I use Hive 3.1.0.3.1.0.0-78.
Create source table with statement:
CREATE TABLE IF NOT EXISTS core.source_jira
(
pk_jira BIGINT DEFAULT SURROGATE_KEY(),
test_1 STRING,
test_2 STRING,
valid BOOLEAN,
PRIMARY KEY (pk_jira) DISABLE NOVALIDATE
)
STORED AS ORC
LOCATION '/warehouse/tablespace/managed/hive/core.db/source_jira'
TBLPROPERTIES (
"orc.compress" = "SNAPPY",
"transactional" = "true",
"NO_AUTO_COMPACTION"="TRUE"
)
;
INSERT INTO TABLE core.source_jira(test_1, test_2, valid)
SELECT "text_1", "text_2", TRUE;
Check table data with select:
SELECT * from core.source_jira
pk_jira
test_1
test_2
valid
1099511627776
text_1
text_2
true
Create destination table with statement:
CREATE TABLE IF NOT EXISTS core.destination_jira
(
pk_jira BIGINT DEFAULT SURROGATE_KEY(),
test_1 STRING,
test_2 STRING,
valid BOOLEAN,
PRIMARY KEY (pk_jira) DISABLE NOVALIDATE
)
STORED AS ORC
LOCATION '/warehouse/tablespace/managed/hive/core.db/destination_jira'
TBLPROPERTIES (
"orc.compress" = "SNAPPY",
"transactional" = "true",
"NO_AUTO_COMPACTION"="TRUE"
)
;
Add column:
ALTER TABLE core.destination_jira ADD COLUMNS (jira_type STRING);
Insert data statement:
insert into table core.destination_jira (test_1, test_2, valid) select test_1, test_2, valid from core.source_jira
I got an error:
SQL Error [40000] [42000]: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
org.jkiss.dbeaver.model.sql.DBSQLException: SQL Error [40000] [42000]: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.executeStatement(JDBCStatementImpl.java:133)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeStatement(SQLQueryJob.java:577)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.lambda$1(SQLQueryJob.java:486)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:172)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeSingleQuery(SQLQueryJob.java:493)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.extractData(SQLQueryJob.java:894)
at org.jkiss.dbeaver.ui.editors.sql.SQLEditor$QueryResultsContainer.readData(SQLEditor.java:3645)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.lambda$0(ResultSetJobDataRead.java:123)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:172)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.run(ResultSetJobDataRead.java:121)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetViewer$ResultSetDataPumpJob.run(ResultSetViewer.java:4949)
at org.jkiss.dbeaver.model.runtime.AbstractJob.run(AbstractJob.java:105)
at org.eclipse.core.internal.jobs.Worker.run(Worker.java:63)
Caused by: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:279)
at org.apache.hive.jdbc.Utils.verifySuccessWithInfo(Utils.java:265)
at org.apache.hive.jdbc.HiveStatement.runAsyncOnServer(HiveStatement.java:303)
at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:244)
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.execute(JDBCStatementImpl.java:329)
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.lambda$0(JDBCStatementImpl.java:131)
at org.jkiss.dbeaver.utils.SecurityManagerUtils.wrapDriverActions(SecurityManagerUtils.java:94)
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.executeStatement(JDBCStatementImpl.java:131)
... 12 more
Caused by: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:335)
at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:199)
at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:262)
at org.apache.hive.service.cli.operation.Operation.run(Operation.java:247)
at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:575)
at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:561)
at sun.reflect.GeneratedMethodAccessor72.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78)
at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36)
at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59)
at com.sun.proxy.$Proxy47.executeStatementAsync(Unknown Source)
at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:315)
at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:566)
at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557)
at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getColNameToDefaultValueMap(SemanticAnalyzer.java:750)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getColForInsertStmtSpec(SemanticAnalyzer.java:4620)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.handleInsertStatementSpec(SemanticAnalyzer.java:4735)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:4580)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:4326)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPostGroupByBodyPlan(SemanticAnalyzer.java:10662)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genBodyPlan(SemanticAnalyzer.java:10601)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:11530)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:11400)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genOPTree(SemanticAnalyzer.java:12202)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:620)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12292)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:360)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:289)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:664)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1869)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1816)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1811)
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:126)
at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:197)
... 26 more
Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:java.lang.NullPointerException
at org.apache.hadoop.hive.ql.metadata.Hive.getEnabledDefaultConstraints(Hive.java:5389)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getColNameToDefaultValueMap(SemanticAnalyzer.java:744)
... 45 more
Caused by: java.lang.NullPointerException: null
Related
I'm querying a MySQL table using PySpark JDBC. The problem I'm facing is that said table has column names with spaces.
In MySQL Workbench I would do the query like this to get the data and that works fine.
select
`some id`
from
table
But when I do it via PySpark like this:
query = 'select `some id` from table'
df = sqlContext.read.format('jdbc') \
.options(
driver = 'com.mysql.jdbc.Driver'
, url = some_connection_url
, dbtable = query
, user = user
, password = password
).load()
I get the following error message:
Py4JJavaError: An error occurred while calling o497.load.
: java.sql.SQLSyntaxErrorException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'select
`some id`
from
some_table WHERE 1=0' at line 1
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:120)
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:97)
at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:122)
at com.mysql.cj.jdbc.ClientPreparedStatement.executeInternal(ClientPreparedStatement.java:953)
at com.mysql.cj.jdbc.ClientPreparedStatement.executeQuery(ClientPreparedStatement.java:1003)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:61)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:210)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:318)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:167)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
I've tried quoting the column names with [] and "" but I get the same error message.
I'm also using the latest MySQL jars: mysql-connector-java-8.0.21.jar and Spark 2.4.
Any insights are highly appreciated.
Enclose your query in brackets () then try to pass to read.format("jdbc").
Example:
query = '(select `some id` from table)e'
df = sqlContext.read.format('jdbc') \
.options(
driver = 'com.mysql.jdbc.Driver'
, url = some_connection_url
, dbtable = query
, user = user
, password = password
).load()
I get a deadlock exception, but when I want to see some info about that from "SHOW ENGINE INNODB STATUS", it contains deadlock information from longer time ago (long before the exception), so it looks like it is just "not catched" by that command.
From the stacktrace I can see, that the last my application code executed was the $Proxy174.attemptToAcquireJobExecution , which is realization of the spring data interface:
#Transactional(TxType.REQUIRES_NEW)
#Modifying
#Query("UPDATE Job SET last_run = NOW(),
last_run_by=:runner
WHERE name = :name
AND (last_run + :interval ) < NOW()")
int attemptToAcquireJobExecution(
#Param("name") String var1,
#Param("interval") int var2,
#Param("runner") String var3);
It operates on such a simple table:
CREATE TABLE `jobs` (
`name` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL,
`last_run_by` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_run` datetime(3) DEFAULT NULL,
`context_data` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
Scheduled tasks, that are executing that command runs every few minutes (on 3 app instances), so there is really minimal risk of collisions, yet deadlock occurs very often... There is no #Transactional anywhere before that operation, so there should be no hanging transaction really. Anyway, pinging those commands yields empty results:
SELECT * FROM INFORMATION_SCHEMA.INNODB_LOCKS ;
SELECT * FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS;
Original stacktrace, my classes marked as "xxx", at the bottom:
com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(NativeConstructorAccessorImpl.java)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
at com.mysql.jdbc.Util.getInstance(Util.java:408)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:951)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3973)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3909)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2527)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2680)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2497)
at com.mysql.jdbc.ConnectionImpl.commit(ConnectionImpl.java:1566)
... 2 frames excluded
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.tomcat.jdbc.pool.ProxyConnection.invoke(ProxyConnection.java:126)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:108)
at org.apache.tomcat.jdbc.pool.DisposableConnectionFacade.invoke(DisposableConnectionFacade.java:81)
at com.sun.proxy.$Proxy121.commit(Unknown Source)
at org.hibernate.resource.jdbc.internal.AbstractLogicalConnectionImplementor.commit(AbstractLogicalConnectionImplementor.java:80)
at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.commit(JdbcResourceLocalTransactionCoordinatorImpl.java:232)
at org.hibernate.engine.transaction.internal.TransactionImpl.commit(TransactionImpl.java:65)
at org.hibernate.jpa.internal.TransactionImpl.commit(TransactionImpl.java:61)
... 49 common frames omitted
Wrapped by: javax.persistence.PersistenceException: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
at org.hibernate.jpa.spi.AbstractEntityManagerImpl.convert(AbstractEntityManagerImpl.java:1692)
at org.hibernate.jpa.spi.AbstractEntityManagerImpl.convert(AbstractEntityManagerImpl.java:1602)
at org.hibernate.jpa.internal.TransactionImpl.commit(TransactionImpl.java:75)
at org.springframework.orm.jpa.JpaTransactionManager.doCommit(JpaTransactionManager.java:517)
... 48 common frames omitted
Wrapped by: org.springframework.orm.jpa.JpaSystemException: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction; nested exception is javax.persistence.PersistenceException: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
at org.springframework.orm.jpa.EntityManagerFactoryUtils.convertJpaAccessExceptionIfPossible(EntityManagerFactoryUtils.java:418)
at org.springframework.orm.jpa.vendor.HibernateJpaDialect.translateExceptionIfPossible(HibernateJpaDialect.java:246)
at org.springframework.orm.jpa.JpaTransactionManager.doCommit(JpaTransactionManager.java:521)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:761)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:730)
at org.springframework.transaction.interceptor.TransactionAspectSupport.commitTransactionAfterReturning(TransactionAspectSupport.java:504)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:292)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:133)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.data.repository.core.support.SurroundingTransactionDetectorMethodInterceptor.invoke(SurroundingTransactionDetectorMethodInterceptor.java:57)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy174.attemptToAcquireJobExecution(Unknown Source)
at xxx.lib.scheduler.JobsService.checkIfJobShouldBeExecuted(JobsService.java:32)
at xxx.lib.scheduler.StandardBatchStarter.startScheduled(StandardBatchStarter.java:38)
at xxx.scheduled.ImportProductDescriptionJob.startScheduled(ImportProductDescriptionJob.java:46)
at xxx.scheduled.ImportProductDescriptionJob$$FastClassBySpringCGLIB$$494978af.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85)
at org.springframework.cloud.sleuth.instrument.scheduling.TraceSchedulingAspect.traceBackgroundThread(TraceSchedulingAspect.java:72)
... 2 frames excluded
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629)
... 21 frames truncated
AND (last_run + :interval ) < NOW()
Probably leads to a syntax error -- or a data error.
Please provide the generated SQL. I suspect it is generating
AND (last_run + 2) < NOW()
But it needs to be something like
AND (last_run + INTERVAL 2 DAY) < NOW()
Also, please provide SHOW CREATE Job.
The query works with mysql perfectly but it fails when used with Sqoop to import Data from MySql to Hive.
select cmpdl.*,ct.value from (SELECT cmp.domain_id,
cmp.model,
cmp.name,
cmp.status,
br.name,
cat.category,
dl.start_date,
dl.end_date,
cmp.created_at,
cmp.updated_at
FROM domain cmp
JOIN brands br ON cmp.brand_id=br.brands_id
JOIN delxxx dl ON cmp.del_id=dl.del_id
JOIN
(SELECT c.domain_id,
group_concat(b.category_name) AS category
FROM domain c
JOIN categories b ON find_in_set(b.category_id,xxxxx)
GROUP BY c.domain_id) cat ON cmp.domain_id=cat.domain_id
WHERE cmp.domain_type='XXXXXX'
AND cmp.category='XXXXXXX'
AND cmp.model != 'XXX') cmpdl join context_targeting ct on cmpdl.domain_id=ct.domain_id and ct.context_targeting_group='xxxxxxx' AND $CONDITIONS;
Error Log
Error: java.io.IOException: SQLException in nextKeyValue
at org.apache.sqoop.mapreduce.db.DBRecordReader.nextKeyValue(DBRecordReader.java:277)
at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapTask.java:556)
at org.apache.hadoop.mapreduce.task.MapContextImpl.nextKeyValue(MapContextImpl.java:80)
at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.nextKeyValue(WrappedMapper.java:91)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.sqoop.mapreduce.AutoProgressMapper.run(AutoProgressMapper.java:64)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:168)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Unknown column 'cmp.domain_id' in 'on clause'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:408)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:389)
at com.mysql.jdbc.Util.getInstance(Util.java:372)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:980)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3835)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3771)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2435)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2582)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2535)
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:1911)
at com.mysql.jdbc.PreparedStatement.executeQuery(PreparedStatement.java:2034)
at org.apache.sqoop.mapreduce.db.DBRecordReader.executeQuery(DBRecordReader.java:111)
at org.apache.sqoop.mapreduce.db.DBRecordReader.nextKeyValue(DBRecordReader.java:235)
... 12 more
I am not able to figure out the exact issue how the alias is getting out of scope and columns are not being found.
There was issue with my sqoop split-by properties
in which i have mentioned cmp.domain_id in place of cmpdl.domain_id due to which the Sqoop Map-Reduce job was failing
I'm using a Liquibase changeset to update null values of the customId column. The sql is a modified version of this answer.
The problem is that it seems to use the same UUID for all rows. My changeset looks like this:
<changeSet id="v3.0.3" author="msp" runAlways="false">
<preConditions onFail="MARK_RAN">
<not>
<sqlCheck expectedResult="0">select count(*) from FOOBAR where customId is null</sqlCheck>
</not>
</preConditions>
<sql>update FOOBAR set customId = concat('cu:', (select uuid())) where customId is null</sql>
</changeSet>
Note that if I'm running this slightly modified version of the query (adding a limit of 2) in the MySQL console, then it works:
update FOOBAR set customId = concat('cu:', (select uuid())) where customId is null limit 2
But if I try to run the very same query (including the limit) with liquibase, I get the following Exception:
Caused by: liquibase.exception.MigrationFailedException: Migration failed for change set changelog/db.changelog-v3.0.3.xml::v3.0.3::msp:
Reason: liquibase.exception.DatabaseException: Error executing SQL update FOOBAR set customId = concat('cu:', (select uuid())) where customId is null limit 2: Duplicate entry 'cu:cde325da-f71f-11e5-95f8-d15a50829a7d' for key 'customId'
at liquibase.changelog.ChangeSet.execute(ChangeSet.java:554)
at liquibase.changelog.visitor.UpdateVisitor.visit(UpdateVisitor.java:43)
at liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:70)
at liquibase.Liquibase.update(Liquibase.java:195)
at liquibase.Liquibase.update(Liquibase.java:174)
at liquibase.integration.spring.SpringLiquibase.performUpdate(SpringLiquibase.java:330)
at liquibase.integration.spring.SpringLiquibase.afterPropertiesSet(SpringLiquibase.java:287)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1625)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1562)
... 77 more
Caused by: liquibase.exception.DatabaseException: Error executing SQL update FOOBAR set customId = concat('cu:', (select uuid())) where customId is null limit 2: Duplicate entry 'cu:cde325da-f71f-11e5-95f8-d15a50829a7d' for key 'customId'
at liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:62)
at liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:122)
at liquibase.database.AbstractJdbcDatabase.execute(AbstractJdbcDatabase.java:1206)
at liquibase.database.AbstractJdbcDatabase.executeStatements(AbstractJdbcDatabase.java:1189)
at liquibase.changelog.ChangeSet.execute(ChangeSet.java:518)
... 85 more
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry 'cu:cde325da-f71f-11e5-95f8-d15a50829a7d' for key 'customId'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.Util.getInstance(Util.java:386)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1039)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4096)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4028)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2490)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2651)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2677)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2627)
at com.mysql.jdbc.StatementImpl.execute(StatementImpl.java:841)
at com.mysql.jdbc.StatementImpl.execute(StatementImpl.java:681)
at com.mchange.v2.c3p0.impl.NewProxyStatement.execute(NewProxyStatement.java:1006)
at liquibase.executor.jvm.JdbcExecutor$ExecuteStatementCallback.doInStatement(JdbcExecutor.java:310)
at liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:55)
... 89 more
Note that there's of course a UNIQUE constraint on customId - but shouldn't select uuid() (also tried uuid() without select) generate a unique value, even when using Liquibase?
The problem can be solved with a small workaround. I split the update into two queries:
<changeSet id="v3.0.3" author="msp" runAlways="false">
<preConditions onFail="MARK_RAN">
<not>
<sqlCheck expectedResult="0">select count(*) from FOOBAR where customId is null</sqlCheck>
</not>
</preConditions>
<sql>update FOOBAR set customId = (select uuid())) where customId is null</sql>
<sql>update FOOBAR set customId = concat('cu:', customId) where length(customId) = 36
</changeSet>
This of course only works for my case where I can be sure that all rows of the first query can be identified by length(customId) = 36.
Can anyone explain why I'm getting this error (full stack trace below):
Caused by: org.apache.openjpa.lib.jdbc.ReportingSQLException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'TYPE = innodb' at line 1** {stmnt 763763032 CREATE TABLE ADMIN_USER (admin_id BIGINT NOT NULL AUTO_INCREMENT, first_name VARCHAR(255), last_name VARCHAR(255), user_name VARCHAR(255), PRIMARY KEY (admin_id)) TYPE = innodb} [code=1064, state=42000]*
I've got a web app with MySQL 5.5.11 and openJPA 1.2.2 and Geronimo 2.2.1 with Tomcat 6. I'm able to depoly the .ear fine. But when I submit my form to persist a Admin #Entity, I get the following error above.
Also, I'm using tranql-connector-mysql-local-1.5.rar (from the Geronimo repository) as my connector.
persistence.xml
<persistence xmlns="http://java.sun.com/xml/ns/persistence" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0">
<persistence-unit name="AReyesPersistUnit" transaction-type="JTA">
<provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
<jta-data-source>AReyesMySql</jta-data-source>
<class>com.areyes.entity.Admin</class>
<properties>
<property name="openjpa.jdbc.SynchronizeMappings" value="buildSchema"/>
</properties>
</persistence-unit>
</persistence>
Admin #Entity
#Entity
#Table (name="ADMIN_USER")
public class Admin implements Serializable {
private static final long serialVersionUID = 1L;
#Id
#GeneratedValue (strategy=GenerationType.IDENTITY)
#Column (name="admin_id")private long id;
#Column (name="user_name")private String userName;
#Column (name="first_name")private String firstName;
#Column (name="last_name")private String lastName;
...followed by getters/setters
Stack trace
2011-06-30 23:15:52,203 ERROR [[AddAdmin]] Servlet.service() for servlet AddAdmin threw exception
javax.ejb.EJBException: The bean encountered a non-application exception; nested exception is:
<openjpa-1.2.2-r422266:898935 nonfatal general error> org.apache.openjpa.persistence.PersistenceException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'TYPE = innodb' at line 1 {stmnt 1243851750 CREATE TABLE ADMIN_USER (admin_id BIGINT NOT NULL AUTO_INCREMENT, first_name VARCHAR(255), last_name VARCHAR(255), user_name VARCHAR(255), PRIMARY KEY (admin_id)) TYPE = innodb} [code=1064, state=42000]
at org.apache.openejb.core.ivm.BaseEjbProxyHandler.convertException(BaseEjbProxyHandler.java:359)
at org.apache.openejb.core.ivm.BaseEjbProxyHandler.invoke(BaseEjbProxyHandler.java:287)
at $Proxy65.create(Unknown Source)
at com.areyes.servlet.AddAdmin.doPost(AddAdmin.java:38)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:713)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:806)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.geronimo.tomcat.GeronimoStandardContext$SystemMethodValve.invoke(GeronimoStandardContext.java:420)
at org.apache.geronimo.tomcat.valve.GeronimoBeforeAfterValve.invoke(GeronimoBeforeAfterValve.java:47)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:555)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:298)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:857)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:588)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:396)
at org.apache.geronimo.pool.ThreadPool$1.run(ThreadPool.java:214)
at org.apache.geronimo.pool.ThreadPool$ContextClassLoaderRunnable.run(ThreadPool.java:344)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: <openjpa-1.2.2-r422266:898935 nonfatal general error> org.apache.openjpa.persistence.PersistenceException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'TYPE = innodb' at line 1 {stmnt 1243851750 CREATE TABLE ADMIN_USER (admin_id BIGINT NOT NULL AUTO_INCREMENT, first_name VARCHAR(255), last_name VARCHAR(255), user_name VARCHAR(255), PRIMARY KEY (admin_id)) TYPE = innodb} [code=1064, state=42000]
at org.apache.openjpa.jdbc.meta.MappingTool.record(MappingTool.java:553)
at org.apache.openjpa.jdbc.meta.MappingTool.record(MappingTool.java:453)
at org.apache.openjpa.jdbc.kernel.JDBCBrokerFactory.synchronizeMappings(JDBCBrokerFactory.java:159)
at org.apache.openjpa.jdbc.kernel.JDBCBrokerFactory.newBrokerImpl(JDBCBrokerFactory.java:119)
at org.apache.openjpa.kernel.AbstractBrokerFactory.newBroker(AbstractBrokerFactory.java:189)
at org.apache.openjpa.kernel.DelegatingBrokerFactory.newBroker(DelegatingBrokerFactory.java:142)
at org.apache.openjpa.persistence.EntityManagerFactoryImpl.createEntityManager(EntityManagerFactoryImpl.java:192)
at org.apache.openjpa.persistence.EntityManagerFactoryImpl.createEntityManager(EntityManagerFactoryImpl.java:56)
at org.apache.geronimo.persistence.CMPEntityManagerTxScoped.createEntityManager(CMPEntityManagerTxScoped.java:74)
at org.apache.geronimo.persistence.CMPEntityManagerTxScoped.getEntityManager(CMPEntityManagerTxScoped.java:55)
at org.apache.geronimo.persistence.CMPEntityManagerTxScoped.persist(CMPEntityManagerTxScoped.java:81)
at com.areyes.session.AdminManager.create(AdminManager.java:27)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.openejb.core.interceptor.ReflectionInvocationContext$Invocation.invoke(ReflectionInvocationContext.java:162)
at org.apache.openejb.core.interceptor.ReflectionInvocationContext.proceed(ReflectionInvocationContext.java:144)
at org.apache.openejb.monitoring.StatsInterceptor.record(StatsInterceptor.java:164)
at org.apache.openejb.monitoring.StatsInterceptor.invoke(StatsInterceptor.java:92)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.openejb.core.interceptor.ReflectionInvocationContext$Invocation.invoke(ReflectionInvocationContext.java:162)
at org.apache.openejb.core.interceptor.ReflectionInvocationContext.proceed(ReflectionInvocationContext.java:144)
at org.apache.openejb.core.interceptor.InterceptorStack.invoke(InterceptorStack.java:122)
at org.apache.openejb.core.stateless.StatelessContainer._invoke(StatelessContainer.java:221)
at org.apache.openejb.core.stateless.StatelessContainer.invoke(StatelessContainer.java:174)
at org.apache.openejb.core.ivm.EjbObjectProxyHandler.businessMethod(EjbObjectProxyHandler.java:217)
at org.apache.openejb.core.ivm.EjbObjectProxyHandler._invoke(EjbObjectProxyHandler.java:77)
at org.apache.openejb.core.ivm.BaseEjbProxyHandler.invoke(BaseEjbProxyHandler.java:282)
... 23 more
Caused by: org.apache.openjpa.lib.jdbc.ReportingSQLException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'TYPE = innodb' at line 1 {stmnt 1243851750 CREATE TABLE ADMIN_USER (admin_id BIGINT NOT NULL AUTO_INCREMENT, first_name VARCHAR(255), last_name VARCHAR(255), user_name VARCHAR(255), PRIMARY KEY (admin_id)) TYPE = innodb} [code=1064, state=42000]
at org.apache.openjpa.lib.jdbc.LoggingConnectionDecorator.wrap(LoggingConnectionDecorator.java:192)
at org.apache.openjpa.lib.jdbc.LoggingConnectionDecorator.access$700(LoggingConnectionDecorator.java:57)
at org.apache.openjpa.lib.jdbc.LoggingConnectionDecorator$LoggingConnection$LoggingStatement.executeUpdate(LoggingConnectionDecorator.java:762)
at org.apache.openjpa.jdbc.schema.SchemaTool.executeSQL(SchemaTool.java:1191)
at org.apache.openjpa.jdbc.schema.SchemaTool.createTable(SchemaTool.java:949)
at org.apache.openjpa.jdbc.schema.SchemaTool.add(SchemaTool.java:526)
at org.apache.openjpa.jdbc.schema.SchemaTool.add(SchemaTool.java:344)
at org.apache.openjpa.jdbc.schema.SchemaTool.run(SchemaTool.java:321)
at org.apache.openjpa.jdbc.meta.MappingTool.record(MappingTool.java:501)
... 54 more
After much digging, this is a bug that has already been fixed. The root issue is that MySQL changed the table_option 'type' to 'engine' in the 5.1ish time frame.
Either you can move to a newer level of OpenJPA(1.3.x, 2.0.x, etc), or don't use OpenJPA to create your tables.