The query works with mysql perfectly but it fails when used with Sqoop to import Data from MySql to Hive.
select cmpdl.*,ct.value from (SELECT cmp.domain_id,
cmp.model,
cmp.name,
cmp.status,
br.name,
cat.category,
dl.start_date,
dl.end_date,
cmp.created_at,
cmp.updated_at
FROM domain cmp
JOIN brands br ON cmp.brand_id=br.brands_id
JOIN delxxx dl ON cmp.del_id=dl.del_id
JOIN
(SELECT c.domain_id,
group_concat(b.category_name) AS category
FROM domain c
JOIN categories b ON find_in_set(b.category_id,xxxxx)
GROUP BY c.domain_id) cat ON cmp.domain_id=cat.domain_id
WHERE cmp.domain_type='XXXXXX'
AND cmp.category='XXXXXXX'
AND cmp.model != 'XXX') cmpdl join context_targeting ct on cmpdl.domain_id=ct.domain_id and ct.context_targeting_group='xxxxxxx' AND $CONDITIONS;
Error Log
Error: java.io.IOException: SQLException in nextKeyValue
at org.apache.sqoop.mapreduce.db.DBRecordReader.nextKeyValue(DBRecordReader.java:277)
at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapTask.java:556)
at org.apache.hadoop.mapreduce.task.MapContextImpl.nextKeyValue(MapContextImpl.java:80)
at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.nextKeyValue(WrappedMapper.java:91)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.sqoop.mapreduce.AutoProgressMapper.run(AutoProgressMapper.java:64)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:168)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Unknown column 'cmp.domain_id' in 'on clause'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:408)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:389)
at com.mysql.jdbc.Util.getInstance(Util.java:372)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:980)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3835)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3771)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2435)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2582)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2535)
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:1911)
at com.mysql.jdbc.PreparedStatement.executeQuery(PreparedStatement.java:2034)
at org.apache.sqoop.mapreduce.db.DBRecordReader.executeQuery(DBRecordReader.java:111)
at org.apache.sqoop.mapreduce.db.DBRecordReader.nextKeyValue(DBRecordReader.java:235)
... 12 more
I am not able to figure out the exact issue how the alias is getting out of scope and columns are not being found.
There was issue with my sqoop split-by properties
in which i have mentioned cmp.domain_id in place of cmpdl.domain_id due to which the Sqoop Map-Reduce job was failing
Related
I use Hive 3.1.0.3.1.0.0-78.
Create source table with statement:
CREATE TABLE IF NOT EXISTS core.source_jira
(
pk_jira BIGINT DEFAULT SURROGATE_KEY(),
test_1 STRING,
test_2 STRING,
valid BOOLEAN,
PRIMARY KEY (pk_jira) DISABLE NOVALIDATE
)
STORED AS ORC
LOCATION '/warehouse/tablespace/managed/hive/core.db/source_jira'
TBLPROPERTIES (
"orc.compress" = "SNAPPY",
"transactional" = "true",
"NO_AUTO_COMPACTION"="TRUE"
)
;
INSERT INTO TABLE core.source_jira(test_1, test_2, valid)
SELECT "text_1", "text_2", TRUE;
Check table data with select:
SELECT * from core.source_jira
pk_jira
test_1
test_2
valid
1099511627776
text_1
text_2
true
Create destination table with statement:
CREATE TABLE IF NOT EXISTS core.destination_jira
(
pk_jira BIGINT DEFAULT SURROGATE_KEY(),
test_1 STRING,
test_2 STRING,
valid BOOLEAN,
PRIMARY KEY (pk_jira) DISABLE NOVALIDATE
)
STORED AS ORC
LOCATION '/warehouse/tablespace/managed/hive/core.db/destination_jira'
TBLPROPERTIES (
"orc.compress" = "SNAPPY",
"transactional" = "true",
"NO_AUTO_COMPACTION"="TRUE"
)
;
Add column:
ALTER TABLE core.destination_jira ADD COLUMNS (jira_type STRING);
Insert data statement:
insert into table core.destination_jira (test_1, test_2, valid) select test_1, test_2, valid from core.source_jira
I got an error:
SQL Error [40000] [42000]: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
org.jkiss.dbeaver.model.sql.DBSQLException: SQL Error [40000] [42000]: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.executeStatement(JDBCStatementImpl.java:133)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeStatement(SQLQueryJob.java:577)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.lambda$1(SQLQueryJob.java:486)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:172)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.executeSingleQuery(SQLQueryJob.java:493)
at org.jkiss.dbeaver.ui.editors.sql.execute.SQLQueryJob.extractData(SQLQueryJob.java:894)
at org.jkiss.dbeaver.ui.editors.sql.SQLEditor$QueryResultsContainer.readData(SQLEditor.java:3645)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.lambda$0(ResultSetJobDataRead.java:123)
at org.jkiss.dbeaver.model.exec.DBExecUtils.tryExecuteRecover(DBExecUtils.java:172)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetJobDataRead.run(ResultSetJobDataRead.java:121)
at org.jkiss.dbeaver.ui.controls.resultset.ResultSetViewer$ResultSetDataPumpJob.run(ResultSetViewer.java:4949)
at org.jkiss.dbeaver.model.runtime.AbstractJob.run(AbstractJob.java:105)
at org.eclipse.core.internal.jobs.Worker.run(Worker.java:63)
Caused by: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:279)
at org.apache.hive.jdbc.Utils.verifySuccessWithInfo(Utils.java:265)
at org.apache.hive.jdbc.HiveStatement.runAsyncOnServer(HiveStatement.java:303)
at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:244)
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.execute(JDBCStatementImpl.java:329)
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.lambda$0(JDBCStatementImpl.java:131)
at org.jkiss.dbeaver.utils.SecurityManagerUtils.wrapDriverActions(SecurityManagerUtils.java:94)
at org.jkiss.dbeaver.model.impl.jdbc.exec.JDBCStatementImpl.executeStatement(JDBCStatementImpl.java:131)
... 12 more
Caused by: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:335)
at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:199)
at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:262)
at org.apache.hive.service.cli.operation.Operation.run(Operation.java:247)
at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:575)
at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:561)
at sun.reflect.GeneratedMethodAccessor72.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78)
at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36)
at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59)
at com.sun.proxy.$Proxy47.executeStatementAsync(Unknown Source)
at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:315)
at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:566)
at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557)
at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.NullPointerException
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getColNameToDefaultValueMap(SemanticAnalyzer.java:750)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getColForInsertStmtSpec(SemanticAnalyzer.java:4620)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.handleInsertStatementSpec(SemanticAnalyzer.java:4735)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:4580)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:4326)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPostGroupByBodyPlan(SemanticAnalyzer.java:10662)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genBodyPlan(SemanticAnalyzer.java:10601)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:11530)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:11400)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genOPTree(SemanticAnalyzer.java:12202)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:620)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12292)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:360)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:289)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:664)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1869)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1816)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1811)
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:126)
at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:197)
... 26 more
Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:java.lang.NullPointerException
at org.apache.hadoop.hive.ql.metadata.Hive.getEnabledDefaultConstraints(Hive.java:5389)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getColNameToDefaultValueMap(SemanticAnalyzer.java:744)
... 45 more
Caused by: java.lang.NullPointerException: null
I'm querying a MySQL table using PySpark JDBC. The problem I'm facing is that said table has column names with spaces.
In MySQL Workbench I would do the query like this to get the data and that works fine.
select
`some id`
from
table
But when I do it via PySpark like this:
query = 'select `some id` from table'
df = sqlContext.read.format('jdbc') \
.options(
driver = 'com.mysql.jdbc.Driver'
, url = some_connection_url
, dbtable = query
, user = user
, password = password
).load()
I get the following error message:
Py4JJavaError: An error occurred while calling o497.load.
: java.sql.SQLSyntaxErrorException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'select
`some id`
from
some_table WHERE 1=0' at line 1
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:120)
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:97)
at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:122)
at com.mysql.cj.jdbc.ClientPreparedStatement.executeInternal(ClientPreparedStatement.java:953)
at com.mysql.cj.jdbc.ClientPreparedStatement.executeQuery(ClientPreparedStatement.java:1003)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:61)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:210)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:318)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:167)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
I've tried quoting the column names with [] and "" but I get the same error message.
I'm also using the latest MySQL jars: mysql-connector-java-8.0.21.jar and Spark 2.4.
Any insights are highly appreciated.
Enclose your query in brackets () then try to pass to read.format("jdbc").
Example:
query = '(select `some id` from table)e'
df = sqlContext.read.format('jdbc') \
.options(
driver = 'com.mysql.jdbc.Driver'
, url = some_connection_url
, dbtable = query
, user = user
, password = password
).load()
I get a deadlock exception, but when I want to see some info about that from "SHOW ENGINE INNODB STATUS", it contains deadlock information from longer time ago (long before the exception), so it looks like it is just "not catched" by that command.
From the stacktrace I can see, that the last my application code executed was the $Proxy174.attemptToAcquireJobExecution , which is realization of the spring data interface:
#Transactional(TxType.REQUIRES_NEW)
#Modifying
#Query("UPDATE Job SET last_run = NOW(),
last_run_by=:runner
WHERE name = :name
AND (last_run + :interval ) < NOW()")
int attemptToAcquireJobExecution(
#Param("name") String var1,
#Param("interval") int var2,
#Param("runner") String var3);
It operates on such a simple table:
CREATE TABLE `jobs` (
`name` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL,
`last_run_by` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_run` datetime(3) DEFAULT NULL,
`context_data` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
Scheduled tasks, that are executing that command runs every few minutes (on 3 app instances), so there is really minimal risk of collisions, yet deadlock occurs very often... There is no #Transactional anywhere before that operation, so there should be no hanging transaction really. Anyway, pinging those commands yields empty results:
SELECT * FROM INFORMATION_SCHEMA.INNODB_LOCKS ;
SELECT * FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS;
Original stacktrace, my classes marked as "xxx", at the bottom:
com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(NativeConstructorAccessorImpl.java)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
at com.mysql.jdbc.Util.getInstance(Util.java:408)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:951)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3973)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3909)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2527)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2680)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2497)
at com.mysql.jdbc.ConnectionImpl.commit(ConnectionImpl.java:1566)
... 2 frames excluded
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.tomcat.jdbc.pool.ProxyConnection.invoke(ProxyConnection.java:126)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:108)
at org.apache.tomcat.jdbc.pool.DisposableConnectionFacade.invoke(DisposableConnectionFacade.java:81)
at com.sun.proxy.$Proxy121.commit(Unknown Source)
at org.hibernate.resource.jdbc.internal.AbstractLogicalConnectionImplementor.commit(AbstractLogicalConnectionImplementor.java:80)
at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.commit(JdbcResourceLocalTransactionCoordinatorImpl.java:232)
at org.hibernate.engine.transaction.internal.TransactionImpl.commit(TransactionImpl.java:65)
at org.hibernate.jpa.internal.TransactionImpl.commit(TransactionImpl.java:61)
... 49 common frames omitted
Wrapped by: javax.persistence.PersistenceException: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
at org.hibernate.jpa.spi.AbstractEntityManagerImpl.convert(AbstractEntityManagerImpl.java:1692)
at org.hibernate.jpa.spi.AbstractEntityManagerImpl.convert(AbstractEntityManagerImpl.java:1602)
at org.hibernate.jpa.internal.TransactionImpl.commit(TransactionImpl.java:75)
at org.springframework.orm.jpa.JpaTransactionManager.doCommit(JpaTransactionManager.java:517)
... 48 common frames omitted
Wrapped by: org.springframework.orm.jpa.JpaSystemException: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction; nested exception is javax.persistence.PersistenceException: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
at org.springframework.orm.jpa.EntityManagerFactoryUtils.convertJpaAccessExceptionIfPossible(EntityManagerFactoryUtils.java:418)
at org.springframework.orm.jpa.vendor.HibernateJpaDialect.translateExceptionIfPossible(HibernateJpaDialect.java:246)
at org.springframework.orm.jpa.JpaTransactionManager.doCommit(JpaTransactionManager.java:521)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:761)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:730)
at org.springframework.transaction.interceptor.TransactionAspectSupport.commitTransactionAfterReturning(TransactionAspectSupport.java:504)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:292)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:133)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.data.repository.core.support.SurroundingTransactionDetectorMethodInterceptor.invoke(SurroundingTransactionDetectorMethodInterceptor.java:57)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy174.attemptToAcquireJobExecution(Unknown Source)
at xxx.lib.scheduler.JobsService.checkIfJobShouldBeExecuted(JobsService.java:32)
at xxx.lib.scheduler.StandardBatchStarter.startScheduled(StandardBatchStarter.java:38)
at xxx.scheduled.ImportProductDescriptionJob.startScheduled(ImportProductDescriptionJob.java:46)
at xxx.scheduled.ImportProductDescriptionJob$$FastClassBySpringCGLIB$$494978af.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85)
at org.springframework.cloud.sleuth.instrument.scheduling.TraceSchedulingAspect.traceBackgroundThread(TraceSchedulingAspect.java:72)
... 2 frames excluded
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629)
... 21 frames truncated
AND (last_run + :interval ) < NOW()
Probably leads to a syntax error -- or a data error.
Please provide the generated SQL. I suspect it is generating
AND (last_run + 2) < NOW()
But it needs to be something like
AND (last_run + INTERVAL 2 DAY) < NOW()
Also, please provide SHOW CREATE Job.
I am trying to use LOAD HADOOP command to copy data from teradata source.
LOAD HADOOP USING JDBC CONNECTION URL 'jdbc:teradata://<<ip_address>>/database=EDW01_V_LV_BASE' WITH PARAMETERS ('user' = 'USR1','password'='pass1') FROM SQL QUERY 'SELECT COL1, COL2, COL3, COL4, COL5, COL6, COl7, COl8, COl9, COL10 FROM POS_RTL_STR WHERE TBL_TMST > '2015-12-15 03:49:33' AND TBL_TMST <= '2015-12-15 14:54:05' AND $CONDITIONS' SPLIT COLUMN COL1 INTO TABLE SCHEMA1.TABLE1 APPEND;.
Now load is failing with the error
2015-12-17 04:02:54,219 ERROR com.ibm.biginsights.ie.db.DBImportImpl [Thread-3] : Error during import
java.lang.RuntimeException: java.sql.SQLException: [Teradata Database] [TeraJDBC 15.00.00.20] [Error 5628] [SQLState HY000] Column 2015-12-15 03:49:33 not found in EDW01_V_LV_BASE.POS_RTL_STR.
at com.ibm.biginsights.ie.sqoop.td.TeradataFastManager.getColumnTypesForRawQuery(TeradataFastManager.java:181)
at org.apache.sqoop.manager.SqlManager.getColumnTypesForQuery(SqlManager.java:234)
at org.apache.sqoop.manager.ConnManager.getColumnTypes(ConnManager.java:304)
at org.apache.sqoop.orm.ClassWriter.getColumnTypes(ClassWriter.java:1833)
at org.apache.sqoop.orm.ClassWriter.generate(ClassWriter.java:1645)
at org.apache.sqoop.tool.CodeGenTool.generateORM(CodeGenTool.java:107)
at org.apache.sqoop.manager.BIImportTool.importTable(BIImportTool.java:97)
at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:605)
at org.apache.sqoop.Sqoop.run(Sqoop.java:143)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:179)
at com.ibm.biginsights.ie.db.SqoopUtils.runSqoopTool(SqoopUtils.java:146)
at com.ibm.biginsights.ie.db.DBImportImpl.importData(DBImportImpl.java:159)
at com.ibm.biginsights.ie.impl.ImporterImpl.executeImport(ImporterImpl.java:504)
at com.ibm.biginsights.ie.impl.ImporterImpl.executePerformImport(ImporterImpl.java:417)
at com.ibm.biginsights.ie.impl.ImporterImpl.performImport(ImporterImpl.java:264)
at com.ibm.biginsights.biga.udf.LoadTool.performImport(LoadTool.java:214)
at com.ibm.biginsights.biga.udf.BIGSQL_DDL.doLoadStatement(BIGSQL_DDL.java:644)
at com.ibm.biginsights.biga.udf.BIGSQL_DDL.processDDL(BIGSQL_DDL.java:207)
Caused by: java.sql.SQLException: [Teradata Database] [TeraJDBC 15.00.00.20] [Error 5628] [SQLState HY000] Column 2015-12-15 03:49:33 not found in EDW01_V_LV_BASE.POS_RTL_STR.
at com.teradata.jdbc.jdbc_4.util.ErrorFactory.makeDatabaseSQLException(ErrorFactory.java:308)
at com.teradata.jdbc.jdbc_4.statemachine.ReceiveInitSubState.action(ReceiveInitSubState.java:109)
at com.teradata.jdbc.jdbc_4.statemachine.StatementReceiveState.subStateMachine(StatementReceiveState.java:307)
at com.teradata.jdbc.jdbc_4.statemachine.StatementReceiveState.action(StatementReceiveState.java:196)
at com.teradata.jdbc.jdbc_4.statemachine.StatementController.runBody(StatementController.java:123)
at com.teradata.jdbc.jdbc_4.statemachine.StatementController.run(StatementController.java:114)
at com.teradata.jdbc.jdbc_4.TDStatement.executeStatement(TDStatement.java:385)
at com.teradata.jdbc.jdbc_4.TDStatement.prepareRequest(TDStatement.java:569)
at com.teradata.jdbc.jdbc_4.TDPreparedStatement.<init>(TDPreparedStatement.java:117)
at com.teradata.jdbc.jdk6.JDK6_SQL_PreparedStatement.<init>(JDK6_SQL_PreparedStatement.java:29)
at com.teradata.jdbc.jdk6.JDK6_SQL_Connection.constructPreparedStatement(JDK6_SQL_Connection.java:81)
at com.teradata.jdbc.jdbc_4.TDSession.prepareStatement(TDSession.java:1357)
at com.teradata.jdbc.jdbc_4.TDSession.prepareStatement(TDSession.java:1401)
at com.teradata.jdbc.jdbc.ManagerConnectionBase.prepareStatement(ManagerConnectionBase.java:302)
at org.apache.sqoop.manager.SqlManager.execute(SqlManager.java:752)
at org.apache.sqoop.manager.SqlManager.execute(SqlManager.java:775)
at com.ibm.biginsights.ie.sqoop.td.TeradataFastManager.executeStmt(TeradataFastManager.java:191)
at com.ibm.biginsights.ie.sqoop.td.TeradataFastManager.getColumnTypesForRawQuery(TeradataFastManager.java:177)
... 18 more
2015-12-17 04:02:54,219 ERROR com.ibm.biginsights.ie.db.DBImportImpl [Thread-3] : [BSL-0-1ae1ada8b]: Error during import (Job Id = ):java.sql.SQLException: [Teradata Database] [TeraJDBC 15.00.00.20] [Error 5628] [SQLState HY000] Column 2015-12-15 03:49:33 not found in EDW01_V_LV_BASE.POS_RTL_STR.
I think it is somewhere wrong in the where clause where I have used value within single quote in where clause. How should it be specified? Or there is any other error?
Please visit: https://www.ibm.com/support/knowledgecenter/?lang=en#!/SSPT3X_4.1.0/com.ibm.swg.im.infosphere.biginsights.db2biga.doc/doc/biga_load_from.html?cp=SSPT3X_4.1.0%2F4-4-0
In the section FROM SQL QUERYselect-statement:
Surround the value for QUERY with single quotes. If a single quote is needed inside the value then use two single quotes.
… FROM SQL QUERY 'SELECT ID, NAME,
STATE, CITY FROM CITIES
WHERE CITY=''San Jose'' AND $CONDITIONS'
...
I am using a PreparedStatement to execute a SQL query. Here is the Query.
select * from DEVICES WHERE device_id in (?,?,?,?)
Here is the error i am getting
com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException:
You have an error in your SQL syntax; check the manual that
corresponds to your MySQL server version for the right
syntax to use near '?,?,?,?)' at line 1
What am i doing wrong here?
Here is the Stacktrace that captured. It looks like the error is occuring when I am executing the query (which is logical).
Here is the Stacktrace that captured. It looks like the error is occuring when I am executing the query (which is logical).
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:409)
at com.mysql.jdbc.Util.getInstance(Util.java:384)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1054)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3562)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3494)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1960)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2114)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2690)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2619)
at com.mysql.jdbc.StatementImpl.executeQuery(StatementImpl.java:1465)
at com.verizonwireless.digitalframe.PictureProcessor.getFilteredFrameList(PictureProcessor.java:715)
at com.verizonwireless.digitalframe.Main.processPictureModule(Main.java:141)
at com.verizonwireless.digitalframe.Main.processRequest(Main.java:78)
at com.verizonwireless.digitalframe.Main.doPost(Main.java:53)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:688)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:770)
at org.apache.catalina.core.StandardWrapper.service(StandardWrapper.java:1550)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:281)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:655)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:595)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:161)
at org.apache.catalina.connector.CoyoteAdapter.doService(CoyoteAdapter.java:331)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:231)
at com.sun.enterprise.v3.services.impl.ContainerMapper$AdapterCallable.call(ContainerMapper.java:317)
at com.sun.enterprise.v3.services.impl.ContainerMapper.service(ContainerMapper.java:195)
at com.sun.grizzly.http.ProcessorTask.invokeAdapter(ProcessorTask.java:860)
at com.sun.grizzly.http.ProcessorTask.doProcess(ProcessorTask.java:757)
at com.sun.grizzly.http.ProcessorTask.process(ProcessorTask.java:1056)
at com.sun.grizzly.http.DefaultProtocolFilter.execute(DefaultProtocolFilter.java:229)
at com.sun.grizzly.DefaultProtocolChain.executeProtocolFilter(DefaultProtocolChain.java:137)
at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:104)
at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:90)
at com.sun.grizzly.http.HttpProtocolChain.execute(HttpProtocolChain.java:79)
at com.sun.grizzly.ProtocolChainContextTask.doCall(ProtocolChainContextTask.java:54)
at com.sun.grizzly.SelectionKeyContextTask.call(SelectionKeyContextTask.java:59)
at com.sun.grizzly.ContextTask.run(ContextTask.java:71)
at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:532)
at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:513)
at java.lang.Thread.run(Thread.java:722)
Oh man what a mistake...
I was using
ResultSet resultSet = statement.executeQuery(query);
I should have been using
ResultSet resultSet = statement.executeQuery();
I works now!