NiFi ExecuteSqlRecord not processing SQL query(Presto) - mysql

I am running a simple Pivot query in Presto via "ExecuteSqlRecord" processor but the moment I run the processor it gives me the error: Unable to execute SQL select.
Query is:
with map_date as
(
SELECT
vin,
epoch,
timestamp,
date,
map_agg(signalName, value) as map_values
from hive.vehicle_signals.vehicle_signals_flat
where date(date) = date('2020-02-28')
and vin = '000015'
and signalName in ('timestamp','epoch','msgId','usec','vlan','vin','msgName','A','B','C','D','E','F','G','H',
'J','K','L','M','N','O','P','R','S','T','U','V','W','X',
'Y','Z','A1','A5','A2','A3','A4','A6','A8','A9','A10','A11','A12')
GROUP BY vin, epoch, timestamp, date
order by timestamp asc
)
SELECT
epoch
, timestamp
, CASE WHEN element_at(map_values, 'A') IS NOT NULL THEN map_values['A'] ELSE NULL END AS A
, CASE WHEN element_at(map_values, 'B') IS NOT NULL THEN map_values['B'] ELSE NULL END AS B
, CASE WHEN element_at(map_values, 'C') IS NOT NULL THEN map_values['C'] ELSE NULL END AS C
, CASE WHEN element_at(map_values, 'D') IS NOT NULL THEN map_values['D'] ELSE NULL END AS D
, CASE WHEN element_at(map_values, 'E') IS NOT NULL THEN map_values['E'] ELSE NULL END AS E
, CASE WHEN element_at(map_values, 'F') IS NOT NULL THEN map_values['F'] ELSE NULL END AS F
, CASE WHEN element_at(map_values, 'G') IS NOT NULL THEN map_values['G'] ELSE NULL END AS G
, CASE WHEN element_at(map_values, 'H') IS NOT NULL THEN map_values['H'] ELSE NULL END AS H
, CASE WHEN element_at(map_values, 'J') IS NOT NULL THEN map_values['J'] ELSE NULL END AS J
, CASE WHEN element_at(map_values, 'K') IS NOT NULL THEN map_values['K'] ELSE NULL END AS K
, CASE WHEN element_at(map_values, 'L') IS NOT NULL THEN map_values['L'] ELSE NULL END AS L
, CASE WHEN element_at(map_values, 'M') IS NOT NULL THEN map_values['M'] ELSE NULL END AS M
, CASE WHEN element_at(map_values, 'N') IS NOT NULL THEN map_values['N'] ELSE NULL END AS N
, CASE WHEN element_at(map_values, 'O') IS NOT NULL THEN map_values['O'] ELSE NULL END AS O
, CASE WHEN element_at(map_values, 'P') IS NOT NULL THEN map_values['P'] ELSE NULL END AS P
, CASE WHEN element_at(map_values, 'R') IS NOT NULL THEN map_values['R'] ELSE NULL END AS R
, CASE WHEN element_at(map_values, 'S') IS NOT NULL THEN map_values['S'] ELSE NULL END AS S
, CASE WHEN element_at(map_values, 'T') IS NOT NULL THEN map_values['T'] ELSE NULL END AS T
, CASE WHEN element_at(map_values, 'U') IS NOT NULL THEN map_values['U'] ELSE NULL END AS U
, CASE WHEN element_at(map_values, 'V') IS NOT NULL THEN map_values['V'] ELSE NULL END AS V
, CASE WHEN element_at(map_values, 'W') IS NOT NULL THEN map_values['W'] ELSE NULL END AS W
, CASE WHEN element_at(map_values, 'X') IS NOT NULL THEN map_values['X'] ELSE NULL END AS X
, CASE WHEN element_at(map_values, 'Y') IS NOT NULL THEN map_values['Y'] ELSE NULL END AS Y
, CASE WHEN element_at(map_values, 'Z') IS NOT NULL THEN map_values['Z'] ELSE NULL END AS Z
, CASE WHEN element_at(map_values, 'A1') IS NOT NULL THEN map_values['A1'] ELSE NULL END AS A1
, CASE WHEN element_at(map_values, 'A2') IS NOT NULL THEN map_values['A2'] ELSE NULL END AS A2
, CASE WHEN element_at(map_values, 'A3') IS NOT NULL THEN map_values['A3'] ELSE NULL END AS A3
, CASE WHEN element_at(map_values, 'A4') IS NOT NULL THEN map_values['A4'] ELSE NULL END AS A4
, CASE WHEN element_at(map_values, 'A5') IS NOT NULL THEN map_values['A5'] ELSE NULL END AS A5
, CASE WHEN element_at(map_values, 'A6') IS NOT NULL THEN map_values['A6'] ELSE NULL END AS A6
, CASE WHEN element_at(map_values, 'A8') IS NOT NULL THEN map_values['A8'] ELSE NULL END AS A8
, CASE WHEN element_at(map_values, 'A9') IS NOT NULL THEN map_values['A9'] ELSE NULL END AS A9
, CASE WHEN element_at(map_values, 'A10') IS NOT NULL THEN map_values['A10'] ELSE NULL END AS A10
, CASE WHEN element_at(map_values, 'A11') IS NOT NULL THEN map_values['A11'] ELSE NULL END AS A11
, CASE WHEN element_at(map_values, 'A12') IS NOT NULL THEN map_values['A12'] ELSE NULL END AS A12
, vin
, date
from map_date
I can run the ABOVE query in ExecuteSqlRecord perfectly fine if I pivot only ~20 fields but when pivoting ~37 fields then the ExecuteSqlRecord processor immediately Fails.
It is interesting to see that I can run the same pivot query with ~37 fields in Presto CLI and QueryDatabaseTable successfully! Please help me achieve this in ExecuteSQLRecord processor as it has the added benefit of writing the output flowfile in any desired format (for my case, I am writing the output of the SQL query as CSV)
Here is the full stack trace of the error,
java.sql.SQLException: Error executing query. No FlowFile to route to failure: java.sql.SQLException: Error executing query
java.sql.SQLException: Error executing query
at com.facebook.presto.jdbc.PrestoStatement.internalExecute(PrestoStatement.java:279)
at com.facebook.presto.jdbc.PrestoStatement.execute(PrestoStatement.java:228)
at com.facebook.presto.jdbc.PrestoPreparedStatement.<init>(PrestoPreparedStatement.java:80)
at com.facebook.presto.jdbc.PrestoConnection.prepareStatement(PrestoConnection.java:129)
at org.apache.commons.dbcp2.DelegatingConnection.prepareStatement(DelegatingConnection.java:303)
at org.apache.commons.dbcp2.DelegatingConnection.prepareStatement(DelegatingConnection.java:303)
at org.apache.nifi.processors.standard.AbstractExecuteSQL.onTrigger(AbstractExecuteSQL.java:237)
at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27)
at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1176)
at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:213)
at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:117)
at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.lang.RuntimeException: Error fetching next at http://presto.testk8s.content.mywebspace.com:8889/b7/journal/20245300229_0354554_08r940_as8y4/2?slug=78234778257265 returned an invalid response: JsonResponse{statusCode=500, statusMessage=Server Error, headers={connection=[close]}, hasValue=false} [Error: ]
at com.facebook.presto.jdbc.internal.client.StatementClientV1.requestFailedException(StatementClientV1.java:446)
at com.facebook.presto.jdbc.internal.client.StatementClientV1.advance(StatementClientV1.java:386)
at com.facebook.presto.jdbc.PrestoResultSet.getColumns(PrestoResultSet.java:1742)
at com.facebook.presto.jdbc.PrestoResultSet.<init>(PrestoResultSet.java:119)
at com.facebook.presto.jdbc.PrestoStatement.internalExecute(PrestoStatement.java:250)
... 17 common frames omitted

Related

How to handle null and NULL value in sql

So I have been facing this weird situation in mysql where somehow null values are inserted in my table.
I'm talking about null value not NULL value.
I have attached image for better understating
as you can see name column have null and mobile_no have NULL
So after using this query
select Case when t1.name IS NULL then 'NA'
when t1.name= 'NA' or 'null' or NULL then 'NA'
else t1.name end as 'Name',
Case when t1.mobile_no IS NULL then 'NA'
when t1.mobile_no= 'NA' or 'null' or NULL then 'NA'
else t1.mobile_noend as 'Mobile no' from student;
after this I'm getting this result
|Name|Mobile no|
----------------
|null|NA |
but I want below result
|Name|Mobile no|
----------------
|NA |NA |

To compare a column with multiple values use col IN (x, y, z), not col = x OR y OR z.
You also can't compare with NULL using = or IN, so that has to be a separate check.
select
Case
when t1.name IS NULL OR t1.name IN ('NA' or 'null') then 'NA'
else t1.name
end as 'Name'

Can you try use IN statement? That should work.
select Case WHEN (t1.name IN ('NA', 'null') OR t1.name IS NULL) THEN 'NA' else t1.name end as 'Name',
Case WHEN (t1.mobile IN ('NA', 'null') OR t1.mobile IS NULL) THEN 'NA' else t1.mobile_no end as 'Mobile no' from student;

mysql combining two tables to produce the following output

hello i had ask the same question before but i want another way. we can use locate and max function to solve the question. but my teacher told me to do like this
my code is like this when '0' then '1' else '0'
select u.id, u.name,
case f.finger when '0' then '1' else '0' end as '0',
case f.finger when '1' then '1' else '0' end as '1',
case f.finger when '2' then '1' else '0' end as '2',
case f.finger when '3' then '1' else '0' end as '3',
case f.finger when '4' then '1' else '0' end as '4',
case f.finger when '5' then '1' else '0' end as '5'
from users u left join user_fingerprints f
on u.id= f.user_id
the code above will resulted in 8 rows.
then i need to combine the rows so that the rows with the same id combine to produce this
from that result then use case function. when the value is 1 then y. when value is zero then n.
can someone give me the answer without the use of max function? thanks
the table and result

You can also do it with SUM() and another CASE EXPRESSION :
SELECT t.id,t.name,
CASE WHEN t.`0` > 0 THEN 'Y' ELSE 'N' as `0`
CASE WHEN t.`1` > 0 THEN 'Y' ELSE 'N' as `1`
....
FROM (
SELECT u.id,u.name,
SUM(CASE WHEN f.finger = '0' then '1' else '0' end) as `0`,
SUM(CASE WHEN f.finger = '1' then '1' else '0' end) as `1`,
....
from users u left join user_fingerprints f
on u.id= f.user_id
GROUP BY u.id,u.name) t

Query to Get Counts of Ids under 2 different Conditions with Case When Statement

So I Have a Table Structure like the one below.
Here are the Conditions:
I call 'Tag' as the IDs with none of Condition(x) as NULL.
I call 'UnTag' as the IDs with any of the Condition(x) is NULL.
I want to Classify 'Classification1' in the below Format:
I want the output in the following structure:
I wrote a code but somehow it is not giving me the desired result.
I Used Case when to Classify the Classification1 and then ran a sub-query for taking out Count.
Please help if there is a different Approach and how to incorporate the Case when in Group by statement.
I wrote this query:
SELECT
CASE
WHEN UPPER(CM1.Classification1) IN ('A', 'C','G') THEN 'X'
WHEN UPPER(CM1.Classification1) IN ('B', 'F') THEN 'Y'
WHEN UPPER(CM1.Classification1) IN ('D', 'E') THEN 'Z'
ELSE 'Undefined' END AS New_Classification,
(SELECT
COUNT(CASE
WHEN UPPER(CM2.Classification1) IN ('A', 'C','G') THEN 'X'
WHEN UPPER(CM2.Classification1) IN ('B', 'F') THEN 'Y'
WHEN UPPER(CM2.Classification1) IN ('D', 'E') THEN 'Z'
ELSE 'Undefined' END)
FROM TABLE1 CM2 WHERE
CM2.Condition1 IS NOT NULL AND
CM2.Condition2 IS NOT NULL AND
CM2.Condition3 IS NOT NULL AND
CM2.Condition4 IS NOT NULL AND
CM1.ID=CM2.ID) AS Count_of_tagged,
(SELECT
COUNT(CASE
WHEN UPPER(CM3.Classification1) IN ('A', 'C','G') THEN 'X'
WHEN UPPER(CM3.Classification1) IN ('B', 'F') THEN 'Y'
WHEN UPPER(CM3.Classification1) IN ('D', 'E') THEN 'Z'
ELSE 'Undefined' END)
FROM TABLE1 CM3 WHERE
(CM3.Condition1 IS NULL OR
CM3.Condition2 IS NULL OR
CM3.Condition3 IS NULL OR
CM3.Condition4 IS NULL) AND
CM1.ID=CM2.ID) AS Count_of_untagged
FROM TABLE1 CM1

You shouldn't need a subquery, as a starting point I came up with:
SELECT CASE Classification1
WHEN 'A' THEN 'X'
WHEN 'B' THEN 'Y'
WHEN 'C' THEN 'X'
WHEN 'D' THEN 'Z'
WHEN 'E' THEN 'Z'
WHEN 'F' THEN 'Y'
WHEN 'G' THEN 'X'
END new_classification,
SUM(
Condition1 IS NOT NULL AND
Condition2 IS NOT NULL AND
Condition3 IS NOT NULL AND
Condition4 IS NOT NULL /* 1 for true, 0 for false */
) tagged_count,
SUM(
Condition1 IS NULL OR
Condition2 IS NULL OR
Condition3 IS NULL OR
Condition4 IS NULL
) untagged_count,
FROM table_name
GROUP BY new_classification

Can give a try at the following sql and tell me if this is what you need
select
CASE
WHEN classification ='A' THEN 'X'
WHEN classification ='B' THEN 'Y'
WHEN classification ='C' THEN 'X'
WHEN classification ='D' THEN 'Z'
WHEN classification ='E' THEN 'Z'
WHEN classification ='F' THEN 'Y'
WHEN classification ='G' THEN 'X'
END as classification,
sum(case when condition1 is not null and condition2 is not null and condition3 is not null and condition4 is not null then 1 else 0 end) as tagged
sum(case when condition1 is null or condition2 is null or condition3 is null or condition4 is null then 1 else 0 end) as untagged
from table
GROUP BY CASE
WHEN classification ='A' THEN 'X'
WHEN classification ='B' THEN 'Y'
WHEN classification ='C' THEN 'X'
WHEN classification ='D' THEN 'Z'
WHEN classification ='E' THEN 'Z'
WHEN classification ='F' THEN 'Y'
WHEN classification ='G' THEN 'X'
END ;

PostgreSQL Error 22012 devided by Zero after Truncate The Table

So after I truncated my table, this query which worked before, now doesn't work. Even I've already filled again my table, It still doesn't work.
Here's my postgreSQL code :
SELECT
k.reviewer AS namareviewer
, COUNT(k.formcode) AS actual
, ROUND((0.2*COUNT(k.formcode))) AS target
, SUM(CASE WHEN k.blibliknowledge != '' THEN 1 ELSE 0 END) AS blibli
, COUNT(CASE WHEN k.solusi != '' THEN 'foo' ELSE NULL END) AS solusi
, ROUND(((COUNT(CASE WHEN k.solusi != '' THEN 'foo' ELSE NULL END)+ SUM(CASE WHEN k.blibliknowledge != '' THEN 1 ELSE 0 END)) /ROUND((0.2*COUNT(k.formcode))))*100,2) as Percentage
FROM kpi k
GROUP
BY k.reviewer
And my KPI Table
That data are the same with the first data which the query already worked. Why The error SQL state : 22012 devided by zero come up after the first table truncated? Thankyou in advanced :D

Use NULLIF to avoid divide by zero error
....
, ROUND(((COUNT(CASE WHEN k.solusi != '' THEN 'foo' ELSE NULL END)+ SUM(CASE WHEN k.blibliknowledge != '' THEN 1 ELSE 0 END))
/NULLIF(ROUND((0.2*COUNT(k.formcode))))*100,2) ,0) as Percentage
........
Complete code :
SELECT
k.reviewer AS namareviewer,
COUNT(k.formcode) AS actual,
ROUND((0.2 * COUNT(k.formcode))) AS target,
SUM(CASE
WHEN k.blibliknowledge != '' THEN 1
ELSE 0
END) AS blibli,
COUNT(CASE
WHEN k.solusi != '' THEN 'foo'
ELSE NULL
END) AS solusi,
ROUND(((COUNT(CASE
WHEN k.solusi != '' THEN 'foo'
ELSE NULL
END) + SUM(
CASE
WHEN
k.blibliknowledge != '' THEN 1
ELSE 0
END)) / NULLIF(ROUND((
0.2 * COUNT(k.formcode))), 0)) * 100, 2) AS Percentage
FROM kpi k
GROUP BY k.reviewer

mysql display results horizontally

I wrote some query and i got following query results.
createdate SITE1 SITE2 SITE3 SITE4 SITE5 SITE6 SITE7 SITE8 SITE9
10/2/2014 63 NULL NULL NULL NULL NULL NULL NULL NULL
10/2/2014 NULL NULL NULL NULL NULL NULL NULL NULL 10
10/3/2014 21 NULL NULL NULL NULL NULL NULL NULL NULL
10/3/2014 NULL NULL NULL NULL NULL NULL NULL NULL 4
Now i want to display like this.
createdate SITE1 SITE2 SITE3 SITE4 SITE5 SITE6 SITE7 SITE8 SITE9
10/2/2014 63 NULL NULL NULL NULL NULL NULL NULL 10
10/3/2014 21 NULL NULL NULL NULL NULL NULL NULL 4
Note: I already grouping SITE NAME and createdate.
Do u have any cool idea or something pls suggest me..
enter code here
select alldata.createdate, count(name),
case when name = 'SITE1' then count(1) end AS SITE1,
case when name = 'SITE2' then count(1) end AS SITE2,
case when name = 'SITE3' then count(1) end AS SITE3,
case when name = 'SITE4' then count(1) end AS SITE4,
case when name = 'SITE5' then count(1) end AS SITE5,
case when name = 'SITE6' then count(1) end AS SITE6,
case when name = 'SITE7' then count(1) end AS SITE7,
case when name = 'SITE8' then count(1) end AS SITE8,
case when name = 'SITE9' then count(1) end AS SITE9,
from
(
select total.createdate as createdate ,total.name as name from
(
select distinct dtr.* , ps.name ,DATE_FORMAT(dt.create_time,'%Y-%m-%d') as createdate from di_dtb_task_result dtr
inner join di_dtb_task_data_detail dt
on dtr.task_id = dt.task_id and dt.user_id <> 10000001
left join dtb_user_info ui
on ui.id = dt.user_id and dtr.task_id = dt.task_id
left join dtb_point_site ps
on ps.id=ui.user_point_site_id and dtr.task_id = dt.task_id
where dtr.status <> 9
-- group by dtr.status ,dtr.task_id , ui.id,ui.user_point_site_id , ps.name
order by dtr.task_id
) total
-- group by total.name , total.createdate
) alldata
group by alldata.createdate ,alldata.name

I think you may want to try max() function
select createdate, max(site1),max(site2),max(site3),max(site4),max(site5),max(site6),max(site7), max(site8),max(site9) from (..your_inner_query..) group by createdate
here you go with a sample sqlfiddle for your reference

Simplest solution:
Just add another group by sql wrapping your sql.
select alldata.createdate, sum(SITE1), sum(SITE2), sum(SITE3), sum(SITE4), sum(SITE5), sum(SITE6), sum(SITE7), sum(SITE8), sum(SITE9)
// inner select
group by alldata.createdate