Select from json_table is slow - json

I got a problem with performance of Insert into Select queries, where i use json_table mechanism. And i think its problem only with select's- since inserts are preety fast.
Im using Oracle 19c database.
I got entire json (that weights over 50mb) stored as clob in a table.
The dynamicly generated queries (i pasted only some of them, there are like 14 queries like that per each dailyReport object), which i run by execute immediate looks like:
INSERT INTO STORING_JSON_TABLE ( ID_ST_PTU ,WART )
SELECT ID_ST_PTU ,WART
FROM STORING_JSON_TABLE,
JSON_TABLE(json_data,'$.dailyReports[0].jpk.content.rapDob.stPTU[1]'
COLUMNS (
ID_ST_PTU PATH '$.id'
,WART PATH '$.wart'
)
)
INSERT INTO STORING_JSON_TABLE ( ID_ST_PTU ,WART )
SELECT ID_ST_PTU ,WART
FROM STORING_JSON_TABLE,
JSON_TABLE(json_data,'$.dailyReports[0].jpk.content.rapDob.stPTU[2]'
COLUMNS (
ID_ST_PTU PATH '$.id'
,WART PATH '$.wart'
)
)
INSERT INTO STORING_JSON_TABLE ( ID_ST_PTU ,WART )
SELECT ID_ST_PTU ,WART
FROM STORING_JSON_TABLE,
JSON_TABLE(json_data,'$.dailyReports[0].jpk.content.rapDob.stPTU[3]'
COLUMNS (
ID_ST_PTU PATH '$.id'
,WART PATH '$.wart'
)
)
Those 3 queries take up to 2 seconds to execute- and i think im doing something wrong.
Ofcourse, i could do one select and then create only Insert's not Insert Into Select's, but still it would be slow- since i have to import around 50 000 dailyReport's each day, and as you can see each dailyReport can contain multiple elements in different array's (i used only one object array as example).
I also tried running them in PLSQL block without EXECUTE IMMEDIATE but they are not working any faster.
I feel like json_table dont pay much attention on the numbers im giving to him, for example in those queries i say that i want to get data only from dailyReport[0], but i feel like it is searching the whole json for that.
I already read about indexes, and i think i found one that could fit my needs, its called: Multivalue Function-Based Index, but seems its not aviable in Oracle 19c, only in Oracle 21.
The json structure looks like:
"dailyReports": [
{
"jpk": {
"naglowek": {
"wersja": "JPK_KASA_v2-0",
"dataJPK": "2022-07-19T21:59:01.17Z"
},
"podmiot1": {
"nip": "XXX",
"nazwaPod": "XXX",
"adresPod": {
"kodPoczt": "XXX",
"miejsc": "XXX XXX",
"nrDomu": "XXX",
"ulica": "XXX"
},
"nrUnik": "XXX",
"nrFabr": "XXX",
"nrEwid": "2022/XXX"
},
"content": [
{
"rapDob": {
"jpkid": 108,
"pamiecChr": 1,
"nrDok": 106,
"nrRap": 6,
"rozpSprzed": "2022-07-19T12:01:54.631Z",
"zakSprzed": "2022-07-19T12:01:54.631Z",
"stPTU": [
{
"id": "A",
"wart": 2300
},
{
"id": "B",
"wart": 800
},
{
"id": "C",
"wart": 500
},
{
"id": "D",
"wart": 0
},
{
"id": "E",
"wart": "ZW"
},
{
"id": "F",
"wart": 0
},
{
"id": "G",
"wart": 0
}
],
"sprzedPar": {
"sumaBrutto": 499,
"sumaPod": 0,
"wartWgPTU": [
{
"idStPTU": "A",
"netto": 0,
"vat": 0
},
{
"idStPTU": "B",
"netto": 0,
"vat": 0
},
{
"idStPTU": "C",
"netto": 0,
"vat": 0
},
{
"idStPTU": "D",
"netto": 499,
"vat": 0
},
{
"idStPTU": "E",
"netto": 0,
"vat": 0
},
{
"idStPTU": "F",
"netto": 0,
"vat": 0
},
{
"idStPTU": "G",
"netto": 0,
"vat": 0
}
],
"sprzedNO": 0
},
"podatekNal": 0,
"sprzedBrutto": 499,
"sprzedNO": 0,
"waluta": "PLN",
"sytAwaryjne": 0,
"zdarzProgramL": 0,
"zdarzProgramO": 0,
"zmBazyTow": 1,
"liczbaPar": 1,
"liczbaParAnul": 0,
"wartParAnul": 0,
"dokNiefisk": 4,
"zakRap": "2022-07-19T21:59:00.935Z",
"nrKasy": "0",
"kasjer": "XXX",
"podpis": {
"rsa": "XXX",
"sha": "XXX",
"jpk": "XXX"
}
}
}
]
}
},
{
"jpk": {
"naglowek": {
"wersja": "JPK_KASA_v2-0",
"dataJPK": "2022-07-22T07:45:58.658Z"
},
"podmiot1": {
"nip": "XXX",
"nazwaPod": "XXXXXX.",
"adresPod": {
"kodPoczt": "XXX",
"miejsc": "XXX",
"nrDomu": "XXX",
"ulica": "XXX"
},
"nrUnik": "XXX",
"nrFabr": "XXX",
"nrEwid": "2022/XXX"
},
"content": [
{
"rapDob": {
"jpkid": XXX,
"pamiecChr": 1,
"nrDok": 112,
"nrRap": 7,
"rozpSprzed": "0001-01-01T00:00:00",
"zakSprzed": "0001-01-01T00:00:00",
"stPTU": [
{
"id": "A",
"wart": 2300
},
{
"id": "B",
"wart": 800
},
{
"id": "C",
"wart": 500
},
{
"id": "D",
"wart": 0
},
{
"id": "E",
"wart": "ZW"
},
{
"id": "F",
"wart": 0
},
{
"id": "G",
"wart": 0
}
],
"sprzedPar": {
"sumaBrutto": 0,
"sumaPod": 0,
"wartWgPTU": [
{
"idStPTU": "A",
"netto": 0,
"vat": 0
},
{
"idStPTU": "B",
"netto": 0,
"vat": 0
},
{
"idStPTU": "C",
"netto": 0,
"vat": 0
},
{
"idStPTU": "D",
"netto": 0,
"vat": 0
},
{
"idStPTU": "E",
"netto": 0,
"vat": 0
},
{
"idStPTU": "F",
"netto": 0,
"vat": 0
},
{
"idStPTU": "G",
"netto": 0,
"vat": 0
}
],
"sprzedNO": 0
},
"podatekNal": 0,
"sprzedBrutto": 0,
"sprzedNO": 0,
"waluta": "PLN",
"sytAwaryjne": 0,
"zdarzProgramL": 0,
"zdarzProgramO": 0,
"zmBazyTow": 0,
"liczbaPar": 0,
"liczbaParAnul": 0,
"wartParAnul": 0,
"dokNiefisk": 5,
"zakRap": "2022-07-22T07:45:58.425Z",
"nrKasy": "0",
"kasjer": "XXX",
"podpis": {
"rsa": "XXX",
"sha": "XXX",
"jpk": "XXX"
}
}
}
]
}
},
etc....
Any ideas how to speed up the queries ?
Execution plan for Select queries:

On 21c, syntax
'$.dailyReports[0].jpk.content.rapDob.stPTU[1,2,3]'
and
'$.dailyReports[0].jpk.content.rapDob.stPTU[1 to 3]'
work,
it will allow you to do 1 INSERT instead of 3.
Don't have hand on a 19c right now to verify.

Related

How do I get back the value from one of the nested dictionary from JSON file

i am new to python and json. I have this issue where i am unable to get the values of 'labels' print out. It is under a nested dictionary 'result' .
Code that i have tried
for i in data['result']:
print("Value:", i['value'])
values = i['value']
print("X:", values['x'])
print("Y:", values['y'])
print("Width:", values['width'])
print("Height:", values['height'])
Output from the above code
However when i try to do print("labels:", values['labels'])
it return a keyerror: 'labels'
for item in values:
print(item)
this is the output shown
x
y
width
height
rotation
x
y
width
height
rotation
labels
May i ask how do i go out to print out the 'labels' value?
The json files is as follows:
{
"id": 9,
"created_username": "",
"created_ago": "3\u00a0weeks, 3\u00a0days",
"completed_by": {
"id": 1,
"first_name": "",
"last_name": "",
"email": ""
},
"task": {
"id": 1,
"data": {
"image": "/data/upload/2/adf8540d-2-33.png"
},
"meta": {},
"created_at": "2022-11-21T14:52:22.478537Z",
"updated_at": "2022-11-22T12:51:18.105745Z",
"is_labeled": true,
"overlap": 1,
"inner_id": 1,
"total_annotations": 1,
"cancelled_annotations": 0,
"total_predictions": 0,
"comment_count": 0,
"unresolved_comment_count": 0,
"last_comment_updated_at": null,
"project": 2,
"updated_by": 1,
"file_upload": 70,
"comment_authors": []
},
"result": [
{
"original_width": 512,
"original_height": 512,
"image_rotation": 0,
"value": {
"x": 50.898958419872464,
"y": 44.0069438675168,
"width": 2.397222452993284,
"height": 2.0975696463691196,
"rotation": 0
},
"id": "pgaiV8NjWC",
"from_name": "rect",
"to_name": "image",
"type": "rectangle",
"origin": "manual"
},
{
"original_width": 512,
"original_height": 512,
"image_rotation": 0,
"value": {
"x": 50.898958419872464,
"y": 44.0069438675168,
"width": 2.397222452993284,
"height": 2.0975696463691196,
"rotation": 0,
"labels" [
"LM"*
]
},
"id": "pgaiV8NjWC",
"from_name": "labels",
"to_name": "image",
"type": "labels",
"origin": "manual"
}
],
"was_cancelled": false,
"ground_truth": false,
"created_at": "2022-11-22T12:51:18.001777Z",
"updated_at": "2022-11-22T12:51:18.001777Z",
"lead_time": 21.556,
"project": 2,
"parent_prediction": null,
"parent_annotation": null
}
Use the json module
import json
dict = json.load(json_filename)
Then you should access the values by the dictionary keys. Be aware that the result value is a list. Therefore the index 1 is necessary.
values = dict['result'][1]['value']
print("X:", values['x'])

MySQL 8 Json document merge data from multiple rows

I've JSON documents in multiple rows as
Row #1
{
"data": {
"level": 1,
"name": "xyz",
"property": "value",
"children": [
{
"level": 2,
"name": "xyz-1"
}
]
}
}
Row #2
{
"data": {
"children": [
{
"level": 2,
"name": "xyz-2"
},
{
"level": 2,
"name": "xyz-3"
}
]
}
}
Row #3
{
"data": {
"children": [
{
"level": 2,
"name": "xyz-4"
}
]
}
}
I want to use MySQL 8 JSON_MERGE_PRESERVE in such a way so I get the result
{
"data": {
"level": 1,
"name": "xyz",
"property": "value",
"children": [
{
"level": 2,
"name": "xyz-1"
},
{
"level": 2,
"name": "xyz-2"
},
{
"level": 2,
"name": "xyz-3"
},
{
"level": 2,
"name": "xyz-4"
}
]
}
}
I've tried
SELECT JSON_MERGE_PRESERVE(
'{ "data": { "level": 1, "name": "xyz", "property": "value", "children": [ { "level": 2, "name": "xyz-1" } ] } }',
'{ "data": { "children": [ { "level": 2, "name": "xyz-2" }, { "level": 2, "name": "xyz-3" } ] } }',
'{ "data": { "children": [ { "level": 2, "name": "xyz-4" } ] } }'
) as json;
But I want to SELECT the JSON data from the table and merge it, something like
[Not working]
SELECT JSON_MERGE_PRESERVE(a.data_json) from
(SELECT data_json FROM data_table
WHERE name = 'abc') as a
The error message is
Error Code: 1582. Incorrect parameter count in the call to native function 'JSON_MERGE_PRESERVE'
One of the many options you can evaluate and use is CTE (Common Table Expressions) - 13.2.13 WITH (Common Table Expressions):
WITH RECURSIVE `cte` AS (
SELECT
1 AS `row`,
`data_json`
FROM
`data_table`
WHERE
`id` = 1
UNION ALL
SELECT
`cte`.`row` + 1 AS `row`,
JSON_MERGE_PRESERVE(`cte`.`data_json`, `data_table`.`data_json`) AS `data_json`
FROM
`data_table`, `cte`
WHERE
`data_table`.`id` = `cte`.`row` + 1
)
SELECT
JSON_PRETTY(`data_json`)
FROM
`cte`
ORDER BY
`row` DESC
LIMIT 1;
See dbfiddle.

Elastic Nested query - display only the first 2 inner hits

How do I change my query to display only the 5 first orders within the orderbook?
My data is structure like this. Order is a nested type.
Orderbook
|_ Orders
This is my query
GET /orderindex/_search
{
"size": 10,
"query": {
"term": { "_type": "orderbook" }
}
}
And this is the result
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 10,
"max_score": 1,
"hits": [
{
"_index": "orderindex",
"_type": "orderbook",
"_id": "1",
"_score": 1,
"_source": {
"id": 1,
"exchange": "Exch1",
"label": "USD/CAD",
"length": 40,
"timestamp": "5/16/2018 4:33:31 AM",
"orders": [
{
"pair1": "USD",
"total": 0.00183244,
"quantity": 61,
"orderbookId": 0,
"price": 0.00003004,
"exchange": "Exch1",
"id": 5063,
"label": "USD/CAD",
"pair2": "CAD",
},
{
"pair1": "USD",
"total": 0.0231154,
"quantity": 770,
"orderbookId": 0,
"price": 0.00003002,
"exchange": "Exch1",
"id": 5064,
"label": "USD/CAD",
"pair2": "CAD",
},
...
..
.
Also, how do I make to query two specific orderbooks by its label name and retrieve only the first 2 orders?
I am now sending of this query, but the problem is it is returning the orderbooks including all its orders and then after this it returns only 2 plus inners hits. How Do I do to return only the 2 inner hits without all the orders that are coming with the orderbook from the first part of the query
GET /orderindex/_search
{
"query": {
"bool": {
"must": [
{
"term": { "_type": "orderbook" }
},
{
"nested": {
"path": "orders",
"query": {
"match_all": {}
},
"inner_hits": {
"size": 2
}
}
}
]
}
}}
Inner hits support the following options:
size
The maximum number of hits to return per inner_hits. By default the
top three matching hits are returned.
Which basically means, that you could do that, by using similar query
{
"query": {
"bool": {
"must": [
{
#replace this one with your query for orderbook
"term": {
"user": "kimchy"
}
},
{
"nested": {
"path": "orders",
"query": {
"match_all": {}
},
"inner_hits": {
"size": 3 #we asks for only 3 inner hits
}
}
}
]
}
}
}
One could also would like to filter _source from results by doing this:
"_source": {
"includes": [ "obj1.*", "obj2.*" ],
"excludes": [ "*.description" ]
}
In your case of orders - it could be useful to excludes orders.*
More information on this one - https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-inner-hits.html

Iterate through Jmeter JSON response

I am using Jmeter and getting the below response
[
{
"paxJourneyID": 9431,
"name": "KIRK JAMES MR",
"gender": "M",
"flight": {
"carrier": "ABC",
"fltNum": "0219",
"depDate": "2017-12-29T00:00:00",
"board": "DXB",
"off": "BAH"
},
"seqNum": 0,
"pnrNum": "5D293N",
"tktNum": "1412100000150",
"priority": {
"code": "",
"entitledClass": ""
},
"groupCode": "",
"poolID": 0,
"poolHeadID": 0,
"isPrimaryPax": true,
"parentID": 0,
"totBagPieces": 0,
"totBagWeight": 0,
"outbound": [],
"class": "Y",
"paxStatusID": 0,
"appStatusID": 1,
"primaryDoc": {
"docTypeID": 1,
"docNum": "B76576557"
}
},
{
"paxJourneyID": 12356,
"name": "GREER JOAN MSTR",
"gender": "M",
"flight": {
"carrier": "ABC",
"fltNum": "0329",
"depDate": "2017-12-29T00:00:00",
"board": "DXB",
"off": "BAH"
},
"seqNum": 0,
"pnrNum": "4WMAIT",
"tktNum": "",
"priority": {
"code": "",
"entitledClass": ""
},
"groupCode": "",
"poolID": 0,
"poolHeadID": 0,
"isPrimaryPax": false,
"parentID": 123,
"totBagPieces": 0,
"totBagWeight": 0,
"outbound": [],
"class": "Y",
"paxStatusID": 0,
"appStatusID": 2,
"primaryDoc": {
"docTypeID": 1,
"docNum": "767"
}
}
]
I have to select the part of response where parentid is 0 use it in next request. How to do the same ?
You can use JSON Extractor and the following JSONPath query:
$..[?(#.parentID == 0)]
Demo:
More information: JMeter's JSON Path Extractor Plugin - Advanced Usage Scenarios

Getting error while parsing json response from a dynamic {System.RuntimeType} variable

I'm working on some code in which uses dynamic variables jsonResponse .
dynamic jsonResponse = JsonConvert.DeserializeObject(response);
This variable contains collection of hotel list in json format. From this collection I am getting roomlist collection in a new variable roomResponseList :
var roomResponseList = jsonResponse["hotels"]["hotels"][rooms].roomResponseList;
I am getting first room detail into **JObject responseRateKeys **:
foreach (var roomByResponse in roomResponseList)
{
JObject responseRateKeys = JObject.Parse(roomByResponse.ToString());
var boardNameListByResponse = responseRateKeys.AsJEnumerable().AsEnumerable()
.Select(t => t["rates"]["boardName"].ToString().Trim())
.Distinct()
.ToList();
}
But when I am trying to get any item list from JObject by using linq lambda, I am getting error,
"Cannot access child value on Newtonsoft.Json.Linq.JProperty."
Value of roomByResponse=
{ "code": "DBL.KG-NM", "name": "DOUBLE KING BED NON SMOKING", "rates": [ { "rateKey": "20171217|20171219|W|256|237403|DBL.KG-NM|ID_B2B_26|RO|IWH25|1~1~0||N#AFF5C93E36054661ADCBC14A78A532AE1007", "rateClass": "NRF", "rateType": "RECHECK", "net": "186.04", "allotment": 99, "paymentType": "AT_WEB", "packaging": false, "boardCode": "RO", "boardName": "ROOM ONLY", "cancellationPolicies": [ { "amount": "149.63", "from": "2017-07-14T03:29:00+05:30" } ], "rooms": 1, "adults": 1, "children": 0, "dailyRates": [ { "offset": 1, "dailyNet": "93.02" }, { "offset": 2, "dailyNet": "93.02" } ] }, { "rateKey": "20171217|20171219|W|256|237403|DBL.KG-NM|ID_B2B_26|BB|IWB25|1~1~0||N#AFF5C93E36054661ADCBC14A78A532AE1007", "rateClass": "NOR", "rateType": "RECHECK", "net": "238.92", "allotment": 99, "paymentType": "AT_WEB", "packaging": false, "boardCode": "BB", "boardName": "BED AND BREAKFAST", "rooms": 1, "adults": 1, "children": 0, "dailyRates": [ { "offset": 1, "dailyNet": "119.46" }, { "offset": 2, "dailyNet": "119.46" } ] }, { "rateKey": "20171217|20171219|W|256|237403|DBL.KG-NM|ID_B2B_26|RO|IWH25|2~2~1|2|N#AFF5C93E36054661ADCBC14A78A532AE1007", "rateClass": "NRF", "rateType": "RECHECK", "net": "372.06", "allotment": 99, "paymentType": "AT_WEB", "packaging": false, "boardCode": "RO", "boardName": "ROOM ONLY", "cancellationPolicies": [ { "amount": "299.25", "from": "2017-07-14T03:29:00+05:30" } ], "rooms": 2, "adults": 2, "children": 1, "childrenAges": "2", "dailyRates": [ { "offset": 1, "dailyNet": "186.03" }, { "offset": 2, "dailyNet": "186.03" } ] }, { "rateKey": "20171217|20171219|W|256|237403|DBL.KG-NM|ID_B2B_26|BB|IWB25|2~2~1|2|N#AFF5C93E36054661ADCBC14A78A532AE1007", "rateClass": "NOR", "rateType": "RECHECK", "net": "477.84", "allotment": 99, "paymentType": "AT_WEB", "packaging": false, "boardCode": "BB", "boardName": "BED AND BREAKFAST", "rooms": 2, "adults": 2, "children": 1, "childrenAges": "2", "dailyRates": [ { "offset": 1, "dailyNet": "238.92" }, { "offset": 2, "dailyNet": "238.92" } ] } ] }
Thank you
Pravesh Singh
change linq to
responseRateKeys["rates"].AsJEnumerable().Select(t=>t["boardName"]).Distinct().ToList()