I'm working with Snowflake, and have a batch of JSON data in a variant datatype. I'm going to paste just the first entry (I hope) out of megabytes of data.
I need to figure out the SQL to extract the data elements from it.
This query:
SELECT
src:value::string AS valuest,
src:value:evaluation_forms as evaluation_forms,
src:value:evaluation_forms:evaluations as evaluations,
src:value:evaluation_forms:evaluations:channel_meta as channel_meta,
src:value:evaluations:channel_meta:agent_first_name as agent_first_name
FROM stage.intuit_quality_raw_table
retrieves JSON data in valuest, JSON data in evaluation_forms, and nulls in evaluations, channel_meta, and agent_first_name.
I suspect that that is because of the '[' characters that occur in from of the evaluations and channel_meta entries, but I just don't know how to deal with that sort of data.
I aplogize the being so long. The first entry is something over 300 lines long, and I'm afraid to try to shorten it because I fear that I will delete something important. I've already added the ending batch of stuff, which I'm also afraid that I might have messed up.
Anyway, here it is:
{"value":
{"evaluation_forms":
[ {"evaluations":
[ {"channel_meta":
{"after_call_work_time": [],
"agent_first_name": ["KATRINA"],
"agent_hung_up": [],
"agent_last_name": ["COX"],
"agent_unique_id": ["LO_00130604"],
"agent_username": [],
"alternate_call_id": [],
"audio_file_name": ["b161aa50-cda9-42e2-ad16-117a9cf08b92.wav"],
"call_direction": ["INBOUND"],
"call_id": ["b161aa50-cda9-42e2-ad16-117a9cf08b92"],
"channel_type": "Call",
"customer_id": [],
"customer_segment": [],
"disposition": [],
"disposition_time": [],
"end_at": ["2023-01-24 12:00:00"],
"file_location": ["blank"],
"handle_time": ["1824"],
"hold_time": [],
"how_call_ended": ["Transferred"],
"how_call_originated": [],
"id": "3cb5a3e5-0ab8-4a38-9f6b-4ab3b2e24351",
"ivr_options": [],
"line": ["Intuit"],
"notes": [],
"queue": ["Product Support"],
"queue_time": [],
"start_at": ["2023-01-24 12:00:00"]
"subqueue": [ "Product Support - Desktop" ],
"supplemental_field_1": ["Expert Service Score: 3"],
"supplemental_field_10": [],
"supplemental_field_2": ["Derived TNPS Score: 0"],
"supplemental_field_3": [],
"supplemental_field_4": [],
"supplemental_field_5": [],
"supplemental_field_6": [],
"supplemental_field_7": [],
"supplemental_field_8": [],
"supplemental_field_9": [],
"talk_time": ["78"],
"team_id": [],
"times_held": ["2"],
"total_time": []
},
"evaluation_list": [
{"evaluation_meta":
{"ack_info":
{"ack_init_time":
null,
"ack_status": "NOT_INITIATED",
"ack_time": null,
"dispute_raised_at": null,
"dispute_raised_by_email": null,
"dispute_resolved_at": null,
"dispute_resolved_by_email": null
},
"agent_email": "lo_00130604#liveopsnation.com",
"agent_id": "63d77af042a6982e25008bf3",
"agent_name": "KATRINA COX",
"agent_status": "ACTIVE",
"created_at": "2023-02-06T02:16:53.637Z",
"evaluation_purpose": null,
"evaluation_type": "Manual",
"evaluator_email": "info#ara-gs.com",
"evaluator_id":"63cb37e42db1035ae08285d1",
"evaluator_name": "RAMON CARPIO JR�",
"evaluator_status": "ACTIVE",
"id": "63e06221ea5e4c7202572252",
"modified_at": "2023-02-06T02:17:35.229Z",
"status": "EDITED"
},
"response": {
"comment": "",
"sections": [
{
"id": "3fe2faeb-e8bc-4a16-9fd1-49166a3eed4d",
"questions": [
{
"comment": "",
"fail": false,
"id": "4c057d99-aae7-4432-8634-6c41e4cde734",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "",
"fail": false,
"id": "f23b5e63-b24c-49f7-aa46-eb7499cdf89d",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "",
"fail": false,
"id": "4798443d-eaf3-46e7-bc9f-1f03ccee871a",
"score_obtained": 1,
"value": "Yes"
}],
"score_obtained": 3
},
{"id": "f5a83974-3be3-4f69-a9de-49866213b383",
"questions": [ {
"comment": "",
"fail": false,
"id": "52ec152b-fce8-464d-a8cf-b4ae82bfb5d9",
"score_obtained": 0,
"value": "N/A"
},
{"comment": "",
"fail": false,
"id": "627ce8fd-2d29-44d9-8c3d-49dfdab9f7b2",
"score_obtained": 0,
"value": "N/A"
},
{ "comment": "",
"fail": false,
"id": "a6efd5fe-42e1-4411-b197-ee715b579281",
"score_obtained": 0,
"value": "N/A"
},
{"comment": "",
"fail":false,
"id": "aea7d958-c27e-44ab-a32f-c2388b8c6aab",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "",
"fail": false,
"id": "860ac0b2-c325-443d-8f91-63879d09bac8",
"score_obtained": 0,
"value": "N/A"
} ],
"score_obtained": 1
},
{"id": "6a5fa573-8dc5-4dbb-b6cd-2e01445c3b4c",
"questions": [
{"comment": "",
"fail": false,
"id": "22e2cc11-c886-4b56-8de6-e97b773de4d0",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "Agent transferred call incorrectly. Another agent advised her of it then call disconnected. ",
"fail":true,
"id": "39b2ab0d-7372-408e-b8e2-886be10239de",
"score_obtained": 0,
"value": "No"}
],
"score_obtained": 1
},
{"id": "d1deed99-e8aa-49f1-8c32-079887fd6838",
"questions": [
{"comment": "",
"fail": false,
"id": "b951c53c-52b9-4a08-b3c6-70298643db22",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "",
"fail": false,
"id": "2321f675-01f9-4975-b7e9-d57feb865d0e",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "",
"fail": false,
"id": "6b0eb24c-f31c-41f0-85d3-56805d42284d",
"score_obtained": 1,
"value": "Yes"
},
{"comment": "",
"fail":false,
"id": "7e5e7a2e-60c0-4416-814e-fe19c9ab2361",
"score_obtained": 1,
"value": "Yes"
},
{
"comment": "",
"fail": false,
"id": "19f6ec0a-4c4c-49b6-962e-d029a0326d27",
"score_obtained": 1,
"value": "No"
} ],
"score_obtained": 5
},
{"id": "6c41a906-3d69-469d-9391-c814a6124ec1",
"questions": [
{"comment": "",
"fail": false,
"id": "d72fca9d-323d-4238-980e-c0aec746e1fc",
"score_obtained": 1,
"value": "Yes"},
{
"comment": "",
"fail": false,
"id": "d1af8d8e-6a97-4111-bad7-a64695f2c3f7",
"score_obtained": 1,
"value": "Yes"
},
{
"comment": "",
"fail":true,
"id": "c1251cb6-c3fe-4f69-bbe5-9510f150d18a",
"score_obtained": 0,
"value": "No"
}],
"score_obtained": 2
},
{
"id": "ca1c275b-1cd0-49c9-a4d7-9fd0544b4bbf",
"questions": [
{
"comment": "",
"fail": true,
"id": "4f3268ba-4e8d-46b8-a50a-69ea9de88a6a",
"score_obtained": 0,
"value": "No"
},
{
"comment": "",
"fail": true,
"id": "cb82d886-8e2c-4d4c-a8f8-b5c769bf289a",
"score_obtained": 0,
"value": "No"
},
{
"comment": "",
"fail": true,
"id": "ea9aa819-6905-4f15-a702-3713bbb033a9",
"score_obtained": 0,
"value": "No"
}
],
"score_obtained": 0
},
{
"id": "5743b6e8-ac6f-4d34-8561-f7e5191d5f37",
"questions": [
{
"comment": "",
"fail": false,
"id": "af04685e-5c73-47be-8282-493960cce3e0",
"score_obtained": 1,
"value": "Yes"
},
{
"comment": "",
"fail": false,
"id": "67b8ea9f-d481-42c2-9343-e586875e0929",
"score_obtained": 1,
"value": "Yes"
},
{
"comment": "",
"fail":false,
"id": "9fc25686-a63c-4db4-b2aa-dcf8666d2147",
"score_obtained": 1,
"value": "Yes"
},
{
"comment": "",
"fail": false,
"id": "410616c0-0d41-4322-b6a3-df11c3c28963",
"score_obtained": 1,
"value": "Yes"
},
{
"comment": "",
"fail": false,
"id": "575c978e-365d-4b83-842e-acfe41de09ae",
"score_obtained": 1,
"value": "Yes"
} ],
"score_obtained": 5
},
"id": "bced1392-24b2-404f-91be-9df6d68e0c8d",
"questions": [
{
"comment": "",
"fail": false,
"id": "e2250777-8bba-4fee-b9b4-1535b42be14c",
"score_obtained": 0,
"value": "No"
},
{
"comment": "",
"fail": false,
"id": "533281a0-63db-4af6-b41e-7b967a456332",
"score_obtained": 0,
"value": "No"
},
{
"comment": "",
"fail": false,
"id": "c9cf8f64-427b-4927-8ffb-02454e6daa43",
"score_obtained": 0,
"value": "No"
},
{
"comment": "",
"fail": false,
"id": "7c4df8ba-0d89-4703-b4fe-3e6f88c3fa76",
"score_obtained": 0,
"value": "No"
},
"comment": "",
"fail": false,
"id": "0bf1e679-a300-475e-9fb8-613317d42570",
"score_obtained": 0,
"value": "No"
},
{
"comment": "Advise agent get the link with forms update
dates to provide clients. No transfer needed to be completed. ",
" fail": false,
"id": "52ba5be6-7def-469a-a31f-7c74c86730ac",
"score_obtained": 0,
"value": ""
}
],
"score_obtained": 0
}
]},
"scores_obtained": {
"auto_fail": false,
"final_score": 17,
"grade_assigned": "Pass",
"percent_score": 77.27,
"total_bonus": 0,
"total_penalties": 0,
"total_points": 22
}
}
]
}
]}]}}
Thanks,
You can use FLATTEN function to parse nested arrays in JSON.
refer to
https://docs.snowflake.com/en/user-guide/querying-semistructured#using-the-flatten-function-to-parse-nested-arrays
To simplify, I have taken a small part of your JSON data and tested it.
CREATE OR REPLACE TABLE xyz
(
src variant
)
AS
SELECT PARSE_JSON(column1) AS src
FROM VALUES
('{"value":
{"evaluation_forms":
[ {"evaluations":
[ {"channel_meta":
{"after_call_work_time": [],
"agent_first_name": ["KATRINA"],
"agent_hung_up": [],
"agent_last_name": ["COX"]
}
} ]
} ]
}
} '
) v;
SELECT
src:value AS valuest,
src:value:evaluation_forms as evaluation_forms,
v.value:evaluations as evaluations,
ve.value:channel_meta as channel_meta,
Fn.value as agent_first_name
FROM
xyz
, LATERAL FLATTEN(INPUT => SRC:value:evaluation_forms) v
, LATERAL FLATTEN(INPUT => v.value:evaluations) ve
, LATERAL FLATTEN(INPUT => ve.value:channel_meta:agent_first_name) fn
;
i am new to python and json. I have this issue where i am unable to get the values of 'labels' print out. It is under a nested dictionary 'result' .
Code that i have tried
for i in data['result']:
print("Value:", i['value'])
values = i['value']
print("X:", values['x'])
print("Y:", values['y'])
print("Width:", values['width'])
print("Height:", values['height'])
Output from the above code
However when i try to do print("labels:", values['labels'])
it return a keyerror: 'labels'
for item in values:
print(item)
this is the output shown
x
y
width
height
rotation
x
y
width
height
rotation
labels
May i ask how do i go out to print out the 'labels' value?
The json files is as follows:
{
"id": 9,
"created_username": "",
"created_ago": "3\u00a0weeks, 3\u00a0days",
"completed_by": {
"id": 1,
"first_name": "",
"last_name": "",
"email": ""
},
"task": {
"id": 1,
"data": {
"image": "/data/upload/2/adf8540d-2-33.png"
},
"meta": {},
"created_at": "2022-11-21T14:52:22.478537Z",
"updated_at": "2022-11-22T12:51:18.105745Z",
"is_labeled": true,
"overlap": 1,
"inner_id": 1,
"total_annotations": 1,
"cancelled_annotations": 0,
"total_predictions": 0,
"comment_count": 0,
"unresolved_comment_count": 0,
"last_comment_updated_at": null,
"project": 2,
"updated_by": 1,
"file_upload": 70,
"comment_authors": []
},
"result": [
{
"original_width": 512,
"original_height": 512,
"image_rotation": 0,
"value": {
"x": 50.898958419872464,
"y": 44.0069438675168,
"width": 2.397222452993284,
"height": 2.0975696463691196,
"rotation": 0
},
"id": "pgaiV8NjWC",
"from_name": "rect",
"to_name": "image",
"type": "rectangle",
"origin": "manual"
},
{
"original_width": 512,
"original_height": 512,
"image_rotation": 0,
"value": {
"x": 50.898958419872464,
"y": 44.0069438675168,
"width": 2.397222452993284,
"height": 2.0975696463691196,
"rotation": 0,
"labels" [
"LM"*
]
},
"id": "pgaiV8NjWC",
"from_name": "labels",
"to_name": "image",
"type": "labels",
"origin": "manual"
}
],
"was_cancelled": false,
"ground_truth": false,
"created_at": "2022-11-22T12:51:18.001777Z",
"updated_at": "2022-11-22T12:51:18.001777Z",
"lead_time": 21.556,
"project": 2,
"parent_prediction": null,
"parent_annotation": null
}
Use the json module
import json
dict = json.load(json_filename)
Then you should access the values by the dictionary keys. Be aware that the result value is a list. Therefore the index 1 is necessary.
values = dict['result'][1]['value']
print("X:", values['x'])
but it should retain the fields in other array elements which did not match the criteria.
I have json response like below
{
"content": [
{
"someId": 1498,
"someKey": {
"anotherKey": "Not Specified",
"keyOne": "",
"keyTwo": null,
"keyThree": null
},
"date": "2009-09-11",
"time": "17:14",
"location": "Not Specified"
},
{
"someId": 1498,
"someKey": {
"anotherKey": "criteria",
"keyOne": "some data",
"keyTwo": "some data for key two",
"keyThree": "some data for key three"
},
"date": "2009-09-12",
"time": "17:15",
"location": "Not Specified"
},...
],
"pageable": {
"sort": {
"sorted": false,
"unsorted": true,
"empty": true
},
"offset": 0,
"pageSize": 10,
"pageNumber": 0,
"unpaged": false,
"paged": true
},
"totalElements": 10630,
"totalPages": 1063,
"last": false,
"size": 10,
"number": 0,
"sort": {
"sorted": false,
"unsorted": true,
"empty": true
},
"first": true,
"numberOfElements": 10,
"empty": false
}
when "anotherKey" value is "criteria", the "keyOne","keyTwo","keyThree" should be omitted, else it should be retained. any solution to this? thanks in advance
I'm want to make my json to csv so that i can upload it on google sheets and make it as json api. Whenever i have change data i will just change it on google sheets. But I'm having problems on converting my json file to csv because it changes the variables whenever i convert it. I'm using https://toolslick.com/csv-to-json-converter to convert my json file to csv.
What is the best way to convert json nested to csv ?
JSON
{
"options": [
{
"id": "1",
"value": "Jumbo",
"shortcut": "J",
"textColor": "#FFFFFF",
"backgroundColor": "#00000"
},
{
"id": "2",
"value": "Hot",
"shortcut": "D",
"textColor": "#FFFFFF",
"backgroundColor": "#FFFFFF"
}
],
"categories": [
{
"id": "1",
"order": 1,
"name": "First Category",
"active": true
},
{
"id": "2",
"order": 2,
"name": "Second Category",
"shortcut": "MT",
"active": true
}
],
"products": [
{
"id": "03c6787c-fc2a-4aa8-93a3-5e0f0f98cfb2",
"categoryId": "1",
"name": "First Product",
"shortcut": "First",
"options": [
{
"optionId": "1",
"price": 23
},
{
"optionId": "2",
"price": 45
}
],
"active": true
},
{
"id": "e8669cea-4c9c-431c-84ba-0b014f0f9bc2",
"categoryId": "2",
"name": "Second Product",
"shortcut": "Second",
"options": [
{
"optionId": "1",
"price": 11
},
{
"optionId": "2",
"price": 20
}
],
"active": true
}
],
"discounts": [
{
"id": "1",
"name": "S",
"type": 1,
"amount": 20,
"active": true
},
{
"id": "2",
"name": "P",
"type": 1,
"amount": 20,
"active": true
},
{
"id": "3",
"name": "G",
"type": 2,
"amount": 5,
"active": true
}
]
}
Using python, this can be easily done or almost done. Maybe this code will help you in some way to understand that.
import json,csv
data = []
with open('your_json_file_here.json') as file:
for line in file:
data.append(json.loads(line))
length = len(data)
with open('create_new_file.csv','w') as f:
writer = csv.writer(f)
writers = csv.DictWriter(f, fieldnames=['header1','header2'])
writers.writeheader()
for iter in range(length):
writer.writerow((data[iter]['specific_col_name1'],data[iter]['specific_col_name2']))
f.close()
Not sure if this is possible but we're trying to create an overall status dashboard in grafana using the singlestat panel. We used templating to group our hosts into two sites and using the packet loss value from hostalive in Icinga2. We'd like the singlestat panel to show the percentage of hosts down but sometimes we get null values. Here's the JSON from our panels:
{
"id": 2,
"title": "Host Group 2",
"span": 6,
"type": "singlestat",
"targets": [
{
"target": "icinga2.$group1.host.hostalive.perfdata.pl.value",
"refId": "A",
"hide": true
},
{
"target": "keepLastValue(averageSeries(#A))",
"refId": "B",
"textEditor": true,
"targetFull": "keepLastValue(averageSeries(icinga2.$group1.host.hostalive.perfdata.pl.value), 10000)"
}
],
"links": [],
"datasource": null,
"maxDataPoints": "",
"interval": null,
"cacheTimeout": null,
"format": "percent",
"prefix": "",
"postfix": "",
"nullText": null,
"valueMaps": [
{
"value": "null",
"op": "=",
"text": "N/A"
}
],
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"rangeMaps": [
{
"from": "null",
"to": "null",
"text": "N/A"
}
],
"mappingType": 1,
"nullPointMode": "connected",
"valueName": "current",
"prefixFontSize": "50%",
"valueFontSize": "80%",
"postfixFontSize": "50%",
"thresholds": "50, 100",
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"sparkline": {
"show": false,
"full": false,
"lineColor": "rgb(31, 120, 193)",
"fillColor": "rgba(31, 118, 189, 0.18)"
},
"gauge": {
"show": false,
"minValue": 0,
"maxValue": 100,
"thresholdMarkers": true,
"thresholdLabels": false
}
}
The polling interval for the hosts is every ten minutes, the grafana board range is "today" and set to refresh every second.
Got it figured out with the keeplastvalue.
"targets": [
{
"hide": true,
"refId": "A",
"target": "exclude(keepLastValue(icinga2.$group1.host.hostalive.perfdata.pl.value, 144), '1-99')",
"textEditor": true
},
{
"refId": "B",
"target": "averageSeries(#A)",
"targetFull": "averageSeries(icinga2.$group1.host.hostalive.perfdata.pl.value)",
"textEditor": false
}
],