parse the nested json have same attribute with jq streaming mode - json

Question

I want to parse the data from the below nested json file, but it has too many "keys" in the json, it makes hard to parse the data
{
"jobname": {
"keys": {
"jobid":"E000295",
"car":"BMW"
},
"property":{
"doctype":"File",
"areadesc":[
{
"areaid":"qaz",
"weather":"hot",
},
{
"areaid":"wsx",
"weather":"code",
},
{
"areaid":"edc",
"weather":"hot",
},
{
"areaid":"rfv",
"weather":"hot",
}
]
},
"toolJobs":[
{
"keys":{
"toolid":"123"
},
"reports":[
{
"keys":{
"oiltype":"a",
"oilcountry":"us"
},
"property":{"reportid":"001"},
"datas":[
{
"keys":{"areaid":"qaz"},
"data":[
{
"time": "2021-01-01",
"value": 1
},
{
"time": "2021-01-02",
"value": 3
},
]
},
{
"keys":{"areaid":"wsx"},
"data":[
{
"time": "2021-01-03",
"value": 5
},
{
"time": "2021-01-04",
"value": 7
},
]
},
]
},
{
"keys":{
"oiltype":"b",
"oilcountry":"china"
},
"property":{"reportid":"002"},
"datas":[
{
"keys":{"areaid":"edc"},
"data":[
{
"time": "2021-01-05",
"value": 2
},
{
"time": "2021-01-06",
"value": 4
},
]
},
{
"keys":{"areaid":"rfv"},
"data":[
{
"time": "2021-01-07",
"value": 6
},
{
"time": "2021-01-08",
"value": 8
},
]
},
]
}
]
}
]
}
}
until now, I can use the below code to get the basic result, but some columns do not have, such as oiltype, oilcountry, reportid, areaid
cat tmp1.json | jq -cn --stream '
[fromstream(
1|truncate_stream(inputs)
| (.[0][:2] | index("keys")) as $ix
| if $ix then .[0] |= .[1+$ix:]
else (.[0] | index("toolJobs")) as $iy | (.[0][$iy:$iy+3] | index("keys")) as $iz
| if $iz then .[0] |= .[1+$iy+$iz:]
else (.[0] | index("data")) as $ik
| if $ik then .[0] |= .[$ik:]
else empty
end
end
end
)] | .[0] as $header | .[1] as $tool | [.[2:][] | ($header+ $tool+.)] | .'
The result is
[
{"jobid":"E000295","car":"BMW","toolid":"123","data":[{"time":"2021-01-01","value":1},{"time":"2021-01-02","value":3}]},
{"jobid":"E000295","car":"BMW","toolid":"123","data":[{"time":"2021-01-03","value":5},{"time":"2021-01-04","value":7}]},
{"jobid":"E000295","car":"BMW","toolid":"123","data":[{"time":"2021-01-05","value":2},{"time":"2021-01-06","value":4}]},
{"jobid":"E000295","car":"BMW","toolid":"123","data":[{"time":"2021-01-07","value":6},{"time":"2021-01-08","value":8}]}]
I also try below code
cat tmp1.json | jq -cn --stream '
[fromstream(
1|truncate_stream(inputs)
| (.[0][:2] | index("keys")) as $ix
| if $ix then .[0] |= .[1+$ix:]
else (.[0] | index("toolJobs")) as $iy | (.[0][$iy:$iy+3] | index("keys")) as $iz
| if $iz then .[0] |= .[1+$iy+$iz:]
else (.[0] | index("data")) as $ik
| if $ik then .[0] |= .[$ik:]
else (.[0] | index("reports")) as $iw | (.[0][$iw:$iw+3] | index("property")) as $ii
| if $ii then (.[0] |= .[$iw+$ii:])
else (.[0] | index("keys")) as $ij
| if $ij then (.[0] |= .[$ij:])
else empty
end
end
end
end
end
)] | .[0] as $header | .[1] as $prjob | [.[2:][] | ($header + $prjob + .)] | .'
but the result is strange
[
{"jobid":"E000295","car":"BMW","property":{"reportid":"001"},"toolid":"123","keys":{"oiltype":"a","oilcountry":"us","areaid":"qaz"},"data":[{"time":"2021-01-01","value":1},{"time":"2021-01-02","value":3}]},
{"jobid":"E000295","car":"BMW","property":{"doctype":"File","areadesc":[{"areaid":"qaz","weather":"hot"},{"areaid":"wsx","weather":"code"},{"areaid":"edc","weather":"hot"},{"areaid":"rfv","weather":"hot"}]},"toolid":"123","keys":{"areaid":"wsx"},"data":[{"time":"2021-01-03","value":5},{"time":"2021-01-04","value":7}]},
{"jobid":"E000295","car":"BMW","property":{"reportid":"002"},"toolid":"123","keys":{"oiltype":"b","oilcountry":"china","areaid":"edc"},"data":[{"time":"2021-01-05","value":2},{"time":"2021-01-06","value":4}]},
{"jobid":"E000295","car":"BMW","property":{"doctype":"File","areadesc":[{"areaid":"qaz","weather":"hot"},{"areaid":"wsx","weather":"code"},{"areaid":"edc","weather":"hot"},{"areaid":"rfv","weather":"hot"}]},"toolid":"123","keys":{"areaid":"rfv"},"data":[{"time":"2021-01-07","value":6},{"time":"2021-01-08","value":8}]}
]
Below is my expected result
[
{
"jobid":"E000295",
"car":"BMW",
"toolid":"123",
"oiltype":"a",
"oilcountry":"us",
"reportid":"001",
"areaid":"qaz",
"data":[
{
"time": "2021-01-01",
"value": 1
},
{
"time": "2021-01-02",
"value": 3
},
]
},
{
"jobid":"E000295",
"car":"BMW",
"toolid":"123",
"oiltype":"a",
"oilcountry":"us",
"reportid":"001",
"areaid":"wsx",
"data":[
{
"time": "2021-01-03",
"value": 5
},
{
"time": "2021-01-04",
"value": 7
},
]
},
{
"jobid":"E000295",
"car":"BMW",
"toolid":"123",
"oiltype":"b",
"oilcountry":"china",
"reportid":"002",
"areaid":"edc",
"data":[
{
"time": "2021-01-05",
"value": 2
},
{
"time": "2021-01-06",
"value": 4
},
]
},
{
"jobid":"E000295",
"car":"BMW",
"toolid":"123",
"oiltype":"b",
"oilcountry":"china",
"reportid":"002",
"areaid":"rfv",
"data":[
{
"time": "2021-01-07",
"value": 6
},
{
"time": "2021-01-08",
"value": 8
},
]
}
]
Does anyone have any idea?

Answer 1

Assuming the input has been corrected, the following "regular" jq program produces the desired result:
[
.jobname
| (.keys + .toolJobs[].keys) as $one
| .toolJobs[]
| .keys as $two
| .reports[]
| (.keys + .property) as $three
| .datas[]
| (.keys + {data}) as $four
| $one + $two + $three + $four
]
If your input is too large, you could reduce the memory requirements by creating a jq-to-jq pipeline, with the first invocation using the above program (or a --stream version of it) but with the outer brackets removed.