How to parse Json response and truncate child nodes - json

This is the JSON response I am trying to parse:
{
"data": {
"Content": {
"id": 26,
"name": "Dashboard1"
},
"List": [
{
"ListContent": {
"id": 178,
"name": "Card-144"
},
"cards": [
{
"id": 1780,
"configuration": {
"id": 7178,
"name": "Emp"
}
}
]
},
{
"ListContent": {
"id": 179,
"name": "Card-14"
},
"cards": [
{
"id": 1798,
"configuration": {
"id": 1789,
"name": "RandomColumns"
}
}
]
},
{
"ListContent": {
"id": 180,
"name": "Card-1"
},
"cards": [
{
"id": 18080,
"configuration": {
"id": 1080,
"allow": true
}
}
]
},
{
"ListContent": {
"id": 181,
"name": "Card-14"
},
"cards": [
{
"id": 18081,
"configuration": {
"id": 1881,
"name": "Functions"
}
}
]
},
{
"ListContent": {
"id": 182,
"name": "Card-1443"
},
"cards": [
{
"id": 1782,
"configuration": {
"id": 1802,
"name": "Emp-O"
}
}
]
}
]
}
}
From the Json, I need to extract "id"s under the "ListContent" nodes and store it in an array. Also, will need to ignore "id"s under the child nodes.
Here is a groovy script I am trying to achieve this with,
def CList = ""
import groovy.json.JsonSlurper
def jsonRespData = context.expand( '${TestStep#Response#$.data.List}' )
def outputResp = new JsonSlurper().parseText(jsonRespData)
outputResp.id.each()
{log.info( ":"+ it)
CList=CList.concat(it.toString()).concat(',')}
log.info (CList)
So, the array that I am expecting is CList [178,179,180,181,182]
but I am currently getting null.
What should be the correct groovy to only read "id" from "ListContent" and write it to an array?
Any help would be really appreciated.
Thanks in advance.

You can just use the (implicit) spread operator like this:
def json = new groovy.json.JsonSlurper().parse('/tmp/x.json' as File)
//
def i = json.data.List.ListContent.id
assert i == [178, 179, 180, 181, 182]
// with explicit spread operator
def e = json.data.List*.ListContent*.id
assert e == [178, 179, 180, 181, 182]

def str = '''
{
"data": {
"Content": {
"id": 26,
"name": "Dashboard1"
},
"List": [
{
"ListContent": {
"id": 178,
"name": "Card-144"
},
"cards": [
{
"id": 1780,
"configuration": {
"id": 7178,
"name": "Emp"
}
}
]
},
{
"ListContent": {
"id": 179,
"name": "Card-14"
},
"cards": [
{
"id": 1798,
"configuration": {
"id": 1789,
"name": "RandomColumns"
}
}
]
},
{
"ListContent": {
"id": 180,
"name": "Card-1"
},
"cards": [
{
"id": 18080,
"configuration": {
"id": 1080,
"allow": true
}
}
]
},
{
"ListContent": {
"id": 181,
"name": "Card-14"
},
"cards": [
{
"id": 18081,
"configuration": {
"id": 1881,
"name": "Functions"
}
}
]
},
{
"ListContent": {
"id": 182,
"name": "Card-1443"
},
"cards": [
{
"id": 1782,
"configuration": {
"id": 1802,
"name": "Emp-O"
}
}
]
}
]
}
}
'''
def outputResp = new groovy.json.JsonSlurper().parseText(str)
outputResp.data.List.collect { it.ListContent.id }
As you already have List from (context.expand( '${TestStep#Response#$.data.List}' )) , you can simply do:
outputResp.collect { it.ListContent.id }
Above returns an ArrayList.

Related

how to merge multiple json files with same structure into one json file with same structure (combined all into one with keeping same structure))

I need to merge file1.json file2.json (could be more) into onefile.json.
version is always the same value in all files. however vulnerabilities array and dependency_files array values different but there might be duplicate/which I want to remove if any after the merge
file1.json:
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
}
]
}
file2.json:
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
onefile.json:
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
I tried a lot with no luck
You could have a reduce on all files, initialized with the first, hence no need for the -n option:
jq '
reduce inputs as {$vulnerabilities, $dependency_files} (.;
.vulnerabilities = (.vulnerabilities + $vulnerabilities | unique_by(.id))
| .dependency_files = (.dependency_files + $dependency_files | unique_by(.name))
)
' file*.json
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
Demo
Using this python code
import json
def merge_dicts(*dicts):
r = {}
skip = 'version'
for item in dicts:
for key, value in item.items():
if (key == skip):
r[skip] = value
else:
r.setdefault(key, []).extend(value)
unique = []
for obj in r[key]:
if obj not in unique:
unique.append(obj)
r[key] = unique
return r
with open("file1.json") as file_1:
data_1 = json.load(file_1)
with open("file2.json") as file_2:
data_2 = json.load(file_2)
with open('data.json', 'w') as merge_file:
json.dump(merge_dicts(data_1, data_2), merge_file, indent = 4)
Result
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
This code is multiple json files support
import os, json
def merge_dicts(*dicts):
r = {}
skip = 'version'
for item in dicts:
for key, value in item.items():
if (key == skip):
r[skip] = value
else:
r.setdefault(key, []).extend(value)
unique = []
for obj in r[key]:
if obj not in unique:
unique.append(obj)
r[key] = unique
return r
json_files = [pos_json for pos_json in os.listdir('./') if pos_json.endswith('.json')]
a = []
print(type(a))
for json_file in json_files:
with open(json_file) as file_item:
read_data = json.load(file_item)
a.append(read_data)
file_item.close()
with open('data.json', 'w') as merge_file:
json.dump(merge_dicts(*tuple(a)), merge_file, indent = 4)

Sort complex JSON object by specific property

How can I sort the given JSON object with property count. I want to sort the entire sub-object. The higher the count value should come on the top an so on.
{
"Resource": [
{
"details": [
{
"value": "3.70"
},
{
"value": "3.09"
}
],
"work": {
"count": 1
}
},
{
"details": [
{
"value": "4"
},
{
"value": "5"
}
],
"work": {
"count": 2
},
{
"details": [
{
"value": "5"
},
{
"value": "5"
}
],
"work": "null"
}
]
}
You can try this example to sort your data:
data = {
"data": {
"Resource": [
{
"details": [{"value": "3.70"}, {"value": "3.09"}],
"work": {"count": 1},
},
{"details": [{"value": "4"}, {"value": "5"}], "work": {"count": 2}},
]
}
}
# sort by 'work'/'count'
data["data"]["Resource"] = sorted(
data["data"]["Resource"], key=lambda r: r["work"]["count"]
)
# sort by 'details'/'value'
for r in data["data"]["Resource"]:
r["details"] = sorted(r["details"], key=lambda k: float(k["value"]))
# pretty print:
import json
print(json.dumps(data, indent=4))
Prints:
{
"data": {
"Resource": [
{
"details": [
{
"value": "3.09"
},
{
"value": "3.70"
}
],
"work": {
"count": 1
}
},
{
"details": [
{
"value": "4"
},
{
"value": "5"
}
],
"work": {
"count": 2
}
}
]
}
}

Read data from hierarchy document in SQL Server

I use SQL Server 2016 and I develop SSIS packages. I call a web service and I have below file.
{
"id": "Voyage",
"fields": {
"CommenceDateGmt": "2018-10-09T04:00:00",
"CommenceDateLocal": "2018-10-09T15:00:00",
"CompleteDateGmt": "2018-11-02T10:30:00",
"CompleteDateLocal": "2018-11-02T20:30:00",
"VoyageStatus": "Completed"
},
"dataSources": [
{
"id": "VoyageItineraries",
"joinType": "toMany",
"values": [
{
"fields": {
"VesselCode": "aaaa",
"VoyageNo": 10000,
"Seq": 258,
"PortFunc": "C",
"PortName": "aaaa (A)"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "aaaa"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": []
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "aaasss"
}
}
]
},
{
"fields": {
"VesselCode": "www",
"VoyageNo": 5454,
"Seq": 5454,
"PortFunc": "L54",
"PortName": "54545"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "fsfsdsd&S"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": [
{
"fields": {
"NominatedLiftQty": 54545.0,
"CargoShortName": "dfdfdf"
}
}
]
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "dfewff"
}
}
]
},
{
"fields": {
"VesselCode": "sdfsdf",
"VoyageNo": 23423,
"Seq": 234,
"PortFunc": "Dd",
"PortName": "fewffe"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "erwerwer"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": [
{
"fields": {
"NominatedLiftQty": 23423.0,
"CargoShortName": "sdfsdf"
}
},
{
"fields": {
"NominatedLiftQty": 23423.0,
"CargoShortName": "sdefsdf"
}
}
]
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "dfsdf"
}
}
]
},
{
"fields": {
"VesselCode": "dfsdf",
"VoyageNo": 234,
"Seq": 32433,
"PortFunc": "L",
"PortName": "sdfsdf"
},
"dataSources": [
{
"id": "VoyageItineraryBerths",
"joinType": "toMany",
"values": [
{
"fields": {
"BerthShortName": "sdfsdf"
}
}
]
},
{
"id": "VoyageCargoHandlings",
"joinType": "toMany",
"values": [
{
"fields": {
"NominatedLiftQty": 234234.0,
"CargoShortName": "sdfsdf"
}
}
]
},
{
"id": "Vessel",
"joinType": "toOne",
"fields": {
"Name": "sdfsdf"
}
}
]
}
]
},
{
"id": "Cargos",
"joinType": "toMany",
"values": [
{
"fields": {
"CoaNo": "sdfsdf"
},
"dataSources": [
{
"id": "Counterparty",
"joinType": "toOne",
"fields": {
"FullName": "sdfsdfsd"
}
}
]
}
]
}
]
}
]
As you can see, I have several table.
I need to transfer this hierarchy data to a table.
Table1Filed1, Table1Filed2, Table1Filed3, Table1Filed4, Table2Field1, Table2Field2, Table2Field3
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data
Data , Data , Data , Data , Data , Data , Data

How to exclude specific fields from JSON using groovy

I would like to exclude the items which don't have productModel property in the below JSON. How can we achieve this in groovy
I tried using hasProperty but not worked for me as expected. If possible can I get some sample snippet
I tried below code - but didn't work as I expected.
response.getAt('myData').getAt('data').getAt('product').hasProperty('productModel').each { println "result ${it}" }
Any help would be really appreciated.
{
"myData": [{
"data": {
"product": {
"productId": "apple",
"productName": "iPhone",
"productModel": "6s"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {}
},
{
"data": {
"product": {
"productId": "apple",
"productName": "iPhone",
"productModel": "7"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {}
},
{
"data": {
"product": {
"productId": "apple",
"productName": "Macbook"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {}
}
],
"metadata": {
"count": 3,
"offset": 0
}
}
If you want to exclude specific fields from JSON object then you have to recreate it using filtered data. The crucial part takes these two lines (assuming that json variable in the below example stores your JSON as text):
def root = new JsonSlurper().parseText(json)
def myData = root.myData.findAll { it.data.product.containsKey('productModel') }
What happens here is we access root.myData list and we filter it using findAll(predicate) method and predicate in this case says that only objects that have key productModel in path data.product are accepted. This findAll() method does not mutate existing list and that is why we store the result in variable myData - after running this method we will end up with a list of size 2.
In next step you have to recreate the object you want to represent as a JSON:
def newJsonObject = [
myData: myData,
metadata: [
count: myData.size(),
offset: 0
]
]
println JsonOutput.prettyPrint(JsonOutput.toJson(newJsonObject))
In this part we create newJsonObject and in the end we convert it to a JSON representation.
Here is the full example:
import groovy.json.JsonOutput
import groovy.json.JsonSlurper
def json = '''{
"myData": [{
"data": {
"product": {
"productId": "apple",
"productName": "iPhone",
"productModel": "6s"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {}
},
{
"data": {
"product": {
"productId": "apple",
"productName": "iPhone",
"productModel": "7"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {}
},
{
"data": {
"product": {
"productId": "apple",
"productName": "Macbook"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {}
}
],
"metadata": {
"count": 3,
"offset": 0
}
}'''
def root = new JsonSlurper().parseText(json)
def myData = root.myData.findAll { it.data.product.containsKey('productModel') }
def newJsonObject = [
myData: myData,
metadata: [
count: myData.size(),
offset: 0
]
]
println JsonOutput.prettyPrint(JsonOutput.toJson(newJsonObject))
And here is the output it produces:
{
"myData": [
{
"data": {
"product": {
"productId": "apple",
"productName": "iPhone",
"productModel": "6s"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [
{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {
}
},
{
"data": {
"product": {
"productId": "apple",
"productName": "iPhone",
"productModel": "7"
},
"statusCode": "active",
"date": "2018-08-07T00:00:00.000Z"
},
"links": [
{
"productUrl": "test"
},
{
"productImage": "test"
}
],
"info": {
}
}
],
"metadata": {
"count": 2,
"offset": 0
}
}

Unique sub-Json of a Json

let me start from a JSon object like
[
{
"id": 32837732,
"composer": {
"id": 536,
"name": "Francis Poulenc"
},
"title": "Of Thee I Sing: Overture (radio version)"
},
{
"id": 32837735,
"composer": {
"id": 245,
"name": "George Gershwin"
},
"title": "Concerto in F : I. Allegro"
},
{
"id": 32837739,
"composer": {
"id": 245,
"name": "George Gershwin"
},
"title": "Concerto in F : II. Adagio"
}
]
is it possible to get, in a clean, declarative way, a Json like
{
'composer': [
{
'id': '536',
'name': 'Francis Poulenc'
},
{
'id': '245',
'name': 'George Gershwin'
},
]
}
That is the unique sub-values for each composer (id and name)?
var arr= [
{
"id": 32837732,
"composer": {
"id": 536,
"name": "Francis Poulenc"
},
"title": "Of Thee I Sing: Overture (radio version)"
},
{
"id": 32837735,
"composer": {
"id": 245,
"name": "George Gershwin"
},
"title": "Concerto in F : I. Allegro"
},
{
"id": 32837739,
"composer": {
"id": 245,
"name": "George Gershwin"
},
"title": "Concerto in F : II. Adagio"
}
];
var composers=[];
arr.forEach(function(arrElem){
if(arrElem.composer && !composers.some(function(comp) { return comp.id === arrElem.composer.id && comp.name === arrElem.composer.name }))
composers.push(arrElem.composer);
});
alert( JSON.stringify( composers ));