Nested dicts and lists / glom lib python - json

I am trying to access deep-nested lists and dictionaries. I am experimenting with the glom library however my Third_KV key doesn't work on the below JSON object when trying to retrieve the "Country"
from glom import glom
target = {
"Result": {
"Topics": [
{
"A": "abc",
"D": 0,
"Questions": [
{
"E": "jklm",
"P": "dsfs",
"Answers": [
{
"first": "string",
"second": "string",
"Country": "CH"
},
{
"first": "string",
"second": "string",
"Country": "NL"
}
]
}
]
}
]
}
}
path = {
"First_KV": ("Result.Topics", ["Questions"]),
"Second_KV": ("Result.Topics", [("Questions", ["Answers"])]),
"Third_KV": ("Result.Topics", [("Questions", "Answers", ["Country"])])
}
countries = glom(target, path["Third_KV"])

Not very clear what final json/array/structure you want, but without relying on any library, can you not use simple map() e.g.
const jsonTest = {
"Result": {
"Topics": [{
"A": "abc",
"D": 0,
"Questions": [{
"E": "jklm",
"P": "dsfs",
"Answers": [{
"first": "CHfirstCountry",
"second": "CHsecondCountry",
"Country": "CH"
},
{
"first": "NLfirstCountry",
"second": "NLsecondCountry",
"Country": "NL"
}
]
}]
}]
}
};
const AnswersArray = jsonTest.Result.Topics[0].Questions[0].Answers;
let dictPerCountry = new Object();
AnswersArray.map((eachElement) => {
dictPerCountry[eachElement.Country] = [eachElement.first, eachElement.second];
});
console.log({
dictPerCountry
});
dictPerCountry will look like so:
{
"dictPerCountry": {
"CH": [
"CHfirstCountry",
"CHsecondCountry"
],
"NL": [
"NLfirstCountry",
"NLsecondCountry"
]
}
}

Answers are of "list" type too and you are missing its square brackets. check below pattern to get the country
pattern = ('Result.Topics', [('Questions', [('Answers', ['Country'])])])
So you need to change your dictionary "path" to be
path = {
"First_KV": ("Result.Topics", ["Questions"]),
"Second_KV": ("Result.Topics", [("Questions", ["Answers"])]),
"Third_KV": ('Result.Topics', [('Questions', [('Answers', ['Country'])])])
}

Related

how to merge multiple json files with same structure into one json file with same structure (combined all into one with keeping same structure))

I need to merge file1.json file2.json (could be more) into onefile.json.
version is always the same value in all files. however vulnerabilities array and dependency_files array values different but there might be duplicate/which I want to remove if any after the merge
file1.json:
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
}
]
}
file2.json:
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
onefile.json:
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
I tried a lot with no luck
You could have a reduce on all files, initialized with the first, hence no need for the -n option:
jq '
reduce inputs as {$vulnerabilities, $dependency_files} (.;
.vulnerabilities = (.vulnerabilities + $vulnerabilities | unique_by(.id))
| .dependency_files = (.dependency_files + $dependency_files | unique_by(.name))
)
' file*.json
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
Demo
Using this python code
import json
def merge_dicts(*dicts):
r = {}
skip = 'version'
for item in dicts:
for key, value in item.items():
if (key == skip):
r[skip] = value
else:
r.setdefault(key, []).extend(value)
unique = []
for obj in r[key]:
if obj not in unique:
unique.append(obj)
r[key] = unique
return r
with open("file1.json") as file_1:
data_1 = json.load(file_1)
with open("file2.json") as file_2:
data_2 = json.load(file_2)
with open('data.json', 'w') as merge_file:
json.dump(merge_dicts(data_1, data_2), merge_file, indent = 4)
Result
{
"version": "x.x.x",
"vulnerabilities": [
{
"id": "0000"
},
{
"id": "11111"
},
{
"id": "2222"
},
{
"id": "3333"
}
],
"dependency_files": [
{
"name": "name0000"
},
{
"name": "name1111"
},
{
"name": "name2222"
},
{
"name": "name3333"
}
]
}
This code is multiple json files support
import os, json
def merge_dicts(*dicts):
r = {}
skip = 'version'
for item in dicts:
for key, value in item.items():
if (key == skip):
r[skip] = value
else:
r.setdefault(key, []).extend(value)
unique = []
for obj in r[key]:
if obj not in unique:
unique.append(obj)
r[key] = unique
return r
json_files = [pos_json for pos_json in os.listdir('./') if pos_json.endswith('.json')]
a = []
print(type(a))
for json_file in json_files:
with open(json_file) as file_item:
read_data = json.load(file_item)
a.append(read_data)
file_item.close()
with open('data.json', 'w') as merge_file:
json.dump(merge_dicts(*tuple(a)), merge_file, indent = 4)

building url query string using n1ql

using couchbase 5
I need to build a query string from this object
[
{
"_id": 190,
"querystring": [
{
"name": "p1",
"value": "val1"
},
{
"name": "p2",
"value": "val2"
}
]
}
]
the expected output should be
p1=val1&p2=val2
can anyone help here?
after few attempts I think I got closer to the solution I need.
[
{
"_id": 190,
"res": [
"company_id=$PREFIJO&",
"user_country=$COUNTRY&",
"offer_unique_code=$PIXEL&",
"pub_id=$PUBID&"
]
}
]
now, how can I convert "res" to a concatenated string of all the array elements?
WITH obj AS ({ "_id": 190, "querystring": [ { "name": "p1", "value": "val1" }, { "name": "p2", "value": "val2" } ] })
SELECT obj._id, CONCAT2("&", ARRAY CONCAT2("=",v.name,v.`value`) FOR v IN obj.querystring END) AS res;
Array of objects
WITH objs AS ([{ "_id": 190, "querystring": [ { "name": "p1", "value": "val1" }, { "name": "p2", "value": "val2" } ] },
{ "_id": 191, "querystring": [ { "name": "p3", "value": "val1" }, { "name": "p4", "value": "val2" } ] }
])
SELECT obj._id, CONCAT2("&", ARRAY CONCAT2("=",v.name,v.`value`) FOR v IN obj.querystring END) AS res FROM objs AS obj ;
Older version where CONCAT2() not available, get array of strings (name=val) and do in application or use the following technique. Assume your name/val doesn't have any replace characters.
WITH objs AS ([{ "_id": 190, "querystring": [ { "name": "p1", "value": "val1" }, { "name": "p2", "value": "val2" } ] },
{ "_id": 191, "querystring": [ { "name": "p3", "value": "val1" }, { "name": "p4", "value": "val2" } ] }
])
SELECT obj._id, replace(replace(replace(encode_json(ARRAY CONCAT(v.name,"=",v.`value`) FOR v IN obj.querystring END),"\",\"","&"),"[\"",""),"\"]","") AS res FROM objs AS obj ;
If single document then have ARRAY of objects then use UNNEST
If there is number , convert to string using TO_STR() before CONCAT operation
https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/stringfun.html#fn-str-concat2

How to get key from ArrayList nested in JSON using Groovy and change its value

I need to be able to find the key quote.orderAttributes[0].attributeDetail.name and set its value to null or any other value I want. I only need to do this for the first element in any list so selecting [0] is fine. I want to be able to use a path such as 'quote.orderAttributes.attributeDetail.name'. But given the amount of time I've spent so far, please advise of any better approaches.
Here is the Json:
{
"source": "source",
"orderId": null,
"Version": null,
"quote": {
"globalTransactionId": "k2o4-6969-1fie-poef",
"quoteStatus": "Not Uploaded",
"events": {
"eventDescription": "event description",
"eventTypeName": "Event Type"
},
"someReport": {
"acceptResultsFlag": "Y",
"orderDate": "2017-06-14",
"orderStatus": "string"
},
"anotherReport": {
"id": 627311,
"orderDate": "2017-06-14"
},
"attributes": [
{
"appliedFlag": "Y",
"attributeDetail": {
"name": "name1",
"value": "value1"
},
"attributeName": "attribute1"
},
{
"appliedFlag": "N",
"attributeDetail": {
"name": "name2",
"value": "value2"
},
"attributeName": "attribute2"
}
],
"orderAttributes": [
{
"appliedFlag": "Y",
"attributeDetail": {
"name": "name3",
"value": "value3"
},
"attributeName": "orderAttribute1"
},
{
"appliedFlag": "N",
"attributeDetail": {
"name": "name4",
"value": "value4"
},
"attributeName": "orderAttribute2"
}
]
}
}
I know the following works but requires that I know which object(s) is an ArrayList and specify its [0] indexed item:
def input = new File("src/test/resources/ShortExample.json")
def json = new JsonSlurper().parse(input)
def option1 = json['quote']["attributes"][0]["attributeDetail"]["name"]
println option1
//or this
//where csvData.fullPath = quote.orderAttributes.attributeDetail.name
def (tkn1, tkn2, tkn3, tkn4) = csvData.fullPath.tokenize('.')
def option2 = json["$tkn1"]["$tkn2"][0]["$tkn3"]["$tkn4"]
println option2
I would like to be able to:
def input = new File("src/test/resources/ShortExample.json")
def json = new JsonSlurper().parse(input)
def changeValueTo = null
def (tkn1, tkn2, tkn3, tkn4) = csvData.fullPath.tokenize('.')
json["$tkn1"]["$tkn2"]["$tkn3"]["$tkn4"] = changeValueTo
I've tried to implement many of the examples on here using recursion, methods creating MapsOrCollections that identify what the object is and then search it for key or value, even trampoline examples.
If you could point me to a good article explaining serialization and deserialization it would be much appreciated too.
Thank you in advance.
as variant:
import groovy.json.*;
def json = '''{
"source": "source",
"orderId": null,
"Version": null,
"quote": {
"globalTransactionId": "k2o4-6969-1fie-poef",
"quoteStatus": "Not Uploaded",
"attributes": [
{
"appliedFlag": "Y",
"attributeDetail": {
"name": "name1",
"value": "value1"
},
"attributeName": "attribute1"
},
{
"appliedFlag": "N",
"attributeDetail": {
"name": "name2",
"value": "value2"
},
"attributeName": "attribute2"
}
]}
}'''
json = new JsonSlurper().parseText(json)
def jsonx(Object json, String expr){
return Eval.me('ROOT',json, expr)
}
println jsonx(json, 'ROOT.quote.attributes[0].attributeDetail.name')
jsonx(json, 'ROOT.quote.attributes[0].attributeDetail.name = null')
println jsonx(json, 'ROOT.quote.attributes[0].attributeDetail.name')
You can access and modify any nested field of JSON object directly, e.g.
json.quote.attributes[0].attributeDetail.name = null
This is possible, because new JsonSlurper().parse(input) returns a groovy.json.internal.LazyMap object. Groovy allows you to access and modify any Map entries using dot notation, e.g.
Map<String, Map<String, Integer>> map = [
lorem: [ipsum: 1, dolor: 2, sit: 3]
]
println map.lorem.ipsum // Prints '1'
map.lorem.ipsum = 10
println map.lorem.ipsum // Prints '10'
You can apply same approach to your example, e.g.
import groovy.json.JsonSlurper
String input = '''{
"source": "source",
"orderId": null,
"Version": null,
"quote": {
"globalTransactionId": "k2o4-6969-1fie-poef",
"quoteStatus": "Not Uploaded",
"events": {
"eventDescription": "event description",
"eventTypeName": "Event Type"
},
"someReport": {
"acceptResultsFlag": "Y",
"orderDate": "2017-06-14",
"orderStatus": "string"
},
"anotherReport": {
"id": 627311,
"orderDate": "2017-06-14"
},
"attributes": [
{
"appliedFlag": "Y",
"attributeDetail": {
"name": "name1",
"value": "value1"
},
"attributeName": "attribute1"
},
{
"appliedFlag": "N",
"attributeDetail": {
"name": "name2",
"value": "value2"
},
"attributeName": "attribute2"
}
],
"orderAttributes": [
{
"appliedFlag": "Y",
"attributeDetail": {
"name": "name3",
"value": "value3"
},
"attributeName": "orderAttribute1"
},
{
"appliedFlag": "N",
"attributeDetail": {
"name": "name4",
"value": "value4"
},
"attributeName": "orderAttribute2"
}
]
}
}'''
def json = new JsonSlurper().parse(input.bytes)
assert json.quote.attributes[0].attributeDetail.name == 'name1'
json.quote.attributes[0].attributeDetail.name = null
assert json.quote.attributes[0].attributeDetail.name == null
I hope it helps.

How to iterate on json fields and insert new values using json4s?

I have a simple json file:
val myJson = {
"field1": [
{
"name": "john",
"lname": "knight"
},
{
"name": "jack",
"lname": "samuel"
},
{
"name": "elinor",
"lname": "cooper"
}
],
"field2": [
{
...
},
{
...
},
{
...
}
],
"field3": [
{
...
},
{
...
},
{
...
}
]
}
and what i want is to be able to iterate on "field1" and for each name to call a method that returns some value and insert this value to the json under "fiel1".
// this returns a list of strings
val kids = getKids("john")
// this is will me the returned value
kids = List("amy", "tom")
now I want to insert it:
{
"field1": [
{
"name": "john",
"lname": "knight"
"kids": ["amy", "tom"]
},
{
"name": "jack",
"lname": "samuel"
"kids": ["edi", "keren"]
},
{
"name": "elinor",
"lname": "cooper"
"kids": ["lilly", "mag"]
}
]
...
but I want to iterate on all the names and do this for each one...how can I accomplish this with json4s?
so lets say i have the parsed json:
val myParsedJson = JsonMethods.parse(myJson)
how do I go from here?
thanks!

How to check if a key exists in a nested JSON object in node?

I've got the following JSON being sent to the server from the browser:
{
"title": "Testing again 2",
"abstract": "An example document",
"_href": "http://google.com",
"tags": [ "person" ],
"attributes": [ {
"id": 1,
"type": "TEXT",
"data": "test"
} ],
"sections": [ {
"id": 1,
"type": "LIST",
"data": [ {
"revision": 124,
"text": "test"
} ]
} ]
}
I need to make sure that the keys "_href", "id" and "revision" are not in the object anyplace at any level.
I found this but it doesn't quite work.
I searched npms.io and found has-any-deep which you can use after JSON.parse ing the JSON.
you need to parse json then check into the data
var str = '{
"title": "Testing again 2",
"abstract": "An example document",
"_href": "http://google.com",
"tags": [ "person" ],
"attributes": [ {
"id": 1,
"type": "TEXT",
"data": "test"
} ],
"sections": [ {
"id": 1,
"type": "LIST",
"data": [ {
"revision": 124,
"text": "test"
} ]
} ]
}';
var jsonObj = JSON.parse(str);
if ( typeof jsonObj._href == 'undefined') {
// check
}
A simple but not 100% foolproof solution would be to parse the JSON to string, and just search for your keys:
var a = JSON.stringify(JSONObject);
var occurs = false;
['"_href"', '"id"', '"version"'].forEach(function(string) {
if(a.indexOf(string) > -1) occurs = true;
});
The issue of course, is if there are values that match
'_href', 'id', 'version' in your JSON. But if you want to use native JS, I guess this is a good bet.
var a = {
"title": "Testing again 2",
"abstract": "An example document",
"tags": [ "person" ],
"attributes": [ {
"type": "TEXT",
"data": "test"
} ],
"sections": [ {
"type": "_href asdad",
"data": [ {
"text": "test"
} ]
} ]
},
b = {
"title": "Testing again 2",
"abstract": "An example document",
"_href": "http://google.com",
"tags": [ "person" ],
"attributes": [ {
"id": 1,
"type": "TEXT",
"data": "test"
} ],
"sections": [ {
"id": 1,
"type": "LIST",
"data": [ {
"revision": 124,
"text": "test"
} ]
} ]
},
aJson = JSON.stringify(a),
bJson = JSON.stringify(b);
var occursa = false, occursb = false;
['"_href"', '"id"', '"version"'].forEach(function(string) {
if(aJson.indexOf(string) > -1) { occursa = true};
});
['"_href"', '"id"', '"version"'].forEach(function(string) {
if(bJson.indexOf(string) > -1) { occursb = true};
});
console.log("a");
console.log(occursa);
console.log("b");
console.log(occursb);
You could use the optional second reviver parameter to JSON.parse for this:
function hasBadProp(json) {
let badProp = false;
JSON.parse(json, (k, v) => {
if ([_href", "id", "revision"].includes(k)) badProp = true;
return v;
});
return badProp;
}