Remove all the values from JSON object with null value Using jsonpath - json

I have a JSON file from which I need to remove node objects which contain a data value of null. Can this be done?
In the sample JSON below, I need to remove the object where all the tag has a null value.
{
"store" : {
"book" : [
{
"category" : "reference",
"author" : "Nigel Rees",
"title" : "Sayings of the Century",
"price" : 8.95
},
{
"category" : "fiction",
"author" : "Evelyn Waugh",
"title" : "Sword of Honour",
"price" : 12.99
},
{
"category" : "fiction",
"author" : null,
"title" : "Moby Dick",
"isbn" : "0-553-21311-3",
"price" : 8.99
},
{
"category" : "fiction",
"author" : null,
"title" : "The Lord of the Rings",
"isbn" : "0-395-19395-8",
"price" : 22.99
}
],
"bicycle" : {
"color" : null,
"price" : null
}
},
"expensive" : 10
}
We tried using JOLT :
[
// Flatten an array of photo objects into a prefixed
// soup of properties.
{
"operation": "modify-overwrite-beta",
"spec": {
"*": "=recursivelySquashNulls"
}
}
]
But this is resulting in removing Null values but also it is not removing empty json fields
Output which we are getting
{
"store": {
"book": [
{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95
},
{
"category": "fiction",
"author": "Evelyn Waugh",
"title": "Sword of Honour",
"price": 12.99
},
{
"category": "fiction",
"title": "Moby Dick",
"isbn": "0-553-21311-3",
"price": 8.99
},
{
"category": "fiction",
"title": "The Lord of the Rings",
"isbn": "0-395-19395-8",
"price": 22.99
}
],
"bicycle": {}
},
"expensive": 10
}
and expected Output is
{
"store": {
"book": [
{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95
},
{
"category": "fiction",
"author": "Evelyn Waugh",
"title": "Sword of Honour",
"price": 12.99
},
{
"category": "fiction",
"title": "Moby Dick",
"isbn": "0-553-21311-3",
"price": 8.99
},
{
"category": "fiction",
"title": "The Lord of the Rings",
"isbn": "0-395-19395-8",
"price": 22.99
}
]
},
"expensive": 10
}
Any Suggestions How we can do this ?

You rather can use the following spec instead of using recursivelySquashNulls by interchanging key-value pairs through use of "$": "...#(0)" pattern
[
{
// convert price values to strings in order to keep the decimal part of them during the interchange operation
"operation": "modify-overwrite-beta",
"spec": {
"store": {
"book": {
"*": {
"price": "=toString"
}
},
"*": {
"price": "=toString"
}
}
}
},
{
// interchange key-value pairs to get rid of the both values null and null object -> { }
"operation": "shift",
"spec": {
"store": {
"book": {
"*": {
"*": {
"$": "&4.&3.&2.#(0)"
}
}
},
"*": {
"*": {
"$": "&3.&2.#(0)"
}
}
},
"*": {
"$": "#(0)"
}
}
},
{
// switch to the original
"operation": "shift",
"spec": {
"store": {
"book": {
"*": {
"*": {
"$": "&4.&3[&2].#(0)"
}
}
},
"*": {
"*": {
"$": "&3.&2.#(0)"
}
}
},
"*": {
"$": "#(0)"
}
}
},
{
// preserve the decimal value as they are
"operation": "modify-overwrite-beta",
"spec": {
"store": {
"book": {
"*": {
"price": "=toDouble"
}
},
"*": {
"price": "=toDouble"
}
}
}
}
]
the demo on the site http://jolt-demo.appspot.com/ is

Related

How to Add an Object to a Json Array Depending a value of another Key in the File Using JoltTransformationJson in NiFi

This is my first time to use JoltTransformationJson, so I have limited knowledge and experience on that. Please help me with this complicated project.
Request:
when the payment.code <> "paid", I have to do the following two things for the file.
to change the payment.code ="denied" and payment.text ="denied"
to add a JSON object to item.ADJ
When the payment.code =="paid", don't need to change anything.
Input :
{
"resourceType": "E",
"id": "11",
"identifier": [
{
"type": {
"coding": [
{
"system": "sys1",
"code": "aaa"
}
]
},
"value": "212"
},
{
"type": {
"coding": [
{
"system": "sys2",
"code": "RRR"
}
]
},
"value": "367"
}
],
"status": "active",
"created": "2021-08-05T02:43:48+00:00",
"outcome": "complete",
"item": [
{
"sequence": 1,
"product": {
"coding": [
{
"system": "example",
"code": "abc",
"display": "ABC"
}
],
"text": "ABC"
},
"servicedDate": "2021-08-04",
"quantity": {
"value": 60
},
"ADJ": [
{
"category": {
"coding": [
{
"system": "code1",
"code": "code1",
"display": "CODE1"
}
],
"text": "CODE1"
},
"amount": {
"value": 46.45,
"currency": "USD"
}
},
{
"category": {
"coding": [
{
"system": "code2",
"code": "code2",
"display": "CODE2"
}
],
"text": "CODE2"
},
"amount": {
"value": 12.04,
"currency": "USD"
}
}
]
}
],
"payment": {
"type": {
"coding": [
{
"system": "http://payment.com",
"code": "reversed/cancelled"
}
],
"text": "cancelled"
}
}
}
My Expected Output :
{
"resourceType": "E",
"id": "11",
"identifier": [
{
"type": {
"coding": [
{
"system": "sys1",
"code": "aaa"
}
]
},
"value": "212"
},
{
"type": {
"coding": [
{
"system": "sys2",
"code": "RRR"
}
]
},
"value": "367"
}
],
"status": "active",
"created": "2021-08-05T02:43:48+00:00",
"outcome": "complete",
"item": [
{
"sequence": 1,
"product": {
"coding": [
{
"system": "example",
"code": "abc",
"display": "ABC"
}
],
"text": "ABC"
},
"servicedDate": "2021-08-04",
"quantity": {
"value": 60
},
"ADJ": [
{
"category": {
"coding": [
{
"system": "code1",
"code": "code1",
"display": "CODE1"
}
],
"text": "CODE1"
},
"amount": {
"value": 46.45,
"currency": "USD"
}
},
{
"category": {
"coding": [
{
"system": "code2",
"code": "code2",
"display": "CODE2"
}
],
"text": "CODE2"
},
"amount": {
"value": 12.04,
"currency": "USD"
}
},
{// new object I want to insert into
"category": {
"coding": [
{
"system": "sys_denail",
"code": "denialreason"
}
],
"reason": {
"coding": [
{
"system": "https://example.com",
"code": "A1"
}
],
"text": "unknown"
}}
}
]
}
],
"payment": {
"type": {
"coding": [
{
"system": "http://payment.com",
"code": "denied" //change the value to denied
}
],
"text": "denied" //change the value to denied
}
}
}
Edit : I've tried to answer the second case by myself to be evaluated after the first case is answered
Welcome to SO, please ask minimal and reproducible questions, and show your effort tried for the future.
What you need is to use a conditional logic along with placeholder values with ampersand symbols depending on the levels of each key name within the tree.
I have partially answered, which will handle the bottom part of your question. Indeed the logic for the rest(inserting an object to the array will be similiar)
So, consider having a look at the following solution
[
{
"operation": "shift",
"spec": {
"*": "&",
"payment": {
"type": {
"coding": {
"*": {
"*": "&4.&3.&2[&1].&",
"code": {
"paid": {
"#1": "&6.&5.&4[&3].&2",
"#(4,text)": "&6.text"
},
"*": {
"#denied": "&6.&5.&4[&3].code",
"#(4,text)": {
"#denied": "&6.text"
}
}
}
}
}
}
}
}
}
]
Edit(for your own answer related to adding an object):
your current idea of using shift after default transformation spec is pretty good, you can rephrase like
[
{
"operation": "default",
"spec": {
"temp_deny": {
"denialreason": {
"category": {
"coding": [
{
"system": "sys_denail",
"code": "denialreason"
}
],
"reason": {
"coding": [
{
"system": "https://example.com",
"code": "A1"
}
],
"text": "unknown"
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"item": {
"*": {
"*": "&2[&1].&",
"ADJ": {
"#": "&3[&2].&",
"#(4,temp_deny)": "&3[&2].&"
}
}
}
}
}
]

Selectively get certain arrays from inside an array in JSON using JOLT

I've been trying to get some fields out of a very long and complicated json format but not getting the output I want.
MY current spec obtains all events from the event array and lists them in the output. I'm unsure how to select specific events and only output those. Am not quite sure of the syntax
My JSON:
{
"rootid": "19718",
"clloadm": "2021-06-01T22:40:02",
"clload": "2021-06-01T21:21:39",
"date": "2021-05-25T21:52:30",
"events": [
{
"done": {
"id": "e0",
"value": "2021-05-29T08:08:19"
},
"id": "e0_event",
"started": {
"id": "e0",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e1",
"value": "2021-05-27T02:20:25"
},
"id": "e1_event",
"started": {
"id": "e1",
"value": "2021-05-27T02:20:25"
},
"status": "complete"
},
{
"done": {
"id": "e2",
"value": "2021-05-29T08:08:19"
},
"id": "e2_event",
"started": {
"id": "e2",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e3",
"value": "2021-05-29T08:08:19"
},
"id": "e3_event",
"started": {
"id": "e3",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e4",
"value": "2021-05-29T08:08:19"
},
"id": "e4_event",
"started": {
"id": "e4",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
}
],
"ids": [
{
"id": "id",
"source": "source",
"value": "value"
},
{
"id": "new_id",
"source": "new_source",
"value": "value"
}
]
}
My Jolt Spec that gets all events for now:
[
{
"operation": "shift",
"spec": {
"rootid": "rootid",
"clloadm": "clloadm",
"clload": "clload",
"date": "date",
"events": {
"*": {
"*": {
"#value": "#id"
}
}
},
"ids": {
"*": {
"#value": "#id"
}
}
}
}
]
The output I get:
{
"rootid" : "19718",
"clloadm" : "2021-06-01T22:40:02",
"clload" : "2021-06-01T21:21:39",
"date" : "2021-05-25T21:52:30",
"e0" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e1" : [ "2021-05-27T02:20:25", "2021-05-27T02:20:25" ],
"e2" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e3" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e4" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"id" : "value",
"new_id" : "value"
}
The output I would like
{
"rootid" : "19718",
"clloadm" : "2021-06-01T22:40:02",
"clload" : "2021-06-01T21:21:39",
"date" : "2021-05-25T21:52:30",
"e0" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e4" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"id" : "value",
"new_id" : "value"
}
You can write the individual keys e0 and e4 as conditional cases for #id key while rewriting the rest of the key-value pairs through "*":"&" representation such as
[
{
"operation": "shift",
"spec": {
"*": "&",
"events": {
"*": {
"*": {
"#id": {
"e0": { "#(2,value)": "&" },
"e4": { "#(2,value)": "&" }
}
}
}
},
"ids": {
"*": {
"#value": "#id"
}
}
}
}
]

How to get the whole json with the result reflected after filter by string using JsonPath

Can I get the whole json with the result reflected after filter by string using JsonPath?
Before JSON
{
"store": {
"book": [
{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95
},
{
"category": "fiction",
"author": "Evelyn Waugh",
"title": "Sword of Honour",
"price": 12.99
}
],
"bicycle": {
"color": "red",
"price": 19.95
}
},
"expensive": 10
}
After JSON (I want this result)
{
"store": {
"book": [
{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95
}
],
"bicycle": {
"color": "red",
"price": 19.95
}
},
"expensive": 10
}
Filter by string for book is done through '$..book[?(#.author =~ /.*REES/i)]'
but can you tell me to get the whole JSON reflected?
You cannot do this using JSONPath; while you want to select the outer-most item based on inner items property, you do not want to retrieve the outer item as a whole. That does not work.
Selecting the outer item can be done like this
$..store[?(#.book[0].author =~ /.*REES/i)]
but this will also return the book at position 2, i.e. index 1.
It sounds like you actually need a JSON transformation like Jolt.
For instance, Jolt's remove transformer would allow you to to remove the second book like this:
[
{
"operation": "remove",
"spec": {
"store": {
"book": {
"array": {
"1": "" //remove book at position 2
}
}
}
}
}
]
Try it online with your own input here.

Jolt Transform - Unable to transform json array into elements are objects with the specified key/value pairs

I'm trying to transform JSON array into elements are objects with the specified key/value pairs
{
"Resource": "sorPersonRole",
"Roleid": "1",
"Timestamp": "2010-06-30 00:00:00.0",
"Release": "Public",
"DOB": "2064-09-05",
"Active": "Y",
"enterprise_id": "a12s33",
"Inactive_enterprise_id": "",
"emp_ID": "123456",
"Inactive_emp_id": "000821972",
"Username": "",
"A_ID": "fsgf1234jhgfs3",
"P_ID": "w123456",
"Is Email Valid": "Y",
"Flag": "N",
"Registered": "spring",
"Description": "mainland corp",
"End Date": null
}
Expected output:
{
"meta" : {
"Resource" : "sorPersonRole",
"Roleid" : "1",
"Timestamp" : "2010-06-30 00:00:00.0",
"Release" : "Public",
"Sorid" : "w123456"
},
"sorAttributes" : {
"DOB" : "2064-09-05",
"Active" : "Y",
"End Date" : null,
"identifiers":
[
{
"type" : "enterprise"
"enterprise_id" : "a12s33",
"Username" : ""
},
{
"type" : "former-enterprise"
"Inactive_enterprise_id" : ""
},
{
"type" : "UID"
"emp_ID" : "123456",
"Inactive_emp_id" : "000821972"
},
{
"type" : "National"
"A_ID" : "fsgf1234jhgfs3"
}
],
"mainLand:com:adhoc" : {
"Is Email Valid" : "Y",
"Flag" : "N",
"Registered" : "spring",
"Description" : "mainland corp"
}
}
}
current Jolt spec: which I am not getting desired output
[
{
"operation": "shift",
"spec": {
"Resource": "meta.&",
"P_ID": "meta.Sorid",
"Roleid": "meta.&",
"Timestamp": "meta.&",
"Release": "meta.&",
"enterprise_id": "sorAttributes.Identifiers.type.enterprise.&",
"Inactive_enterprise_id": "sorAttributes.Identifiers.type.former-enterprise.&",
"emp_ID": "sorAttributes.Identifiers.type.UID.&",
"Inactive_emp_id": "sorAttributes.Identifiers.type.UID.&",
"Username": "sorAttributes.Identifiers.type.enterprise.&",
"A_ID": "sorAttributes.Identifiers.type.National.&",
"Is Email Valid": "sorAttributes.mainLand:com:adhoc.&",
"Flag": "sorAttributes.mainLand:com:adhoc.&",
"Registered": "sorAttributes.mainLand:com:adhoc.&",
"Description": "sorAttributes.mainLand:com:adhoc.&",
"*": "sorAttributes.&"
}
}
]
I have tried the different JsonSpecs provided on different websites, could able to match expected output. Also tried using two-shift operations but no luck, Any help or suggestion will be appreciated.
Thanks.
This can help,
For the nodes to be shifted into the identifier array, shift one level more.
[
{
"operation": "shift",
"spec": {
"Resource": "meta.&",
"Roleid": "meta.&",
"Timestamp": "meta.&",
"Release": "meta.&",
"P_ID": "meta.Sorid",
"DOB": "sorAttributes.&",
"Active": "sorAttributes.&",
"End Date": "sorAttributes.&",
"Is Email Valid": "sorAttributes.mainLand:com:adhoc.&",
"Flag": "sorAttributes.mainLand:com:adhoc.&",
"Registered": "sorAttributes.mainLand:com:adhoc.&",
"Description": "sorAttributes.mainLand:com:adhoc.&",
"enterprise_id": {
"#enterprise": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&",
"#(1,Username)": "sorAttributes.identifiers[#2].Username"
},
"Inactive_enterprise_id": {
"#former-enterprise": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&"
},
"Inactive_emp_id": {
"#UID": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&",
"#(1,emp_ID)": "sorAttributes.identifiers[#2].emp_ID"
},
"A_ID": {
"#National": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&"
}
}
}, {
"operation": "modify-overwrite-beta",
"spec": {
"*": "=recursivelySquashNulls"
}
}
]

Json Jolt - Shifting data to multiple sub-arrays

I am trying to do a number of shifts to copy some data into multiple arrays and I am having trouble getting the data to correct array.
This is what I have
{
"input": {
"rating_date": "2018-08-06",
"Rates": {
"name": "Generic ratings Card",
"version": "v1",
"value": "2600.00",
"name1": [
{
"hits": "321",
"genre": "fiction",
"ratingName": "name1"
},
{
"hits": "654",
"genre": "fiction",
"ratingName": "name1"
}
],
"name2": [
{
"hits": "123",
"genre": "nonfiction",
"ratingName": "name2"
},
{
"hits": "456",
"genre": "fiction",
"ratingName": "name2"
}
]
}
},
"spec": {
"operation": "shift",
"spec": {
"rating_date": "rating_by_date[#0].date",
"Rates": {
"name*": {
"$": "rating_by_date[#2].ratecards[#1].type",
"*": {
"#": "rating_by_date[#3].ratecards[#2].rates[#0].&"
}
}
}
}
},
"expected": {
"rating_by_date": [
{
"date": "2018-08-06",
"ratecards": [
{
"type": "name1",
"rates": [
{
"hits": "321",
"genre": "fiction",
"ratingName": "name1"
},
{
"hits": "654",
"genre": "fiction",
"ratingName": "name1"
}
]
},
{
"type": "name2",
"rates": [
{
"hits": "123",
"genre": "nonfiction",
"ratingName": "name2"
},
{
"hits": "456",
"genre": "fiction",
"ratingName": "name2"
}
]
}
]
}
]
}
}
The idea is to take the objects in the rates array and transform them (in a later shift) into Key-Value pairs, which I already know how to do.
Now, the spec I have is not shifting the data into appropriate arrays and I end up with something like this:
{
"rating_by_date" : [ {
"date" : "2018-08-06",
"ratecards" : [ {
"type" : "name1",
"rates" : [ {
"0" : {
"hits" : "321",
"genre" : "fiction",
"ratingName" : "name1"
}
} ]
}, {
"rates" : [ {
"1" : {
"hits" : "654",
"genre" : "fiction",
"ratingName" : "name1"
}
} ]
} ]
}, {
"ratecards" : [ {
"type" : "name2",
"rates" : [ {
"0" : {
"hits" : "123",
"genre" : "nonfiction",
"ratingName" : "name2"
}
} ]
}, {
"rates" : [ {
"1" : {
"hits" : "456",
"genre" : "fiction",
"ratingName" : "name2"
}
} ]
} ]
} ]
}
Instead of creating new objects in the same array, it is creating new arrays altogether. I am clearly not understanding exactly how to reference array structures on the RHS, I'd appreciate a clarification.
I managed to get it to work, but I'm not sure why it would not work in one shift step. Here is what I had to do to make it work:
[
{
"operation": "shift",
"spec": {
"rating_date": "rating_by_date[0].date",
"Rates": {
"name*": {
"$": "rating_by_date[0].ratecards.&.type",
"#": "rating_by_date[0].ratecards.&.rates"
}
}
}
},
{
"operation": "shift",
"spec": {
"rating_by_date": {
"*": {
"ratecards": {
"*": {
"#": "rating_by_date[&3].ratecards2[]"
}
},
"*": "rating_by_date[&1].&"
}
}
}
}
]