Jolt List Transformation - json

I am trying to transform a JSON using jolt without adding new items as well as remove white space from the key value. additionally, I would like to know how to get map to current list.
input :
[
{
"id": "9164339966213408050",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"Total BYOP Tier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468326"
}
}
],
"chars": {
"Total BYOP Tier": "60"
}
},
{
"id": "9164339966213408051",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"Total BYOP Tier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468325"
}
}
],
"chars": {
"Total BYOP Tier": "60"
}
}
]
expected output :
[
{
"id": "9164339966213408050",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468326",
"TotalBYOPTier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468326"
}
}
],
"chars": {
"TotalBYOPTier": "60"
}
},
{
"id": "9164339966213408051",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468325",
"TotalBYOPTier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468325"
}
}
],
"chars": {
"TotalBYOPTier": "60"
}
}
]
spec tried :
[
{
"operation": "shift",
"spec": {
"*": {
"#": "CustomerProductArray"
}
}
},
{
"operation": "shift",
"spec": {
"CustomerProductArray": {
"*": {
"extendedChars": {
"*": {
"* * *": "CustomerProductArray.&(0,1)&(0,2)&(0,3)",
"*": {
"name": "CustomerProductArray[].extendedChars[]..name",
"isMultiple": "CustomerProductArray[].extendedChars[].isMultiple",
"tomsId": "CustomerProductArray[].extendedChars[].tomsId",
"values": "CustomerProductArray[].extendedChars[].values"
}
}
},
"chars": {
"* * *": "CustomerProductArray[].chars.&(0,1)&(0,2)&(0,3)"
}
}
}
}
}
]

Use the below spec as you want to replicate the content of the Total BYOP Tier array, nested inside and outside the object
[
{
"operation": "shift",
"spec": {
"*": {
"*": "&1.&",
"extendedChars": {
"*": {
"Total BYOP Tier": {
"*": {
"#": "&5.&4[&3].&" // &5 : (going up the tree five levels) replicates the index of the outermost array, &4[&3] : stands for "extendedChars"(&4) along strolling through with its indexes([&3])
},
"#": "&4.&3[&2].&1"
}
}
}
}
}
},
{
// get rid of object labels
"operation": "shift",
"spec": {
"*": ""
}
}
]
the demo on the site http://jolt-demo.appspot.com/ is

Related

JOLT filter - Remove fields from nested array based on condition for two different JSON values

I have to make a JOLT filter to delete a field based on condition, also the filter has to work with two different JSON structures.
So, here is the case.
I have these JSONs:
JSON 1:
{
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key2": "val2",
"key3": "val3",
"key4": "val4",
"key5": "val5",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
}
}
JSON 2:
{
"responseMessage": {
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key2": "val2",
"key3": "val3",
"key4": "val4",
"key5": "val5",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
},
"state": {
"state1": "state1"
}
},
"status": [],
"notes": []
}
I need the filter to remove for example key2 and key5 from the attributes_here only if the service value is THIS_SERVICE and keep the same structure for the JSON.
Desired output:
JSON 1:
{
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key3": "val3",
"key4": "val4",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
}
}
JSON 2:
{
"responseMessage": {
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key3": "val3",
"key4": "val4",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
},
"state": {
"state1": "state1"
}
},
"status": [],
"notes": []
}
Any help would be appreciated. Thank you and I wish you a great day!
You can use the following spec as an option :
[
{
"operation": "shift",
"spec": {
"responseMessage": { // distinguish the elements whether they have an extra wrapper
"*": "&1_&"
},
"*": "&"
}
},
{ // accumulate the respective attributes by their "service" value within the "attributes_here" object
"operation": "shift",
"spec": {
"*": "&",
"*ayload": {
"*": "&1.&",
"elements": {
"*": {
"*": "&3.&2[#2].&",
"attributes": {
"*": {
"*": "&5.&4[#4].&2[#2].&",
"attributes_here": {
"*": "&6.&5[#5].&3[#3].&1.&",
"key2|key5": "&6.&5[#5].&3[#3].&1.#4,service.&"
}
}
}
}
}
}
}
},
{// get rid of the keys with "THIS_SERVICE"
"operation": "remove",
"spec": {
"*ayload": {
"elements": {
"*": {
"attributes": {
"*": {
"attributes_here": {
"THIS_SERVICE": ""
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*_*": {
"#": "&(1,1).&(1,2)" // the value of the first and second asterisks, after going one level up the tree, respectively
},
"*": "&"
}
}
]
Please try the below jolt spec:
[
{
"operation": "shift",
"spec": {
"#false": "has_responseMessage",
"header": "&",
"payload": "&",
"data": "&",
"responseMessage": {
"*": "&",
"#(2,status)": "rest_&.status",
"#(2,notes)": "rest_&.notes",
"#true": "has_responseMessage"
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"payload": {
"*": "&1.&",
"elements": {
"*": { // 0, 1
"*": "&3.&2[&1].&",
"service": {
"#": "&4.&3[&2].&1",
"THIS_SERVICE": {
"#(2,attributes)": "&5.&4[&3].attributes_temp",
"#attributes_temp": "&5.&4[&3].attributes_key"
},
"#attributes": "&4.&3[&2].attributes_key"
}
}
}
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"has_responseMessage": "=lastElement",
"payload": {
"elements": {
"*": {
"attributes_key": "=lastElement"
}
}
}
}
},
{
"operation": "remove",
"spec": {
"payload": {
"elements": {
"*": {
"attributes_temp": {
"*": {
"attributes_here": {
"key2": "",
"key5": ""
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"payload": {
"*": "&1.&",
"elements": {
"*": { // 0, 1
"service": "&3.&2[&1].&",
"something": "&3.&2[&1].&",
"attributes_key": {
"attributes_temp": {
"#(2,attributes_temp)": "&5.&4[&3].attributes"
},
"attributes": {
"#(2,attributes)": "&5.&4[&3].attributes"
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"has_responseMessage": {
"true": {
"#(3,header)": "responseMessage.header",
"#(3,payload)": "responseMessage.payload",
"#(3,data)": "responseMessage.data",
"#(3,state)": "responseMessage.state"
},
"false": {
"#(3,header)": "header",
"#(3,payload)": "payload",
"#(3,data)": "data"
}
},
"rest_responseMessage": {
"*": "&"
}
}
}
]
You can remove your fields in the remove operation as you want.

Transform complex data by jolt processor in apache nifi

I have following expected input and expected output and i want to transform data by jolt processor
Input JSON
{
"subscriptionId": "63",
"data": [
{
"type": "demo",
"Data": {
"transactionId": "598958",
"type": "json",
"xyz": "pqr",
"name": "john"
},
"nameData": [
{
"name": "Ama",
"nameCount": "215",
"genderData": [
{
"gender": "Male",
"genderCount": "140"
},
{
"gender": "Female",
"genderCount": "75"
}
]
},
{
"name": "Aedes",
"nameCount": "161",
"genderData": [
{
"gender": "Female",
"genderCount": "134"
},
{
"gender": "Male",
"genderCount": "27"
}
]
},
{
"name": "Culex",
"nameCount": "2610",
"genderData": [
{
"gender": "Male",
"genderCount": "1926"
},
{
"gender": "Female",
"genderCount": "684"
}
]
},
{
"name": "Kamp",
"nameCount": "465",
"genderData": [
{
"gender": "Male",
"genderCount": "465"
}
]
}
]
}
]
}
Expected Output JSON
{
"transactionId": "598958",
"type": "json",
"xyz": "pqr",
"name": "john",
"alert_array": [{
"abc": "123",
"xyz": "pqrs",
"properties": [{
"key": "Ama",
"value": "215",
"object": [{
"key": "Male",
"value": "140"
},
{
"key": "Female",
"value": "75"
}
]
},
{
"key": "Aedes",
"value": "161",
"object": [{
"key": "Male",
"value": "134"
},
{
"key": "Female",
"value": "27"
}
]
},
{
"key": "Culex",
"value": "2610",
"object": [{
"key": "Male",
"value": "1926"
},
{
"key": "Female",
"value": "684"
}
]
},
{
"key": "Kamp",
"value": "465",
"object": [{
"key": "Male",
"value": "465"
}]
},
{
"key": "type",
"value": "demo"
}
]
}]
}
I have the above following input JSON with nameData being an array and having genderData another array within it.
Need to convert the input JSON to expected JSON output as shown above.
please suggest JOLT spec to transform the above JSON.
You can start with determining the nodes alert_array,properties,object as diving to the innermost attributes within the shift transformation such as
[
{
"operation": "shift",
"spec": {
"data": {
"*": {
"Da*": "",
"nameD*": {
"#123": "alert_array[0].abc", // # wildcard is used to add fixed values
"#pqrs": "alert_array[0].xyz",
"*": {
"name": "alert_array[0].properties[&1].key",
"nameC*": "alert_array[0].properties[&1].value",
"*": "alert_array[0].properties[&1].object"
}
}
}
}
}
},
{
// rename the innermost attributes
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"key": "#(1,gender)",
"value": "#(1,genderCount)"
}
}
}
}
}
}
}
},
{
// get rid of the former names of the innermost attributes
"operation": "remove",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"gender": "",
"genderCount": ""
}
}
}
}
}
}
}
}
]
the demo on the site http://jolt-demo.appspot.com/ is
Edit : You can add two more shift transformation specs in order to put the lately desired object
{
"key" : "type",
"value" : "demo"
}
such as
[
{
"operation": "shift",
"spec": {
"data": {
"*": {
"Da*": "",
"nameD*": {
"#123": "alert_array[0].abc", // # wildcard is used to add fixed values
"#pqrs": "alert_array[0].xyz",
"*": {
"name": "alert_array[0].properties[&1].key",
"nameC*": "alert_array[0].properties[&1].value",
"*": "alert_array[0].properties[&1].object"
}
}
}
}
}
},
{
// rename the innermost attributes
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"key": "#(1,gender)",
"value": "#(1,genderCount)"
}
}
}
}
}
}
}
},
{
// get rid of the former names of the innermost attributes
"operation": "remove",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"gender": "",
"genderCount": ""
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"alert_array": {
"*": {
"*": "&2[&1].&",
"pro*": {
"#type": "&3[&2].&1.&2.key",
"#demo": "&3[&2].&1.&2.value",
"*": "&3[&2].&1.&"
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"alert_array": {
"*": {
"*": "&2[&1].&",
"pro*": {
"*": {
"#": "&4[&3].&2"
}
}
}
}
}
}
]

How to write JOLT Spec for nested arrays

I am trying to transform a JSON using JOLT. This JSON consists of nested arrays and I am not able to transform it correctly. Can someone please help. Thanks.
{
"root": [
{
"id": "1234",
"password": "key1234",
"devices": [
{
"details": {
"deviceType": "tv-iot",
"deviceId": "tv-iot-111"
}
},
{
"details": {
"deviceType": "machine-iot",
"deviceId": "machine-iot-999"
}
}
]
},
{
"id": "6789",
"password": "key6789",
"devices": [
{
"details": {
"deviceType": "phone-iot",
"deviceId": "phone-iot-111"
}
},
{
"details": {
"deviceType": "mobile-iot",
"deviceId": "mobile-iot-999"
}
}
]
}
]
}
This is the spec that I have written.
[
{
"operation": "shift",
"spec": {
"root": {
"*": {
"id": "[&1].userid",
"password": "[&1].pwd",
"devices": {
"*": {
"details": {
"deviceType": "[&2].deviceCategory",
"deviceId": "[&2].deviceUniqueValue"
}
}
}
}
}
}
}
]
The expected JSON that I am looking for is:
[
{
"userid": "1234",
"pwd": "key1234",
"devices": [
{
"details": {
"deviceCategory": "tv-iot",
"deviceUniqueValue": "tv-iot-111"
}
},
{
"details": {
"deviceCategory": "machine-iot",
"deviceUniqueValue": "machine-iot-999"
}
}
]
},
{
"userid": "6789",
"pwd": "key6789",
"devices": [
{
"details": {
"deviceCategory": "phone-iot",
"deviceUniqueValue": "phone-iot-111"
}
},
{
"details": {
"deviceCategory": "mobile-iot",
"deviceUniqueValue": "mobile-iot-999"
}
}
]
}
]
However, I get this wrong output. Somehow, my nested objects are getting transformed into list.
[
{
"userid" : "1234",
"pwd" : "key1234",
"deviceCategory" : [ "tv-iot", "phone-iot" ],
"deviceUniqueValue" : [ "tv-iot-111", "phone-iot-111" ]
},
{
"deviceCategory" : [ "machine-iot", "mobile-iot" ],
"deviceUniqueValue" : [ "machine-iot-999", "mobile-iot-999" ],
"userid" : "6789",
"pwd" : "key6789"
}
]
I am unable to figure out what is wrong. Can someone please help?
UPDATE(Solution): Was able to come up with a shorter spec that works as well !
[
{
"operation": "shift",
"spec": {
"root": {
"*": {
"id": "[&1].userId",
"password": "[&1].pwd",
"*": "[&1].&"
}
}
}
},
{
"operation": "shift",
"spec": {
"*": {
"devices": {
"*": {
"details": {
"deviceType": "[&4].&3.[&2].&1.deviceCategory",
"deviceId": "[&4].&3.[&2].&1.deviceUniqueVal"
}
}
},
"*": "[&1].&"
}
}
}
]
You can start by deep diving into the innermost object while partitioning the sub-objects by id values through a shift transformation such as
[
{
"operation": "shift",
"spec": {
"root": {
"*": {
"devices": {
"*": {
"details": {
"*": {
"#(4,id)": "#(5,id).userid",
"#(4,password)": "#(5,id).pwd",
"#": "#(5,id).devicedetails[&3].&2.&1"
}
}
}
}
}
}
}
},
{
// get rid of top level object names
"operation": "shift",
"spec": {
"*": ""
}
},
{
// get rid of repeating components of each arrays respectively
"operation": "cardinality",
"spec": {
"*": {
"us*": "ONE",
"pwd": "ONE"
}
}
},
{
// determine new key names for attributes respectively
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": {
"*": {
"*": {
"deviceCategory": "=(#(1,deviceType))",
"deviceUniqueValue": "=(#(1,deviceId))"
}
}
}
}
}
},
{
// get rid of extra elements generated
"operation": "remove",
"spec": {
"*": {
"*": {
"*": {
"*": {
"deviceType": "",
"deviceId": ""
}
}
}
}
}
}
]

Jolt specification to transform JSON array within array

I'm trying to transform a JSON file into something that can later be transformed to csv or xml. My JSON file looks like this:
{
"meta": {
"contentType": "Response"
},
"content": {
"_type": "Response",
"data": {
"_type": "ObjectList",
"meta": {
"_type": "ObjectListMeta"
},
"DataObjects": [
{
"head": {
"fields": {
"id": {
"value": "24"
},
"name": {
"value": ""
}
}
},
"table": [
{
"number": 1,
"fields": {
"height": {
"value": 500,
"text": "500"
},
"average": {
"value": -2,
"text": "-2"
}
}
},
{
"number": 2,
"fields": {
"height": {
"value": 99999,
"text": "99999"
},
"average": {
"value": -5,
"text": "-5"
}
}
}
]
}
]
}
}
}
I want the output to look like this:
[
{
"id": "24",
"name": "",
"height": 500,
"average": -2
},
{
"id": "24",
"name": "",
"height": 99999,
"average": -5
}
]
I tried the following JOLT specification:
[
{
"operation": "shift",
"spec": {
"content": {
"data": {
"DataObjects": {
"*": {
"head": {
"fields": {
"id": {
"value": "[&4].id"
},
"name": {
"value": "[&4].name"
}
}
},
"table": {
"*": {
"fields": {
"height": {
"value": "[&5].height"
},
"average": {
"value": "[&5].average"
}
}
}
}
}
}
}
}
}
}
]
But I only get the following output:
[
{
"id": "24",
"name": "",
"height" : [ 500, 99999 ],
"average" : [ -2, -5 ]
}
]
Which is not exactly what I want. How do I have to modify my spec to get the output I need?
You can combine those attributes under the table list as having multiple respective objects for one head.fields object such as
[
{
"operation": "shift",
"spec": {
"content": {
"*": {
"DataObjects": {
"*": {
"table": {
"*": {
"#(2,head.fields.id.value)": "[&].id",
"#(2,head.fields.name.value)": "[&].name",
"#(0,fields.height.value)": "[&].height",
"#(0,fields.average.value)": "[&].average"
}
}
}
}
}
}
}
}
]

Jolt specification needed for the below input and expected output

I need to split the data list into two separate idExists and idNotExists lists, based on whether id exists or not.
Can someone help me with Jolt Specification to achieve the below results?
Input:
{
"data": [
{
"name": "a",
"id": "100"
},
{
"name": "b",
"id": "101"
},
{
"name": "c"
}
]
}
Desired Output:
{
"IdExists": [
{
"name": "a",
"id": "100"
},
{
"name": "b",
"id": "101"
}
],
"IdNotExists": [
{
"name": "c"
}
]
}
I have tried with the below spec but it is not working as expected
[
{
"operation": "modify-default-beta",
"spec": {
"data": {
"": {
"id": "NotExist"
}
}
}
},
{
"operation": "shift",
"spec": {
"data": {
"": {
"id": {
"10*": {
"#2": "IdExists[]"
},
"*": {
"#2": "IdNotExists[]"
}
}
}
}
}
}
]
I had some modification on your spec, added * as it was missing in the selectors.
Modified shifting condition to NotExist and finally shift only name to the IdNotExists array.
[
{
"operation": "modify-default-beta",
"spec": {
"data": {
"*": {
"id": "NotExist"
}
}
}
},
{
"operation": "shift",
"spec": {
"data": {
"*": {
"id": {
"NotExist": {
"#(2,name)": "IdNotExists[].name"
},
"*": {
"#2": "IdExists[]"
}
}
}
}
}
}
]