Jolt Transformation order of fields - json

I've following input json
{
"NAME1": "Mueller",
"NAME2": "Max",
"STREET1": "Adlerstr.",
"STREET2": "1",
"COUNTRY": "Munich"
}
I renamed the name of the fields and concatinate the street* fields
[
{
"operation": "shift",
"spec": {
"NAME1": "name",
"NAME2": "surName",
"STREET1": "tmp_street1",
"STREET2": "tmp_street2",
"COUNTRY": "country"
}
},
{
"operation": "modify-default-beta",
"spec": {
"street": "=concat(#(1,tmp_street1),' ',#(1,tmp_street2))"
}
},
{
"operation": "remove",
"spec": {
"tmp_*": ""
}
}
]
I getting the following output
{
"name" : "Mueller",
"surName" : "Max",
"country" : "Munich", // Should be after the street field
"street" : "Adlerstr. 1"
}
Is it possible to move the street field on the 3rd position in the json output ?


Yes, street field can be moved to the 3rd position of the json output, with only two operations.
Check this spec,
[
{
"operation": "modify-default-beta",
"spec": {
"street": "=concat(#(1,STREET1),' ',#(1,STREET2))"
}
},
{
"operation": "shift",
"spec": {
"NAME1": "name",
"NAME2": "surName",
"street": "street",
"COUNTRY": "country"
}
}
]

Related

How to correct this jolt transformation

I have an input JSON like this.
{
"trackingNumber": "1ZEA83550362028861",
"localActivityDate": "20210324",
"localActivityTime": "183500",
"scheduledDeliveryDate": "20220525",
"actualDeliveryDate": "20220729",
"actualdeliveryTime": "183500",
"gmtActivityDate": "20210324",
"gmtActivityTime": "223500",
"activityStatus": {
"type": "G",
"code": "OR",
"description": "Origin Scan"
},
"activityLocation": {
"city": "RANDALLSTOWN,",
"stateProvince": "MD",
"postalCode": "21133",
"country": "US"
}
}
I have written a jolt transformation for this JSON
[
{
"operation": "shift",
"spec": {
"trackingNumber": "transformedPayload.trackingInfo",
"localActivityDate": "tmp_Date",
"localActivityTime": "tmp_Time",
"scheduledDeliveryDate": "tmp_App",
"actualDeliveryDate": "tmp_Del_Date",
"actualdeliveryTime": "tmp_Del_Time",
"activityStatus": {
"type": "transformedPayload.events.type",
"code": "transformedPayload.events.statusCode",
"description": "transformedPayload.events.statusDescription"
},
"activityLocation": {
"city": "transformedPayload.address.city",
"stateProvince": "transformedPayload.address.state",
"postalCode": "transformedPayload.address.postalCode",
"country": "transformedPayload.address.country"
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"tmp_Year": "=substring(#(1,tmp_Date),0,4)",
"tmp_Month": "=substring(#(1,tmp_Date),4,6)",
"tmp_Day": "=substring(#(1,tmp_Date),6,8)",
"tmp_Hours": "=substring(#(1,tmp_Time),0,2)",
"tmp_Minutes": "=substring(#(1,tmp_Time),2,4)",
"tmp_Seconds": "=substring(#(1,tmp_Time),4,6)",
"timeStamp": "=concat(#(1,tmp_Year),'-',#(1,tmp_Month),'-',#(1,tmp_Day),'T',#(1,tmp_Hours),':',#(1,tmp_Minutes),':',#(1,tmp_Seconds),'Z')",
"tmp_App_Year": "=substring(#(1,tmp_App),0,4)",
"tmp_App_Month": "=substring(#(1,tmp_App),4,6)",
"tmp_App_Day": "=substring(#(1,tmp_App),6,8)",
"appointmentTime": "=concat(#(1,tmp_App_Year),'-',#(1,tmp_App_Month),'-',#(1,tmp_App_Day))",
"tmp__Del_Year": "=substring(#(1,tmp_Del_Date),0,4)",
"tmp_Del_Month": "=substring(#(1,tmp_Del_Date),4,6)",
"tmp_Del_Day": "=substring(#(1,tmp_Del_Date),6,8)",
"tmp_Del_Hours": "=substring(#(1,tmp_Del_Time),0,2)",
"tmp_Del_Minutes": "=substring(#(1,tmp_Del_Time),2,4)",
"tmp_Del_Seconds": "=substring(#(1,tmp_Del_Time),4,6)",
"deliveryTime": "=concat(#(1,tmp__Del_Year),'-',#(1,tmp_Del_Month),'-',#(1,tmp_Del_Day),'T',#(1,tmp_Del_Hours),':',#(1,tmp_Del_Minutes),':',#(1,tmp_Del_Seconds),'Z')"
}
},
{
"operation": "remove",
"spec": {
"tmp_*": ""
}
}
]
This transforms the data into this format.
{
"transformedPayload" : {
"trackingInfo" : "1ZEA83550362028861",
"events" : {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan"
},
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
}
},
"timeStamp" : "2021-03-24T18:35:00Z",
"appointmentTime" : "2022-05-25",
"deliveryTime" : "2022-07-29T18:35:00Z"
}
What changes do i need to make in the transformation such that the timestamp, appointmentTime and deliveryTime are also nested under transformedPayload i.e it looks like this (correct format).
{
"transformedPayload" : {
"trackingInfo" : "1ZEA83550362028861",
"events" : {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan"
},
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"timeStamp" : "2021-03-24T18:35:00Z",
"appointmentTime" : "2022-05-25",
"deliveryTime" : "2022-07-29T18:35:00Z"
}
}
This is my first time doing a jolt transformation so i am confused on how to resolve this. Any help is appreciated.

You are already so close to solution,I can offer the following spec similar to yours to the desired output :
[
{
"operation": "modify-overwrite-beta",
"spec": {
"tsY": "=substring(#(1,localActivityDate),0,4)",
"tsM": "=substring(#(1,localActivityDate),4,6)",
"tsD": "=substring(#(1,localActivityDate),6,8)",
"tsH": "=substring(#(1,localActivityTime),0,2)",
"tsMi": "=substring(#(1,localActivityTime),2,4)",
"tsS": "=substring(#(1,localActivityTime),4,6)",
"timeStamp": "=concat(#(1,tsY),'-',#(1,tsM),'-',#(1,tsD),'T',#(1,tsH),':',#(1,tsMi),':',#(1,tsS),'Z')",
"aTY": "=substring(#(1,scheduledDeliveryDate),0,4)",
"aTM": "=substring(#(1,scheduledDeliveryDate),4,6)",
"aTD": "=substring(#(1,scheduledDeliveryDate),6,8)",
"appointmentTime": "=concat(#(1,aTY),'-',#(1,aTM),'-',#(1,aTD))",
"dTY": "=substring(#(1,actualDeliveryDate),0,4)",
"dTM": "=substring(#(1,actualDeliveryDate),4,6)",
"dTD": "=substring(#(1,actualDeliveryDate),6,8)",
"dTH": "=substring(#(1,actualdeliveryTime),0,2)",
"dTMi": "=substring(#(1,actualdeliveryTime),2,4)",
"dTS": "=substring(#(1,actualdeliveryTime),4,6)",
"deliveryTime": "=concat(#(1,dTY),'-',#(1,dTM),'-',#(1,dTD),'T',#(1,dTH),':',#(1,dTMi),':',#(1,dTS),'Z')"
}
},
{
"operation": "shift",
"spec": {
"*Number": "&(0,1)Info",
"activityStatus": {
"*": "events.&"
},
"activityLocation": {
"*": "address.&"
},
"timeStamp": "&",
"appointmentTime": "&",
"deliveryTime": "&"
}
}
]

Rearraning of fields in nested JSON using Jolt

I have a requirement where re-arrangement of fields need to be done in a nested JSON which has array objects.
Below is the sample input:
{
"_id": "1234",
"Name": "John",
"City": "New Jersey",
"Order": [
{
"item": "pizza",
"qty": 2
},
{
"item": "coke",
"qty": 2
}
],
"State": "NJ",
"Phone": "67755321",
"Paymenttype": "Card"
}
My Jolt is able to populate all fields, but unable to figure out how to get the 'order data' in same manner in output
My code:
[
{
"operation": "shift",
"spec": {
"_id": "_id",
"Name": "Name",
"State": "State",
"Phone": "Phone"
}
}
]
Expected output:
{
"_id" : "1234",
"Name" : "John",
"State" : "NJ",
"Phone" : "67755321",
"Order": [
{
"item": "pizza",
"qty": 2
},
{
"item": "coke",
"qty": 2
}
],
"Paymenttype": "Card"
}

You can just use ampersand substitutions for the values of the attributes such as
[
{
"operation": "shift",
"spec": {
"_id": "&",
"Name": "&",
"State": "&",
"Phone": "&",
"Order": "&",
"Paymenttype": "&"
}
}
]
the demo on the site https://jolt-demo.appspot.com/ is :
even using "*" wildcards within the keys might be practical(depending on the existing key names) such as
[
{
"operation": "shift",
"spec": {
"_id": "&",
"N*": "&",
"St*": "&",
"Ph*": "&",
"Or*": "&",
"Pa*": "&"
}
}
]

JSON Transformation using JOLT: Nested Array and Concatenation

I have the following input JSON
[
{
"name": "Project1",
"Addresses": [
[
"B24",
"Niyam Street",
"67897",
"New York"
],
[
"A14",
"Prinston Str",
"London"
]
]
},
{
"name": "Project2",
"Addresses": [
[
"123",
"Portland Street",
"234"
],
[
"Lalbag",
"Kolaba"
],
[
"8th Avenue",
"3rd Signal"
]
]
}
]
I would like to transform it into like the below:
[
{
"name": "Project1",
"Addresses": [
"B24 Niyam Street 67897 New York",
"A14 Prinston Str London"
]
},
{
"name": "Project2",
"Addresses": [
"123 Portland Street 234",
"Lalbag Kolaba",
"8th Avenue 3rd Signal"
]
}
]
The Addresses attribute value is a two-dimensional array of dynamic size.
Could you please help me with a valid jolt spec or some hints to achieve it? I am lost. Thank you so much.

You can use combination of shift and modify transformations such as
[
{
"operation": "shift",
"spec": {
"*": {
"*": "&", // the attributes other than "Addresses"
"Addresses": {
"*": "&1.&2.&" // determine the objects with indexed key names(0,1,2...) where &2 represents going two level up the tree to get the outermost indices of the objects of the main array, & for indices of the "Addresses" array, and &1 to keep the key name "Addresses" to transfer to the upcoming specs
}
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": {
"*": "=join(' ',#(1,&))" // concatenate all components of each array respectively at once
}
}
}
},
{
// combine the attributes back again
"operation": "shift",
"spec": {
"*": {
"*": "[&].&1"
},
"A*": {
"*": {
"*": "[&1].&2"
}
}
}
}
]
the demo on the site http://jolt-demo.appspot.com/ is

How to transform a flat JSON to nested JSON using NiFi

Input JSON :
{
"type": "mbrInfo",
"csId": 123456789,
"insTS": "14-07-201911:55",
"seqId": 1234565,
"title": "Mr",
"fName": "Amit",
"mName": "",
"lName": "V",
"suffix": "Engg",
"lvlId": "P",
"lvlType": "LAC",
"acctStatus": "20",
"enrlDT": "2016-08-29",
"vrsnId": 1
}
Expected Output JSON:
{
"type": "mbrInfo",
"csId": 123456789,
"insTS": "14-07-201911:55",
"seqId": 1234565,
"name" : [{
"title": "Mr",
"fName": "Amit",
"mName": "",
"lName": "V",
"suffix": "Engg"}],
"lvlId": "P",
"lvlType": "LAC",
"acctStatus": "20",
"enrlDT": "2016-08-29",
"vrsnId": 1
}.
Currently I am using JOLTtransformJSON processor with JOLT Spec as :
[
{
"operation": "shift",
"spec": {
"name": {
"$": "[#1]",
"#.title": "[#1].title",
"#.fName": "[#1].fName",
"#.mName": "[#1].mName",
"#.lName": "[#1].lName",
"#.suffix": "[#1].suffix"
}
}
}
]
But all I am getting is either NULL or the original JSON (with diff spec) as output.
Thanks in advance.

Is the intent to put all the name fields into a 1-element array containing an object. This JOLT spec puts them into an object at the name field:
[
{
"operation": "shift",
"spec": {
"title": "name.title",
"fName": "name.fName",
"mName": "name.mName",
"lName": "name.lName",
"suffix": "name.suffix",
"*": "&"
}
}
]
...and this spec puts them into a 1-element array at the name field:
[
{
"operation": "shift",
"spec": {
"title": "name[0].title",
"fName": "name[0].fName",
"mName": "name[0].mName",
"lName": "name[0].lName",
"suffix": "name[0].suffix",
"*": "&"
}
}
]
I don't see any other place in the input to get an index into the array, so I just used 0.

jolt concatenate array elements to single object

I started with Jolt, but I can't concatenate the elements of the array to single string,
I have json like this:
{
"partNb": "1234",
"partDescriptions": [
{
"country": "GB",
"language": "en",
"content": "1 tool description in en_GB"
},
{
"country": "GB",
"language": "en",
"content": "2 tool description in en_GB"
}
]
}
and with jolt spec:
[
{
"operation": "shift",
"spec": {
"partNb": "id",
"partDescriptions": {
"*": {
"content": "description"
}
}
}
}
]
For this I have this output:
{
"id" : "1234",
"description" : [ "1 tool description in en_GB", "2 tool description in en_GB" ]
}
but how to get result like this?:
{
"id" : "1234",
"description" : "1 tool description in en_GB , 2 tool description in en_GB"
}

Spec
[
{
"operation": "modify-overwrite-beta",
"spec": {
"description": "=join(', ',#(1,description))"
}
}
]

To get only content fields concatenated into descriptions:
[
{
"operation": "shift",
"spec": {
"partDescriptions": {
"*": {
"content": {
"#": "content"
}
}
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"description": "=join(', ', #(2,content))"
}
}
]
Output:
{
"content" : [ "1 tool description in en_GB", "2 tool description in en_GB" ],
"description" : "1 tool description in en_GB, 2 tool description in en_GB"
}