How to correct this jolt transformation - json

I have an input JSON like this.
{
"trackingNumber": "1ZEA83550362028861",
"localActivityDate": "20210324",
"localActivityTime": "183500",
"scheduledDeliveryDate": "20220525",
"actualDeliveryDate": "20220729",
"actualdeliveryTime": "183500",
"gmtActivityDate": "20210324",
"gmtActivityTime": "223500",
"activityStatus": {
"type": "G",
"code": "OR",
"description": "Origin Scan"
},
"activityLocation": {
"city": "RANDALLSTOWN,",
"stateProvince": "MD",
"postalCode": "21133",
"country": "US"
}
}
I have written a jolt transformation for this JSON
[
{
"operation": "shift",
"spec": {
"trackingNumber": "transformedPayload.trackingInfo",
"localActivityDate": "tmp_Date",
"localActivityTime": "tmp_Time",
"scheduledDeliveryDate": "tmp_App",
"actualDeliveryDate": "tmp_Del_Date",
"actualdeliveryTime": "tmp_Del_Time",
"activityStatus": {
"type": "transformedPayload.events.type",
"code": "transformedPayload.events.statusCode",
"description": "transformedPayload.events.statusDescription"
},
"activityLocation": {
"city": "transformedPayload.address.city",
"stateProvince": "transformedPayload.address.state",
"postalCode": "transformedPayload.address.postalCode",
"country": "transformedPayload.address.country"
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"tmp_Year": "=substring(#(1,tmp_Date),0,4)",
"tmp_Month": "=substring(#(1,tmp_Date),4,6)",
"tmp_Day": "=substring(#(1,tmp_Date),6,8)",
"tmp_Hours": "=substring(#(1,tmp_Time),0,2)",
"tmp_Minutes": "=substring(#(1,tmp_Time),2,4)",
"tmp_Seconds": "=substring(#(1,tmp_Time),4,6)",
"timeStamp": "=concat(#(1,tmp_Year),'-',#(1,tmp_Month),'-',#(1,tmp_Day),'T',#(1,tmp_Hours),':',#(1,tmp_Minutes),':',#(1,tmp_Seconds),'Z')",
"tmp_App_Year": "=substring(#(1,tmp_App),0,4)",
"tmp_App_Month": "=substring(#(1,tmp_App),4,6)",
"tmp_App_Day": "=substring(#(1,tmp_App),6,8)",
"appointmentTime": "=concat(#(1,tmp_App_Year),'-',#(1,tmp_App_Month),'-',#(1,tmp_App_Day))",
"tmp__Del_Year": "=substring(#(1,tmp_Del_Date),0,4)",
"tmp_Del_Month": "=substring(#(1,tmp_Del_Date),4,6)",
"tmp_Del_Day": "=substring(#(1,tmp_Del_Date),6,8)",
"tmp_Del_Hours": "=substring(#(1,tmp_Del_Time),0,2)",
"tmp_Del_Minutes": "=substring(#(1,tmp_Del_Time),2,4)",
"tmp_Del_Seconds": "=substring(#(1,tmp_Del_Time),4,6)",
"deliveryTime": "=concat(#(1,tmp__Del_Year),'-',#(1,tmp_Del_Month),'-',#(1,tmp_Del_Day),'T',#(1,tmp_Del_Hours),':',#(1,tmp_Del_Minutes),':',#(1,tmp_Del_Seconds),'Z')"
}
},
{
"operation": "remove",
"spec": {
"tmp_*": ""
}
}
]
This transforms the data into this format.
{
"transformedPayload" : {
"trackingInfo" : "1ZEA83550362028861",
"events" : {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan"
},
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
}
},
"timeStamp" : "2021-03-24T18:35:00Z",
"appointmentTime" : "2022-05-25",
"deliveryTime" : "2022-07-29T18:35:00Z"
}
What changes do i need to make in the transformation such that the timestamp, appointmentTime and deliveryTime are also nested under transformedPayload i.e it looks like this (correct format).
{
"transformedPayload" : {
"trackingInfo" : "1ZEA83550362028861",
"events" : {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan"
},
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"timeStamp" : "2021-03-24T18:35:00Z",
"appointmentTime" : "2022-05-25",
"deliveryTime" : "2022-07-29T18:35:00Z"
}
}
This is my first time doing a jolt transformation so i am confused on how to resolve this. Any help is appreciated.

You are already so close to solution,I can offer the following spec similar to yours to the desired output :
[
{
"operation": "modify-overwrite-beta",
"spec": {
"tsY": "=substring(#(1,localActivityDate),0,4)",
"tsM": "=substring(#(1,localActivityDate),4,6)",
"tsD": "=substring(#(1,localActivityDate),6,8)",
"tsH": "=substring(#(1,localActivityTime),0,2)",
"tsMi": "=substring(#(1,localActivityTime),2,4)",
"tsS": "=substring(#(1,localActivityTime),4,6)",
"timeStamp": "=concat(#(1,tsY),'-',#(1,tsM),'-',#(1,tsD),'T',#(1,tsH),':',#(1,tsMi),':',#(1,tsS),'Z')",
"aTY": "=substring(#(1,scheduledDeliveryDate),0,4)",
"aTM": "=substring(#(1,scheduledDeliveryDate),4,6)",
"aTD": "=substring(#(1,scheduledDeliveryDate),6,8)",
"appointmentTime": "=concat(#(1,aTY),'-',#(1,aTM),'-',#(1,aTD))",
"dTY": "=substring(#(1,actualDeliveryDate),0,4)",
"dTM": "=substring(#(1,actualDeliveryDate),4,6)",
"dTD": "=substring(#(1,actualDeliveryDate),6,8)",
"dTH": "=substring(#(1,actualdeliveryTime),0,2)",
"dTMi": "=substring(#(1,actualdeliveryTime),2,4)",
"dTS": "=substring(#(1,actualdeliveryTime),4,6)",
"deliveryTime": "=concat(#(1,dTY),'-',#(1,dTM),'-',#(1,dTD),'T',#(1,dTH),':',#(1,dTMi),':',#(1,dTS),'Z')"
}
},
{
"operation": "shift",
"spec": {
"*Number": "&(0,1)Info",
"activityStatus": {
"*": "events.&"
},
"activityLocation": {
"*": "address.&"
},
"timeStamp": "&",
"appointmentTime": "&",
"deliveryTime": "&"
}
}
]

Related

How to rename field names in a nested array created from input JSON using JOLT transformation without changing anything else

I started with an input JSON as such.
{
"trackingNumber": "1ZEA83550362028861",
"localActivityDate": "20210324",
"localActivityTime": "183500",
"scheduledDeliveryDate": "20210324",
"actualDeliveryDate": "20210324",
"actualdeliveryTime": "183500",
"gmtActivityDate": "20210324",
"gmtActivityTime": "223500",
"activityStatus": {
"type": "G",
"code": "OR",
"description": "Origin Scan"
},
"activityLocation": {
"city": "RANDALLSTOWN,",
"stateProvince": "MD",
"postalCode": "21133",
"country": "US"
}
}
This is the JOLT transformation spec that i have written as of now.
[
{
"operation": "modify-overwrite-beta",
"spec": {
"tsY": "=substring(#(1,localActivityDate),0,4)",
"tsM": "=substring(#(1,localActivityDate),4,6)",
"tsD": "=substring(#(1,localActivityDate),6,8)",
"tsH": "=substring(#(1,localActivityTime),0,2)",
"tsMi": "=substring(#(1,localActivityTime),2,4)",
"tsS": "=substring(#(1,localActivityTime),4,6)",
"timeStamp": "=concat(#(1,tsY),'-',#(1,tsM),'-',#(1,tsD),'T',#(1,tsH),':',#(1,tsMi),':',#(1,tsS),'Z')",
"aTY": "=substring(#(1,scheduledDeliveryDate),0,4)",
"aTM": "=substring(#(1,scheduledDeliveryDate),4,6)",
"aTD": "=substring(#(1,scheduledDeliveryDate),6,8)",
"appointmentTime": "=concat(#(1,aTY),'-',#(1,aTM),'-',#(1,aTD))",
"dTY": "=substring(#(1,actualDeliveryDate),0,4)",
"dTM": "=substring(#(1,actualDeliveryDate),4,6)",
"dTD": "=substring(#(1,actualDeliveryDate),6,8)",
"dTH": "=substring(#(1,actualdeliveryTime),0,2)",
"dTMi": "=substring(#(1,actualdeliveryTime),2,4)",
"dTS": "=substring(#(1,actualdeliveryTime),4,6)",
"deliveryTime": "=concat(#(1,dTY),'-',#(1,dTM),'-',#(1,dTD),'T',#(1,dTH),':',#(1,dTMi),':',#(1,dTS),'Z')"
}
},
{
"operation": "shift",
"spec": {
"*Number": "transformedPayload.&(0,1)Info",
"activityStatus": {
"*": "transformedPayload.events.&"
},
"activityLocation": {
"*": "transformedPayload.address.&"
},
"timeStamp": "transformedPayload.events[0].&",
"appointmentTime": "transformedPayload.events[1].&",
"deliveryTime": "transformedPayload.events[2].&",
"activityStatus": {
"type": "transformedPayload.events[0].type",
"code": "transformedPayload.events[0].statusCode",
"description": "transformedPayload.events[0].statusDescription"
},
"activityLocation": {
"city": "transformedPayload.address.city",
"stateProvince": "transformedPayload.address.state",
"postalCode": "transformedPayload.address.postalCode",
"country": "transformedPayload.address.country"
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"metaData": {
"domain": "LTL",
"eventType": "statusUpdate",
"version": "v1"
},
"transformedPayload": {
"events": {
"[1]": {
"statusCode": "AB",
"statusDescription": "Delivery Scheduled"
},
"[2]": {
"statusCode": "D1",
"statusDescription": "Delivered"
}
}
}
}
}
]
The resultant JSON created by this transformation looks like this.
{
"transformedPayload" : {
"events" : [ {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan",
"timeStamp" : "2021-03-24T18:35:00Z"
}, {
"appointmentTime" : "2021-03-24",
"statusCode" : "AB",
"statusDescription" : "Delivery Scheduled"
}, {
"deliveryTime" : "2021-03-24T18:35:00Z",
"statusCode" : "D1",
"statusDescription" : "Delivered"
} ],
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"trackingInfo" : "1ZEA83550362028861"
},
"metaData" : {
"domain" : "LTL",
"eventType" : "statusUpdate",
"version" : "v1"
}
}
I just need a small tweak in this where the appointmentTime and the deliveryTime fields in the index [1] and [2] of the events array also need to be named as "timestamp" (as seen in the [0]th index). So that finally the correct output JSON looks something like this.
{
"transformedPayload" : {
"events" : [ {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan",
"timeStamp" : "2021-03-24T18:35:00Z"
}, {
"timestamp" : "2021-03-24",
"statusCode" : "AB",
"statusDescription" : "Delivery Scheduled"
}, {
"timestamp" : "2021-03-24T18:35:00Z",
"statusCode" : "D1",
"statusDescription" : "Delivered"
} ],
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"trackingInfo" : "1ZEA83550362028861"
},
"metaData" : {
"domain" : "LTL",
"eventType" : "statusUpdate",
"version" : "v1"
}
}
I have tried renaming the field in the shift operation itself but that did not work. I am completely new to JOLT transformation so it seems a bit tricky doing this small change. So any help is appreciated. Thanks
Just convert the related lines within the shift transformation spec from
"appointmentTime": "transformedPayload.events[1].&",
"deliveryTime": "transformedPayload.events[2].&",
to
"appointmentTime": "transformedPayload.events[1].timestamp",
"deliveryTime": "transformedPayload.events[2].timestamp",
instead of the replicating operator & used in the previous one.

json - jolt : How to extract common value to key

I am trying to convert a JSON to different format using JOLT (using NiFi JoltTransformJson processor).
Input Json
[
{
"date": "202001010000",
"name": "test1",
"val": "1",
"status": "0"
},
{
"date": "202001010000",
"name": "test2",
"val": "2",
"status": "0"
},
{
"date": "202001010001",
"name": "test1",
"val": "3",
"status": "0"
},
{
"date": "202001010001",
"name": "test2",
"val": "4",
"status": "0"
}
]
and I want to Output like
{
"202001010000" : [ {
"name" : "test1",
"val" : "1",
"status" : "0"
}, {
"name" : "test2",
"val" : "2",
"status" : "0"
}
],
"202001010001" : [ {
"name" : "test1",
"val" : "3",
"status" : "0"
}, {
"name" : "test2",
"val" : "4",
"status" : "0"
}
]
}
I'm trying to convert JSON format using Jolt Transform but it can't.
Make the date node value as key for each object in the array.
Remove the date node from the object.
Spec :
[
{
"operation": "shift",
"spec": {
"*": {
"date": {
"#1": "#(2,date)"
}
}
}
},
{
"operation": "remove",
"spec": {
"*": {
"*": {
"date": ""
}
}
}
}
]

Jolt Transformation order of fields

I've following input json
{
"NAME1": "Mueller",
"NAME2": "Max",
"STREET1": "Adlerstr.",
"STREET2": "1",
"COUNTRY": "Munich"
}
I renamed the name of the fields and concatinate the street* fields
[
{
"operation": "shift",
"spec": {
"NAME1": "name",
"NAME2": "surName",
"STREET1": "tmp_street1",
"STREET2": "tmp_street2",
"COUNTRY": "country"
}
},
{
"operation": "modify-default-beta",
"spec": {
"street": "=concat(#(1,tmp_street1),' ',#(1,tmp_street2))"
}
},
{
"operation": "remove",
"spec": {
"tmp_*": ""
}
}
]
I getting the following output
{
"name" : "Mueller",
"surName" : "Max",
"country" : "Munich", // Should be after the street field
"street" : "Adlerstr. 1"
}
Is it possible to move the street field on the 3rd position in the json output ?
Yes, street field can be moved to the 3rd position of the json output, with only two operations.
Check this spec,
[
{
"operation": "modify-default-beta",
"spec": {
"street": "=concat(#(1,STREET1),' ',#(1,STREET2))"
}
},
{
"operation": "shift",
"spec": {
"NAME1": "name",
"NAME2": "surName",
"street": "street",
"COUNTRY": "country"
}
}
]

Jolt Transform - Unable to transform json array into elements are objects with the specified key/value pairs

I'm trying to transform JSON array into elements are objects with the specified key/value pairs
{
"Resource": "sorPersonRole",
"Roleid": "1",
"Timestamp": "2010-06-30 00:00:00.0",
"Release": "Public",
"DOB": "2064-09-05",
"Active": "Y",
"enterprise_id": "a12s33",
"Inactive_enterprise_id": "",
"emp_ID": "123456",
"Inactive_emp_id": "000821972",
"Username": "",
"A_ID": "fsgf1234jhgfs3",
"P_ID": "w123456",
"Is Email Valid": "Y",
"Flag": "N",
"Registered": "spring",
"Description": "mainland corp",
"End Date": null
}
Expected output:
{
"meta" : {
"Resource" : "sorPersonRole",
"Roleid" : "1",
"Timestamp" : "2010-06-30 00:00:00.0",
"Release" : "Public",
"Sorid" : "w123456"
},
"sorAttributes" : {
"DOB" : "2064-09-05",
"Active" : "Y",
"End Date" : null,
"identifiers":
[
{
"type" : "enterprise"
"enterprise_id" : "a12s33",
"Username" : ""
},
{
"type" : "former-enterprise"
"Inactive_enterprise_id" : ""
},
{
"type" : "UID"
"emp_ID" : "123456",
"Inactive_emp_id" : "000821972"
},
{
"type" : "National"
"A_ID" : "fsgf1234jhgfs3"
}
],
"mainLand:com:adhoc" : {
"Is Email Valid" : "Y",
"Flag" : "N",
"Registered" : "spring",
"Description" : "mainland corp"
}
}
}
current Jolt spec: which I am not getting desired output
[
{
"operation": "shift",
"spec": {
"Resource": "meta.&",
"P_ID": "meta.Sorid",
"Roleid": "meta.&",
"Timestamp": "meta.&",
"Release": "meta.&",
"enterprise_id": "sorAttributes.Identifiers.type.enterprise.&",
"Inactive_enterprise_id": "sorAttributes.Identifiers.type.former-enterprise.&",
"emp_ID": "sorAttributes.Identifiers.type.UID.&",
"Inactive_emp_id": "sorAttributes.Identifiers.type.UID.&",
"Username": "sorAttributes.Identifiers.type.enterprise.&",
"A_ID": "sorAttributes.Identifiers.type.National.&",
"Is Email Valid": "sorAttributes.mainLand:com:adhoc.&",
"Flag": "sorAttributes.mainLand:com:adhoc.&",
"Registered": "sorAttributes.mainLand:com:adhoc.&",
"Description": "sorAttributes.mainLand:com:adhoc.&",
"*": "sorAttributes.&"
}
}
]
I have tried the different JsonSpecs provided on different websites, could able to match expected output. Also tried using two-shift operations but no luck, Any help or suggestion will be appreciated.
Thanks.
This can help,
For the nodes to be shifted into the identifier array, shift one level more.
[
{
"operation": "shift",
"spec": {
"Resource": "meta.&",
"Roleid": "meta.&",
"Timestamp": "meta.&",
"Release": "meta.&",
"P_ID": "meta.Sorid",
"DOB": "sorAttributes.&",
"Active": "sorAttributes.&",
"End Date": "sorAttributes.&",
"Is Email Valid": "sorAttributes.mainLand:com:adhoc.&",
"Flag": "sorAttributes.mainLand:com:adhoc.&",
"Registered": "sorAttributes.mainLand:com:adhoc.&",
"Description": "sorAttributes.mainLand:com:adhoc.&",
"enterprise_id": {
"#enterprise": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&",
"#(1,Username)": "sorAttributes.identifiers[#2].Username"
},
"Inactive_enterprise_id": {
"#former-enterprise": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&"
},
"Inactive_emp_id": {
"#UID": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&",
"#(1,emp_ID)": "sorAttributes.identifiers[#2].emp_ID"
},
"A_ID": {
"#National": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&"
}
}
}, {
"operation": "modify-overwrite-beta",
"spec": {
"*": "=recursivelySquashNulls"
}
}
]

JOLT - Flattening a simple JSON array

I want this:
{
"DATA": [
{
"SPEED" : "hi",
"ANGLE" : "180",
"GRIT" : "2000"
},
{
"SPEED" : "mid",
"ANGLE" : "180",
"GRIT" : "2000"
},
{
"SPEED" : "lo",
"ANGLE" : "180",
"GRIT" : "2000"
}
]
}
To become this:
{
"SPEED" : "hi",
"ANGLE" : "180",
"GRIT" : "2000"
},
{
"SPEED" : "mid",
"ANGLE" : "180",
"GRIT" : "2000"
},
{
"SPEED" : "lo",
"ANGLE" : "180",
"GRIT" : "2000"
}
But with this:
[
{
"operation": "shift",
"spec": {
"DATA": {
"*": {
"*": "&"
}
}
}
}
]
I get this:
{
"SPEED" : [ "hi", "mid", "lo" ],
"ANGLE" : [ "180", "180", "180" ],
"GRIT" : [ "2000", "2000", "2000" ]
}
I'm very new to JOLT and since I can't find any documentation, I come to you for help.
Stackoverflow won't let me post this question since it contains mostly code, but that lies in the nature of this question, namely describing what input I have, what I want it to be and what I've tried.
EDIT:
#Pratik Jaiswal
Thanks for your answer, but it looks like I can do the same with
[
{
"operation": "shift",
"spec": {
"DATA": {
"*": {
"#": ""
}
}
}
}
]
This doesn't remove the outer brackets, though. Is that even possible with JOLT? Or do I just have to get rid of them in another way?
[
{
"operation": "shift",
"spec": {
"DATA": {
"*": {
"SPEED": "[&1].SPEED",
"ANGLE": "[&1].ANGLE",
"GRIT": "[&1].GRIT"
}
}
}
}
]
Test the spec here http://jolt-demo.appspot.com/#inception