Json transformation using jolt string concatenation in an array - json

I need to do a string concatenation on the element in 7 and 8th position in an array as aircraft id and create a json message using jolt.Below is the sample spec,input,actual output and expected output.Can you please help
iam using jolt framework version 0.0.20
Input :
[
[
"20083366",
"12018098",
"Air Transport Europe",
"2017",
"1",
"60",
"OM",
"ATM",
"Test",
"",
"",
"7600000",
"",
"0"
],
[
"20083366",
"12018098",
"Air Transport Europe",
"2017",
"2",
"75",
"OM",
"ATM",
"",
"",
"",
"7600000",
"",
"0"
]
]
Jolt Spec :
[
{
"operation": "shift",
"spec": {
"*": {
"7": "[&1].countryCodeAircraftId",
"8": "[&1].aircraftId",
"9": "[&1].aircraftTypeCode",
"1": "[&1].insuredId",
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"*": {
"14": "=concat(#(1,countryCodeAircraftId),' ',#(1,aircraftId))"
}
}
}
]
Actual output:
[ {
"countryCodeAircraftId" : "ATM",
"aircraftIds" : "Test",
"aircraftTypeCode" : "",
"insuredId" : "12018098",
"aircraftId" : "ATM Test"
}, {
"countryCodeAircraftId" : "ATM",
"aircraftIds" : "",
"aircraftTypeCode" : "",
"insuredId" : "12018098",
"aircraftId" : "ATM "
} ]
Expected output:
[ {
"aircraftId" : "ATM Test",
"aircraftTypeCode" : "",
"insuredId" : "12018098"
}, {
"aircraftId" : "",
"aircraftTypeCode" : "",
"insuredId" : "12018098",
} ]


Jolt version 0.0.20 does not have the "modify-default-beta" operation. It was added in 0.0.22.

Related

How to rename field names in a nested array created from input JSON using JOLT transformation without changing anything else

I started with an input JSON as such.
{
"trackingNumber": "1ZEA83550362028861",
"localActivityDate": "20210324",
"localActivityTime": "183500",
"scheduledDeliveryDate": "20210324",
"actualDeliveryDate": "20210324",
"actualdeliveryTime": "183500",
"gmtActivityDate": "20210324",
"gmtActivityTime": "223500",
"activityStatus": {
"type": "G",
"code": "OR",
"description": "Origin Scan"
},
"activityLocation": {
"city": "RANDALLSTOWN,",
"stateProvince": "MD",
"postalCode": "21133",
"country": "US"
}
}
This is the JOLT transformation spec that i have written as of now.
[
{
"operation": "modify-overwrite-beta",
"spec": {
"tsY": "=substring(#(1,localActivityDate),0,4)",
"tsM": "=substring(#(1,localActivityDate),4,6)",
"tsD": "=substring(#(1,localActivityDate),6,8)",
"tsH": "=substring(#(1,localActivityTime),0,2)",
"tsMi": "=substring(#(1,localActivityTime),2,4)",
"tsS": "=substring(#(1,localActivityTime),4,6)",
"timeStamp": "=concat(#(1,tsY),'-',#(1,tsM),'-',#(1,tsD),'T',#(1,tsH),':',#(1,tsMi),':',#(1,tsS),'Z')",
"aTY": "=substring(#(1,scheduledDeliveryDate),0,4)",
"aTM": "=substring(#(1,scheduledDeliveryDate),4,6)",
"aTD": "=substring(#(1,scheduledDeliveryDate),6,8)",
"appointmentTime": "=concat(#(1,aTY),'-',#(1,aTM),'-',#(1,aTD))",
"dTY": "=substring(#(1,actualDeliveryDate),0,4)",
"dTM": "=substring(#(1,actualDeliveryDate),4,6)",
"dTD": "=substring(#(1,actualDeliveryDate),6,8)",
"dTH": "=substring(#(1,actualdeliveryTime),0,2)",
"dTMi": "=substring(#(1,actualdeliveryTime),2,4)",
"dTS": "=substring(#(1,actualdeliveryTime),4,6)",
"deliveryTime": "=concat(#(1,dTY),'-',#(1,dTM),'-',#(1,dTD),'T',#(1,dTH),':',#(1,dTMi),':',#(1,dTS),'Z')"
}
},
{
"operation": "shift",
"spec": {
"*Number": "transformedPayload.&(0,1)Info",
"activityStatus": {
"*": "transformedPayload.events.&"
},
"activityLocation": {
"*": "transformedPayload.address.&"
},
"timeStamp": "transformedPayload.events[0].&",
"appointmentTime": "transformedPayload.events[1].&",
"deliveryTime": "transformedPayload.events[2].&",
"activityStatus": {
"type": "transformedPayload.events[0].type",
"code": "transformedPayload.events[0].statusCode",
"description": "transformedPayload.events[0].statusDescription"
},
"activityLocation": {
"city": "transformedPayload.address.city",
"stateProvince": "transformedPayload.address.state",
"postalCode": "transformedPayload.address.postalCode",
"country": "transformedPayload.address.country"
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"metaData": {
"domain": "LTL",
"eventType": "statusUpdate",
"version": "v1"
},
"transformedPayload": {
"events": {
"[1]": {
"statusCode": "AB",
"statusDescription": "Delivery Scheduled"
},
"[2]": {
"statusCode": "D1",
"statusDescription": "Delivered"
}
}
}
}
}
]
The resultant JSON created by this transformation looks like this.
{
"transformedPayload" : {
"events" : [ {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan",
"timeStamp" : "2021-03-24T18:35:00Z"
}, {
"appointmentTime" : "2021-03-24",
"statusCode" : "AB",
"statusDescription" : "Delivery Scheduled"
}, {
"deliveryTime" : "2021-03-24T18:35:00Z",
"statusCode" : "D1",
"statusDescription" : "Delivered"
} ],
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"trackingInfo" : "1ZEA83550362028861"
},
"metaData" : {
"domain" : "LTL",
"eventType" : "statusUpdate",
"version" : "v1"
}
}
I just need a small tweak in this where the appointmentTime and the deliveryTime fields in the index [1] and [2] of the events array also need to be named as "timestamp" (as seen in the [0]th index). So that finally the correct output JSON looks something like this.
{
"transformedPayload" : {
"events" : [ {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan",
"timeStamp" : "2021-03-24T18:35:00Z"
}, {
"timestamp" : "2021-03-24",
"statusCode" : "AB",
"statusDescription" : "Delivery Scheduled"
}, {
"timestamp" : "2021-03-24T18:35:00Z",
"statusCode" : "D1",
"statusDescription" : "Delivered"
} ],
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"trackingInfo" : "1ZEA83550362028861"
},
"metaData" : {
"domain" : "LTL",
"eventType" : "statusUpdate",
"version" : "v1"
}
}
I have tried renaming the field in the shift operation itself but that did not work. I am completely new to JOLT transformation so it seems a bit tricky doing this small change. So any help is appreciated. Thanks

Just convert the related lines within the shift transformation spec from
"appointmentTime": "transformedPayload.events[1].&",
"deliveryTime": "transformedPayload.events[2].&",
to
"appointmentTime": "transformedPayload.events[1].timestamp",
"deliveryTime": "transformedPayload.events[2].timestamp",
instead of the replicating operator & used in the previous one.

How to correct this jolt transformation

I have an input JSON like this.
{
"trackingNumber": "1ZEA83550362028861",
"localActivityDate": "20210324",
"localActivityTime": "183500",
"scheduledDeliveryDate": "20220525",
"actualDeliveryDate": "20220729",
"actualdeliveryTime": "183500",
"gmtActivityDate": "20210324",
"gmtActivityTime": "223500",
"activityStatus": {
"type": "G",
"code": "OR",
"description": "Origin Scan"
},
"activityLocation": {
"city": "RANDALLSTOWN,",
"stateProvince": "MD",
"postalCode": "21133",
"country": "US"
}
}
I have written a jolt transformation for this JSON
[
{
"operation": "shift",
"spec": {
"trackingNumber": "transformedPayload.trackingInfo",
"localActivityDate": "tmp_Date",
"localActivityTime": "tmp_Time",
"scheduledDeliveryDate": "tmp_App",
"actualDeliveryDate": "tmp_Del_Date",
"actualdeliveryTime": "tmp_Del_Time",
"activityStatus": {
"type": "transformedPayload.events.type",
"code": "transformedPayload.events.statusCode",
"description": "transformedPayload.events.statusDescription"
},
"activityLocation": {
"city": "transformedPayload.address.city",
"stateProvince": "transformedPayload.address.state",
"postalCode": "transformedPayload.address.postalCode",
"country": "transformedPayload.address.country"
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"tmp_Year": "=substring(#(1,tmp_Date),0,4)",
"tmp_Month": "=substring(#(1,tmp_Date),4,6)",
"tmp_Day": "=substring(#(1,tmp_Date),6,8)",
"tmp_Hours": "=substring(#(1,tmp_Time),0,2)",
"tmp_Minutes": "=substring(#(1,tmp_Time),2,4)",
"tmp_Seconds": "=substring(#(1,tmp_Time),4,6)",
"timeStamp": "=concat(#(1,tmp_Year),'-',#(1,tmp_Month),'-',#(1,tmp_Day),'T',#(1,tmp_Hours),':',#(1,tmp_Minutes),':',#(1,tmp_Seconds),'Z')",
"tmp_App_Year": "=substring(#(1,tmp_App),0,4)",
"tmp_App_Month": "=substring(#(1,tmp_App),4,6)",
"tmp_App_Day": "=substring(#(1,tmp_App),6,8)",
"appointmentTime": "=concat(#(1,tmp_App_Year),'-',#(1,tmp_App_Month),'-',#(1,tmp_App_Day))",
"tmp__Del_Year": "=substring(#(1,tmp_Del_Date),0,4)",
"tmp_Del_Month": "=substring(#(1,tmp_Del_Date),4,6)",
"tmp_Del_Day": "=substring(#(1,tmp_Del_Date),6,8)",
"tmp_Del_Hours": "=substring(#(1,tmp_Del_Time),0,2)",
"tmp_Del_Minutes": "=substring(#(1,tmp_Del_Time),2,4)",
"tmp_Del_Seconds": "=substring(#(1,tmp_Del_Time),4,6)",
"deliveryTime": "=concat(#(1,tmp__Del_Year),'-',#(1,tmp_Del_Month),'-',#(1,tmp_Del_Day),'T',#(1,tmp_Del_Hours),':',#(1,tmp_Del_Minutes),':',#(1,tmp_Del_Seconds),'Z')"
}
},
{
"operation": "remove",
"spec": {
"tmp_*": ""
}
}
]
This transforms the data into this format.
{
"transformedPayload" : {
"trackingInfo" : "1ZEA83550362028861",
"events" : {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan"
},
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
}
},
"timeStamp" : "2021-03-24T18:35:00Z",
"appointmentTime" : "2022-05-25",
"deliveryTime" : "2022-07-29T18:35:00Z"
}
What changes do i need to make in the transformation such that the timestamp, appointmentTime and deliveryTime are also nested under transformedPayload i.e it looks like this (correct format).
{
"transformedPayload" : {
"trackingInfo" : "1ZEA83550362028861",
"events" : {
"type" : "G",
"statusCode" : "OR",
"statusDescription" : "Origin Scan"
},
"address" : {
"city" : "RANDALLSTOWN,",
"state" : "MD",
"postalCode" : "21133",
"country" : "US"
},
"timeStamp" : "2021-03-24T18:35:00Z",
"appointmentTime" : "2022-05-25",
"deliveryTime" : "2022-07-29T18:35:00Z"
}
}
This is my first time doing a jolt transformation so i am confused on how to resolve this. Any help is appreciated.

You are already so close to solution,I can offer the following spec similar to yours to the desired output :
[
{
"operation": "modify-overwrite-beta",
"spec": {
"tsY": "=substring(#(1,localActivityDate),0,4)",
"tsM": "=substring(#(1,localActivityDate),4,6)",
"tsD": "=substring(#(1,localActivityDate),6,8)",
"tsH": "=substring(#(1,localActivityTime),0,2)",
"tsMi": "=substring(#(1,localActivityTime),2,4)",
"tsS": "=substring(#(1,localActivityTime),4,6)",
"timeStamp": "=concat(#(1,tsY),'-',#(1,tsM),'-',#(1,tsD),'T',#(1,tsH),':',#(1,tsMi),':',#(1,tsS),'Z')",
"aTY": "=substring(#(1,scheduledDeliveryDate),0,4)",
"aTM": "=substring(#(1,scheduledDeliveryDate),4,6)",
"aTD": "=substring(#(1,scheduledDeliveryDate),6,8)",
"appointmentTime": "=concat(#(1,aTY),'-',#(1,aTM),'-',#(1,aTD))",
"dTY": "=substring(#(1,actualDeliveryDate),0,4)",
"dTM": "=substring(#(1,actualDeliveryDate),4,6)",
"dTD": "=substring(#(1,actualDeliveryDate),6,8)",
"dTH": "=substring(#(1,actualdeliveryTime),0,2)",
"dTMi": "=substring(#(1,actualdeliveryTime),2,4)",
"dTS": "=substring(#(1,actualdeliveryTime),4,6)",
"deliveryTime": "=concat(#(1,dTY),'-',#(1,dTM),'-',#(1,dTD),'T',#(1,dTH),':',#(1,dTMi),':',#(1,dTS),'Z')"
}
},
{
"operation": "shift",
"spec": {
"*Number": "&(0,1)Info",
"activityStatus": {
"*": "events.&"
},
"activityLocation": {
"*": "address.&"
},
"timeStamp": "&",
"appointmentTime": "&",
"deliveryTime": "&"
}
}
]

json - jolt : How to extract common value to key

I am trying to convert a JSON to different format using JOLT (using NiFi JoltTransformJson processor).
Input Json
[
{
"date": "202001010000",
"name": "test1",
"val": "1",
"status": "0"
},
{
"date": "202001010000",
"name": "test2",
"val": "2",
"status": "0"
},
{
"date": "202001010001",
"name": "test1",
"val": "3",
"status": "0"
},
{
"date": "202001010001",
"name": "test2",
"val": "4",
"status": "0"
}
]
and I want to Output like
{
"202001010000" : [ {
"name" : "test1",
"val" : "1",
"status" : "0"
}, {
"name" : "test2",
"val" : "2",
"status" : "0"
}
],
"202001010001" : [ {
"name" : "test1",
"val" : "3",
"status" : "0"
}, {
"name" : "test2",
"val" : "4",
"status" : "0"
}
]
}
I'm trying to convert JSON format using Jolt Transform but it can't.

Make the date node value as key for each object in the array.
Remove the date node from the object.
Spec :
[
{
"operation": "shift",
"spec": {
"*": {
"date": {
"#1": "#(2,date)"
}
}
}
},
{
"operation": "remove",
"spec": {
"*": {
"*": {
"date": ""
}
}
}
}
]

Jolt Transform - Unable to transform json array into elements are objects with the specified key/value pairs

I'm trying to transform JSON array into elements are objects with the specified key/value pairs
{
"Resource": "sorPersonRole",
"Roleid": "1",
"Timestamp": "2010-06-30 00:00:00.0",
"Release": "Public",
"DOB": "2064-09-05",
"Active": "Y",
"enterprise_id": "a12s33",
"Inactive_enterprise_id": "",
"emp_ID": "123456",
"Inactive_emp_id": "000821972",
"Username": "",
"A_ID": "fsgf1234jhgfs3",
"P_ID": "w123456",
"Is Email Valid": "Y",
"Flag": "N",
"Registered": "spring",
"Description": "mainland corp",
"End Date": null
}
Expected output:
{
"meta" : {
"Resource" : "sorPersonRole",
"Roleid" : "1",
"Timestamp" : "2010-06-30 00:00:00.0",
"Release" : "Public",
"Sorid" : "w123456"
},
"sorAttributes" : {
"DOB" : "2064-09-05",
"Active" : "Y",
"End Date" : null,
"identifiers":
[
{
"type" : "enterprise"
"enterprise_id" : "a12s33",
"Username" : ""
},
{
"type" : "former-enterprise"
"Inactive_enterprise_id" : ""
},
{
"type" : "UID"
"emp_ID" : "123456",
"Inactive_emp_id" : "000821972"
},
{
"type" : "National"
"A_ID" : "fsgf1234jhgfs3"
}
],
"mainLand:com:adhoc" : {
"Is Email Valid" : "Y",
"Flag" : "N",
"Registered" : "spring",
"Description" : "mainland corp"
}
}
}
current Jolt spec: which I am not getting desired output
[
{
"operation": "shift",
"spec": {
"Resource": "meta.&",
"P_ID": "meta.Sorid",
"Roleid": "meta.&",
"Timestamp": "meta.&",
"Release": "meta.&",
"enterprise_id": "sorAttributes.Identifiers.type.enterprise.&",
"Inactive_enterprise_id": "sorAttributes.Identifiers.type.former-enterprise.&",
"emp_ID": "sorAttributes.Identifiers.type.UID.&",
"Inactive_emp_id": "sorAttributes.Identifiers.type.UID.&",
"Username": "sorAttributes.Identifiers.type.enterprise.&",
"A_ID": "sorAttributes.Identifiers.type.National.&",
"Is Email Valid": "sorAttributes.mainLand:com:adhoc.&",
"Flag": "sorAttributes.mainLand:com:adhoc.&",
"Registered": "sorAttributes.mainLand:com:adhoc.&",
"Description": "sorAttributes.mainLand:com:adhoc.&",
"*": "sorAttributes.&"
}
}
]
I have tried the different JsonSpecs provided on different websites, could able to match expected output. Also tried using two-shift operations but no luck, Any help or suggestion will be appreciated.
Thanks.

This can help,
For the nodes to be shifted into the identifier array, shift one level more.
[
{
"operation": "shift",
"spec": {
"Resource": "meta.&",
"Roleid": "meta.&",
"Timestamp": "meta.&",
"Release": "meta.&",
"P_ID": "meta.Sorid",
"DOB": "sorAttributes.&",
"Active": "sorAttributes.&",
"End Date": "sorAttributes.&",
"Is Email Valid": "sorAttributes.mainLand:com:adhoc.&",
"Flag": "sorAttributes.mainLand:com:adhoc.&",
"Registered": "sorAttributes.mainLand:com:adhoc.&",
"Description": "sorAttributes.mainLand:com:adhoc.&",
"enterprise_id": {
"#enterprise": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&",
"#(1,Username)": "sorAttributes.identifiers[#2].Username"
},
"Inactive_enterprise_id": {
"#former-enterprise": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&"
},
"Inactive_emp_id": {
"#UID": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&",
"#(1,emp_ID)": "sorAttributes.identifiers[#2].emp_ID"
},
"A_ID": {
"#National": "sorAttributes.identifiers[#2].type",
"#": "sorAttributes.identifiers[#2].&"
}
}
}, {
"operation": "modify-overwrite-beta",
"spec": {
"*": "=recursivelySquashNulls"
}
}
]

Jolt transforming to array

I am getting null in my output json. Please find my spec and details below. The input json can have n numbers of COMPINFO . pls suggest
my input.json is
{
"valid": "true",
"message": "",
"data": {
"COMPINFO": [
{
"ORGID": "",
"SITEID": "BWDEMO",
"COMPID": "C2014",
"COMP_DESC": "Cherokee High School",
"ASSETTYPE": "MANUFACTURING",
"BUILDING": "Main",
"FLR_LEVEL": "Ground",
"ROOM_SPCE": "100"
},
{
"ORGID": "",
"SITEID": "BWDEMO",
"COMPID": "9001B",
"COMP_DESC": "Sludge Pump",
"ASSETTYPE": "FACILITY",
"BUILDING": "Main",
"FLR_LEVEL": "Production",
"ROOM_SPCE": "100"
}
]
}
}
my Spec.json is
[
{
"operation": "shift",
"spec": {
"data": {
"COMPINFO": {
"*": {
"COMPID": "[&1].COMPID",
"ORGID": "[&1].ORGID",
"COMP_DESC": "[&1].DESCRIPTION",
"BUILDING": "[&1].LOCATIONS.[&1].Building",
"FLR_LEVEL": "[&1].LOCATIONS.[&1].Floor_Level",
"ROOM_SPCE": "[&1].LOCATIONS.[&1].ROOM_SPCE",
"SITEID": "[&1].SITEID",
"ASSETTYPE": "[&1].ASSETTYPE"
}
}
}
}
}
]
expected output should be :
[
{
"COMPID" : "C2014",
"ORGID" : "",
"DESCRIPTION" : "Cherokee High School",
"LOCATIONS" : [ {
"Building" : "Main",
"Floor_Level" : "Ground",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "MANUFACTURING"
}, {
"COMPID" : "9001B",
"ORGID" : "",
"DESCRIPTION" : "Sludge Pump",
"LOCATIONS" : [{
"Building" : "Main",
"Floor_Level" : "Production",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "FACILITY"
}
]
but getting null (highlighted (Bold)) :
[
{
"COMPID" : "C2014",
"ORGID" : "",
"DESCRIPTION" : "Cherokee High School",
"LOCATIONS" : [ {
"Building" : "Main",
"Floor_Level" : "Ground",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "MANUFACTURING"
}, {
"COMPID" : "9001B",
"ORGID" : "",
"DESCRIPTION" : "Sludge Pump",
"LOCATIONS" : [ null, {
"Building" : "Main",
"Floor_Level" : "Production",
"ROOM_SPCE" : "100"
} ],
"SITEID" : "BWDEMO",
"ASSETTYPE" : "FACILITY"
}
]
can someone help quickly ??
Thanks in advance .ssssssssssssssssssssss

It looks like you there is only one "location" per input item, and that you just want them to alway be the first element in a location array.
If so that is easy. If you are wanting to "group" your data and have some of the locations array have multiple items, that is a harder transform.
Spec for the simple version
[
{
"operation": "shift",
"spec": {
"data": {
"COMPINFO": {
"*": {
"COMPID": "[&1].COMPID",
"ORGID": "[&1].ORGID",
"COMP_DESC": "[&1].DESCRIPTION",
"BUILDING": "[&1].LOCATIONS[0].Building",
"FLR_LEVEL": "[&1].LOCATIONS[0].Floor_Level",
"ROOM_SPCE": "[&1].LOCATIONS[0].ROOM_SPCE",
"SITEID": "[&1].SITEID",
"ASSETTYPE": "[&1].ASSETTYPE"
}
}
}
}
}
]
There are examples around for the harder version.