Json Jolt - Shifting data to multiple sub-arrays - json

I am trying to do a number of shifts to copy some data into multiple arrays and I am having trouble getting the data to correct array.
This is what I have
{
"input": {
"rating_date": "2018-08-06",
"Rates": {
"name": "Generic ratings Card",
"version": "v1",
"value": "2600.00",
"name1": [
{
"hits": "321",
"genre": "fiction",
"ratingName": "name1"
},
{
"hits": "654",
"genre": "fiction",
"ratingName": "name1"
}
],
"name2": [
{
"hits": "123",
"genre": "nonfiction",
"ratingName": "name2"
},
{
"hits": "456",
"genre": "fiction",
"ratingName": "name2"
}
]
}
},
"spec": {
"operation": "shift",
"spec": {
"rating_date": "rating_by_date[#0].date",
"Rates": {
"name*": {
"$": "rating_by_date[#2].ratecards[#1].type",
"*": {
"#": "rating_by_date[#3].ratecards[#2].rates[#0].&"
}
}
}
}
},
"expected": {
"rating_by_date": [
{
"date": "2018-08-06",
"ratecards": [
{
"type": "name1",
"rates": [
{
"hits": "321",
"genre": "fiction",
"ratingName": "name1"
},
{
"hits": "654",
"genre": "fiction",
"ratingName": "name1"
}
]
},
{
"type": "name2",
"rates": [
{
"hits": "123",
"genre": "nonfiction",
"ratingName": "name2"
},
{
"hits": "456",
"genre": "fiction",
"ratingName": "name2"
}
]
}
]
}
]
}
}
The idea is to take the objects in the rates array and transform them (in a later shift) into Key-Value pairs, which I already know how to do.
Now, the spec I have is not shifting the data into appropriate arrays and I end up with something like this:
{
"rating_by_date" : [ {
"date" : "2018-08-06",
"ratecards" : [ {
"type" : "name1",
"rates" : [ {
"0" : {
"hits" : "321",
"genre" : "fiction",
"ratingName" : "name1"
}
} ]
}, {
"rates" : [ {
"1" : {
"hits" : "654",
"genre" : "fiction",
"ratingName" : "name1"
}
} ]
} ]
}, {
"ratecards" : [ {
"type" : "name2",
"rates" : [ {
"0" : {
"hits" : "123",
"genre" : "nonfiction",
"ratingName" : "name2"
}
} ]
}, {
"rates" : [ {
"1" : {
"hits" : "456",
"genre" : "fiction",
"ratingName" : "name2"
}
} ]
} ]
} ]
}
Instead of creating new objects in the same array, it is creating new arrays altogether. I am clearly not understanding exactly how to reference array structures on the RHS, I'd appreciate a clarification.

I managed to get it to work, but I'm not sure why it would not work in one shift step. Here is what I had to do to make it work:
[
{
"operation": "shift",
"spec": {
"rating_date": "rating_by_date[0].date",
"Rates": {
"name*": {
"$": "rating_by_date[0].ratecards.&.type",
"#": "rating_by_date[0].ratecards.&.rates"
}
}
}
},
{
"operation": "shift",
"spec": {
"rating_by_date": {
"*": {
"ratecards": {
"*": {
"#": "rating_by_date[&3].ratecards2[]"
}
},
"*": "rating_by_date[&1].&"
}
}
}
}
]

Related

A JOLT transformation question about mapping reference

This is an input that needs to be transformed using Jolt Transformation to obtain the expected output.
I am attempting to create a jolt transformation for the below input:
{
"flights": [
{
"id": "123",
"route": "BJS-SIN"
},
{
"id": "456",
"route": "SIN-PEK"
},
{
"id": "789",
"route": "SIN-BJS"
}
],
"prices": [
{
"id": "abc",
"amount": 560
},
{
"id": "def",
"amount": 780
}
],
"solutions": [
{
"price-ref": "abc",
"flights-ref": [
"123",
"456"
]
},
{
"price-ref": "def",
"flights-ref": [
"123",
"789"
]
}
]
}
Desired output would like :
{
"solutions": [
{
"flights": [
{
"id": "123",
"route": "BJS-SIN"
},
{
"id": "456",
"route": "SIN-PEK"
}
],
"price": {
"id": "abc",
"amount": 560
}
},
{
"flights": [
{
"id": "123",
"route": "BJS-SIN"
},
{
"id": "789",
"route": "SIN-BJS"
}
],
"price": {
"id": "def",
"amount": 780
}
}
]
}
As the output, all the data should be constructed refer to the solutions node. How about the mapping expression in JOLT?
all the values from the source json are generated randomly and the keys names are fixed.
I had tried many times but could not find the right spec, so please help.
You can use this spec:
[
{
"operation": "shift",
"spec": {
"*": "&",
"flights|prices": {
"*": {
"*": "&2.#(1,id).&"
}
}
}
},
{
"operation": "shift",
"spec": {
"solutions": {
"*": {
"flights*": {
"*": {
"*": {
"#(6,flights.&)": "&5[&4].flights[&2]"
}
}
},
"price*": {
"*": {
"#(5,prices.&)": "&4[&3].price"
}
}
}
}
}
}
]

Selectively get certain arrays from inside an array in JSON using JOLT

I've been trying to get some fields out of a very long and complicated json format but not getting the output I want.
MY current spec obtains all events from the event array and lists them in the output. I'm unsure how to select specific events and only output those. Am not quite sure of the syntax
My JSON:
{
"rootid": "19718",
"clloadm": "2021-06-01T22:40:02",
"clload": "2021-06-01T21:21:39",
"date": "2021-05-25T21:52:30",
"events": [
{
"done": {
"id": "e0",
"value": "2021-05-29T08:08:19"
},
"id": "e0_event",
"started": {
"id": "e0",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e1",
"value": "2021-05-27T02:20:25"
},
"id": "e1_event",
"started": {
"id": "e1",
"value": "2021-05-27T02:20:25"
},
"status": "complete"
},
{
"done": {
"id": "e2",
"value": "2021-05-29T08:08:19"
},
"id": "e2_event",
"started": {
"id": "e2",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e3",
"value": "2021-05-29T08:08:19"
},
"id": "e3_event",
"started": {
"id": "e3",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
},
{
"done": {
"id": "e4",
"value": "2021-05-29T08:08:19"
},
"id": "e4_event",
"started": {
"id": "e4",
"value": "2021-05-29T08:08:19"
},
"status": "complete"
}
],
"ids": [
{
"id": "id",
"source": "source",
"value": "value"
},
{
"id": "new_id",
"source": "new_source",
"value": "value"
}
]
}
My Jolt Spec that gets all events for now:
[
{
"operation": "shift",
"spec": {
"rootid": "rootid",
"clloadm": "clloadm",
"clload": "clload",
"date": "date",
"events": {
"*": {
"*": {
"#value": "#id"
}
}
},
"ids": {
"*": {
"#value": "#id"
}
}
}
}
]
The output I get:
{
"rootid" : "19718",
"clloadm" : "2021-06-01T22:40:02",
"clload" : "2021-06-01T21:21:39",
"date" : "2021-05-25T21:52:30",
"e0" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e1" : [ "2021-05-27T02:20:25", "2021-05-27T02:20:25" ],
"e2" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e3" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e4" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"id" : "value",
"new_id" : "value"
}
The output I would like
{
"rootid" : "19718",
"clloadm" : "2021-06-01T22:40:02",
"clload" : "2021-06-01T21:21:39",
"date" : "2021-05-25T21:52:30",
"e0" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"e4" : [ "2021-05-29T08:08:19", "2021-05-29T08:08:19" ],
"id" : "value",
"new_id" : "value"
}
You can write the individual keys e0 and e4 as conditional cases for #id key while rewriting the rest of the key-value pairs through "*":"&" representation such as
[
{
"operation": "shift",
"spec": {
"*": "&",
"events": {
"*": {
"*": {
"#id": {
"e0": { "#(2,value)": "&" },
"e4": { "#(2,value)": "&" }
}
}
}
},
"ids": {
"*": {
"#value": "#id"
}
}
}
}
]

JOLT Transformation - Extract field and put it to the top level

I have an SAP Idoc with additional customer specific fields as JSON
{
"E1EDL20": [
{
"VBELN": "1234567890",
"VSTEL": "ZO01",
"LGNUM": "123",
"NTGEW": "100.660",
"ANZPK": "00002",
"/XYZ/XYZ1234_E1EDL20": {
"ACTION": "POSTED",
"CONSIGNMENT": "12345678",
"FREIGHT_ORDER_EXTERNAL": "S123456789",
"IN_YARD": "X",
"IN_YARD_DATE": "20201123",
"IN_YARD_TIME": "100923"
},
"E1ADRM1": [
{
"PARTNER_Q": "LF",
"PARTNER_ID": "0000100000",
"/XYZ/XYZ1234_E1ADRM1": {
"SUPPLIER": "X"
}
},
{
"PARTNER_Q": "OSP",
"PARTNER_ID": "ZO01",
"/XYZ/XYZ1234_E1ADRM1": {
"SUPPLIER": "X"
}
}
]
}
]
}
I want to extract the first field of an array and put it to the top level.
This is the expected output:
{
"deliveryInternal" : "1234567890",
"receivingPoint" : "ZO01",
"warehouseNumber" : "123",
"netWeight" : "100.66",
"numberOfPackages" : "00002",
"type": "LF",
"action" : "POSTED",
"consignment" : "12345678",
"freightOrderExternal" : "S123456789",
"inYard" : "X",
"inYardDate" : "00002"
}
Does anyone have an idea how the problem can be solved ?
You can use shift operation with index-based keys as below
[
{
"operation": "shift",
"spec": {
"E1EDL20": {
"0": {
"VBELN": "deliveryInternal",
"VSTEL": "receivingPoint",
"/XYZ/XYZ1234_E1EDL20": {
"ACTION": "action"
},
"E1ADRM1": {
"0": {
"PARTNER_Q": "type"
}
}
}
}
}
}
]

Jolt transformation combining multiple values in an array of objects when multiple objects are in input

I have a JSON input :
{
"Name": "travel.0.0",
"Documents": [
{
"Id": 6732210,
"Name": "Passport",
"Fields": [
{
"Name": "First Name",
"Type": "text"
}
],
"Variants": [
{
"Name": "diplomatic_passport",
"TrainingSamples": [
{
"Name": "First Name",
"Value": "VGhlIFBPU1QsIFBVVCBhbmQgUEFUQ0ggcmVxdWVzdHMgY2FuIGhhdmUgdGhlIHJlcXVlc3QgYm9keSAocGF5bG9hZCksIHN1Y2ggYXMgSlNPTiBvciBYTUwgZGF0YS4gSW4gU3dhZ2dlciB0ZXJtcywgdGhlIHJlcXVlc3QgYm9keSBpcyBjYWxsZWQgYSBib2R5IHBhcmFtZXRlci4gVGhlcmUgY2FuIGJlIG9ubHkgb25lIGJvZHkgcGFyYW1ldGVyLCBhbHRob3VnaCB0aGUgb3BlcmF0aW9uIG1heSBoYXZlIG90aGVyIHBhcmFtZXRlcnMgKHBhdGgsIHF1ZXJ5LCBoZWFkZXIpLg==",
"BoundingBoxes": [
{
"Name": "string",
"Dimensions": [
{
"PageIndex": 0,
"Left": 0,
"Top": 0,
"Right": 0,
"Bottom": 0
}
]
}
]
}
]
}
]
}
]
}
My JOLT Spec is :
[
{
"operation": "shift",
"spec": {
"Name": "ProjectName",
"Documents": {
"*": {
"Name": {
"#": ["DocumentDefinitions[#].Name", "DocumentDefinitions[#].SectionDefinitions[#].Name"]
},
"Variants": {
"*": {
"Name": "DocumentDefinitions[#].SectionDefinitions[#].VarientNames[&1]",
"TrainingSamples": {
"0": {
"Name": "DocumentDefinitions[#].SectionDefinitions[#].ImageSources[#].ImageName.",
"Value": "DocumentDefinitions[#].SectionDefinitions[#].ImageSources[#].ImageValue."
}
}
}
},
"Fields": {
"*": {
"Name": "DocumentDefinitions[#].SectionDefinitions[#].Fields[&1].Name.",
"Type": "DocumentDefinitions[#].SectionDefinitions[#].Fields[&1].Type."
}
}
}
}
}
}
]
My Output is :
{
"ProjectName" : "travel.0.0",
"DocumentDefinitions" : [ {
"Name" : "Passport",
"SectionDefinitions" : [ {
"Name" : "Passport",
"VarientNames" : [ "diplomatic_passport" ],
"ImageSources" : [ {
"ImageName" : "First Name",
"ImageValue" : "VGhlIFBPU1QsIFBVVCBhbmQgUEFUQ0ggcmVxdWVzdHMgY2FuIGhhdmUgdGhlIHJlcXVlc3QgYm9keSAocGF5bG9hZCksIHN1Y2ggYXMgSlNPTiBvciBYTUwgZGF0YS4gSW4gU3dhZ2dlciB0ZXJtcywgdGhlIHJlcXVlc3QgYm9keSBpcyBjYWxsZWQgYSBib2R5IHBhcmFtZXRlci4gVGhlcmUgY2FuIGJlIG9ubHkgb25lIGJvZHkgcGFyYW1ldGVyLCBhbHRob3VnaCB0aGUgb3BlcmF0aW9uIG1heSBoYXZlIG90aGVyIHBhcmFtZXRlcnMgKHBhdGgsIHF1ZXJ5LCBoZWFkZXIpLg=="
} ],
"Fields" : [ {
"Name" : "First Name",
"Type" : "text"
} ]
} ]
} ]
}
This output looks fine , until i have only one JSON object inside my input array of documents . However , when i put another object , it looks somewhat like this which is not desired . It is clubbing all values of both objects together as below :
{
"ProjectName" : "travel.0.0",
"DocumentDefinitions" : [ {
"Name" : [ "Passport", "Passport" ],
"SectionDefinitions" : [ {
"Name" : [ "Passport", "Passport" ],
"VarientNames" : [ [ "diplomatic_passport", "diplomatic_passport" ] ],
"ImageSources" : [ {
"ImageName" : [ "First Name", "First Name" ],
"ImageValue" : [ "VGhlIFBPU1QsIFBVVCBhbmQgUEFUQ0ggcmVxdWVzdHMgY2FuIGhhdmUgdGhlIHJlcXVlc3QgYm9keSAocGF5bG9hZCksIHN1Y2ggYXMgSlNPTiBvciBYTUwgZGF0YS4gSW4gU3dhZ2dlciB0ZXJtcywgdGhlIHJlcXVlc3QgYm9keSBpcyBjYWxsZWQgYSBib2R5IHBhcmFtZXRlci4gVGhlcmUgY2FuIGJlIG9ubHkgb25lIGJvZHkgcGFyYW1ldGVyLCBhbHRob3VnaCB0aGUgb3BlcmF0aW9uIG1heSBoYXZlIG90aGVyIHBhcmFtZXRlcnMgKHBhdGgsIHF1ZXJ5LCBoZWFkZXIpLg==", "VGhlIFBPU1QsIFBVVCBhbmQgUEFUQ0ggcmVxdWVzdHMgY2FuIGhhdmUgdGhlIHJlcXVlc3QgYm9keSAocGF5bG9hZCksIHN1Y2ggYXMgSlNPTiBvciBYTUwgZGF0YS4gSW4gU3dhZ2dlciB0ZXJtcywgdGhlIHJlcXVlc3QgYm9keSBpcyBjYWxsZWQgYSBib2R5IHBhcmFtZXRlci4gVGhlcmUgY2FuIGJlIG9ubHkgb25lIGJvZHkgcGFyYW1ldGVyLCBhbHRob3VnaCB0aGUgb3BlcmF0aW9uIG1heSBoYXZlIG90aGVyIHBhcmFtZXRlcnMgKHBhdGgsIHF1ZXJ5LCBoZWFkZXIpLg==" ]
} ],
"Fields" : [ {
"Name" : [ "First Name", "First Name" ],
"Type" : [ "text", "text" ]
} ]
} ]
} ]
}
Please suggest what changes should i do in my JOLT spec , such that each transformed object in the array , get listed as a seprate object , and not get clubbed .
Check this spec,
[
{
"operation": "shift",
"spec": {
"Name": "ProjectName",
"Documents": {
"*": {
"Name": "DocumentDefinitions[&1].Name",
"Variants": {
"*": {
"Name": "DocumentDefinitions[&3].SectionDefinitions[&1].VarientNames[]",
"TrainingSamples": {
"*": {
"Name": "DocumentDefinitions[&5].SectionDefinitions[&1].ImageSources[&1].ImageName",
"Value": "DocumentDefinitions[&5].SectionDefinitions[&1].ImageSources[&1].ImageValue"
}
}
}
},
"Fields": {
"*": {
"Name": "DocumentDefinitions[&3].SectionDefinitions[&3].Fields[&1].Name",
"Type": "DocumentDefinitions[&3].SectionDefinitions[&3].Fields[&1].Type"
}
}
}
}
}
}
]

jolt format with array of strings

Trying to make a Jolt script that will put in a single number line, then a array of strings in one single array and a tag on the end of that array. These are the values that I have been working with.
JSON INPUT
[
{
"foo": "111",
"bar": "222",
"sun": "333",
"ListofStrings": [
"Dog",
"Train"
],
"ID": "BLAH"
},
{
"foo": "999",
"bar": "222",
"sun": "777",
"ListofStrings": [
"CAT",
"PLANE"
],
"ID": "HAHA"
}
]
JOLT SPEC This is what I have been working with that prints out the ListofStrings but this is the one that works stably.
[
{
"operation": "shift",
"spec": {
"*": {
"foo": "input[].number",
"bar": "input[].number",
"sun": "input[].number",
"ListofStrings": "input[].List"
}
}
},
{
"operation": "default",
"spec": {
"app_id": "test",
"input": {
"*": {
"app_id": "test"
}
}
}
}
]
CURRENT OUTPUT
{
"input" : [ {
"number" : "111"
}, {
"number" : "222"
}, {
"number" : "333"
}, {
"List" : [ "Dog", "Train" ]
}, {
"number" : "999"
}, {
"number" : "222"
}, {
"number" : "777"
}, {
"List" : [ "Cat", "Car" ]
} ],
"app_id" : "test"
}
DESIRED OUTPUT
{
"input" : [ {
"number" : "111"
"List" : [ "Dog", "Train" ]
"ID": "BLAH_foo"
}, {
"number" : "222"
"List" : [ "Dog", "Train" ]
"ID": "BLAH_bar"
}, {
"number" : "333"
"List" : [ "Dog", "Train" ]
"ID": "BLAH_sun"
}, {
"number" : "999"
"List" : [ "Cat", "Car" ]
"ID": "HAHA_foo"
}, {
"number" : "222"
"List" : [ "Cat", "Car" ]
"ID": "HAHA_bar"
}, {
"number" : "777"
"List" : [ "Cat", "Car" ]
"ID": "HAHA_sun"
} ],
"app_id" : "test"
}
Check this spec
[
//Converting list to Map
{
"operation": "shift",
"spec": {
"*": {
"ListofStrings": null,
"*": {
"#": "#1.number",
"#(1,ListofStrings)": "#1.list"
}
}
}
},
//Shift the number and list to the input array
{
"operation": "shift",
"spec": {
"*": {
"$": "input[#2].number",
"#(0,list)": "input[#2].List"
}
}
}, {
"operation": "default",
"spec": {
"app_id": "test",
"input": {
"*": {
"app_id": "test"
}
}
}
}
]
Edit 1
Add the ID node to the map using first shift operation "ID": null, and "#(1,ID)": "#1.ID". Then shift the ID node to the input array in the second shift operation "#(0,ID)": "input[#2].ID".
[
//Converting list to Map
{
"operation": "shift",
"spec": {
"*": {
"ListofStrings": null,
"ID": null,
"*": {
"#": "#1.number",
"#(1,ListofStrings)": "#1.list",
"#(1,ID)": "#1.ID"
}
}
}
},
//Shift the number and list to the input array
{
"operation": "shift",
"spec": {
"*": {
"$": "input[#2].number",
"#(0,list)": "input[#2].List",
"#(0,ID)": "input[#2].ID"
}
}
}, {
"operation": "default",
"spec": {
"app_id": "test",
"input": {
"*": {
"app_id": "test"
}
}
}
}
]