Jolt how to do this transform? Concat and set in array - json

I have an input like this
{
"employee": {
"id": "123456",
"firstName": "John",
"prefFirstName": "Doe",
"lastName": "John",
"prefLastName": "John",
"mgrFirstName": "Mitch",
"mgrPrefFirstName": "Mitch",
"mgrLastName": "McEvoy",
"mgrPrefLastName": "McEvoy",
"gmFirstName": "GMName",
"gmPrefFirstName": "GMName",
"gmLastName": "GMLastName",
"gmPrefLastName": "GMLastName",
"dirFirstName": "DirFirstName",
"dirPrefFirstName": "DirFirstName",
"dirLastName": "DirLastName",
"dirPrefLastName": "DirLastName",
}
}
Expected output should be like this, I can't figure out the fields in bold.
"managers" : [ {
"chnl" : "eml",
"nm": "Mitch McEvoy",
“mgrlbl”: “Direct Manager”
}, {
"chnl" : "eml",
"eml" : "",
"nm": "GMName GMLastName",
“mgrlbl”: “General Manager”
}, {
"chnl" : "eml",
"eml" : "",
"nm": "DirFirstName DirLastName",
“mgrlbl”: “Director”
} ],
"nhstdt" : "2020-11-23",
"nhmob" : "",
"nheml" : "empp#web.com",
"mgreml" : [ "testttt#web.com", "testttt#web.com" ],
"mgrchnl" : "email",
"chnl" : "mob",
"nhnm" : "Doe John",
"mgrnm" : "Mitch McEvoy"
I have a transform like this but can't figure out how to get the desired output:
[
{
"operation": "shift",
"spec": {
"employee": {
"prefFirstName": {
"*": {
"#1": "tmpFirstName",
"#(2,prefLastName)": "tmpLastName"
},
"": {
"#(2,firstName)": "tmpFirstName",
"#(2,lastName)": "tmpLastName"
}
},
"mgrPrefFirstName": {
"*": {
"#1": "tmpMgrFirstName",
"#(2,mgrPrefLastName)": "tmpMgrLastName"
},
"": {
"#(2,mgrFirstName)": "tmpMgrFirstName",
"#(2,mgrLastName)": "tmpMgrLastName"
}
},
"mgrName": "managers[0].nm",
"mgrBusEmail": "managers[0].eml",
"gmName": "managers[1].nm",
"gmBusEmail": "managers[1].eml",
"dirName": "managers[2].nm",
"dirBusEmail": "managers[2].eml",
"#eml": ["managers[0].chnl", "managers[1].chnl", "managers[2].chnl"]
}
}
},
{
"operation": "modify-default-beta",
"spec": {
"nhnm": "=concat(#(1,tmpFirstName),' ',#(1,tmpLastName))",
"mgrnm": "=concat(#(1,tmpMgrFirstName),' ',#(1,tmpMgrLastName))"
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"mode": ["=toInteger", 0]
}
}
]
But I am not able to figure out how do I concatenate mgrFirstName and mgrLastName, gmFirstName and gmLastName, dirFirstName and dirLastName and after concatenation I need to set this in the array managers[0], managers[1] and managers[2] respectively.
I am not able to figure how do I modify an array which is in my shift spec after I concat the stirngs.

You can do the concatenation first with a modify-default-beta spec, then set the desired fields in the array. This should get you started:
[
{
"operation": "modify-default-beta",
"spec": {
"employee": {
"mgrFullName": "=concat(#(1,mgrFirstName),' ',#(1,mgrLastName))",
"gmFullName": "=concat(#(1,gmFirstName),' ',#(1,gmLastName))",
"dirFullName": "=concat(#(1,dirFirstName),' ',#(1,dirLastName))"
}
}
},
{
"operation": "shift",
"spec": {
"employee": {
"mgrFullName": "managers[0].nm",
"mgrBusEmail": "managers[0].eml",
"gmFullName": "managers[1].nm",
"gmBusEmail": "managers[1].eml",
"dirFullName": "managers[2].nm",
"dirBusEmail": "managers[2].eml",
"#eml": ["managers[0].chnl", "managers[1].chnl", "managers[2].chnl"],
"#Direct Manager": "managers[0].mgrlbl",
"#General Manager": "managers[1].mgrlbl",
"#Director": "managers[2].mgrlbl"
}
}
}
]

Related

How does the jolt wildcards work for RHS?

I'm trying to understand the working of '#' wild card.
I have used the '#' in the spec while preparing the productList and it works and I got the output as per my expectation. But I'm not sure about the working of it.
Can anyone please help me to understand the working of it?
Here is the input JSON
{
"orders": [
{
"order_parts": [
{
"id": "0001",
"items": [
{
"id": "00101",
"goodIdentificationList": [
{
"goodIdentificationTypeId": "UPCA",
"idValue": "42684666380437"
},
{
"idValue": "V-ASHBY",
"goodIdentificationTypeId": "SHOPIFY_PROD_SKU"
}
],
"productName": "BLACK / 6 / 809"
},
{
"id": "00102",
"goodIdentificationList": [
{
"goodIdentificationTypeId": "SHOPIFY_PROD_ID",
"idValue": "42684666380437"
},
{
"idValue": "V-ASHBY",
"goodIdentificationTypeId": "UPCA"
}
],
"productName": "BLACK / 6 / 809"
}
]
},
{
"id": "0002",
"items": [
{
"id": "00103",
"goodIdentificationList": [
{
"goodIdentificationTypeId": "SHOPIFY_PROD_ID",
"idValue": "42684666380437"
},
{
"idValue": "V-ASHBY",
"goodIdentificationTypeId": "UPCA"
}
],
"productName": "BLACK / 6 / 809"
}
]
}
]
}
]
}
Expected by the below spec:-
Check every map of goodIdentificationList, get the idValue where goodIdentificationTypeId - UPCA, and put in the productList as gtin.
Get the id from the items list and put it in the productList as an itemId.
Get the productName from the items list and put it in the productList as a name.
Jolt Spec is like below
[
{
"operation": "shift",
"spec": {
"orders": {
"*": {
"order_parts": {
"*": {
"items": {
"*": {
"goodIdentificationList": {
"*": {
"goodIdentificationTypeId": {
"UPCA": {
"#(2,idValue)": "[&5].productList.[#8].gtin"
}
}
}
},
"id": "[&1].productList.[#4].itemId",
"productName": "[&1].productList.[#4].name"
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": {
"productList": {
"*": "productList.[]"
}
}
}
}
]
By the above spec, I'm able to prepare the productList as I was expecting.
But want to understand the working of '#' here.
Output JSON
{
"productList" : [ {
"itemId" : "00101",
"name" : "BLACK / 6 / 809"
}, {
"itemId" : "00103",
"name" : "BLACK / 6 / 809"
}, {
"itemId" : "00102",
"name" : "BLACK / 6 / 809"
} ]
}
Any help will be appreciated.
Thanks!
You principally need this spec
[
{
"operation": "shift",
"spec": {
"order*": {
"*": {
"order*": {
"*": {
"items": {
"*": {
"id": "&3.&1.&",
"prod*": "&3.&1.&"
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": {
"*": "productList[]"
}
}
}
]
in which, the combination of the indexes of "items" array(&1) vs. the indexes of "order_parts" array(&3) handles separation of three individual objects, and then then tiding up innermost part while tagging the desired value(productList), and combining those newly generated objects will spontaneously form an array.
Replacement of
"id": "&3.&1.&",
"prod*": "&3.&1.&"
with
"id": "&3[&1].&",
"prod*": "&3[&1].&"
or
"id": "&3[#2].&",
"prod*": "&3[#2].&"
would handle exactly the same for this current case. But if there was no array the identifiers [&1] or [#2] would produce square brackets. The difference for right-hand-size usage of them is [&1] will traverse {, while [#2] will traverse both { and the current : characters in order to reach the target within the current tree, and [&1] will use indexes 0,1,2, which starts from zero, everytime and this will generate some null components for generated arrays for some cases, while [#2] won't. Eg. On the RHS of the spec, # is only valid in the the context of an array, like "[#2]".What "[#2]" means is, go up the three levels(including the colon at the current level) and ask that node how many matches it has had, and then use that as an index in the arrays.

Using variable from previous spec block in JOLT

I need to convert input attributes into SOLR request containing three sections, one of them is solrOptions.
in my request I have:
{
...
"firstTranDateRange": "IN LAST 365 DAYS",
"lastTranDateRange": "In last 60 days",
...
}
In the output I need to have following:
{
"searchAttributes": { ... }
"searchOptions": { ... }
"solrOptions: {
"anm_boost" : "1.0",
"pnm_boost" : "1.0",
"lastTranDays" : {
"min" : "00000",
"max": "00060"
},
"firstTranDays" : {
"min" : "00000",
"max" : "00365"
}
}
}
I wrote the following spec:
[
{
"spec": {
"searchAttrbutes": {
// filling up some other attributes not related to the question
"ltdr": "=substring(#(1,lastTranDateRange),8,3)",
"ltdr1": "=trim",
"ftdr": "=substring(#(1,firstTranDateRange),8,3)",
"ftdr1": "=trim"
}
},
"operation": "modify-default-beta"
},
{
"spec": {
"solrOptions": {
"anm_boost": "1.0",
"pnm_boost": "1.0"
},
"searchOptions": {
"maxRecords": 1,
...
}
},
"operation": "default"
},
{
"spec": {
"lastTranDateRange": {
"#00000": "solrOptions.lastTranDays.min",
"#(2,ltdr1)": "solrOptions.lastTranDays.max"
},
"firstTranDateRange": {
"#00000": "solrOptions.firstTranDays.min",
"#(2,ftdr1)": "solrOptions.firstTranDays.max"
},
...
},
"operation": "shift"
}
]
I haven't figured out how to prepend number with proper number of zeros. Help in this will be also appreciated. However, it is not working in the first place. The variables I defined do not appear in the output. In the output I am getting
{
"solrOptions": {
"anm_boost": "1.0",
"pnm_boost": "1.0",
"wcNameOn": "true",
"fuzzyNameOn": "true",
"lastTranDays": {
"min": "00000"
},
"firstTranDays": {
"min": "00000"
}
},
"searchOptions": {
"maxRecords": "10",
"matchIndicators": "TRUE",
...
}
}
Can someone please tell me what I am doing incorrectly.
Thanks.
You can start with getting rid of "searchAttributes" from modify transformation presuming the input is exactly
{
"firstTranDateRange": "IN LAST 365 DAYS",
"lastTranDateRange": "In last 60 days"
}
otherwise keep it, and converting that from default to overwrite in order to overwrite all existing attributes. Btw, substring in Jolt is interestingly different function unlike to their usage in some well-known DBMS products, here ending index should be provided as the last argument.
Moreover leftPad function will be needed for your case to prepend the values with zeroes.
* wildcard is used to represent all or rest of the expressions(key names), and &1 to climb one(1) level tree up and grab the related key name(solrOptions)
Thus, you can use the following combination of specs:
[
{
"operation": "modify-overwrite-beta",
"spec": {
"*": "=substring(#(1,&),8,11)" // overwrite all attributes at once by using * and & wildcards
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"lastTranDateRange": "=trim", //individual manipulation needed as not having 3 digits
"*": "=leftPad(#(1,&),5,'0')"
}
},
{
"operation": "default",
"spec": {
"solrOptions": {
"anm_boost": "1.0",
"pnm_boost": "1.0"
},
"searchOptions": {
"maxRecords": 1
}
}
},
{
"operation": "shift",
"spec": {
//"searchAttributes": "&",
"search*": "&",
"solr*": {
"*": "&1.&",
"#(1,firstTranDateRange)": "&1.firstTranDays.max",
"#(1,lastTranDateRange)": "&1.lastTranDays.max"
}
}
},
{
"operation": "default",
"spec": {
"solrOptions": {
"firstTranDays": {
"min": "00000"
},
"lastTranDays": {
"min": "00000"
}
}
}
}
]
to yield the output :
{
//"searchAttributes": "{....}",
"solrOptions" : {
"firstTranDays" : {
"max" : "00365",
"min" : "00000"
},
"lastTranDays" : {
"max" : "00060",
"min" : "00000"
},
"pnm_boost" : "1.0",
"anm_boost" : "1.0"
},
"searchOptions" : {
"maxRecords" : 1
}
}

JOLT shift transformation to filter values in array

I want to use a JOLT transformation to do two things:
filter the elements in the array called myarray so that only elements remain which have a "v_518" attribute
filter out all attributes of the remaining elements except for "v_518" and "lfdn"
Input:
{
"isError": false,
"isValid": true,
"myarray": [
{
"p_0001": "1",
"p_0002": "1",
"p_0003": "1",
"p_0004": "1",
"v_518": "0,214506186",
"lfdn": 89709
},
{
"p_0001": "2",
"p_0002": "1",
"p_0003": "1",
"v_518": "0,3823236",
"lfdn": 89710
},
{
"p_0001": "3",
"p_0002": "1",
"p_0003": "1",
"lfdn": 89711
}
],
"errorMessage": null,
"exceptionMessage": null,
"innerExceptionMessage": null
}
Desired output:
{
"isError": false,
"isValid": true,
"myarray": [
{
"v_518": "0,214506186",
"lfdn": 89709
},
{
"v_518": "0,3823236",
"lfdn": 89710
}
],
"errorMessage": null,
"exceptionMessage": null,
"innerExceptionMessage": null
}
What I tried so far, but doesn't work as intended:
[
{
"operation": "shift",
"spec": {
"isError": "isError",
"isValid": "isValid",
"myarray": {
// loop thru all the elements in value array
"*": {
"v_518": {
// if the value "v_518" exists
// grab the whole object and write it out to
// a v_518_array array.
"#(1,v_518)": "v_518_array",
"#(1,lfdn)": "v_518_array"
}
}
},
"errorMessage": "errorMessage",
"exceptionMessage": "exceptionMessage",
"innerExceptionMessage": "innerExceptionMessage"
}
}
]
I tried working with the examples in http://jolt-demo.appspot.com/#andrewkcarter2 but I couldn't figure out how to do it.
I was able to solve my issue. This answer was the hint I needed to get the ball rolling: https://stackoverflow.com/a/38154541/1561441
The key is referencing the array you are currently transforming via "value" = "array[&1].value".
In my mind I spent way too much time on this issue. Does anyone know of a good documentation for the Jolt syntax? I couldn't find a satisfactory one by googling myself.
[
{
"operation": "shift",
"spec": {
"isError": "isError",
"isValid": "isValid",
"myarray": {
// loop thru all the elements in value array
"*": {
"v_518": {
// if the value "v_518" exists
// grab the whole object and write it out to
// a v_518_array array.
"#1": "v_518_array"
}
}
},
"errorMessage": "errorMessage",
"exceptionMessage": "exceptionMessage",
"innerExceptionMessage": "innerExceptionMessage"
}
},
{
"operation": "shift",
//Transform array: https://stackoverflow.com/questions/37865871/how-do-i-transform-an-array-using-jolt
"spec": {
"v_518_array": {
// loop thru all the elements in value array
"*": {
"v_518": "v_518_array[&1].v_518",
"lfdn": "v_518_array[&1].lfdn"
}
}
}
}
]
Here's a slightly better solution:
[
{
"operation": "shift",
"spec": {
"isError": "isError",
"isValid": "isValid",
"myarray": {
// loop thru all the elements in value array
"*": {
"v_518": {
// if the value "v_518" exists
// grab the whole object and write it out to
// a v_518_array array.
"#1": "v_518_array"
}
}
},
"errorMessage": "errorMessage",
"exceptionMessage": "exceptionMessage",
"innerExceptionMessage": "innerExceptionMessage"
}
},
{
"operation": "shift",
//Transform array: https://stackoverflow.com/questions/37865871/how-do-i-transform-an-array-using-jolt
"spec": {
"v_518_array": {
// loop thru all the elements in value array
"*": {
"v_518": "&2[&1].v_518", //notice the generic shorthand here
"lfdn": "&2[&1].lfdn"
}
}
}
}
]
Still there's a briefer, and more dynamic method to figure out your issue such as
[
{
"operation": "shift",
"spec": {
"myarray": {
"*": {
"v_518": {
"#(1,&)": "&3.[&2].&",
"#(1,lfdn)": "&3.[&2].lfdn"
}
}
},
"*": "&"
}
}
]

JOLT tranform elements based on key

I am trying to find out correct JOLT specification file for below:
Input file:
{
"network": "A",
"ips": {
"subnet1": "1.1.1.1",
"subnet2": "1.1.1.2",
"subnet3": "1.1.1.3",
"subnet4": "1.1.1.4"
}
}
Output file:
{
"allValues": [
{
"network": "A",
"ips": {
"subnet1": "1.1.1.1",
"subnet2": "1.1.1.2"
}
},
{
"network": "A",
"ips": {
"subnet3": "1.1.1.3",
"subnet4": "1.1.1.4"
}
}
]
}
Based on subnet values, I want to group the IPs(subnet1 and subnet2 in one) and (subnet3 and subnet4 in another), while all should have network value(A).
Can somebody please help.
If you know which subnets belong together (subnet1 and subnet2 for example), you can use something like the following:
[
{
"operation": "shift",
"spec": {
"ips": {
"subnet1": "allValues[0].ips.subnet1",
"subnet2": "allValues[0].ips.subnet2",
"subnet3": "allValues[1].ips.subnet3",
"subnet4": "allValues[1].ips.subnet4"
},
"network": "allValues[0].network",
"#(0,network)": "allValues[1].network"
}
}
]

How to get multiple array value as object

Hi I am new to JOLT transformation. I need to transform the input json using JOLT to get below seen output. Please help me in below transformation:
input:
{
"image": [
"content1",
"content2",
"content3"
],
"legal": [
"legal1",
"legal2",
"legal3"
],
"hyper": [
"hyper1",
"hyper2",
"hyper3"
]
}
output:
[
{
"image": "content1",
"legal": "legal1",
"hyper": "hyper1"
},
{
"image": "content1",
"legal": "legal1",
"hyper": "hyper1"
},
{
"image": "content1",
"legal": "legal1",
"hyper": "hyper1"
}
]
Spec
[
{
"operation": "shift",
"spec": {
"*": { // image, legal, etc
"*": { // array
"*": { // content1, legal1, etc
"$": "[&2].&3" // grab "content1" and use it as output
// send it to an output doc that is a top level array
// indexed by looking 3 levels up the tree [&2]
}
}
}
}
}
]