I'm trying to transform a JSON file into something that can later be transformed to csv or xml. My JSON file looks like this:
{
"meta": {
"contentType": "Response"
},
"content": {
"_type": "Response",
"data": {
"_type": "ObjectList",
"meta": {
"_type": "ObjectListMeta"
},
"DataObjects": [
{
"head": {
"fields": {
"id": {
"value": "24"
},
"name": {
"value": ""
}
}
},
"table": [
{
"number": 1,
"fields": {
"height": {
"value": 500,
"text": "500"
},
"average": {
"value": -2,
"text": "-2"
}
}
},
{
"number": 2,
"fields": {
"height": {
"value": 99999,
"text": "99999"
},
"average": {
"value": -5,
"text": "-5"
}
}
}
]
}
]
}
}
}
I want the output to look like this:
[
{
"id": "24",
"name": "",
"height": 500,
"average": -2
},
{
"id": "24",
"name": "",
"height": 99999,
"average": -5
}
]
I tried the following JOLT specification:
[
{
"operation": "shift",
"spec": {
"content": {
"data": {
"DataObjects": {
"*": {
"head": {
"fields": {
"id": {
"value": "[&4].id"
},
"name": {
"value": "[&4].name"
}
}
},
"table": {
"*": {
"fields": {
"height": {
"value": "[&5].height"
},
"average": {
"value": "[&5].average"
}
}
}
}
}
}
}
}
}
}
]
But I only get the following output:
[
{
"id": "24",
"name": "",
"height" : [ 500, 99999 ],
"average" : [ -2, -5 ]
}
]
Which is not exactly what I want. How do I have to modify my spec to get the output I need?
You can combine those attributes under the table list as having multiple respective objects for one head.fields object such as
[
{
"operation": "shift",
"spec": {
"content": {
"*": {
"DataObjects": {
"*": {
"table": {
"*": {
"#(2,head.fields.id.value)": "[&].id",
"#(2,head.fields.name.value)": "[&].name",
"#(0,fields.height.value)": "[&].height",
"#(0,fields.average.value)": "[&].average"
}
}
}
}
}
}
}
}
]
Related
I have to make a JOLT filter to delete a field based on condition, also the filter has to work with two different JSON structures.
So, here is the case.
I have these JSONs:
JSON 1:
{
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key2": "val2",
"key3": "val3",
"key4": "val4",
"key5": "val5",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
}
}
JSON 2:
{
"responseMessage": {
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key2": "val2",
"key3": "val3",
"key4": "val4",
"key5": "val5",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
},
"state": {
"state1": "state1"
}
},
"status": [],
"notes": []
}
I need the filter to remove for example key2 and key5 from the attributes_here only if the service value is THIS_SERVICE and keep the same structure for the JSON.
Desired output:
JSON 1:
{
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key3": "val3",
"key4": "val4",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
}
}
JSON 2:
{
"responseMessage": {
"header": {
"key": "value"
},
"payload": {
"someArray": [
{
"key1": "val1"
}
],
"elements": [
{
"service": "serviceOne",
"something": "somethingValue"
},
{
"service": "THIS_SERVICE",
"attributes": [
{
"something": "somethingValue",
"attributes_here": {
"key1": "val1",
"key3": "val3",
"key4": "val4",
"key6": "val6"
}
}
]
}
]
},
"data": {
"key1": "val1"
},
"state": {
"state1": "state1"
}
},
"status": [],
"notes": []
}
Any help would be appreciated. Thank you and I wish you a great day!
You can use the following spec as an option :
[
{
"operation": "shift",
"spec": {
"responseMessage": { // distinguish the elements whether they have an extra wrapper
"*": "&1_&"
},
"*": "&"
}
},
{ // accumulate the respective attributes by their "service" value within the "attributes_here" object
"operation": "shift",
"spec": {
"*": "&",
"*ayload": {
"*": "&1.&",
"elements": {
"*": {
"*": "&3.&2[#2].&",
"attributes": {
"*": {
"*": "&5.&4[#4].&2[#2].&",
"attributes_here": {
"*": "&6.&5[#5].&3[#3].&1.&",
"key2|key5": "&6.&5[#5].&3[#3].&1.#4,service.&"
}
}
}
}
}
}
}
},
{// get rid of the keys with "THIS_SERVICE"
"operation": "remove",
"spec": {
"*ayload": {
"elements": {
"*": {
"attributes": {
"*": {
"attributes_here": {
"THIS_SERVICE": ""
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*_*": {
"#": "&(1,1).&(1,2)" // the value of the first and second asterisks, after going one level up the tree, respectively
},
"*": "&"
}
}
]
Please try the below jolt spec:
[
{
"operation": "shift",
"spec": {
"#false": "has_responseMessage",
"header": "&",
"payload": "&",
"data": "&",
"responseMessage": {
"*": "&",
"#(2,status)": "rest_&.status",
"#(2,notes)": "rest_&.notes",
"#true": "has_responseMessage"
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"payload": {
"*": "&1.&",
"elements": {
"*": { // 0, 1
"*": "&3.&2[&1].&",
"service": {
"#": "&4.&3[&2].&1",
"THIS_SERVICE": {
"#(2,attributes)": "&5.&4[&3].attributes_temp",
"#attributes_temp": "&5.&4[&3].attributes_key"
},
"#attributes": "&4.&3[&2].attributes_key"
}
}
}
}
}
},
{
"operation": "modify-overwrite-beta",
"spec": {
"has_responseMessage": "=lastElement",
"payload": {
"elements": {
"*": {
"attributes_key": "=lastElement"
}
}
}
}
},
{
"operation": "remove",
"spec": {
"payload": {
"elements": {
"*": {
"attributes_temp": {
"*": {
"attributes_here": {
"key2": "",
"key5": ""
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"payload": {
"*": "&1.&",
"elements": {
"*": { // 0, 1
"service": "&3.&2[&1].&",
"something": "&3.&2[&1].&",
"attributes_key": {
"attributes_temp": {
"#(2,attributes_temp)": "&5.&4[&3].attributes"
},
"attributes": {
"#(2,attributes)": "&5.&4[&3].attributes"
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"has_responseMessage": {
"true": {
"#(3,header)": "responseMessage.header",
"#(3,payload)": "responseMessage.payload",
"#(3,data)": "responseMessage.data",
"#(3,state)": "responseMessage.state"
},
"false": {
"#(3,header)": "header",
"#(3,payload)": "payload",
"#(3,data)": "data"
}
},
"rest_responseMessage": {
"*": "&"
}
}
}
]
You can remove your fields in the remove operation as you want.
I am trying to transform a JSON using jolt without adding new items as well as remove white space from the key value. additionally, I would like to know how to get map to current list.
input :
[
{
"id": "9164339966213408050",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"Total BYOP Tier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468326"
}
}
],
"chars": {
"Total BYOP Tier": "60"
}
},
{
"id": "9164339966213408051",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"Total BYOP Tier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468325"
}
}
],
"chars": {
"Total BYOP Tier": "60"
}
}
]
expected output :
[
{
"id": "9164339966213408050",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468326",
"TotalBYOPTier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468326"
}
}
],
"chars": {
"TotalBYOPTier": "60"
}
},
{
"id": "9164339966213408051",
"actualEndDate": "2022-08-15T14:47:09+03:00",
"extendedChars": [
{
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468325",
"TotalBYOPTier": {
"values": [
{
"value": "60"
}
],
"name": "Total BYOP Tier",
"isMultiple": false,
"tomsId": "9162933742365468325"
}
}
],
"chars": {
"TotalBYOPTier": "60"
}
}
]
spec tried :
[
{
"operation": "shift",
"spec": {
"*": {
"#": "CustomerProductArray"
}
}
},
{
"operation": "shift",
"spec": {
"CustomerProductArray": {
"*": {
"extendedChars": {
"*": {
"* * *": "CustomerProductArray.&(0,1)&(0,2)&(0,3)",
"*": {
"name": "CustomerProductArray[].extendedChars[]..name",
"isMultiple": "CustomerProductArray[].extendedChars[].isMultiple",
"tomsId": "CustomerProductArray[].extendedChars[].tomsId",
"values": "CustomerProductArray[].extendedChars[].values"
}
}
},
"chars": {
"* * *": "CustomerProductArray[].chars.&(0,1)&(0,2)&(0,3)"
}
}
}
}
}
]
Use the below spec as you want to replicate the content of the Total BYOP Tier array, nested inside and outside the object
[
{
"operation": "shift",
"spec": {
"*": {
"*": "&1.&",
"extendedChars": {
"*": {
"Total BYOP Tier": {
"*": {
"#": "&5.&4[&3].&" // &5 : (going up the tree five levels) replicates the index of the outermost array, &4[&3] : stands for "extendedChars"(&4) along strolling through with its indexes([&3])
},
"#": "&4.&3[&2].&1"
}
}
}
}
}
},
{
// get rid of object labels
"operation": "shift",
"spec": {
"*": ""
}
}
]
the demo on the site http://jolt-demo.appspot.com/ is
I have following expected input and expected output and i want to transform data by jolt processor
Input JSON
{
"subscriptionId": "63",
"data": [
{
"type": "demo",
"Data": {
"transactionId": "598958",
"type": "json",
"xyz": "pqr",
"name": "john"
},
"nameData": [
{
"name": "Ama",
"nameCount": "215",
"genderData": [
{
"gender": "Male",
"genderCount": "140"
},
{
"gender": "Female",
"genderCount": "75"
}
]
},
{
"name": "Aedes",
"nameCount": "161",
"genderData": [
{
"gender": "Female",
"genderCount": "134"
},
{
"gender": "Male",
"genderCount": "27"
}
]
},
{
"name": "Culex",
"nameCount": "2610",
"genderData": [
{
"gender": "Male",
"genderCount": "1926"
},
{
"gender": "Female",
"genderCount": "684"
}
]
},
{
"name": "Kamp",
"nameCount": "465",
"genderData": [
{
"gender": "Male",
"genderCount": "465"
}
]
}
]
}
]
}
Expected Output JSON
{
"transactionId": "598958",
"type": "json",
"xyz": "pqr",
"name": "john",
"alert_array": [{
"abc": "123",
"xyz": "pqrs",
"properties": [{
"key": "Ama",
"value": "215",
"object": [{
"key": "Male",
"value": "140"
},
{
"key": "Female",
"value": "75"
}
]
},
{
"key": "Aedes",
"value": "161",
"object": [{
"key": "Male",
"value": "134"
},
{
"key": "Female",
"value": "27"
}
]
},
{
"key": "Culex",
"value": "2610",
"object": [{
"key": "Male",
"value": "1926"
},
{
"key": "Female",
"value": "684"
}
]
},
{
"key": "Kamp",
"value": "465",
"object": [{
"key": "Male",
"value": "465"
}]
},
{
"key": "type",
"value": "demo"
}
]
}]
}
I have the above following input JSON with nameData being an array and having genderData another array within it.
Need to convert the input JSON to expected JSON output as shown above.
please suggest JOLT spec to transform the above JSON.
You can start with determining the nodes alert_array,properties,object as diving to the innermost attributes within the shift transformation such as
[
{
"operation": "shift",
"spec": {
"data": {
"*": {
"Da*": "",
"nameD*": {
"#123": "alert_array[0].abc", // # wildcard is used to add fixed values
"#pqrs": "alert_array[0].xyz",
"*": {
"name": "alert_array[0].properties[&1].key",
"nameC*": "alert_array[0].properties[&1].value",
"*": "alert_array[0].properties[&1].object"
}
}
}
}
}
},
{
// rename the innermost attributes
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"key": "#(1,gender)",
"value": "#(1,genderCount)"
}
}
}
}
}
}
}
},
{
// get rid of the former names of the innermost attributes
"operation": "remove",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"gender": "",
"genderCount": ""
}
}
}
}
}
}
}
}
]
the demo on the site http://jolt-demo.appspot.com/ is
Edit : You can add two more shift transformation specs in order to put the lately desired object
{
"key" : "type",
"value" : "demo"
}
such as
[
{
"operation": "shift",
"spec": {
"data": {
"*": {
"Da*": "",
"nameD*": {
"#123": "alert_array[0].abc", // # wildcard is used to add fixed values
"#pqrs": "alert_array[0].xyz",
"*": {
"name": "alert_array[0].properties[&1].key",
"nameC*": "alert_array[0].properties[&1].value",
"*": "alert_array[0].properties[&1].object"
}
}
}
}
}
},
{
// rename the innermost attributes
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"key": "#(1,gender)",
"value": "#(1,genderCount)"
}
}
}
}
}
}
}
},
{
// get rid of the former names of the innermost attributes
"operation": "remove",
"spec": {
"*": {
"*": {
"*": {
"*": {
"*": {
"*": {
"gender": "",
"genderCount": ""
}
}
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"alert_array": {
"*": {
"*": "&2[&1].&",
"pro*": {
"#type": "&3[&2].&1.&2.key",
"#demo": "&3[&2].&1.&2.value",
"*": "&3[&2].&1.&"
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": "&",
"alert_array": {
"*": {
"*": "&2[&1].&",
"pro*": {
"*": {
"#": "&4[&3].&2"
}
}
}
}
}
}
]
I has this mapping for the índex on elastic, i was try to get the max value of a day for a specific sensor, but my query get the value of all the sensors.
"sensor": {
"type": "nested",
"properties": {
"type": {
"type": "integer"
},
"name": {
"type": "keyword"
},
"number": {
"type": "integer"
},
"values": {
"type": "nested",
"properties": {
"type": {
"type": "text"
},
"value": {
"type": "float"
},
"unit": {
"type": "text"
},
"Lmin": {
"type": "float"
},
"Lmax": {
"type": "float"
}
}
}
}
An this is the map of objects,
I need only the max and the min value of the las day from the sensor number 13, i try it but ever i get the max of all sensors.
{"query": {
"nested": {
"path": "sensor",
"query": {
"nested": {
"path": "sensor.values",
"query": {
"bool": {
"must": [
{
"match": {
"sensor.values.type": "TEMPERATURE"
}
}
]
}
}
}
}
}
},
"aggs": {
"agg_by_type": {
"nested": {
"path": "sensor.values"
},
"aggs": {
"max_value": {
"max": {
"field": "sensor.values.value"
}
}
}
}
}
}
I'm new in elasticsearch, can someone help whit this please?, thanks.
You need to also add the nested filter in the aggregation part to only aggregate the relevant nested documents, i.e. the ones related to TEMPERATURE, like this:
{
"query": {
"nested": {
"path": "sensor",
"query": {
"nested": {
"path": "sensor.values",
"query": {
"bool": {
"must": [
{
"match": {
"sensor.values.type": "TEMPERATURE"
}
}
]
}
}
}
}
}
},
"aggs": {
"agg_by_type": {
"nested": {
"path": "sensor.values"
},
"aggs": {
"temperature_only": {
"filter": {
"match": {
"sensor.values.type": "TEMPERATURE"
}
},
"aggs": {
"max_value": {
"max": {
"field": "sensor.values.value"
}
}
}
}
}
}
}
}
After a few days of work in another projects, i back to try make this query and finally i can do it, now i can get the data per day, hours, and by type of sensor, thanks for your help.
This is my code if somebody are trying the same.
{
"query": {
"bool": {
"filter": [
{
"bool": {
"must": [
{
"match": {
"mac": "34:ab:95:8f:84:c0"
}
}
],
"filter": [
{
"range": {
"timestamp": {
"gte": "2021-08-10",
"lt": "2021-08-25"
}
}
}
]
}
},
{
"nested": {
"path": "sensor",
"query": {
"bool": {
"must": [
{
"match": {
"sensor.type": 1
}
}
],
"should": [
{
"nested": {
"path": "sensor.values",
"query": {
"bool": {
"must": [
{
"match": {
"sensor.values.type": "HUMIDITY"
}
},
{
"match": {
"sensor.values.type": "TEMPERATURE"
}
}
]
}
}
}
}
]
}
}
}
}
]
}
},
"aggs": {
"values_per_day": {
"date_histogram": {
"field": "timestamp",
"fixed_interval": "1d",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"aggs": {
"agg_type": {
"nested": {
"path": "sensor"
},
"aggs": {
"type_only": {
"filter": {
"match": {
"sensor.type": 1
}
},
"aggs": {
"agg_by_type": {
"nested": {
"path": "sensor.values"
},
"aggs": {
"temperature_only": {
"filter": {
"match": {
"sensor.values.type": "TEMPERATURE"
}
},
"aggs": {
"max_value": {
"max": {
"field": "sensor.values.value"
}
},
"min_value": {
"min": {
"field": "sensor.values.value"
}
}
}
},
"humedity_only": {
"filter": {
"match": {
"sensor.values.type": "HUMIDITY"
}
},
"aggs": {
"max_value": {
"max": {
"field": "sensor.values.value"
}
},
"min_value": {
"min": {
"field": "sensor.values.value"
}
}
}
}
}
}
}
}
}
}
}
}
},
"from": 0,
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}
I need to split the data list into two separate idExists and idNotExists lists, based on whether id exists or not.
Can someone help me with Jolt Specification to achieve the below results?
Input:
{
"data": [
{
"name": "a",
"id": "100"
},
{
"name": "b",
"id": "101"
},
{
"name": "c"
}
]
}
Desired Output:
{
"IdExists": [
{
"name": "a",
"id": "100"
},
{
"name": "b",
"id": "101"
}
],
"IdNotExists": [
{
"name": "c"
}
]
}
I have tried with the below spec but it is not working as expected
[
{
"operation": "modify-default-beta",
"spec": {
"data": {
"": {
"id": "NotExist"
}
}
}
},
{
"operation": "shift",
"spec": {
"data": {
"": {
"id": {
"10*": {
"#2": "IdExists[]"
},
"*": {
"#2": "IdNotExists[]"
}
}
}
}
}
}
]
I had some modification on your spec, added * as it was missing in the selectors.
Modified shifting condition to NotExist and finally shift only name to the IdNotExists array.
[
{
"operation": "modify-default-beta",
"spec": {
"data": {
"*": {
"id": "NotExist"
}
}
}
},
{
"operation": "shift",
"spec": {
"data": {
"*": {
"id": {
"NotExist": {
"#(2,name)": "IdNotExists[].name"
},
"*": {
"#2": "IdExists[]"
}
}
}
}
}
}
]