Chainlink multi-variable responses - results not getting appended - ethereum

I'm trying to implement a multi-variable job on my local node, however I'm finding that only 1 of the 3 results are being added to the chainlink_result_collection array in the resultcollect task. My definition is very similar to the example one linked, the main difference being the use of an external adapter
Running the above example smart contract and spec on my node works, and gives the following output. Take note of the chainlink_result_collection array being appended to after each run. Have linked the output file to stay under the 30k size limit for SO posts
However when I try to run my multi-variable job, I get the following output. Take note of the chainlink_result_collection after each resultcollect task. You can see each time it's being set to the latest return value, instead of being appended to like the previous job output. So the length of the chainlink_result_collection array at the end is 1 instead of 3 elements
{
"id": "c27b2627-3aeb-4acb-8b84-1e191854b25d",
"jobId": "332118053343434c8d519e4c3a8c3fef",
"result": {
"data": {
"result": "0x0000000000000000000000000000000000000000000000020000000000002060",
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"__chainlink_result_collection__": [
"0x0000000000000000000000000000000000000000000000020000000000002060"
]
},
"error": "insertEthTx failed while constructing EthTx data: number of collectors 2 != number of types in ABI encoding 4"
},
"status": "errored",
"taskRuns": [
{
"id": "5b7a842a-c5bb-47bb-86d6-9ffc9a8d6288",
"result": {
"data": {
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"roundAnswer": "36893488147419111519",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"laterRoundAnswer": "36893488147419111520",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"earlierRoundAnswer": "36893488147419111518"
},
"error": null
},
"status": "completed",
"task": {
"ID": 83,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "historical-price",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.169781+09:30",
"UpdatedAt": "2021-08-27T16:16:00.169781+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 3
},
{
"id": "ce1394bd-5d9e-4150-8fd2-7d251b69a81d",
"result": {
"data": {
"result": "36893488147419111519"
},
"error": null
},
"status": "completed",
"task": {
"ID": 84,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "copy",
"confirmations": null,
"params": {
"copyPath": [
"roundAnswer"
]
},
"CreatedAt": "2021-08-27T16:16:00.170343+09:30",
"UpdatedAt": "2021-08-27T16:16:00.170343+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "da32538d-b640-4bcb-8acb-fe9aafa344b0",
"result": {
"data": {
"result": "0x000000000000000000000000000000000000000000000002000000000000205f"
},
"error": null
},
"status": "completed",
"task": {
"ID": 85,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethuint256",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.170885+09:30",
"UpdatedAt": "2021-08-27T16:16:00.170885+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "372cdf12-8752-492c-9c2d-ff3354d878af",
"result": {
"data": {
"result": "0x000000000000000000000000000000000000000000000002000000000000205f",
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"__chainlink_result_collection__": [
"0x000000000000000000000000000000000000000000000002000000000000205f"
]
},
"error": null
},
"status": "completed",
"task": {
"ID": 86,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "resultcollect",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.171458+09:30",
"UpdatedAt": "2021-08-27T16:16:00.171458+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "0c8e18c2-c352-4af0-a282-a19fbcf03c2d",
"result": {
"data": {
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"roundAnswer": "36893488147419111519",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"laterRoundAnswer": "36893488147419111520",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"earlierRoundAnswer": "36893488147419111518"
},
"error": null
},
"status": "completed",
"task": {
"ID": 87,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "historical-price",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.172138+09:30",
"UpdatedAt": "2021-08-27T16:16:00.172138+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "48482e08-bed1-4956-8aa8-3ff6f2d9f38d",
"result": {
"data": {
"result": "36893488147419111518"
},
"error": null
},
"status": "completed",
"task": {
"ID": 88,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "copy",
"confirmations": null,
"params": {
"copyPath": [
"earlierRoundAnswer"
]
},
"CreatedAt": "2021-08-27T16:16:00.172638+09:30",
"UpdatedAt": "2021-08-27T16:16:00.172638+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "825ed16c-b662-4fb2-ac2d-abeee3259cfc",
"result": {
"data": {
"result": "0x000000000000000000000000000000000000000000000002000000000000205e"
},
"error": null
},
"status": "completed",
"task": {
"ID": 89,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethuint256",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.173151+09:30",
"UpdatedAt": "2021-08-27T16:16:00.173151+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "58ade305-bc91-4542-83b5-8b4e0018eebf",
"result": {
"data": {
"result": "0x000000000000000000000000000000000000000000000002000000000000205e",
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"__chainlink_result_collection__": [
"0x000000000000000000000000000000000000000000000002000000000000205e"
]
},
"error": null
},
"status": "completed",
"task": {
"ID": 90,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "resultcollect",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.173677+09:30",
"UpdatedAt": "2021-08-27T16:16:00.173677+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "d7982f84-03be-4f41-a6ad-ee487c84fc6b",
"result": {
"data": {
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"roundAnswer": "36893488147419111519",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"laterRoundAnswer": "36893488147419111520",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"earlierRoundAnswer": "36893488147419111518"
},
"error": null
},
"status": "completed",
"task": {
"ID": 91,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "historical-price",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.174159+09:30",
"UpdatedAt": "2021-08-27T16:16:00.174159+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "cf3a1caa-723e-4b2f-ae50-6096007dd02e",
"result": {
"data": {
"result": "36893488147419111520"
},
"error": null
},
"status": "completed",
"task": {
"ID": 92,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "copy",
"confirmations": null,
"params": {
"copyPath": [
"laterRoundAnswer"
]
},
"CreatedAt": "2021-08-27T16:16:00.174617+09:30",
"UpdatedAt": "2021-08-27T16:16:00.174617+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "53d76d5c-01b2-4e0d-8515-9f1a4901635b",
"result": {
"data": {
"result": "0x0000000000000000000000000000000000000000000000020000000000002060"
},
"error": null
},
"status": "completed",
"task": {
"ID": 93,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethuint256",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.175071+09:30",
"UpdatedAt": "2021-08-27T16:16:00.175071+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "1fb908bb-9539-4617-9395-4e5796e214c2",
"result": {
"data": {
"result": "0x0000000000000000000000000000000000000000000000020000000000002060",
"address": "0xb6EFEcE462EA6118A0A7EC1F2a3c7033b1F82967",
"dataPrefix": "0xc23231a259fa1d606b3b1f8d31d6b169af467af60dbf0ac4e0014eb49553ccc70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f08086217d2ab6f717fd7a12cd3c53a94efa6761b2cdfe81000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000061288b88",
"unixDateTime": "1625097600",
"functionSelector": "0x4ab0d190",
"aggregatorAddress": "0x9326BFA02ADD2366b30bacB125260Af641031331",
"__chainlink_result_collection__": [
"0x0000000000000000000000000000000000000000000000020000000000002060"
]
},
"error": null
},
"status": "completed",
"task": {
"ID": 94,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "resultcollect",
"confirmations": null,
"params": {
},
"CreatedAt": "2021-08-27T16:16:00.175515+09:30",
"UpdatedAt": "2021-08-27T16:16:00.175515+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
},
{
"id": "8763850e-80a6-4a1b-af04-aabfb69ff2c9",
"result": {
"data": {
},
"error": "insertEthTx failed while constructing EthTx data: number of collectors 2 != number of types in ABI encoding 4"
},
"status": "errored",
"task": {
"ID": 95,
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethtx",
"confirmations": 1,
"params": {
"abiEncoding": [
"bytes32",
"uint80",
"uint80",
"uint80"
]
},
"CreatedAt": "2021-08-27T16:16:00.175982+09:30",
"UpdatedAt": "2021-08-27T16:16:00.175982+09:30",
"DeletedAt": null
},
"minimumConfirmations": 3,
"confirmations": 1
}
],
"createdAt": "2021-08-27T16:16:52.308136+09:30",
"finishedAt": "2021-08-27T16:18:59.22246+09:30",
"updatedAt": "2021-08-27T16:18:59.259186+09:30",
"creationHeight": "26956178",
"observedHeight": "26956180",
"payment": "0",
"initiator": {
"id": 7,
"jobSpecId": "33211805-3343-434c-8d51-9e4c3a8c3fef",
"type": "runlog",
"params": {
"address": "0xb6efece462ea6118a0a7ec1f2a3c7033b1f82967"
}
},
"type": "Direct request job run"
}
This is my job definition:
{
"name": "multi-word2",
"initiators": [
{
"id": 7,
"jobSpecId": "33211805-3343-434c-8d51-9e4c3a8c3fef",
"type": "runlog",
"params": {
"address": "0xb6efece462ea6118a0a7ec1f2a3c7033b1f82967"
}
}
],
"tasks": [
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "historical-price"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "copy",
"params": {
"copyPath": [
"roundAnswer"
]
}
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethuint256"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "resultcollect"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "historical-price"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "copy",
"params": {
"copyPath": [
"earlierRoundAnswer"
]
}
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethuint256"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "resultcollect"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "historical-price"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "copy",
"params": {
"copyPath": [
"laterRoundAnswer"
]
}
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethuint256"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "resultcollect"
},
{
"jobSpecId": "332118053343434c8d519e4c3a8c3fef",
"type": "ethtx",
"confirmations": 1,
"params": {
"abiEncoding": [
"bytes32",
"uint80",
"uint80",
"uint80"
]
}
}
]
}

the issue is that the 'chainlink_result_collection' array is not passed through from the resultsCollect to the EA (external adapter) call, whereas it is from resultsCollect to a normal HTTP Get task. I verified this by creating a job that contains 3 requests (EA, HTTPGET, EA), and observed that at the resultsCollect for the HTTPGet, the array correctly had 2 values, however on the final request (EA), the array didn't get passed through to the EA task, and the end result was just the 1 return value from the final EA
There's an open issue on this at the Chainlink GitHub

Related

Flutter dynamic data from api to make Widgets

In my case the json will be different every time and i have to make widget like "inputtype": "dropdown", "inputtype": "radiobutton" but if i craete model it don't go as i wants it to.
{
"fields": [
{
"id": 31,
"name": "make",
"isrequired": "required",
"valuetype": "text",
"priority": 1,
"inputtype": "dropdown",
"max_min": [],
"rangeable": "false",
"choices": [
{
"id": 46,
"name": "Samsung",
"categoryextrafield_id": 31,
"created_at": "2021-12-29T01:30:47.000000Z",
"updated_at": "2021-12-29T01:30:47.000000Z",
"priority": 10
},
{
"id": 47,
"name": "Dell",
"categoryextrafield_id": 31,
"created_at": "2021-12-29T01:30:52.000000Z",
"updated_at": "2021-12-29T01:30:52.000000Z",
"priority": 20
},
{
"id": 48,
"name": "IBM",
"categoryextrafield_id": 31,
"created_at": "2021-12-29T01:31:09.000000Z",
"updated_at": "2021-12-29T01:31:09.000000Z",
"priority": 30
},
{
"id": 49,
"name": "Acer",
"categoryextrafield_id": 31,
"created_at": "2021-12-29T01:31:24.000000Z",
"updated_at": "2021-12-29T01:31:24.000000Z",
"priority": 40
}
],
"available": []
},
{
"id": 32,
"name": "model",
"isrequired": "required",
"valuetype": "text",
"priority": 2,
"inputtype": "textfield",
"max_min": [],
"rangeable": "false",
"choices": [],
"available": [
{
"model": "a51"
},
{
"model": "y9s"
},
{
"model": "a31"
},
{
"model": "yS10"
},
{
"model": "Y10S"
},
{
"model": "A551"
},
{
"model": "node8"
},
{
"model": "s9"
},
{
"model": null
},
{
"model": "2021"
},
{
"model": "2020"
},
{
"model": "2010"
},
{
"model": "Civic"
},
{
"model": "2019"
},
{
"model": "Daewooy9"
},
{
"model": "corei5"
},
{
"model": "corei9"
},
{
"model": "corei3"
},
{
"model": "corei11"
}
]
},
{
"id": 29,
"name": "features",
"isrequired": "required",
"valuetype": "text",
"priority": 3,
"inputtype": "checkbox",
"max_min": [],
"rangeable": "false",
"choices": [
{
"id": 41,
"name": "Bluetooth",
"categoryextrafield_id": 29,
"created_at": "2021-12-29T01:19:00.000000Z",
"updated_at": "2021-12-29T01:19:00.000000Z",
"priority": 1
},
{
"id": 42,
"name": "Fingerprint",
"categoryextrafield_id": 29,
"created_at": "2021-12-29T01:19:10.000000Z",
"updated_at": "2021-12-29T01:19:10.000000Z",
"priority": 10
},
{
"id": 43,
"name": "LedDisplay",
"categoryextrafield_id": 29,
"created_at": "2021-12-29T01:19:35.000000Z",
"updated_at": "2021-12-29T01:19:35.000000Z",
"priority": 15
}
],
"available": []
},
{
"id": 30,
"name": "condition",
"isrequired": "required",
"valuetype": "text",
"priority": 4,
"inputtype": "radiobutton",
"max_min": [],
"rangeable": "false",
"choices": [
{
"id": 44,
"name": "Used",
"categoryextrafield_id": 30,
"created_at": "2021-12-29T01:20:31.000000Z",
"updated_at": "2021-12-29T01:20:31.000000Z",
"priority": 10
},
{
"id": 45,
"name": "New",
"categoryextrafield_id": 30,
"created_at": "2021-12-29T01:20:38.000000Z",
"updated_at": "2021-12-29T01:20:38.000000Z",
"priority": 20
}
],
"available": []
}
]
}

JQ Select Outer Objects and Child Based on Child

I am trying to select the credentials object and specific parent level objects only if the key credhub-ref exist within the credential object following JSON:
{
"total_results": 23,
"total_pages": 1,
"prev_url": null,
"next_url": null,
"resources": [
{
"metadata": {
"guid": "cd",
"url": "/v2/service_bindings/cd",
"created_at": "2019-03-04T21:18:53Z",
"updated_at": "2019-03-04T21:18:53Z"
},
"entity": {
"app_guid": "9c",
"service_instance_guid": "66",
"credentials": {
"credhub-ref": "/c/54"
},
"binding_options": {},
"gateway_data": null,
"gateway_name": "",
"syslog_drain_url": null,
"volume_mounts": [],
"name": null,
"last_operation": {
"type": "create",
"state": "succeeded",
"description": "",
"updated_at": "2019-03-04T21:18:53Z",
"created_at": "2019-03-04T21:18:53Z"
},
"app_url": "/v2/apps/9c",
"service_instance_url": "/v2/service_instances/66",
"service_binding_parameters_url": "/v2/service_bindings/cd"
}
},
{
"metadata": {
"guid": "cd",
"url": "/v2/service_bindings/cd",
"created_at": "2019-03-11T08:49:22Z",
"updated_at": "2019-03-11T08:49:22Z"
},
"entity": {
"app_guid": "ea",
"service_instance_guid": "86",
"credentials": {},
"binding_options": {},
"gateway_data": null,
"gateway_name": "",
"syslog_drain_url": null,
"volume_mounts": [],
"name": null,
"last_operation": {
"type": "create",
"state": "succeeded",
"description": "",
"updated_at": "2019-03-11T08:49:22Z",
"created_at": "2019-03-11T08:49:22Z"
},
"app_url": "/v2/apps/ea",
"service_instance_url": "/v2/service_instances/86",
"service_binding_parameters_url": "/v2/service_bindings/cd"
}
},
{
"metadata": {
"guid": "e0",
"url": "/v2/service_bindings/e0",
"created_at": "2019-03-19T20:07:25Z",
"updated_at": "2019-03-19T20:07:25Z"
},
"entity": {
"app_guid": "73",
"service_instance_guid": "52",
"credentials": {
"hostname": "10.13.7.64",
"port": 3306,
"name": "cf_52",
"username": "w",
"password": "w",
"uri": "mysql://",
"jdbcUrl": "jdbc:mysql://10.193.78.64:"
},
"binding_options": {},
"gateway_data": null,
"gateway_name": "",
"syslog_drain_url": null,
"volume_mounts": [],
"name": null,
"last_operation": {
"type": "create",
"state": "succeeded",
"description": "",
"updated_at": "2019-03-19T20:07:25Z",
"created_at": "2019-03-19T20:07:25Z"
},
"app_url": "/v2/apps/73",
"service_instance_url": "/v2/service_instances/52",
"service_binding_parameters_url": "/v2/service_bindings/e0"
}
},
{
"metadata": {
"guid": "18",
"url": "/v2/service_bindings/18",
"created_at": "2019-03-19T20:07:27Z",
"updated_at": "2019-03-19T20:07:27Z"
},
"entity": {
"app_guid": "73",
"service_instance_guid": "ae",
"credentials": {
"credhub-ref": "/c/54"
},
"binding_options": {},
"gateway_data": null,
"gateway_name": "",
"syslog_drain_url": null,
"volume_mounts": [],
"name": null,
"last_operation": {
"type": "create",
"state": "succeeded",
"description": "",
"updated_at": "2019-03-19T20:07:27Z",
"created_at": "2019-03-19T20:07:27Z"
},
"app_url": "/v2/apps/73",
"service_instance_url": "/v2/service_instances/aec",
"service_binding_parameters_url": "/v2/service_bindings/18"
}
}
]
}
The current query only returns the credentials object:
jq '.resources[].entity.credentials | select(.["credhub-ref"])' test.json
Results in:
{
"credhub-ref": "/c/42"
}
{
"credhub-ref": "/c/54"
}
I need a query that outputs the parent objects app_guid and service_instance_guid too like the following:
}
"app_guid": "9c",
"service_instance_guid": "66",
"credentials": {
"credhub-ref": "/c/54"
},
{
"app_guid": "73",
"service_instance_guid": "ae",
"credentials": {
"credhub-ref": "/c/54"
}
A few good examples of how this can be done would go a long way for helping me better understand how JQ works.
This is one of many ways way to get the expected output:
jq '.resources[].entity | select(.credentials."credhub-ref") | {app_guid, service_instance_guid, credentials}' test.json
It will search for .credentials.credhub-ref in every entity object and will return app_guid, service_instance_guid, credentials as object if found.
Output:
{
"app_guid": "9c",
"service_instance_guid": "66",
"credentials": {
"credhub-ref": "/c/54"
}
}
{
"app_guid": "73",
"service_instance_guid": "ae",
"credentials": {
"credhub-ref": "/c/54"
}
}

Filtering output to needed values in json

I have below output generated as part of gathering all attributes about openstack VM. I am trying to work a json filter, which would let me select just values of interest. (tenant name, tenant ID or cloud name and so on).
I am new to filters, so all the help is appreciated. I tried different ways but not close to what i want.
N_ID": true,
"NAME_ATTR": "name",
"OS-DCF:diskConfig": "AUTO",
"OS-EXT-AZ:availability_zone": "cloud-b",
"OS-EXT-STS:power_state": 1,
"OS-EXT-STS:task_state": null,
"OS-EXT-STS:vm_state": "active",
"OS-SRV-USG:launched_at": "2018-06-05T22:45:50.000000",
"OS-SRV-USG:terminated_at": null,
"accessIPv4": "192.168.0.1",
"accessIPv6": "",
"addresses": {
"tenant-internal-direct-net": [
{
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:73:c6:9e",
"OS-EXT-IPS:type": "fixed",
"addr": "192.168.0.1",
"version": 4
}
]
},
"az": "cloud-1-b",
"cloud": "cloud-1",
"config_drive": "",
"created": "2018-06-05T22:45:37Z",
"flavor": {
"id": "15",
"name": "2vCPUx8GB"
},
"hostId": "f12acb7a6a4050672a7710da4ccb5987916ac46648fc667496c4a169",
"human_id": "myhost-ost1",
"id": "ae12d6cd-c9b8-4908-a0f9-6af614f01197",
"image": {
"id": ""
},
"interface_ip": "192.168.0.1",
"key_name": "pz-rhel7",
"metadata": {},
"name": "myhost-ost1",
"networks": {
"tenant-internal-direct-net": [
"192.168.0.1"
]
},
"os-extended-volumes:volumes_attached": [
{
"id": "179749a7-ba9b-4cc1-aebd-dcc8dcb7f4e1"
}
],
"private_v4": "",
"progress": 0,
"public_v4": "192.168.0.1",
"public_v6": "",
"region": "cloud-1",
"request_ids": [],
"security_groups": [
{
"description": "",
"id": "55991c66-beb5-4750-b8f6-06025a1ce7d4",
"name": "SSHping",
"security_group_rules": [
{
"direction": "ingress",
"ethertype": "IPv4",
"id": "53fa8fca-d54c-4462-9f5e-a748fed86885",
"port_range_max": null,
"port_range_min": null,
"protocol": "icmp",
"remote_ip_prefix": "0.0.0.0/0",
"security_group_id": "55991c66-beb5-4750-b8f6-06025a1ce7d4"
},
{
"direction": "ingress",
"ethertype": "IPv4",
"id": "841446f5-a7bf-46e9-91c2-f505c5842bc6",
"port_range_max": 22,
"port_range_min": 22,
"protocol": "tcp",
"remote_ip_prefix": "0.0.0.0/0",
"security_group_id": "55991c66-beb5-4750-b8f6-06025a1ce7d4"
}
]
},
{
"description": "Default security group",
"id": "987efb61-d7f3-47d5-9362-42f66d1b104b",
"name": "default",
"security_group_rules": [
{
"direction": "ingress",
"ethertype": "IPv4",
"id": "79da1cc2-2128-4937-95ca-d0fc757bbfb3",
"port_range_max": 22,
"port_range_min": 22,
"protocol": "tcp",
"remote_ip_prefix": "0.0.0.0/0",
"security_group_id": "987efb61-d7f3-47d5-9362-42f66d1b104b"
},
{
"direction": "ingress",
"ethertype": "IPv4",
"id": "901d230e-9cd5-486c-aa89-c279b359fac9",
"port_range_max": 1,
"port_range_min": 1,
"protocol": "tcp",
"remote_ip_prefix": "0.0.0.0/0",
"security_group_id": "987efb61-d7f3-47d5-9362-42f66d1b104b"
},
{
"direction": "ingress",
"ethertype": "IPv4",
"id": "950c6b8c-e129-4138-8dd5-92254c5bd6c8",
"port_range_max": null,
"port_range_min": null,
"protocol": null,
"remote_ip_prefix": null,
"security_group_id": "987efb61-d7f3-47d5-9362-42f66d1b104b"
},
{
"direction": "ingress",
"ethertype": "IPv4",
"id": "cf20d16d-0b8a-4bd7-b6b4-cab6d5968a44",
"port_range_max": null,
"port_range_min": null,
"protocol": null,
"remote_ip_prefix": null,
"security_group_id": "987efb61-d7f3-47d5-9362-42f66d1b104b"
}
]
}
],
"status": "ACTIVE",
"tenant_id": "99f1f837a8004092926887d46188e726",
"updated": "2018-06-05T22:45:50Z",
"user_id": "35f6e9300037f9c6d23f08c5af8b324c4ab23688d2a146bd6aac601377ed72ef",
"volumes": [
{
"HUMAN_ID": false,
"NAME_ATTR": "name",
"attachments": [
{
"attached_at": "2018-06-05T22:45:43.000000",
"attachment_id": "4602ad6b-8652-4702-910b-e4e82a894a32",
"device": "/dev/vda",
"host_name": null,
"id": "179749a7-ba9b-4cc1-aebd-dcc8dcb7f4e1",
"server_id": "ae12d6cd-c9b8-4908-a0f9-6af614f01197",
"volume_id": "179749a7-ba9b-4cc1-aebd-dcc8dcb7f4e1"
}
],
"availability_zone": "nova",
"bootable": true,
"consistencygroup_id": null,
"created_at": "2018-06-05T22:45:39.000000",
"description": "",
"device": "/dev/vda",
"display_description": "",
"display_name": "",
"encrypted": false,
"human_id": null,
"id": "179749a7-ba9b-4cc1-aebd-dcc8dcb7f4e1",
"links": [
{
"href": "http://cloud-1.mycloud.com:8776/v2/99f1f837a8004092926887d46188e726/volumes/179749a7-ba9b-4cc1-aebd-dcc8dcb7f4e1",
"rel": "self"
},
{
"href": "http://cloud-1.mycloud.com:8776/99f1f837a8004092926887d46188e726/volumes/179749a7-ba9b-4cc1-aebd-dcc8dcb7f4e1",
"rel": "bookmark"
}
],
"metadata": {
"attached_mode": "rw",
"readonly": "False"
},
"multiattach": false,
"name": "",
"os-vol-tenant-attr:tenant_id": "99f1f837a8004092926887d46188e726",
"replication_status": "disabled",
"request_ids": [],
"size": 11,
"snapshot_id": null,
"source_volid": null,
"status": "in-use",
"updated_at": "2018-06-05T22:45:43.000000",
"user_id": "35f6e9300037f9c6d23f08c5af8b324c4ab23688d2a146bd6aac601377ed72ef",
"volume_image_metadata": {
"checksum": "2d3f40fa841b1c35c10e74207d1221f8",
"container_format": "bare",
"disk_format": "raw",
"hw_disk_bus": "scsi",
"hw_qemu_guest_agent": "yes",
"hw_scsi_model": "virtio-scsi",
"hw_vif_multiqueue_enabled": "true",
"hw_watchdog_action": "reset",
"image_id": "fe463cc5-fecf-4aad-a77e-70987f421169",
"image_name": "RHEL-7_4-MG",
"min_disk": "0",
"min_ram": "0",
"size": "10737418240"
},
"volume_type": "ceph-standard",
"x_openstack_request_ids": []
}
],
"x_openstack_request_ids": []
}

Logstash nested array JSON parsing

I have a JSON log of my application which contains elements in a nested array form, here is the sample of it:-
{
"msgs": [{
"ts": "2017-09-04T07:07:45.6229372Z",
"tid": 25,
"eid": 1,
"lvl": "Information",
"cat": "Microsoft.AspNetCore.Hosting.Internal.WebHost",
"msg": {
"cnt": "Request starting HTTP/1.1 POST http://localhost:20001/Processor text/xml; charset=utf-8 685",
"Protocol": "HTTP/1.1",
"Method": "POST",
"ContentType": "text/xml; charset=utf-8",
"ContentLength": 685,
"Scheme": "http",
"Host": "localhost:20001",
"PathBase": "",
"Path": "/Processor",
"QueryString": ""
}
},
{
"ts": "2017-09-04T07:07:45.6229372Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.Service.ServiceHostMiddleware",
"msg": {
"cnt": "REQ"
},
"data": {
"Headers": {
"Connection": "Keep-Alive",
"Content-Length": "685",
"Content-Type": "text/xml; charset=utf-8",
"Accept-Encoding": "gzip, deflate",
"Expect": "100-continue",
"Host": "localhost:20001",
"SOAPAction": "\"http://servereps.mtxeps.com/TransactionService/SendTransaction\""
}
}
},
{
"ts": "2017-09-04T07:07:45.6239372Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.ProcessorTransactionService",
"msg": {
"cnt": "REQ"
},
"data": {
"Trace": "Aa10031<1C>Ab1<1C>Ac000101<1C>Ad20170616145857<1C>Ae10011<1C>AhVT00009<1C>ArY<1C>Be1812<1C>BfS<1C>BnDB<1C>Bo400296<1C>Bp4803<1C>BqDebit Card<1C>Br16<1C>Da300<1C>Dc0<1C>Dk840<1C>Ga01<1C>Gb1<1C>GfG<1C>GhB/GsF>O<1C>GkOE1<1C>Ia7325980B6B284759<1C>Ib8765432100324A00313<1C>Ic0043<1C>Ig3|1|MX2015-06<1C>IjgnkA7MdNlE2EB1c2B3jlz1G3Kf2U5S3x2H9W6ldIY7QpmlUzYcNHGg==|FFFF987654323D2007CD<1C>Nc5<1C>Oa18"
}
},
{
"ts": "2017-09-04T07:07:45.6249373Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
"msg": {
"cnt": "REQ"
},
"data": {
"request": {
"Version": "1.0",
"Server": {
"LogicalDatacenterId": 0,
"PhysicalDatacenterId": 0,
"UniqueId": "G00008D4F35EDADCD368",
"UniversalTimestamp": "2017-09-04T07:07:45.6249373Z"
},
"State": {
"Status": "InFlight"
},
"Tenant": {
"CompanyNumber": 10031,
"StoreNumber": 1,
"HostType": {
"Code": 7,
"Name": "ProdConcordHC"
},
"MerchantNumber": "",
"TerminalId": ""
},
"PointOfInteraction": {
"Client": {
"Type": "OpenEPS",
"CardType": "DB",
"CardName": "Debit Card",
"CommandSequence": "B/GsF>O"
},
"Lane": {
"Type": "Attended",
"Number": 1
},
"Terminal": {
"EMVCapabilities": "FullEMV",
"SerialNumber": "0043"
},
"PointOfSale": {
"ReferenceNumber": "VT00009",
"CashierNumber": "1"
},
"LocalTimestamp": "2017-06-16T14:58:57"
},
"Request": {
"MessageType": {
"Code": 0,
"Name": "Standard"
},
"TenderType": {
"Code": 1,
"Name": "Debit"
},
"TransactionType": {
"Code": 1,
"Name": "Purchase"
},
"EntryMode": "Swiped",
"AuditId": 10011,
"CardType": {
"TenderType": {
"Code": 0,
"Name": "Unknown"
},
"Code": 0,
"Name": "Unknown"
},
"AccountNumberFirstSix": "400296",
"AccountNumberLastFour": "4803",
"AccountNumberLength": 16,
"Card": {
"EncryptedTrack": {
"EncryptionType": "Hardware",
"EncryptionKeySerialNumber": "FFFF987654323D2007CD",
"EncryptedValue": "gnkA7MdNlE2EB1c2B3jlz1G3Kf2U5S3x2H9W6ldIY7QpmlUzYcNHGg=="
},
"EncryptedPIN": {
"EncryptionType": "PassThrough",
"EncryptionKeySerialNumber": "8765432100324A00313",
"EncryptedValue": "7325980B6B284759"
}
},
"Currency": {
"Number": 840
},
"Amount": 3.0,
"CashbackAmount": 0.0,
"AllowPartialAuthorization": true,
"Host": {
"HostType": {
"Code": 7,
"Name": "ProdConcordHC"
},
"Values": {
"HostProfileId": 1,
"CompanyNumber": 10031,
"StoreNumber": 1,
"StoreProfileId": 1,
"NumericStateCode": 0,
"AlphaStateCode": "NH",
"CheckAuthService": "3"
}
}
},
"Trace": {
"Path": [{
"Type": "OpenEPS",
"Value": "OE1"
},
{
"NodeId": 0,
"Type": "Gateway",
"Value": "GAT"
},
{
"NodeId": 0,
"ElapsedSeconds": 0.7458587,
"Type": "Tenant",
"Value": "TEN"
}]
}
},
"timeoutInMilliseconds": 45000
}
},
{
"ts": "2017-09-04T07:07:45.6259373Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TenantResolutionClient",
"msg": {
"cnt": "SEND"
},
"data": {
"RequestUri": "http://153.71.66.148:20002/TenantConfigurationService/v1/Resolution",
"Method": "POST",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:45 GMT",
"User-Agent": "ConnectedPayments/1.0",
"x-ms-request-root-id": "6627de44-43a027c7567ddccd",
"x-ms-request-id": "|6627de44-43a027c7567ddccd.1.",
"Request-Id": "|6627de44-43a027c7567ddccd.1.",
"Correlation-Context": "UniqueId=G00008D4F35EDADCD368",
"Content-Type": "application/json; charset=utf-8",
"Content-Length": "181"
}
}
},
{
"ts": "2017-09-04T07:07:46.37098Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TenantResolutionClient",
"msg": {
"cnt": "RECV"
},
"data": {
"StatusCode": 200,
"ReasonPhrase": "OK",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:46 GMT",
"Transfer-Encoding": "chunked",
"X-Unique-Id": "G00008D4F35EDADCD368",
"X-Node-Id": "0",
"Content-Type": "application/json; charset=utf-8"
}
}
},
{
"ts": "2017-09-04T07:07:46.37098Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
"msg": {
"cnt": "Resolution performed."
},
"data": {
"Response": {
"Content": {
"Company": {
"Number": 10031,
"Name": "Sprouts"
},
"TransactionProfile": {
"TransactionProfileId": 1,
"CompanyNumber": 10031,
"Number": 1000,
"Description": "Transaction Profile 1000"
},
"TransactionValidation": {
"TransactionValidationId": 1,
"TransactionProfileId": 1,
"MessageTypeCode": 0,
"TenderTypeCode": 1,
"TransactionTypeCode": 1,
"CardTypeCode": 0
},
"Store": {
"Number": 1,
"Name": "Store 1",
"QueueLaneNumber": 0
},
"StoreProfile": {
"StoreProfileId": 1,
"CompanyNumber": 10031,
"StoreNumber": 1,
"TransactionProfileId": 1,
"HostTypeCode": 7,
"MerchantNumber": "",
"TerminalId": ""
},
"HostProfile": {
"HostProfileId": 1,
"CompanyNumber": 10031,
"StoreNumber": 1,
"StoreProfileId": 1,
"NumericStateCode": 0,
"AlphaStateCode": "NH",
"CheckAuthService": "3"
}
},
"NodeId": 0,
"StatusCode": 200
},
"ElapsedSeconds": 0.7458587
}
},
{
"ts": "2017-09-04T07:07:46.37198Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TransactionAuthorizationClient",
"msg": {
"cnt": "SEND"
},
"data": {
"RequestUri": "http://ser22vvm211:8082/OrchestrationService/v1/Authorization",
"Method": "POST",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:46 GMT",
"User-Agent": "ConnectedPayments/1.0",
"x-ms-request-root-id": "6627de44-43a027c7567ddccd",
"x-ms-request-id": "|6627de44-43a027c7567ddccd.2.",
"Request-Id": "|6627de44-43a027c7567ddccd.2.",
"Correlation-Context": "UniqueId=G00008D4F35EDADCD368",
"Content-Type": "application/json; charset=utf-8",
"Content-Length": "1821"
}
}
},
{
"ts": "2017-09-04T07:07:49.8941815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TransactionAuthorizationClient",
"msg": {
"cnt": "RECV"
},
"data": {
"StatusCode": 200,
"ReasonPhrase": "OK",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:49 GMT",
"Server": "Apache-Coyote/1.1",
"Content-Length": "1672",
"Content-Type": "application/json"
}
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
"msg": {
"cnt": "RSP"
},
"data": {
"ResponseTransaction": {
"Version": null,
"Server": {
"LogicalDatacenterId": 0,
"PhysicalDatacenterId": 0,
"TransactionId": 10909,
"UniqueId": "G00008D4F35EDADCD368",
"UniversalTimestamp": "2017-09-04T07:07:45.6249373Z"
},
"State": null,
"Tenant": {
"CompanyNumber": 10031,
"StoreNumber": 1,
"HostType": {
"Code": 7,
"Name": "ProdConcordHC"
},
"MerchantNumber": "",
"TerminalId": ""
},
"PointOfInteraction": {
"Client": {
"Type": "B/GsF>O",
"CardType": "DB",
"CardName": "Debit Card",
"CommandSequence": "B/GsF>O",
"Version": null
},
"Lane": {
"Type": "Attended",
"Number": 1
},
"Terminal": {
"EMVCapabilities": "Contact, ContactlessDisabled",
"Type": null,
"SerialNumber": null,
"EMVKernelVersion": null,
"EMVIdentifierCAPK": null,
"PINCapabilities": null
},
"PointOfSale": {
"CashierNumber": "1",
"ReferenceNumber": null
},
"LocalTimestamp": "2017-06-16T14:58:57",
"LocalTimeZoneOffset": null
},
"Request": {
"MessageType": null,
"EntryMode": null,
"CashbackAmount": 0.00,
"AllowPartialAuthorization": true,
"TenderType": null,
"TransactionType": null,
"ReversalType": null,
"ReferenceId": null,
"AuditId": null,
"AccountNumberFirstSix": null,
"AccountNumberLastFour": null,
"AccountNumberLength": null,
"CardType": null,
"PreviousTransaction": null,
"Card": null,
"Check": null,
"Identification": null,
"Currency": null,
"Amount": null,
"TipAmount": null,
"Host": null
},
"Response": {
"ErrorCode": 96,
"ErrorMessage": "E2G",
"IsApproved": false,
"ResponseCode": "96",
"HostResponseCode": "E2G",
"HostResponseMessage": "EDIT ER:OPT DATA",
"ApprovedAmount": 0.00,
"ApprovedCashbackAmount": 0.0,
"IsApprovedLocally": false,
"AuthorizationCode": null,
"ResponseMessage": null,
"BalanceAmount": null,
"Check": null,
"Currency": null,
"Card": null,
"Host": null
},
"Trace": {
"ElapsedSeconds": 4.2694351,
"Path": [{
"NodeId": 0,
"ElapsedSeconds": 4.2694351,
"Type": "Gateway",
"Value": "GAT"
}]
},
"Queue": null,
"SensitiveData": null,
"PreviousTransaction": null,
"Trace": null
}
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.ProcessorTransactionService",
"msg": {
"cnt": "RSP"
},
"data": {
"Trace": "Aa10031<1C>Ab1<1C>Ac000101<1C>Ad20170616145857<1C>Ae10011<1C>Af96<1C>ArY<1C>BnDB<1C>BqDebit Card<1C>Db0<1C>Dc0<1C>Ga1<1C>Gb1<1C>GfG<1C>GhB/GsF>O<1C>GkGAT<1C>Jb96<1C>Ka7<1C>MbE2G<1C>Mg0<1C>Nc5<1C>Ya10909<1C>Yb0<1C>YcG00008D4F35EDADCD368"
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.Service.ServiceHostMiddleware",
"msg": {
"cnt": "RSP"
},
"data": {
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:49 GMT",
"Transfer-Encoding": "chunked",
"Content-Type": "text/xml; charset=utf-8",
"X-Unique-Id": "G00008D4F35EDADCD368",
"X-Node-Id": "0"
}
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"eid": 2,
"lvl": "Information",
"cat": "Microsoft.AspNetCore.Hosting.Internal.WebHost",
"msg": {
"cnt": "Request finished in 4272.8738ms 200 text/xml; charset=utf-8",
"ElapsedMilliseconds": 4272.8738,
"StatusCode": 200,
"ContentType": "text/xml; charset=utf-8"
}
}],
"RequestId": "G00008D4F35EDADCD368",
"RequestPath": "/Processor",
"Action": "http://servereps.mtxeps.com/TransactionService/SendTransaction",
"Contract": "NCR.CP.GatewayService.IProcessorTransactionContract",
"OperationName": "SendTransaction",
"MethodName": "SendTransaction"
}
Here "msgs" element contains nested array object into it, so to create field of every elements inside the array object i had followed the mutate and split approach both:-
Mutate filter:-
mutate {
add_field => {
"ts" => "%{[doc][msgs][0][ts]}"
"tid1" => "%{[doc][msgs][0][tid]}"
"eid1" => "%{[doc][msgs][0][eid]}"
"lvl1" => "%{[doc][msgs][0][lvl]}"
"cat1" => "%{[doc][msgs][0][cat]}"
"msg1" => "%{[doc][msgs][0][msg]}"
"data" => "%{[doc][msgs][1][data]}"
"actual-message" =>"%{[doc][msgs][3][data][Trace]}"
"error" =>"%{[doc][msgs][5][ex][exs][0][ec]}"
"error-type" =>"%{[doc][msgs][5][ex][exs][0][typ]}"
}
}
This approach is static as my logs is occurring dynamically i.e position of error is not fixed as it may come in 6th array element or may be at 7th or 8th,
so i ignored this approach and followed the split filter approach:-
Split Filter in logstash config:-
input{
file{
path=>"C:\Logs\GatewayService\GatewayService-Processor.Transactions-20170830.slog"
}
}
split {
field=>"msgs"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "g_index"
}
}
now it is showing error as:
[2017-09-19T17:40:07,557][WARN ][logstash.filters.split ] Only String and Array types are splittable. field:msg is of type = NilClass
so how should i able to retrieve every element in this JSON to be a field so as to visualize in kibana by applying metrics in dashboard.

Convert json to include name and children values to feed D3

I am trying to get json converted from:
{
"Devices": [
{
"Udid": "7a2b0e6c928f2321a75e423ba23ae93d",
"SerialNumber": "RF1D232ZLEE",
"MacAddress": "40F232726FC8",
"Imei": "3576342323280150",
"EasId": "SEC1BC252327E92B",
"AssetNumber": "7a2b0e23223928f2321a75e423ba23ae93d",
"DeviceFriendlyName": "gel1 Android Android 5.0.1 ZLEE ",
"LocationGroupId": {
"Id": {
"Value": 19529
},
"Name": "Group Express"
},
"LocationGroupName": "Group Express",
"UserId": {
"Name": ""
},
"UserName": "",
"UserEmailAddress": "",
"Ownership": "S",
"PlatformId": {
"Id": {
"Value": 5
},
"Name": "Android"
},
"Platform": "Android",
"ModelId": {
"Id": {
"Value": 5
},
"Name": "samsung GT-I9505"
},
"Model": "samsung GT-I9505",
"OperatingSystem": "5.0.1",
"PhoneNumber": "+447881867010",
"LastSeen": "2016-07-06T14:01:03.590",
"EnrollmentStatus": "Unenrolled",
"ComplianceStatus": "NotAvailable",
"CompromisedStatus": false,
"LastEnrolledOn": "2016-06-15T16:01:38.763",
"LastComplianceCheckOn": "0001-01-01T00:00:00.000",
"LastCompromisedCheckOn": "2016-07-06T13:58:26.183",
"IsSupervised": false,
"DeviceMCC": {
"SIMMCC": "234",
"CurrentMCC": "234"
},
"AcLineStatus": 0,
"VirtualMemory": 0,
"Id": {
"Value": 23459
}
},
{
"Udid": "c5f94db71d406dae7f881d3edf059e",
"SerialNumber": "",
"MacAddress": "000C300F9108",
"Imei": "",
"EasId": "D80DB85EC411C8E9B28BC292A603F05C2C0EEEC8",
"AssetNumber": "c592f93db71d406dae7f881d3edf059e",
"DeviceFriendlyName": "user Windows 10 WinRT 10.0.10240 ",
"LocationGroupId": {
"Id": {
"Value": 18498
},
"Name": "Business Solutions"
},
"LocationGroupName": "Business Solutions",
"UserId": {
"Name": ""
},
"UserName": "",
"UserEmailAddress": "",
"Ownership": "C",
"PlatformId": {
"Id": {
"Value": 12
},
"Name": "WinRT"
},
"Platform": "WinRT",
"ModelId": {
"Id": {
"Value": 50
},
"Name": "Windows 10"
},
"Model": "Windows 10",
"OperatingSystem": "10.0.10240",
"PhoneNumber": "",
"LastSeen": "2016-05-03T10:54:07.650",
"EnrollmentStatus": "Unenrolled",
"ComplianceStatus": "NotAvailable",
"CompromisedStatus": false,
"LastEnrolledOn": "2016-01-29T16:41:57.760",
"LastComplianceCheckOn": "0001-01-01T00:00:00.000",
"LastCompromisedCheckOn": "0001-01-01T00:00:00.000",
"IsSupervised": false,
"DeviceMCC": {
"SIMMCC": "",
"CurrentMCC": ""
},
"AcLineStatus": 0,
"VirtualMemory": 0,
"Id": {
"Value": 23545
}
}
],
"Page": 0,
"PageSize": 500,
"Total": 13}
To something like:
{"name": "Devices",
"children": [
{"name":"Udid", "size":"7f0dsda63274692ea4f0b66fec67a020158"},
{"name":"SerialNumber", "size":"P988KJSPQF938"},
{"name":"MacAddress", "size":"1HJUSUD031C4"},
{"name":"Imei", "size":""},
{"name":"EasId", "size":"ApKJSPQF193"},
{"name":"AssetNumber", "size":"7f0cda636b3305fea4f0b66fec9997267a020158"},
{"name":"DeviceFriendlyName", "size":"TMcKenz iPad iOS 7.1.4 F193 "},
{"name":"LocationGroupId",
"children": [
{"name":"Id","size":7488},
{"name":"Name","size":"MCM"}
]
},
{"name":"UserId",
"children": [
{"name":"Id","size":6418},
{"name":"Name","size":"Tom McKenz"}
]
},
{"name":"UserName", "size":"TMcKenz"},
{"name":"UserEmailAddress", "size":"TMcKenz#awaw.com"}
]
}
Not sure what is the best practice here: is it possible to use D3.nest or do you need to iterate through all the nodes and change to 'name' and 'children' accordingly.