JQ Error: Cannot iterate over string while trying to map IP and Ports - json

I have a below json output, I would love to remove duplicate data and map the data in a table format using jq. I am using the below query but I keep getting an error;
Cannot iterate over string ("78.45.196...)
e.t.c.
Json data
[
{
"ip": "78.45.196.23",
"timestamp": "1616566245",
"ports": [
{
"port": 5060,
"proto": "tcp",
"status": "open",
"reason": "syn-ack",
"ttl": 50
}
]
},
{
"ip": "67.89.378.82",
"timestamp": "1616566255",
"ports": [
{
"port": 2000,
"proto": "tcp",
"status": "open",
"reason": "syn-ack",
"ttl": 50
}
]
},
{
"ip": "67.89.378.82",
"timestamp": "1616566255",
"ports": [
{
"port": 2080,
"proto": "tcp",
"status": "open",
"reason": "syn-ack",
"ttl": 50
}
]
},
{
"ip": "78.45.196.23",
"timestamp": "1616566245",
"ports": [
{
"port": 5060,
"proto": "tcp",
"status": "open",
"reason": "syn-ack",
"ttl": 50
}
]
},
{
"ip": "67.89.378.82",
"timestamp": "1616566255",
"ports": [
{
"port": 2000,
"proto": "tcp",
"status": "open",
"reason": "syn-ack",
"ttl": 50
}
]
},
{
"ip": "78.45.196.23",
"timestamp": "1616566245",
"ports": [
{
"port": 5080,
"proto": "tcp",
"status": "open",
"reason": "syn-ack",
"ttl": 50
}
]
}
]
My query
jq -r '.[][] | group_by(.ip) | map({ip: .ip, ports: map(.ports[].port) | add | unique})' jsonfile.json
Expected output
To remove duplicates and get ip and ports.
Or can one explain to me how to get unique values from both IP and ports.
[
{"ip:" "67.89.378.82", "ports:"[2000, 2080]},
{"ip:" "78.45.196.23", "ports:"[5060, 5080]}
]

Construct your desired JSON immediately following the group_by()
group_by(.ip) |
map
(
{
ip: .[0].ip,
ports: [ .[].ports[].port ] | unique
}
)
jq play link
Follow-up question to discard IPs that have only port as 0
group_by(.ip) |
map
(
{
ip: .[0].ip,
ports: [ .[].ports[] | select(.port != 0 ).port ] | unique
} |
select(.ports | length > 0)
)

Related

JSON Schema - validate schema on root if property in another subschema contains a fix value

I would like to validate the following json:
{
"endpoints": [
{
"ip_name": "plantuml_ip",
"ip": "172.18.0.2",
"ports": [
{
"name": "plantuml_port",
"port": 8080,
"proto": "HTTPS_TERM"
}
]
}
],
"subdomains": [
{
"name": "plantuml_port",
"value": "plantuml"
}
]
}
If proto contains value HTTPS_TERM, then subdomains with the same name should exists (plantuml_port). Here is an another valid example:
Valid:
{
"endpoints": [{
"ip_name": "plantuml_ip",
"ip": "172.18.0.2",
"ports": [{
"name": "plantuml_port",
"port": 8080,
"proto": "HTTPS_TERM"
}, {
"name": "random_port",
"port": 8181,
"proto": "HTTPS_TERM"
}, {
"name": "no_subdomain",
"port": 999,
"proto": "NO_SUBDOMAIN"
}]
}],
"subdomains": [{
"name": "plantuml_port",
"value": "plantuml"
}, {
"name": "random_port",
"value": "random"
}]
}
It is something like this possible? Thanks in advance!

How to print nested JSON array data in a tabular format?

I want to read the status of clusters and servers inside it.
Below is the sample json file
"data": [{
"id": 7865,
"timeCreated": 1602589399294,
"timeUpdated": 1602748892149,
"name": "gw-ext-1",
"type": "CLUSTER",
"status": "RUNNING",
"multicastEnabled": false,
"primaryNodeId": 546,
"servers": [{
"id": 768,
"timeCreated": 1602589028419,
"timeUpdated": 1602747941321,
"name": "gw-jpg208765-1",
"type": "SERVER",
"serverType": "GATEWAY",
"status": "RUNNING",
"addresses": [{
"networkInterface": "eng123"
},
{
"networkInterface": "eng124"
}],
"clusterId": 098,
"clusterName": "gw-ext-1",
"currentClusteringPort": 897,
"runtimeInformation": {
"Information": {
"runtime": {
"name": "abctech",
"version": "1.6.8"
},
"specification": {
"vendor": "rrr",
"name": "rrrt",
"version": "1.8.89"
}
},
"osInformation": {
"name": "LX",
"version": "35",
"architecture": "klh"
},
"mExpirationDate": 098765589283662
}
},
{
"id": 876,
"timeCreated": 1602589007370,
"timeUpdated": 1602748894901,
"name": "gw-jpg208765-2",
"type": "SERVER",
"serverType": "GATEWAY",
"mVersion": "3.9.1",
"gaVersion": "3.9.1",
"agentVersion": "1.9.5",
"ExpirationDate": 32521996800000,
"ExpirationDate": 1665661007000,
"status": "DISCONNECTED",
"addresses": [{
"networkInterface": "engg"
},
{
"networkInterface": "engg"
}],
"clusterId": 768,
"clusterName": "gw-ext-1",
"serverPort": 987,
"currentClusteringPort": 987,
"runtimeInformation": {
"abcInfo": {
"runtime": {
"name": "abc",
"version": "1.2.3"
},
"specification": {
"vendor": "RRR",
"name": "RTR",
"version": "1.8.0"
}
},
"osInformation": {
"name": "LX",
"version": "4.78",
"architecture": "eng"
},
"ExpirationDate": 8765478999765
}
}],
"visibilityMap": {
"mapNodes": [{
"serverId": 765,
"visibleNodeIds": [765,
876],
"unknownNodeIps": []
},
{
"serverId": 876,
"visibleNodeIds": [765,
876],
"unknownNodeIps": []
}]
}
},
{
"id": 7865,
"timeCreated": 1602589399294,
"timeUpdated": 1602748892149,
"name": "gw-ext-2",
"type": "CLUSTER",
"status": "RUNNING",
"multicastEnabled": false,
"primaryNodeId": 546,
"servers": [{
"id": 768,
"timeCreated": 1602589028419,
"timeUpdated": 1602747941321,
"name": "gw-jpg208766-1",
"type": "SERVER",
"serverType": "GATEWAY",
"status": "RUNNING",
"addresses": [{
"networkInterface": "eng123"
},
{
"networkInterface": "eng124"
}],
"clusterId": 098,
"clusterName": "gw-ext-2",
"currentClusteringPort": 897,
"runtimeInformation": {
"Information": {
"runtime": {
"name": "abctech",
"version": "1.6.8"
},
"specification": {
"vendor": "rrr",
"name": "rrrt",
"version": "1.8.89"
}
},
"osInformation": {
"name": "LX",
"version": "35",
"architecture": "klh"
},
"mExpirationDate": 098765589283662
}
},
{
"id": 876,
"timeCreated": 1602589007370,
"timeUpdated": 1602748894901,
"name": "gw-jpg208766-2",
"type": "SERVER",
"serverType": "GATEWAY",
"mVersion": "3.9.1",
"gaVersion": "3.9.1",
"agentVersion": "1.9.5",
"ExpirationDate": 32521996800000,
"ExpirationDate": 1665661007000,
"status": "DISCONNECTED",
"addresses": [{
"networkInterface": "engg"
},
{
"networkInterface": "engg"
}],
"clusterId": 768,
"clusterName": "gw-ext-2",
"serverPort": 987,
"currentClusteringPort": 987,
"runtimeInformation": {
"abcInfo": {
"runtime": {
"name": "abc",
"version": "1.2.3"
},
"specification": {
"vendor": "RRR",
"name": "RTR",
"version": "1.8.0"
}
},
"osInformation": {
"name": "LX",
"version": "4.78",
"architecture": "eng"
},
"ExpirationDate": 8765478999765
}
}],
"visibilityMap": {
"mapNodes": [{
"serverId": 765,
"visibleNodeIds": [765,
876],
"unknownNodeIps": []
},
{
"serverId": 876,
"visibleNodeIds": [765,
876],
"unknownNodeIps": []
}]
}
}]
So in each cluster we have two servers and this json continues to have around 15 clusters.
I want to filter out the status of each cluster and server in below format
name cluster/server status
gw-ext-1 CLUSTER RUNNING
gw-jpg208765-1 SERVER RUNNING
gw-jpg208765-2 SERVER DISCONNECTED
similarly for other clusters also.
I tried few things but its not giving me the servers .. it gives only cluster's details
target_id=echo \$targetIdResponse | ${env.WORKSPACE}/jq -r '.data[] | [.name, .type, .status]'
OR
target_id=echo \$targetIdResponse | ${env.WORKSPACE}/jq -r '.data[] | [.name, .type, .status, .servers.name, .servers.type, .servers.status]'
where $targetIdResponse contains my json data
I want to know how i can filter the above json to get the required data.
You need to have the header array the required fields in a separate array and put them together in a tabular format using #tsv
jq -r '[ "name", "cluster/server", "status" ],
( .data[] | [.name, .type, .status] ),
( .data[].servers[] | [ .name, .type, .status ] ) | #tsv'
The requirement was modified since originally posted to have the server information exactly below the cluster information
jq -r '[ "name", "cluster/server", "status" ],
( .data[] | [.name, .type, .status], ( .servers[] | [.name, .type, .status] ) ) | #tsv'

How to avoid generating all combinations of selected data while constructing an object?

My original JSON is given below.
[
{
"id": "1",
"name": "AA_1",
"total": "100002",
"files": [
{
"filename": "8665b987ab48511eda9e458046fbc42e.csv",
"filename_original": "some.csv",
"status": "3",
"total": "100002",
"time": "2020-08-24 23:25:49"
}
],
"status": "3",
"created": "2020-08-24 23:25:49",
"filenames": "8665b987ab48511eda9e458046fbc42e.csv",
"is_append": "0",
"is_deleted": "0",
"comment": null
},
{
"id": "4",
"name": "AA_2",
"total": "43806503",
"files": [
{
"filename": "1b4812fe634938928953dd40db1f70b2.csv",
"filename_original": "other.csv",
"status": "3",
"total": "21903252",
"time": "2020-08-24 23:33:43"
},
{
"filename": "63ab85fef2412ce80ae8bd018497d8bf.csv",
"filename_original": "some.csv",
"status": "2",
"total": 0,
"time": "2020-08-24 23:29:30"
}
],
"status": "2",
"created": "2020-08-24 23:35:51",
"filenames": "1b4812fe634938928953dd40db1f70b2.csv&&63ab85fef2412ce80ae8bd018497d8bf.csv",
"is_append": "0",
"is_deleted": "0",
"comment": null
}
]
From this JSON I want to create new objects by combining fields from objects which have status: 2 and their files which also have the same pair, status: 2.
So, I am expecting a JSON array as below.
[
{
"id": "4",
"name": "AA_2",
"file_filename": "63ab85fef2412ce80ae8bd018497d8bf.csv",
"file_status": 2
}
]
So far I tried with this JQ filter:
.[]|select(.status=="2")|[{id:.id,file_filename:.files[].filename,file_status:.files[].status}]
But this produces some invalid data.
[
{
"id": "4", # want to remove this as file.status != 2
"file_filename": "1b4812fe634938928953dd40db1f70b2.csv",
"file_status": "3"
},
{
"id": "4",
"file_filename": "1b4812fe634938928953dd40db1f70b2.csv",
"file_status": "2"
},
{
"id": "4", # Repeat
"file_filename": "63ab85fef2412ce80ae8bd018497d8bf.csv",
"file_status": "3"
},
{
"id": "4", # Repeat
"file_filename": "63ab85fef2412ce80ae8bd018497d8bf.csv",
"file_status": "2"
}
]
How do I filter the new JSON using JQ and remove these duplicate objects?
By applying [] operator to files twice, you're running into a combinatorial explosion. That needs to be avoided, for example:
[ .[] | select(.status == "2") | {id, name} + (.files[] | select(.status == "2") | {file_filename: .filename, file_status: .status}) ]
Online demo

Is it possible to use jq to sort_by then sort_by again by sub item

I have the following json:
[
{
"SG": [
{
"Id": "17",
"GroupName": "fistGN",
"Permissions": [
{
"Port": 80,
"Protocol": "tcp"
},
{
"Port": 8080,
"Protocol": "tcp"
},
{
"Port": 5080,
"Protocol": "tcp"
}
]
},
{
"Id": "1",
"GroupName": "secondGN",
"Permissions": [
{
"Port": 80,
"Protocol": "tcp"
},
{
"Port": 8080,
"Protocol": "tcp"
},
{
"Port": 5080,
"Protocol": "tcp"
}
]
}
]
}
]
Is it possible to sort_by GroupName and then for each group by Permissions Port using a single command?
This is what I'm trying to do but that is not working as expected:
jq -s -S '.[].SG |= sort_by(.GroupName, .Permissions[].Port)' myfile.json
jq solution:
jq '.[0].SG |= (map(.Permissions |= sort_by(.Port)) | sort_by(.GroupName))' myfile.json
(map(.Permissions |= sort_by(.Port)) | sort_by(.GroupName)) - compound expression to separate 2 sort operations:
map(.Permissions |= sort_by(.Port)) - get a new array where each internal .Permissions array is sorted by key "Port" value
sort_by(.GroupName) - sort SG array items by "GroupName" key
The output:
[
{
"SG": [
{
"Id": "17",
"GroupName": "fistGN",
"Permissions": [
{
"Port": 80,
"Protocol": "tcp"
},
{
"Port": 5080,
"Protocol": "tcp"
},
{
"Port": 8080,
"Protocol": "tcp"
}
]
},
{
"Id": "1",
"GroupName": "secondGN",
"Permissions": [
{
"Port": 80,
"Protocol": "tcp"
},
{
"Port": 5080,
"Protocol": "tcp"
},
{
"Port": 8080,
"Protocol": "tcp"
}
]
}
]
}
]

Create array from multi-dimensional one with jq

I would like to use 'jq' json processor to transform a json structure to an array of simple objects.
My structure is like this:
{"nsgs": [
{
"comments": "text1",
"properties": {
"securityRules": [
{
"name": "1",
"properties": {
"protocol": "TCP",
"sourcePortRange": "*"
}
},
{
"name": "2",
"properties": {
"protocol": "UDP",
"sourcePortRange": "*"
}
}
]
}
},
{
"comments": "text2",
"properties": {
"securityRules": [
{
"name": "3",
"properties": {
"protocol": "TCP",
"sourcePortRange": "*"
}
},
{
"name": "4",
"properties": {
"protocol": "UDP",
"sourcePortRange": "*"
}
}
]
}
}
]}
And what I want to get is:
[
{ "comments": "text1",
"name": "1",
"protocol": "TCP",
"sourcePortRange": "*"
},
{ "comments": "text1",
"name": "2",
"protocol": "UDP",
"sourcePortRange": "*"
},
{ "comments": "text2",
"name": "3",
"protocol": "TCP",
"sourcePortRange": "*"
},
{ "comments": "text2",
"name": "4",
"protocol": "UDP",
"sourcePortRange": "*"
}
]
I tried lots of approaches but nothing helps.
Will appreciate any help.
Here is another solution:
.nsgs | map({comments} + (.properties.securityRules[] | {name}+.properties))
The following filter, laid out here for easy reading, will aggregate the input as requested:
.nsgs
| map(.comments as $comments
| .properties.securityRules[]
| {comments: $comments,
name,
protocol: .properties.protocol,
sourcePortRange: .properties.sourcePortRange } )
If you wanted to avoid the repetition in the last two lines, you could replace the last four lines with:
| {comments: $comments, name }
+ (.properties | {protocol, sourcePortRange} ) )