Change json object with jq - json

how can I update servers object in my json using jq?
"tags": [],
"servers": [],
"components": {
"securitySchemes": {
"cookie": {
"type": "apiKey",
"in": "cookie",
"name": "store",
"description": "Store API"
}
},
I need to have servers like that:
"servers": [
{
"url": "http://localhost:5000",
"description": "Localhost server"
}
],


echo '{"tags": [],
"servers": [],
"components": {
"securitySchemes": {
"cookie": {
"type": "apiKey",
"in": "cookie",
"name": "store",
"description": "Store API"
}
}
}}' | jq '.servers = [{url:"http://localhost:5000", description: "Localhost server"}]'
Or, for a fixed json file (adding missing }):
jq '.servers = [{url:"http://localhost:5000", description: "Localhost server"}]' file.json

Related

Unable to parse JSON data for Oracle Integration Cloud using JQ

need help to parse the JSON data received from Oracle Integration Cloud. The expected output is mentioned below alongwith the command i am trying to use.
JQ command
jq '[{id: .id},{integrations: [.integrations[]|{code: .code, version: .version, dependencies: .dependencies|{connections: .connections[]|{id: .id, status: .status}}, .dependencies|{lookups: .lookups}}]}]' output.json
Error :
jq: error: syntax error, unexpected FIELD (Unix shell quoting issues?) at , line 1:
[{id: .id},{integrations: [.integrations[]|{code: .code, version: .version, dependencies: .dependencies|{connections: .connections[]|{id: .id, status: .status}}, .dependencies|{lookups: .lookups}}]}]
Note : If i run below command to fetch only connections data it works fine
jq '[{id: .id},{integrations: [.integrations[]|{code: .code, version: .version, dependencies: .dependencies|{connections: .connections[]|{id: .id, status: .status}}}]}]' output.json
Expected Output:
[
{
"id": "SAMPLE_PACKAGE"
},
{
"integrations": [
{
"code": "HELLO_INTEGRATION",
"version": "01.00.0000",
"dependencies": {
"connections": {
"id": "HELLO_WORLD1",
"status": "CONFIGURED"
}
}
},
{
"code": "HELLO_INTEGRATIO_LOOKUP",
"version": "01.00.0000",
"dependencies": {
"connections": {
"id": "HELLO_WORLD1",
"status": "CONFIGURED"
},
"lookups": {
"name": "COMMON_LOOKUP_VARIABLES",
"status": "CONFIGURED"
}
}
},
{
"code": "HI_INTEGRATION",
"version": "01.00.0000",
"dependencies": {
"connections": {
"id": "HELLO_WORLD1",
"status": "CONFIGURED"
}
}
}
]
}
]
output.json file contains
{
"bartaType": "DEVELOPED",
"countOfIntegrations": 3,
"id": "SAMPLE_PACKAGE",
"integrations": [
{
"code": "HELLO_INTEGRATION",
"dependencies": {
"connections": [
{
"id": "HELLO_WORLD1",
"lockedFlag": false,
"name": "Hello World1",
"role": "SOURCE",
"status": "CONFIGURED",
"type": "rest",
"usage": 6
}
]
},
"description": "",
"eventSubscriptionFlag": false,
"filmstrip": [
{
"code": "HELLO_WORLD1",
"iconUrl": "/images/rest/rest_icon_46.png",
"name": "Hello World1",
"role": "SOURCE",
"status": "CONFIGURED"
}
],
"id": "HELLO_INTEGRATION|01.00.0000",
"lockedFlag": false,
"name": "HELLO_INTEGRATION",
"pattern": "Orchestration",
"patternDescription": "Map Data",
"payloadTracingEnabledFlag": true,
"publishFlag": false,
"scheduleApplicable": false,
"scheduleDefined": false,
"status": "ACTIVATED",
"style": "FREEFORM",
"styleDescription": "Orchestration",
"tempCopyExists": false,
"tracingEnabledFlag": true,
"version": "01.00.0000",
"warningMsg": "ACTIVATE_PUBLISH_NO_CONN"
},
{
"code": "HELLO_INTEGRATIO_LOOKUP",
"dependencies": {
"connections": [
{
"id": "HELLO_WORLD1",
"lockedFlag": false,
"name": "Hello World1",
"role": "SOURCE",
"status": "CONFIGURED",
"type": "rest",
"usage": 6
}
],
"lookups": [
{
"lockedFlag": false,
"name": "COMMON_LOOKUP_VARIABLES",
"status": "CONFIGURED",
"usage": 1
}
]
},
"description": "",
"eventSubscriptionFlag": false,
"filmstrip": [
{
"code": "HELLO_WORLD1",
"iconUrl": "/images/rest/rest_icon_46.png",
"name": "Hello World1",
"role": "SOURCE",
"status": "CONFIGURED"
}
],
"id": "HELLO_INTEGRATIO_LOOKUP|01.00.0000",
"lockedFlag": false,
"name": "HELLO_INTEGRATION_LOOKUP",
"pattern": "Orchestration",
"patternDescription": "Map Data",
"payloadTracingEnabledFlag": true,
"publishFlag": false,
"scheduleApplicable": false,
"scheduleDefined": false,
"status": "ACTIVATED",
"style": "FREEFORM",
"styleDescription": "Orchestration",
"tempCopyExists": false,
"tracingEnabledFlag": true,
"version": "01.00.0000",
"warningMsg": "ACTIVATE_PUBLISH_NO_CONN"
},
{
"code": "HI_INTEGRATION",
"dependencies": {
"connections": [
{
"id": "HELLO_WORLD1",
"lockedFlag": false,
"name": "Hello World1",
"role": "SOURCE",
"status": "CONFIGURED",
"type": "rest",
"usage": 6
}
]
},
"description": "",
"eventSubscriptionFlag": false,
"filmstrip": [
{
"code": "HELLO_WORLD1",
"iconUrl": "/images/rest/rest_icon_46.png",
"name": "Hello World1",
"role": "SOURCE",
"status": "CONFIGURED"
}
],
"id": "HI_INTEGRATION|01.00.0000",
"lockedFlag": false,
"name": "HI_INTEGRATION",
"pattern": "Orchestration",
"patternDescription": "Map Data",
"payloadTracingEnabledFlag": true,
"publishFlag": false,
"scheduleApplicable": false,
"scheduleDefined": false,
"status": "ACTIVATED",
"style": "FREEFORM",
"styleDescription": "Orchestration",
"tempCopyExists": false,
"tracingEnabledFlag": true,
"version": "01.00.0000",
"warningMsg": "ACTIVATE_PUBLISH_NO_CONN"
}
],
"isCloneAllowed": false,
"isViewAllowed": false,
"name": "SAMPLE_PACKAGE",
"type": "DEVELOPED"
}

The problem is that the lookups key is not always present so, you cannot use the [] on it. So, instead you can use the map function and provide a default before piping to the map function like below
[
{ id: .id },
{
integrations: [
.integrations[]|{
id: .id,
code: .code,
dependencies: {
connections: (.dependencies.connections//[]|map({id,status}))[0],
lookups: (.dependencies.lookups//[]|map({name,status}))[0]
}
}
]
}
]
The (.dependencies.lookups//[]|map({name,status}))[0] has the effect of passing an empty array to the map function which results in a null value when accessing the first element.
See in action https://jqplay.org/s/zQBkHtnzOd1

The provided JQ statement works fine for single elements in the array , but incase the array contains multiple elements it only fetches the first element. Also i updated the dependencies object to capture all the arrays ( connections,lookups,certificates,libraries,integrations)
Below is the modified one. Please suggest for any better options.
[
{ id: .id },
{
integrations: [
.integrations[]|{
id: .id,
code: .code,
dependencies: {
connections: (.dependencies.connections//[]|map({id,status})),
lookups: (.dependencies.lookups//[]|map({name,status})),
certificates: (.dependencies.certificates//[]|map({id,status})),
libraries: (.dependencies.libraries//[]|map({code,status,version})),
integrations: (.dependencies.integrations//[]|map({code,version}))
}
}
]
}
]|del(..|select(.==[]))
Note: To remove the empty arrays del function is added which is giving the below output :
[
{
"id": "SAMPLE_PACKAGE"
},
{
"integrations": [
{
"id": "HELLO_INTEGRATION|01.00.0000",
"code": "HELLO_INTEGRATION",
"dependencies": {
"connections": [
{
"id": "HELLO_WORLD1",
"status": "CONFIGURED"
},
{
"id": "HELLO_WORLD2",
"status": "CONFIGURED"
}
]
}
},
{
"id": "HELLO_INTEGRATIO_LOOKUP|01.00.0000",
"code": "HELLO_INTEGRATIO_LOOKUP",
"dependencies": {
"connections": [
{
"id": "HELLO_WORLD1",
"status": "CONFIGURED"
}
],
"lookups": [
{
"name": "COMMON_LOOKUP_VARIABLES",
"status": "CONFIGURED"
}
]
}
},
{
"id": "HI_INTEGRATION|01.00.0000",
"code": "HI_INTEGRATION",
"dependencies": {
"connections": [
{
"id": "HELLO_WORLD1",
"status": "CONFIGURED"
}
]
}
}
]
}
]

storing json output in bash from cloudfromation

I am using aws ecs query to get list of properties being used by the current running task.
command -
cft = "aws ecs describe-tasks --cluster arn:aws:ecs:us-west-2:4984314772:cluster/secrets --tasks arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b
I am storing this in an output variable
output= $( eval $cft)
Output:
"tasks": [
{
"attachments": [
{
"id": "da8a1312-8278-46d5-8e3b-6b6a1d96f820",
"type": "ElasticNetworkInterface",
"status": "ATTACHED",
"details": [
{
"name": "subnetId",
"value": "subnet-0a151f2eb959ad4"
},
{
"name": "networkInterfaceId",
"value": "eni-081948e3666253f"
},
{
"name": "macAddress",
"value": "02:2a:9i:5c:4a:77"
},
{
"name": "privateDnsName",
"value": "ip-172-56-17-177.us-west-2.compute.internal"
},
{
"name": "privateIPv4Address",
"value": "172.56.17.177"
}
]
}
],
"availabilityZone": "us-west-2a",
"clusterArn": "arn:aws:ecs:us-west-2:4984314772:cluster/secrets",
"containers": [
{
"taskArn": "arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b",
"name": "nginx",
"image": "nginx",
"lastStatus": "PENDING",
"networkInterfaces": [
{
"attachmentId": "da8a1312-8278-46d5-6b6a1d96f820",
"privateIpv4Address": "172.31.17.176"
}
],
"healthStatus": "UNKNOWN",
"cpu": "0"
}
],
"cpu": "256",
"createdAt": "2020-12-10T18:00:16.320000+05:30",
"desiredStatus": "RUNNING",
"group": "family:nginx",
"healthStatus": "UNKNOWN",
"lastStatus": "PENDING",
"launchType": "FARGATE",
"memory": "512",
"overrides": {
"containerOverrides": [
{
"name": "nginx"
}
],
"inferenceAcceleratorOverrides": []
},
"platformVersion": "1.4.0",
"tags": [],
"taskArn": "arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b",
"taskDefinitionArn": "arn:aws:ecs:us-west-2:4984314772:task-definition/nginx:17",
"version": 2
}
],
"failures": []
}
now if do an echo of $output.tasks[0].containers[0] nothing happens it prints the entire thing again, i want to store the result in output variable and refer different parameter like we do in json format.

You will need to use a json parser such as jq and so:
eval $cft | jq '.tasks[].containers[]'
To avoid using eval you could simple pipe the aws command into jq and so:
aws ecs describe-tasks --cluster arn:aws:ecs:us-west-2:4984314772:cluster/secrets --tasks arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b | jq '.tasks[].containers[]'
or:
cft=$(aws ecs describe-tasks --cluster arn:aws:ecs:us-west-2:4984314772:cluster/secrets --tasks arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b | jq '.tasks[].containers[]')
echo $cft | jq '.tasks[].containers[]'

Merging JSON files using jenkins groovy

I have multiple JSON files with the following structures
FILE1.json
{
"Plugins": [
{
"Name": "Async Http Client",
"Installation": [
{
"Version": "1.7.24.2",
"Enabled": true,
"URL": "http://wiki.jenkins-ci.org/display/JENKINS/Async+Http+Client+Plugin",
"Server": "Server1"
}
]
},
{
"Name": "Common API for Blue Ocean",
"Installation": [
{
"Version": "1.10.2",
"Enabled": true,
"URL": "https://wiki.jenkins-ci.org/display/JENKINS/Blue+Ocean+Plugin",
"Server": "Server1"
}
]
},
.
.
.
]
}
FILE2.json
{
"Plugins": [
{
"Name": "Async Http Client",
"Installation": [
{
"Version": "1.7.24.1",
"Enabled": true,
"URL": "http://wiki.jenkins-ci.org/display/JENKINS/Async+Http+Client+Plugin",
"Server": "Server2"
}
]
},
{
"Name": "Common API for Blue Ocean",
"Installation": [
{
"Version": "1.10.2",
"Enabled": true,
"URL": "https://wiki.jenkins-ci.org/display/JENKINS/Blue+Ocean+Plugin",
"Server": "Server2"
}
]
},
.
.
.
]
}
FILE3.json
{
"Plugins": [
{
"Name": "Common API for Blue Ocean",
"Installation": [
{
"Version": "1.10.1",
"Enabled": true,
"URL": "https://wiki.jenkins-ci.org/display/JENKINS/Blue+Ocean+Plugin",
"Server": "Server3"
}
]
},
{
"Name": "Support Core Plugin",
"Installation": [
{
"Version": "2.54",
"Enabled": true,
"URL": "https://wiki.jenkins.io/display/JENKINS/Support+Core+Plugin",
"Server": "Server3"
}
]
},
.
.
.
]
}
I want to merge them into a single file with the following structure using groovy
MergedFile.json
{
"Plugins": [
{
"Name": "Async Http Client",
"Installation": [
{
"Version": "1.7.24.2",
"Enabled": true,
"URL": "http://wiki.jenkins-ci.org/display/JENKINS/Async+Http+Client+Plugin",
"Server": "Server1"
},
{
"Version": "1.7.24.1",
"Enabled": true,
"URL": "http://wiki.jenkins-ci.org/display/JENKINS/Async+Http+Client+Plugin",
"Server": "Server2"
}
]
},
{
"Name": "Common API for Blue Ocean",
"Installation": [
{
"Version": "1.10.2",
"Enabled": true,
"URL": "https://wiki.jenkins-ci.org/display/JENKINS/Blue+Ocean+Plugin",
"Server": "Server1"
},
{
"Version": "1.10.2",
"Enabled": true,
"URL": "https://wiki.jenkins-ci.org/display/JENKINS/Blue+Ocean+Plugin",
"Server": "Server2"
},
{
"Version": "1.10.1",
"Enabled": true,
"URL": "https://wiki.jenkins-ci.org/display/JENKINS/Blue+Ocean+Plugin",
"Server": "Server3"
}
]
},
{
"Name": "Support Core Plugin",
"Installation": [
{
"Version": "2.54",
"Enabled": true,
"URL": "https://wiki.jenkins.io/display/JENKINS/Support+Core+Plugin",
"Server": "Server3"
}
]
},
.
.
.
]
}
I am looking for a jenkins pipeline script using jenkins groovy. The file structure of the 3 files are the same. Only the file content maybe different. If there is any common "Name", "Installation" is merged under the same "Name" like mentioned above.
Any help is appreciated!
Edit:
I've tried something like this..
files = findFiles(glob: "*.json")
def input
def mergedjson = readJSON file:"${files[0]}"
for (f=1;f<files.size();f++)
{
input=readJSON file:"${files[f]}"
i = input.Plugins.size()
j = mergedjson.Plugins.size()
for (x=0;x<i;x++)
{
def json_status=0
for (y=0;y<j;y++)
{
if (input.Plugins[x].Name == mergedjson.Plugins[y].Name)
{
mergedjson.Plugins[y].Installation += input.Plugins[x].Installation
json_status=1
break;
}
}
if (json_status == 0)
mergedjson.Plugins[j++] = input.Plugins[x]
}
}
inventory=JsonOutput.prettyPrint(JsonOutput.toJson(mergedjson))
writeFile([file: "Inventory.json", text: inventory])
I am getting the following error
java.lang.StackOverflowError
at java.lang.reflect.InvocationTargetException.(InvocationTargetException.java:72)
at sun.reflect.GeneratedMethodAccessor544.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
at groovy.lang.MetaBeanProperty.getProperty(MetaBeanProperty.java:59)
at groovy.lang.PropertyValue.getValue(PropertyValue.java:42)
at org.codehaus.groovy.runtime.DefaultGroovyMethods.getProperties(DefaultGroovyMethods.java:389)
at groovy.json.JsonOutput.getObjectProperties(JsonOutput.java:327)
at groovy.json.JsonOutput.writeObject(JsonOutput.java:320)
at groovy.json.JsonOutput.writeMap(JsonOutput.java:458)
at groovy.json.JsonOutput.writeObject(JsonOutput.java:321)
at groovy.json.JsonOutput.writeMap(JsonOutput.java:458)
at groovy.json.JsonOutput.writeObject(JsonOutput.java:321)
at groovy.json.JsonOutput.writeMap(JsonOutput.java:458)
at groovy.json.JsonOutput.writeObject(JsonOutput.java:321)
.
.
.

Parsing Git Json with Regular Express

I am taking a Github json file and parsing it with Java's regular expression library JsonPath. I am having a problem parsing arrays that do not have labels.
I need to send a email every time a particular file is changed in our repository.
Here is the Git Json:
{
"trigger": "push",
"payload": {
"type": "GitPush",
"before": "xxxxxxxx",
"after": "yyyyyyyy",
"branch": "branch-name",
"ref": "refs/heads/branch-name",
"repository": {
"id": 42,
"name": "repo",
"title": "repo",
"type": "GitRepository"
},
"beanstalk_user": {
"type": "Owner",
"id": 42,
"login": "username",
"email": "user#example.org",
"name": "Name Surname"
},
"commits": [
{
"type": "GitCommit",
"id": "ffffffff",
"message": "Important changes.",
"branch": "branch-name",
"author": {
"name": "Name Surname",
"email": "user#example.org"
},
"beanstalk_user": {
"type": "Owner",
"id": 42,
"login": "username",
"email": "user#example.org",
"name": "Name Surname"
},
"changed_files": {
"added": [
"NEWFILE",
],
"deleted": [
"Gemfile",
"NEWFILE"
],
"modified": [
"README.md",
"NEWFILE"
],
"copied": [
]
},
"changeset_url": "https://subdomain.github.com/repository-name/changesets/ffffffff",
"committed_at": "2014/08/18 13:30:29 +0000",
"parents": [
"afafafaf"
]
}
]
}
}
This is the expression I am using: to get the commits
$..changed_files
This return the whole changed files part but I can not explicitly choose the name "NEWFILE"
I tried
$..changed_files.*[?(#.added == "NEWFILE")]
$..changed_files.*[?(#.*== "NEWFILE")]
It just returns a empty array.
I just want it to return Newfile and what type of change. Any Ideas?

You can use the following JsonPath to retrieve the commits which list "NEWFILE" as an added file :
$.payload.commits[?(#.changed_files.added.indexOf("NEWFILE") != -1)]

Google Home actions Sync payload issue

Below is the payload generated for SYNC intent. When tested in smart home validator it parses the json without any issue.
Sync Response Payload:
{
"payload": {
"agentUserId": "myhome",
"devices": [
{
"traits": [
"action.devices.traits.OnOff"
],
"willReportState": false,
"name": {
"defaultNames": [],
"name": "Sky Light",
"nicknames": []
},
"attributes": [],
"customData": "",
"id": "BED_LIGHT",
"type": "action.devices.types.LIGHT",
"deviceInfo": {
"swVersion": "1.0",
"model": "Light",
"manufacturer": "Connected Smart Life",
"hwVersion": "1.0"
}
},
{
"traits": [
"action.devices.traits.OnOff"
],
"willReportState": false,
"name": {
"defaultNames": [],
"name": "Ruthu Room Round Light",
"nicknames": []
},
"attributes": [],
"customData": "",
"id": "SWITCH94",
"type": "action.devices.types.LIGHT",
"deviceInfo": {
"swVersion": "1.0",
"model": "Light",
"manufacturer": "Connected Smart Life",
"hwVersion": "1.0"
}
}
]
},
"requestId": "18338945788402571764"
}
But, Google Assistant Actions failed to parse this and the log shows below erros:
{
insertId: "1ha0kvhg1kf0a0t"
logName: "projects/connectedghome/logs/actions.googleapis.com%2Factions"
receiveTimestamp: "2018-05-05T21:04:27.318451222Z"
resource: {…}
severity: "ERROR"
textPayload: "SYNC: Request ID 18338945788402571764 update devices failed: INVALID_ARGUMENT. Detail: Error: [Expect a map object but found: []] while parsing JSON [{"payload":{"agentUserId":"myhome","devices":[{"attributes":[],"customData":"","deviceInfo":{"hwVersion":"1.0","manufacturer":"Connected Smart Life","model":"Light","swVersion":"1.0"},"id":"BED_LIGHT","name":{"defaultNames":[],"name":"Sky Light","nicknames":[]},"traits":["action.devices.traits.OnOff"],"type":"action.devices.types.LIGHT","willReportState":false},{"attributes":[],"customData":"","deviceInfo":{"hwVersion":"1.0","manufacturer":"Connected Smart Life","model":"Light","swVersion":"1.0"},"id":"SWITCH94","name":{"defaultNames":[],"name":"Ruthu Room Round Light","nicknames":[]},"traits":["action.devices.traits.OnOff"],"type":"action.devices.types.LIGHT","willReportState":false}]},"requestId":"18338945788402571764"}]"
timestamp: "2018-05-05T21:04:27.284259780Z"
}
Am I missing something here?

Attributes should be a JSON object or not defined.