I am trying to filter docker-machine's output using the following jq filter.
docker-machine inspect default | jq '{ConfigVersion, .Driver.{MachineName, CPU, Memory}, DriverName}'
The original json for the first command is here
{
"ConfigVersion": 3,
"Driver": {
"IPAddress": "192.168.99.100",
"MachineName": "default",
"SSHUser": "docker",
"SSHPort": 52314,
"SSHKeyPath": "/Users/apatil/.docker/machine/machines/default/id_rsa",
"StorePath": "/Users/apatil/.docker/machine",
"SwarmMaster": false,
"SwarmHost": "tcp://0.0.0.0:3376",
"SwarmDiscovery": "",
"VBoxManager": {},
"HostInterfaces": {},
"CPU": 2,
"Memory": 5120,
"DiskSize": 20000,
"NatNicType": "82540EM",
"Boot2DockerURL": "",
"Boot2DockerImportVM": "",
"HostDNSResolver": false,
"HostOnlyCIDR": "192.168.99.1/24",
"HostOnlyNicType": "82540EM",
"HostOnlyPromiscMode": "deny",
"UIType": "headless",
"HostOnlyNoDHCP": false,
"NoShare": false,
"DNSProxy": true,
"NoVTXCheck": false,
"ShareFolder": ""
},
"DriverName": "virtualbox",
"HostOptions": {
"Driver": "",
"Memory": 0,
"Disk": 0,
"EngineOptions": {
"ArbitraryFlags": [],
"Dns": null,
"GraphDir": "",
"Env": [],
"Ipv6": false,
"InsecureRegistry": [],
"Labels": [],
"LogLevel": "",
"StorageDriver": "",
"SelinuxEnabled": false,
"TlsVerify": true,
"RegistryMirror": [],
"InstallURL": "https://get.docker.com"
},
"SwarmOptions": {
"IsSwarm": false,
"Address": "",
"Discovery": "",
"Agent": false,
"Master": false,
"Host": "tcp://0.0.0.0:3376",
"Image": "swarm:latest",
"Strategy": "spread",
"Heartbeat": 0,
"Overcommit": 0,
"ArbitraryFlags": [],
"ArbitraryJoinFlags": [],
"Env": null,
"IsExperimental": false
},
"AuthOptions": {
"CertDir": "/Users/apatil/.docker/machine/certs",
"CaCertPath": "/Users/apatil/.docker/machine/certs/ca.pem",
"CaPrivateKeyPath": "/Users/apatil/.docker/machine/certs/ca-key.pem",
"CaCertRemotePath": "",
"ServerCertPath": "/Users/apatil/.docker/machine/machines/default/server.pem",
"ServerKeyPath": "/Users/apatil/.docker/machine/machines/default/server-key.pem",
"ClientKeyPath": "/Users/apatil/.docker/machine/certs/key.pem",
"ServerCertRemotePath": "",
"ServerKeyRemotePath": "",
"ClientCertPath": "/Users/apatil/.docker/machine/certs/cert.pem",
"ServerCertSANs": [],
"StorePath": "/Users/apatil/.docker/machine/machines/default"
}
},
"Name": "default"
}
I am getting the following error from jq for the command above
$ docker-machine inspect default | jq '{ConfigVersion, .Driver.{MachineName, CPU, Memory}, DriverName}'
jq: error: syntax error, unexpected FIELD (Unix shell quoting issues?) at <top-level>, line 1:
{ConfigVersion, .Driver.{MachineName, CPU, Memory}, DriverName}
jq: error: syntax error, unexpected '}', expecting $end (Unix shell quoting issues?) at <top-level>, line 1:
{ConfigVersion, .Driver.{MachineName, CPU, Memory}, DriverName}
jq: 2 compile errors
Fixed it using
$ docker-machine inspect default |
jq '{ConfigVersion,
Driver: (.Driver|{MachineName, CPU, Memory}),
DriverName}'
{
"ConfigVersion": 3,
"Driver": {
"MachineName": "default",
"CPU": 2,
"Memory": 5120
},
"DriverName": "virtualbox"
}
Related
I have a JSON output that I receive from cURL, I would like to pipe to jq to extract useful information and display the csv data in a terminal.
Sample JSON response:
{
"operation": "GET CLIENT SESSIONS",
"outcome": "SUCCESS",
"result": {
"Session": [
{
"sessionID": "E1B7190EF32AA9F6E63265ADBB88D1A9D6F4254457A0.production",
"requestID": "",
"sessionPoolID": "8LNIVzRTSo-jJV-CdPMP0Q",
"agentID": "",
"ablSessionID": "",
"lastAccessStr": "2022-04-01T10:36:14.745-0700",
"elapsedTimeMs": 1675892,
"sessionState": "AVAILABLE",
"requestState": "READY",
"sessionType": "SESSION_FREE",
"adapterType": "APSV",
"bound": false,
"clientConnInfo": null,
"agentConnInfo": null
},
{
"sessionID": "E1B7190EF32AA9F6E63265ADBB88D1A9D6F4254457A0.production",
"requestID": "",
"sessionPoolID": "8LNIVzRTSo-jJV-CdPMP0Q",
"agentID": "",
"ablSessionID": "",
"lastAccessStr": "2022-04-01T10:36:14.745-0700",
"elapsedTimeMs": 1675892,
"sessionState": "AVAILABLE",
"requestState": "READY",
"sessionType": "SESSION_FREE",
"adapterType": "APSV",
"bound": false,
"clientConnInfo": null,
"agentConnInfo": null
}
]
},
"errmsg": "",
"versionStr": "v1.0.0 ( 2021-10-29 )",
"versionNo": 1
}
I am able to extract information using this query:
jq '.result.Session[] | ([.lastAccessStr, .elapsedTimeMs, .clientConnInfo]) | #csv'
The problem I am running into, sometimes the response can have additional nested data for clientConnInfo and agentConnInfo as seen in the example below.
{
"operation": "GET CLIENT SESSIONS",
"outcome": "SUCCESS",
"result": {
"Session": [
{
"sessionID": "E1B7190EF32AA9F6E63265ADBB88D1A9D6F4254457A0.production",
"requestID": "",
"sessionPoolID": "8LNIVzRTSo-jJV-CdPMP0Q",
"agentID": "",
"ablSessionID": "",
"lastAccessStr": "2022-04-01T10:36:14.745-0700",
"elapsedTimeMs": 1675892,
"sessionState": "AVAILABLE",
"requestState": "READY",
"sessionType": "SESSION_FREE",
"adapterType": "APSV",
"bound": false,
"clientConnInfo": null,
"agentConnInfo": null
},
{
"sessionID": "26691913A73E55175D233F86D219B4AEFA4AD14AE9E4.production",
"requestID": "ROOT:a:000002a6",
"sessionPoolID": "8LNIVzRTSo-jJV-CdPMP0Q",
"agentID": "qsrLXAxsRRanJio6dYOC2Q",
"ablSessionID": "",
"lastAccessStr": "2022-04-01T11:04:08.902-0700",
"elapsedTimeMs": 1735,
"sessionState": "RESERVED",
"requestState": "RUNNING",
"sessionType": "SESSION_FREE",
"adapterType": "APSV",
"bound": false,
"clientConnInfo": {
"clientName": "xxxxx",
"requestID": "ROOT:a:000002a6",
"sessionID": "26691913A73E55175D233F86D219B4AEFA4AD14AE9E4.production",
"adapterType": "APSV",
"reqStartTimeStr": "2022-04-01T11:04:08.902-0700",
"elapsedTimeMs": 1735,
"executerThreadId": "thd-8",
"requestUrl": "xxxxx",
"requestProcedure": "xxxxx.p",
"httpSessionId": "26691913A73E55175D233F86D219B4AEFA4AD14AE9E4.production"
},
"agentConnInfo": {
"agentID": "qsrLXAxsRRanJio6dYOC2Q",
"connID": "AR2k7gnYSiKPCk2DEHpaSg",
"connPoolID": "v0Lh7XcITsOSfoF_RjXXig",
"state": "RESERVED",
"agentAddr": "xxxxx",
"localAddr": "xxxxx"
}
}
]
},
"errmsg": "",
"versionStr": "v1.0.0 ( 2021-10-29 )",
"versionNo": 1
}
If I try to use the same query, it dies with below error:
jq: error (at <stdin>:0): object ({"clientNam...) is not valid in a csv row
The desired output is to capture this:
.lastAccessStr
.elapsedTimeMs
.clientName (if available in output)
.requestID (if available in output)
.agentAddr (if available in output)
.localAddr (if available in output)
I have been trying to make this work using https://jqplay.org without any luck.
Can anyone give me some examples on how I would go about making this work?
User pmf has got it in the comments:
Use ? to ignore an error, and // to provide an alternative if the first one is null, false or inexistent. Define every column at question along the lines of (.clientConnInfo.clientName? // "none")
this is Json: I try to use
jq -r '.[] | .DBSnapshotIdentifier'
But it doesn't work for me
{
"DBSnapshots": [
{
"DBSnapshotIdentifier": "auto-lims-final-snapshot",
"DBInstanceIdentifier": "auto-lims",
"SnapshotCreateTime": "2018-08-15T09:59:23.332000+00:00",
"Engine": "postgres",
"AllocatedStorage": 50,
"Status": "available",
"Port": 5432,
"AvailabilityZone": "us-east-2a",
"VpcId": "vpc-e799fc8f",
"InstanceCreateTime": "2018-04-09T08:28:03.565000+00:00",
"MasterUsername": "postgres",
"EngineVersion": "9.6.6",
"LicenseModel": "postgresql-license",
"SnapshotType": "manual",
"OptionGroupName": "default:postgres-9-6",
"PercentProgress": 100,
"StorageType": "gp2",
"Encrypted": false,
"DBSnapshotArn": "arn:aws:rds:us-east-2:833682533595:snapshot:auto-lims-final-snapshot",
"IAMDatabaseAuthenticationEnabled": false,
"ProcessorFeatures": [],
"DbiResourceId": "db-ZL5T2TA2PJVG6CVOJRO7HUOAXQ",
"TagList": [
{
"Key": "Environment",
"Value": "auto"
},
{
"Key": "Application",
"Value": "LIMS"
}
],
"SnapshotTarget": "region"
},
{
"DBSnapshotIdentifier": "automation-lims-before-postgres-12-5",
"DBInstanceIdentifier": "automation-lims",
"Engine": "postgres",
"AllocatedStorage": 500,
"Status": "available",
"Port": 5432,
"AvailabilityZone": "us-east-2b",
"VpcId": "vpc-09fa88d2884ee2083",
"InstanceCreateTime": "2019-12-26T11:19:41.947000+00:00",
"MasterUsername": "lims",
"EngineVersion": "9.6.20",
"LicenseModel": "postgresql-license",
"SnapshotType": "manual",
"OptionGroupName": "default:postgres-9-6",
"PercentProgress": 100
}
]
}
How I can get only nested lines.
You first need to select the DBSnapshots object, try:
.DBSnapshots[] | .DBSnapshotIdentifier
This will output:
"auto-lims-final-snapshot"
"automation-lims-before-postgres-12-5"
As you can try in this online demo
To include the .TagList.Key we can add the following:
.DBSnapshots[] | .DBSnapshotIdentifier, .TagList[]?.Key
This will now output
"auto-lims-final-snapshot"
"Environment"
"Application"
"automation-lims-before-postgres-12-5"
As you can try in this online demo
If you'd like the DBSnapshotIdentifier to be on the same line as the TagList we can use string interpolation like so:
.DBSnapshots[] | "\(.DBSnapshotIdentifier) - \(.TagList[]?.Key)"
Thiis will output:
"auto-lims-final-snapshot - Environment"
"auto-lims-final-snapshot - Application"
As this demo demostrates
The .[]? from the above command mean:
Like .[], but no errors will be output if . is not an array or object.
More information in JQ's documentation
I just started using jq and json files, and I'm trying to parse a specific file.
I'm tring to do it with jq in command line, but if there's any other way to do it properly, I'm in to give it a try.
The file itself looks like this :
{
"Status": "ok",
"Code": 200,
"Message": "",
"Result": [
{
"ID": 123456,
"Activity": 27,
"Name": Example1",
"Coordinate": {
"Galaxy": 1,
"System": 22,
"Position": 3
},
"Administrator": false,
"Inactive": false,
"Vacation": false,
"HonorableTarget": false,
"Debris": {
"Metal": 0,
"Crystal": 0,
"RecyclersNeeded": 0
},
"Moon": null,
"Player": {
"ID": 111111,
"Name": "foo",
"Rank": 4
},
"Alliance": null
},
{
"ID": 223344,
"Activity": 17,
"Name": "Example2",
"Coordinate": {
"Galaxy": 3,
"System": 44,
"Position": 5
},
"Administrator": false,
"Inactive": false,
"Vacation": false,
"StrongPlayer": false,
"HonorableTarget": false,
"Debris": {
"Metal": 0,
"Crystal": 0,
"RecyclersNeeded": 0
},
"Moon": null,
"Player": {
"ID": 765432,
"Name": "Player 2",
"Rank": 3
},
"Alliance": null
},
(...)
]
}
I would need to extract information based on the galaxy/system/position.
For example, having a script with the proper filters in it and execute something like that :
./parser --galaxy=1 --system=22 --position=3
And it would give me :
ID : 123456
Name : Example1
Activity : 27
...
I tried to do that with curl to grab my json file and jq to parse my file, but I have no idea how I can make that kind of request.
The following should be sufficient to get you on your way.
First, let's assume the JSON is in a file name galaxy.json; second, let's assume the file galaxy.jq contains the following:
.Result[]
| select(.Coordinate | (.Galaxy==$galaxy and .System==$system and .Position==$position))
Then the invocation:
jq -f so-galaxy.jq --argjson galaxy 1 --argjson system 22 --argjson position 3 galaxy.json
would yield the corresponding object:
{
"ID": 123456,
"Activity": 27,
"Name": "Example1",
"Coordinate": {
"Galaxy": 1,
"System": 22,
"Position": 3
},
"Administrator": false,
"Inactive": false,
"Vacation": false,
"HonorableTarget": false,
"Debris": {
"Metal": 0,
"Crystal": 0,
"RecyclersNeeded": 0
},
"Moon": null,
"Player": {
"ID": 111111,
"Name": "foo",
"Rank": 4
},
"Alliance": null
}
Key: Value format
If you want the output to be in key: value format, simply add -r to the command-line options, and append the following to the jq filter:
| to_entries[]
| "\(.key): \(.value)"
Output
ID: 123456
Activity: 27
Name: Example1
Coordinate: {"Galaxy":1,"System":22,"Position":3}
Administrator: false
Inactive: false
Vacation: false
HonorableTarget: false
Debris: {"Metal":0,"Crystal":0,"RecyclersNeeded":0}
Moon: null
Player: {"ID":111111,"Name":"foo","Rank":4}
Alliance: null
I have a JSON file name emoji.json located in my local mac PC
my JSON file has this structure
[
{
"name": "COPYRIGHT SIGN",
"unified": "00A9-FE0F",
"non_qualified": "00A9",
"docomo": "E731",
"au": "E558",
"softbank": "E24E",
"google": "FEB29",
"image": "00a9-fe0f.png",
"sheet_x": 0,
"sheet_y": 12,
"s": "copyright",
"short_names": [
"copyright"
],
"text": null,
"texts": null,
"c": "Symbols",
"o": 128,
"added_in": "1.1",
"has_img_twitter": false,
"has_img_emojione": false,
"has_img_messenger": false
},
{
"name": "REGISTERED SIGN",
"unified": "00AE-FE0F",
"non_qualified": "00AE",
"docomo": "E736",
"au": "E559",
"softbank": "E24F",
"google": "FEB2D",
"image": "00ae-fe0f.png",
"sheet_x": 0,
"sheet_y": 13,
"s": "registered",
"short_names": [
"registered"
],
"text": null,
"texts": null,
"c": "Symbols",
"o": 129,
"added_in": "1.1",
"has_img_twitter": false,
"has_img_emojione": false,
"has_img_messenger": false
}
...
]
and i want to remove some unwanted values so the file should became like blow.
and i want to save this file in JSON format in my PC
[
{
"s": "copyright",
"c": "Symbols",
"o": 128
},
{
"s": "registered",
"c": "Symbols",
"o": 129
}
]
i know JQ could do that and i check documentation and https://jqplay.org/
and i test and found out map({ s, c , o }) could do that but i dint find how to input file and export file
jq + mv approach:
jq 'map({s, c, o})' emoji.json > tmp_json && mv tmp_json emoji.json
After banging my head on my desk for a couple days. Here is where I am.
The Goal: Create an entry in the /etc/fstab file using the "UUID=" as my mount point.
In this case there will be x number of disks on y number of hosts so
just putting the information in a vars file unfortunately not
possible. Also there will not be any partitions or lvms, for this
application we need JBOD with filesystems.
My Approach: After some trial and error I have reached a point where my disks are all formatted and they have UUID's. So I am attempting to loop over my devices and then return my UUID to that I can use the mount module to append to the fstab.
Stuck at: After getting my list of devices the best way I have discovered to create a list of device UUID's is using the blkid command and registering the results. (see below) My thought was if I could access the nested UUID within 'results2'. Then I could pair the device name/UUID in a new fact for easier access later.
#######
#get new facts
#######
- command: "/sbin/blkid -o device"
register: "result1"
- shell: '/sbin/blkid "{{ item }}" -o value'
register: "result2"
with_items: "{{ result1.stdout_lines }}"
#- debug: var="{{ result2.results }}"
- debug: "{{item}}"
with_together:
- "{{result1.stdout_lines}}"
- "{{result2.results}}"
Output
PLAY [disk-test-node] **********************************************************
TASK [setup] *******************************************************************
ok: [31.212.212.123]
TASK [common : set_fact] *******************************************************
task path: /ansible/roles/common/tasks/main.yml:5
ok: [31.212.212.123] => (item={u'uuid': u'', u'size_total': 8319852544, u'mount': u'/', u'size_available': 7187308544, u'fstype': u'ext4', u'device': u'/dev/xvda1', u'options': u'rw'}) => {"ansible_facts": {"mounted_drives": "/dev/xvda1"}, "changed": false, "item": {"device": "/dev/xvda1", "fstype": "ext4", "mount": "/", "options": "rw", "size_available": 7187308544, "size_total": 8319852544, "uuid": ""}}
TASK [common : command] ********************************************************
task path: /ansible/roles/common/tasks/main.yml:39
changed: [31.212.212.123] => {"changed": true, "cmd": ["/sbin/blkid", "-o", "device"], "delta": "0:00:00.003729", "end": "2016-11-15 22:11:13.348422", "rc": 0, "start": "2016-11-15 22:11:13.344693", "stderr": "", "stdout": "/dev/xvda1\n/dev/xvdd\n/dev/xvde\n/dev/xvdb\n/dev/xvdc\n/dev/xvdh\n/dev/xvdi\n/dev/xvdf\n/dev/xvdg", "stdout_lines": ["/dev/xvda1", "/dev/xvdd", "/dev/xvde", "/dev/xvdb", "/dev/xvdc", "/dev/xvdh", "/dev/xvdi", "/dev/xvdf", "/dev/xvdg"], "warnings": []}
TASK [common : command] ********************************************************
task path: /ansible/roles/common/tasks/main.yml:42
changed: [31.212.212.123] => (item=/dev/xvda1) => {"changed": true, "cmd": "/sbin/blkid \"/dev/xvda1\" -o value", "delta": "0:00:00.002619", "end": "2016-11-15 22:11:15.693323", "item": "/dev/xvda1", "rc": 0, "start": "2016-11-15 22:11:15.690704", "stderr": "", "stdout": "cdbab22a-45d6-4cce-95a3-681f42187a46\next4", "stdout_lines": ["cdbab22a-45d6-4cce-95a3-681f42187a46", "ext4"], "warnings": []}
changed: [31.212.212.123] => (item=/dev/xvdd) => {"changed": true, "cmd": "/sbin/blkid \"/dev/xvdd\" -o value", "delta": "0:00:00.002580", "end": "2016-11-15 22:11:17.984917", "item": "/dev/xvdd", "rc": 0, "start": "2016-11-15 22:11:17.982337", "stderr": "", "stdout": "61b23af1-44dd-46c4-8e4b-2af2bd928f98\next4", "stdout_lines": ["61b23af1-44dd-46c4-8e4b-2af2bd928f98", "ext4"], "warnings": []}
Its right there!
You can see the UUID in "stdout" and "stdout_lines[0]"
Final Debug Loop after pairing
TASK [common : debug] **********************************************************
task path: /ansible/roles/common/tasks/main.yml:48
ok: [31.212.212.123] => (item=[u'/dev/xvda1', {'_ansible_parsed': True, '_ansible_item_result': True, u'stdout': u'cdbab22a-45d6-4cce-95a3-681f42187a46\next4', '_ansible_no_log': False, u'warnings': [], u'changed': True, u'rc': 0, u'end': u'2016-11-15 22:11:15.693323', u'start': u'2016-11-15 22:11:15.690704', u'cmd': u'/sbin/blkid "/dev/xvda1" -o value', 'item': u'/dev/xvda1', u'delta': u'0:00:00.002619', 'invocation': {'module_name': u'command', u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'_raw_params': u'/sbin/blkid "/dev/xvda1" -o value', u'removes': None, u'creates': None, u'chdir': None}}, 'stdout_lines': [u'cdbab22a-45d6-4cce-95a3-681f42187a46', u'ext4'], u'stderr': u''}]) => {
"<type 'list'>": "VARIABLE IS NOT DEFINED!",
"item": [
"/dev/xvda1",
{
"_ansible_item_result": true,
"_ansible_no_log": false,
"_ansible_parsed": true,
"changed": true,
"cmd": "/sbin/blkid \"/dev/xvda1\" -o value",
"delta": "0:00:00.002619",
"end": "2016-11-15 22:11:15.693323",
"invocation": {
"module_args": {
"_raw_params": "/sbin/blkid \"/dev/xvda1\" -o value",
"_uses_shell": true,
"chdir": null,
"creates": null,
"executable": null,
"removes": null,
"warn": true
},
"module_name": "command"
},
"item": "/dev/xvda1",
"rc": 0,
"start": "2016-11-15 22:11:15.690704",
"stderr": "",
"stdout": "cdbab22a-45d6-4cce-95a3-681f42187a46\next4",
"stdout_lines": [
"cdbab22a-45d6-4cce-95a3-681f42187a46",
"ext4"
],
"warnings": []
}
]
}
ok: [31.212.212.123] => (item=[u'/dev/xvdd', {'_ansible_parsed': True, '_ansible_item_result': True, u'stdout': u'61b23af1-44dd-46c4-8e4b-2af2bd928f98\next4', '_ansible_no_log': False, u'warnings': [], u'changed': True, u'rc': 0, u'end': u'2016-11-15 22:11:17.984917', u'start': u'2016-11-15 22:11:17.982337', u'cmd': u'/sbin/blkid "/dev/xvdd" -o value', 'item': u'/dev/xvdd', u'delta': u'0:00:00.002580', 'invocation': {'module_name': u'command', u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'_raw_params': u'/sbin/blkid "/dev/xvdd" -o value', u'removes': None, u'creates': None, u'chdir': None}}, 'stdout_lines': [u'61b23af1-44dd-46c4-8e4b-2af2bd928f98', u'ext4'], u'stderr': u''}]) => {
"<type 'list'>": "VARIABLE IS NOT DEFINED!",
"item": [
"/dev/xvdd",
{
"_ansible_item_result": true,
"_ansible_no_log": false,
"_ansible_parsed": true,
"changed": true,
"cmd": "/sbin/blkid \"/dev/xvdd\" -o value",
"delta": "0:00:00.002580",
"end": "2016-11-15 22:11:17.984917",
"invocation": {
"module_args": {
"_raw_params": "/sbin/blkid \"/dev/xvdd\" -o value",
"_uses_shell": true,
"chdir": null,
"creates": null,
"executable": null,
"removes": null,
"warn": true
},
"module_name": "command"
},
"item": "/dev/xvdd",
"rc": 0,
"start": "2016-11-15 22:11:17.982337",
"stderr": "",
"stdout": "61b23af1-44dd-46c4-8e4b-2af2bd928f98\next4",
"stdout_lines": [
"61b23af1-44dd-46c4-8e4b-2af2bd928f98",
"ext4"
],
"warnings": []
}
]
}
May I advice another solution?
- shell: "blkid | sed 's/:.*UUID=\"\\([0-9a-f-]\\+\\).*/ \\1/'"
register: devs
- debug: msg="dev={{ item.split()[0] }}, uuid={{ item.split()[1] }}"
with_items: "{{ devs.stdout_lines }}"
First task will return device_name UUID pairs separated with space.
Second task uses python split() function to get device_name and UUID as separate strings.