Mac Terminal: Delete Entire Column From Json File - json

How can I use mac terminal to delete an entire column from a json File.
The Json structure is as follows:
[{
"recordid": "6a0a9c66f8e0292a54c9f023c93732f1b41d8943",
"fields": {
"city": "Cove",
"zip": "71937",
"dst": 1,
"geopoint": [
34.398483,
-94.39398
],
"longitude": -94.39398,
"state": "AR",
"latitude": 34.398483,
"timezone": -6
},
"geometry": {
"type": "Point",
"coordinates": [
-94.39398,
34.398483
]
},
"record_timestamp": "2018-02-09T09:33:38.603-07:00"
},
{
"recordid": "37e2c801aafc7befde9734bcb1b1f83a5645ad0f",
"fields": {
"city": "Edgemont",
"zip": "72044",
"dst": 1,
"geopoint": [
35.624351,
-92.16056
],
"longitude": -92.16056,
"state": "AR",
"latitude": 35.624351,
"timezone": -6
},
"geometry": {
"type": "Point",
"coordinates": [
-92.16056,
35.624351
]
},
"record_timestamp": "2018-02-09T09:33:38.603-07:00"
}]
Using Terminal, how can I remove both and all columns including geopoint and geometry attributes while saving the file with the rest which I would wanna keep.

Use jq, map trough the JSON, delete .geometry and .fields.geopoint;
jq 'map(del(.fields.geopoint, .geometry))'
Result;
[
{
"recordid": "6a0a9c66f8e0292a54c9f023c93732f1b41d8943",
"fields": {
"city": "Cove"
}
},
{
"recordid": "6a0a9c66f8e0292a54c9f023c93732f1b41d8342",
"fields": {
"city": "Edgemont"
}
}
]
cat json.json | jq 'map(del(.fields.geopoint, .geometry))' > new.json
mv new.json json.json # Overwrites original file
Try it online!
cat mainzip.json | jq 'map(del(.datasetid, .fields.city, .fields.dst, .fields.geopoint, .fields.state, .fields.timezone, .type.point, .geometry, .record_timestamp))' > temporary_mainzip.json
mv temporary_mainzip.json mainzip.json

Related

How can I convert nested JSON to CSV on command line?

I have a JSON file who I am trying to convert to CSV using jq, but I've been having a lot of problems, this is the JSON:
{
"nhits": 2,
"parameters": {
"dataset": "real-time-bezettingen-fietsenstallingen-gent",
"rows": 10,
"start": 0,
"facet": [
"facilityname"
],
"format": "json",
"timezone": "UTC"
},
"records": [
{
"datasetid": "real-time-bezettingen-fietsenstallingen-gent",
"recordid": "d471594688a931ba8d81f8d883874a08cee84775",
"fields": {
"id": "48-2",
"freeplaces": 71,
"facilityname": "Braunplein",
"geo_point_2d": [
51.05406845807926,
3.723722319130363
],
"time": "2022-11-10T12:18:01+00:00",
"totalplaces": 116,
"occupiedplaces": 45,
"bezetting": 38
},
"geometry": {
"type": "Point",
"coordinates": [
3.723722319130363,
51.05406845807926
]
},
"record_timestamp": "2022-11-10T12:18:04.838Z"
},
{
"datasetid": "real-time-bezettingen-fietsenstallingen-gent",
"recordid": "d0121748cf31c7e1c02d99712bdf07cb33156689",
"fields": {
"id": "48-1",
"freeplaces": 65,
"facilityname": "Korenmarkt",
"geo_point_2d": [
51.05388288288933,
3.7214177570400473
],
"time": "2022-11-10T12:18:01+00:00",
"totalplaces": 235,
"occupiedplaces": 170,
"bezetting": 72
},
"geometry": {
"type": "Point",
"coordinates": [
3.7214177570400473,
51.05388288288933
]
},
"record_timestamp": "2022-11-10T12:18:04.838Z"
}
],
"facet_groups": [
{
"name": "facilityname",
"facets": [
{
"name": "Braunplein",
"count": 1,
"state": "displayed",
"path": "Braunplein"
},
{
"name": "Korenmarkt",
"count": 1,
"state": "displayed",
"path": "Korenmarkt"
}
]
}
]
}
I only want to have the columns facilityname, time, totalplaces, occupiedplaces and bezetting, I tried converting using the following command:
jq -r '["naam", "tijd", "totaalAantalPlaatsen", "bezettePlaatsen", "bezetting"] , .records[] | (.fields[] | [.facilityname, .time, .totalplaces, .occupiedplaces, .bezetting]) | #csv' data.json
But I get the error:
jq: error (at data.json:0): Cannot index array with string "fields"
Does anyone know what I'm doing wrong?
You just need some parentheses around the .records[] ... part
jq -r '
["name", "tijd", "totaalAantalPlaatsen", "bezettePlaatsen", "bezetting"],
(.records[].fields | [.facilityname, .time, .totalplaces, .occupiedplaces, .bezetting])
| #csv
' file.json

Cannot get jq to query json object [duplicate]

This question already has answers here:
How to use jq when the variable has reserved characters?
(3 answers)
Closed 6 months ago.
I have a JSON file that I am trying to query with jq. I am unable to retrieve the observations. I am trying to retieve each of the "observations using the following command and not able to get to the result:
cat sample3.json | jq .dataSets[0].series.0:0:0:0:0.observations.0[0]
I am able to retieve up to the series using:
cat sample3.json | jq .dataSets[0].series
But once I try to drill down further I am getting a compile error:
$ cat sample3.json | jq .dataSets[0].series.0:0:0:0:0
jq: error: syntax error, unexpected LITERAL, expecting end of file (Unix shell quoting issues?) at <top-level>, line 1:
.dataSets[0].series.0:0:0:0:0
jq: 1 compile error
I am not sure what I am doing wrong here....
The input file is:
{
"header": {
"id": "b8be2cd5-33bf-4687-9e81-eb032f6f8a71",
"test": false,
"prepared": "2022-09-01T13:30:57.013+02:00",
"sender": {
"id": "ECB"
}
},
"dataSets": [
{
"action": "Replace",
"validFrom": "2022-09-01T13:30:57.013+02:00",
"series": {
"0:0:0:0:0": {
"attributes": [
0,
null,
0,
null,
null,
null,
null,
null,
null,
null,
null,
null,
0,
null,
0,
null,
0,
0,
0,
0
],
"observations": {
"0": [
1.4529,
0,
0,
null,
null
],
"1": [
1.4472,
0,
0,
null,
null
],
"2": [
1.4591,
0,
0,
null,
null
]
}
}
}
}
],
"structure": {
"links": [
{
"title": "Exchange Rates",
"rel": "dataflow",
"href": "https://sdw-wsrest.ecb.europa.eu:443/service/dataflow/ECB/EXR/1.0"
}
],
"name": "Exchange Rates",
"dimensions": {
"series": [
{
"id": "FREQ",
"name": "Frequency",
"values": [
{
"id": "D",
"name": "Daily"
}
]
},
{
"id": "CURRENCY",
"name": "Currency",
"values": [
{
"id": "AUD",
"name": "Australian dollar"
}
]
},
{
"id": "CURRENCY_DENOM",
"name": "Currency denominator",
"values": [
{
"id": "EUR",
"name": "Euro"
}
]
},
{
"id": "EXR_TYPE",
"name": "Exchange rate type",
"values": [
{
"id": "SP00",
"name": "Spot"
}
]
},
{
"id": "EXR_SUFFIX",
"name": "Series variation - EXR context",
"values": [
{
"id": "A",
"name": "Average"
}
]
}
],
"observation": [
{
"id": "TIME_PERIOD",
"name": "Time period or range",
"role": "time",
"values": [
{
"id": "2022-08-29",
"name": "2022-08-29",
"start": "2022-08-29T00:00:00.000+02:00",
"end": "2022-08-29T23:59:59.999+02:00"
},
{
"id": "2022-08-30",
"name": "2022-08-30",
"start": "2022-08-30T00:00:00.000+02:00",
"end": "2022-08-30T23:59:59.999+02:00"
},
{
"id": "2022-08-31",
"name": "2022-08-31",
"start": "2022-08-31T00:00:00.000+02:00",
"end": "2022-08-31T23:59:59.999+02:00"
}
]
}
]
},
"attributes": {
"series": [
{
"id": "TIME_FORMAT",
"name": "Time format code",
"values": [
{
"name": "P1D"
}
]
},
{
"id": "BREAKS",
"name": "Breaks",
"values": []
},
{
"id": "COLLECTION",
"name": "Collection indicator",
"values": [
{
"id": "A",
"name": "Average of observations through period"
}
]
},
{
"id": "COMPILING_ORG",
"name": "Compiling organisation",
"values": []
},
{
"id": "DISS_ORG",
"name": "Data dissemination organisation",
"values": []
},
{
"id": "DOM_SER_IDS",
"name": "Domestic series ids",
"values": []
},
{
"id": "PUBL_ECB",
"name": "Source publication (ECB only)",
"values": []
},
{
"id": "PUBL_MU",
"name": "Source publication (Euro area only)",
"values": []
},
{
"id": "PUBL_PUBLIC",
"name": "Source publication (public)",
"values": []
},
{
"id": "UNIT_INDEX_BASE",
"name": "Unit index base",
"values": []
},
{
"id": "COMPILATION",
"name": "Compilation",
"values": []
},
{
"id": "COVERAGE",
"name": "Coverage",
"values": []
},
{
"id": "DECIMALS",
"name": "Decimals",
"values": [
{
"id": "4",
"name": "Four"
}
]
},
{
"id": "NAT_TITLE",
"name": "National language title",
"values": []
},
{
"id": "SOURCE_AGENCY",
"name": "Source agency",
"values": [
{
"id": "4F0",
"name": "European Central Bank (ECB)"
}
]
},
{
"id": "SOURCE_PUB",
"name": "Publication source",
"values": []
},
{
"id": "TITLE",
"name": "Title",
"values": [
{
"name": "Australian dollar/Euro"
}
]
},
{
"id": "TITLE_COMPL",
"name": "Title complement",
"values": [
{
"name": "ECB reference exchange rate, Australian dollar/Euro, 2:15 pm (C.E.T.)"
}
]
},
{
"id": "UNIT",
"name": "Unit",
"values": [
{
"id": "AUD",
"name": "Australian dollar"
}
]
},
{
"id": "UNIT_MULT",
"name": "Unit multiplier",
"values": [
{
"id": "0",
"name": "Units"
}
]
}
],
"observation": [
{
"id": "OBS_STATUS",
"name": "Observation status",
"values": [
{
"id": "A",
"name": "Normal value"
}
]
},
{
"id": "OBS_CONF",
"name": "Observation confidentiality",
"values": [
{
"id": "F",
"name": "Free"
}
]
},
{
"id": "OBS_PRE_BREAK",
"name": "Pre-break observation value",
"values": []
},
{
"id": "OBS_COM",
"name": "Observation comment",
"values": []
}
]
}
}
}
The .foo syntax cannot be used if the key name has anything but alphanumeric characters or the underscore, or if the first character of the key name is numeric.
Assuming you are using a recent version of jq,
you can always use the form: ."foo", which is actually an abbreviation of the basic form, .["foo"].
So assuming you're using a sufficiently recent version of jq, your query could begin with:
.dataSets[0].series."0:0:0:0:0"
If you are presenting the jq query on a command line, then you may have to escape the double-quotes appropriately, e.g. in a bash shell, by enclosing the jq query in single-quotes.

storing json output in bash from cloudfromation

I am using aws ecs query to get list of properties being used by the current running task.
command -
cft = "aws ecs describe-tasks --cluster arn:aws:ecs:us-west-2:4984314772:cluster/secrets --tasks arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b
I am storing this in an output variable
output= $( eval $cft)
Output:
"tasks": [
{
"attachments": [
{
"id": "da8a1312-8278-46d5-8e3b-6b6a1d96f820",
"type": "ElasticNetworkInterface",
"status": "ATTACHED",
"details": [
{
"name": "subnetId",
"value": "subnet-0a151f2eb959ad4"
},
{
"name": "networkInterfaceId",
"value": "eni-081948e3666253f"
},
{
"name": "macAddress",
"value": "02:2a:9i:5c:4a:77"
},
{
"name": "privateDnsName",
"value": "ip-172-56-17-177.us-west-2.compute.internal"
},
{
"name": "privateIPv4Address",
"value": "172.56.17.177"
}
]
}
],
"availabilityZone": "us-west-2a",
"clusterArn": "arn:aws:ecs:us-west-2:4984314772:cluster/secrets",
"containers": [
{
"taskArn": "arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b",
"name": "nginx",
"image": "nginx",
"lastStatus": "PENDING",
"networkInterfaces": [
{
"attachmentId": "da8a1312-8278-46d5-6b6a1d96f820",
"privateIpv4Address": "172.31.17.176"
}
],
"healthStatus": "UNKNOWN",
"cpu": "0"
}
],
"cpu": "256",
"createdAt": "2020-12-10T18:00:16.320000+05:30",
"desiredStatus": "RUNNING",
"group": "family:nginx",
"healthStatus": "UNKNOWN",
"lastStatus": "PENDING",
"launchType": "FARGATE",
"memory": "512",
"overrides": {
"containerOverrides": [
{
"name": "nginx"
}
],
"inferenceAcceleratorOverrides": []
},
"platformVersion": "1.4.0",
"tags": [],
"taskArn": "arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b",
"taskDefinitionArn": "arn:aws:ecs:us-west-2:4984314772:task-definition/nginx:17",
"version": 2
}
],
"failures": []
}
now if do an echo of $output.tasks[0].containers[0] nothing happens it prints the entire thing again, i want to store the result in output variable and refer different parameter like we do in json format.
You will need to use a json parser such as jq and so:
eval $cft | jq '.tasks[].containers[]'
To avoid using eval you could simple pipe the aws command into jq and so:
aws ecs describe-tasks --cluster arn:aws:ecs:us-west-2:4984314772:cluster/secrets --tasks arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b | jq '.tasks[].containers[]'
or:
cft=$(aws ecs describe-tasks --cluster arn:aws:ecs:us-west-2:4984314772:cluster/secrets --tasks arn:aws:ecs:us-west-2:4984314772:task/secrets/86855757eec4487f9d4475a1f7c4cb0b | jq '.tasks[].containers[]')
echo $cft | jq '.tasks[].containers[]'

How to verify that i get only one key from JSON response?

I get a response in JSON and need to make sure that i get only one 'signal' in signal_events. I'm using Ruby with rspec.
"signal_events": [
{
"id": "587e9ae969702d10bd5a0000",
"created_at": 1484692201,
"geo": {
"type": "Point",
"coordinates": [
-153.45703125,
59.67773438
]
},
"expires_at": 1484778601,
"geohashes": [
"bddg",
"bdeh"
],
"signal": {
"id": "587e9ae969702d0911060000",
"created_at": 1484692201.24,
"expires_at": 1484778601.24,
"signal_at": 1484691607,
"source": "usgs",
"updated_at": 1484692144,
"magnitude": 2,
"radius": 6.36107901750268,
"event_name": "earthquake",
"tsunami": "no"
},
"signal_type": "earthquake",
"centroid": {
"type": "Point",
"coordinates": [
-153.45703125,
59.67773438
]
},
"location": {
"country": "United States",
"country_code": "US",
"city": "Kenai Peninsula Borough",
"region": "Kenai Peninsula Borough",
"region_code": "AK"
}
},
Hash can not contain duplicated keys, so you will always have one value for any given key.

D3 - tree nodes not built from external JSON file

I'm having difficulty processing external JSON data to build/dispaly a tree. The issue is that nodes are not built from received JSON file. The code only looks as at the JSON data as one object and does not create leafs(other nodes)
When adding the same JSON data as array inside HTML file, the same code creates nodes and displays the tree correctly.
Here is an extract of the code in question:
<body onload="myFunction()">
<script type="text/javascript" >
function myFunction()
{
d3.json("/data/mydata.json", function (error, data) {
var root2 = [data];
root = root2[0];
........... other stuff comes here .....
var nodes = tree.nodes(root.reverse());
The browser debugger gives me the following:
nodes
[Array[1]0: Object
childern: Array[2]
name: "Max"
parent: "null"
proto: Object
depth: 0
length: 1
x: 230
y: 0
proto: Array[0]
JOSN file is at the bottom
I appreciate your feedback in this regard.
[{
"name": "Max",
"parent": "null",
"size": 4938,
"childern": [
{
"name": "David",
"parent": "Max",
"Childern": [
{
"name": "Craig",
"parent": "David",
"size": 3938
},
{
"name": "Robin",
"parent": "David",
"size": 3812
},
{
"name": "Anna",
"parent": "David",
"size": 743
}
]
},
{
"name": "Peter",
"parent": "Max",
"Childern": [
{
"name": "Jeff",
"parent": "Peter",
"size": 3534
},
{
"name": "Buffy",
"parent": "Peter",
"size": 5731
}
]
}
]
}
]
Your external JSON file appears to have the 'children' element with a capital 'C' for the lower tiers of data, but with lower case 'c' at the top tier.
Additionally they were misspelled as 'Childern' which may have been causing problems (you will want to check your JavaScript to see how you spelled it there and make it consistent).
I was able to do a quick experiment with an external JSON file and I can confirm that you will get a problem similar to what you describe. So use lower case characters (or at least consistent characters and spelling) for the word 'children' in your data.
Try with the data per below;
[{
"name": "Max",
"parent": "null",
"size": 4938,
"children": [
{
"name": "David",
"parent": "Max",
"children": [
{
"name": "Craig",
"parent": "David",
"size": 3938
},
{
"name": "Robin",
"parent": "David",
"size": 3812
},
{
"name": "Anna",
"parent": "David",
"size": 743
}
]
},
{
"name": "Peter",
"parent": "Max",
"children": [
{
"name": "Jeff",
"parent": "Peter",
"size": 3534
},
{
"name": "Buffy",
"parent": "Peter",
"size": 5731
}
]
}
]
}