Related
Question: I would like to know how I can iterate over an object to print out the name of each subnet and its cidr range in the subnets object.
I know how I can print out the vnet name and cidr range:
jq '.[] | select(.vnet) | {name: .vnet.name, cidr: .vnet.address_space[0]}'
I have looked online and found that you can convert the object to an array, but I am unsure how I can look through that array to print out the values:
jq '.[] | to_entries | map_values(.value)'
Essentially the desired OUTPUT should look something like this:
{
"name": "asdf1",
"cidr": "10....."
},
{
"name": "asdf2",
"cidr": "10....."
}
//...and so forth
Sample INPUT:
{
"route_tables": {
"asdf": {
"disable_bgp_route_propagation": true,
"name": "az_afw",
"resource_group_name": "vnet-spoke",
"route_entries": {
"re1": {
"name": "rt-rfc-10-8",
"next_hop_in_ip_address": "10.0.0.0",
"next_hop_type": "VirtualAppliance",
"prefix": "10.0.0.0/8"
},
"re2": {
"name": "rt-rfc-172-12",
"next_hop_in_ip_address": "10.0.0.0",
"next_hop_type": "VirtualAppliance",
"prefix": "172.16.0.0/12"
}
}
}
},
"vnet_peering_settings": {
"peer1": {
"peer_to_source": {
"allow_forwarded_traffic": true,
"allow_gateway_transit": false,
"allow_virtual_network_access": true,
"use_remote_gateways": true
},
"source_to_peer": {
"allow_forwarded_traffic": true,
"allow_gateway_transit": true,
"allow_virtual_network_access": true,
"use_remote_gateways": false
}
}
},
"vnet_spoke_object": {
"specialsubnets": {},
"subnets": {
"objectAsdf1": {
"cidr": "10.0.0.1/24",
"enforce_private_link_endpoint_network_policies": false,
"enforce_private_link_service_network_policies": false,
"name": "asdf1",
"nsg_creation": true,
"nsg_inbound": [],
"nsg_outbound": [],
"route": null,
"service_endpoints": []
},
"objectAsdf2": {
"cidr": "10.0.0.1/24",
"enforce_private_link_endpoint_network_policies": false,
"enforce_private_link_service_network_policies": false,
"name": "asdf2",
"nsg_creation": true,
"nsg_inbound": [],
"nsg_outbound": [],
"route": "asdf",
"service_endpoints": [
"Microsoft.EventHub"
]
},
"objectAsdf3": {
"cidr": "10.0.0.1/24",
"enforce_private_link_endpoint_network_policies": false,
"enforce_private_link_service_network_policies": false,
"name": "asdf3",
"nsg_creation": true,
"nsg_inbound": [],
"nsg_outbound": [],
"route": "asdf",
"service_endpoints": []
},
"objectAsdf4": {
"cidr": "10.0.0.1/24",
"enforce_private_link_endpoint_network_policies": false,
"enforce_private_link_service_network_policies": false,
"name": "asdf4",
"nsg_creation": true,
"nsg_inbound": [],
"nsg_outbound": [],
"route": "asdf",
"service_endpoints": []
}
},
"vnet": {
"address_space": [
"10.0.0.0/16"
],
"ddos_id": "placeholder",
"dns": [
"10.0.0.1",
"10.0.0.1"
],
"enable_ddos_std": false,
"name": "asdf"
}
}
}
Start with the root path and traverse down and use to_entries to get past the varying key names and use the .value field only
jq '.vnet_spoke_object.subnets | to_entries[].value | { name, cidr }'
or collect it as an array of objects
jq '.vnet_spoke_object.subnets | to_entries | map(.value | { name, cidr })'
jqplay - Demo
I have a json file with the following input
{
"Arg":"room=Rhasspy rhasspyName",
"Results": [
{
"Name":"TV",
"Internals": { },
"Readings": { },
"Attributes": { "rhasspyName": "TV" }
},
{
"Name":"dyTest01",
"Internals": { },
"Readings": { },
"Attributes": { "rhasspyName": "radio" }
},
{
"Name":"enoAcPC01",
"Internals": { },
"Readings": { },
"Attributes": { "rhasspyName": "pc" }
} ],
"totalResultsReturned":3
}
With jq '.Results | .[] | .["Attributes"] | .rhasspyName' -r I can get a list like
TV
radio
pc
How can I take this input and create a new json looking like
{"Devices":["TV","radio","pc"]}
Put them into an array and pair that with Devices key in an object.
$ jq '{Devices:[.Results[].Attributes.rhasspyName]}' file
{
"Devices": [
"TV",
"radio",
"pc"
]
}
To create a new file with that JSON value, redirect JQ's stdout to a file, like:
jq '{Devices:[.Results[].Attributes.rhasspyName]}' file > newfile
I have read properties with jq from a json object and have stored them to variables.
I want to now read these variables and essentially find and replace a word inside the string with a global shell variable.
I've set my json ID's from my JSON file
# Set Json ID's
TARGET_ID=$(jq '.DefaultCacheBehavior.TargetOriginId' distconfig.json)
DOMAIN_NAME=$(jq '.Origins.Items[0].DomainName' distconfig.json)
ORIGIN_ID=$(jq '.Origins.Items[0].Id' distconfig.json)
echo "$TARGET_ID"
echo "$DOMAIN_NAME"
echo "$ORIGIN_ID"
This returns
"S3-Website-stag4.example.io.s3-website.us-east-2.amazonaws.com"
"stag4.example.io.s3-website.us-east-2.amazonaws.com"
"S3-Website-stag4.example.io.s3-website.us-east-2.amazonaws.com"
I have my location id variable and would like to write it to find and replace all stag4 references in those 3 ID's.
Then I would like to write those 3 ID's to the initial json object, or create a temp version of it.
Example, if:
$DOMAIN_NAME is"stag4.example.io.s3-website.us-east-2.amazonaws.com"
I would like to essentially have it set to:
$LOCATION_NAME="stag6"
DOMAIN_LOCATION="example.io"
"$DOMAIN_NAME=S3-Website-\$LOCATION_NAME\.example.io.s3-website.us-east-2.amazonaws.com"
"$TARGET_ID=\$LOCATION_NAME\.example.io.s3-website.us-east-2.amazonaws.com"
"$ORIGIN_ID=S3-Website-\$LOCATION_NAME\.example.io.s3-website.us-east-2.amazonaws.com"
Then write those 3 to the temp or new json file so I can run my cloudformation command:
aws cloudfront create-distribution --distribution-config file://disttemp.json
I have now built out the proper variables from the initial json file like so:
$LOCATION_NAME="stag6"
DOMAIN_LOCATION="example.io"
echo "Build New IDs"
TARGET_ID_BUILT="S3-Website-$LOCATION_NAME.$DOMAIN_LOCATION.s3-website.us-east-2.amazonaws.com"
DOMAIN_NAME_BUILT="$LOCATION_NAME.$DOMAIN_LOCATION.s3-website.us-east-2.amazonaws.com"
ORIGIN_ID_BUILT="S3-Website-$LOCATION_NAME.$DOMAIN_LOCATION.s3-website.us-east-2.amazonaws.com"
echo "$TARGET_ID_BUILT"
echo "$DOMAIN_NAME_BUILT"
echo "$ORIGIN_ID_BUILT"
How do I write these variables to the json file with jq?
EDIT: Sample of distconfig.json requested – domain/creds swapped to example
{
"CallerReference": "my-test-distribution-2",
"Comment": "",
"CacheBehaviors": {
"Quantity": 0
},
"IsIPV6Enabled": true,
"Logging": {
"Bucket": "",
"Prefix": "",
"Enabled": false,
"IncludeCookies": false
},
"WebACLId": "",
"Origins": {
"Items": [
{
"OriginPath": "",
"CustomOriginConfig": {
"OriginSslProtocols": {
"Items": [
"TLSv1",
"TLSv1.1",
"TLSv1.2"
],
"Quantity": 3
},
"OriginProtocolPolicy": "http-only",
"OriginReadTimeout": 30,
"HTTPPort": 80,
"HTTPSPort": 443,
"OriginKeepaliveTimeout": 5
},
"CustomHeaders": {
"Quantity": 0
},
"Id": "S3-Website-stag4.example.io.s3-website.us-east-2.amazonaws.com",
"DomainName": "stag4.example.io.s3-website.us-east-2.amazonaws.com"
}
],
"Quantity": 1
},
}
"DefaultRootObject": "",
"PriceClass": "PriceClass_All",
"Enabled": true,
"DefaultCacheBehavior": {
"TrustedSigners": {
"Enabled": false,
"Quantity": 0
},
"LambdaFunctionAssociations": {
"Quantity": 0
},
"TargetOriginId": "S3-Website-stag4.example.io.s3-website.us-east-2.amazonaws.com",
"ViewerProtocolPolicy": "redirect-to-https",
"ForwardedValues": {
"Headers": {
"Quantity": 0
},
"Cookies": {
"Forward": "none"
},
"QueryStringCacheKeys": {
"Quantity": 0
},
"QueryString": false
},
"MaxTTL": 31536000,
"SmoothStreaming": false,
"DefaultTTL": 86400,
"AllowedMethods": {
"Items": [
"HEAD",
"GET"
],
"CachedMethods": {
"Items": [
"HEAD",
"GET"
],
"Quantity": 2
},
"Quantity": 2
},
"MinTTL": 0,
"Compress": true
},
"ViewerCertificate": {
"SSLSupportMethod": "sni-only",
"ACMCertificateArn": "xxxx",
"MinimumProtocolVersion": "TLSv1.1_2016",
"Certificate": "xxxx",
"CertificateSource": "acm"
},
"CustomErrorResponses": {
"Quantity": 0
},
"HttpVersion": "http2",
"Restrictions": {
"GeoRestriction": {
"RestrictionType": "none",
"Quantity": 0
}
},
"Aliases": {
"Quantity": 0
}
}
You should use sed to do the substitution and then inject the value back into the JSON.
echo $TARGET_ID | sed 's/stag4/stag5/g'
Outputs
S3-Website-stag5.example.io.s3-website.us-east-2.amazonaws.com
Next we'll put the value back into the original JSON, this will technically output a new JSON and does not edit the file, however, you can easily solve for this on the output by temporarily saving to a tmp file.
We will use the --arg flag to reference our bash variable and set the new value for our field
cat distconfig.json | jq --arg TARGET_ID $TARGET_ID '.DefaultCacheBehavior.TargetOriginId = $TARGET_ID' > tmp.json && mv tmp.json distconfig.json
The examples here don't go nearly far enough in explaining how to produce a more complicated structure...
If I want to end up with something like:
{
"data": {
"type": "mobile_screens",
"id": "1",
"attributes": {
"title": "Watch"
},
"relationships": {
"mobile_screen_components": {
"data": [
{
"id": "1_1",
"type": "mobile_screen_components"
},
{
"id": "1_2",
"type": "mobile_screen_components"
},
...
]
}
}
},
"included": [
{
"id": "1_1",
"type": "mobile_screen_components",
"attributes": {
"title": "Featured Playlist",
"display_type": "shelf"
},
"relationships": {
"playlist": {
"data": {
"id": "938973798001",
"type": "playlists"
}
}
}
},
{
"id": "938973798001",
"type": "playlists",
"relationships": {
"videos": {
"data": [
{
"id": "5536725488001",
"type": "videos"
},
{
"id": "5535943875001",
"type": "videos"
}
]
}
}
},
{
"id": "5536725488001",
"type": "videos",
"attributes": {
"duration": 78321,
"live_stream": false,
"thumbnail": {
"width": 1280,
"url":
"http://xxx.jpg?pubId=694940094001",
"height": 720
},
"last_published_date": "2017-08-09T18:26:04.899Z",
"streams": [
{
"url":
"http://xxx.m3u8",
"mime_type": "MP4"
}
],
"last_modified_date": "2017-08-09T18:26:27.621Z",
"description": "xxx",
"fn__media_tags": [
"weather",
"personality"
],
"created_date": "2017-08-09T18:23:16.830Z",
"title": "NOAA predicts most active hurricane season since 2010",
"fn__tve_authentication_required": false
}
},
...,
]
}
what is the most simple data structure and serializer I can set up?
I get stumped after something like:
const mobile_screen_components = responses.map((currentValue, index) => {
id[`id_${index}`];
});
const dataSet = {
id: 1,
title: 'Watch',
mobile_screen_components,
};
const ScreenSerializer = new JSONAPISerializer('mobile_screens', {
attributes: ['title', 'mobile_screen_components'],
mobile_screen_components: {
ref: 'id',
}
});
Which only gives me:
{
"data": {
"type": "mobile_screens",
"id": "1",
"attributes": { "title": "Watch" },
"relationships": {
"mobile-screen-components": {
"data": [
{ "type": "mobile_screen_components", "id": "1_0" },
{ "type": "mobile_screen_components", "id": "1_1" },
{ "type": "mobile_screen_components", "id": "1_2" },
{ "type": "mobile_screen_components", "id": "1_3" },
{ "type": "mobile_screen_components", "id": "1_4" },
{ "type": "mobile_screen_components", "id": "1_5" }
]
}
}
}
}
I have no idea how to get the "included" sibling to "data." etc.
So, the question is:
what is the most simple data structure and serializer I can set up?
Below is the simplest object that can be converted to JSON similar to JSON in the question using jsonapi-serializer:
let dataSet = {
id: '1',
title: 'Watch',
mobile_screen_components: [
{
id: '1_1',
title: 'Featured Playlists',
display_type: 'shelf',
playlists: {
id: 938973798001,
videos: [
{
id: 5536725488001,
duration: 78321,
live_stream: false
},
{
id: 5535943875001,
duration: 52621,
live_stream: true
}
]
}
}
]
};
To serialize this object to JSON API, I used the following code:
let json = new JSONAPISerializer('mobile_screen', {
attributes: ['id', 'title', 'mobile_screen_components'],
mobile_screen_components: {
ref: 'id',
attributes: ['id', 'title', 'display_type', 'playlists'],
playlists: {
ref: 'id',
attributes: ['id', 'videos'],
videos: {
ref: 'id',
attributes: ['id', 'duration', 'live_stream']
}
}
}
}).serialize(dataSet);
console.log(JSON.stringify(json, null, 2));
The first parameter of JSONAPISerializer constructor is the resource type.
The second parameter is the serialization options.
Each level of the options equals to the level of the nested object in serialized object.
ref - if present, it's considered as a relationships.
attributes - an array of attributes to show.
Introduction
First of all we have to understand the JSON API document data structure
[0.1] Refering to the top level (object root keys) :
A document MUST contain at least one of the following top-level
members:
data: the document’s “primary data”
errors: an array of error objects
meta: a meta object that contains non-standard meta-information.
A document MAY contain any of these top-level members:
jsonapi: an object describing the server’s implementation
links: a links object related to the primary data.
included: an array of resource objects that are related to the primary data and/or each other (“included resources”).
[0.2]
The document’s “primary data” is a representation of the resource or
collection of resources targeted by a request.
Primary data MUST be either:
a single resource identifier object, or
null, for requests that target single resources
an array of resource identifier
objects, or an empty array ([]), for reqs. that target
collections
Example
The following primary data is a single resource object:
{
"data": {
"type": "articles",
"id": "1",
"attributes": {
// ... this article's attributes
},
"relationships": {
// ... this article's relationships
}
}
}
In the (jsonapi-serializer) documentation : Available serialization option (opts argument)
So in order to add the included (top-level member) I performed the following test :
var JsonApiSerializer = require('jsonapi-serializer').Serializer;
const DATASET = {
id:23,title:'Lifestyle',slug:'lifestyle',
subcategories: [
{description:'Practices for becoming 31337.',id:1337,title:'Elite'},
{description:'Practices for health.',id:69,title:'Vitality'}
]
}
const TEMPLATE = {
topLevelLinks:{self:'http://example.com'},
dataLinks:{self:function(collection){return 'http://example.com/'+collection.id}},
attributes:['title','slug','subcategories'],
subcategories:{ref:'id',attributes:['id','title','description']}
}
let SERIALIZER = new JsonApiSerializer('pratices', DATASET, TEMPLATE)
console.log(SERIALIZER)
With the following output :
{ links: { self: 'http://example.com' },
included:
[ { type: 'subcategories', id: '1337', attributes: [Object] },
{ type: 'subcategories', id: '69', attributes: [Object] } ],
data:
{ type: 'pratices',
id: '23',
links: { self: 'http://example.com/23' },
attributes: { title: 'Lifestyle', slug: 'lifestyle' },
relationships: { subcategories: [Object] } } }
As you may observe, the included is correctly populated.
NOTE : If you need more help with your dataSet, edit your question with the original data.
I try reading a JSON file from R using rjson but keep getting errors. I validated the JSON file using various online validators. Here is the content of the JSON file:
{
"scenarios": [
{
"files": {
"type1": "/home/blah/Desktop/temp/scen_0.type1",
"type2": "/home/blah/Desktop/temp/scen_0.type2"
},
"ID": "scen_0",
"arr": [],
"TypeToElementStatsFilename": {
"type1": "/home/blah/Desktop/temp/scen_0.type1.elements",
"type2": "/home/blah/Desktop/temp/scen_0.type2.elements"
}
}
],
"randomSeed": "39327314969888",
"zone": {
"length": 1000000,
"start": 1
},
"instanceFilename": "/home/blah/bloo/data/XY112.zip",
"txtFilename": "/home/blah/bloo/data/XY112.txt",
"nSimulations": 2,
"TypeTodbFilename": {
"type1": "/home/blah/bloo/data/map.type1.oneAmb.XY112.out"
},
"arr": {
"seg11": {
"length": 1000,
"start": 147000
},
"seg12": {
"length": 1000,
"start": 153000
},
"seg5": {
"length": 1000,
"start": 145000
},
"seg6": {
"length": 1000,
"start": 146000
},
"seg1": {
"length": 100,
"start": 20000
}
},
"outPath": "/home/blah/Desktop/temp",
"instanceID": "XY112",
"arrIds": [
"seg5",
"seg6",
"seg1",
"seg11",
"seg12"
],
"truth": {
"files": {
"type1": "/home/blah/Desktop/temp/truth.type1",
"type2": "/home/blah/Desktop/temp/truth.type2"
},
"ID": "truth",
"TypeToElementStatsFilename": {
"type1": "/home/blah/Desktop/temp/truth.type1.elements",
"type2": "/home/blah/Desktop/temp/truth.type2.elements"
}
}
}
And the error:
> json_file <- "~/json"
> json_data <- fromJSON(paste(readLines(json_file), collapse=""))
Error in fromJSON(paste(readLines(json_file), collapse = "")) :
unexpected character: :
RJSON freaks out about empty arrays.
fromJSON( '{ "arr": [ ] }')
Error in fromJSON("{ \"arr\": [ ] }") : unexpected character: :
You can try the fromJSON function in the RJSONIO package hosted at http://www.omegahat.org. It seems to read the file fine.
There's a fix for this.
Create a new function to replace the existing getURL function used in RCurl and you should have your solution.
myGetURL <- function(...) {
rcurlEnv <- getNamespace("RCurl")
mapUnicodeEscapes <- get("mapUnicodeEscapes", rcurlEnv)
unlockBinding("mapUnicodeEscapes", rcurlEnv)
assign("mapUnicodeEscapes", function(str) str, rcurlEnv)
on.exit({
assign("mapUnicodeEscapes", mapUnicodeEscapes, rcurlEnv)
lockBinding("mapUnicodeEscapes", rcurlEnv)
}, add = TRUE)
return(getURL(...))
}
Test:
> json <- myGetURL("http://abicky.net/hatena/rcurl/a.json")
> cat(json, fill = TRUE)
{"a":"\\\"\u0030\\\""}
> fromJSON(json)
$a
[1] "\\\"0\\\""