Convert a JSON value to another JSON shape? - json

I have the following JSON file that I need to transform.
Original JSON file:
{
"configs": [
{
"ignore": "this",
"key": "abc123",
"default": false
},
{
"ignore": "that",
"key": "def123",
"default": "#F5F5F5"
},
{
"type": "color",
"key": "ghi123",
"default": "#3d4246"
}
]
}
I want to convert the JSON to look like this:
{
"configs": [
{
"abc123": false
},
{
"def123": "#F5F5F5"
},
{
"ghi123": "#3d4246"
}
]
}
How can I do this?
require 'json'
original_json = JSON.parse("...") #original JSON shape

There's likely a cleaner way, but here's a shot:
require 'json'
original_json = JSON.parse('{
"configs": [
{
"ignore": "this",
"key": "abc123",
"default": false
},
{
"ignore": "that",
"key": "def123",
"default": "#F5F5F5"
},
{
"type": "color",
"key": "ghi123",
"default": "#3d4246"
}
]
}')
original_json['configs'].each do |h|
k = h["key"]
v = h["default"]
h.clear
h[k] = v
end
original_json.to_json
#=> {"configs":[{"abc123":false},{"def123":"#F5F5F5"},{"ghi123":"#3d4246"}]}

With Enumerable#each_with_object you can create new hash as you want. Then you can convert it to json.
original_hash = JSON.parse(original_json)
new_hash = original_hash[:configs].each_with_object({ configs: [] }) do |item, new_hash|
new_hash[:configs] << {
item[:key] => item[:default]
# add another key value pair if you want
}
end
new_hash.to_json
#=> {
# "configs": [
# {
# "abc123": false
# },
# {
# "def123": "#F5F5F5"
# },
# {
# "ghi123": "#3d4246"
# }
# ]
# }

Related

How to search a JSON hash using a Regex?

I'm using an API to return all Macros to me, I am trying to return all the "macros" where "actions" contains a "value" matching my Regexp Pattern which I will link below.
I've tried below and other methods, but it returns me nil for present values. Any tips appreciated
macros["value"].select { |m| m['key'] == 'value' }.first['/^DE([0-9a-zA-Z]\s?){20}$/gm']
API result snippet:
jsObj
=> {"macros"=>
[{"url"=>"https://s/1900002708354.json",
"id"=>1900002708354,
"title"=>"Append Signature",
"active"=>true,
"updated_at"=>"2021-10-22T14:11:15Z",
"created_at"=>"2021-10-22T14:11:15Z",
"position"=>10001,
"description"=>"This macro appends a signature to the message ",
"actions"=>[{"field"=>"comment_value_html", "value"=>"<p>Mit besten Grüßen,</p><p>{{current_user.name}} [{{ticket.account}}] <br></p><p><br></p><p>{{dc.signature_email}}<br></p><p><br></p>"}],
"restriction"=>nil},
{"url"=>"949.json",
"id"=>59071949,
"title"=>"information",
"description"=>nil,
"actions"=>[{"field"=>"priority", "value"=>"low"}, {"field"=>"comment_value", "value"=>"DE89370400440532013000" "DE89 3704
0044 0532 0130 00"
"}],
"restriction"=>nil},
Desired Result:
{
"macros": [
{
"url": "x.json",
"id": 1900002708354,
"actions": [
{
"field": "comment_value_html",
"value": "DE89 3704 0044 0532 0130 00"
}
],
"restriction": null
},
{
"url": "x.json",
"id": 59071949,
"actions": [
{
"field": "priority",
"value": "low"
},
{
"field": "comment_value",
"value": "DE89 3704 0044 0532 0130 00
"
}
],
"restriction": null
},
Given that macros is the JSON object containing the macros data, you can use
macros.select { |m| m["actions"].any? { |w| /\ADE(?:[0-9a-zA-Z]\s?){20}\z/.match?(w["value"]) } }
Here is a Ruby demo:
require 'json'
j = <<-DATA
{
"macros": [
{
"url": "x.json",
"id": 1900002708354,
"actions": [
{
"field": "comment_value_html",
"value": "DE11111111111222222220"
}
],
"restriction": null
},
{
"url": "x.json",
"id": 59071949,
"actions": [
{
"field": "priority",
"value": "low"
},
{
"field": "comment_value",
"value": "DE12345678901234567890"
}
],
"restriction": null
}
]}
DATA
jsObj = JSON.parse(j)
macros = jsObj['macros']
puts jsObj['macros'].select { |m| m["actions"].any? { |w| /\ADE(?:[0-9a-zA-Z]\s?){20}\z/.match?(w["value"]) } }
Output:
{"url"=>"x.json", "id"=>1900002708354, "actions"=>[{"field"=>"comment_value_html", "value"=>"DE11111111111222222220"}], "restriction"=>nil}
{"url"=>"x.json", "id"=>59071949, "actions"=>[{"field"=>"priority", "value"=>"low"}, {"field"=>"comment_value", "value"=>"DE12345678901234567890"}], "restriction"=>nil}
The .select { |m| m["actions"].any? { |w| /\ADE(?:[0-9a-zA-Z]\s?){20}\z/.match?(w["value"]) } } main part gets all actions nodes that contain an array with a value key whose value matches the regex given.

How to get required json output from complex nested json format.?

My original file is in CSV format which I have converted to python JSON array to JSON Sring.
jsonfile
<class 'list'>
<class 'dict'>
[
{
"key": "timestamp",
"source": "eia007",
"turnover": "65million",
"url": "abc.com",
"record": "",
"loc.reg": "nord000",
"loc.count": "abs39i5",
"loc.town": "cold54",
"co.gdp": "nscrt77",
"co.pop.min": "min50",
"co.pop.max": "max75",
"co.rev": "",
"chain.system": "5t5t5",
"chain.type": "765ef",
"chain.strat": "",
}
]
I would like to get the output as below:
{
"timestamp001": {
"key": "timestamp001",
"phNo": "ner007",
"turnover": "65million",
"url": "abc.com",
"record": "",
"loc": {
"reg": "nord000",
"count": "abs39i5",
"town": "cold54"
},
"co": {
"form": "nscrt77",
"pop": {
"min": "min50",
"max": "max75"
},
"rev: ""
},
"chain":{
"system": "5t5t5",
"type": "765ef",
"strat": ""
}
...
}
...
}
]
I have tried different options; tried to enumerate, but cannot get the required output. Please help me with this. Thanks in advance.
You can use something like this to create the nested dict:
import json
def unflatten(somedict):
unflattened = {}
for key, value in somedict.items():
splitkey = key.split(".")
print(f"doing {key} {value} {splitkey}")
# subdict is the dict that goes deeper in the nested structure
subdict = unflattened
for subkey in splitkey[:-1]:
# if this is the first time we see this key, add it
if subkey not in subdict:
subdict[subkey] = {}
# shift the subdict a level deeper
subdict = subdict[subkey]
# add the value
subdict[splitkey[-1]] = value
return unflattened
data = {
"key": "timestamp",
"source": "eia007",
"turnover": "65million",
"url": "abc.com",
"record": "",
"loc.reg": "nord000",
"loc.count": "abs39i5",
"loc.town": "cold54",
"co.gdp": "nscrt77",
"co.pop.min": "min50",
"co.pop.max": "max75",
"co.rev": "",
"chain.system": "5t5t5",
"chain.type": "765ef",
"chain.strat": "",
}
unflattened = unflatten(data)
print(json.dumps(unflattened, indent=4))
Which produces:
{
"key": "timestamp",
"source": "eia007",
"turnover": "65million",
"url": "abc.com",
"record": "",
"loc": {
"reg": "nord000",
"count": "abs39i5",
"town": "cold54"
},
"co": {
"gdp": "nscrt77",
"pop": {
"min": "min50",
"max": "max75"
},
"rev": ""
},
"chain": {
"system": "5t5t5",
"type": "765ef",
"strat": ""
}
}
Cheers!

Reconstructing JSON with jq

I have a JSON like this (sample.json):
{
"sheet1": [
{
"hostname": "sv001",
"role": "web",
"ip1": "172.17.0.3"
},
{
"hostname": "sv002",
"role": "web",
"ip1": "172.17.0.4"
},
{
"hostname": "sv003",
"role": "db",
"ip1": "172.17.0.5",
"ip2": "172.18.0.5"
}
],
"sheet2": [
{
"hostname": "sv004",
"role": "web",
"ip1": "172.17.0.6"
},
{
"hostname": "sv005",
"role": "db",
"ip1": "172.17.0.7"
},
{
"hostname": "vsv006",
"role": "db",
"ip1": "172.17.0.8"
}
],
"sheet3": []
}
I want to extract data like this:
sheet1
jq '(something command)' sample.json
{
"web": {
"hosts": [
"172.17.0.3",
"172.17.0.4"
]
},
"db": {
"hosts": [
"172.17.0.5"
]
}
}
Is it possible to perform the reconstruction with jq map?
(I will reuse the result for ansible inventory.)
Here's a short, straight-forward and efficient solution -- efficient in part because it avoids group_by by courtesy of the following generic helper function:
def add_by(f;g): reduce .[] as $x ({}; .[$x|f] += [$x|g]);
.sheet1
| add_by(.role; .ip1)
| map_values( {hosts: .} )
Output
This produces the required output:
{
"web": {
"hosts": [
"172.17.0.3",
"172.17.0.4"
]
},
"db": {
"hosts": [
"172.17.0.5"
]
}
}
If the goal is to regroup the ips by their roles within each sheet you could do this:
map_values(
reduce group_by(.role)[] as $g ({};
.[$g[0].role].hosts = [$g[] | del(.hostname, .role)[]]
)
)
Which produces something like this:
{
"sheet1": {
"db": {
"hosts": [
"172.17.0.5",
"172.18.0.5"
]
},
"web": {
"hosts": [
"172.17.0.3",
"172.17.0.4"
]
}
},
"sheet2": {
"db": {
"hosts": [
"172.17.0.7",
"172.17.0.8"
]
},
"web": {
"hosts": [
"172.17.0.6"
]
}
},
"sheet3": {}
}
https://jqplay.org/s/3VpRc5l4_m
If you want to flatten all to a single object keeping only unique ips, you can keep everything mostly the same, you'll just need to flatten the inputs prior to grouping and remove the map_values/1 call.
$ jq -n '
reduce ([inputs[][]] | group_by(.role)[]) as $g ({};
.[$g[0].role].hosts = ([$g[] | del(.hostname, .role)[]] | unique)
)
'
{
"db": {
"hosts": [
"172.17.0.5",
"172.17.0.7",
"172.17.0.8",
"172.18.0.5"
]
},
"web": {
"hosts": [
"172.17.0.3",
"172.17.0.4",
"172.17.0.6"
]
}
}
https://jqplay.org/s/ZGj1wC8hU3

Parse Complex JSON -- Map

I need to parse the complex JSON (below) IN SCALA to get the values of "expression" and "value" in "measure" key i.e I need List (COUNT, COUNT_DISTINCT ...) and List (1,tbl1.USER_ID ...).
I tried multiple options, but it is not working. Any help is appreciated
{
"uuid": "uuidddd",
"last_modified": 1559080222953,
"version": "2.6.1.0",
"name": "FULL_DAY_2_mand_date",
"is_draft": false,
"model_name": "FULL_DAY_1_may05",
"description": "",
"null_string": null,
"dimensions": [
{
"name": "PLATFORM",
"table": "tbl1",
"column": "PLATFORM",
"derived": null
},
{
"name": "OS_VERSION",
"table": "tbl1",
"column": "OS_VERSION",
"derived": null
}
],
"measures": [
{
"name": "_COUNT_",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1"
},
"returntype": "bigint"
}
},
{
"name": "UU",
"function": {
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
"value": "tbl1.USER_ID"
},
"returntype": "hllc(12)"
}
},
{
"name": "CONT_SIZE",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "tbl1.SIZE"
},
"returntype": "bigint"
}
},
{
"name": "CONT_COUNT",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "tbl1.COUNT"
},
"returntype": "bigint"
}
}
],
"dictionaries": [],
"rowkey": {
"rowkey_columns": [
{
"column": "tbl1.OS_VERSION",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "tbl1.PLATFORM",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "tbl1.DEVICE_FAMILY",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
}
]
},
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"_COUNT_",
"CONT_SIZE",
"CONT_COUNT"
]
}
]
},
{
"name": "F2",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"UU"
]
}
]
}
]
},
"aggregation_groups": [
{
"includes": [
"tbl1.PLATFORM",
"tbl1.OS_VERSION"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [
"tbl1.DATE_HR"
],
"joint_dims": []
}
}
],
"signature": "ttrrs==",
"notify_list": [],
"status_need_notify": [
"ERROR",
"DISCARDED",
"SUCCEED"
],
"partition_date_start": 0,
"partition_date_end": 3153600000000,
"auto_merge_time_ranges": [
604800000,
2419200000
],
"volatile_range": 0,
"retention_range": 0,
"engine_type": 4,
"storage_type": 2,
"override_kylin_properties": {
"job.queuename": "root.production.P0",
"is-mandatory-only-valid": "true"
},
"cuboid_black_list": [],
"parent_forward": 3,
"mandatory_dimension_set_list": [],
"snapshot_table_desc_list": []
}
This is a snippet of the code I tried, and it is giving a null list
import org.json4s._
import org.json4s.jackson.JsonMethods._
implicit val formats = org.json4s.DefaultFormats
case class Function (
expression: String,
parameter: Parameter,
returntype: String
)
case class Parameter (
`type`: String,
value: String
)
case class Measures (
name: String,
function: Function
)
case class AllMeasuresData(uuid: String, measure: List[Measures])
val data = parse(tmp).extract[AllMeasuresData]
val names = data.measure.map(_.name)
println(names)
case class AllMeasuresData(uuid: String, measure: List[Measures])
val data = parse(tmp).extract[AllMeasuresData]
val names = data.measure.map(_.name)
println(names)
There are couple typos in your ADT:
Here is what you need:
case class Function (
expression: String,
parameter: Parameter,
returntype: String
)
case class Parameter (
`type`: String,
value: String
)
case class Measures (
name: String,
function: Function
)
case class AllMeasuresData(uuid: String, measures: List[Measures])
There is also an extra comma int the json, here is the correct:
{
"uuid":"uuidddd",
"last_modified":1559080222953,
"version":"2.6.1.0",
"name":"FULL_DAY_2_mand_date",
"is_draft":false,
"model_name":"FULL_DAY_1_may05",
"description":"",
"null_string":null,
"dimensions":[
{
"name":"PLATFORM",
"table":"tbl1",
"column":"PLATFORM",
"derived":null
},
{
"name":"OS_VERSION",
"table":"tbl1",
"column":"OS_VERSION",
"derived":null
} // There was an extra trailing comma here
],
"measures":[
{
"name":"_COUNT_",
"function":{
"expression":"COUNT",
"parameter":{
"type":"constant",
"value":"1"
},
"returntype":"bigint"
}
},
{
"name":"UU",
"function":{
"expression":"COUNT_DISTINCT",
"parameter":{
"type":"column",
"value":"tbl1.USER_ID"
},
"returntype":"hllc(12)"
}
},
{
"name":"CONT_SIZE",
"function":{
"expression":"SUM",
"parameter":{
"type":"column",
"value":"tbl1.SIZE"
},
"returntype":"bigint"
}
},
{
"name":"CONT_COUNT",
"function":{
"expression":"SUM",
"parameter":{
"type":"column",
"value":"tbl1.COUNT"
},
"returntype":"bigint"
}
}
],
"dictionaries":[
],
"rowkey":{
"rowkey_columns":[
{
"column":"tbl1.OS_VERSION",
"encoding":"dict",
"encoding_version":1,
"isShardBy":false
},
{
"column":"tbl1.PLATFORM",
"encoding":"dict",
"encoding_version":1,
"isShardBy":false
},
{
"column":"tbl1.DEVICE_FAMILY",
"encoding":"dict",
"encoding_version":1,
"isShardBy":false
}
]
},
"hbase_mapping":{
"column_family":[
{
"name":"F1",
"columns":[
{
"qualifier":"M",
"measure_refs":[
"_COUNT_",
"CONT_SIZE",
"CONT_COUNT"
]
}
]
},
{
"name":"F2",
"columns":[
{
"qualifier":"M",
"measure_refs":[
"UU"
]
}
]
}
]
},
"aggregation_groups":[
{
"includes":[
"tbl1.PLATFORM",
"tbl1.OS_VERSION"
],
"select_rule":{
"hierarchy_dims":[
],
"mandatory_dims":[
"tbl1.DATE_HR"
],
"joint_dims":[
]
}
}
],
"signature":"ttrrs==",
"notify_list":[
],
"status_need_notify":[
"ERROR",
"DISCARDED",
"SUCCEED"
],
"partition_date_start":0,
"partition_date_end":3153600000000,
"auto_merge_time_ranges":[
604800000,
2419200000
],
"volatile_range":0,
"retention_range":0,
"engine_type":4,
"storage_type":2,
"override_kylin_properties":{
"job.queuename":"root.production.P0",
"is-mandatory-only-valid":"true"
},
"cuboid_black_list":[
],
"parent_forward":3,
"mandatory_dimension_set_list":[
],
"snapshot_table_desc_list":[
]
}
Now you can run:
val data = parse(tmp).extract[AllMeasuresData]
val names = data.measures.map(_.name)
println(names)
// Displays
// List(_COUNT_, UU, CONT_SIZE, CONT_COUNT)
Your Parameter class does not match the JSON because you have used type1 rather than type as the field name. Use backticks to use "type" as a field name even though it is a reserved word:
case class Parameter (
`type`: String,
value: String
)
You also need to change the Function class as it has returntype1 rather than returntype:
case class Function (
expression: String,
parameter: Parameter,
returntype: String
)
The names of the fields in Scala must exactly match the names of the fields in the JSON. Extra fields in the JSON are ignored, but all the fields in the Scala must have matching fields in the JSON. If there are optional fields in the JSON then the Scala field type should be Option[...].

Convert Nested Dictionary to a graphable format

So I'm trying to convert a nested dictionary like:
A = {
"root":
{
"child1":
{
"child11":"hmm",
"child12":"not_hmm"
},
"child2":"hello"
}
}
To this:
{
"name":"root",
"children": [
{"name":"child1",
"children" :
[{"name":"child11",
"children":[{"name":"hmm"}]}
{"name":"child12",
"children":[{"name":"not_hmm"}]}
]
},
{"name":"child2",
"children":[{"name":"hello"}]
}
]
}
I need this, since I'm trying to visualize it with this graph drawing template: Collapsible Tree
I'm having some trouble creating a recursive method that is capable of this transformation.
Preferably in python3. So far I have:
def visit(node, parent=None):
B = {}
for k,v in node.items():
B["name"]=k
B["children"] = []
if isinstance(v,dict):
print("Key value pair is",k,v)
B["children"].append(visit(v,k))
new_dict = {}
new_dict["name"]=v
return [new_dict]
C = visit(A) # This should have the final result
But its wrong. Any help is appreciated.
We'll have a function that takes a root (assuming it has only one entry), and returns a dict, as well as a helper function that returns lists of dicts.
def convert(d):
for k, v in d.items():
return {"name": k, "children": convert_helper(v)}
def convert_helper(d):
if isinstance(d, dict):
return [{"name": k, "children": convert_helper(v)} for k, v in d.items()]
else:
return [{"name": d}]
which gives us
json.dumps(convert(A), indent=2)
{
"name": "root",
"children": [
{
"name": "child1",
"children": [
{
"name": "child11",
"children": [
{
"name": "hmm"
}
]
},
{
"name": "child12",
"children": [
{
"name": "not_hmm"
}
]
}
]
},
{
"name": "child2",
"children": [
{
"name": "hello"
}
]
}
]
}