Groovy collect nested elements along with outer element - json

Using Groovy, requirement is to collect a map's nested element values, along with its top-level element value.
Unsure if a recursive method is needed.
Sample JSON
{
"items": [
{
"attribute": "Type",
"options":
[
{
"label": "Type1",
"value": "1"
},
{
"label": "Type2",
"value": "2"
}
]
},
{
"attribute": "Size",
"options": [
{
"label": "SizeA",
"value": "A"
},
{
"label": "SizeB",
"value": "B"
}
]
}
]
}
Expected output after collect
[
{attribute=Type,label=Type1,value=1},
{attribute=Type,label=Type2,value=2},
{attribute=Size,label=SizeA,value=A},
{attribute=Size,label=SizeB,value=B}
]

You can solve this combining multiple lists of options obtained through the collect method using a collectMany.
See the following snippet of code:
def input = """
{
"items": [
{
"attribute": "Type",
"options": [
{
"label": "Type1",
"value": "1"
},
{
"label": "Type2",
"value": "2"
}
]
},
{
"attribute": "Size",
"options": [
{
"label": "SizeA",
"value": "A"
},
{
"label": "SizeB",
"value": "B"
}
]
} ]
}"""
def json = new groovy.json.JsonSlurper().parseText(input)
/* collectMany directly flatten the two sublists
* [ [ [ type1 ], [ type2 ] ], [ [ sizeA ], [ sizeB ] ] ]
* into
* [ [type1], [type2], [sizeA], [sizeB] ]
*/
def result = json.items.collectMany { item ->
// collect returns a list of N (in this example, N=2) elements
// [ [ attribute1: ..., label1: ..., value1: ... ],
// [ attribute2: ..., label2: ..., value2: ... ] ]
item.options.collect { option ->
// return in [ attribute: ..., label: ..., value: ... ]
[ attribute: item.attribute ] + option
}
}
assert result == [
[ attribute: "Type", label: "Type1", value: "1" ],
[ attribute: "Type", label: "Type2", value: "2" ],
[ attribute: "Size", label: "SizeA", value: "A" ],
[ attribute: "Size", label: "SizeB", value: "B" ],
]

Thanks to Giuseppe!
I had solved this using a less groovy method:
def result = [];//Create new list with top-level element and nested elements
json.items.each {item ->
result.addAll(item.options.collect{ option ->
[attribute: /"${item?.attribute?:''}"/, label: /"${option?.label?:''}"/, value: /"${option?.value?:''}"/ ]
})
};

Related

Jsonpath: get the list of elements that contain at least one id from a list of ids

With the following json:
{
"elements": [
{
"ids": [
{
"id": "A"
},
{
"id": "B"
}
],
"value": "one"
},
{
"ids": [
{
"id": "C"
},
{
"id": "D"
}
],
"value": "two"
},
{
"ids": [
{
"id": "E",
},
{
"id": "F",
}
],
"value": "three"
}
]
}
What would be the jsonpath to return the list of the elements containing at least an id from the list ['A','C']?
I can get the desired result if I ask specifically for each id:
$.elements[?('A' in #.ids.*.id || 'C' in #.ids.*.id)]
[
{
"ids" : [
{
"id" : "A"
},
{
"id" : "B"
}
],
"value" : "one"
},
{
"ids" : [
{
"id" : "C"
},
{
"id" : "D"
}
],
"value" : "two"
}
]
but in my scenario I need to indicate the values of the ids within a list ['A','C']
Thanks in advance!
use the anyof filter operator
$.elements[?(#.ids.*.id anyof ['A','C'])]

Efficient way of converting terraform JSON schema to AWS API JSON schema

I'm super new to Go and I'm trying to convert a JSON schema which contains terraform specific attribute names to AWS API specific JSON schema in Go.
For example:
The input is of type:
{
"egress": [
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Port 443",
"from_port": 443,
"protocol": "tcp",
"to_port": 443
}
],
"ingress": [
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Port 443",
"from_port": 443,
"protocol": "tcp",
"to_port": 443
}
],
"name": "my_sec_group",
"vpc_id": "${aws_vpc.my_vpc.id}"
}
And the desired output should be matching the AWS API object which would be something like this:
{
"SecurityGroups": [
{
"GroupName": "my_sec_group",
"IpPermissions": [
{
"FromPort": 443,
"IpProtocol": "tcp",
"IpRanges": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Port 443"
}
],
"ToPort": 443
}
],
"IpPermissionsEgress": [
{
"FromPort": 443,
"IpProtocol": "tcp",
"IpRanges": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Port 443"
}
],
"ToPort": 443
}
],
"VpcId": "${aws_vpc.my_vpc.id}"
}
]
}
What I have tried till now:
I. Create a generic JSON spec and use it to convert the input JSON to desired output
My generic JSON spec looks something like this:
{
"aws_security_group": [
{
"terraform_attribute": "name",
"attribute_type": "string",
"aws_attribute": "GroupName"
},
{
"terraform_attribute": "vpc_id",
"attribute_type": "string",
"aws_attribute": "VpcId"
},
{
"terraform_attribute": "description",
"attribute_type": "string",
"aws_attribute": "Description"
},
{
"terraform_attribute": "ingress",
"attribute_type": "list",
"aws_attribute": "IpPermissions",
"list_items": [
{
"terraform_attribute": "from_port",
"attribute_type": "string",
"aws_attribute": "FromPort"
},
{
"terraform_attribute": "to_port",
"attribute_type": "string",
"aws_attribute": "ToPort"
},
{
"terraform_attribute": "protocol",
"attribute_type": "string",
"aws_attribute": "IpProtocol"
},
{
"terraform_attribute": "cidr_blocks",
"attribute_type": "list",
"aws_attribute": "IpRanges",
"list_items": [
{
"terraform_attribute": "cidr_blocks.value",
"attribute_type": "string",
"aws_attribute": "CidrIp"
},
{
"terraform_attribute": "description",
"attribute_type": "string",
"aws_attribute": "Description"
}
]
}
]
},
{
"terraform_attribute": "egress",
"attribute_type": "list",
"aws_attribute": "IpPermissionsEgress",
"list_items": [
{
"terraform_attribute": "from_port",
"attribute_type": "string",
"aws_attribute": "FromPort"
},
{
"terraform_attribute": "to_port",
"attribute_type": "string",
"aws_attribute": "ToPort"
},
{
"terraform_attribute": "protocol",
"attribute_type": "string",
"aws_attribute": "IpProtocol"
},
{
"terraform_attribute": "cidr_blocks",
"attribute_type": "list",
"aws_attribute": "IpRanges",
"list_items": [
{
"terraform_attribute": "cidr_blocks.value",
"attribute_type": "string",
"aws_attribute": "CidrIp"
},
{
"terraform_attribute": "description",
"attribute_type": "string",
"aws_attribute": "Description"
}
]
}
]
}
]
}
Then I'm using this spec to convert the input JSON to desired output by recursively visiting each block within the input something like this:
for key, val := range configCopy {
for _, detail := range resourceDetails {
detail := detail.(map[string]interface{})
if detail["attribute_type"] == "string" && detail["terraform_attribute"] == key {
delete(configCopy, key)
configCopy[detail["aws_attribute"].(string)] = val
break
} else if detail["attribute_type"] == "list" && detail["terraform_attribute"] == key {
delete(configCopy, key)
configCopy[detail["aws_attribute"].(string)] =
buildListValue(val, detail["list_items"].([]interface{}))
}
}
}
func buildListValue(val interface{}, listItemVal []interface{}) []map[string]interface{} {
v := reflect.ValueOf(val)
var result []map[string]interface{}
fmt.Println("Val Kind:", v.Kind(), "Val String: ", v.String())
valCopy := make([]interface{}, v.Len())
if v.Kind() == reflect.Slice {
for i := 0; i < v.Len(); i++ {
valCopy[i] = v.Index(i).Interface()
}
}
for _, eachVal := range valCopy {
res := make(map[string]interface{})
f := reflect.ValueOf(eachVal)
if f.Kind() == reflect.Map {
for _, key := range f.MapKeys() {
fmt.Println("Key:", key.String(), " value:", f.MapIndex(key).Interface())
for _, listItem := range listItemVal {
listItem := listItem.(map[string]interface{})
if listItem["attribute_type"] == "string" && listItem["terraform_attribute"] == key.String() {
res[listItem["aws_attribute"].(string)] = f.MapIndex(key).Interface()
break
} else if listItem["attribute_type"] == "list" && listItem["terraform_attribute"] == key.String() {
res[listItem["aws_attribute"].(string)] =
buildListValue(f.MapIndex(key).Interface(), listItem["list_items"].([]interface{}))
break
}
}
}
} else if f.Kind() == reflect.String {
for _, listItem := range listItemVal {
listItem := listItem.(map[string]interface{})
if strings.HasSuffix(listItem["terraform_attribute"].(string), ".value") {
res[listItem["aws_attribute"].(string)] = f.String()
}
}
}
if len(res) > 0 {
result = append(result, res)
}
}
return result
}
Where configCopy is the input JSON. This approach works, but the problems associated with this approach are:
It is a time consuming process as I need to manually create a spec for each resource type.
Error prone as the transformation code is completely dependent on the manual spec being defined.
II. Tried using some open-source JSON transformers in Go like kazaam
Project kazaam takes a JSON string and a schema spec as input and produces the result using the given spec very similar to project Jolt.
I tried using a spec like this to convert the above input JSON:
[
{
"operation": "shift",
"spec": {
"SecurityGroups.Description": "description",
"SecurityGroups.GroupName": "name",
"SecurityGroups.VpcId": "vpc_id",
"SecurityGroups.IpPermissions.IpProtocol": "ingress[*].protocol",
"SecurityGroups.IpPermissions.FromPort": "ingress[*].from_port",
"SecurityGroups.IpPermissions.ToPort": "ingress[*].to_port",
"SecurityGroups.IpPermissions.IpRanges.CidrIp": "ingress[*].cidr_blocks[*]",
"SecurityGroups.IpPermissions.IpRanges.Description": "ingress[*].description",
"SecurityGroups.IpPermissionsEgress.IpProtocol": "egress[*].protocol",
"SecurityGroups.IpPermissionsEgress.FromPort": "egress[*].from_port",
"SecurityGroups.IpPermissionsEgress.ToPort": "egress[*].to_port",
"SecurityGroups.IpPermissionsEgress.IpRanges.CidrIp": "egress[*].cidr_blocks[*]",
"SecurityGroups.IpPermissionsEgress.IpRanges.Description": "egress[*].description"
}
}
]
But the result produced was something like the one shown below which is not what I was expecting:
{
"SecurityGroups": {
"IpPermissionsEgress": {
"FromPort": [
443
],
"ToPort": [
443
],
"IpRanges": {
"Description": [
"Port 443"
],
"CidrIp": [
[
"0.0.0.0/0"
]
]
},
"IpProtocol": [
"tcp"
]
},
"VpcId": "${aws_vpc.my_vpc.id}",
"IpPermissions": {
"FromPort": [
443
],
"ToPort": [
443
],
"IpRanges": {
"CidrIp": [
[
"0.0.0.0/0"
]
],
"Description": [
"Port 443"
]
},
"IpProtocol": [
"tcp"
]
},
"Description": null,
"GroupName": "my_sec_group"
}
}
Which is not as expected. I may be doing something wrong while constructing the kazaam spec, but I'm not really sure.
Is there any better/efficient way to address this use case? Any suggestions on this would be of great help.

How to parse an array of json in scala play framework?

I have an array of json objects like this
[
{
"events": [
{
"type": "message",
"attributes": [
{
"key": "action",
"value": "withdraw_reward"
},
{
"key": "sender",
"value": "bob"
},
{
"key": "module",
"value": "distribution"
},
{
"key": "sender",
"value": "bob"
}
]
},
{
"type": "credit",
"attributes": [
{
"key": "recipient",
"value": "ross"
},
{
"key": "sender",
"value": "bob"
},
{
"key": "amount",
"value": "100"
}
]
},
{
"type": "rewards",
"attributes": [
{
"key": "amount",
"value": "100"
},
{
"key": "validator",
"value": "sarah"
}
]
}
]
},
{
"events": [
{
"type": "message",
"attributes": [
{
"key": "action",
"value": "withdraw_reward"
},
{
"key": "sender",
"value": "bob"
},
{
"key": "module",
"value": "distribution"
},
{
"key": "sender",
"value": "bob"
}
]
},
{
"type": "credit",
"attributes": [
{
"key": "recipient",
"value": "ross"
},
{
"key": "sender",
"value": "bob"
},
{
"key": "amount",
"value": "100"
}
]
},
{
"type": "rewards",
"attributes": [
{
"key": "amount",
"value": "200"
},
{
"key": "validator",
"value": "Ryan"
}
]
}
]
}
]
How to traverse through the types, check if it's type equals to rewards and then go through the attributes and verify if the validator equals to sarah and fetch the value of the key amount? Pretty new to scala and play framework. Any help would be great. Thanks
You could parse your JSON into a structure of case classes for easier handling and then extract the wanted field like so:
val json =
"""[
{"events":[
{
"type":"message","attributes":[
{"key":"action","value":"withdraw_reward"},
{"key":"sender","value":"bob"},
{"key":"module","value":"distribution"},
{"key":"sender","value":"bob"}
]},
{
"type":"credit","attributes":[
{"key":"recipient","value":"ross"},
{"key":"sender","value":"bob"},
{"key":"amount","value":"100"}
]},
{
"type":"rewards","attributes":[
{"key":"amount","value":"100"},
{"key":"validator","value":"sara"}
]}
]
},
{"events":[
{
"type":"message","attributes":[
{"key":"action","value":"withdraw_reward"},
{"key":"sender","value":"bob"},
{"key":"module","value":"distribution"},
{"key":"sender","value":"bob"}
]},
{
"type":"credit","attributes":[
{"key":"recipient","value":"ross"},
{"key":"sender","value":"bob"},
{"key":"amount","value":"100"}
]},
{
"type":"rewards","attributes":[
{"key":"amount","value":"200"},
{"key":"validator","value":"Ryan"}
]}
]
}
]
"""
case class EventWrapper(events: Seq[Event])
case class KeyValue(key: String, value: String)
case class Event(`type`: String, attributes: Seq[KeyValue])
import play.api.libs.json._
implicit val kvReads: Reads[KeyValue] = Json.reads[KeyValue]
implicit val eventReads: Reads[Event] = Json.reads[Event]
implicit val eventWrapperReads: Reads[EventWrapper] = Json.reads[EventWrapper]
val rewardAmountsValidatedBySara = Json
.parse(json)
.as[Seq[EventWrapper]]
.flatMap {
_.events.collect {
case Event(t, attributes) if t == "rewards" && attributes.contains(KeyValue("validator", "sara")) =>
attributes.collect {
case KeyValue("amount", value) => value
}
}.flatten
}
val amount = rewardAmountsValidatedBySara.head
For your example, rewardAmountsValidatedBySara would yield a List of Strings containing only the String "100". Which you could retrieve (potentially unsafe) with .head as shown above.
Normally you would not do this, as it could throw an exception on an empty List, so it would be better to use .headOption which returns an Option which you can then handle safely.
Note that the implicit Reads are Macros, which automatically translate into Code, that instructs the Play Json Framework how to read the JsValue into the defined case classes, see the documentation for more info.

Extract from json with | jq by a given word

Can somebody help me to extract with | jq the following:
{
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"pod": "dev-cds-5c97cf7f78-sw6b9"
},
"values": [
[
1588204800,
"0.3561394483796914"
],
[
1588215600,
"0.3607968456046861"
],
[
1588226400,
"0.3813882532417868"
],
[
1588237200,
"0.6264355815408573"
]
]
},
{
"metric": {
"pod": "uat-cds-66ccc9685-b5tvh"
},
"values": [
[
1588204800,
"0.9969746974696218"
],
[
1588215600,
"0.7400881057270005"
],
[
1588226400,
"1.2298959318837195"
],
[
1588237200,
"0.9482296838254507"
]
]
}
]
}
}
I need to obtain all-values individually by given word dev-cds and not all the name dev-cds-5c97cf7f78-sw6b9.
Result desired:
{
"metric": {
"pod": "dev-cds-5c97cf7f78-sw6b9"
},
"values": [
[
1588204800,
"0.3561394483796914"
],
[
1588215600,
"0.3607968456046861"
],
[
1588226400,
"0.3813882532417868"
],
[
1588237200,
"0.6264355815408573"
]
]
}
You should first iterate over the result array. Check if the pod inside, metric object has the value that contains "dev-cds".
.data.result[] | if .metric.pod | contains("dev-cds") then . else empty end
https://jqplay.org/s/54OH83qHKP

Groovy: Convert Array to JSon

Am new to Groovy and am having trouble converting an array to JSON. The JSON computed should have all the values from my array list, but it is storing only the last one. Here is the code:
def arraylist = [["0",2],["1",8],["2",6],["3",8],["4",3]]
def arraysize = arraylist.size()
def builder = new groovy.json.JsonBuilder()
builder ({
cols([
{
"id" "hours"
"label" "Hours"
"type" "string"
},
{
"id" "visitor"
"label" "Visitors"
"type" "number"
}
])
rows([
{
for( i in 0..< arraysize )
{
c([
{
"v" arraylist[i][0]
},
{
"v" arraylist[i][1]
}
])
}//for
}
])
})
println builder.toPrettyString()
Can try running the code here:
http://groovyconsole.appspot.com/
Expected output is here:
{
"cols": [
{
"id": "hours",
"label": "Hours",
"type": "string"
},
{
"id": "visitor",
"label": "Visitors",
"type": "number"
}
],
"rows": [
{
"c": [
{
"v": "0"
},
{
"v": 2
}
]
},
{
"c": [
{
"v": "1"
},
{
"v": 8
}
]
},
{
"c": [
{
"v": "2"
},
{
"v": 6
}
]
},
{
"c": [
{
"v": "3"
},
{
"v": 8
}
]
},
{
"c": [
{
"v": "4"
},
{
"v": 3
}
]
}
]
}
Something like this seems to give the result you wanted:
def arraylist = [["0",2],["1",8],["2",6],["3",8],["4",3]]
def builder = new groovy.json.JsonBuilder()
builder {
cols( [
[ id: "hours", label: "Hours", type: "string" ],
[ id: "visitor", label: "Visitors", type: "number" ] ] )
rows( arraylist.collect { pair -> [ c: pair.collect { item -> [ v: item ] } ] } )
}
println builder.toPrettyString()