recursively flatten a map golang - json

Using the below json Im trying to flatten it for ease of accessibility.
Example having 2 resource there can be n number of in this structure
"config": {
"type": "r1",
"properties": {
"p1": "10",
"p2": "10"
},
"connected": [
{
"type": "r3",
"properties": {
"p1": "10",
"p2": "10"
},
"connected": [
{}
},
],
}
Custom flatterning logic
func keyValuePairs(m interface{}) map[string]interface{} {
kvs := make(map[string]interface{})
if reflect.ValueOf(m).Kind() == reflect.Map {
mp, ok := m.(map[string]interface{})
if ok {
var key string
var value interface{}
for k, v := range mp {
switch k {
case "type":
key = v.(string)
case "properties":
value = v
case "connected":
if collection, ok := v.([]interface{}); ok {
for _, c := range collection {
for nk, nv := range keyValuePairs(c) {
nnv, _ := nv.(map[string]interface{})
_, ok := nnv["connectedTo"]
if !ok {
nnv["connectedTo"] = key
}
kvs[nk] = nv
}
}
}
default:
for nk, nv := range keyValuePairs(v) {
kvs[nk] = nv
}
}
}
if key != "" {
kvs[key] = value
}
} else {
for k, v := range m.(map[string]interface{}) {
kvs[k] = v
}
}
}
return kvs
}
The desired output is kinda fluctuationg, some runs I get the output what I require and some executions "connectedTo" attribute is empty.
{
"r1": {
"p1": "10",
"p2": "10"
},
"r3" : {
"connectedTo": "r1",
"p1": "10",
"p2": "10"
},
}
I think the executions are not sequential. Please do correct me if Im wrong.

Iteration order over a map is random. For details, see Why are iterations over maps random? and How to iterate maps in insertion order?
Now look at your loop:
var key string
var value interface{}
for k, v := range mp {
switch k {
case "type":
key = v.(string)
case "properties":
value = v
case "connected":
if collection, ok := v.([]interface{}); ok {
for _, c := range collection {
for nk, nv := range keyValuePairs(c) {
nnv, _ := nv.(map[string]interface{})
_, ok := nnv["connectedTo"]
if !ok {
nnv["connectedTo"] = key
}
kvs[nk] = nv
}
}
}
default:
for nk, nv := range keyValuePairs(v) {
kvs[nk] = nv
}
}
}
You are using the key variable in the "connected" branch, but the order in which "type" and "connected" will be reached is non-deterministic (random).
The "type" branch is what sets key, but if "connected" is reached first, key will be empty.
You must not rely on map iteration order.
An easy fix is to first get the value associated with"type" first and assign it to key (which you use in the "connected" branch), before the loop.
For example:
key, _ := mp["type"].(string)
value := mp["properties"]
// Now process other properties:
for k, v := range mp {
switch k {
case "type", "properties": // Already handled
case "connected":
if collection, ok := v.([]interface{}); ok {
for _, c := range collection {
for nk, nv := range keyValuePairs(c) {
nnv, _ := nv.(map[string]interface{})
_, ok := nnv["connectedTo"]
if !ok {
nnv["connectedTo"] = key
}
kvs[nk] = nv
}
}
}
default:
for nk, nv := range keyValuePairs(v) {
kvs[nk] = nv
}
}
}

Related

Add values to array object in json file

I need to add an array of string to a json file.
i.e let's say i have the following json file:
{
"user": {
"role": "admin",
"email": "admin#domain.com"
},
"region": [
{
"location": "EU",
"currency": "EUR",
"countries": [
{
"FR": [
{
"time": "morning",
"cities": []
}
]
}
]
},
{
"location": "NA",
"currency": "USD",
"countries": [
{
"USA": [
{
"time": "evening",
"cities": []
}
]
}
]
}
]
}
Now lets say I have a list of cities that I want to add to region -> countries -> FR
This is my approach, but not sure how to update the cities field.
I have commented the section in the code that need to store to update the list of cities for FA.
func test (jsonFilePath, cityNames string) error {
jsonBytes, err := ioutil.ReadFile(jsonFile)
if err != nil {
return err
}
fmt.Printf("template json file is %s\n", jsonBytes)
var result map[string]interface{}
err = json.Unmarshal(jsonBytes, &result)
if err != nil {
return err
}
if _, exists := result["region"]; exists {
fmt.Printf("Found region\n")
countries := result["region"].([]interface{})
for _, country := range countries {
m := country.(map[string]interface{})
if _, exists := m["FR"]; exists {
/*How to add cityNames to the cities array of the FA?*/
}
}
} else {
errMsg := "region field is missing"
return errors.New(errMsg)
}
jsonBytes, err = json.Marshal(result)
if err != nil {
return err
}
fmt.Printf("the new json file is %s\n", string(jsonBytes))
// Write back to file
err = ioutil.WriteFile("newJson.json", jsonBytes, 0644)
return err
}
Note: The json file has other fields which should not be updated, only Cities filed should be updated; so the preference is to update the Cities field only without defining a struct
Normally it's better (i.e. easier and more efficient) to use structs instead of reflection. There are even websites that can generate a struct from a JSON sample.
But here's how to do it using maps of interfaces:
var result map[string]interface{}
err := json.Unmarshal(jsonBytes, &result)
if err != nil {
return nil, err
}
regions, ok := result["region"].([]interface{})
if !ok {
return nil, errors.New("region not found")
}
for i := range regions {
region, ok := regions[i].(map[string]interface{})
if !ok {
continue
}
countries, ok := region["countries"].([]interface{})
if !ok {
continue
}
for j := range countries {
country, ok := countries[j].(map[string]interface{})
if !ok {
continue
}
fr, ok := country["FR"].([]interface{})
if !ok {
continue
}
for k := range fr {
time, ok := fr[k].(map[string]interface{})
if !ok {
continue
}
cities, ok := time["cities"].([]interface{})
if !ok {
continue
}
time["cities"] = append(cities, cityNames)
}
}
}
Note that in Go maps do not preserve the order, so the order of fields in the output will be different (and also random between runs).

Golang: unmarshal json object into a slice

I have JSON like
{
"a": {"key": "a", "value": 1,},
"b": {"key": "b", "value": 1,},
}
Is there a way to unmarshal it into []*struct {Key string; Value int}, preserving the order of the structures?
If I unmarshal the JSON into map[string]*struct {Key string; Value int} and then convert the map into a slice, I'll lose the order of the structures.
To preserve order, use Decoder.Token and Decoder.More to walk through the top-level JSON object.
r := strings.NewReader(`
{
"a": {"key": "a", "value": 1},
"b": {"key": "b", "value": 1}
}`)
d := json.NewDecoder(r)
t, err := d.Token()
if err != nil || t != json.Delim('{') {
log.Fatal("expected object")
}
var result []*element
for d.More() {
k, err := d.Token()
if err != nil {
log.Fatal(err)
}
var v element
if err := d.Decode(&v); err != nil {
log.Fatal(err)
}
result = append(result, &v)
fmt.Println("key:", k, "value:", v)
}
Run it on the Go Playground
Change the calls to log.Fatal to the error handling appropriate for your application.
This answer edits the JSON in the question to make the JSON valid.
The field names in the struct element type must be exported.
The easiest way that I found was to use jsonparser.ObjectEach:
import "github.com/buger/jsonparser"
...
var ss []*struct{Key string; Value int}
err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
var s struct{Key string; Value int}
if err := json.Unmarshal(value, &s); err != nil {
return err
}
*ss = append(*ss, &s)
return nil
})
You could use the map[string]interface{} to unmarshal the json string.
The code is this
func test() {
jsonStr := `
{
"a": {"key": "a", "value": 1},
"b": {"key": "b", "value": 1}
}`
var mapResult map[string]interface{}
err := json.Unmarshal([]byte(jsonStr), &mapResult)
if err != nil {
fmt.Println("JsonToMapDemo err: ", err)
}
fmt.Println(mapResult)
}
the output is:
map[a:map[key:a value:1] b:map[key:b value:1]]

Golang json unmarshal according to key value pair

I have json as following
"data": [
{
"id": "recent_search",
"items": [],
"name": ""
},
{
"id": "popular_search",
"items": [],
"name": ""
},
{
"id": "digital",
"items": [],
"name": "DIGITAL"
}
]
and the structs are as follows:
type universeTypeData struct {
Recent universeSearchInfo
Popular universeSearchInfo
Digital universeSearchInfo
}
type universeSearchInfo struct {
ID string `json:"id"`
Name string `json:"name"`
Items []universeSearchItem `json:"items"`
}
I want to unmarshal my json as "id" with value "recent_search" map to Recent, "id" with value "popular_search" map to Popular. Is there any way of doing this in golang?
My approach of doing it is
for _, v := range result.Data {
if v.ID == "in_category" {
finalResult.Universe.InCategory.ID = v.ID
finalResult.Universe.InCategory.Name = v.Name
for _, abc := range v.Items {
finalResult.Universe.InCategory.Items = append(finalResult.Universe.InCategory.Items, abc)
}
}
if v.ID == "recent_search" {
finalResult.Universe.Recent.ID = v.ID
finalResult.Universe.Recent.Name = v.Name
for _, abc := range v.Items {
finalResult.Universe.Recent.Items = append(finalResult.Universe.Recent.Items, abc)
}
}
if v.ID == "popular_search" {
finalResult.Universe.Popular.ID = v.ID
finalResult.Universe.Popular.Name = v.Name
for _, abc := range v.Items {
finalResult.Universe.Popular.Items = append(finalResult.Universe.Popular.Items, abc)
}
}
Is there any better way of doing it?
You want to unmurshal JSON array into Go struct which is not natural mapping. Any way, you most likely should be first unmurshal in slice and then parse this slice. Some workaround is to use json.Decoder
dec := json.NewDecoder(JSONdataReader)
var res universeTypeData
// read open bracket
dec.Token()
// while the array contains values
for dec.More() {
var m universeSearchInfo
// decode an array value
dec.Decode(&m)
switch m.ID {
case "recent_search":
res.Recent = m
case "popular_search":
res.Popular = m
case "digital":
res.Digital = m
}
}
// read closing bracket
dec.Token()
which allow you to decode on the fly, in one pass, without consuming intermediate slice representation. Working example
Implement Unmarshaler interface:
Unmarshaler is the interface implemented by types that can unmarshal a
JSON description of themselves. The input can be assumed to be a valid
encoding of a JSON value. UnmarshalJSON must copy the JSON data if it
wishes to retain the data after returning.
json unmarshaler interface assign the value from json to struct after parsing the result and applying conditions to fetch the value.
package main
import (
"encoding/json"
"fmt"
)
type Details struct {
Data []universeSearchInfo `json:"data"`
}
type universeTypeData struct {
Recent universeSearchInfo
Popular universeSearchInfo
Digital universeSearchInfo
}
type universeSearchInfo struct {
ID string `json:"id"`
Name string `json:"name"`
Items []string `json:"items"`
}
func main() {
var result universeTypeData
jsonBytes := []byte(`{"data": [
{
"id": "recent_search",
"items": [],
"name": ""
},
{
"id": "popular_search",
"items": [],
"name": ""
},
{
"id": "digital",
"items": [],
"name": "DIGITAL"
}
]}`)
if err := json.Unmarshal(jsonBytes, &result); err != nil {
fmt.Println(err)
}
fmt.Println(result)
}
func (universeData *universeTypeData) UnmarshalJSON(data []byte) error {
var result Details
if err := json.Unmarshal(data, &result); err != nil {
return err
}
for _,value := range result.Data{
switch value.ID {
case "recent_search":
universeData.Recent = value
}
}
return nil
}
Working code on Go Playground

Custom config loading in Go

I'm using JSON files to store/load my config. Let's say I have the following:
type X interface
// implements interface X
type Y struct {
Value string
}
// implements interface X
type Z struct {
Value string
}
type Config struct {
interfaceInstance X `json:"X"`
}
Config file example:
{
"config1": {
"X": {
"type": "Z",
"Value": "value_1"
}
},
"config2": {
"X": {
"type": "Y",
"Value": "value_2"
}
}
}
I want to be able to define config files something like this example, and be able to dynamically load the JSON as either struct Y or struct Z. Any suggestions on how to accomplish this? I'm using a simple json.Decoder to load the JSON as a struct.
decoder := json.NewDecoder(file)
err = decoder.Decode(&config)
One possible strategy would be to implement json.Unmarshaler for the Config type in such a way that you first unmarshal into a generic object and inspect the "type" attribute then, switching on the type string, unmarshal the same byte array into the known type and assign to the "interfaceInstance" member of the config.
For example (Go Playground):
// Note the slightly different JSON here...
var jsonstr = `{
"config1": {
"type": "Z",
"Value": "value_1"
},
"config2": {
"type": "Y",
"Value": "value_2"
}
}`
func main() {
config := map[string]Config{}
err := json.Unmarshal([]byte(jsonstr), &config)
if err != nil {
panic(err)
}
fmt.Printf("OK: %#v\n", config)
// OK: map[string]main.Config{
// "config1": main.Config{interfaceInstance:main.Z{Value:"value_1"}},
// "config2": main.Config{interfaceInstance:main.Y{Value:"value_2"}},
// }
}
func (c *Config) UnmarshalJSON(bs []byte) error {
// Unmarshal into an object to inspect the type.
var obj map[string]interface{}
err := json.Unmarshal(bs, &obj)
if err != nil {
return err
}
// Unmarshal again into the target type.
configType := obj["type"].(string)
switch configType {
case "Y":
var y Y
if err = json.Unmarshal(bs, &y); err == nil {
c.interfaceInstance = y
}
case "Z":
var z Z
if err = json.Unmarshal(bs, &z); err == nil {
c.interfaceInstance = z
}
default:
return fmt.Errorf("unexpected type %q", configType)
}
return err
}

Making minimal modification to JSON data without a structure in golang

I have a solr response in JSON format which looks like this:
{
"responseHeader": {
"status": 0,
"QTime": 0,
"params": {
"q": "solo",
"wt": "json"
}
},
"response": {
"numFound": 2,
"start": 0,
"docs": [
{
<Large nested JSON element>
},
{
<Large nested JSON element>
}
]
}
}
Now, in my Golang app, I would like to quickly remove the "responseHeader" so that I can return the "response" alone. How can I do this without creating large structures?
Edit 1
The answer by evanmcdonnal was the solution to this problem, but it had some minor typos, this is what I ended up using:
var temp map[string]interface{}
if err := json.Unmarshal(body, &temp); err != nil {
panic(err.Error())
}
result, err := json.Marshal(temp["response"])
Here's a really brief example of how to do this quickly and easily. The steps are; unmarshal into the universal map[string]interface{} then, assuming no errors, marshal only the inner object which you want.
var temp := &map[string]interface{}
if err := json.Unmarshal(input, temp); err != nil {
return err;
}
return json.Marshal(temp["response"])
I wrote a package µjson to do exactly that: performing generic transformations on JSON documents without unmarshalling them.
Run on Go Playground
input := []byte(`
{
"responseHeader": {
"status": 0,
"QTime": 0,
"params": {
"q": "solo",
"wt": "json"
}
},
"response": {
"numFound": 2,
"start": 0,
"docs": [
{ "name": "foo" },
{ "name": "bar" }
]
}
}`)
blacklistFields := [][]byte{
[]byte(`"responseHeader"`), // note the quotes
}
b := make([]byte, 0, 1024)
err := ujson.Walk(input, func(_ int, key, value []byte) bool {
for _, blacklist := range blacklistFields {
if bytes.Equal(key, blacklist) {
// remove the key and value from the output
return false
}
}
// write to output
if len(b) != 0 && ujson.ShouldAddComma(value, b[len(b)-1]) {
b = append(b, ',')
}
if len(key) > 0 {
b = append(b, key...)
b = append(b, ':')
}
b = append(b, value...)
return true
})
if err != nil {
panic(err)
}
fmt.Printf("%s", b)
// Output: {"response":{"numFound":2,"start":0,"docs":[{"name":"foo"},{"name":"bar"}]}}
You can read more about it on the blog post. I put the answer here just in case someone else might need it.