merge JSON objects recursively - json

Here's an example of what I need:
Default JSON:
{
"name": "John",
"greetings": {
"first": "hi",
"second": "hello"
}
}
merged with the changes:
{
"name": "Jane",
"greetings": {
"first": "hey"
}
}
should become:
{
"name": "Jane",
"greetings": {
"first": "hey",
"second": "hello"
}
}
Here's what I've tried:
package main
import (
"encoding/json"
"fmt"
"reflect"
)
func MergeJSON(defaultJSON, changedJSON string) string {
var defaultJSONDecoded map[string]interface{}
defaultJSONUnmarshalErr := json.Unmarshal([]byte(defaultJSON), &defaultJSONDecoded)
if defaultJSONUnmarshalErr != nil {
panic("Error unmarshalling first JSON")
}
var changedJSONDecoded map[string]interface{}
changedJSONUnmarshalErr := json.Unmarshal([]byte(changedJSON), &changedJSONDecoded)
if changedJSONUnmarshalErr != nil {
panic("Error unmarshalling second JSON")
}
for key, _ := range defaultJSONDecoded {
checkKeyBeforeMerging(key, defaultJSONDecoded[key], changedJSONDecoded[key], changedJSONDecoded)
}
mergedJSON, mergedJSONErr := json.Marshal(changedJSONDecoded)
if mergedJSONErr != nil {
panic("Error marshalling merging JSON")
}
return string(mergedJSON)
}
func checkKeyBeforeMerging(key string, defaultMap interface{}, changedMap interface{}, finalMap map[string]interface{}) {
if !reflect.DeepEqual(defaultMap, changedMap) {
switch defaultMap.(type) {
case map[string]interface{}:
//Check that the changed map value doesn't contain this map at all and is nil
if changedMap == nil {
finalMap[key] = defaultMap
} else if _, ok := changedMap.(map[string]interface{}); ok { //Check that the changed map value is also a map[string]interface
defaultMapRef := defaultMap.(map[string]interface{})
changedMapRef := changedMap.(map[string]interface{})
for newKey, _ := range defaultMapRef {
checkKeyBeforeMerging(newKey, defaultMapRef[newKey], changedMapRef[newKey], finalMap)
}
}
default:
//Check if the value was set, otherwise set it
if changedMap == nil {
finalMap[key] = defaultMap
}
}
}
}
func main() {
defaultJSON := `{"name":"John","greetings":{"first":"hi","second":"hello"}}`
changedJSON := `{"name":"Jane","greetings":{"first":"hey"}}`
mergedJSON := MergeJSON(defaultJSON, changedJSON)
fmt.Println(mergedJSON)
}
The code above returns the following:
{
"greetings": {
"first": "hey"
},
"name": "Jane",
"second": "hello"
}
So basically any changes should be applied to the default and return the full JSON. I also need this to work recursively.
How can I fix this? I can see where I went wrong, I'm just not sure how to make it work recursively.
Thanks

Your issue in the posted code is with your recursive call:
checkKeyBeforeMerging(newKey, defaultMapRef[newKey], changedMapRef[newKey], finalMap)
The reference to finalMap should actually be the nested part of the merged map. Meaning replace finalMap with something like finalMap[key].(map[string]interface{}).

I recently come across the same need, here is my solution:
// override common json by input json
func Override(input, common interface{}) {
switch inputData := input.(type) {
case []interface{}:
switch commonData := common.(type) {
case []interface{}:
for idx, v := range inputData {
Override(v, commonData[idx])
}
}
case map[string]interface{}:
switch commonData := common.(type) {
case map[string]interface{}:
for k, v := range commonData {
switch reflect.TypeOf(v).Kind() {
case reflect.Slice, reflect.Map:
Override(inputData[k], v)
default:
// do simply replacement for primitive type
_, ok := inputData[k]
if !ok {
inputData[k] = v
}
}
}
}
}
return
}
unmarshal json before call:
var commmon interface{}
if err = json.Unmarshal("common data", &commmon); err != nil {
logger.Error(err)
return
}
var input interface{}
if err = json.Unmarshal("input data", &input); err != nil {
logger.Error(err)
return
}
Override(input, commmon)

Related

How to write custom splitFunc for bufio.Scaner that scan json objects

I have a code like this
scanner := bufio.NewScanner(reader)
scanner.Split(splitJSON)
for scanner.Scan() {
bb := scanner.Bytes()
}
I would like to get from Scanner only valid JSON objects one at a time. In some case in Scanner may be bytes that represent struct like this
{
"some_object": "name",
"some_fileds": {}
}
{
"some_object":
}
I need only the first part of this
{
"some_object": "name",
"some_fileds": {}
}
For the other, I should wait for the end of JSON object.
I have a function like this, but it's horrible and doesn't work.
func splitJSON(
bb []byte, atEOF bool,
) (advance int, token []byte, err error) {
print(string(bb))
if len(bb) < 10 {
return 0, nil, nil
}
var nested, from, to int
var end bool
for i, b := range bb {
if string(b) == "{" {
if end {
to = i
break
}
if nested == 0 {
from = i
}
nested++
}
if string(b) == "}" {
nested--
if nested == 0 {
to = i
end = true
}
}
}
if atEOF {
return len(bb), bb, nil
}
return len(bb[from:to]), bb[from:to], nil
}
UPD
It was decided by this splitFunc
func splitJSON(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
reader := bytes.NewReader(data)
dec := json.NewDecoder(reader)
var raw json.RawMessage
if err := dec.Decode(&raw); err != nil {
return 0, nil, nil
}
return len(raw) + 1, raw, nil
}
Use json.Decoder for this. Each Decoder.Decode() call will decode the next JSON-encoded value from the input, JSON objects in your case.
If you don't want to decode the JSON objects just need the JSON data (byte slice), use a json.RawMessage to unmarshal into.
For example:
func main() {
reader := strings.NewReader(src)
dec := json.NewDecoder(reader)
for {
var raw json.RawMessage
if err := dec.Decode(&raw); err != nil {
if err == io.EOF {
break
}
fmt.Printf("Error:", err)
return
}
fmt.Println("Next:", string(raw))
}
}
const src = `{
"some_object": "name",
"some_fileds": {}
}
{
"some_object": "foo"
}`
This will output (try it on the Go Playground):
Next: {
"some_object": "name",
"some_fileds": {}
}
Next: {
"some_object": "foo"
}

How to parse json without a defined stucture?

I want to parse this JSON :
{
"error": null,
"id": "tutu",
"result": {
"param1": 559,
"param2": "yo",
"param3": {"tab":["a", "b"], "param4":"hello"},
}
}
The problem is that I want a flexible solution if the JSON structure changes: I want to be able to access each field with a key system (like in Javascript) without knowing in advance the JSON structure:
fmt.Println(jsonObj["result"]["param3"]["tab"][1])
Is it possible to do it ?
First, in your case unmarshall directly into map[string]interface{} to save an assertion.
var jsonObj map[string]interface{}
err := json.Unmarshal(b, &jsonObj)
Second, remember to check the assertion
result, ok := jsonObj["result"].(map[string]interface{})
if !ok {
panic("not json obj")
}
param3, ok := result["param3"].(map[string]interface{})
if !ok {
panic("not json obj")
}
tab, ok := param3["tab"].([]interface{})
if !ok {
panic("not json arr")
}
Lastly, you can declare a new type and "hide" the assertions in its methods
type AnyObj map[string]interface{}
func (obj AnyObj) MustObject(name string) AnyObj {
v, ok := obj[name].(map[string]interface{})
if !ok {
panic("not json obj")
}
return AnyObj(v)
}
func (obj AnyObj) MustArray(name string) []interface{} {
v, ok := obj[name].([]interface{})
if !ok {
panic("not json arr")
}
return v
}
Then use like this:
func main() {
var jsonObj AnyObj
err := json.Unmarshal(b, &jsonObj)
if (err != nil) {
panic(err);
}
tab := jsonObj.MustObject("result").MustObject("param3").MustArray("tab")
fmt.Println(tab[1])
}
check it here https://play.golang.org/p/ucdMZ0VEKcr
Yes, it is possible, although the fact that you need to assert the types makes it a bit unwieldy:
package main
import (
"encoding/json"
"fmt"
)
func main() {
b := []byte(`
{
"error": null,
"id": "tutu",
"result": {
"param1": 559,
"param2": "yo",
"param3": {"tab":["a", "b"], "param4": "hello"}
}
}
`)
var jsonObj interface{}
err := json.Unmarshal(b, &jsonObj)
if err != nil {
panic(err)
}
msg := jsonObj.(map[string]interface{})
result := msg["result"].(map[string]interface{})
param3 := result["param3"].(map[string]interface{})
tab := param3["tab"].([]interface{})
fmt.Println(tab[1])
}
This prints
b
like you would expect.
Note that this program will panic: not just if the JSON fails to parse, but also if the JSON does not exactly match the program's expectations: missing keys, different types and so on.
The Generic JSON with interface section of the JSON and Go article on The Go Blog has an example of checking the actual type of the thing at runtime; as was suggested in the comments, it's easy to forget.
this is mine used fastjson:
package main
import (
"fmt"
"github.com/valyala/fastjson"
)
type ParseValue struct {
*fastjson.Value
}
func Parse(b []byte) (*ParseValue, error) {
var p fastjson.Parser
v, err := p.ParseBytes(b)
if err != nil {
return nil, err
}
return &ParseValue{v}, nil
}
func (p *ParseValue) GetString(keys ...string) string {
return string(p.GetStringBytes(keys...))
}
func main() {
b := []byte(`
{
"error": null,
"id": "tutu",
"result": {
"param1": 559,
"param2": "yo",
"param3": {"tab":["a", "b"], "param4": "hello"}
}
}
`)
v, err := Parse(b)
if err != nil {
panic(err)
}
r := v.GetString("result", "param3", "tab", "1")
fmt.Println(r)
rr := v.GetUint("result", "param1")
fmt.Println(rr)
// output:
// b
// 559
}

How to write all decoded data in a JSON file?

I try to parse an XML data to a JSON file, but when I begin writing marshaled data to a JSON, it just rewrites data in the JSON file and, as a result, I have the file with last XML element. How to write the whole data into the JSON file?
Snippet of code that parses XML and marshal data to JSON
decoder := xml.NewDecoder(file)
resultData := map[string]map[string]string{}
for {
t, _ := decoder.Token()
if t == nil {
break
}
switch et := t.(type) {
case xml.StartElement:
if et.Name.Local == "profile" {
var object XMLProfile
decoder.DecodeElement(&object, &et)
resultData = map[string]map[string]string{
object.ProfileName: {},
}
for _, val := range object.Fields {
resultData[object.ProfileName][val.Name] = val.Value
}
}
}
}
if out, err := json.MarshalIndent(resultData, "", "\t"); err != nil {
panic(err)
} else {
_ = ioutil.WriteFile("test.json", out, 0644)
}
Expect JSON:
{
"Profile 1": {
"role": "user"
},
"Profile 2": {
"role": "user"
},
"Profile 3": {
"role": "admin"
}
}
Actual JSON:
{
"Profile 3": {
"role": "admin"
}
}
Seems like you are recreating the resultData after each iteration in the nodes named "profile". As that happens, only the last one will reach the code where you write the JSON.
Try this:
decoder := xml.NewDecoder(file)
resultData := map[string]map[string]string{}
for {
t, _ := decoder.Token()
if t == nil {
break
}
switch et := t.(type) {
case xml.StartElement:
if et.Name.Local == "profile" {
var object XMLProfile
decoder.DecodeElement(&object, &et)
resultData[object.ProfileName] = map[string]string{}
for _, val := range object.Fields {
resultData[object.ProfileName][val.Name] = val.Value
}
}
}
}
if out, err := json.MarshalIndent(resultData, "", "\t"); err != nil {
panic(err)
} else {
_ = ioutil.WriteFile("test.json", out, 0644)
}
I would also check if no duplicate ProfileName happens to appear in the XML, as it would override the previous entry.

Parse JSON HTTP response using golang

I am trying to get the value of say "ip" from my following curl output:
{
"type":"example",
"data":{
"name":"abc",
"labels":{
"key":"value"
}
},
"subsets":[
{
"addresses":[
{
"ip":"192.168.103.178"
}
],
"ports":[
{
"port":80
}
]
}
]
}
I have found many examples in the internet to parse json output of curl requests and I have written the following code, but that doesn't seem to return me the value of say "ip"
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
)
type svc struct {
Ip string `json:"ip"`
}
func main() {
url := "http://myurl.com"
testClient := http.Client{
Timeout: time.Second * 2, // Maximum of 2 secs
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatal(err)
}
res, getErr := testClient.Do(req)
if getErr != nil {
log.Fatal(getErr)
}
body, readErr := ioutil.ReadAll(res.Body)
if readErr != nil {
log.Fatal(readErr)
}
svc1 := svc{}
jsonErr := json.Unmarshal(body, &svc1)
if jsonErr != nil {
log.Fatal(jsonErr)
}
fmt.Println(svc1.Ip)
}
I would appreciate if anyone could provide me hints on what I need to add to my code to get the value of say "ip".
You can create structs which reflect your json structure and then decode your json.
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
)
type Example struct {
Type string `json:"type,omitempty"`
Subsets []Subset `json:"subsets,omitempty"`
}
type Subset struct {
Addresses []Address `json:"addresses,omitempty"`
}
type Address struct {
IP string `json:"IP,omitempty"`
}
func main() {
m := []byte(`{"type":"example","data": {"name": "abc","labels": {"key": "value"}},"subsets": [{"addresses": [{"ip": "192.168.103.178"}],"ports": [{"port": 80}]}]}`)
r := bytes.NewReader(m)
decoder := json.NewDecoder(r)
val := &Example{}
err := decoder.Decode(val)
if err != nil {
log.Fatal(err)
}
// If you want to read a response body
// decoder := json.NewDecoder(res.Body)
// err := decoder.Decode(val)
// Subsets is a slice so you must loop over it
for _, s := range val.Subsets {
// within Subsets, address is also a slice
// then you can access each IP from type Address
for _, a := range s.Addresses {
fmt.Println(a.IP)
}
}
}
The output would be:
192.168.103.178
By decoding this to a struct, you can loop over any slice and not limit yourself to one IP
Example here:
https://play.golang.org/p/sWA9qBWljA
One approach is to unmarshal the JSON to a map, e.g. (assumes jsData contains JSON string)
obj := map[string]interface{}{}
if err := json.Unmarshal([]byte(jsData), &obj); err != nil {
log.Fatal(err)
}
Next, implement a function for searching the value associated with a key from the map recursively, e.g.
func find(obj interface{}, key string) (interface{}, bool) {
//if the argument is not a map, ignore it
mobj, ok := obj.(map[string]interface{})
if !ok {
return nil, false
}
for k, v := range mobj {
//key match, return value
if k == key {
return v, true
}
//if the value is a map, search recursively
if m, ok := v.(map[string]interface{}); ok {
if res, ok := find(m, key); ok {
return res, true
}
}
//if the value is an array, search recursively
//from each element
if va, ok := v.([]interface{}); ok {
for _, a := range va {
if res, ok := find(a, key); ok {
return res,true
}
}
}
}
//element not found
return nil,false
}
Note, that the above function return an interface{}. You need to convert it to appropriate type, e.g. using type switch:
if ip, ok := find(obj, "ip"); ok {
switch v := ip.(type) {
case string:
fmt.Printf("IP is a string -> %s\n", v)
case fmt.Stringer:
fmt.Printf("IP implements stringer interface -> %s\n", v.String())
case int:
default:
fmt.Printf("IP = %v, ok = %v\n", ip, ok)
}
}
A working example can be found at https://play.golang.org/p/O5NUi4J0iR
Typically in these situations you will see people describe all of these sub struct types. If you don't actually need to reuse the definition of any sub structs (like as a type for a function argument), then you don't need to define them. You can just use one definition for the whole response. In addition, in some cases you don't need to define a type at all, you can just do it at the time of declaration:
package main
import "encoding/json"
const s = `
{
"subsets": [
{
"addresses": [
{"ip": "192.168.103.178"}
]
}
]
}
`
func main() {
var svc struct {
Subsets []struct {
Addresses []struct { Ip string }
}
}
json.Unmarshal([]byte(s), &svc)
ip := svc.Subsets[0].Addresses[0].Ip
println(ip == "192.168.103.178")
}
You can write your own decoder or use existing third-party decoders.
For instance, github.com/buger/jsonparser could solve your problem by iterating throw array (two times).
package main
import (
"github.com/buger/jsonparser"
"fmt"
)
var data =[]byte(`{
"type":"example",
"data":{
"name":"abc",
"labels":{
"key":"value"
}
},
"subsets":[
{
"addresses":[
{
"ip":"192.168.103.178"
}
],
"ports":[
{
"port":80
}
]
}
]
}`)
func main() {
jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
v, _, _, err := jsonparser.Get(value, "ip")
if err != nil {
return
}
fmt.Println("ip: ", string(v[:]))
}, "addresses")
}, "subsets")
}
Output: ip: 192.168.103.178

create a map of string from a JSON with unknow key-values in Go

I try to create a map of strings from a JSON with an undefined number of unknow key-values.
Here is my example JSON file:
{
"localhost":
{
"tag": "dev_latest",
"vhost": "localhost.com"
},
"development":
{
"tag": "dev_latest",
"vhost": "dev.com"
}
}
I want to create a map[string]string with value like this:
config := map[string]string{
"localhost-tag": "dev_latest",
"localhost-vhost": "localhost.com,
"development-tag": "dev_latest,
...
}
To parse a JSON with "github.com/jmoiron/jsonq" with known values, is quite easy, but in this case, localhost can be anything and tag can be any other thing.
My entry point in my Go code is like this:
func ParseJson(){
configPath := GetConfigPath()
b, err := ioutil.ReadFile(configPath)
// Here, I need to create my map of strings..
return configKeyStr
}
Any help will be really appreciate.
Thanks!
Easy to do. Simply convert.
package main
import (
"encoding/json"
"fmt"
"log"
)
const s = `
{
"localhost":
{
"tag": "dev_latest",
"vhost": "localhost.com"
},
"development":
{
"tag": "dev_latest",
"vhost": "dev.com"
}
}
`
func main() {
var m map[string]interface{}
err := json.Unmarshal([]byte(s), &m)
if err != nil {
log.Fatal(err)
}
mm := make(map[string]string)
for k, v := range m {
mm[k] = fmt.Sprint(v)
}
fmt.Println(mm)
}
UPDATE
Wrote flatten (maybe works as charm)
package main
import (
"encoding/json"
"fmt"
"log"
"reflect"
)
const s = `
{
"localhost":
{
"tag": "dev_latest",
"vhost": "localhost.com"
},
"development":
{
"tag": "dev_latest",
"vhost": "dev.com"
}
}
`
func flatten(m map[string]interface{}) map[string]string {
mm := make(map[string]string)
for k, v := range m {
switch reflect.TypeOf(v).Kind() {
case reflect.Map:
mv := flatten(v.(map[string]interface{}))
for kk, vv := range mv {
mm[k+"-"+kk] = vv
}
case reflect.Array, reflect.Slice:
for kk, vv := range m {
if reflect.TypeOf(vv).Kind() == reflect.Map {
mv := flatten(vv.(map[string]interface{}))
for kkk, vvv := range mv {
mm[k+"-"+kkk] = vvv
}
} else {
mm[k+"-"+kk] = fmt.Sprint(vv)
}
}
default:
mm[k] = fmt.Sprint(v)
}
}
return mm
}
func main() {
var m map[string]interface{}
err := json.Unmarshal([]byte(s), &m)
if err != nil {
log.Fatal(err)
}
b, _ := json.MarshalIndent(flatten(m), "", " ")
println(string(b))
}
You can't have this automatically, but you can range over the "internal" maps, and combine the outer keys with the inner keys using simple string concatenation (+ operator). Also it's recommended to unmarshal directly into a value of map[string]map[string]string so you don't need to use type assertions. Also no need to use any external libraries for this, the standard encoding/json package is perfectly enough for this.
Example:
var mm map[string]map[string]string
if err := json.Unmarshal([]byte(src), &mm); err != nil {
panic(err)
}
config := map[string]string{}
for mk, m := range mm {
for k, v := range m {
config[mk+"-"+k] = v
}
}
fmt.Println(config)
Output is as expected (try it on the Go Playground):
map[localhost-tag:dev_latest localhost-vhost:localhost.com
development-tag:dev_latest development-vhost:dev.com]
Since in the question you mentioned undefined number of unknown key-values, you may need to deal with JSON document with unknown number of nesting level and having a value other than string. In this case, you need to Unmarshal json to map[string]interface{}, then use recursion to make flat map. Once the json document unmrashaled to map[string]interface{}, use the following function:
func flatMap(src map[string]interface{}, baseKey, sep string, dest map[string]string) {
for key, val := range src {
if len(baseKey) != 0 {
key = baseKey + sep + key
}
switch val := val.(type) {
case map[string]interface{}:
flatMap(val, key, sep, dest)
case string:
dest[key] = val
case fmt.Stringer:
dest[key] = val.String()
default:
//TODO: You may need to handle ARRAY/SLICE
//simply convert to string using `Sprintf`
//NOTE: modify as needed.
dest[key] = fmt.Sprintf("%v", val)
}
}
}
The working solution adapted from mattn answer at https://play.golang.org/p/9SQsbAUFdY
As pointed by mattn, you may have problem when you want to writeback the configuration value. In that case, use the existing library/framework.