How to parse json without a defined stucture? - json

I want to parse this JSON :
{
"error": null,
"id": "tutu",
"result": {
"param1": 559,
"param2": "yo",
"param3": {"tab":["a", "b"], "param4":"hello"},
}
}
The problem is that I want a flexible solution if the JSON structure changes: I want to be able to access each field with a key system (like in Javascript) without knowing in advance the JSON structure:
fmt.Println(jsonObj["result"]["param3"]["tab"][1])
Is it possible to do it ?

First, in your case unmarshall directly into map[string]interface{} to save an assertion.
var jsonObj map[string]interface{}
err := json.Unmarshal(b, &jsonObj)
Second, remember to check the assertion
result, ok := jsonObj["result"].(map[string]interface{})
if !ok {
panic("not json obj")
}
param3, ok := result["param3"].(map[string]interface{})
if !ok {
panic("not json obj")
}
tab, ok := param3["tab"].([]interface{})
if !ok {
panic("not json arr")
}
Lastly, you can declare a new type and "hide" the assertions in its methods
type AnyObj map[string]interface{}
func (obj AnyObj) MustObject(name string) AnyObj {
v, ok := obj[name].(map[string]interface{})
if !ok {
panic("not json obj")
}
return AnyObj(v)
}
func (obj AnyObj) MustArray(name string) []interface{} {
v, ok := obj[name].([]interface{})
if !ok {
panic("not json arr")
}
return v
}
Then use like this:
func main() {
var jsonObj AnyObj
err := json.Unmarshal(b, &jsonObj)
if (err != nil) {
panic(err);
}
tab := jsonObj.MustObject("result").MustObject("param3").MustArray("tab")
fmt.Println(tab[1])
}
check it here https://play.golang.org/p/ucdMZ0VEKcr

Yes, it is possible, although the fact that you need to assert the types makes it a bit unwieldy:
package main
import (
"encoding/json"
"fmt"
)
func main() {
b := []byte(`
{
"error": null,
"id": "tutu",
"result": {
"param1": 559,
"param2": "yo",
"param3": {"tab":["a", "b"], "param4": "hello"}
}
}
`)
var jsonObj interface{}
err := json.Unmarshal(b, &jsonObj)
if err != nil {
panic(err)
}
msg := jsonObj.(map[string]interface{})
result := msg["result"].(map[string]interface{})
param3 := result["param3"].(map[string]interface{})
tab := param3["tab"].([]interface{})
fmt.Println(tab[1])
}
This prints
b
like you would expect.
Note that this program will panic: not just if the JSON fails to parse, but also if the JSON does not exactly match the program's expectations: missing keys, different types and so on.
The Generic JSON with interface section of the JSON and Go article on The Go Blog has an example of checking the actual type of the thing at runtime; as was suggested in the comments, it's easy to forget.

this is mine used fastjson:
package main
import (
"fmt"
"github.com/valyala/fastjson"
)
type ParseValue struct {
*fastjson.Value
}
func Parse(b []byte) (*ParseValue, error) {
var p fastjson.Parser
v, err := p.ParseBytes(b)
if err != nil {
return nil, err
}
return &ParseValue{v}, nil
}
func (p *ParseValue) GetString(keys ...string) string {
return string(p.GetStringBytes(keys...))
}
func main() {
b := []byte(`
{
"error": null,
"id": "tutu",
"result": {
"param1": 559,
"param2": "yo",
"param3": {"tab":["a", "b"], "param4": "hello"}
}
}
`)
v, err := Parse(b)
if err != nil {
panic(err)
}
r := v.GetString("result", "param3", "tab", "1")
fmt.Println(r)
rr := v.GetUint("result", "param1")
fmt.Println(rr)
// output:
// b
// 559
}

Related

How do I json unmarshal slice inside a slice

I am trying to unmarshal some pretty ugly json but can't figure out how. I have:
package main
import "fmt"
import "encoding/json"
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
MyData []struct {
ID string `json:"id"`
Value int `json:"value"`
}
}
func main() {
b := `[
{
"id": 1,
"key": "my_key"
},
[
{
"id": "some_id",
"value": 12
},
{
"id": "anorther_id",
"value": 13
}
]
]`
var pk []PublicKey
err := json.Unmarshal([]byte(b), &pk)
if err != nil {
fmt.Println(err)
}
fmt.Println(pk)
}
For the result I am getting:
[{1 my_key []} {0 []}]
The second slice is empty when it shouldn't be.
EDIT:
The error I get is:
json: cannot unmarshal array into Go struct field PublicKey.key of type main.PublicKey
https://play.golang.org/p/cztXOchiiS5
That is some truly hideous JSON! I have two approaches to handling the mixed array elements and I like the 2nd one better. Here's the first approach using interface and a type switch:
package main
import (
"encoding/json"
"errors"
"fmt"
)
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
}
type MyData struct {
ID string `json:"id"`
Value int `json:"value"`
}
type MixedData struct {
Key []PublicKey
MyData [][]MyData
}
func (md *MixedData) UnmarshalJSON(b []byte) error {
md.Key = []PublicKey{}
md.MyData = [][]MyData{}
var obj []interface{}
err := json.Unmarshal([]byte(b), &obj)
if err != nil {
return err
}
for _, o := range obj {
switch o.(type) {
case map[string]interface{}:
m := o.(map[string]interface{})
id, ok := m["id"].(float64)
if !ok {
return errors.New("public key id must be an int")
}
pk := PublicKey{}
pk.ID = int(id)
pk.Key, ok = m["key"].(string)
if !ok {
return errors.New("public key key must be a string")
}
md.Key = append(md.Key, pk)
case []interface{}:
a := o.([]interface{})
myData := make([]MyData, len(a))
for i, x := range a {
m, ok := x.(map[string]interface{})
if !ok {
return errors.New("data array contains unexpected object")
}
val, ok := m["value"].(float64)
if !ok {
return errors.New("data value must be an int")
}
myData[i].Value = int(val)
myData[i].ID, ok = m["id"].(string)
if !ok {
return errors.New("data id must be a string")
}
md.MyData = append(md.MyData, myData)
}
default:
// got something unexpected, handle somehow
}
}
return nil
}
func main() {
b := `[
{
"id": 1,
"key": "my_key"
},
[
{
"id": "some_id",
"value": 12
},
{
"id": "another_id",
"value": 13
}
]
]`
m := MixedData{}
err := json.Unmarshal([]byte(b), &m)
if err != nil {
fmt.Println(err)
}
fmt.Println(m)
}
https://play.golang.org/p/g8d_AsH-pYY
Hopefully there aren't any unexpected other elements, but they can be handled similarly.
Here is the second that relies more on Go's internal JSON parsing with the help of json.RawMessage. It makes the same assumptions about the contents of the array. It assumes that any objects will Unmarshal into PublicKey instances and any arrays consist of only MyData instances. I also added how to marshal back into the target JSON for symmetry:
package main
import (
"encoding/json"
"fmt"
"os"
)
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
}
type MyData struct {
ID string `json:"id"`
Value int `json:"value"`
}
type MixedData struct {
Keys []PublicKey
MyData [][]MyData
}
func (md *MixedData) UnmarshalJSON(b []byte) error {
md.Keys = []PublicKey{}
md.MyData = [][]MyData{}
obj := []json.RawMessage{}
err := json.Unmarshal([]byte(b), &obj)
if err != nil {
return err
}
for _, o := range obj {
switch o[0] {
case '{':
pk := PublicKey{}
err := json.Unmarshal(o, &pk)
if err != nil {
return err
}
md.Keys = append(md.Keys, pk)
case '[':
myData := []MyData{}
err := json.Unmarshal(o, &myData)
if err != nil {
return err
}
md.MyData = append(md.MyData, myData)
default:
// got something unexpected, handle somehow
}
}
return nil
}
func (md *MixedData) MarshalJSON() ([]byte, error) {
out := make([]interface{}, len(md.Keys)+len(md.MyData))
i := 0
for _, x := range md.Keys {
out[i] = x
i++
}
for _, x := range md.MyData {
out[i] = x
i++
}
return json.Marshal(out)
}
func main() {
b := `[
{
"id": 1,
"key": "my_key"
},
[
{
"id": "some_id",
"value": 12
},
{
"id": "another_id",
"value": 13
}
]
]`
m := MixedData{}
err := json.Unmarshal([]byte(b), &m)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Println(m)
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
if err := enc.Encode(m); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
https://play.golang.org/p/ryZzaWKNcN0
Here's an approach that combines json.RawMessage with the trick of using the default unmarshaler in a type that implements json.Unmarshaler by creating a new temporary type that aliases the target type.
The idea is that we unmarshal the incoming array into a raw message and ensure that the array length is what we expect. Then we unmarshal the individual array elements into the custom struct types using their JSON tag annotations. The end result is that we can unmarshal the PublicKey type in the usual way and the UnmarshalJSON code is not terribly difficult to follow once you understand the tricks.
For example (Go Playground):
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
Data []MyData
}
type MyData struct {
ID string `json:"id"`
Value int `json:"value"`
}
func (pk *PublicKey) UnmarshalJSON(bs []byte) error {
// Unmarshal into a RawMessage so we can inspect the array length.
var rawMessage []json.RawMessage
err := json.Unmarshal(bs, &rawMessage)
if err != nil {
return err
}
if len(rawMessage) != 2 {
return fmt.Errorf("expected array of length 2, got %d", len(rawMessage))
}
// Parse the first object as PublicKey using the default unmarshaler
// using a temporary type that is an alias for the target type.
type PublicKey2 PublicKey
var pk2 PublicKey2
err = json.Unmarshal(rawMessage[0], &pk2)
if err != nil {
return err
}
// Parse the second object as []MyData in the usual way.
err = json.Unmarshal(rawMessage[1], &pk2.Data)
if err != nil {
return err
}
// Finally, assign the aliased object to the target object.
*pk = PublicKey(pk2)
return nil
}
func main() {
var pk PublicKey
err := json.Unmarshal([]byte(jsonstr), &pk)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n", pk)
// main.PublicKey{ID:1, Key:"my_key", Data:[]main.MyData{main.MyData{ID:"some_id", Value:12}, main.MyData{ID:"anorther_id", Value:13}}}
}

merge JSON objects recursively

Here's an example of what I need:
Default JSON:
{
"name": "John",
"greetings": {
"first": "hi",
"second": "hello"
}
}
merged with the changes:
{
"name": "Jane",
"greetings": {
"first": "hey"
}
}
should become:
{
"name": "Jane",
"greetings": {
"first": "hey",
"second": "hello"
}
}
Here's what I've tried:
package main
import (
"encoding/json"
"fmt"
"reflect"
)
func MergeJSON(defaultJSON, changedJSON string) string {
var defaultJSONDecoded map[string]interface{}
defaultJSONUnmarshalErr := json.Unmarshal([]byte(defaultJSON), &defaultJSONDecoded)
if defaultJSONUnmarshalErr != nil {
panic("Error unmarshalling first JSON")
}
var changedJSONDecoded map[string]interface{}
changedJSONUnmarshalErr := json.Unmarshal([]byte(changedJSON), &changedJSONDecoded)
if changedJSONUnmarshalErr != nil {
panic("Error unmarshalling second JSON")
}
for key, _ := range defaultJSONDecoded {
checkKeyBeforeMerging(key, defaultJSONDecoded[key], changedJSONDecoded[key], changedJSONDecoded)
}
mergedJSON, mergedJSONErr := json.Marshal(changedJSONDecoded)
if mergedJSONErr != nil {
panic("Error marshalling merging JSON")
}
return string(mergedJSON)
}
func checkKeyBeforeMerging(key string, defaultMap interface{}, changedMap interface{}, finalMap map[string]interface{}) {
if !reflect.DeepEqual(defaultMap, changedMap) {
switch defaultMap.(type) {
case map[string]interface{}:
//Check that the changed map value doesn't contain this map at all and is nil
if changedMap == nil {
finalMap[key] = defaultMap
} else if _, ok := changedMap.(map[string]interface{}); ok { //Check that the changed map value is also a map[string]interface
defaultMapRef := defaultMap.(map[string]interface{})
changedMapRef := changedMap.(map[string]interface{})
for newKey, _ := range defaultMapRef {
checkKeyBeforeMerging(newKey, defaultMapRef[newKey], changedMapRef[newKey], finalMap)
}
}
default:
//Check if the value was set, otherwise set it
if changedMap == nil {
finalMap[key] = defaultMap
}
}
}
}
func main() {
defaultJSON := `{"name":"John","greetings":{"first":"hi","second":"hello"}}`
changedJSON := `{"name":"Jane","greetings":{"first":"hey"}}`
mergedJSON := MergeJSON(defaultJSON, changedJSON)
fmt.Println(mergedJSON)
}
The code above returns the following:
{
"greetings": {
"first": "hey"
},
"name": "Jane",
"second": "hello"
}
So basically any changes should be applied to the default and return the full JSON. I also need this to work recursively.
How can I fix this? I can see where I went wrong, I'm just not sure how to make it work recursively.
Thanks
Your issue in the posted code is with your recursive call:
checkKeyBeforeMerging(newKey, defaultMapRef[newKey], changedMapRef[newKey], finalMap)
The reference to finalMap should actually be the nested part of the merged map. Meaning replace finalMap with something like finalMap[key].(map[string]interface{}).
I recently come across the same need, here is my solution:
// override common json by input json
func Override(input, common interface{}) {
switch inputData := input.(type) {
case []interface{}:
switch commonData := common.(type) {
case []interface{}:
for idx, v := range inputData {
Override(v, commonData[idx])
}
}
case map[string]interface{}:
switch commonData := common.(type) {
case map[string]interface{}:
for k, v := range commonData {
switch reflect.TypeOf(v).Kind() {
case reflect.Slice, reflect.Map:
Override(inputData[k], v)
default:
// do simply replacement for primitive type
_, ok := inputData[k]
if !ok {
inputData[k] = v
}
}
}
}
}
return
}
unmarshal json before call:
var commmon interface{}
if err = json.Unmarshal("common data", &commmon); err != nil {
logger.Error(err)
return
}
var input interface{}
if err = json.Unmarshal("input data", &input); err != nil {
logger.Error(err)
return
}
Override(input, commmon)

How can I know that the field is set to null?

I want to output an error if the field in json contains a value of null. How can I do it? I have tried "encoding/json". Maybe I need another library.
Code example:
package main
import (
"encoding/json"
"fmt"
"strings"
)
type Item struct {
Value *int
}
func main() {
var jsonBlob = `[
{},
{"Value": null},
{"Value": 0},
{"Value": 1}
]`
var items []Item
err := json.NewDecoder(strings.NewReader(jsonBlob)).Decode(&items)
if err != nil {
fmt.Println("error:", err)
}
for _, a := range items {
if a.Value != nil {
fmt.Println(*a.Value)
} else {
fmt.Println(a.Value)
}
}
}
I got:
<nil>
<nil>
0
1
I want:
<nil>
<error>
0
1
Please help. Many thanks!
If you want to control how a type is unmarshaled, you can implement json.Unmarshaler
Since a map allows you to tell the difference between an unset value, and a null value, unmarshaling first into a generic map[string]interface{} will allow you to inspect the values without tokenizing the JSON.
type Item struct {
Value *int
}
func (i *Item) UnmarshalJSON(b []byte) error {
tmp := make(map[string]interface{})
err := json.Unmarshal(b, &tmp)
if err != nil {
return err
}
val, ok := tmp["Value"]
if ok && val == nil {
return errors.New("Value cannot be nil")
}
if !ok {
return nil
}
f, ok := val.(float64)
if !ok {
return fmt.Errorf("unexpected type %T for Value", val)
}
n := int(f)
i.Value = &n
return nil
}
https://play.golang.org/p/MNDsQpfEJA

Parse JSON HTTP response using golang

I am trying to get the value of say "ip" from my following curl output:
{
"type":"example",
"data":{
"name":"abc",
"labels":{
"key":"value"
}
},
"subsets":[
{
"addresses":[
{
"ip":"192.168.103.178"
}
],
"ports":[
{
"port":80
}
]
}
]
}
I have found many examples in the internet to parse json output of curl requests and I have written the following code, but that doesn't seem to return me the value of say "ip"
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
)
type svc struct {
Ip string `json:"ip"`
}
func main() {
url := "http://myurl.com"
testClient := http.Client{
Timeout: time.Second * 2, // Maximum of 2 secs
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatal(err)
}
res, getErr := testClient.Do(req)
if getErr != nil {
log.Fatal(getErr)
}
body, readErr := ioutil.ReadAll(res.Body)
if readErr != nil {
log.Fatal(readErr)
}
svc1 := svc{}
jsonErr := json.Unmarshal(body, &svc1)
if jsonErr != nil {
log.Fatal(jsonErr)
}
fmt.Println(svc1.Ip)
}
I would appreciate if anyone could provide me hints on what I need to add to my code to get the value of say "ip".
You can create structs which reflect your json structure and then decode your json.
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
)
type Example struct {
Type string `json:"type,omitempty"`
Subsets []Subset `json:"subsets,omitempty"`
}
type Subset struct {
Addresses []Address `json:"addresses,omitempty"`
}
type Address struct {
IP string `json:"IP,omitempty"`
}
func main() {
m := []byte(`{"type":"example","data": {"name": "abc","labels": {"key": "value"}},"subsets": [{"addresses": [{"ip": "192.168.103.178"}],"ports": [{"port": 80}]}]}`)
r := bytes.NewReader(m)
decoder := json.NewDecoder(r)
val := &Example{}
err := decoder.Decode(val)
if err != nil {
log.Fatal(err)
}
// If you want to read a response body
// decoder := json.NewDecoder(res.Body)
// err := decoder.Decode(val)
// Subsets is a slice so you must loop over it
for _, s := range val.Subsets {
// within Subsets, address is also a slice
// then you can access each IP from type Address
for _, a := range s.Addresses {
fmt.Println(a.IP)
}
}
}
The output would be:
192.168.103.178
By decoding this to a struct, you can loop over any slice and not limit yourself to one IP
Example here:
https://play.golang.org/p/sWA9qBWljA
One approach is to unmarshal the JSON to a map, e.g. (assumes jsData contains JSON string)
obj := map[string]interface{}{}
if err := json.Unmarshal([]byte(jsData), &obj); err != nil {
log.Fatal(err)
}
Next, implement a function for searching the value associated with a key from the map recursively, e.g.
func find(obj interface{}, key string) (interface{}, bool) {
//if the argument is not a map, ignore it
mobj, ok := obj.(map[string]interface{})
if !ok {
return nil, false
}
for k, v := range mobj {
//key match, return value
if k == key {
return v, true
}
//if the value is a map, search recursively
if m, ok := v.(map[string]interface{}); ok {
if res, ok := find(m, key); ok {
return res, true
}
}
//if the value is an array, search recursively
//from each element
if va, ok := v.([]interface{}); ok {
for _, a := range va {
if res, ok := find(a, key); ok {
return res,true
}
}
}
}
//element not found
return nil,false
}
Note, that the above function return an interface{}. You need to convert it to appropriate type, e.g. using type switch:
if ip, ok := find(obj, "ip"); ok {
switch v := ip.(type) {
case string:
fmt.Printf("IP is a string -> %s\n", v)
case fmt.Stringer:
fmt.Printf("IP implements stringer interface -> %s\n", v.String())
case int:
default:
fmt.Printf("IP = %v, ok = %v\n", ip, ok)
}
}
A working example can be found at https://play.golang.org/p/O5NUi4J0iR
Typically in these situations you will see people describe all of these sub struct types. If you don't actually need to reuse the definition of any sub structs (like as a type for a function argument), then you don't need to define them. You can just use one definition for the whole response. In addition, in some cases you don't need to define a type at all, you can just do it at the time of declaration:
package main
import "encoding/json"
const s = `
{
"subsets": [
{
"addresses": [
{"ip": "192.168.103.178"}
]
}
]
}
`
func main() {
var svc struct {
Subsets []struct {
Addresses []struct { Ip string }
}
}
json.Unmarshal([]byte(s), &svc)
ip := svc.Subsets[0].Addresses[0].Ip
println(ip == "192.168.103.178")
}
You can write your own decoder or use existing third-party decoders.
For instance, github.com/buger/jsonparser could solve your problem by iterating throw array (two times).
package main
import (
"github.com/buger/jsonparser"
"fmt"
)
var data =[]byte(`{
"type":"example",
"data":{
"name":"abc",
"labels":{
"key":"value"
}
},
"subsets":[
{
"addresses":[
{
"ip":"192.168.103.178"
}
],
"ports":[
{
"port":80
}
]
}
]
}`)
func main() {
jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
v, _, _, err := jsonparser.Get(value, "ip")
if err != nil {
return
}
fmt.Println("ip: ", string(v[:]))
}, "addresses")
}, "subsets")
}
Output: ip: 192.168.103.178

How do I decode the following JSON?

I have a JSON object of the format
{
"results": [
{
"hits": [
{
"title": "Juliette DELAUNAY",
"author:url": "abc.com"
}
]
}
]
}
To decode in my go program, I have made the following structs
type results struct{
Result []result `json:"results"`
}
type result struct{
Hits []hit `json:"hits"`
}
type hit struct{
Name string `json:"title"`
Url string `json:"author:url"`
}
var m =make(map[string]string)
var t results
But when I try to do the following,
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(&t)
if err != nil {
fmt.Println(err)
}
for _,x := range t.Result[0].Hits{
m[x.Name] = x.Url
fmt.Println(x.Name,x.Url)
}
It gives a runtime error saying index is out of range. What am I doing wrong? Are my structs incorrect for the given json?
EDIT : The JSON file I need to decode
var jsonStr = []byte(`{"requests":[{"indexName":"recherchepepitesAtoZ","params":"query=x&hitsPerPage=2817&maxValuesPerFacet=42&page=0&facets=%5B%22field_frenchtech_hub_pepite%22%2C%22field_categorie%22%2C%22field_frenchtech_hub_pepite%22%5D&tagFilters="}]}`)
req, err := http.NewRequest("POST", "http://6y0slgl8yj-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%20(lite)%203.20.4%3Binstantsearch.js%201.10.4%3BJS%20Helper%202.18.0&x-algolia-application-id=6Y0SLGL8YJ&x-algolia-api-key=6832a361e1e1628f8ddb2483623104c6", bytes.NewBuffer(jsonStr))
//req.Header.Set("X-Custom-Header", "application/x-www-form-urlencoded")
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
Here is a slightly modified version that works on my machine and go playground:
GoPlayground
package main
import (
"encoding/json"
"fmt"
"strings"
)
type results struct {
Result []result `json:"results"`
}
type result struct {
Hits []hit `json:"hits"`
}
type hit struct {
Name string `json:"title"`
Url string `json:"author:url"`
}
var m = make(map[string]string)
func main() {
jsonSample := `{
"results": [
{
"hits": [
{
"title": "Juliette DELAUNAY",
"author:url": "abc.com"
}
]
}
]
}`
var t results
decoder := json.NewDecoder(strings.NewReader(jsonSample))
err := decoder.Decode(&t)
if err != nil {
fmt.Println(err)
}
for _, x := range t.Result[0].Hits {
m[x.Name] = x.Url
fmt.Println(x.Name, x.Url)
}
}