Unmarshal nested map with int keys - json

If I have a map, I can Marshal it no problem:
package main
import (
"encoding/json"
"os"
)
type object map[int]interface{}
func main() {
obj := object{
1: "one", 2: object{3: "three"},
}
buf, err := json.Marshal(obj)
if err != nil {
panic(err)
}
os.Stdout.Write(buf) // {"1":"one","2":{"3":"three"}}
}
However I want to do the reverse. I tried this:
package main
import (
"encoding/json"
"fmt"
)
func main() {
buf := []byte(`{"1":"one","2":{"3":"three"}}`)
var obj map[int]interface{}
json.Unmarshal(buf, &obj)
// map[int]interface {}{1:"one", 2:map[string]interface {}{"3":"three"}}
fmt.Printf("%#v\n", obj)
}
Only the top level has the correct type. Is it possible to do what I am wanting?

JSON keys are never anything but strings, that's how the spec is defined, and you can see that in the output of you marshal. So when you try the reverse with interface{} as the top level map's value type, the type information for the nested objects is lost. You'd need a custom map type that implements UnmarshalJSON to be able to do what you want.
For example:
type IntKeyMap map[int]interface{}
func (m *IntKeyMap) UnmarshalJSON(data []byte) error {
raw := map[int]json.RawMessage{}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
for k, v := range raw {
// check if the value is a nested object
if len(v) > 0 && v[0] == '{' && v[len(v)-1] == '}' {
// The following assumes that the nested JSON object's
// key strings represent integers, if that is not the
// case this block will fail.
obj := IntKeyMap{}
if err := json.Unmarshal([]byte(v), &obj); err != nil {
return err
}
(*m)[k] = obj
} else {
var i interface{}
if err := json.Unmarshal([]byte(v), &i); err != nil {
return err
}
(*m)[k] = i
}
}
return nil
}
https://go.dev/play/p/lmyhqD__Uod

Related

Can't generate scalar JSON in gqlgen

I have a GQL scheme:
extend type MyType #key(fields: "id") {
id: ID! #external
properties: JSON #external
myField: String! #requires(fields: "properties")
}
scalar JSON
In graph/model/model.go:
package model
import (
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
)
type JSON map[string]interface{}
// UnmarshalGQL implements the graphql.Unmarshaler interface
func (b *JSON) UnmarshalGQL(v interface{}) error {
*b = make(map[string]interface{})
byteData, err := json.Marshal(v)
if err != nil {
panic("FAIL WHILE MARSHAL SCHEME")
}
tmp := make(map[string]interface{})
err = json.Unmarshal(byteData, &tmp)
if err != nil {
panic("FAIL WHILE UNMARSHAL SCHEME")
//return fmt.Errorf("%v", err)
}
*b = tmp
return nil
}
// MarshalGQL implements the graphql.Marshaler interface
func (b JSON) MarshalGQL(w io.Writer) {
byteData, err := json.Marshal(b)
if err != nil {
panic("FAIL WHILE MARSHAL SCHEME")
}
_, _ = w.Write(byteData)
}
But when I run go run github.com/99designs/gqlgen generate
error:
generating core failed: type.gotpl: template: type.gotpl:52:28: executing "type.gotpl" at <$type.Elem.GO>: nil pointer evaluating *config.TypeReference.
GOexit status 1
I just need to get map[string]interface{} which called JSON. I knew there's scalar Map, but for apollo federation that field must be called JSON.
it's should to replace MarshalGQL to MarshalJSON like:
type JSON map[string]interface{}
func MarshalJSON(b JSON) graphql.Marshaler {
return graphql.WriterFunc(func(w io.Writer) {
byteData, err := json.Marshal(b)
if err != nil {
log.Printf("FAIL WHILE MARSHAL JSON %v\n", string(byteData))
}
_, err = w.Write(byteData)
if err != nil {
log.Printf("FAIL WHILE WRITE DATA %v\n", string(byteData))
}
})
}
func UnmarshalJSON(v interface{}) (JSON, error) {
byteData, err := json.Marshal(v)
if err != nil {
return JSON{}, fmt.Errorf("FAIL WHILE MARSHAL SCHEME")
}
tmp := make(map[string]interface{})
err = json.Unmarshal(byteData, &tmp)
if err != nil {
return JSON{}, fmt.Errorf("FAIL WHILE UNMARSHAL SCHEME")
}
return tmp, nil
}

How do you modify this struct in Golang to accept two different results?

For the following JSON response {"table_contents":[{"id":100,"description":"text100"},{"id":101,"description":"text101"},{"id":1,"description":"text1"}]}
All you have to do is to produce the following code to execute it properly and be able to reads fields from the struct, such as:
package main
import (
"fmt"
"encoding/json"
)
type MyStruct1 struct {
TableContents []struct {
ID int
Description string
} `json:"table_contents"`
}
func main() {
result:= []byte(`{"table_contents":[{"id":100,"description":"text100"},{"id":101,"description":"text101"},{"id":1,"description":"text1"}]}`)
var container MyStruct1
err := json.Unmarshal(result, &container)
if err != nil {
fmt.Println(" [0] Error message: " + err.Error())
return
}
for i := range container.TableContents {
fmt.Println(container.TableContents[i].Description)
}
}
But how do you deal with the following JSON response? {"table_contents":[[{"id":100,"description":"text100"},{"id":101,"description":"text101"}],{"id":1,"description":"text1"}]} You can either get this response or the one above, it is important to modify the struct to accept both.
I did something like this, with the help of internet:
package main
import (
"fmt"
"encoding/json"
)
type MyStruct1 struct {
TableContents []TableContentUnion `json:"table_contents"`
}
type TableContentClass struct {
ID int
Description string
}
type TableContentUnion struct {
TableContentClass *TableContentClass
TableContentClassArray []TableContentClass
}
func main() {
result:= []byte(`{"table_contents":[[{"id":100,"description":"text100"},{"id":101,"description":"text101"}],{"id":1,"description":"text1"}]}`)
var container MyStruct1
err := json.Unmarshal(result, &container)
if err != nil {
fmt.Println(" [0] Error message: " + err.Error())
return
}
for i := range container.TableContents {
fmt.Println(container.TableContents[i])
}
}
but it does not go past the error message :(
[0] Error message: json: cannot unmarshal array into Go struct field MyStruct1.table_contents of type main.TableContentUnion*
Been struggling to come up with a solution for hours. If someone could help I would be happy. Thank you for reading. Let me know if you have questions
Inside table_contents you have two type options (json object or list of json objects). What you can do is to unmarshall into an interface and then run type-check on it when using it:
type MyStruct1 struct {
TableContents []interface{} `json:"table_contents"`
}
...
for i := range container.TableContents {
switch container.TableContents[i].(type){
case map[string]interface{}:
fmt.Println("json object")
case []interface{}:
fmt.Println("list")
}
}
From there you can use some library (e.g. https://github.com/mitchellh/mapstructure) to map unmarshalled struct to your TableContentClass type. See PoC playground here: https://play.golang.org/p/NhVUhQayeL_C
Custom UnmarshalJSON function
You can also create a custom UnmarshalJSON function on the object that has the 2 possibilities. In you case that would be TableContentUnion.
In the custom unmarshaller you can then decide how to unmarshal the content.
func (s *TableContentUnion) UnmarshalJSON(b []byte) error {
// Note that we get `b` as bytes, so we can also manually check to see
// if it is an array (starts with `[`) or an object (starts with `{`)
var jsonObj interface{}
if err := json.Unmarshal(b, &jsonObj); err != nil {
return err
}
switch jsonObj.(type) {
case map[string]interface{}:
// Note: instead of using json.Unmarshal again, we could also cast the interface
// and build the values as in the example above
var tableContentClass TableContentClass
if err := json.Unmarshal(b, &tableContentClass); err != nil {
return err
}
s.TableContentClass = &tableContentClass
case []interface{}:
// Note: instead of using json.Unmarshal again, we could also cast the interface
// and build the values as in the example above
if err := json.Unmarshal(b, &s.TableContentClassArray); err != nil {
return err
}
default:
return errors.New("TableContentUnion.UnmarshalJSON: unknown content type")
}
return nil
}
The rest then works like in your test code that was failing before. Here the working Go Playground
Unmarshal to map and manually build struct
You can always unmarshal a json (with an object at the root) into a map[string]interface{}. Then you can iterate things and further unmarshal them after checking what type they are.
Working example:
func main() {
result := []byte(`{"table_contents":[[{"id":100,"description":"text100"},{"id":101,"description":"text101"}],{"id":1,"description":"text1"}]}`)
var jsonMap map[string]interface{}
err := json.Unmarshal(result, &jsonMap)
if err != nil {
fmt.Println(" [0] Error message: " + err.Error())
return
}
cts, ok := jsonMap["table_contents"].([]interface{})
if !ok {
// Note: nil or missing 'table_contents" will also lead to this path.
fmt.Println("table_contents is not a slice")
return
}
var unions []TableContentUnion
for _, content := range cts {
var union TableContentUnion
if contents, ok := content.([]interface{}); ok {
for _, content := range contents {
contCls := parseContentClass(content)
if contCls == nil {
continue
}
union.TableContentClassArray = append(union.TableContentClassArray, *contCls)
}
} else {
contCls := parseContentClass(content)
union.TableContentClass = contCls
}
unions = append(unions, union)
}
container := MyStruct1{
TableContents: unions,
}
for i := range container.TableContents {
fmt.Println(container.TableContents[i])
}
}
func parseContentClass(value interface{}) *TableContentClass {
m, ok := value.(map[string]interface{})
if !ok {
return nil
}
return &TableContentClass{
ID: int(m["id"].(float64)),
Description: m["description"].(string),
}
}
This is most useful if the json has too many variations. For cases like this it might also make sense sometimes to switch to a json package that works differently like https://github.com/tidwall/gjson which gets values based on their path.
Use json.RawMessage to capture the varying parts of the JSON document. Unmarshal each raw message as appropriate.
func (ms *MyStruct1) UnmarshalJSON(data []byte) error {
// Declare new type with same base type as MyStruct1.
// This breaks recursion in call to json.Unmarshal below.
type x MyStruct1
v := struct {
*x
// Override TableContents field with raw message.
TableContents []json.RawMessage `json:"table_contents"`
}{
// Unmarshal all but TableContents directly to the
// receiver.
x: (*x)(ms),
}
err := json.Unmarshal(data, &v)
if err != nil {
return err
}
// Unmarahal raw elements as appropriate.
for _, tcData := range v.TableContents {
if bytes.HasPrefix(tcData, []byte{'{'}) {
var v TableContentClass
if err := json.Unmarshal(tcData, &v); err != nil {
return err
}
ms.TableContents = append(ms.TableContents, v)
} else {
var v []TableContentClass
if err := json.Unmarshal(tcData, &v); err != nil {
return err
}
ms.TableContents = append(ms.TableContents, v)
}
}
return nil
}
Use it like this:
var container MyStruct1
err := json.Unmarshal(result, &container)
if err != nil {
// handle error
}
Run it on the Go playground.
This approach does not add any outside dependencies. The function code does not need to modified when fields are added or removed from MyStruct1 or TableContentClass.

Unmarshall only approved fields

I'm trying to give permissions to users to edit certain fields of a struct.
This fields will vary depending on the action, the context, and the role the current user has.
At the moment I'm doing this in an imperative way but it's rather tedious and not scalable.
I thought having a list of approved_fields could be a nice and more scalable solution but I have no idea how to go through with this. I think reflection is the way to go but I don't know enough yet.
Any inspiration would help.
type Foo struct {
Bar1 int
Bar2 int
Bar3 int
}
foo := Foo{}
approved_fields := []string{"Bar1", "Bar2"}
decode(json_data, &foo, approved_fields)
foo2 := Foo{}
approved_fields := []string{"Bar1", "Bar3"}
decode(json_data, &foo2, approved_fields)
Here's how I would think about solving it. There may be something more efficient since I am unmarshalling the whole item before selectively choosing fields.
import (
"encoding/json"
"log"
"reflect"
"github.com/pkg/errors"
)
func fieldByName(s string, v interface{}) (reflect.Value, error) {
el := reflect.ValueOf(v).Elem()
fbn := el.FieldByName(s)
if !fbn.IsValid() {
return fbn, errors.Errorf("does not have field named %s", s)
}
return fbn, nil
}
func decode(data []byte, v interface{}, approvedFields []string) error {
typeOf := reflect.TypeOf(v)
if typeOf.Kind() == reflect.Ptr {
typeOf = typeOf.Elem()
}
if typeOf.Kind() == reflect.Slice {
return errors.New("does not support slices")
}
newItem := reflect.New(typeOf)
newItemInterface := newItem.Interface()
if err := json.Unmarshal(data, &newItemInterface); err != nil {
return errors.Wrap(err, "json unmarshall")
}
for _, approvedField := range approvedFields {
fbn, err := fieldByName(approvedField, v)
if err != nil {
return errors.Wrap(err, "field by name")
}
val, _ := fieldByName(approvedField, newItemInterface)
fbn.Set(val)
}
return nil
}
a test:
func TestBar1Bar2(t *testing.T) {
var json_data []byte
{
f := &Foo{
Bar1: 1,
Bar2: 2,
Bar3: 3,
}
json_data, _ = json.Marshal(f)
}
approved_fields := []string{"Bar1", "Bar2"}
f := &Foo{}
err := decode(json_data, f, approved_fields)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, f.Bar1, 1)
assert.Equal(t, f.Bar2, 2)
assert.Equal(t, f.Bar3, 0)
}

How to compare JSON with varying order?

I'm attempting to implement testing with golden files, however, the JSON my function generates varies in order but maintains the same values. I've implemented the comparison method used here:
How to compare two JSON requests?
But it's order dependent. And as stated here by brad:
JSON objects are unordered, just like Go maps. If
you're depending on the order that a specific implementation serializes your JSON
objects in, you have a bug.
I've written some sample code that simulated my predicament:
package main
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"reflect"
"time"
)
type example struct {
Name string
Earnings float64
}
func main() {
slice := GetSlice()
gfile, err := ioutil.ReadFile("testdata/example.golden")
if err != nil {
fmt.Println(err)
fmt.Println("Failed reading golden file")
}
testJSON, err := json.Marshal(slice)
if err != nil {
fmt.Println(err)
fmt.Println("Error marshalling slice")
}
equal, err := JSONBytesEqual(gfile, testJSON)
if err != nil {
fmt.Println(err)
fmt.Println("Error comparing JSON")
}
if !equal {
fmt.Println("Restults don't match JSON")
} else {
fmt.Println("Success!")
}
}
func GetSlice() []example {
t := []example{
example{"Penny", 50.0},
example{"Sheldon", 70.0},
example{"Raj", 20.0},
example{"Bernadette", 200.0},
example{"Amy", 250.0},
example{"Howard", 1.0}}
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(t), func(i, j int) { t[i], t[j] = t[j], t[i] })
return t
}
func JSONBytesEqual(a, b []byte) (bool, error) {
var j, j2 interface{}
if err := json.Unmarshal(a, &j); err != nil {
return false, err
}
if err := json.Unmarshal(b, &j2); err != nil {
return false, err
}
return reflect.DeepEqual(j2, j), nil
}
func WriteTestSliceToFile(arr []example, filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println("failed creating file: %s", err)
}
datawriter := bufio.NewWriter(file)
marshalledStruct, err := json.Marshal(arr)
if err != nil {
fmt.Println("Error marshalling json")
fmt.Println(err)
}
_, err = datawriter.Write(marshalledStruct)
if err != nil {
fmt.Println("Error writing to file")
fmt.Println(err)
}
datawriter.Flush()
file.Close()
}
JSON arrays are ordered. The json.Marshal function preserves order when encoding a slice to a JSON array.
JSON objects are not ordered. The json.Marshal function writes object members in sorted key order as described in the documentation.
The bradfitz comment JSON object ordering is not relevant to this question:
The application in the question is working with a JSON array, not a JSON object.
The package was updated to write object fields in sorted key order a couple of years after Brad's comment.
To compare slices while ignoring order, sort the two slices before comparing. This can be done before encoding to JSON or after decoding from JSON.
sort.Slice(slice, func(i, j int) bool {
if slice[i].Name != slice[j].Name {
return slice[i].Name < slice[j].Name
}
return slice[i].Earnings < slice[j].Earnings
})
For unit testing, you could use assert.JSONEq from Testify. If you need to do it programatically, you could follow the code of the JSONEq function.
https://github.com/stretchr/testify/blob/master/assert/assertions.go#L1551

Golang Converting JSON

map[key:2073933158088]
I need to grab the key out of this data structure as a string, but I can't seem to figure out how!
Help with this overly simple question very much appreciated.
The value above is encapsulated in the variable named data.
I have tried: data.key, data[key], data["key"], data[0] and none of these seem to be appropriate calls.
To define data I sent up a JSON packet to a queue on IronMQ. I then pulled the message from the queue and manipulated it like this:
payloadIndex := 0
for index, arg := range(os.Args) {
if arg == "-payload" {
payloadIndex = index + 1
}
}
if payloadIndex >= len(os.Args) {
panic("No payload value.")
}
payload := os.Args[payloadIndex]
var data interface{}
raw, err := ioutil.ReadFile(payload)
if err != nil {
panic(err.Error())
}
err = json.Unmarshal(raw, &data)
Design your data type to match json structure. This is how can you achieve this:
package main
import (
"fmt"
"encoding/json"
)
type Data struct {
Key string `json:"key"`
}
func main() {
data := new(Data)
text := `{ "key": "2073933158088" }`
raw := []byte(text)
err := json.Unmarshal(raw, data)
if err != nil {
panic(err.Error())
}
fmt.Println(data.Key)
}
Since the number in the json is unquoted, it's not a string, Go will panic if you try to just handle it as a string (playground: http://play.golang.org/p/i-NUwchJc1).
Here's a working alternative:
package main
import (
"fmt"
"encoding/json"
"strconv"
)
type Data struct {
Key string `json:"key"`
}
func (d *Data) UnmarshalJSON(content []byte) error {
var m map[string]interface{}
err := json.Unmarshal(content, &m)
if err != nil {
return err
}
d.Key = strconv.FormatFloat(m["key"].(float64), 'f', -1, 64)
return nil
}
func main() {
data := new(Data)
text := `{"key":2073933158088}`
raw := []byte(text)
err := json.Unmarshal(raw, data)
if err != nil {
panic(err.Error())
}
fmt.Println(data.Key)
}
You can see the result in the playground: http://play.golang.org/p/5hU3hdV3kK