How to unmarshall different json format - json

I read lots of similar questions but not able to find the right solution yet. Hope someone can give me better idea.
My error response could be one of these format:
var errProxy = `{"errors":{"id": "1", "message": "failed to resolve the ip", "status": "failed"}}`
var errServer = `{"errors": "failed to ping the dns server."}`
I am trying to solve this by using two structs and trying to unmarshall one after another until err is nil and return:
// ErrorServer and ErrorProxy is my two struct to represent my above two error response.
type ErrorServer struct {
Error string `json:"errors"`
}
type ErrorResponse struct {
ID string `json:"id"`
Message string `json:"message"`
Status string `json:"status"`
}
type ErrorProxy struct {
ErrorResponse
}
This is the function I used to parse it:
func parseErrorResponse(body io.ReadCloser) (string, error) {
var errServer ErrorServer
var errProxy ErrorProxy
err := json.NewDecoder(body).Decode(&errServer)
if err != nil {
err = json.NewDecoder(body).Decode(&errProxy)
if err != nil {
return "", err
}
return errProxy.Error, nil
}
return errServer.Errors, nil
}

I think a custom unmarshal function is good for this case
type ErrorServer struct {
Error string
}
type ErrorResponse struct {
ID string
Message string
Status string
}
type ErrorProxy struct {
Err ErrorResponse
}
func parseErrorResponse(body io.Reader) (interface{}, error) {
data := make(map[string]interface{})
if err := json.NewDecoder(body).Decode(&data); err != nil {
return nil, err
}
// Check type of errors field
switch errors := data["errors"].(type) {
case string:
// Unmarshal for ErrorServer
errServer := &ErrorServer{
Error: errors,
}
return errServer, nil
case map[string]interface{}:
// Unmarshal for ErrorProxy
errProxy := &ErrorProxy{
Err: ErrorResponse{
ID: errors["id"].(string),
Message: errors["message"].(string),
Status: errors["status"].(string),
},
}
return errProxy, nil
default:
return nil, fmt.Errorf(`failed to parse "errors" field`)
}
}
func main() {
body := bytes.NewReader([]byte(`{"errors": "failed to ping the dns server."}`))
//body := bytes.NewReader([]byte(`{"errors":{"id": "1", "message": "failed to resolve the ip", "status": "failed"}}`))
parsedErr, _ := parseErrorResponse(body)
switch err := parsedErr.(type) {
case *ErrorServer:
fmt.Printf("err server: %+v \n", err)
case *ErrorProxy:
fmt.Printf("err response: %+v \n", err)
}
}

Another way to do that is using type assertion. You can unmarshal that json string into map[string]interface{} and check whether that value interface{} is a string or map[string]interface{}. Depends on its type, you know which kind of error it is and construct a struct from that.

Funny enough, I just gave the answer to this problem here: Use a customized UnmarshalJSON function. How do you modify this struct in Golang to accept two different results?
Applied to your situation:
package main
import (
"encoding/json"
"fmt"
"log"
)
type Answer struct {
Errors Error `json:"errors"`
}
type ErrorResponse struct {
ID string `json:"id"`
Message string `json:"message"`
Status string `json:"status"`
}
type Error struct {
Error string
ErrorResponse ErrorResponse
}
func (s *Error) UnmarshalJSON(b []byte) error {
if len(b) == 0 {
// no data, nothing to do
return nil
}
if b[0] == '{' {
// is object
return json.Unmarshal(b, &s.ErrorResponse)
}
return json.Unmarshal(b, &s.Error)
}
func main() {
var errProxy = []byte(`{"errors":{"id": "1", "message": "failed to resolve the ip", "status": "failed"}}`)
var errServer = []byte(`{"errors": "failed to ping the dns server."}`)
var answ Answer
if err := json.Unmarshal(errProxy, &answ); err != nil {
log.Fatal(err)
}
fmt.Println(answ)
answ = Answer{}
if err := json.Unmarshal(errServer, &answ); err != nil {
log.Fatal(err)
}
fmt.Println(answ)
}
Go PlayGround
Key in the above example is the type that contains both variations. We can also simplify that as both errors could be contained within the ErrorResponse type:
type Answer struct {
Errors ErrorResponse `json:"errors"`
}
type ErrorResponse struct {
ID string
Message string
Status string
}
func (s *ErrorResponse) UnmarshalJSON(b []byte) error {
if len(b) == 0 {
// no data, nothing to do
return nil
}
if b[0] == '{' {
// is object
var tmp struct {
ID string `json:"id"`
Message string `json:"message"`
Status string `json:"status"`
}
if err := json.Unmarshal(b, &tmp); err != nil {
return err
}
s.ID = tmp.ID
s.Message = tmp.Message
s.Status = tmp.Status
return nil
}
return json.Unmarshal(b, &s.Message)
}
The Error struct is not needed here.

Related

How to unmarshal escaped JSON string with UnmarshalJSON

I'm trying to unmarshal the following JSON string
token = `{
"id": 1,
"token": {
"id": 2248637,
"metadata": {
"name": "Name #1",
"formats": "[{\"mimeType\": \"model/gltf-binary\", \"uri\": \"uri1\", {\"mimeType\": \"image/gif\", \"uri\": \"uri2\"}]"
}
}`
I can unmarshal it with 2 phases like this. However, I would like to use custom unmarshalJSON but I fail. I got error
My code as follow:
type FileFormat struct {
MIMEType string
URI string
}
func (format *FileFormat) UnmarshalJSON(data []byte) error {
var aux []interface{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
format.MIMEType = aux[0].(string)
format.URI = aux[1].(string)
return nil
}
type TokenMetadata struct {
Name string `json:"name"`
Formats []FileFormat `json:"formats"`
}
type Token struct {
ID TokenID `json:"tokenId"`
Metadata TokenMetadata `json:"metadata"`
}
func main() {
var tokenRes OwnedToken
if err := json.Unmarshal([]byte(token), &tokenRes); err != nil {
fmt.Println(err)
}
}
And the error is
json: cannot unmarshal string into Go struct field TokenMetadata.token.metadata.formats of type []main.FileFormat
How can I fix this problem? Many thanks!
The JSON array of file formats is double encoded. Declare a Go type corresponding to the array. Double decode in the UnmarshalJSON method for that type.
type FileFormats []FileFormat
func (ff *FileFormats) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
return json.Unmarshal(
[]byte(s),
(*[]FileFormat)(ff))
}
type TokenMetadata struct {
Name string `json:"name"`
Formats FileFormats `json:"formats"`
}
Note: The conversion from from *FileFormats to *[]FileFormat is required to prevent recursion.

How to handle unmarshaling to a custom interface whose type could only be determined after unmarshaling

I have a json response like this
{
"foo" : "bar",
"object" : {
"type" : "action",
"data" : "somedata"
}
}
Here the object could be one of multiple types. I define the types and have them implement a common interface.
type IObject interface {
GetType() string
}
type Action struct {
Type string `json:"type"`
Data string `json:"data"`
}
func (a Action) GetType() string {
return "action"
}
type Activity struct {
Type string `json:"type"`
Duration int `json:"duration"`
}
func (a Activity) GetType() string {
return "activity"
}
And a response struct
type Response struct {
Foo string `json:"foo"`
Object IObject `json:"object"`
}
As the type information of a struct that implements IObject is contained within the struct, there is no way to learn in without unmarshaling. I also cannot change the structure of the json response. Currently I am dealing with this problem using a custom unmarshaller:
func UnmarshalObject(m map[string]interface{}, object *IObject) error {
if m["type"] == "action" {
b, err := json.Marshal(m)
if err != nil {
return err
}
action := Action{}
if err = json.Unmarshal(b, &action); err != nil {
return err
}
*object = action
return nil
}
if m["type"] == "activity" {
b, err := json.Marshal(m)
if err != nil {
return err
}
activity := Activity{}
if err = json.Unmarshal(b, &activity); err != nil {
return err
}
*object = activity
return nil
}
return errors.New("unknown actor type")
}
func (r *Response) UnmarshalJSON(data []byte) error {
raw := struct {
Foo string `json:"foo"`
Object interface{} `json:"object"`
}{}
err := json.Unmarshal(data, &raw)
if err != nil {
return err
}
r.Foo = raw.Foo
if err = UnmarshalObject(raw.Object.(map[string]interface{}), &r.Object); err != nil
{
return err
}
return nil
}
So what I do is basically
Unmarshall the object into an interface{}
Typecast to map[string]interface{}
Read the "type" value to determine the type
Create a new instance of the determined type
Marshal back to json
Unmarshal again to the new instance of the determined type
Assign the instance to the field
This feels off and I am not comfortable with it. Especially the marshaling/unmarshaling back and forth. Is there a more elegant way to solve this problem?
You can use json.RawMessage.
func (r *Response) UnmarshalJSON(data []byte) error {
var raw struct {
Foo string `json:"foo"`
Object json.RawMessage `json:"object"`
}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
r.Foo = raw.Foo
var obj struct {
Type string `json:"type"`
}
if err := json.Unmarshal(raw.Object, &obj); err != nil {
return err
}
switch obj.Type {
case "action":
r.Object = new(Action)
case "activity":
r.Object = new(Activity)
}
return json.Unmarshal(raw.Object, r.Object)
}
https://go.dev/play/p/6dqiybS4zNp

Unmarshaling JSON in golang

i'm having a lot of trouble getting my program to work. I want to unmarshal something pretty simple, but it's giving me a lot of issues, unfortunately.
Here is the response that I want to unmarshal:
{"error":[],"result":{"XXBTZUSD":[[1647365820,"39192.0","39192.0","39191.9","39191.9","39191.9","0.18008008",10],[1647365880,"39186.1","39186.1","39172.0","39176.0","39174.4","0.13120077",10]],"last":1647408900}}
I've wrote these structs to help with unmarshalling
type Resp struct {
Error []string `json:"error"`
Result Trades `json:"result"`
}
type Trades struct {
Pair []OHLC `json:"XXBTZUSD"`
Last float64 `json:"last"`
}
type OHLC struct {
Time float64
Open string
High string
Low string
Close string
Vwa string
Volume string
Count float64
}
I have a function call that makes the http request and then unmarshals the data. For whatever reason, my code will end before even starting the function call for the http request and subsequent unmarshalling when the Pair type is []OHLC or []*OHLC. If I change the Pair type to interface{}, then it runs. i want to make it work with the OHLC struct instead though. Below is the complete code:
package main
import (
"fmt"
"net/http"
//"strings"
"io/ioutil"
"encoding/json"
)
type Resp struct {
Error []string `json:"error"`
Result Trades `json:"result"`
}
type Trades struct {
Pair []OHLC `json:"XXBTZUSD"`
Last float64 `json:"last"`
}
type OHLC struct {
TT float64
Open string
High string
Low string
Close string
Vwap string
Volume string
Count float64
}
/*func main() {
var data = [...]Trade{
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
}
}*/
func main() {
fmt.Println("in main");
getOhlc()
}
func getOhlc() {
fmt.Println("in ohlc func")
resp, err := http.Get("https://api.kraken.com/0/public/OHLC?pair=XXBTZUSD");
if err != nil {
fmt.Errorf("error after request")
return;
}
defer resp.Body.Close();
body, err := ioutil.ReadAll(resp.Body);
if err != nil {
fmt.Errorf("error when reading")
return;
}
var jsonData Resp;
err = json.Unmarshal(body, &jsonData);
if err != nil {
fmt.Errorf("error when unmarshalling")
return
}
if(len(jsonData.Error) > 0) {
fmt.Errorf("error");
return;
}
fmt.Println(jsonData);
}
Any ideas about what might be happening?
"Any ideas about what might be happening?"
The elements in the "XXBTZUSD" JSON array are arrays themselves, i.e. "XXBTZUSD" is an array of arrays. The OHLC type is a struct type. The stdlib will not, by itself, unmarshal a JSON array into a Go struct. Go structs can be used to unmarshal JSON objects. JSON arrays can be unmarshaled into Go slices or arrays.
You would clearly see that that's the issue if you would just print the error from json.Unmarshal:
json: cannot unmarshal array into Go struct field
Trades.result.XXBTZUSD of type main.OHLC
https://go.dev/play/p/D4tjXZVzDI_w
If you want to unmarshal a JSON array into a Go struct you have to have the Go struct type implement a the json.Unmarshaler interface.
func (o *OHLC) UnmarshalJSON(data []byte) error {
// first unmarshal the array into a slice of raw json
raw := []json.RawMessage{}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
// create a function that unmarshals each raw json element into a field
unmarshalFields := func(raw []json.RawMessage, fields ...interface{}) error {
if len(raw) != len(fields) {
return errors.New("bad number of elements in json array")
}
for i := range raw {
if err := json.Unmarshal([]byte(raw[i]), fields[i]); err != nil {
return err
}
}
return nil
}
// call the function
return unmarshalFields(
raw,
&o.Time,
&o.Open,
&o.High,
&o.Low,
&o.Close,
&o.Vwa,
&o.Volume,
&o.Count,
)
}
https://go.dev/play/p/fkFKLkaNaSU
Your code had some issues:
Remove semicolons from end of lines, it's redundant.
fmt.Errorf return error, and not print it, every time check your error and propagate it.
We can convert array of numbers and string to struct in golang.
for achieving your desired output we need to first convert to intermediate container and then convert to our wanted output:
package main
import (
"errors"
"fmt"
"log"
"net/http"
//"strings"
"encoding/json"
"io/ioutil"
)
type Resp struct {
Error []string `json:"error"`
Result Trades `json:"result"`
}
type IntermediateResp struct {
Error []string `json:"error"`
Result IntermediateTrades `json:"result"`
}
type IntermediateTrades struct {
Pair [][]interface{} `json:"XXBTZUSD"`
Last int `json:"last"`
}
type Trades struct {
Pair []OHLC `json:"result"`
Last int `json:"last"`
}
type OHLC struct {
TT float64
Open string
High string
Low string
Close string
Vwap string
Volume string
Count float64
}
/*func main() {
var data = [...]Trade{
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
}
}*/
func main() {
fmt.Println("in main")
err := getOhlc()
if err != nil {
log.Fatal(err)
}
}
func buildOHLC(l []interface{}) (*OHLC, error) {
if len(l) < 8 {
return nil, errors.New("short list")
}
return &OHLC{
TT: l[0].(float64),
Open: l[1].(string),
High: l[2].(string),
Low: l[3].(string),
Close: l[4].(string),
Vwap: l[5].(string),
Volume: l[6].(string),
Count: l[7].(float64),
}, nil
}
func convert(r IntermediateResp) (*Resp, error) {
result := &Resp{Error: r.Error, Result: Trades{Pair: make([]OHLC, len(r.Result.Pair)), Last: r.Result.Last}}
for i, v := range r.Result.Pair {
ohlc, err := buildOHLC(v)
if err != nil {
return nil, err
}
result.Result.Pair[i] = *ohlc
}
return result, nil
}
func getOhlc() error {
fmt.Println("in ohlc func")
resp, err := http.Get("https://api.kraken.com/0/public/OHLC?pair=XXBTZUSD")
if err != nil {
return fmt.Errorf("error after request, %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
fmt.Println(string(body))
if err != nil {
return fmt.Errorf("error when reading %v", err)
}
var jsonData IntermediateResp
err = json.Unmarshal(body, &jsonData)
if err != nil {
return fmt.Errorf("error when unmarshalling %v", err)
}
if len(jsonData.Error) > 0 {
return fmt.Errorf("error")
}
convertedOhlc, err := convert(jsonData)
if err != nil {
return fmt.Errorf("error when convertedOhlc %v", err)
}
fmt.Println(convertedOhlc)
return nil
}
We define IntermediateResp and IntermediateTrades for Unmarshaling json and then convert it to actual Resp.
I think aother way is using custom Unmarshal for Trades struct.

How to unmarshal emtpy string as nil

If I have some JSON data like this:
{
nullableID: ""
}
How can I get unmarshalling this struct:
help := struct {
ID *primitive.ObjectID `json:"nullableID",omitempty`
}{}
To decode into help such that help.ID == nil
Have ObjectID implement the Unmarshaler interface and check for an empty string:
func (o *ObjectID) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(data, o); err != nil {
return err
}
if string(*o) == "" {
o = nil
}
return nil
}
If the ObjectID type is imported from another package, you can create a new type that wraps that type:
// objID is a copy of primitive.ObjectID but with it's own json unmarshalling.
type objID struct {
*primitive.ObjectID
}
func (o *objID) UnmarshalJSON(data []byte) error {
// Same implementation as above
}
This article explains it in a lot more detail.

Unmarshaling json in Go: required field?

Is it possible to generate an error if a field was not found while parsing a JSON input using Go?
I could not find it in documentation.
Is there any tag that specifies the field as required?
There is no tag in the encoding/json package that sets a field to "required". You will either have to write your own MarshalJSON() method, or do a post check for missing fields.
To check for missing fields, you will have to use pointers in order to distinguish between missing/null and zero values:
type JsonStruct struct {
String *string
Number *float64
}
Full working example:
package main
import (
"fmt"
"encoding/json"
)
type JsonStruct struct {
String *string
Number *float64
}
var rawJson = []byte(`{
"string":"We do not provide a number"
}`)
func main() {
var s *JsonStruct
err := json.Unmarshal(rawJson, &s)
if err != nil {
panic(err)
}
if s.String == nil {
panic("String is missing or null!")
}
if s.Number == nil {
panic("Number is missing or null!")
}
fmt.Printf("String: %s Number: %f\n", *s.String, *s.Number)
}
Playground
You can also override the unmarshalling for a specific type (so a required field buried in a few json layers) without having to make the field a pointer. UnmarshalJSON is defined by the Unmarshaler interface.
type EnumItem struct {
Named
Value string
}
func (item *EnumItem) UnmarshalJSON(data []byte) (err error) {
required := struct {
Value *string `json:"value"`
}{}
all := struct {
Named
Value string `json:"value"`
}{}
err = json.Unmarshal(data, &required)
if err != nil {
return
} else if required.Value == nil {
err = fmt.Errorf("Required field for EnumItem missing")
} else {
err = json.Unmarshal(data, &all)
item.Named = all.Named
item.Value = all.Value
}
return
}
Here is another way by checking your customized tag
you can create a tag for your struct like:
type Profile struct {
Name string `yourprojectname:"required"`
Age int
}
Use reflect to check if the tag is assigned required value
func (p *Profile) Unmarshal(data []byte) error {
err := json.Unmarshal(data, p)
if err != nil {
return err
}
fields := reflect.ValueOf(p).Elem()
for i := 0; i < fields.NumField(); i++ {
yourpojectTags := fields.Type().Field(i).Tag.Get("yourprojectname")
if strings.Contains(yourpojectTags, "required") && fields.Field(i).IsZero() {
return errors.New("required field is missing")
}
}
return nil
}
And test cases are like:
func main() {
profile1 := `{"Name":"foo", "Age":20}`
profile2 := `{"Name":"", "Age":21}`
var profile Profile
err := profile.Unmarshal([]byte(profile1))
if err != nil {
log.Printf("profile1 unmarshal error: %s\n", err.Error())
return
}
fmt.Printf("profile1 unmarshal: %v\n", profile)
err = profile.Unmarshal([]byte(profile2))
if err != nil {
log.Printf("profile2 unmarshal error: %s\n", err.Error())
return
}
fmt.Printf("profile2 unmarshal: %v\n", profile)
}
Result:
profile1 unmarshal: {foo 20}
2009/11/10 23:00:00 profile2 unmarshal error: required field is missing
You can go to Playground to have a look at the completed code
You can just implement the Unmarshaler interface to customize how your JSON gets unmarshalled.
you can also make use of JSON schema validation.
package main
import (
"encoding/json"
"fmt"
"github.com/alecthomas/jsonschema"
"github.com/xeipuuv/gojsonschema"
)
type Bird struct {
Species string `json:"birdType"`
Description string `json:"what it does" jsonschema:"required"`
}
func main() {
var bird Bird
sc := jsonschema.Reflect(&bird)
b, _ := json.Marshal(sc)
fmt.Println(string(b))
loader := gojsonschema.NewStringLoader(string(b))
documentLoader := gojsonschema.NewStringLoader(`{"birdType": "pigeon"}`)
schema, err := gojsonschema.NewSchema(loader)
if err != nil {
panic("nop")
}
result, err := schema.Validate(documentLoader)
if err != nil {
panic("nop")
}
if result.Valid() {
fmt.Printf("The document is valid\n")
} else {
fmt.Printf("The document is not valid. see errors :\n")
for _, err := range result.Errors() {
// Err implements the ResultError interface
fmt.Printf("- %s\n", err)
}
}
}
Outputs
{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Bird","definitions":{"Bird":{"required":["birdType","what it does"],"properties":{"birdType":{"type":"string"},"what it does":{"type":"string"}},"additionalProperties":false,"type":"object"}}}
The document is not valid. see errors :
- (root): what it does is required
code example taken from Strict JSON parsing