Related
i'm having a lot of trouble getting my program to work. I want to unmarshal something pretty simple, but it's giving me a lot of issues, unfortunately.
Here is the response that I want to unmarshal:
{"error":[],"result":{"XXBTZUSD":[[1647365820,"39192.0","39192.0","39191.9","39191.9","39191.9","0.18008008",10],[1647365880,"39186.1","39186.1","39172.0","39176.0","39174.4","0.13120077",10]],"last":1647408900}}
I've wrote these structs to help with unmarshalling
type Resp struct {
Error []string `json:"error"`
Result Trades `json:"result"`
}
type Trades struct {
Pair []OHLC `json:"XXBTZUSD"`
Last float64 `json:"last"`
}
type OHLC struct {
Time float64
Open string
High string
Low string
Close string
Vwa string
Volume string
Count float64
}
I have a function call that makes the http request and then unmarshals the data. For whatever reason, my code will end before even starting the function call for the http request and subsequent unmarshalling when the Pair type is []OHLC or []*OHLC. If I change the Pair type to interface{}, then it runs. i want to make it work with the OHLC struct instead though. Below is the complete code:
package main
import (
"fmt"
"net/http"
//"strings"
"io/ioutil"
"encoding/json"
)
type Resp struct {
Error []string `json:"error"`
Result Trades `json:"result"`
}
type Trades struct {
Pair []OHLC `json:"XXBTZUSD"`
Last float64 `json:"last"`
}
type OHLC struct {
TT float64
Open string
High string
Low string
Close string
Vwap string
Volume string
Count float64
}
/*func main() {
var data = [...]Trade{
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
}
}*/
func main() {
fmt.Println("in main");
getOhlc()
}
func getOhlc() {
fmt.Println("in ohlc func")
resp, err := http.Get("https://api.kraken.com/0/public/OHLC?pair=XXBTZUSD");
if err != nil {
fmt.Errorf("error after request")
return;
}
defer resp.Body.Close();
body, err := ioutil.ReadAll(resp.Body);
if err != nil {
fmt.Errorf("error when reading")
return;
}
var jsonData Resp;
err = json.Unmarshal(body, &jsonData);
if err != nil {
fmt.Errorf("error when unmarshalling")
return
}
if(len(jsonData.Error) > 0) {
fmt.Errorf("error");
return;
}
fmt.Println(jsonData);
}
Any ideas about what might be happening?
"Any ideas about what might be happening?"
The elements in the "XXBTZUSD" JSON array are arrays themselves, i.e. "XXBTZUSD" is an array of arrays. The OHLC type is a struct type. The stdlib will not, by itself, unmarshal a JSON array into a Go struct. Go structs can be used to unmarshal JSON objects. JSON arrays can be unmarshaled into Go slices or arrays.
You would clearly see that that's the issue if you would just print the error from json.Unmarshal:
json: cannot unmarshal array into Go struct field
Trades.result.XXBTZUSD of type main.OHLC
https://go.dev/play/p/D4tjXZVzDI_w
If you want to unmarshal a JSON array into a Go struct you have to have the Go struct type implement a the json.Unmarshaler interface.
func (o *OHLC) UnmarshalJSON(data []byte) error {
// first unmarshal the array into a slice of raw json
raw := []json.RawMessage{}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
// create a function that unmarshals each raw json element into a field
unmarshalFields := func(raw []json.RawMessage, fields ...interface{}) error {
if len(raw) != len(fields) {
return errors.New("bad number of elements in json array")
}
for i := range raw {
if err := json.Unmarshal([]byte(raw[i]), fields[i]); err != nil {
return err
}
}
return nil
}
// call the function
return unmarshalFields(
raw,
&o.Time,
&o.Open,
&o.High,
&o.Low,
&o.Close,
&o.Vwa,
&o.Volume,
&o.Count,
)
}
https://go.dev/play/p/fkFKLkaNaSU
Your code had some issues:
Remove semicolons from end of lines, it's redundant.
fmt.Errorf return error, and not print it, every time check your error and propagate it.
We can convert array of numbers and string to struct in golang.
for achieving your desired output we need to first convert to intermediate container and then convert to our wanted output:
package main
import (
"errors"
"fmt"
"log"
"net/http"
//"strings"
"encoding/json"
"io/ioutil"
)
type Resp struct {
Error []string `json:"error"`
Result Trades `json:"result"`
}
type IntermediateResp struct {
Error []string `json:"error"`
Result IntermediateTrades `json:"result"`
}
type IntermediateTrades struct {
Pair [][]interface{} `json:"XXBTZUSD"`
Last int `json:"last"`
}
type Trades struct {
Pair []OHLC `json:"result"`
Last int `json:"last"`
}
type OHLC struct {
TT float64
Open string
High string
Low string
Close string
Vwap string
Volume string
Count float64
}
/*func main() {
var data = [...]Trade{
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
Trade{5, "op", "hi", "lo", "cl", "vw", "vo", 2},
}
}*/
func main() {
fmt.Println("in main")
err := getOhlc()
if err != nil {
log.Fatal(err)
}
}
func buildOHLC(l []interface{}) (*OHLC, error) {
if len(l) < 8 {
return nil, errors.New("short list")
}
return &OHLC{
TT: l[0].(float64),
Open: l[1].(string),
High: l[2].(string),
Low: l[3].(string),
Close: l[4].(string),
Vwap: l[5].(string),
Volume: l[6].(string),
Count: l[7].(float64),
}, nil
}
func convert(r IntermediateResp) (*Resp, error) {
result := &Resp{Error: r.Error, Result: Trades{Pair: make([]OHLC, len(r.Result.Pair)), Last: r.Result.Last}}
for i, v := range r.Result.Pair {
ohlc, err := buildOHLC(v)
if err != nil {
return nil, err
}
result.Result.Pair[i] = *ohlc
}
return result, nil
}
func getOhlc() error {
fmt.Println("in ohlc func")
resp, err := http.Get("https://api.kraken.com/0/public/OHLC?pair=XXBTZUSD")
if err != nil {
return fmt.Errorf("error after request, %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
fmt.Println(string(body))
if err != nil {
return fmt.Errorf("error when reading %v", err)
}
var jsonData IntermediateResp
err = json.Unmarshal(body, &jsonData)
if err != nil {
return fmt.Errorf("error when unmarshalling %v", err)
}
if len(jsonData.Error) > 0 {
return fmt.Errorf("error")
}
convertedOhlc, err := convert(jsonData)
if err != nil {
return fmt.Errorf("error when convertedOhlc %v", err)
}
fmt.Println(convertedOhlc)
return nil
}
We define IntermediateResp and IntermediateTrades for Unmarshaling json and then convert it to actual Resp.
I think aother way is using custom Unmarshal for Trades struct.
So I'm trying to get the number of pigeons from a JSON having this format. This JSON contains a lot of bird types and each one is defined by his color and last contact:
{
"url": "http://localhost:9001/",
"pigeons": [
{
"color": "white",
"lastContact": "2020-03-23T14:46:20.806Z"
},
{
"color": "grey",
"lastContact": "2020-03-23T14:46:20.807Z"
}
],
"parrots": [
{
"color": "green",
"lastContact": "2020-03-23T14:46:20.806Z"
}
]
}
Already made this piece of code that gets the JSON from the API, but since I don't have any experience in Go, can you guys help me to count the number of pigeons from here? I don't really care about the number of other bird types.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
)
type pigeons struct {
Number int `json:"something"`
}
func main() {
url := "http://localhost:9001"
birdsClient := http.Client{
Timeout: time.Second * 2, // Maximum of 2 secs
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatal(err)
}
res, getErr := birdsClient.Do(req)
if getErr != nil {
log.Fatal(getErr)
}
body, readErr := ioutil.ReadAll(res.Body)
if readErr != nil {
log.Fatal(readErr)
}
pigeons1 := pigeons{}
jsonErr := json.Unmarshal(body, &pigeons1)
if jsonErr != nil {
log.Fatal(jsonErr)
}
fmt.Println(pigeons1.Number)
}
In the JSON document that comes back, pigeons is an array and it looks like the length of that array is the number of pigeons. So if you unmarshal that into a struct that accepts and array for pigeons, you can get the length of it:
type pigeons struct {
Pigeons []interface{} `json:"pigeons"`
}
Above, you can unmarshal the pigeons field to an interface array because you don't care about the contents of the field. If you need to process contents, you need a separate struct and use the array of that. Then:
var p pigeons
json.Unmarshal(body, &p)
fmt.Printf("%d",len(p.Pigeons))
I am trying to unmarshal some pretty ugly json but can't figure out how. I have:
package main
import "fmt"
import "encoding/json"
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
MyData []struct {
ID string `json:"id"`
Value int `json:"value"`
}
}
func main() {
b := `[
{
"id": 1,
"key": "my_key"
},
[
{
"id": "some_id",
"value": 12
},
{
"id": "anorther_id",
"value": 13
}
]
]`
var pk []PublicKey
err := json.Unmarshal([]byte(b), &pk)
if err != nil {
fmt.Println(err)
}
fmt.Println(pk)
}
For the result I am getting:
[{1 my_key []} {0 []}]
The second slice is empty when it shouldn't be.
EDIT:
The error I get is:
json: cannot unmarshal array into Go struct field PublicKey.key of type main.PublicKey
https://play.golang.org/p/cztXOchiiS5
That is some truly hideous JSON! I have two approaches to handling the mixed array elements and I like the 2nd one better. Here's the first approach using interface and a type switch:
package main
import (
"encoding/json"
"errors"
"fmt"
)
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
}
type MyData struct {
ID string `json:"id"`
Value int `json:"value"`
}
type MixedData struct {
Key []PublicKey
MyData [][]MyData
}
func (md *MixedData) UnmarshalJSON(b []byte) error {
md.Key = []PublicKey{}
md.MyData = [][]MyData{}
var obj []interface{}
err := json.Unmarshal([]byte(b), &obj)
if err != nil {
return err
}
for _, o := range obj {
switch o.(type) {
case map[string]interface{}:
m := o.(map[string]interface{})
id, ok := m["id"].(float64)
if !ok {
return errors.New("public key id must be an int")
}
pk := PublicKey{}
pk.ID = int(id)
pk.Key, ok = m["key"].(string)
if !ok {
return errors.New("public key key must be a string")
}
md.Key = append(md.Key, pk)
case []interface{}:
a := o.([]interface{})
myData := make([]MyData, len(a))
for i, x := range a {
m, ok := x.(map[string]interface{})
if !ok {
return errors.New("data array contains unexpected object")
}
val, ok := m["value"].(float64)
if !ok {
return errors.New("data value must be an int")
}
myData[i].Value = int(val)
myData[i].ID, ok = m["id"].(string)
if !ok {
return errors.New("data id must be a string")
}
md.MyData = append(md.MyData, myData)
}
default:
// got something unexpected, handle somehow
}
}
return nil
}
func main() {
b := `[
{
"id": 1,
"key": "my_key"
},
[
{
"id": "some_id",
"value": 12
},
{
"id": "another_id",
"value": 13
}
]
]`
m := MixedData{}
err := json.Unmarshal([]byte(b), &m)
if err != nil {
fmt.Println(err)
}
fmt.Println(m)
}
https://play.golang.org/p/g8d_AsH-pYY
Hopefully there aren't any unexpected other elements, but they can be handled similarly.
Here is the second that relies more on Go's internal JSON parsing with the help of json.RawMessage. It makes the same assumptions about the contents of the array. It assumes that any objects will Unmarshal into PublicKey instances and any arrays consist of only MyData instances. I also added how to marshal back into the target JSON for symmetry:
package main
import (
"encoding/json"
"fmt"
"os"
)
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
}
type MyData struct {
ID string `json:"id"`
Value int `json:"value"`
}
type MixedData struct {
Keys []PublicKey
MyData [][]MyData
}
func (md *MixedData) UnmarshalJSON(b []byte) error {
md.Keys = []PublicKey{}
md.MyData = [][]MyData{}
obj := []json.RawMessage{}
err := json.Unmarshal([]byte(b), &obj)
if err != nil {
return err
}
for _, o := range obj {
switch o[0] {
case '{':
pk := PublicKey{}
err := json.Unmarshal(o, &pk)
if err != nil {
return err
}
md.Keys = append(md.Keys, pk)
case '[':
myData := []MyData{}
err := json.Unmarshal(o, &myData)
if err != nil {
return err
}
md.MyData = append(md.MyData, myData)
default:
// got something unexpected, handle somehow
}
}
return nil
}
func (md *MixedData) MarshalJSON() ([]byte, error) {
out := make([]interface{}, len(md.Keys)+len(md.MyData))
i := 0
for _, x := range md.Keys {
out[i] = x
i++
}
for _, x := range md.MyData {
out[i] = x
i++
}
return json.Marshal(out)
}
func main() {
b := `[
{
"id": 1,
"key": "my_key"
},
[
{
"id": "some_id",
"value": 12
},
{
"id": "another_id",
"value": 13
}
]
]`
m := MixedData{}
err := json.Unmarshal([]byte(b), &m)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Println(m)
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
if err := enc.Encode(m); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
https://play.golang.org/p/ryZzaWKNcN0
Here's an approach that combines json.RawMessage with the trick of using the default unmarshaler in a type that implements json.Unmarshaler by creating a new temporary type that aliases the target type.
The idea is that we unmarshal the incoming array into a raw message and ensure that the array length is what we expect. Then we unmarshal the individual array elements into the custom struct types using their JSON tag annotations. The end result is that we can unmarshal the PublicKey type in the usual way and the UnmarshalJSON code is not terribly difficult to follow once you understand the tricks.
For example (Go Playground):
type PublicKey struct {
ID int `json:"id"`
Key string `json:"key"`
Data []MyData
}
type MyData struct {
ID string `json:"id"`
Value int `json:"value"`
}
func (pk *PublicKey) UnmarshalJSON(bs []byte) error {
// Unmarshal into a RawMessage so we can inspect the array length.
var rawMessage []json.RawMessage
err := json.Unmarshal(bs, &rawMessage)
if err != nil {
return err
}
if len(rawMessage) != 2 {
return fmt.Errorf("expected array of length 2, got %d", len(rawMessage))
}
// Parse the first object as PublicKey using the default unmarshaler
// using a temporary type that is an alias for the target type.
type PublicKey2 PublicKey
var pk2 PublicKey2
err = json.Unmarshal(rawMessage[0], &pk2)
if err != nil {
return err
}
// Parse the second object as []MyData in the usual way.
err = json.Unmarshal(rawMessage[1], &pk2.Data)
if err != nil {
return err
}
// Finally, assign the aliased object to the target object.
*pk = PublicKey(pk2)
return nil
}
func main() {
var pk PublicKey
err := json.Unmarshal([]byte(jsonstr), &pk)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n", pk)
// main.PublicKey{ID:1, Key:"my_key", Data:[]main.MyData{main.MyData{ID:"some_id", Value:12}, main.MyData{ID:"anorther_id", Value:13}}}
}
I have different structures that share a field and I need to decode a JSON file into its corresponding structure in Go.
Example:
type Dog struct {
AnimalType string //will always be "dog"
BarkLoudnessLevel int
}
type Cat struct {
AnimalType string //will always be "cat"
SleepsAtNight bool
}
If I am receiving one of these structures as a JSON string, what would be the most elegant way of parsing it into its proper structure?
So, there are a couple ways of doing this, but the easiest is probably deserializing the payload twice and having conditional branches based off of the "AnimalType" attribute in your payload. Here's a simple example using an intermediate deserialization model:
package main
import (
"fmt"
"encoding/json"
)
type Dog struct {
AnimalType string //will always be "dog"
BarkLoudnessLevel int
}
type Cat struct {
AnimalType string //will always be "cat"
SleepsAtNight bool
}
var (
payloadOne = `{"AnimalType":"dog","BarkLoudnessLevel":1}`
payloadTwo = `{"AnimalType":"cat","SleepsAtNight":false}`
)
func main() {
parseAnimal(payloadOne)
parseAnimal(payloadTwo)
}
func parseAnimal(payload string) {
animal := struct{
AnimalType string
}{}
if err := json.Unmarshal([]byte(payload), &animal); err != nil {
panic(err)
}
switch animal.AnimalType {
case "dog":
dog := Dog{}
if err := json.Unmarshal([]byte(payload), &dog); err != nil {
panic(err)
}
fmt.Printf("Got a dog: %v\n", dog)
case "cat":
cat := Cat{}
if err := json.Unmarshal([]byte(payload), &cat); err != nil {
panic(err)
}
fmt.Printf("Got a cat: %v\n", cat)
default:
fmt.Println("Unknown animal")
}
}
See it in action here.
IMO a better way of approaching this is moving the "metadata" for the payload into a parent structure, though this requires modifying the expected json payload. So, for example, if you were working with payloads that looked like:
{"AnimalType":"dog", "Animal":{"BarkLoudnessLevel": 1}}
Then you could use something like json.RawMessage to partially parse the structure and then conditionally parse the rest as needed (rather than parsing everything twice)--also results in a nicer separation of structure attributes. Here's an example of how you'd do that:
package main
import (
"encoding/json"
"fmt"
)
type Animal struct {
AnimalType string
Animal json.RawMessage
}
type Dog struct {
BarkLoudnessLevel int
}
type Cat struct {
SleepsAtNight bool
}
var (
payloadOne = `{"AnimalType":"dog", "Animal":{"BarkLoudnessLevel": 1}}`
payloadTwo = `{"AnimalType":"cat", "Animal":{"SleepsAtNight": false}}`
)
func main() {
parseAnimal(payloadOne)
parseAnimal(payloadTwo)
}
func parseAnimal(payload string) {
animal := &Animal{}
if err := json.Unmarshal([]byte(payload), &animal); err != nil {
panic(err)
}
switch animal.AnimalType {
case "dog":
dog := Dog{}
if err := json.Unmarshal(animal.Animal, &dog); err != nil {
panic(err)
}
fmt.Printf("Got a dog: %v\n", dog)
case "cat":
cat := Cat{}
if err := json.Unmarshal(animal.Animal, &cat); err != nil {
panic(err)
}
fmt.Printf("Got a cat: %v\n", cat)
default:
fmt.Println("Unknown animal")
}
}
And in action here.
I am trying to get the value of say "ip" from my following curl output:
{
"type":"example",
"data":{
"name":"abc",
"labels":{
"key":"value"
}
},
"subsets":[
{
"addresses":[
{
"ip":"192.168.103.178"
}
],
"ports":[
{
"port":80
}
]
}
]
}
I have found many examples in the internet to parse json output of curl requests and I have written the following code, but that doesn't seem to return me the value of say "ip"
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
)
type svc struct {
Ip string `json:"ip"`
}
func main() {
url := "http://myurl.com"
testClient := http.Client{
Timeout: time.Second * 2, // Maximum of 2 secs
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatal(err)
}
res, getErr := testClient.Do(req)
if getErr != nil {
log.Fatal(getErr)
}
body, readErr := ioutil.ReadAll(res.Body)
if readErr != nil {
log.Fatal(readErr)
}
svc1 := svc{}
jsonErr := json.Unmarshal(body, &svc1)
if jsonErr != nil {
log.Fatal(jsonErr)
}
fmt.Println(svc1.Ip)
}
I would appreciate if anyone could provide me hints on what I need to add to my code to get the value of say "ip".
You can create structs which reflect your json structure and then decode your json.
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
)
type Example struct {
Type string `json:"type,omitempty"`
Subsets []Subset `json:"subsets,omitempty"`
}
type Subset struct {
Addresses []Address `json:"addresses,omitempty"`
}
type Address struct {
IP string `json:"IP,omitempty"`
}
func main() {
m := []byte(`{"type":"example","data": {"name": "abc","labels": {"key": "value"}},"subsets": [{"addresses": [{"ip": "192.168.103.178"}],"ports": [{"port": 80}]}]}`)
r := bytes.NewReader(m)
decoder := json.NewDecoder(r)
val := &Example{}
err := decoder.Decode(val)
if err != nil {
log.Fatal(err)
}
// If you want to read a response body
// decoder := json.NewDecoder(res.Body)
// err := decoder.Decode(val)
// Subsets is a slice so you must loop over it
for _, s := range val.Subsets {
// within Subsets, address is also a slice
// then you can access each IP from type Address
for _, a := range s.Addresses {
fmt.Println(a.IP)
}
}
}
The output would be:
192.168.103.178
By decoding this to a struct, you can loop over any slice and not limit yourself to one IP
Example here:
https://play.golang.org/p/sWA9qBWljA
One approach is to unmarshal the JSON to a map, e.g. (assumes jsData contains JSON string)
obj := map[string]interface{}{}
if err := json.Unmarshal([]byte(jsData), &obj); err != nil {
log.Fatal(err)
}
Next, implement a function for searching the value associated with a key from the map recursively, e.g.
func find(obj interface{}, key string) (interface{}, bool) {
//if the argument is not a map, ignore it
mobj, ok := obj.(map[string]interface{})
if !ok {
return nil, false
}
for k, v := range mobj {
//key match, return value
if k == key {
return v, true
}
//if the value is a map, search recursively
if m, ok := v.(map[string]interface{}); ok {
if res, ok := find(m, key); ok {
return res, true
}
}
//if the value is an array, search recursively
//from each element
if va, ok := v.([]interface{}); ok {
for _, a := range va {
if res, ok := find(a, key); ok {
return res,true
}
}
}
}
//element not found
return nil,false
}
Note, that the above function return an interface{}. You need to convert it to appropriate type, e.g. using type switch:
if ip, ok := find(obj, "ip"); ok {
switch v := ip.(type) {
case string:
fmt.Printf("IP is a string -> %s\n", v)
case fmt.Stringer:
fmt.Printf("IP implements stringer interface -> %s\n", v.String())
case int:
default:
fmt.Printf("IP = %v, ok = %v\n", ip, ok)
}
}
A working example can be found at https://play.golang.org/p/O5NUi4J0iR
Typically in these situations you will see people describe all of these sub struct types. If you don't actually need to reuse the definition of any sub structs (like as a type for a function argument), then you don't need to define them. You can just use one definition for the whole response. In addition, in some cases you don't need to define a type at all, you can just do it at the time of declaration:
package main
import "encoding/json"
const s = `
{
"subsets": [
{
"addresses": [
{"ip": "192.168.103.178"}
]
}
]
}
`
func main() {
var svc struct {
Subsets []struct {
Addresses []struct { Ip string }
}
}
json.Unmarshal([]byte(s), &svc)
ip := svc.Subsets[0].Addresses[0].Ip
println(ip == "192.168.103.178")
}
You can write your own decoder or use existing third-party decoders.
For instance, github.com/buger/jsonparser could solve your problem by iterating throw array (two times).
package main
import (
"github.com/buger/jsonparser"
"fmt"
)
var data =[]byte(`{
"type":"example",
"data":{
"name":"abc",
"labels":{
"key":"value"
}
},
"subsets":[
{
"addresses":[
{
"ip":"192.168.103.178"
}
],
"ports":[
{
"port":80
}
]
}
]
}`)
func main() {
jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
v, _, _, err := jsonparser.Get(value, "ip")
if err != nil {
return
}
fmt.Println("ip: ", string(v[:]))
}, "addresses")
}, "subsets")
}
Output: ip: 192.168.103.178