Parse nested json array from a file - json

I have a json file sample.json containing a json array as follows:
[
{
"time": "2021-01-04T00:11:32.362Z",
"extra_data": {
"id": "123"
},
"info": "event123"
},
{
"time": "2021-01-05T00:11:32.362Z",
"extra_data": {
"id": "456"
},
"info": "event456"
},
{
"time": "2021-01-06T00:11:32.362Z",
"extra_data": {
"id": "789"
},
"info": "event789"
}
]
I am trying to unmarshal this json array so that for each json object (i.e. event), I can access the id and info values. This is what I have so far:
func main() {
file, err := ioutil.ReadFile("/Users/janedoe/Downloads/sample.json")
var events Event
json.Unmarshal([]byte(file), &events)
fmt.Println(reflect.TypeOf(events))
// Reading each value by its key for each event
fmt.Println("Event123_Time :", events.Timestamp,
"\nEvent123_ExtraData_Id :", events.ExtraData.Id,
"\nEvent123_Info :", events.Info)
}
type Event struct {
Time string `json:"time"`
ExtraData ExtraData `json:"extra_data"`
Info string `json:"info"`
}
type ExtraData struct {
Id string `json:"id"`
}
The output that I am getting is:
main.Event
Event123_Time :
Event123_ExtraData_Id :
Event123_Info :
I am not getting any values, implying that the marshalling is not happening as expected. How can I fix this?

Since your json is an array, I think you want:
var events []Event
not
var events Event
Here's a version with that change:
func main() {
file, err := ioutil.ReadFile("/Users/janedoe/Downloads/sample.json")
if err != nil {
panic(err)
}
var events []Event
if err := json.Unmarshal([]byte(file), &events); err != nil {
panic(err)
}
fmt.Println(reflect.TypeOf(events))
for i, event := range events {
// Reading each value by its key for each event
fmt.Println(i, "Event123_Time :", event.Time,
"\nEvent123_ExtraData_Id :", event.ExtraData.Id,
"\nEvent123_Info :", event.Info)
}
}
type Event struct {
Time string `json:"time"`
ExtraData ExtraData `json:"extra_data"`
Info string `json:"info"`
}
type ExtraData struct {
Id string `json:"id"`
}

Related

Trouble mapping json to golang struct

I have a json stream as follows ...
[
{
"page": 1,
"pages": 7,
"per_page": "2000",
"total": 13200
},
[
{
"indicator": {
"id": "SP.POP.TOTL",
"value": "Population, total"
},
"country": {
"id": "1A",
"value": "Arab World"
},
"value": null,
"decimal": "0",
"date": "2019"
},
{
"indicator": {
"id": "SP.POP.TOTL",
"value": "Population, total"
},
"country": {
"id": "1A",
"value": "Arab World"
},
"value": "419790588",
"decimal": "0",
"date": "2018"
},
...
]
]
And I'm trying to decode it ... so I have the following struct ... but I keep getting
"cannot unmarshal array into Go value of type struct { P struct ... "
type Message []struct {
P struct {
Page int
}
V []struct {
Indicator struct {
Id string
Value string
}
Country struct {
Value string
}
Value string
Decimal string
Date string
}
}
My struct looks to match the json ... but obviously not! Any ideas?
Since your JSON array have two different types first unmarshal them into a slice of json.RawMessage which is []byte as underlying type so that we can unmarshal again JSON array data.
So unmarshal data for P and V struct type using index directly (predict) or detect if object(starting with '{') then unmarshal into P and array(starting with '[') then unmarshal into V. Now prepare your Message using those data.
type Message struct {
PageData P
ValData []V
}
type P struct {
Page int
}
type V struct {
Indicator struct {
Id string
Value string
}
Country struct {
Value string
}
Value string
Decimal string
Date string
}
func main() {
var rawdata []json.RawMessage
json.Unmarshal([]byte(jsonData), &rawdata)
var pageData P
json.Unmarshal(rawdata[0], &pageData)
var valData []V
json.Unmarshal(rawdata[1], &valData)
res := Message{pageData, valData}
fmt.Println(res)
}
var jsonData = `[...]` //your json data
Full code in Go Playground
As poWar said, the JSON you actually have is a list of objects whose types do not conform to each other. You must therefore unmarshal into something capable of holding different object types, such as interface{} or—since there is an outer array—[]interface{}.
You can also, if you like, decode into a []json.RawMessage. The underlying json.RawMessage itself has underlying type []byte so that it's basically the undecoded "inner" JSON. In at least some cases this is going to be more work than just decoding directly to []interface{} and checking each resulting interface, but you can, if you wish, decode to struct once you have the JSON separated out. For instance:
func main() {
var x []json.RawMessage
err := json.Unmarshal(input, &x)
if err != nil {
fmt.Printf("err = %v\n", err)
return
}
if len(x) != 2 {
fmt.Println("unexpected input")
return
}
var page struct {
Page int
}
err = json.Unmarshal(x[0], &page)
if err != nil {
fmt.Printf("unable to unmarshal page part: %v\n", err)
return
}
fmt.Printf("page = %d\n", page.Page)
// ...
}
Here on the Go Playground is a more complete example. See also Eklavya's answer.
Looking at your struct, your corresponding JSON should look something like this.
[
{
"P": {"page": 1},
"V": [
{
"Indicator": {"Id": ...},
"Country": {"Value":""},
"Value": "",
...
}
]
},
...
]
The JSON structure you are trying to Unmarshal looks like a list of objects where each object is not of the same type. You can start unmarshalling them into interfaces and defining each interface based on the object being unmarhsalled.
package main
import (
"encoding/json"
"log"
)
type Message []interface{}
func main() {
data := `[{"page":1,"pages":7,"per_page":"2000","total":13200},[{"indicator":{"id":"SP.POP.TOTL","value":"Population, total"},"country":{"id":"1A","value":"Arab World"},"value":null,"decimal":"0","date":"2019"},{"indicator":{"id":"SP.POP.TOTL","value":"Population, total"},"country":{"id":"1A","value":"Arab World"},"value":"419790588","decimal":"0","date":"2018"}]]`
var m Message
if err := json.Unmarshal([]byte(data), &m); err != nil {
log.Fatalf("could not unmarshal")
}
log.Printf("message: %v", m)
}
Output:
message: [map[page:1 pages:7 per_page:2000 total:13200] [map[country:map[id:1A value:Arab World] date:2019 decimal:0 indicator:map[id:SP.POP.TOTL value:Population, total] value:<nil>] map[country:map[id:1A value:Arab World] date:2018 decimal:0 indicator:map[id:SP.POP.TOTL value:Population, total] value:419790588]]]
[Edit]: Ideally you should change your JSON to be structured better for unmarshalling. If you do not have control on it, then your corresponding Go structure is just embedded maps of string to interfaces, which you will have to manually type cast and access.

Issue parsing JSON file with Go

I make a GET request, and receive a JSON file, that I cannot parse.
Here is the data I have to parse
{
"codeConv": "ACC00000321",
"start": "2019-07-01T00:00:00Z",
"end": "2019-08-21T00:00:00Z",
"details": [
{
"idPrm": "30000000123456",
"keys": [
{
"timestamp": "2019-07-01T00:00:00Z",
"value": 0
},
{
"timestamp": "2019-07-01T00:30:00Z",
"value": 0
},
...
]
},
{
"idPrm": "30000000123457",
"keys": [
{
"timestamp": "2019-07-01T00:00:00Z",
"value": 1
},
{
"timestamp": "2019-07-01T00:30:00Z",
"value": 2
},
...
]
}
]
}
Here are my objects:
type APIMain struct {
CodeConv string `json:"codeConv"`
Start string `json:"start"`
End []Keys `json:"end"`
Details []APIData `json:"details"`
}
//APIData match the data we receive from api
type APIData struct {
Prm string `json:"idPrm"`
Keys []Keys `json:"keys"`
}
type Keys struct {
Timestamp string `json:"timestamp"`
Value string `json:"value"`
}
and here is the method to get data with basic auth:
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
if login != "" && password != "" {
req.SetBasicAuth(login, password)
}
response, err := client.Do(req)
//defer response.Body.Close()
if err != nil {
return nil, err
}
if response.StatusCode != 200 {
fmt.Println(response.Body)
panic(response.Status)
}
err = json.NewDecoder(response.Body).Decode(&result)
fmt.Println("result", result) // result is empty array
How can I see if the problem is in a request, or if the problem is in parsing ?
I have get a response.Body object, but it needs to be decoded.
I fixed it using: https://mholt.github.io/json-to-go/
which generated this structure:
type AutoGenerated struct {
CodeConv string `json:"codeConv"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
Details []struct {
IDPrm string `json:"idPrm"`
Keys []struct {
Timestamp time.Time `json:"timestamp"`
Value float64 `json:"value"`
} `json:"keys"`
} `json:"details"`
}
Thanks for your comments
Great time saver !

Unmarshalling complex json in Go

So I am trying to fetch the analytics of an app by pinging and endpoint. I make the GET request which is successfull (no errors there) but I am unable to decode the JSON
I need to to decode the following json into structs
{
"noResultSearches": {
"results": [
{
"count": 1,
"key": "\"note 9\""
},
{
"count": 1,
"key": "nokia"
}
]
},
"popularSearches": {
"results": [
{
"count": 4,
"key": "6"
},
{
"count": 2,
"key": "\"note 9\""
},
{
"count": 1,
"key": "nokia"
}
]
},
"searchVolume": {
"results": [
{
"count": 7,
"key": 1537401600000,
"key_as_string": "2018/09/20 00:00:00"
}
]
}
}
For which I am using the following structs
type analyticsResults struct {
Count int `json:"count"`
Key string `json:"key"`
}
type analyticsVolumeResults struct {
Count int `json:"count"`
Key int64 `json:"key"`
DateAsStr string `json:"key_as_string"`
}
type analyticsPopularSearches struct {
Results []analyticsResults `json:"results"`
}
type analyticsNoResultSearches struct {
Results []analyticsResults `json:"results"`
}
type analyticsSearchVolume struct {
Results []analyticsVolumeResults `json:"results"`
}
type overviewAnalyticsBody struct {
NoResultSearches analyticsNoResultSearches `json:"noResultSearches"`
PopularSearches analyticsPopularSearches `json:"popularSearches"`
SearchVolume analyticsSearchVolume `json:"searchVolume"`
}
I make a GET request to an endpoint and then use the response body to decode the json but I get an error. Following is a part of the code that stays in my ShowAnalytics function
func ShowAppAnalytics(app string) error {
spinner.StartText("Fetching app analytics")
defer spinner.Stop()
fmt.Println()
req, err := http.NewRequest("GET", "<some-endpoint>", nil)
if err != nil {
return err
}
resp, err := session.SendRequest(req)
if err != nil {
return err
}
spinner.Stop()
var res overviewAnalyticsBody
dec := json.NewDecoder(resp.Body)
err = dec.Decode(&res)
if err != nil {
return err
}
fmt.Println(res)
return nil
}
json: cannot unmarshal array into Go struct field
overviewAnalyticsBody.noResultSearches of type
app.analyticsNoResultSearches
What am I doing wrong here? Why do I get this error?
EDIT: After you edited, your current code works as-is. Check it out here: Go Playground.
Original answer follows.
There is some inconsistency between the code you posted and the error you get.
I tried it on the Go Playground (here's your version), and I get the following error:
json: cannot unmarshal number into Go struct field analyticsVolumeResults.key of type string
We get this error because in the JSON searchVolume.results.key is a number:
"key": 1537401600000,
And you used string in the Go model:
Key string `json:"key"`
If we change it to int64:
Key int64 `json:"key"`
It works, and prints (try it on the Go Playground):
{{[{1 "note 9"} {1 nokia}]} {[{4 6} {2 "note 9"} {1 nokia}]} {[{7 1537401600000 2018/09/20 00:00:00}]}}
If that key may sometimes be a number and sometimes a string, you may also use json.Number in the Go model:
Key json.Number `json:"key"`

Golang json unmarshal according to key value pair

I have json as following
"data": [
{
"id": "recent_search",
"items": [],
"name": ""
},
{
"id": "popular_search",
"items": [],
"name": ""
},
{
"id": "digital",
"items": [],
"name": "DIGITAL"
}
]
and the structs are as follows:
type universeTypeData struct {
Recent universeSearchInfo
Popular universeSearchInfo
Digital universeSearchInfo
}
type universeSearchInfo struct {
ID string `json:"id"`
Name string `json:"name"`
Items []universeSearchItem `json:"items"`
}
I want to unmarshal my json as "id" with value "recent_search" map to Recent, "id" with value "popular_search" map to Popular. Is there any way of doing this in golang?
My approach of doing it is
for _, v := range result.Data {
if v.ID == "in_category" {
finalResult.Universe.InCategory.ID = v.ID
finalResult.Universe.InCategory.Name = v.Name
for _, abc := range v.Items {
finalResult.Universe.InCategory.Items = append(finalResult.Universe.InCategory.Items, abc)
}
}
if v.ID == "recent_search" {
finalResult.Universe.Recent.ID = v.ID
finalResult.Universe.Recent.Name = v.Name
for _, abc := range v.Items {
finalResult.Universe.Recent.Items = append(finalResult.Universe.Recent.Items, abc)
}
}
if v.ID == "popular_search" {
finalResult.Universe.Popular.ID = v.ID
finalResult.Universe.Popular.Name = v.Name
for _, abc := range v.Items {
finalResult.Universe.Popular.Items = append(finalResult.Universe.Popular.Items, abc)
}
}
Is there any better way of doing it?
You want to unmurshal JSON array into Go struct which is not natural mapping. Any way, you most likely should be first unmurshal in slice and then parse this slice. Some workaround is to use json.Decoder
dec := json.NewDecoder(JSONdataReader)
var res universeTypeData
// read open bracket
dec.Token()
// while the array contains values
for dec.More() {
var m universeSearchInfo
// decode an array value
dec.Decode(&m)
switch m.ID {
case "recent_search":
res.Recent = m
case "popular_search":
res.Popular = m
case "digital":
res.Digital = m
}
}
// read closing bracket
dec.Token()
which allow you to decode on the fly, in one pass, without consuming intermediate slice representation. Working example
Implement Unmarshaler interface:
Unmarshaler is the interface implemented by types that can unmarshal a
JSON description of themselves. The input can be assumed to be a valid
encoding of a JSON value. UnmarshalJSON must copy the JSON data if it
wishes to retain the data after returning.
json unmarshaler interface assign the value from json to struct after parsing the result and applying conditions to fetch the value.
package main
import (
"encoding/json"
"fmt"
)
type Details struct {
Data []universeSearchInfo `json:"data"`
}
type universeTypeData struct {
Recent universeSearchInfo
Popular universeSearchInfo
Digital universeSearchInfo
}
type universeSearchInfo struct {
ID string `json:"id"`
Name string `json:"name"`
Items []string `json:"items"`
}
func main() {
var result universeTypeData
jsonBytes := []byte(`{"data": [
{
"id": "recent_search",
"items": [],
"name": ""
},
{
"id": "popular_search",
"items": [],
"name": ""
},
{
"id": "digital",
"items": [],
"name": "DIGITAL"
}
]}`)
if err := json.Unmarshal(jsonBytes, &result); err != nil {
fmt.Println(err)
}
fmt.Println(result)
}
func (universeData *universeTypeData) UnmarshalJSON(data []byte) error {
var result Details
if err := json.Unmarshal(data, &result); err != nil {
return err
}
for _,value := range result.Data{
switch value.ID {
case "recent_search":
universeData.Recent = value
}
}
return nil
}
Working code on Go Playground

Unmarshalling json arrays as json objects

I have to unmarshal a series of Json objects, but one of the objects contain a json array which is not really structured in a good way.
"labels": [
{
"key": "owner",
"value": "harry"
},
{
"key": "group",
"value": "student"
}
]
I am unmarshalling it using this struct -
type StudentDetails struct {
Id string `json:"id"`
Name string `json:"name"`
Labels []Label `json:"labels,omitempty"`
}
type Label struct {
Key string `json:"key"`
Value string `json:"value"`
}
And I have to access it using x.Labels[0].key == "owner" inside a for loop which is very annoying.
I want to be able to do x.Labels.Owner == "harry" instead. How do I go about achieving this? The rest of JSON is unmarshalled fine using the default unmarshal function, so I don't think writing custom function will be good option.
With the constraints you have here, this is about as close as you will get (run in playground):
package main
import (
"encoding/json"
"fmt"
)
func main() {
j := `
{
"id": "42",
"name": "Marvin",
"labels": [
{
"key": "owner",
"value": "harry"
},
{
"key": "group",
"value": "student"
}
]
}`
d := StudentDetails{}
err := json.Unmarshal([]byte(j), &d)
if err != nil {
panic(err)
}
fmt.Println(d.Labels["owner"])
fmt.Println(d.Labels["group"])
}
type StudentDetails struct {
Id string `json:"id"`
Name string `json:"name"`
Labels Labels `json:"labels"`
}
type Labels map[string]string
func (l *Labels) UnmarshalJSON(b []byte) error {
a := []map[string]string{}
err := json.Unmarshal(b, &a)
if err != nil {
return err
}
t := map[string]string{}
for _, m := range a {
t[m["key"]] = m["value"]
}
*l = t
return nil
}
How about to define custom []Label type and add function on it.
For instance
type Labels []Label
func (l Labels) Owner() string {
if len(l) > 1 {
return l[0].Value
}
return ""
}