Golang Converting JSON - json

map[key:2073933158088]
I need to grab the key out of this data structure as a string, but I can't seem to figure out how!
Help with this overly simple question very much appreciated.
The value above is encapsulated in the variable named data.
I have tried: data.key, data[key], data["key"], data[0] and none of these seem to be appropriate calls.
To define data I sent up a JSON packet to a queue on IronMQ. I then pulled the message from the queue and manipulated it like this:
payloadIndex := 0
for index, arg := range(os.Args) {
if arg == "-payload" {
payloadIndex = index + 1
}
}
if payloadIndex >= len(os.Args) {
panic("No payload value.")
}
payload := os.Args[payloadIndex]
var data interface{}
raw, err := ioutil.ReadFile(payload)
if err != nil {
panic(err.Error())
}
err = json.Unmarshal(raw, &data)

Design your data type to match json structure. This is how can you achieve this:
package main
import (
"fmt"
"encoding/json"
)
type Data struct {
Key string `json:"key"`
}
func main() {
data := new(Data)
text := `{ "key": "2073933158088" }`
raw := []byte(text)
err := json.Unmarshal(raw, data)
if err != nil {
panic(err.Error())
}
fmt.Println(data.Key)
}

Since the number in the json is unquoted, it's not a string, Go will panic if you try to just handle it as a string (playground: http://play.golang.org/p/i-NUwchJc1).
Here's a working alternative:
package main
import (
"fmt"
"encoding/json"
"strconv"
)
type Data struct {
Key string `json:"key"`
}
func (d *Data) UnmarshalJSON(content []byte) error {
var m map[string]interface{}
err := json.Unmarshal(content, &m)
if err != nil {
return err
}
d.Key = strconv.FormatFloat(m["key"].(float64), 'f', -1, 64)
return nil
}
func main() {
data := new(Data)
text := `{"key":2073933158088}`
raw := []byte(text)
err := json.Unmarshal(raw, data)
if err != nil {
panic(err.Error())
}
fmt.Println(data.Key)
}
You can see the result in the playground: http://play.golang.org/p/5hU3hdV3kK

Related

Unmarshal nested map with int keys

If I have a map, I can Marshal it no problem:
package main
import (
"encoding/json"
"os"
)
type object map[int]interface{}
func main() {
obj := object{
1: "one", 2: object{3: "three"},
}
buf, err := json.Marshal(obj)
if err != nil {
panic(err)
}
os.Stdout.Write(buf) // {"1":"one","2":{"3":"three"}}
}
However I want to do the reverse. I tried this:
package main
import (
"encoding/json"
"fmt"
)
func main() {
buf := []byte(`{"1":"one","2":{"3":"three"}}`)
var obj map[int]interface{}
json.Unmarshal(buf, &obj)
// map[int]interface {}{1:"one", 2:map[string]interface {}{"3":"three"}}
fmt.Printf("%#v\n", obj)
}
Only the top level has the correct type. Is it possible to do what I am wanting?
JSON keys are never anything but strings, that's how the spec is defined, and you can see that in the output of you marshal. So when you try the reverse with interface{} as the top level map's value type, the type information for the nested objects is lost. You'd need a custom map type that implements UnmarshalJSON to be able to do what you want.
For example:
type IntKeyMap map[int]interface{}
func (m *IntKeyMap) UnmarshalJSON(data []byte) error {
raw := map[int]json.RawMessage{}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
for k, v := range raw {
// check if the value is a nested object
if len(v) > 0 && v[0] == '{' && v[len(v)-1] == '}' {
// The following assumes that the nested JSON object's
// key strings represent integers, if that is not the
// case this block will fail.
obj := IntKeyMap{}
if err := json.Unmarshal([]byte(v), &obj); err != nil {
return err
}
(*m)[k] = obj
} else {
var i interface{}
if err := json.Unmarshal([]byte(v), &i); err != nil {
return err
}
(*m)[k] = i
}
}
return nil
}
https://go.dev/play/p/lmyhqD__Uod

Golang Read JSON from S3 into struct in memory

I have a JSON file in S3 that takes the format of the following struct:
type StockInfo []struct {
Ticker string `json:"ticker"`
BoughtPrice string `json:"boughtPrice"`
NumberOfShares string `json:"numberOfShares"`
}
and I want to read it into a struct value from S3. In python the code would look something like this:
import boto3
import json
s3 = boto3.client('s3', 'us-east-1')
obj = s3.get_object(Bucket=os.environ["BucketName"], Key=os.environ["Key"])
fileContents = obj['Body'].read().decode('utf-8')
json_content = json.loads(fileContents)
However I'm kinda stuck on how to make this happen in Go. I've gotten this far:
package main
import (
"archive/tar"
"bytes"
"fmt"
"log"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/joho/godotenv"
)
type StockInfo []struct {
Ticker string `json:"ticker"`
BoughtPrice string `json:"boughtPrice"`
NumberOfShares string `json:"numberOfShares"`
}
func init() {
// loads values from .env into the system
if err := godotenv.Load(); err != nil {
log.Print("No .env file found")
}
return
}
func main() {
// Store the PATH environment variable in a variable
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
if err != nil {
panic(err)
}
s3Client := s3.New(sess)
bucket := "ian-test-bucket-go-python"
key := "StockInfo.json"
requestInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
result, err := s3Client.GetObject(requestInput)
if err != nil {
fmt.Println(err)
}
fmt.Println(result)
which returns to me the body/object buffer, but im not sure how to read that into a string so I can marshal it into my struct. I found this code in a similar question:
requestInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
buf := new(aws.WriteAtBuffer)
numBytes, _ := *s3manager.Downloader.Download(buf, requestInput)
tr := tar.NewReader(bytes.NewReader(buf.Bytes()))
but I get the following errors:
not enough arguments in call to method expression s3manager.Downloader.Download
have (*aws.WriteAtBuffer, *s3.GetObjectInput)
want (s3manager.Downloader, io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader))
multiple-value s3manager.Downloader.Download() in single-value context
Can anyone point me in the right direction? kinda frustrating how hard it seems to do this compared to python.
I was able to do it with the following code:
requestInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
result, err := s3Client.GetObject(requestInput)
if err != nil {
fmt.Println(err)
}
defer result.Body.Close()
body1, err := ioutil.ReadAll(result.Body)
if err != nil {
fmt.Println(err)
}
bodyString1 := fmt.Sprintf("%s", body1)
var s3data StockInfo
decoder := json.NewDecoder(strings.NewReader(bodyString1))
err = decoder.Decode(&s3data)
if err != nil {
fmt.Println("twas an error")
}
fmt.Println(s3data)
Alternative solution using json.Unmarshal
besed on aws-sdk-go-v2
...
params := &s3.GetObjectInput{
Bucket: aws.String(s3Record.S3.Bucket.Name),
Key: aws.String(s3Record.S3.Object.Key),
}
result, _ := client.GetObject(context.TODO(), params)
if err != nil {
panic(err)
}
defer result.Body.Close()
// capture all bytes from upload
b, err := ioutil.ReadAll(result.Body)
if err != nil {
panic(err)
}
var temp StockInfo
if err = json.Unmarshal(b, &temp); err != nil {
panic(err)
}
ftm.Println("res: ",b)

How to compare JSON with varying order?

I'm attempting to implement testing with golden files, however, the JSON my function generates varies in order but maintains the same values. I've implemented the comparison method used here:
How to compare two JSON requests?
But it's order dependent. And as stated here by brad:
JSON objects are unordered, just like Go maps. If
you're depending on the order that a specific implementation serializes your JSON
objects in, you have a bug.
I've written some sample code that simulated my predicament:
package main
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"reflect"
"time"
)
type example struct {
Name string
Earnings float64
}
func main() {
slice := GetSlice()
gfile, err := ioutil.ReadFile("testdata/example.golden")
if err != nil {
fmt.Println(err)
fmt.Println("Failed reading golden file")
}
testJSON, err := json.Marshal(slice)
if err != nil {
fmt.Println(err)
fmt.Println("Error marshalling slice")
}
equal, err := JSONBytesEqual(gfile, testJSON)
if err != nil {
fmt.Println(err)
fmt.Println("Error comparing JSON")
}
if !equal {
fmt.Println("Restults don't match JSON")
} else {
fmt.Println("Success!")
}
}
func GetSlice() []example {
t := []example{
example{"Penny", 50.0},
example{"Sheldon", 70.0},
example{"Raj", 20.0},
example{"Bernadette", 200.0},
example{"Amy", 250.0},
example{"Howard", 1.0}}
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(t), func(i, j int) { t[i], t[j] = t[j], t[i] })
return t
}
func JSONBytesEqual(a, b []byte) (bool, error) {
var j, j2 interface{}
if err := json.Unmarshal(a, &j); err != nil {
return false, err
}
if err := json.Unmarshal(b, &j2); err != nil {
return false, err
}
return reflect.DeepEqual(j2, j), nil
}
func WriteTestSliceToFile(arr []example, filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println("failed creating file: %s", err)
}
datawriter := bufio.NewWriter(file)
marshalledStruct, err := json.Marshal(arr)
if err != nil {
fmt.Println("Error marshalling json")
fmt.Println(err)
}
_, err = datawriter.Write(marshalledStruct)
if err != nil {
fmt.Println("Error writing to file")
fmt.Println(err)
}
datawriter.Flush()
file.Close()
}
JSON arrays are ordered. The json.Marshal function preserves order when encoding a slice to a JSON array.
JSON objects are not ordered. The json.Marshal function writes object members in sorted key order as described in the documentation.
The bradfitz comment JSON object ordering is not relevant to this question:
The application in the question is working with a JSON array, not a JSON object.
The package was updated to write object fields in sorted key order a couple of years after Brad's comment.
To compare slices while ignoring order, sort the two slices before comparing. This can be done before encoding to JSON or after decoding from JSON.
sort.Slice(slice, func(i, j int) bool {
if slice[i].Name != slice[j].Name {
return slice[i].Name < slice[j].Name
}
return slice[i].Earnings < slice[j].Earnings
})
For unit testing, you could use assert.JSONEq from Testify. If you need to do it programatically, you could follow the code of the JSONEq function.
https://github.com/stretchr/testify/blob/master/assert/assertions.go#L1551

Unmarshalling to same struct but different json name

I am trying to unmarshal a particular json data, perform some data transformations and then marshal the data and send it. However I want to marshal it with different json variable name.
Can I marshal the data to another json name like have xyz instead of abc
{"abc":1}
to
{"xyz":1}
package main
import (
"fmt"
"encoding/json"
)
type SomeStruct struct {
SomeField int `json:"abc"`
}
func main(){
jsonData := []byte(`{"abc":1}`)
strct := SomeStruct{}
json.Unmarshal([]byte(jsonData), &strct)
fmt.Println(strct)
x, err := json.Marshal(strct)
if err != nil {
fmt.Println("errs", err)
}
finalValue := string(x)
fmt.Println(finalValue)
}
Define two structures, one for your input, one for your output, and convert like this:
package main
import (
"fmt"
"encoding/json"
)
type inStruct struct {
SomeField int `json:"abc"`
}
type outStruct struct {
SomeField int `json:"xyz"`
}
func main(){
jsonData := []byte(`{"abc":1}`)
strct := inStruct{}
if err := json.Unmarshal(jsonData, &strct); err != nil {
log.Fatal(err)
}
// Edit to reflect mkopriva's comment
// outStruct := &outStruct{SomeField: strct.SomeField}
outStruct := outStruct(strct)
x, err := json.Marshal(outStruct)
if err != nil {
log.Fatal(err)
}
finalValue := string(x)
fmt.Println(finalValue)
}
You can potentially implement a custom ToJSON method for the struct, as follows:
package main
import (
"encoding/json"
"fmt"
)
type SomeStruct struct {
SomeField int `json:"abc"`
}
func (s *SomeStruct) ToJSON() string {
return fmt.Sprintf("{\"xyz\":%d}", s.SomeField)
}
func main() {
jsonData := []byte(`{"abc":1}`)
strct := SomeStruct{}
json.Unmarshal([]byte(jsonData), &strct)
fmt.Println(strct)
x, err := json.Marshal(strct)
if err != nil {
fmt.Println("errs", err)
}
finalValue := string(x)
fmt.Println(finalValue)
fmt.Println("custom ToJSON", strct.ToJSON())
}
Playground link: https://play.golang.org/p/mjW0dBPN59Q
It might not be flexible in the long run though. Personally, for requirement like this, I would prefer the solution posted by #Clément

Golang - Cannot access map in []interface{}

I have used json.Unmarshal and extracted json content. I then managed to get one layer deeper into the []interface{} by using the following code:
response, err := http.Get("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?key=2B2A0C37AC20B5DC2234E579A2ABB11C&steamids=76561198132612090")
content, err := ioutil.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
panic(0)
}
var decoded map[string]interface{}
if err := json.Unmarshal(content, &decoded); err != nil {
panic(0)
}
players := decoded["response"].(map[string]interface{})["players"]
if err != nil {
panic(0)
}
Variable players' type is []interface {} and content is [map[personaname:Acidic]].
How do I access this map? I've tried players["personaname"] but that doesn't seem to work. Any ideas?
Defining a struct type with the expected schema will make your life easier when you want to get the data from it:
package main
import "fmt"
//import "net/http"
//import "io/ioutil"
import "encoding/json"
// you don't need to define everything, only what you need
type Player struct {
Steamid string
Communityvisibilitystate int
Personaname string
Lastlogoff int64 // time.Unix(Lastlogoff, 0)
Profileurl string
Avatar string
Avatarmedium string
Avatarfull string
Personastate int
Realname string
Primaryclanid string
Timecreated int64 // time.Unix(Timecreated, 0)
Personastateflags int
//Loccountrycode string // e.g. if you don't need this
}
func main() {
/*response, err := http.Get("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?key=2B2A0C37AC20B5DC2234E579A2ABB11C&steamids=76561198132612090")
if err != nil {
panic(err)
}
content, err := ioutil.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
panic(0)
}*/
content := []byte(`{
"response": {
"players": [
{
"steamid": "76561198132612090",
"communityvisibilitystate": 3,
"profilestate": 1,
"personaname": "Acidic",
"lastlogoff": 1459489924,
"profileurl": "http://steamcommunity.com/id/ari9/",
"avatar": "https://steamcdn-a.akamaihd.net/steamcommunity/public/images/avatars/bc/bc50a4065c31c606e51dfad329341b2d1f1ac4d3.jpg",
"avatarmedium": "https://steamcdn-a.akamaihd.net/steamcommunity/public/images/avatars/bc/bc50a4065c31c606e51dfad329341b2d1f1ac4d3_medium.jpg",
"avatarfull": "https://steamcdn-a.akamaihd.net/steamcommunity/public/images/avatars/bc/bc50a4065c31c606e51dfad329341b2d1f1ac4d3_full.jpg",
"personastate": 3,
"realname": "Ari Seyhun",
"primaryclanid": "103582791440552060",
"timecreated": 1397199406,
"personastateflags": 0,
"loccountrycode": "TR"
}
]
}
}`)
var decoded struct {
Response struct {
Players []Player
}
}
if err := json.Unmarshal(content, &decoded); err != nil {
panic(err)
}
fmt.Printf("%#v\n", decoded.Response.Players)
}
http://play.golang.org/p/gVPRwLFunF
You can also create a new named type from time.Time for Timecreated and Lastlogoff with its own UnmarshalJSON function, and immediately convert it to time.Time using time.Unix()
Players is a JSON array. Thus you have to convert it to a slice of interface.
Then you can access any element of the slice and casting it to a map[string]interface{} type.
Here's the working example
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
)
func main() {
response, err := http.Get("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?key=2B2A0C37AC20B5DC2234E579A2ABB11C&steamids=76561198132612090")
content, err := ioutil.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
panic(0)
}
var decoded map[string]interface{}
if err := json.Unmarshal(content, &decoded); err != nil {
panic(0)
}
players := decoded["response"].(map[string]interface{})["players"]
if err != nil {
panic(0)
}
sliceOfPlayers := players.([]interface{})
fmt.Println((sliceOfPlayers[0].(map[string]interface{}))["personaname"])
}