preserve int64 values when parsing json in Go - json

I'm processing a json POST in Go that contains an array of objects containing 64bit integers. When using json.Unmarshal these values seem to be converted to a float64 which isn't very helpful.
body := []byte(`{"tags":[{"id":4418489049307132905},{"id":4418489049307132906}]}`)
var dat map[string]interface{}
if err := json.Unmarshal(body, &dat); err != nil {
panic(err)
}
tags := dat["tags"].([]interface{})
for i, tag := range tags {
fmt.Println("tag: ", i, " id: ", tag.(map[string]interface{})["id"].(int64))
}
Is there any way to preserve the original int64 in the output of json.Unmarshal?
Go Playground of above code

Solution 1
You can use a Decoder and UseNumber to decode your numbers without loss :
The Number type is defined like this :
// A Number represents a JSON number literal.
type Number string
which means you can easily convert it :
package main
import (
"encoding/json"
"fmt"
"bytes"
"strconv"
)
func main() {
body := []byte("{\"tags\":[{\"id\":4418489049307132905},{\"id\":4418489049307132906}]}")
dat := make(map[string]interface{})
d := json.NewDecoder(bytes.NewBuffer(body))
d.UseNumber()
if err := d.Decode(&dat); err != nil {
panic(err)
}
tags := dat["tags"].([]interface{})
n := tags[0].(map[string]interface{})["id"].(json.Number)
i64, _ := strconv.ParseUint(string(n), 10, 64)
fmt.Println(i64) // prints 4418489049307132905
}
Solution 2
You can also decode into a specific structure tailored to your needs :
package main
import (
"encoding/json"
"fmt"
)
type A struct {
Tags []map[string]uint64 // "tags"
}
func main() {
body := []byte("{\"tags\":[{\"id\":4418489049307132905},{\"id\":4418489049307132906}]}")
var a A
if err := json.Unmarshal(body, &a); err != nil {
panic(err)
}
fmt.Println(a.Tags[0]["id"]) // logs 4418489049307132905
}
Personally I generally prefer this solution which feels more structured and easier to maintain.
Caution
A small note if you use JSON because your application is partly in JavaScript : JavaScript has no 64 bits integers but only one number type, which is the IEEE754 double precision float. So you wouldn't be able to parse this JSON in JavaScript without loss using the standard parsing function.

easier one:
body := []byte(`{"tags":[{"id":4418489049307132905},{"id":4418489049307132906}]}`)
var dat map[string]interface{}
if err := json.Unmarshal(body, &dat); err != nil {
panic(err)
}
tags := dat["tags"].([]interface{})
for i, tag := range tags {
fmt.Printf("tag: %v, id: %.0f", i, tag.(map[string]interface{})["id"].(float64))
}

I realize this is very old, but this is the solution I ended up using
/*
skipping previous code, this is just converting the float
to an int, if the value is the same with or without what's
after the decimal points
*/
f := tag.(map[string]interface{})["id"].(float64)
if math.Floor(f) == f {
fmt.Println("int tag: ", i, " id: ", int64(f))
} else {
fmt.Println("tag: ", i, " id: ", f)
}

Related

Unmarshaling from JSON key containing a single quote

I feel quite puzzled by this.
I need to load some data (coming from a French database) that is serialized in JSON and in which some keys have a single quote.
Here is a simplified version:
package main
import (
"encoding/json"
"fmt"
)
type Product struct {
Name string `json:"nom"`
Cost int64 `json:"prix d'achat"`
}
func main() {
var p Product
err := json.Unmarshal([]byte(`{"nom":"savon", "prix d'achat": 170}`), &p)
fmt.Printf("product cost: %d\nerror: %s\n", p.Cost, err)
}
// product cost: 0
// error: %!s(<nil>)
Unmarshaling leads to no errors however the "prix d'achat" (p.Cost) is not correctly parsed.
When I unmarshal into a map[string]any, the "prix d'achat" key is parsed as I would expect:
package main
import (
"encoding/json"
"fmt"
)
func main() {
blob := map[string]any{}
err := json.Unmarshal([]byte(`{"nom":"savon", "prix d'achat": 170}`), &blob)
fmt.Printf("blob: %f\nerror: %s\n", blob["prix d'achat"], err)
}
// blob: 170.000000
// error: %!s(<nil>)
I checked the json.Marshal documentation on struct tags and I cannot find any issue with the data I'm trying to process.
Am I missing something obvious here?
How can I parse a JSON key containing a single quote using struct tags?
Thanks a lot for any insight!
I didn't find anything in the documentation, but the JSON encoder considers single quote to be a reserved character in tag names.
func isValidTag(s string) bool {
if s == "" {
return false
}
for _, c := range s {
switch {
case strings.ContainsRune("!#$%&()*+-./:;<=>?#[]^_{|}~ ", c):
// Backslash and quote chars are reserved, but
// otherwise any punctuation chars are allowed
// in a tag name.
case !unicode.IsLetter(c) && !unicode.IsDigit(c):
return false
}
}
return true
}
I think opening an issue is justified here. In the meantime, you're going to have to implement json.Unmarshaler and/or json.Marshaler. Here is a start:
func (p *Product) UnmarshalJSON(b []byte) error {
type product Product // revent recursion
var _p product
if err := json.Unmarshal(b, &_p); err != nil {
return err
}
*p = Product(_p)
return unmarshalFieldsWithSingleQuotes(p, b)
}
func unmarshalFieldsWithSingleQuotes(dest interface{}, b []byte) error {
// Look through the JSON tags. If there is one containing single quotes,
// unmarshal b again, into a map this time. Then unmarshal the value
// at the map key corresponding to the tag, if any.
var m map[string]json.RawMessage
t := reflect.TypeOf(dest).Elem()
v := reflect.ValueOf(dest).Elem()
for i := 0; i < t.NumField(); i++ {
tag := t.Field(i).Tag.Get("json")
if !strings.Contains(tag, "'") {
continue
}
if m == nil {
if err := json.Unmarshal(b, &m); err != nil {
return err
}
}
if j, ok := m[tag]; ok {
if err := json.Unmarshal(j, v.Field(i).Addr().Interface()); err != nil {
return err
}
}
}
return nil
}
Try it on the playground: https://go.dev/play/p/aupACXorjOO

Read extern JSON file

I am trying to read the following JSON file:
{
"a":1,
"b":2,
"c":3
}
I have tried this but I found that I had to write each field of the JSON file into a struct but I really don't want to have all my JSON file in my Go code.
import (
"fmt"
"encoding/json"
"io/ioutil"
)
type Data struct {
A string `json:"a"`
B string `json:"b"`
C string `json:"c"`
}
func main() {
file, _ := ioutil.ReadFile("/path/to/file.json")
data := Data{}
if err := json.Unmarshal(file ,&data); err != nil {
panic(err)
}
for _, letter := range data.Letter {
fmt.Println(letter)
}
}
Is there a way to bypass this thing with something like json.load(file) in Python?
If you only want to support integer values, you could unmarshal your data into a map[string]int. Note that the order of a map is not defined, so the below program's output is non-deterministic for the input.
package main
import (
"fmt"
"encoding/json"
"io/ioutil"
)
func main() {
file, _ := ioutil.ReadFile("/path/to/file.json")
var data map[string]int
if err := json.Unmarshal(file ,&data); err != nil {
panic(err)
}
for letter := range data {
fmt.Println(letter)
}
}
You can unmarshal any JSON data in this way:
var data interface{}
if err := json.Unmarshal(..., &data); err != nil {
// handle error
}
Though, in this way you should handle all the reflection-related stuffs
since you don't know what type the root data is, and its fields.
Even worse, your data might not be map at all.
It can be any valid JSON data type like array, string, integer, etc.
Here's a playground link: https://play.golang.org/p/DiceOv4sATO
It's impossible to do anything as simple as in Python, because Go is strictly typed, so it's necessary to pass your target into the unmarshal function.
What you've written could otherwise be shortened, slightly, to something like this:
func UnmarshalJSONFile(path string, i interface{}) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return json.NewDecoder(f).Decode(i)
}
But then to use it, you would do this:
func main() {
data := Data{}
if err := UnmarshalJSONFile("/path/to/file.json", &data); err != nil {
panic(err)
}
}
But you can see that the UnmarshalJSONFile is so simple, it hardly warrants a standard library function.

merge two map[string]interface{} from json

I have two json inputs built this way
"count: 1 result: fields"
I would like to concatenate the fields that I find within result without using a defined structure. I have tried in many ways but most of the time the result is an error about the type Interface {} or the last map overwritten the data
I would like both the "result" and the first and second map fields to be merged within the result in output.
oracle, err := http.Get("http://XXX:8080/XXXX/"+id)
if err != nil {
panic(err)
}
defer oracle.Body.Close()
mysql, err := http.Get("http://XXX:3000/XXX/"+id)
if err != nil {
panic(err)
}
defer mysql.Body.Close()
oracleJSON, err := ioutil.ReadAll(oracle.Body)
if err != nil {
panic(err)
}
mysqlJSON, err := ioutil.ReadAll(mysql.Body)
if err != nil {
panic(err)
}
var oracleOUT map[string]interface{}
var mysqlOUT map[string]interface{}
json.Unmarshal(oracleJSON, &oracleOUT)
json.Unmarshal(mysqlJSON, &mysqlOUT)
a := oracleOUT["result"]
b := mysqlOUT["result"]
c.JSON(http.StatusOK, gin.H{"result": ????})
this is an example of json
{"count":1,"result":{"COD_DIPENDENTE":"00060636","MATRICOLA":"60636","COGNOME":"PIPPO"}}
If i have two json like this the result of the function it should be
`"result":{"COD_DIPENDENTE":"00060636","MATRICOLA":"60636","COGNOME":"PIPPO","COD_DIPENDENTE":"00060636","MATRICOLA":"60636","COGNOME":"PIPPO"}}`
The output you are looking for is not valid JSON. However with a small change you can output something very similar to your example that is valid JSON.
You probably do want to use a defined structure for the portion of the input that has a known structure, so that you can extract the more abstract "result" section more easily.
If you start at the top of the input structure using a map[string]interface{} then you'll have to do a type assertion on the "result" key. For example:
var input map[string]interface{}
err = json.Unmarshal(data, &input)
if err != nil {
return err
}
keys, ok := input["result"].(map[string]interface{})
if !ok {
return errors.New("wasn't the type we expected")
}
However if you used a defined structure for the top level you can do it like the following which feels much cleaner.
type Input struct {
Count int `json:"count"`
Result map[string]interface{} `json:"result"`
}
var input Input
err = json.Unmarshal(data, &input)
if err != nil {
return err
}
// from here you can use input.Result directly without a type assertion
To generate output that has duplicate keys, you could use an array of objects with a single key/value pair in each, then you end up with a valid JSON structure that does not overwrite keys. Here's how to do that (playground link):
package main
import (
"encoding/json"
"fmt"
)
type Input struct {
Count int `json:"count"`
Result map[string]interface{} `json:"result"`
}
type Output struct {
Count int `json:"count"`
Result []map[string]interface{} `json:"result"`
}
var inputdata = [][]byte{
[]byte(`{"count":1,"result":{"COD_DIPENDENTE":"00060636", "MATRICOLA":"60636", "COGNOME":"PIPPO"}}`),
[]byte(`{"count":1,"result":{"COD_DIPENDENTE":"00060636", "MATRICOLA":"60636", "COGNOME":"PIPPO"}}`),
}
func main() {
inputs := make([]Input, len(inputdata))
for i := range inputs {
err := json.Unmarshal(inputdata[i], &inputs[i])
if err != nil {
panic(err)
}
}
var out Output
out.Count = len(inputs)
for _, input := range inputs {
for k, v := range input.Result {
out.Result = append(out.Result, map[string]interface{}{k: v})
}
}
outdata, _ := json.Marshal(out)
fmt.Println(string(outdata))
}
Which produces output that looks like this when formatted:
{
"count": 2,
"result": [
{"MATRICOLA": "60636"},
{"COGNOME": "PIPPO"},
{"COD_DIPENDENTE": "00060636"},
{"COGNOME": "PIPPO"},
{"COD_DIPENDENTE": "00060636"},
{"MATRICOLA": "60636"}
]
}

Getting json dynamic key name as string?

For example:
{"id":
{"12345678901234":
{"Account":"asdf",
"Password":"qwerty"
"LastSeen":"1397621470",
}
}
}
A program I've been trying to make needs to get the id as a string and then later use it to check the time in LastSeen.
I've tried using simplejson and jsonq,but still cant figure out how to do that.
You can use RawMessage and make it much simpiler (play with it) :
package main
import (
"encoding/json"
"fmt"
)
var data []byte = []byte(`{"id": {"12345678901234": {"Account":"asdf", "Password":"qwerty", "LastSeen":"1397621470"}}}`)
type Message struct {
Id string
Info struct {
Account string
Password string
LastSeen string
}
}
func main() {
var (
tmpmsg struct {
Data map[string]json.RawMessage `json:"id"`
}
msg Message
)
if err := json.Unmarshal(data, &tmpmsg); err != nil {
panic(err) //you probably wanna use or something instead
}
for id, raw := range tmpmsg.Data {
msg.Id = id
if err := json.Unmarshal(raw, &msg.Info); err != nil {
panic(err)
}
}
fmt.Printf("%+v\n", msg)
}
Looking at the Golang blog post on JSON here it can be done using the encoding/json package. I created a small program to do this as follows:
package main
import (
"encoding/json"
"fmt"
)
var data []byte = []byte(`{"id": {"12345678901234": {"Account":"asdf", "Password":"qwerty", "LastSeen":"1397621470"}}}`)
type Message struct {
id string
LastSeen int64
}
var m Message
func main() {
var i interface {}
err := json.Unmarshal(data, &i)
if err != nil {
println("Error decoding data")
fmt.Printf("%s", err.Error())
return
}
m := i.(map[string]interface{})
for k, v := range m {
println(k)
im := v.(map[string]interface{})
for ik, iv := range im {
println("\t", ik)
jm := iv.(map[string]interface{})
for jk, jv := range jm {
println("\t\t", jk, ": ", jv.(string))
}
}
}
}
I apologise if this is poor in terms of Go best practices and such, I am new to the language. And I know that some elements of this aren't entirely necessary like the Message type definition but this works, at least on your data.

strict json parser in golang

In Go, I have some JSON from third-party API and then I'm trying to parse it:
b := []byte(`{"age":21,"married":true}`)
var response_hash map[string]string
_ = json.Unmarshal(b, &response_hash)
But it fails (response_hash becomes empty) and Unmarshal can handle only JSON with quotes:
b := []byte(`{"age":"21","married":"true"}`)
var response_hash map[string]string
_ = json.Unmarshal(b, &response_hash)
Is there any way to read the first version of JSON from third-party API?
Go is a strongly typed language. You need to specify what types the JSON encoder is to expect. You're creating a map of string values but your two json values are an integer and a boolean value.
Here's a working example with a struct.
http://play.golang.org/p/oI1JD1UUhu
package main
import (
"fmt"
"encoding/json"
)
type user struct {
Age int
Married bool
}
func main() {
src_json := []byte(`{"age":21,"married":true}`)
u := user{}
err := json.Unmarshal(src_json, &u)
if err != nil {
panic(err)
}
fmt.Printf("Age: %d\n", u.Age)
fmt.Printf("Married: %v\n", u.Married)
}
If you want to do it in a more dynamic manner you can use a map of interface{} values. You just have to do type assertions when you use the values.
http://play.golang.org/p/zmT3sPimZC
package main
import (
"fmt"
"encoding/json"
)
func main() {
src_json := []byte(`{"age":21,"married":true}`)
// Map of interfaces can receive any value types
u := map[string]interface{}{}
err := json.Unmarshal(src_json, &u)
if err != nil {
panic(err)
}
// Type assert values
// Unmarshal stores "age" as a float even though it's an int.
fmt.Printf("Age: %1.0f\n", u["age"].(float64))
fmt.Printf("Married: %v\n", u["married"].(bool))
}