Golang parse JSON with unknown keys [duplicate] - json
This question already has answers here:
How to unmarshal JSON with unknown fieldnames to struct [duplicate]
(1 answer)
Partly JSON unmarshal into a map in Go
(3 answers)
Unmarshal JSON with some known, and some unknown field names
(8 answers)
Closed 3 years ago.
I'm trying to unmarshal json with dynamic field keys to the struct
That json returned from storcli utility for linux.
One part of the code work well, but if json data contains many structs I can't unmarshal it.
I think that is because DriveDetailedInformation struct do not have all json tags.
Can anybody help me?
package main
import (
"fmt"
"encoding/json"
)
type jsonStruct struct {
Controllers []struct {
CommandStatus struct {
Controller int `json:"Controller"`
Status string `json:"Status"`
Description string `json:"Description"`
} `json:"Command Status"`
ResponseData map[string]*json.RawMessage `json:"Response Data"`
} `json:"Controllers"`
}
type DriveStruct struct {
EIDSlt string `json:"EID:Slt"`
DID int `json:"DID"`
State string `json:"State"`
DG int `json:"DG"`
Size string `json:"Size"`
Intf string `json:"Intf"`
Med string `json:"Med"`
SED string `json:"SED"`
PI string `json:"PI"`
SeSz string `json:"SeSz"`
Model string `json:"Model"`
Sp string `json:"Sp"`
}
type DriveDetailedInformation struct {
DriveState map[string]DriveStateStruct
DriveDeviceAttributes map[string]DriveDeviceAttributesStruct
DrivePoliciesSettings map[string]DrivePoliciesSettingsStruct
InquiryData string `json:"Inquiry Data"`
}
type DriveStateStruct struct {
ShieldCounter int `json:"Shield Counter"`
MediaErrorCount int `json:"Media Error Count"`
OtherErrorCount int `json:"Other Error Count"`
BBMErrorCount int `json:"BBM Error Count"`
DriveTemperature string `json:"Drive Temperature"`
PredictiveFailureCount int `json:"Predictive Failure Count"`
SMARTAlertFlaggedByDrive string `json:"S.M.A.R.T alert flagged by drive"`
}
type DriveDeviceAttributesStruct struct {
SN string `json:"SN"`
ManufacturerID string `json:"Manufacturer Id"`
ModelNumber string `json:"Model Number"`
NANDVendor string `json:"NAND Vendor"`
WWN string `json:"WWN"`
FirmwareRevision string `json:"Firmware Revision"`
RawSize string `json:"Raw size"`
CoercedSize string `json:"Coerced size"`
NonCoercedSize string `json:"Non Coerced size"`
DeviceSpeed string `json:"Device Speed"`
LinkSpeed string `json:"Link Speed"`
NCQSetting string `json:"NCQ setting"`
WriteCache string `json:"Write cache"`
SectorSize string `json:"Sector Size"`
ConnectorName string `json:"Connector Name"`
}
type DrivePoliciesSettingsStruct struct {
DrivePosition string `json:"Drive position"`
EnclosurePosition int `json:"Enclosure position"`
ConnectedPortNumber string `json:"Connected Port Number"`
SequenceNumber int `json:"Sequence Number"`
CommissionedSpare string `json:"Commissioned Spare"`
EmergencySpare string `json:"Emergency Spare"`
LastPredictiveFailureEventSequenceNumber int `json:"Last Predictive Failure Event Sequence Number"`
SuccessfulDiagnosticsCompletionOn string `json:"Successful diagnostics completion on"`
SEDCapable string `json:"SED Capable"`
SEDEnabled string `json:"SED Enabled"`
Secured string `json:"Secured"`
Locked string `json:"Locked"`
NeedsEKMAttention string `json:"Needs EKM Attention"`
PIEligible string `json:"PI Eligible"`
Certified string `json:"Certified"`
WidePortCapable string `json:"Wide Port Capable"`
PortInformation []struct {
Port int `json:"Port"`
Status string `json:"Status"`
Linkspeed string `json:"Linkspeed"`
SASAddress string `json:"SAS address"`
} `json:"Port Information"`
}
var jsonData = `
{
"Controllers":[
{
"Command Status":{
"Controller":0,
"Status":"Success",
"Description":"Show Drive Information Succeeded."
},
"Response Data":{
"Drive /c0/e31/s0":[
{
"EID:Slt":"31:0",
"DID":19,
"State":"Onln",
"DG":0,
"Size":"9.094 TB",
"Intf":"SATA",
"Med":"HDD",
"SED":"N",
"PI":"N",
"SeSz":"512B",
"Model":"ST10000DM0004-1ZC101",
"Sp":"U"
}
],
"Drive /c0/e31/s0 - Detailed Information":{
"Drive /c0/e31/s0 State":{
"Shield Counter":0,
"Media Error Count":0,
"Other Error Count":0,
"BBM Error Count":0,
"Drive Temperature":" 25C (77.00 F)",
"Predictive Failure Count":0,
"S.M.A.R.T alert flagged by drive":"No"
},
"Drive /c0/e31/s0 Device attributes":{
"SN":" ZA23V0DH",
"Manufacturer Id":"ATA ",
"Model Number":"ST10000DM0004-1ZC101",
"NAND Vendor":"NA",
"WWN":"5000c500a5ad06b6",
"Firmware Revision":"DN01 ",
"Raw size":"9.095 TB [0x48c400000 Sectors]",
"Coerced size":"9.094 TB [0x48c300000 Sectors]",
"Non Coerced size":"9.094 TB [0x48c300000 Sectors]",
"Device Speed":"6.0Gb/s",
"Link Speed":"6.0Gb/s",
"NCQ setting":"N/A",
"Write cache":"N/A",
"Sector Size":"512B",
"Connector Name":""
},
"Drive /c0/e31/s0 Policies/Settings":{
"Drive position":"DriveGroup:0, Span:0, Row:0",
"Enclosure position":0,
"Connected Port Number":"0(path0) ",
"Sequence Number":2,
"Commissioned Spare":"No",
"Emergency Spare":"No",
"Last Predictive Failure Event Sequence Number":0,
"Successful diagnostics completion on":"N/A",
"SED Capable":"No",
"SED Enabled":"No",
"Secured":"No",
"Locked":"No",
"Needs EKM Attention":"No",
"PI Eligible":"No",
"Certified":"No",
"Wide Port Capable":"No",
"Port Information":[
{
"Port":0,
"Status":"Active",
"Linkspeed":"6.0Gb/s",
"SAS address":"0x5003048001927c6c"
}
]
},
"Inquiry Data":""
},
"Drive /c0/e31/s1":[
{
"EID:Slt":"31:1",
"DID":20,
"State":"Onln",
"DG":0,
"Size":"9.094 TB",
"Intf":"SATA",
"Med":"HDD",
"SED":"N",
"PI":"N",
"SeSz":"512B",
"Model":"ST10000DM0004-1ZC101",
"Sp":"U"
}
],
"Drive /c0/e31/s1 - Detailed Information":{
"Drive /c0/e31/s1 State":{
"Shield Counter":0,
"Media Error Count":0,
"Other Error Count":0,
"BBM Error Count":0,
"Drive Temperature":" 25C (77.00 F)",
"Predictive Failure Count":0,
"S.M.A.R.T alert flagged by drive":"No"
},
"Drive /c0/e31/s1 Device attributes":{
"SN":" ZA23MCVS",
"Manufacturer Id":"ATA ",
"Model Number":"ST10000DM0004-1ZC101",
"NAND Vendor":"NA",
"WWN":"5000c500a5acc582",
"Firmware Revision":"DN01 ",
"Raw size":"9.095 TB [0x48c400000 Sectors]",
"Coerced size":"9.094 TB [0x48c300000 Sectors]",
"Non Coerced size":"9.094 TB [0x48c300000 Sectors]",
"Device Speed":"6.0Gb/s",
"Link Speed":"6.0Gb/s",
"NCQ setting":"N/A",
"Write cache":"N/A",
"Sector Size":"512B",
"Connector Name":""
},
"Drive /c0/e31/s1 Policies/Settings":{
"Drive position":"DriveGroup:0, Span:0, Row:1",
"Enclosure position":0,
"Connected Port Number":"0(path0) ",
"Sequence Number":2,
"Commissioned Spare":"No",
"Emergency Spare":"No",
"Last Predictive Failure Event Sequence Number":0,
"Successful diagnostics completion on":"N/A",
"SED Capable":"No",
"SED Enabled":"No",
"Secured":"No",
"Locked":"No",
"Needs EKM Attention":"No",
"PI Eligible":"No",
"Certified":"No",
"Wide Port Capable":"No",
"Port Information":[
{
"Port":0,
"Status":"Active",
"Linkspeed":"6.0Gb/s",
"SAS address":"0x5003048001927c6d"
}
]
},
"Inquiry Data":""
}
}
}
]
}
`
func main() {
var f jsonStruct
err := json.Unmarshal([]byte(jsonData), &f)
if err != nil {
fmt.Println("Error parsing JSON: ", err)
}
for _, controller := range f.Controllers {
for k, v := range controller.ResponseData {
///THAT IS WORK
var ds []DriveStruct
if err := json.Unmarshal(*v, &ds); err == nil {
fmt.Println(k, ds)
}
///THAT IS NOT WORK WHY?
var dds DriveDetailedInformation
if err := json.Unmarshal(*v, &dds); err == nil {
fmt.Println(k, dds)
}
}
}
}
Because your key value is not correct. Below is the code I modified.
type DriveDetailedInformation struct {
DriveState DriveStateStruct `json:"Drive /c0/e31/s0 State"`
DriveDeviceAttributes DriveDeviceAttributesStruct `json:"Drive /c0/e31/s0 Device attributes"`
DrivePoliciesSettings DrivePoliciesSettingsStruct `json:"Drive /c0/e31/s0 Policies/Settings"`
InquiryData string `json:"Inquiry Data"`
}
type DriveDetailedInformation1 struct {
DriveState DriveStateStruct `json:"Drive /c0/e31/s1 State"`
DriveDeviceAttributes DriveDeviceAttributesStruct `json:"Drive /c0/e31/s1 Device attributes"`
DrivePoliciesSettings DrivePoliciesSettingsStruct `json:"Drive /c0/e31/s1 Policies/Settings"`
InquiryData string `json:"Inquiry Data"`
}
func main() {
var f jsonStruct
err := json.Unmarshal([]byte(jsonData), &f)
if err != nil {
fmt.Println("Error parsing JSON: ", err)
}
for _, controller := range f.Controllers {
for k, v := range controller.ResponseData {
switch k {
case "Drive /c0/e31/s0","Drive /c0/e31/s1":
var ds []DriveStruct
if err := json.Unmarshal(*v, &ds); err == nil {
fmt.Println(k, ds)
}
case "Drive /c0/e31/s1 - Detailed Information":
var dds1 DriveDetailedInformation1
if err := json.Unmarshal(*v, &dds1); err == nil {
fmt.Println(k, dds1)
}
case "Drive /c0/e31/s0 - Detailed Information":
var dds DriveDetailedInformation
if err := json.Unmarshal(*v, &dds); err == nil {
fmt.Println(k, dds)
}
}
}
}
}
I think Response Data can be parsed in a structure.
Related
How to create a struct with different structs of one members of struct in go?
I am getting a json response from a server with three fix json object field and one field with different json objects.
Example - Response 1
{
status : "success",
error-code : "",
message : "login successful",
data : {
token : "token value",
refresh-token : "refresh token value"
}
}
Example - Response 2
{
status : "success",
error-code : "",
message : "user data fetched",
data : {
first-name: "josh",
last-name : "walter",
age : "30",
phone: 1234567890,
address : "x street, dc"
}
}
for above json response created struct as follows
type loginData struct {
Token string `json:"token"`
RefreshToken string `json:"refresh-token"`
}
type userdata {
FirstName string `json:"first-name"`
LastName string `json:"refresh-token"`
Age string `json:"age"`
Phone string `json:"phone"`
Address string `json:"address"`
}
type response struct {
Status string `json:"status"`
ErrorCode string `json:"error-code"`
RespMessage string `json:"message"`
RespData string `json:"data"`
}
How to add logindata struct while unmarshaling during login response and userdata struct while unmarshaling userdata response in "RespData" field in response struct
First change the RespData field's type:
type response struct {
Status string `json:"status"`
ErrorCode string `json:"error-code"`
RespMessage string `json:"message"`
RespData interface{} `json:"data"`
}
Then, depending on what request you are making, set the RespData field to a pre-allocated instance of a pointer to the expected type:
r, err := http.Get("https://some_api.com/loginData")
if err != nil {
return err
}
defer r.Body.Close()
// check r.StatusCode and make sure it's correct
data := loginData{}
resp := response{RespData: &data}
if err := json.NewDecoder(r.Body).Decode(&resp); err != nil {
return err
}
fmt.Println(data)
According to me, you should have two different structs to do this. One reason is separation of concern, as the structure of these responses may change in the future and these are responses from two different apis so its better to maintain different response objects.
type loginData struct {
Token string `json:"token"`
RefreshToken string `json:"refresh-token"`
}
type userdata struct {
FirstName string `json:"first-name"`
LastName string `json:"refresh-token"`
Age string `json:"age"`
Phone string `json:"phone"`
Address string `json:"address"`
}
type response struct {
Status string `json:"status"`
ErrorCode string `json:"error-code"`
RespMessage string `json:"message"`
}
type userResponse struct {
response
RespData userdata `json:"data"`
}
type loginResponse struct {
response
RespData loginData `json:"data"`
}
#mkopriva as you said tried this suggestion it worked.
package main
import (
"encoding/json"
"fmt"
"strings"
)
type loginData struct {
Token string `json:"token"`
RefreshToken string `json:"refresh-token"`
}
type userdata struct {
FirstName string `json:"first-name"`
LastName string `json:"last-name"`
Age string `json:"age"`
Phone string `json:"phone"`
Address string `json:"address"`
}
type response struct {
Status string `json:"status"`
ErrorCode string `json:"error-code"`
RespMessage string `json:"message"`
RespData interface{} `json:"data"`
}
func main() {
jsonresp_1 := `{
"status" : "success",
"error-code" : "",
"message" : "login successful",
"data" : {
"token" : "token value",
"refresh-token" : "refresh token value"
}
}`
jsonresp_2 := `{
"status" : "success",
"error-code" : "",
"message" : "user data fetched",
"data" : {
"first-name": "josh",
"last-name" : "walter",
"age" : "30",
"phone": "1234567890",
"address" : "x street, dc"
}
}`
resp1 := strings.NewReader(jsonresp_1)
data1 := loginData{}
respdata1 := response{RespData: &data1}
if err := json.NewDecoder(resp1).Decode(&respdata1); err != nil {
fmt.Println(err)
}
fmt.Printf("token : %s \nrefreshtoken : %s \n", data1.Token, data1.RefreshToken)
fmt.Println("-------------------")
resp2 := strings.NewReader(jsonresp_2)
data2 := userdata{}
respdata2 := response{RespData: &data2}
if err := json.NewDecoder(resp2).Decode(&respdata2); err != nil {
fmt.Println(err)
}
fmt.Printf("firstname : %s \nlastname : %s ", data2.FirstName, data2.LastName)
}
Output
⚡ ⇒ go run main.go
token : token value
refreshtoken : refresh token value
-------------------
firstname : josh
lastname : walter
Impossible insert data into mysql table with Point data type using Gorm customized data type
In my mysql db I have table "restaurant". This table has field "position" with type POINT. So, when I try to insert data in to table using customized data type like in docs https://gorm.io/docs/create.html#Create-From-SQL-Expr/Context-Valuer and to get inserted id, but I got error:
"sql: converting argument $6 type: unsupported type models.Point, a struct"
I'm new in go and sorry for my English.
Code:
Model
type Restaurant struct {
ID uint
Name string `json:"name"`
EmployeeId uint `json:"employee_id"`
Phone string `json:"phone"`
Address string `json:"address"`
ImagesUrl *string `json:"images_url"`
Position Point `json:"position" gorm:"type:Point"`
WorkDays string `json:"work_days"`
StartWorkTime string `json:"start_work_time"`
EndWorkTime string `json:"end_work_time"`
Blocked bool `json:"blocked"`
}
Point type
type Point struct {
LatLong string `json:"lat_long"`
//Lat float64
//Long float64
}
func (p *Point) Scan(v interface{}) error {
fmt.Printf("Scan value %v", v)
return nil
}
func (p Point) GormDataType() string {
return "geometry"
}
func (p Point) GormValue(ctx context.Context, db *gorm.DB) clause.Expr {
return clause.Expr{
SQL: "ST_PointFromText(?, 4326)",
Vars: []interface{}{fmt.Sprintf("POINT(%s)", p.LatLong)},
}
}
body request data:
{
"name": "Restaurant 1",
"employee_id": 9,
"phone": "123465",
"address": "Test Address 91",
"work_days": "sdfsfsd",
"start_work_time": "00:00:00",
"end_work_time": "00:00:00",
"position": {
"lat_long": "38.590559 68.755866"
}
}
Calling method after parsing body to Restaurant type
func NewRestaurant(restaurant *Restaurant) (uint, error) {
err := GetDB().Create(restaurant).Error
if err != nil {
return 0, err
}
return restaurant.ID, nil
}
dependency
import (
"gorm.io/driver/mysql"
"gorm.io/gorm"
)
gorm build this query :
INSERT INTO `restaurant`
(`name`,`employee_id`,`phone`,`address`,`images_url`,`position`,`work_days`,`start_work_time`,`end_work_time`,`blocked`)
VALUES
("Restaurant 1",9,"123465","Test Address 91",NULL,"{38.590559 68.755866}","sdfsfsd","00:00:00","00:00:00",false)
Clean way to conditionally unmarshal JSON to struct
I'm sending requests to a JSON API, and it either returns an error...
{
"error": {
"code": 404,
"message": "Document not found.",
"status": "NOT_FOUND"
}
}
or the data.
{
"name": "projectname",
"fields": {
"userId": {
"stringValue": "erw9384rjidfge"
}
},
"createTime": "2018-06-28T00:52:25.638791Z",
"updateTime": "2018-06-28T00:52:25.638791Z"
}
Here are the corresponding structs
type HttpError struct {
Code int `json:"code"`
Message string `json:"message"`
Status string `json:"status"`
}
type Document struct {
Name string `json:"name"`
Fields struct {
UserID struct {
StringValue string `json:"stringValue"`
} `json:"userId"`
} `json:"fields"`
CreateTime time.Time `json:"createTime"`
UpdateTime time.Time `json:"updateTime"`
}
Once I get the response, how do I cleanly/concisely unmarshal to the correct struct? I've seen a lot of ugly solutions (maybe Go's fault instead of the writers).
func getDocument() {
resp, _ := httpClient.Get("example.com")
defer resp.Body.Close()
bodyBytes, _ := ioutil.ReadAll(resp.Body)
var data map[string]interface{}
// How to unmarshal to either HttpError or Document??
err = json.Unmarshal([]byte(bodyBytes), &data)
}
By the way, I can't use the Go Firestore client library because reasons.
You can use an struct type inside your unmarshal method; with pointers to establish what's been unmarshalled.
Note: This code assumes there is no overlap of top level json keys... error / name / fields / etc.
type outer struct {
*HttpError `json:"error"`
*Document
}
var out outer
if err := json.Unmarshal(bodyBytes, &out); err != nil {
// error handling
}
if out.HttpErr != nil {
// handle error json case
}
// Here you can use out.Document, probably worth check if it is nil first.
Runnable example
json: cannot unmarshal array into Go value of type main.Data
Json is -
{
"apiAddr":"abc",
"data":
[
{
"key":"uid1",
"name":"test",
"commandList":["dummy cmd"],
"frequency":"1",
"deviceList":["dev1"],
"lastUpdatedBy": "user",
"status":"Do something"
}
]
}
And the code to unmarshall is -
type Data struct {
APIAddr string `json:"apiAddr"`
Data []Template `json:"data"`
}
type Template struct {
Key string `json:"key"`
Name string `json:"name"`
CommandList []string `json:"commandList"`
Frequency string `json:"frequency"`
DeviceList []string `json:"deviceList"`
LastUpdatedBy string `json:"lastUpdatedBy"`
Status string `json:"status"`
}
raw, err := ioutil.ReadFile(*testFile)
if err != nil {
return
}
var testTemplates Data
err = json.Unmarshal(raw, &testTemplates)
if err != nil {
return
}
where testFile is the json file.
I am getting this error
json: cannot unmarshal array into Go value of type main.Data.
Looking at the existing questions in stackoverflow, looks like I am doing all right.Anyone?
Made a few modification and Unmarshaling worked just fine.
package main
import (
"encoding/json"
"fmt"
)
var raw = ` {
"apiAddr":"abc",
"data":
[
{
"key":"uid1",
"name":"test",
"commandList":["dummy cmd"],
"frequency":"1",
"deviceList":["dev1"],
"lastUpdatedBy": "user",
"status":"Do something"
}
]
}`
func main() {
var testTemplates Data
err := json.Unmarshal([]byte(raw), &testTemplates)
if err != nil {
return
}
fmt.Println("Hello, playground", testTemplates)
}
type Data struct {
APIAddr string `json:"apiAddr"`
Data []Template `json:"data"`
}
type Template struct {
Key string `json:"key"`
Name string `json:"name"`
CommandList []string `json:"commandList"`
Frequency string `json:"frequency"`
DeviceList []string `json:"deviceList"`
LastUpdatedBy string `json:"lastUpdatedBy"`
Status string `json:"status"`
}
You can run it in Playground as well: https://play.golang.org/p/TSmUnFYO97-
json: cannot unmarshal string into Go value of type map[string]interface {}
I have been working on converting some crypto pool software over to work with an incompatible coin. I believe I have it all about done. But this error keeps popping up and I just can't seem to figure out what the problem is. Here is my code:
type GetBalanceReply struct {
Unspent string `json:"unspent"`
}
type SendTransactionReply struct {
Hash string `json:"hash"`
}
type RPCClient struct {
sync.RWMutex
Url string
Name string
Account string
Password string
sick bool
sickRate int
successRate int
client *http.Client
}
type GetBlockReply struct {
Difficulty string `json:"bits"`
Hash string `json:"hash"`
MerkleTreeHash string `json:"merkle_tree_hash"`
Nonce string `json:"nonce"`
PrevHash string `json:"previous_block_hash"`
TimeStamp string `json:"time_stamp"`
Version string `json:"version"`
Mixhash string `json:"mixhash"`
Number string `json:"number"`
TransactionCount string `json:"transaction_count"`
}
type GetBlockReplyPart struct {
Number string `json:"number"`
Difficulty string `json:"bits"`
}
type TxReceipt struct {
TxHash string `json:"hash"`
}
type Tx struct {
Gas string `json:"gas"`
GasPrice string `json:"gasPrice"`
Hash string `json:"hash"`
}
type JSONRpcResp struct {
Id *json.RawMessage `json:"id"`
Result *json.RawMessage `json:"result"`
Balance *json.RawMessage `json:"balance"`
Transaction *json.RawMessage `json:"transaction"`
Error map[string]interface{} `json:"error"`
}
func (r *RPCClient) GetPendingBlock() (*GetBlockReplyPart, error) {
rpcResp, err := r.doPost(r.Url, "fetchheaderext", []interface{}{r.Account, r.Password, "pending"})
if err != nil {
return nil, err
}
if rpcResp.Result != nil {
var reply *GetBlockReplyPart
err = json.Unmarshal(*rpcResp.Result, &reply)
return reply, err
}
return nil, nil
}
func (r *RPCClient) GetBlockByHeight(height int64) (*GetBlockReply, error) {
//params := []interface{}{fmt.Sprintf("0x%x", height), true}
params := []interface{}{"-t", height}
return r.getBlockBy("fetch-header", params)
}
Whenever I run that manually in the wallet it displays:
"result": {
"bits": "7326472509313",
"hash": "060d0f6157d08bb294ad30f97a2c15c821ff46236281f118d65576b9e4a0ba27",
"merkle_tree_hash": "36936f36718e134e1eecaef05e66ebc4079811d8ee5a543f36d370335adc0801",
"nonce": "0",
"previous_block_hash": "10f4da59558fe41bab50a15864d1394462cd90836aecf52524f4cbce02a74a27",
"time_stamp": "1520718416",
"version": "1",
"mixhash": "0",
"number": "1011160",
"transaction_count": "1"
}'
But, the code errors out with "json: cannot unmarshal string into Go value of type map[string]interface {}"
Can anyone help me figure this one out? I've been hours on trying to figure it out on my own.
Try changing the type of the Error field to interface{} if you are not sure about the error structure, or change it to string if its always a string.
This should get rid of the error.
The error comes from unmarshaling JSONRpcResp. The input data has something like {error: "this is the error message", ...} and unmarshaling this into a map[string]interface{} will simply not work.