Write to []string from multiple json files using iotil.ReadAll() - json

I have a problem with getting information from json files during reading it from directory. I don't understand, why when i wrote code it's not working at all.
func FilePathWalkDir(root string) ([]string, error) {
var files []string
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
files = append(files, path)
}
return nil
})
return files, err
}
var s []string
func main() {
var (
files []string
err error
)
files, err = FilePathWalkDir("D:/Go/Go_project/Go_pro/files")
if err != nil {
panic(err)
}
for _, file := range files {
//fmt.Println("Index for json:", index)
jsonFile, err := os.Open(file)
if err != nil {
log.Println("Error:", err)
}
defer jsonFile.Close()
byteValue, _ := ioutil.ReadAll(jsonFile)
_ = json.Unmarshal([]byte(byteValue), &s)
log.Printf("Unmarshaled: %v", s)
}
}
After this, i got:
2020/06/21 13:10:03 Unmarshaled: []
2020/06/21 13:10:03 Unmarshaled: []
2020/06/21 13:10:03 Unmarshaled: []
Json files:
First:
{
"name":"Kate",
"date":"2013-04-23T19:24:59.511Z",
"data":"is nice"
}
Second:
{
"name":"Gleison",
"date":"2012-04-23T19:25:00.511Z",
"data":"is a good person"
}
Third:
{
"name":"Rodrigo",
"date":"2013-04-23T20:24:59.511Z",
"data":"is kind"
}

You are trying to unmarshal in slice type []string, while data inside file is map type map[string]string.
Slice type is: ["1", "2", "3"], and Map type is: {"name": "Andrew", "age": 33"}.
Please, read about slices and maps.
func FilePathWalkDir(root string) ([]string, error) {
var files []string
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
files = append(files, path)
}
return nil
})
return files, err
}
// var s []string !
var s map[string]string
func main() {
files, err := FilePathWalkDir("D:/Go/Go_project/Go_pro/files")
if err != nil {
panic(err)
}
for _, file := range files {
jsonFile, err := os.Open(file)
if err != nil {
// ???
log.Println("Error:", err)
}
defer jsonFile.Close()
byteValue, _ := ioutil.ReadAll(jsonFile)
if err := json.Unmarshal([]byte(byteValue), &s); err != nil {
// always check errors
panic(err)
}
log.Printf("Unmarshaled: %v", s)
}
}

If you want to convert the map[string] string to the [] string, you can use the following method
var strs []string
strs = append(s[key],strs)
...
But this method will make you lose the map's key.
Why do you use []string instead of map.

Related

Can't generate scalar JSON in gqlgen

I have a GQL scheme:
extend type MyType #key(fields: "id") {
id: ID! #external
properties: JSON #external
myField: String! #requires(fields: "properties")
}
scalar JSON
In graph/model/model.go:
package model
import (
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
)
type JSON map[string]interface{}
// UnmarshalGQL implements the graphql.Unmarshaler interface
func (b *JSON) UnmarshalGQL(v interface{}) error {
*b = make(map[string]interface{})
byteData, err := json.Marshal(v)
if err != nil {
panic("FAIL WHILE MARSHAL SCHEME")
}
tmp := make(map[string]interface{})
err = json.Unmarshal(byteData, &tmp)
if err != nil {
panic("FAIL WHILE UNMARSHAL SCHEME")
//return fmt.Errorf("%v", err)
}
*b = tmp
return nil
}
// MarshalGQL implements the graphql.Marshaler interface
func (b JSON) MarshalGQL(w io.Writer) {
byteData, err := json.Marshal(b)
if err != nil {
panic("FAIL WHILE MARSHAL SCHEME")
}
_, _ = w.Write(byteData)
}
But when I run go run github.com/99designs/gqlgen generate
error:
generating core failed: type.gotpl: template: type.gotpl:52:28: executing "type.gotpl" at <$type.Elem.GO>: nil pointer evaluating *config.TypeReference.
GOexit status 1
I just need to get map[string]interface{} which called JSON. I knew there's scalar Map, but for apollo federation that field must be called JSON.
it's should to replace MarshalGQL to MarshalJSON like:
type JSON map[string]interface{}
func MarshalJSON(b JSON) graphql.Marshaler {
return graphql.WriterFunc(func(w io.Writer) {
byteData, err := json.Marshal(b)
if err != nil {
log.Printf("FAIL WHILE MARSHAL JSON %v\n", string(byteData))
}
_, err = w.Write(byteData)
if err != nil {
log.Printf("FAIL WHILE WRITE DATA %v\n", string(byteData))
}
})
}
func UnmarshalJSON(v interface{}) (JSON, error) {
byteData, err := json.Marshal(v)
if err != nil {
return JSON{}, fmt.Errorf("FAIL WHILE MARSHAL SCHEME")
}
tmp := make(map[string]interface{})
err = json.Unmarshal(byteData, &tmp)
if err != nil {
return JSON{}, fmt.Errorf("FAIL WHILE UNMARSHAL SCHEME")
}
return tmp, nil
}

How can I write one after another JSON data

I am working on a website scraper. I can send only 1 JSON data to JSON file regularly. I want to write one after another JSON data, so I need to keep hundreds of data in a single JSON file. like this
[
{
"id": 1321931,
"name": "Mike"
},
{
"id": 32139219,
"name": "Melissa"
},
{
"id": 8421921,
"name": "Jordan"
},
{
"id": 4291901,
"name": "David"
}
]
but output like this. When I send new data, just the first JSON data update itself.
[
{
"id": 1,
"name": "Mike"
}
]
here is the code:
package main
import (
"encoding/json"
"fmt"
"html/template"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"strings"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Name string `json:"name"`
}
var tpl *template.Template
var name string
var stonf Info
var allInfos []Info
var id int
var co = colly.NewCollector()
func main() {
fmt.Println("Started...")
allInfos = make([]Info, 1)
id = rand.Intn((99999 - 10000) + 10000)
// Reading Data From Json
data, err := ioutil.ReadFile("stocky.json")
if err != nil {
fmt.Println("ERROR 1 JSON", err)
}
// Unmarshal JSON data
var d []Info
err = json.Unmarshal([]byte(data), &d)
if err != nil {
fmt.Println(err)
}
tpl, _ = tpl.ParseGlob("templates/*.html")
http.HandleFunc("/mete", hellloHandleFunc)
staticHandler := http.FileServer(http.Dir("./css/"))
http.Handle("/css/", http.StripPrefix("/css", staticHandler))
http.ListenAndServe("localhost:8080", nil)
}
func hellloHandleFunc(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
log.Fatal(err)
}
allInfos[0].ID = id // JSON-PRO
// GET Price - Fiyat GETİR
co.OnHTML("div#dp", func(p *colly.HTMLElement) {
name = p.ChildText("h1#title")
})
requestLink := strings.TrimSpace(r.FormValue("input-link"))
co.Visit(requestLink)
// FIRST DATA JSON
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
stonf = Info{
Name: name,
}
fmt.Println("Index Running")
tpl.ExecuteTemplate(w, "form-copy.html", stonf)
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
Here is a solution which appends new Info to the list and store in file.
The solution will perform properly only for relatively small list. For large lists, the overhead of writing the entire file each time may be too high. In such case i propose to change the format to ndjson. It will allow to write only the current Info struct instead of the whole list.
I've also added synchronization mechanism to avoid race conditions in case you send multiple HTTP requests at the same time.
I assumed that the identifier must be generated separately for each request, and it is not a problem if collision occur.
package main
import (
"encoding/json"
"fmt"
"html/template"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"strings"
"sync"
"github.com/gocolly/colly"
)
type (
Info struct {
ID int `json:"id"`
Name string `json:"name"`
}
Infos struct {
List []Info
sync.Mutex
}
)
var (
infos *Infos
tpl *template.Template
co = colly.NewCollector()
)
func main() {
fmt.Println("Started...")
var err error
infos, err = readInfos()
if err != nil {
log.Fatal(err)
}
tpl, _ = tpl.ParseGlob("templates/*.html")
http.HandleFunc("/mete", hellloHandleFunc)
staticHandler := http.FileServer(http.Dir("./css/"))
http.Handle("/css/", http.StripPrefix("/css", staticHandler))
if err := http.ListenAndServe("localhost:8080", nil); err != nil {
log.Fatal(err)
}
}
func hellloHandleFunc(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
log.Fatal(err)
}
stonf := Info{
ID: rand.Intn((99999 - 10000) + 10000),
}
// GET Price - Fiyat GETİR
co.OnHTML("div#dp", func(p *colly.HTMLElement) {
stonf.Name = p.ChildText("h1#title")
})
requestLink := strings.TrimSpace(r.FormValue("input-link"))
if err := co.Visit(requestLink); err != nil {
log.Fatal(err)
}
if err := infos.AppendAndWrite(stonf); err != nil {
log.Fatal(err)
}
// FIRST DATA JSON
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(stonf)
fmt.Println("Index Running")
tpl.ExecuteTemplate(w, "form-copy.html", stonf)
}
func readInfos() (*Infos, error) {
// Reading Data From Json
data, err := ioutil.ReadFile("stocky.json")
if err != nil {
return nil, err
}
var r []Info
// Unmarshal JSON data
err = json.Unmarshal([]byte(data), &r)
if err != nil {
return nil, err
}
return &Infos{List: r}, nil
}
func (i *Infos) AppendAndWrite(info Info) error {
i.Lock()
defer i.Unlock()
i.List = append(i.List, info)
if err := i.storeLocked(); err != nil {
return fmt.Errorf("storing info list failed: %w", err)
}
return nil
}
func (i *Infos) storeLocked() error {
dataFile, err := json.MarshalIndent(i.List, "", " ")
if err != nil {
return fmt.Errorf("could not marshal infos JSON: %w", err)
}
err = ioutil.WriteFile("stocky.json", dataFile, 0666)
if err != nil {
return fmt.Errorf("could not write 'stocky.json' file: %w", err)
}
return nil
}
There is a standard called JSON lines (https://jsonlines.org/) consisting on only one JSON per line instead of wrapping all in a JSON array.
JSON library from Go stdlib works pretty well with JSON lines on both cases, reading and writing.
Write multiple JSON (one per line):
e := json.NewEncoder(yourWriterFile)
e.Encode(object1)
e.Encode(object2)
//...
Read multiple JSON (one per line or concatenated):
d := json.NewDecoder(yourReaderFile)
d.Decode(&object1)
d.Decode(&object2)
//...
More info: https://pkg.go.dev/encoding/json

Golang Read JSON from S3 into struct in memory

I have a JSON file in S3 that takes the format of the following struct:
type StockInfo []struct {
Ticker string `json:"ticker"`
BoughtPrice string `json:"boughtPrice"`
NumberOfShares string `json:"numberOfShares"`
}
and I want to read it into a struct value from S3. In python the code would look something like this:
import boto3
import json
s3 = boto3.client('s3', 'us-east-1')
obj = s3.get_object(Bucket=os.environ["BucketName"], Key=os.environ["Key"])
fileContents = obj['Body'].read().decode('utf-8')
json_content = json.loads(fileContents)
However I'm kinda stuck on how to make this happen in Go. I've gotten this far:
package main
import (
"archive/tar"
"bytes"
"fmt"
"log"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/joho/godotenv"
)
type StockInfo []struct {
Ticker string `json:"ticker"`
BoughtPrice string `json:"boughtPrice"`
NumberOfShares string `json:"numberOfShares"`
}
func init() {
// loads values from .env into the system
if err := godotenv.Load(); err != nil {
log.Print("No .env file found")
}
return
}
func main() {
// Store the PATH environment variable in a variable
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
if err != nil {
panic(err)
}
s3Client := s3.New(sess)
bucket := "ian-test-bucket-go-python"
key := "StockInfo.json"
requestInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
result, err := s3Client.GetObject(requestInput)
if err != nil {
fmt.Println(err)
}
fmt.Println(result)
which returns to me the body/object buffer, but im not sure how to read that into a string so I can marshal it into my struct. I found this code in a similar question:
requestInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
buf := new(aws.WriteAtBuffer)
numBytes, _ := *s3manager.Downloader.Download(buf, requestInput)
tr := tar.NewReader(bytes.NewReader(buf.Bytes()))
but I get the following errors:
not enough arguments in call to method expression s3manager.Downloader.Download
have (*aws.WriteAtBuffer, *s3.GetObjectInput)
want (s3manager.Downloader, io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader))
multiple-value s3manager.Downloader.Download() in single-value context
Can anyone point me in the right direction? kinda frustrating how hard it seems to do this compared to python.
I was able to do it with the following code:
requestInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
result, err := s3Client.GetObject(requestInput)
if err != nil {
fmt.Println(err)
}
defer result.Body.Close()
body1, err := ioutil.ReadAll(result.Body)
if err != nil {
fmt.Println(err)
}
bodyString1 := fmt.Sprintf("%s", body1)
var s3data StockInfo
decoder := json.NewDecoder(strings.NewReader(bodyString1))
err = decoder.Decode(&s3data)
if err != nil {
fmt.Println("twas an error")
}
fmt.Println(s3data)
Alternative solution using json.Unmarshal
besed on aws-sdk-go-v2
...
params := &s3.GetObjectInput{
Bucket: aws.String(s3Record.S3.Bucket.Name),
Key: aws.String(s3Record.S3.Object.Key),
}
result, _ := client.GetObject(context.TODO(), params)
if err != nil {
panic(err)
}
defer result.Body.Close()
// capture all bytes from upload
b, err := ioutil.ReadAll(result.Body)
if err != nil {
panic(err)
}
var temp StockInfo
if err = json.Unmarshal(b, &temp); err != nil {
panic(err)
}
ftm.Println("res: ",b)

detect duplicate in JSON String Golang

I have JSON string like
"{\"a\": \"b\", \"a\":true,\"c\":[\"field_3 string 1\",\"field3 string2\"]}"
how to detect the duplicate attribute in this json string using Golang
Use the json.Decoder to walk through the JSON. When an object is found, walk through keys and values checking for duplicate keys.
func check(d *json.Decoder, path []string, dup func(path []string) error) error {
// Get next token from JSON
t, err := d.Token()
if err != nil {
return err
}
// Is it a delimiter?
delim, ok := t.(json.Delim)
// No, nothing more to check.
if !ok {
// scaler type, nothing to do
return nil
}
switch delim {
case '{':
keys := make(map[string]bool)
for d.More() {
// Get field key.
t, err := d.Token()
if err != nil {
return err
}
key := t.(string)
// Check for duplicates.
if keys[key] {
// Duplicate found. Call the application's dup function. The
// function can record the duplicate or return an error to stop
// the walk through the document.
if err := dup(append(path, key)); err != nil {
return err
}
}
keys[key] = true
// Check value.
if err := check(d, append(path, key), dup); err != nil {
return err
}
}
// consume trailing }
if _, err := d.Token(); err != nil {
return err
}
case '[':
i := 0
for d.More() {
if err := check(d, append(path, strconv.Itoa(i)), dup); err != nil {
return err
}
i++
}
// consume trailing ]
if _, err := d.Token(); err != nil {
return err
}
}
return nil
}
Here's how to call it:
func printDup(path []string) error {
fmt.Printf("Duplicate %s\n", strings.Join(path, "/"))
return nil
}
...
data := `{"a": "b", "a":true,"c":["field_3 string 1","field3 string2"], "d": {"e": 1, "e": 2}}`
if err := check(json.NewDecoder(strings.NewReader(data)), nil, printDup); err != nil {
log.Fatal(err)
}
The output is:
Duplicate a
Duplicate d/e
Run it on the Playground
Here's how to generate an error on the first duplicate key:
var ErrDuplicate = errors.New("duplicate")
func dupErr(path []string) error {
return ErrDuplicate
}
...
data := `{"a": "b", "a":true,"c":["field_3 string 1","field3 string2"], "d": {"e": 1, "e": 2}}`
err := check(json.NewDecoder(strings.NewReader(data)), nil, dupErr)
if err == ErrDuplicate {
fmt.Println("found a duplicate")
} else if err != nil {
// some other error
log.Fatal(err)
}
One that would probably work well would be to simply decode, reencode, then check the length of the new json against the old json:
https://play.golang.org/p/50P-x1fxCzp
package main
import (
"encoding/json"
"fmt"
)
func main() {
jsn := []byte("{\"a\": \"b\", \"a\":true,\"c\":[\"field_3 string 1\",\"field3 string2\"]}")
var m map[string]interface{}
err := json.Unmarshal(jsn, &m)
if err != nil {
panic(err)
}
l := len(jsn)
jsn, err = json.Marshal(m)
if err != nil {
panic(err)
}
if l != len(jsn) {
panic(fmt.Sprintf("%s: %d (%d)", "duplicate key", l, len(jsn)))
}
}
The right way to do it would be to re-implement the json.Decode function, and store a map of keys found, but the above should work (especially if you first stripped any spaces from the json using jsn = bytes.Replace(jsn, []byte(" "), []byte(""), -1) to guard against false positives.

Golang error: interface conversion: interface {} is bool/float..., not string

I am trying to decode an arbitrary JSON using Golang, so I unmarshal the incoming JSON in a map[string]interface{} as shown in the code below:
func JsonHandler(jsonRequest []byte) {
// Creating the maps for JSON
var m interface{}
// Parsing/Unmarshalling JSON encoding/json
if err := json.Unmarshal([]byte(jsonRequest), &m); err != nil {
panic(err)
}
//Creating an output file for writing
f, err := os.OpenFile("/home/dorrahadrich/Desktop/output.txt", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
panic(err)
}
defer f.Close()
ParseJson(m, f, err)
}
func ParseJson(m interface{}, f *os.File, err error) {
switch v := m.(interface{}).(type){
case map[string]interface{}:
ParseMap (m.(map[string]interface{}),f,err)
fmt.Println(v)
case []interface{}:
ParseArray (m.([]interface{}),f,err)
fmt.Println(v)
default:
}
}
func ParseMap(aMap map[string]interface{}, f *os.File, err error) {
for key, val := range aMap {
switch val.(type) {
case map[string]interface{}:
if _, err = f.WriteString(key + "={\n"); err != nil {
panic(err)
}
ParseMap(val.(map[string]interface{}), f, err)
//Close brackets
if _, err = f.WriteString("};\n"); err != nil {
panic(err)
}
case []interface{}:
//Write to file
if _, err = f.WriteString(key + "={\n"); err != nil {
panic(err)
}
ParseArray(val.([]interface{}), f, err)
//Close brackets
if _, err = f.WriteString("};\n"); err != nil {
panic(err)
}
default:
otherValues(key, val.(interface{}), f , err)
}
}
}
func ParseArray(anArray []interface{}, f *os.File, err error) {
for _, val := range anArray {
switch val.(type) {
case map[string]interface{}:
ParseMap(val.(map[string]interface{}), f, err)
case []interface{}:
ParseArray(val.([]interface{}), f, err)
default:
}
}
}
func otherValues(key string, other interface{}, f *os.File, err error) {
if _, err = f.WriteString(key); err != nil {
panic(err)
}
if _, err = f.WriteString("="); err != nil {
panic(err)
}
switch other.(interface{}).(type) {
case string:
if _, err = f.WriteString(other.(string)); err != nil {
panic(err)
}
case float64:
if _, err = f.WriteString(strconv.FormatFloat(other.(float64), 'f', -1, 64)); err != nil {
panic(err)
}
case bool:
if _, err = f.WriteString(strconv.FormatBool(other.(bool))); err != nil {
panic(err)
}
default:
}
}
The problem is that whenever a JSON contains a bool/int/float or any not string value the program panics saying that it fails converting an interface to the given type! Please note that the JSON is arbitrary so I don't have any idea about the keys nor the values, I can't unmrashal into an interface nor access the values giving a path.
The error says it all:
interface conversion: interface{} is bool/float64
when you are unmarshalling json the values for int and bool which are not of interface type. In your switch add case for bool/float64/string too. Since json is arbitrary unmarshal them using interface{}.
func otherValues(other interface{}, f *os.File, err error) {
switch bb := other.(interface{}).(type) {
case string:
fmt.Println("This is a string")
case float64:
fmt.Println("this is a float")
case bool:
fmt.Println("this is a boolean")
default:
fmt.Printf("Default value is of type %v", bb)
}
}
Use file.Write in place of file.WriteString
func (f *File) Write(b []byte) (n int, err error)
Write writes len(b) bytes to the File. It returns the number of bytes
written and an error, if any. Write returns a non-nil error when n !=
len(b).