How to save a JSON file into csv - json

I am trying to save this astros.json file into csv file.
Everything is working I don't have any error but my code is not going into my for loop.
So the astros.csv file is created successfully but there are no datas inside.
package main
import (
"encoding/json"
"encoding/csv"
"fmt"
"io/ioutil"
"os"
"net/http"
"strconv"
)
type People struct {
Name string
Craft string
}
type General struct {
//People []People
Number int
Message string
}
func main() {
// Reading data from JSON File
response, err := http.Get("http://api.open-notify.org/astros.json")
if err != nil {
fmt.Printf("The Http request failed with error %s\n", err)
}
data,_ := ioutil.ReadAll(response.Body)
//fmt.Println(string(data))
// Unmarshal JSON data
var general []General
json.Unmarshal([]byte(data), &general)
//fmt.Printf("First person: %s, Message: %s", general.People[0].Name, general.Message)
// Create a csv file
csvdatafile, err := os.Create("./astros.csv")
if err != nil {
fmt.Println(err)
}
defer csvdatafile.Close()
// Write Unmarshaled json data to CSV file
w := csv.NewWriter(csvdatafile)
for _, obj := range general {
fmt.Println("Are you going into the for ?")
var record []string
//record = append(record, obj.People)
record = append(record, strconv.Itoa(obj.Number), obj.Message)
record = append(record, obj.Message)
//record = append(record, []string{People})
w.Write(record)
fmt.Println("Are you coming here")
record = nil
}
w.Flush()
fmt.Println("Appending succed")
}

It is important to always check for errors; if you change your code to:
err = json.Unmarshal([]byte(data), &general)
if err != nil {
fmt.Printf("Error in unmarshall %s\n", err)
return
}
Then you would see the problem; json.Unmarshal is returning cannot unmarshal object into Go value of type []main.General. This means that general is nil so your loop is never entered.
The reason for this is apparent when you compare the JSON:
{
"people": [{
"name": "Christina Koch",
"craft": "ISS"
}, {
"name": "Alexander Skvortsov",
"craft": "ISS"
}, {
"name": "Luca Parmitano",
"craft": "ISS"
}, {
"name": "Andrew Morgan",
"craft": "ISS"
}, {
"name": "Oleg Skripochka",
"craft": "ISS"
}, {
"name": "Jessica Meir",
"craft": "ISS"
}
],
"number": 6,
"message": "success"
}
With the variable you are unmarshalling into; []General where General is:
type General struct {
//People []People
Number int
Message string
}
It looks like you had the right idea at some point with the []People but then commented it out.
If you uncomment []People, unmarshal into a General (rather than []General) then make your loop for _, obj := range general.People { it should work as expected. This assumes that you want to iterate through the people; general.Message and general.Number are also available (but you dont need a loop to access these).
See this example (have removed file operation as the playground does not support that).

you could try something like this:
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"net/http"
"os"
"strconv"
)
type People struct {
Name string
Craft string
}
type Response struct {
People []People
Number int
Message string
}
func main() {
// Reading data from JSON File
r, err := http.Get("http://api.open-notify.org/astros.json")
if err != nil {
fmt.Printf("The Http request failed with error %s\n", err)
}
var response Response
err = json.NewDecoder(r.Body).Decode(&response)
if err != nil {
fmt.Printf("decoding error%s\n", err)
return
}
// Create a csv file
csvdatafile, err := os.Create("./astros.csv")
if err != nil {
fmt.Println(err)
}
defer csvdatafile.Close()
// Write Unmarshaled json data to CSV file
w := csv.NewWriter(csvdatafile)
for _, obj := range response.People {
fmt.Println("Are you going into the for ?")
var record []string
//record = append(record, obj.People)
record = append(record, strconv.Itoa(response.Number), response.Message)
record = append(record, obj.Name)
// record = append(record, []string{obj})
w.Write(record)
fmt.Println("Are you coming here")
record = nil
}
w.Flush()
fmt.Println("Appending succed")
}

Related

inputting json file from shell i converted using marshal the file but now i have to add metadata uuid and time

Im inputing JSON file from shell and converting to specific format i did that part on bottom but im trying to include like metadat like time ,uuid and couple of other thing to it then printout the new json file.My issue i cant put the new info to data part of the data
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"time"
// "github.com/google/uuid"
)
func help() {
fmt.Println("help")
}
type Event struct{
Specversion string `json:"specversion"`
ID string `json:"id"` /// uuid
Source string `json:"source"`
// Type bool `json:"type"`
Time time.Now `json:"time"`
Data string `json:"data"` /// this part supposed to come from the file
}
func main(){
if len(os.Args) < 1 { //checking for provided argument if the first arg satisfy then use read file
fmt.Println("Usage : " + os.Args[0] + " file name")
help()
os.Exit(1)
}
file, err := ioutil.ReadFile(os.Args[1])
if err != nil {
fmt.Println("Cannot read the file or file does not exists")
os.Exit(1)
}
out, err := json.Marshal(map[string]string{"data": string(file)})
if err != nil {
panic(err)
}
//fmt.Println(string(out))
e := Event{
Specversion: "1.0",
//id: uuid.New(),
Source: "CSS",
//type: true,
Time: time.Now(),
Data: data.(string),
}
fmt.Println(string(out))
fmt.Println(string(e))
//fmt.Println(string(id))
}
The outcome should be this
{
"specversion": "1.0",
"id": "ce20b92d-b45b-hhd7-a544-8fe4d8f7ff06",
"source": "CCS",
"type": "EVENT",
"time": "2022-08-09T23:59:08.468903+00:00",
"data": "{\"mac_address\": null, \".................
original file from input
"mac_address": "",
"serial_number": "j",
"device_type": "STORAGE",
"device_model": "VIRTUAL",
"part_number": "VIRTUAL",
"extra_attributes": [
{
"attribute_type": "ENTITLEMENT_ID",
"attribute_value": "H6MI9JYNJCOSBL1GTjfgjfjY6P"
The application marshals a map with the data, but not the metadata. The application configures a value with the metadata and data, but does nothing with it. Pull those pieces together. Something like this:
file, err := ioutil.ReadFile(os.Args[1])
if err != nil {
fmt.Println("Cannot read the file or file does not exists")
os.Exit(1)
}
e := Event{
Specversion: "1.0",
Source: "CSS",
Time: time.Now(),
Data: string(file),
}
out, _ := json.MarshalIndent(e, "", " ")
fmt.Println(string(out))
Run the program on the playground.
You can try the approach below for this:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"time"
// "github.com/google/uuid"
)
func help() {
fmt.Println("help")
}
type Event struct {
Specversion string `json:"specversion"`
ID string `json:"id"` /// uuid
Source string `json:"source"`
// Type bool `json:"type"`
Time time.Now `json:"time"`
Data map[string]interface{} `json:"data"` /// this part supposed to come from the file
}
func main() {
if len(os.Args) < 1 { //checking for provided argument if the first arg satisfy then use read file
fmt.Println("Usage : " + os.Args[0] + " file name")
help()
os.Exit(1)
}
file, err := ioutil.ReadFile(os.Args[1])
if err != nil {
fmt.Println("Cannot read the file or file does not exists")
os.Exit(1)
}
res := map[string]interface{}{}
err = json.Unmarshal(file, &res)
if err != nil {
panic(err)
}
e := Event{
Specversion: "1.0",
//id: uuid.New(),
Source: "CSS",
//type: true,
Time: time.Now(),
Data: res,
}
fmt.Println(e)
}
Defining your Event struct data type as map sting interface helps.
Hope this works.

How can I write one after another JSON data

I am working on a website scraper. I can send only 1 JSON data to JSON file regularly. I want to write one after another JSON data, so I need to keep hundreds of data in a single JSON file. like this
[
{
"id": 1321931,
"name": "Mike"
},
{
"id": 32139219,
"name": "Melissa"
},
{
"id": 8421921,
"name": "Jordan"
},
{
"id": 4291901,
"name": "David"
}
]
but output like this. When I send new data, just the first JSON data update itself.
[
{
"id": 1,
"name": "Mike"
}
]
here is the code:
package main
import (
"encoding/json"
"fmt"
"html/template"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"strings"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Name string `json:"name"`
}
var tpl *template.Template
var name string
var stonf Info
var allInfos []Info
var id int
var co = colly.NewCollector()
func main() {
fmt.Println("Started...")
allInfos = make([]Info, 1)
id = rand.Intn((99999 - 10000) + 10000)
// Reading Data From Json
data, err := ioutil.ReadFile("stocky.json")
if err != nil {
fmt.Println("ERROR 1 JSON", err)
}
// Unmarshal JSON data
var d []Info
err = json.Unmarshal([]byte(data), &d)
if err != nil {
fmt.Println(err)
}
tpl, _ = tpl.ParseGlob("templates/*.html")
http.HandleFunc("/mete", hellloHandleFunc)
staticHandler := http.FileServer(http.Dir("./css/"))
http.Handle("/css/", http.StripPrefix("/css", staticHandler))
http.ListenAndServe("localhost:8080", nil)
}
func hellloHandleFunc(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
log.Fatal(err)
}
allInfos[0].ID = id // JSON-PRO
// GET Price - Fiyat GETİR
co.OnHTML("div#dp", func(p *colly.HTMLElement) {
name = p.ChildText("h1#title")
})
requestLink := strings.TrimSpace(r.FormValue("input-link"))
co.Visit(requestLink)
// FIRST DATA JSON
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
stonf = Info{
Name: name,
}
fmt.Println("Index Running")
tpl.ExecuteTemplate(w, "form-copy.html", stonf)
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
Here is a solution which appends new Info to the list and store in file.
The solution will perform properly only for relatively small list. For large lists, the overhead of writing the entire file each time may be too high. In such case i propose to change the format to ndjson. It will allow to write only the current Info struct instead of the whole list.
I've also added synchronization mechanism to avoid race conditions in case you send multiple HTTP requests at the same time.
I assumed that the identifier must be generated separately for each request, and it is not a problem if collision occur.
package main
import (
"encoding/json"
"fmt"
"html/template"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"strings"
"sync"
"github.com/gocolly/colly"
)
type (
Info struct {
ID int `json:"id"`
Name string `json:"name"`
}
Infos struct {
List []Info
sync.Mutex
}
)
var (
infos *Infos
tpl *template.Template
co = colly.NewCollector()
)
func main() {
fmt.Println("Started...")
var err error
infos, err = readInfos()
if err != nil {
log.Fatal(err)
}
tpl, _ = tpl.ParseGlob("templates/*.html")
http.HandleFunc("/mete", hellloHandleFunc)
staticHandler := http.FileServer(http.Dir("./css/"))
http.Handle("/css/", http.StripPrefix("/css", staticHandler))
if err := http.ListenAndServe("localhost:8080", nil); err != nil {
log.Fatal(err)
}
}
func hellloHandleFunc(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
log.Fatal(err)
}
stonf := Info{
ID: rand.Intn((99999 - 10000) + 10000),
}
// GET Price - Fiyat GETİR
co.OnHTML("div#dp", func(p *colly.HTMLElement) {
stonf.Name = p.ChildText("h1#title")
})
requestLink := strings.TrimSpace(r.FormValue("input-link"))
if err := co.Visit(requestLink); err != nil {
log.Fatal(err)
}
if err := infos.AppendAndWrite(stonf); err != nil {
log.Fatal(err)
}
// FIRST DATA JSON
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(stonf)
fmt.Println("Index Running")
tpl.ExecuteTemplate(w, "form-copy.html", stonf)
}
func readInfos() (*Infos, error) {
// Reading Data From Json
data, err := ioutil.ReadFile("stocky.json")
if err != nil {
return nil, err
}
var r []Info
// Unmarshal JSON data
err = json.Unmarshal([]byte(data), &r)
if err != nil {
return nil, err
}
return &Infos{List: r}, nil
}
func (i *Infos) AppendAndWrite(info Info) error {
i.Lock()
defer i.Unlock()
i.List = append(i.List, info)
if err := i.storeLocked(); err != nil {
return fmt.Errorf("storing info list failed: %w", err)
}
return nil
}
func (i *Infos) storeLocked() error {
dataFile, err := json.MarshalIndent(i.List, "", " ")
if err != nil {
return fmt.Errorf("could not marshal infos JSON: %w", err)
}
err = ioutil.WriteFile("stocky.json", dataFile, 0666)
if err != nil {
return fmt.Errorf("could not write 'stocky.json' file: %w", err)
}
return nil
}
There is a standard called JSON lines (https://jsonlines.org/) consisting on only one JSON per line instead of wrapping all in a JSON array.
JSON library from Go stdlib works pretty well with JSON lines on both cases, reading and writing.
Write multiple JSON (one per line):
e := json.NewEncoder(yourWriterFile)
e.Encode(object1)
e.Encode(object2)
//...
Read multiple JSON (one per line or concatenated):
d := json.NewDecoder(yourReaderFile)
d.Decode(&object1)
d.Decode(&object2)
//...
More info: https://pkg.go.dev/encoding/json

Convert JSON file into csv in golang

this following code is working successfully, it converts my json file into a csv file. But I would like to add titles to each of my columns into the csv file. Unfortunately, I can't find out how to do that. If anyone has an idea it would be very helpful.
Regards
package main
import (
"encoding/json"
"encoding/csv"
"fmt"
"io/ioutil"
"os"
"net/http"
"strconv"
)
type People struct {
Name string
Craft string
}
type General struct {
People []People
Number int
Message string
}
func main() {
// Reading data from JSON File
response, err := http.Get("http://api.open-notify.org/astros.json")
if err != nil {
fmt.Printf("The Http request failed with error %s\n", err)
}
data,_ := ioutil.ReadAll(response.Body)
//fmt.Println(string(data))
// Unmarshal JSON data
var general General
json.Unmarshal([]byte(data), &general)
//fmt.Printf("First person: %s, Message: %s", general.People[0].Name, general.Message)
// Create a csv file
csvdatafile, err := os.Create("./astros.csv")
if err != nil {
fmt.Println(err)
}
defer csvdatafile.Close()
// Write Unmarshaled json data to CSV file
w := csv.NewWriter(csvdatafile)
for _, obj := range general.People {
fmt.Println("Are you going into the for ?")
var record []string
record = append(record, strconv.Itoa(general.Number), general.Message)
record = append(record, obj.Name, obj.Craft)
w.Write(record)
fmt.Println("Are you coming here")
record = nil
}
w.Flush()
fmt.Println("Appending succed")
}
Well if you want it just for this example can you just write the column headers to the file before the for statement, that is:
w := csv.NewWriter(csvdatafile)
//new code
var header []string
header = append(header, "Number")
header = append(header, "Message")
header = append(header, "Name")
header = append(header, "Craft")
w.Write(header)
for _, obj := range general.People {

Parsing a JSON file in Go

I am having trouble parsing a JSON file in Go.
I am not getting any errors, but I am not getting an output.
I have tried a few different methods, but I can't seem to get any to work.
Any help would be greatly appreciated. Thanks in advance.
package simplefiles
import (
"encoding/json"
"fmt"
"os"
)
//PluginInfo - struct for plugins.json
var PluginInfo struct {
LatestVersion string `json:"latest_version"`
LastUpdated string `json:"last_updated"`
Popular bool `json:"popular"`
Info []string `json:"Info"`
}
//ParsePlugins - parses plugins.json
func ParsePlugins() {
pluginsFile, err := os.Open("test.json")
if err != nil {
fmt.Println("opening config file", err.Error())
}
jsonParser := json.NewDecoder(pluginsFile)
if err = jsonParser.Decode(&PluginInfo); err != nil {
fmt.Println("parsing config file", err.Error())
} else {
fmt.Printf(PluginInfo.LastUpdated)
}
return
}
JSON Sample:
{
"my-test-site":{
"latest_version":"6.4.5",
"last_updated":"2016-05-22T00:23:00.000Z",
"popular":true,
"infomation":[
{
"id":6043,
"title":"Test info 1",
"created_at":"2014-08-01T10:58:35.000Z",
"updated_at":"2015-05-15T13:47:24.000Z",
"published_date":null,
"references":{
"url":[
"http://samplesite1.com",
"http://samplesite2.com"
]
},
"site_type":"info",
"fixed_v":"1.10"
}
]
},
"another-test-site":{
"latest_version":"2.1.0",
"last_updated":"2016-06-12T08:36:00.000Z",
"popular":false,
"infomation":[
{
"id":6044,
"title":"Test site 2 info",
"created_at":"2014-08-01T10:58:35.000Z",
"updated_at":"2015-05-15T13:47:24.000Z",
"published_date":null,
"references":{
"otherinfo":[
"blah blah blah"
]
},
"site_type":"search",
"fixed_v":"1.2.0"
}
]
}
}
Your problem is that your JSON data is a map of string to the struct type you defined, not the struct type directly. If you modify your code slightly as below it works, but you need to index into the map to get each struct value:
package main
import (
"encoding/json"
"fmt"
"os"
)
//PluginInfo - struct for plugins.json
var PluginInfo map[string]struct { // NOTICE map of string to struct
LatestVersion string `json:"latest_version"`
LastUpdated string `json:"last_updated"`
Popular bool `json:"popular"`
Info []string `json:"Info"`
}
//ParsePlugins - parses plugins.json
func ParsePlugins() {
pluginsFile, err := os.Open("test.json")
if err != nil {
fmt.Println("opening config file", err.Error())
}
jsonParser := json.NewDecoder(pluginsFile)
if err = jsonParser.Decode(&PluginInfo); err != nil {
fmt.Println("parsing config file", err.Error())
} else {
for key, val := range PluginInfo {
fmt.Printf("Key %q, last updated %s\n", key, val.LastUpdated)
}
}
return
}
func main() {
ParsePlugins()
}

Write Struct to Json File using Struct Fields (not json keys)

How can I read a json file into a struct, and then Marshal it back out to a json string with the Struct fields as keys (rather than the original json keys)?
(see Desired Output to Json File below...)
Code:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
)
type Rankings struct {
Keyword string `json:"keyword"`
GetCount uint32 `json:"get_count"`
Engine string `json:"engine"`
Locale string `json:"locale"`
Mobile bool `json:"mobile"`
}
func main() {
var jsonBlob = []byte(`
{"keyword":"hipaa compliance form", "get_count":157, "engine":"google", "locale":"en-us", "mobile":false}
`)
rankings := Rankings{}
err := json.Unmarshal(jsonBlob, &rankings)
if err != nil {
// nozzle.printError("opening config file", err.Error())
}
rankingsJson, _ := json.Marshal(rankings)
err = ioutil.WriteFile("output.json", rankingsJson, 0644)
fmt.Printf("%+v", rankings)
}
Output on screen:
{Keyword:hipaa compliance form GetCount:157 Engine:google Locale:en-us Mobile:false}
Output to Json File:
{"keyword":"hipaa compliance form","get_count":157,"engine":"google","locale":"en-us","mobile":false}
Desired Output to Json File:
{"Keyword":"hipaa compliance form","GetCount":157,"Engine":"google","Locale":"en-us","Mobile":false}
If I understand your question correctly, all you want to do is remove the json tags from your struct definition.
So:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
)
type Rankings struct {
Keyword string
GetCount uint32
Engine string
Locale string
Mobile bool
}
func main() {
var jsonBlob = []byte(`
{"keyword":"hipaa compliance form", "get_count":157, "engine":"google", "locale":"en-us", "mobile":false}
`)
rankings := Rankings{}
err := json.Unmarshal(jsonBlob, &rankings)
if err != nil {
// nozzle.printError("opening config file", err.Error())
}
rankingsJson, _ := json.Marshal(rankings)
err = ioutil.WriteFile("output.json", rankingsJson, 0644)
fmt.Printf("%+v", rankings)
}
Results in:
{Keyword:hipaa compliance form GetCount:0 Engine:google Locale:en-us Mobile:false}
And the file output is:
{"Keyword":"hipaa compliance form","GetCount":0,"Engine":"google","Locale":" en-us","Mobile":false}
Running example at http://play.golang.org/p/dC3s37HxvZ
Note: GetCount shows 0, since it was read in as "get_count". If you want to read in JSON that has "get_count" vs. "GetCount", but output "GetCount" then you'll have to do some additional parsing.
See Go- Copy all common fields between structs for additional info about this particular situation.
Try to change the json format in the struct
type Rankings struct {
Keyword string `json:"Keyword"`
GetCount uint32 `json:"Get_count"`
Engine string `json:"Engine"`
Locale string `json:"Locale"`
Mobile bool `json:"Mobile"`
}
An accourance happened by just using json.Marshal() / json.MarshalIndent().
It overwrites the existing file, which in my case was suboptimal. I just wanted to add content to current file, and keep old content.
This writes data through a buffer, with bytes.Buffer type.
This is what I gathered up so far:
package srf
import (
"bytes"
"encoding/json"
"os"
)
func WriteDataToFileAsJSON(data interface{}, filedir string) (int, error) {
//write data as buffer to json encoder
buffer := new(bytes.Buffer)
encoder := json.NewEncoder(buffer)
encoder.SetIndent("", "\t")
err := encoder.Encode(data)
if err != nil {
return 0, err
}
file, err := os.OpenFile(filedir, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
return 0, err
}
n, err := file.Write(buffer.Bytes())
if err != nil {
return 0, err
}
return n, nil
}
This is the execution of the function, together with the standard json.Marshal() or json.MarshalIndent() which overwrites the file
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
minerals "./minerals"
srf "./srf"
)
func main() {
//array of Test struct
var SomeType [10]minerals.Test
//Create 10 units of some random data to write
for a := 0; a < 10; a++ {
SomeType[a] = minerals.Test{
Name: "Rand",
Id: 123,
A: "desc",
Num: 999,
Link: "somelink",
People: []string{"John Doe", "Aby Daby"},
}
}
//writes aditional data to existing file, or creates a new file
n, err := srf.WriteDataToFileAsJSON(SomeType, "test2.json")
if err != nil {
log.Fatal(err)
}
fmt.Println("srf printed ", n, " bytes to ", "test2.json")
//overrides previous file
b, _ := json.MarshalIndent(SomeType, "", "\t")
ioutil.WriteFile("test.json", b, 0644)
}
Why is this useful?
File.Write() returns bytes written to the file! So this is perfect if you want to manage memory or storage.
WriteDataToFileAsJSON() (numberOfBytesWritten, error)