I want to parse a JSON file that contains a feature collection of a country's regions.
I am using this package https://github.com/tidwall/geojson
I opened the file like this:
jsonFile, err := os.Open("filename.json")
if err != nil {
return nil, err
}
defer jsonFile.Close()
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
And then I parse the file using this:
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
but it returns a single geojson.Object and I want a list of features
Can someone help me with this
Problem solved
// open json file
jsonFile, err := os.Open(filename)
if err != nil {
return nil, err
}
defer jsonFile.Close()
// read the file
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
// parse into a single geojson.Object
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
// typecast geojson.Object into geojson.FeatureCollection
fc, ok := obj.(*geojson.FeatureCollection)
if !ok {
return nil, errors.Newf(errors.Internal, nil, "cannot convert into feature collection")
}
Related
I wrote some code which hits one public API and saves the JSON output in a file. But the data is storing line by line into the file instead of a single JSON format.
For eg.
Current Output:
{"ip":"1.1.1.1", "Country":"US"}
{"ip":"8.8.8.8", "Country":"IN"}
Desired Output:
[
{"ip":"1.1.1.1", "Country":"US"},
{"ip":"8.8.8.8", "Country":"IN"}
]
I know this should be pretty simple and i am missing out something.
My Current Code is:
To read IP from file and hit the API one by one on each IP.
func readIPfromFile(filename string, outFile string, timeout int) {
data := jsonIn{}
//open input file
jsonFile, err := os.Open(filename) //open input file
...
...
jsonData := bufio.NewScanner(jsonFile)
for jsonData.Scan() {
// marshal json data & check for logs
if err := json.Unmarshal(jsonData.Bytes(), &data); err != nil {
log.Fatal(err)
}
//save to file
url := fmt.Sprintf("http://ipinfo.io/%s", data.Host)
GetGeoIP(url, outFile, timeout)
}
}
To make HTTP Request with custom request header and call write to file function.
func GetGeoIP(url string, outFile string, timeout int) {
geoClient := http.Client{
Timeout: time.Second * time.Duration(timeout), // Timeout after 5 seconds
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("accept", "application/json")
res, getErr := geoClient.Do(req)
if getErr != nil {
log.Fatal(getErr)
}
if res.Body != nil {
defer res.Body.Close()
}
body, readErr := ioutil.ReadAll(res.Body)
if readErr != nil {
log.Fatal(readErr)
}
jsonout := jsonOut{}
jsonErr := json.Unmarshal(body, &jsonout)
if jsonErr != nil {
log.Fatal(jsonErr)
}
file, _ := json.Marshal(jsonout)
write2file(outFile, file)
}
To Write data to file:
func write2file(outFile string, file []byte) {
f, err := os.OpenFile(outFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
log.Fatal(err)
}
defer f.Close()
if _, err = f.WriteString(string(file)); err != nil {
log.Fatal(err)
}
if _, err = f.WriteString("\n"); err != nil {
log.Fatal(err)
}
I know, i can edit f.WriteString("\n"); to f.WriteString(","); to add comma but still adding [] in the file is challenging for me.
First, please do not invent a new way of json marshaling, just use golang built-in encoding/json or other library on github.
Second, if you want to create a json string that represents an array of object, you need to create the array of objects in golang and marshal it into string (or more precisely, into array of bytes)
I create a simple as below, but please DIY if possible.
https://go.dev/play/p/RR_ok-fUTb_4
I have link like this: https://storage.googleapis.com/data/test_file.csv.zip
the content is one csv file on the protected zip with password. How can I read data from the csv?
I have try this but its error
func ReadCSVZIPFromURL(fileURL string) (data [][]string, err error) {
resp, err := http.Get(fileURL)
if err != nil {
return data, errors.AddTrace(err)
}
defer resp.Body.Close()
r, err := zip.OpenReader(resp.Body)
if err != nil {
log.Fatal(err)
}
defer r.Close()
for _, file := range r.File {
if file.IsEncrypted() {
file.SetPassword("password")
}
reader := csv.NewReader(file)
data, err := reader.ReadAll()
}
return data, nil
}
i have solve the problem, this the solve. the idea is we copy the byte to zip reader, and we will get the ioReader, the we we read ioReader use csv library.
to encrypt and decrypt data using password we use this library "github.com/alexmullins/zip"
func GetCSVFromZipURL(ctx context.Context, fileURL, filePassword string) (ioReader io.Reader, err error) {
span, ctx := tracer.StartSpanFromContext(ctx)
defer span.Finish()
resp, err := http.Get(fileURL)
if err != nil {
return ioReader, errors.AddTrace(err)
}
defer resp.Body.Close()
buf := &bytes.Buffer{}
_, err = io.Copy(buf, resp.Body)
if err != nil {
return ioReader, errors.AddTrace(err)
}
b := bytes.NewReader(buf.Bytes())
r, err := zip.NewReader(b, int64(b.Len()))
if err != nil {
return ioReader, errors.AddTrace(err)
}
for _, f := range r.File {
if f.IsEncrypted() {
f.SetPassword(filePassword)
}
ioReader, err = f.Open()
if err != nil {
return ioReader, errors.AddTrace(err)
}
return ioReader, nil
}
return ioReader, nil
}
func getUserBenefitListFromCSV(ioReader io.Reader) (userBenefitList []UserBenefit, err error) {
reader := csv.NewReader(ioReader)
row := 1
for {
csvRowsStr, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return userBenefitList, errors.AddTrace(err)
}
// check if 1st row (header), skip
if row == 1 {
row++
continue
}
if len(csvRowsStr) > 0 {
userID, err := strconv.ParseInt(csvRowsStr[0], 10, 64)
if err != nil {
return userBenefitList, errors.AddTrace(err)
}
catalogID := 0
if len(csvRowsStr) > 1 {
catalogID, err = strconv.ParseInt(csvRowsStr[1], 10, 64)
if err != nil {
return userBenefitList, errors.AddTrace(err)
}
}
userBenefitTemp := UserBenefit{
UserID: userID,
CatalogID: catalogID,
}
userBenefitList = append(userBenefitList, userBenefitTemp)
}
}
return userBenefitList, nil
}
I want to continuously write json objects to a file. To be able to read it, I need to wrap them into an array. I don't want to read the whole file, for simple appending. So what I' doing now:
comma := []byte(", ")
file, err := os.OpenFile(erp.TransactionsPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
return err
}
transaction, err := json.Marshal(t)
if err != nil {
return err
}
transaction = append(transaction, comma...)
file.Write(transaction)
But with this implementation I will need to add []scopes by hand(or via some script) before reading. How can I add an object before closing scope on each writing?
You don't need to wrap the JSON objects into an array, you can just write them as-is. You may use json.Encoder to write them to the file, and you may use json.Decoder to read them. Encoder.Encode() and Decoder.Decode() encode and decode individual JSON values from a stream.
To prove it works, see this simple example:
const src = `{"id":"1"}{"id":"2"}{"id":"3"}`
dec := json.NewDecoder(strings.NewReader(src))
for {
var m map[string]interface{}
if err := dec.Decode(&m); err != nil {
if err == io.EOF {
break
}
panic(err)
}
fmt.Println("Read:", m)
}
It outputs (try it on the Go Playground):
Read: map[id:1]
Read: map[id:2]
Read: map[id:3]
When writing to / reading from a file, pass the os.File to json.NewEncoder() and json.NewDecoder().
Here's a complete demo which creates a temporary file, uses json.Encoder to write JSON objects into it, then reads them back with json.Decoder:
objs := []map[string]interface{}{
map[string]interface{}{"id": "1"},
map[string]interface{}{"id": "2"},
map[string]interface{}{"id": "3"},
}
file, err := ioutil.TempFile("", "test.json")
if err != nil {
panic(err)
}
// Writing to file:
enc := json.NewEncoder(file)
for _, obj := range objs {
if err := enc.Encode(obj); err != nil {
panic(err)
}
}
// Debug: print file's content
fmt.Println("File content:")
if data, err := ioutil.ReadFile(file.Name()); err != nil {
panic(err)
} else {
fmt.Println(string(data))
}
// Reading from file:
if _, err := file.Seek(0, io.SeekStart); err != nil {
panic(err)
}
dec := json.NewDecoder(file)
for {
var obj map[string]interface{}
if err := dec.Decode(&obj); err != nil {
if err == io.EOF {
break
}
panic(err)
}
fmt.Println("Read:", obj)
}
It outputs (try it on the Go Playground):
File content:
{"id":"1"}
{"id":"2"}
{"id":"3"}
Read: map[id:1]
Read: map[id:2]
Read: map[id:3]
I'm getting JSON resonse from an external API with the following way:
func Request(url string, contentType string) []byte {
resp, err := http.Get(url)
resp.Header.Set("Content-Type", contentType)
if err != nil {
log.Fatal(err)
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
log.Fatal(err)
}
return body
}
url := fmt.Sprintf("https://example.com/api/category/%s", category)
contentType := "application/json"
body := Request(url, contentType)
res := &JSONRespStruct{}
err := json.Unmarshal([]byte(body), res)
if err != nil {
log.Fatal(err)
}
The problem if I start to benchmark my site with go-wrk, the server crashes with the following error message:
2018/01/02 12:13:35 invalid character '<' looking for beginning of value
I think the code try to parse the JSON response as HTML. How I can force to get the response as a JSON?
You probably want to set the header on the request. Setting the header on the response has no impact.
func Request(url string, contentType string) []byte {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", contentType)
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
return body
}
I build a client and a server in golang both are using this functions to encrypt/decrypt
func encrypt(text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
b := base64.StdEncoding.EncodeToString(text)
ciphertext := make([]byte, aes.BlockSize+len(b))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
cfb := cipher.NewCFBEncrypter(block, iv)
cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
return ciphertext, nil
}
func decrypt(text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
if len(text) < aes.BlockSize {
return nil, errors.New("ciphertext too short")
}
iv := text[:aes.BlockSize]
text = text[aes.BlockSize:]
cfb := cipher.NewCFBDecrypter(block, iv)
cfb.XORKeyStream(text, text)
data, err := base64.StdEncoding.DecodeString(string(text))
if err != nil {
return nil, err
}
return data, nil
}
so yeah I make a normal post request
url := "https://"+configuration.Server+configuration.Port+"/get"
// TODO maybe bugs rest here
ciphertext, err := encrypt([]byte(*getUrl))
if err != nil {
fmt.Println("Error: " + err.Error())
}
fmt.Println(string(ciphertext))
values := map[string]interface{}{"url": *getUrl, "urlCrypted": ciphertext}
jsonValue, _ := json.Marshal(values)
jsonStr := bytes.NewBuffer(jsonValue)
req, err := http.NewRequest("POST", url, jsonStr)
and the servercode is as following
requestContent := getRequestContentFromRequest(req)
url := requestContent["url"].(string)
undecryptedUrl := requestContent["urlCrypted"].(string)
decryptedurl, err := decrypt([]byte(undecryptedUrl))
if err != nil {
fmt.Println("Error: " + err.Error())
}
fmt.Println(decryptedurl)
where getRequestContentFromRequest is as following
func getRequestContentFromRequest(req *http.Request)
map[string]interface{} {
buf := new(bytes.Buffer)
buf.ReadFrom(req.Body)
data := buf.Bytes()
var requestContent map[string]interface{}
err := json.Unmarshal(data, &requestContent)
if err != nil {
fmt.Println(err)
}
return requestContent
}
Now to the problem.
If I encrypt my string in the client and decrypt it direct after that everything is fine.
But, when I send the encrypted string to the server and try to decrypt it with literrally the same function as in the client, the decrypt function throws an error.
Error: illegal base64 data at input byte 0
I think the Problem is the unmarshalling of the JSON.
Thanks for help.
P.S.
Repos are
github.com/BelphegorPrime/goSafeClient and github.com/BelphegorPrime/goSafe
UPDATE
Example JSON
{"url":"facebook2.com","urlCrypted":"/}\ufffd\ufffd\ufffdgP\ufffdN뼞\ufffd\u0016\ufffd)\ufffd\ufffd\ufffdy\u001c\u000f\ufffd\ufffd\ufffdep\ufffd\rY\ufffd\ufffd$\ufffd\ufffd"}
UPDATE2
I made a playground here
The problem is that you encode in base64 twice. The first time in the encrypt function and the second time during the JSON marshalling. byte slices are automatically converted into base64 strings by the encoding/json marshaller.
The solution is to decode the base64 string before calling decrypt.
Example on the Go PlayGround
EDIT
Working solution here