Read CSV file on protected zip - csv

I have link like this: https://storage.googleapis.com/data/test_file.csv.zip
the content is one csv file on the protected zip with password. How can I read data from the csv?
I have try this but its error
func ReadCSVZIPFromURL(fileURL string) (data [][]string, err error) {
resp, err := http.Get(fileURL)
if err != nil {
return data, errors.AddTrace(err)
}
defer resp.Body.Close()
r, err := zip.OpenReader(resp.Body)
if err != nil {
log.Fatal(err)
}
defer r.Close()
for _, file := range r.File {
if file.IsEncrypted() {
file.SetPassword("password")
}
reader := csv.NewReader(file)
data, err := reader.ReadAll()
}
return data, nil
}

i have solve the problem, this the solve. the idea is we copy the byte to zip reader, and we will get the ioReader, the we we read ioReader use csv library.
to encrypt and decrypt data using password we use this library "github.com/alexmullins/zip"
func GetCSVFromZipURL(ctx context.Context, fileURL, filePassword string) (ioReader io.Reader, err error) {
span, ctx := tracer.StartSpanFromContext(ctx)
defer span.Finish()
resp, err := http.Get(fileURL)
if err != nil {
return ioReader, errors.AddTrace(err)
}
defer resp.Body.Close()
buf := &bytes.Buffer{}
_, err = io.Copy(buf, resp.Body)
if err != nil {
return ioReader, errors.AddTrace(err)
}
b := bytes.NewReader(buf.Bytes())
r, err := zip.NewReader(b, int64(b.Len()))
if err != nil {
return ioReader, errors.AddTrace(err)
}
for _, f := range r.File {
if f.IsEncrypted() {
f.SetPassword(filePassword)
}
ioReader, err = f.Open()
if err != nil {
return ioReader, errors.AddTrace(err)
}
return ioReader, nil
}
return ioReader, nil
}
func getUserBenefitListFromCSV(ioReader io.Reader) (userBenefitList []UserBenefit, err error) {
reader := csv.NewReader(ioReader)
row := 1
for {
csvRowsStr, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return userBenefitList, errors.AddTrace(err)
}
// check if 1st row (header), skip
if row == 1 {
row++
continue
}
if len(csvRowsStr) > 0 {
userID, err := strconv.ParseInt(csvRowsStr[0], 10, 64)
if err != nil {
return userBenefitList, errors.AddTrace(err)
}
catalogID := 0
if len(csvRowsStr) > 1 {
catalogID, err = strconv.ParseInt(csvRowsStr[1], 10, 64)
if err != nil {
return userBenefitList, errors.AddTrace(err)
}
}
userBenefitTemp := UserBenefit{
UserID: userID,
CatalogID: catalogID,
}
userBenefitList = append(userBenefitList, userBenefitTemp)
}
}
return userBenefitList, nil
}

Related

Parsing GeoJson data in golang

I want to parse a JSON file that contains a feature collection of a country's regions.
I am using this package https://github.com/tidwall/geojson
I opened the file like this:
jsonFile, err := os.Open("filename.json")
if err != nil {
return nil, err
}
defer jsonFile.Close()
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
And then I parse the file using this:
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
but it returns a single geojson.Object and I want a list of features
Can someone help me with this
Problem solved
// open json file
jsonFile, err := os.Open(filename)
if err != nil {
return nil, err
}
defer jsonFile.Close()
// read the file
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
// parse into a single geojson.Object
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
// typecast geojson.Object into geojson.FeatureCollection
fc, ok := obj.(*geojson.FeatureCollection)
if !ok {
return nil, errors.Newf(errors.Internal, nil, "cannot convert into feature collection")
}

How to process a request that has multiple inputs and multiple files at the same time

Building a backend go server that can take a form with multiple inputs and 3 of them have multiple file inputs. I searched and it states that if you want to make something like this work you don't want to use the typical
if err := r.ParseMultipartForm(32 << 20); err != nil {
fmt.Println(err)
}
// get a reference to the fileHeaders
files := r.MultipartForm.File["coverArt"]
and instead you should use
mr, err := r.MultipartReader()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
Standard form-data:
Name
Email
Cover art photos (multiple files)
Profile photos (multiple files)
2 Audio files (2 songs)
2 Videos (personal intro, recording of person in a cappella)
HTML Form
<form method="post" enctype="multipart/form-data" action="/upload">
<input type="text" name="name">
<input type="text" name="email">
<input name="coverArt" type="file" multiple />
<input name="profile" type="file" multiple />
<input type="file" name="songs" multiple />
<input type="file" name="videos" multiple/>
<button type="submit">Upload File</button>
</form>
Go Code:
func FilePOST(w http.ResponseWriter, r *http.Request) error {
fmt.Println("File Upload Endpoint Hit")
mr, err := r.MultipartReader()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
for {
part, err := mr.NextPart()
// This is OK, no more parts
if err == io.EOF {
break
}
// Some error
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
// CoverArt 'files' part
if part.FormName() == "coverArt" {
name := part.FileName()
outfile, err := os.Create("uploads/" + name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
defer outfile.Close()
_, err = io.Copy(outfile, part)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
}
// Profile Pic 'files' part
if part.FormName() == "profile" {
name := part.FileName()
outfile, err := os.Create("uploads/" + name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
defer outfile.Close()
_, err = io.Copy(outfile, part)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
}
// Songs 'files' part
if part.FormName() == "songs" {
name := part.FileName()
outfile, err := os.Create("uploads/" + name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
defer outfile.Close()
_, err = io.Copy(outfile, part)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
}
// Video 'files' part
if part.FormName() == "videos" {
name := part.FileName()
outfile, err := os.Create("uploads/" + name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
defer outfile.Close()
_, err = io.Copy(outfile, part)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
// return
}
}
}
fmt.Println("done")
return nil
}
Go Server Error:
go run main.go [15:58:21]
now serving at the following location www.localhost:3000
File Upload Endpoint Hit
INFO[0009] POST /upload elapsed="680.422µs" host= method=POST path=/upload query=
2021/07/14 15:58:32 http: panic serving [::1]:62924: runtime error: invalid memory address or nil pointer dereference
It is hard to guess where your code panics. Probably the reason is that your program continue to execute when error occurs. For example if creation of file fails, outfile.Close() will panic as the outfile is nil.
Both approaches support multiple files for single field. The difference is in how they handle memory. The streaming version reads small portions of data from the network and writes it to a file when you call io.Copy. The other variant loads all the data into memory when you call ParseMultiForm(), so it requires as much memory as the size of the files you want to transfer. Below you will find working examples for both variants.
Streaming variant:
func storeFile(part *multipart.Part) error {
name := part.FileName()
outfile, err := os.Create("uploads/" + name)
if err != nil {
return err
}
defer outfile.Close()
_, err = io.Copy(outfile, part)
if err != nil {
return err
}
return nil
}
func filePOST(w http.ResponseWriter, r *http.Request) error {
fmt.Println("File Upload Endpoint Hit")
mr, err := r.MultipartReader()
if err != nil {
return err
}
for {
part, err := mr.NextPart()
// This is OK, no more parts
switch {
case errors.Is(err, io.EOF):
fmt.Println("done")
return nil
case err != nil:
// Some error
return err
default:
switch part.FormName() {
case "coverArt", "profile", "songs", "videos":
if err := storeFile(part); err != nil {
return err
}
}
}
}
}
func main() {
http.HandleFunc("/upload", func(writer http.ResponseWriter, request *http.Request) {
err := filePOST(writer, request)
if err != nil {
http.Error(writer, err.Error(), http.StatusInternalServerError)
log.Println("Error", err)
}
})
if err := http.ListenAndServe(":8080", nil); err != nil {
log.Fatal(err)
}
}
And version with ParseMultipartForm, which reads data to memory.
func storeFile(part *multipart.FileHeader) error {
name := part.Filename
infile, err := part.Open()
if err != nil {
return err
}
defer infile.Close()
outfile, err := os.Create("uploads/" + name)
if err != nil {
return err
}
defer outfile.Close()
_, err = io.Copy(outfile, infile)
if err != nil {
return err
}
return nil
}
func FilePOST(w http.ResponseWriter, r *http.Request) error {
fmt.Println("File Upload Endpoint Hit")
if err := r.ParseMultipartForm(2 << 24); err != nil {
return err
}
for _, fileType := range []string{"coverArt", "profile", "songs", "videos"} {
uploadedFiles, exists := r.MultipartForm.File[fileType]
if !exists {
continue
}
for _, file := range uploadedFiles {
if err := storeFile(file); err != nil {
return err
}
}
}
return nil
}
func main() {
http.HandleFunc("/upload", func(writer http.ResponseWriter, request *http.Request) {
err := FilePOST(writer, request)
if err != nil {
http.Error(writer, err.Error(), http.StatusInternalServerError)
log.Println("Error", err)
}
})
if err := http.ListenAndServe(":8080", nil); err != nil {
log.Fatal(err)
}
}

Golang JSON weird behaviour

Fairly unremarkable code here:
type Response struct {
Status string `json:"status"`
Msg string `json:"msg"`
ErrorCode string `json:"error-code"`
Data interface{} `json:"data"`
Eof bool `json:"eof"`
}
func main() {
data := [][]string{
[]string{"description", "BILL NO 2362 EXTRA 5%"},
}
r := &Response{
Data: data,
}
str, _ := json.Marshal(r)
fmt.Println(string(str))
}
It prints the following:
{"status":"","msg":"","error-code":"","data":[["description","BILL NO 2362 EXTRA 5%"]],"eof":false}
No problem so far. However if I try to marshal JSON from exactly the same data, but fetched from a DB, I get a weird output:
func TestJson(t *testing.T) {
var pool *sql.DB
pool, err := sql.Open("mysql", "user:pass#tcp(127.0.0.1:3306)/db")
if err != nil {
t.Errorf("%s\n", err.Error())
}
defer pool.Close()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
if err := pool.PingContext(ctx); err != nil {
t.Errorf("%s\n", err.Error())
}
data := fetchAll(t, ctx, pool)
fmt.Println(data)
res := &Response{
Data: data,
}
str, err := json.Marshal(res)
if err != nil {
t.Errorf("%s\n", err.Error())
}
fmt.Printf(string(str))
}
func fetchAll(t *testing.T, ctx context.Context, pool *sql.DB) [][]string {
rows, err := pool.QueryContext(ctx, "select description from vouchers where id = 1")
if err != nil {
t.Fatalf("%s\n", err.Error())
}
cols, err := rows.Columns()
if err != nil {
t.Fatalf("%s\n", err.Error())
}
vals := make([]interface{}, len(cols))
var results [][]string
for rows.Next() {
for i := range cols {
vals[i] = &vals[i]
}
err = rows.Scan(vals...)
if err != nil {
t.Fatalf("%s\n", err.Error())
}
var r []string
for i, c := range cols {
r = append(r, c)
var v string
if vals[i] == nil {
v = "NULL"
} else {
b, _ := vals[i].([]byte)
v = string(b)
}
fmt.Printf("byte len %d\n", len(v))
fmt.Printf("rune len %d\n", utf8.RuneCountInString(v))
r = append(r, v)
}
results = append(results, r)
}
if rows.Err() != nil {
t.Fatalf("%s\n", err.Error())
}
return results
}
The test code prints this:
=== RUN TestJson
byte len 21
rune len 21
[[description BILL NO 2362 EXTRA 5%]]
{"status":"","msg":"","error-code":"","data":[["description","BILL NO 2362 EXTRA 5%!"(MISSING)]],"eof":false}--- PASS: TestJson (0.35s)
PASS
ok command-line-arguments 0.836s
Note the corrupted JSON. I printed the rune length of the string to make sure the string contains
just plain ascii characters. The print before the marshal confirms that we are looking at exactly the same string. Why on earth is the output different ?
If I remove the last "%" then life returns to normal:
byte len 20
rune len 20
[[description BILL NO 2362 EXTRA 5]]
{"status":"","msg":"","error-code":"","data":[["description","BILL NO 2362 EXTRA 5"]],"eof":false}--- PASS: TestJson (0.35s)
PASS
ok command-line-arguments 0.534s
Based on the suggestions modify the program as follows:
func TestJson(t *testing.T) {
var pool *sql.DB
pool, err := sql.Open("mysql", "user:pass#tcp(127.0.0.1:3306)/db")
if err != nil {
t.Errorf("%s\n", err.Error())
}
defer pool.Close()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
if err := pool.PingContext(ctx); err != nil {
t.Errorf("%s\n", err.Error())
}
data := fetchAll(t, ctx, pool)
fmt.Println(data)
res := &Response{
Data: data,
}
str, err := json.Marshal(res)
if err != nil {
t.Errorf("%s\n", err.Error())
}
fmt.Print(string(str))
}
func fetchAll(t *testing.T, ctx context.Context, pool *sql.DB) [][]string {
rows, err := pool.QueryContext(ctx, "select description from vouchers where id = 1")
if err != nil {
t.Fatalf("%s\n", err.Error())
}
cols, err := rows.Columns()
if err != nil {
t.Fatalf("%s\n", err.Error())
}
vals := make([]interface{}, len(cols))
var results [][]string
for rows.Next() {
for i := range cols {
vals[i] = &vals[i]
}
err = rows.Scan(vals...)
if err != nil {
t.Fatalf("%s\n", err.Error())
}
var r []string
for i, c := range cols {
r = append(r, c)
var v string
if vals[i] == nil {
v = "NULL"
} else {
b, _ := vals[i].([]byte)
v = string(b)
}
fmt.Printf("byte len %d\n", len(v))
fmt.Printf("rune len %d\n", utf8.RuneCountInString(v))
r = append(r, v)
}
results = append(results, r)
}
if rows.Err() != nil {
t.Fatalf("%s\n", err.Error())
}
return results
}

Golang error: interface conversion: interface {} is bool/float..., not string

I am trying to decode an arbitrary JSON using Golang, so I unmarshal the incoming JSON in a map[string]interface{} as shown in the code below:
func JsonHandler(jsonRequest []byte) {
// Creating the maps for JSON
var m interface{}
// Parsing/Unmarshalling JSON encoding/json
if err := json.Unmarshal([]byte(jsonRequest), &m); err != nil {
panic(err)
}
//Creating an output file for writing
f, err := os.OpenFile("/home/dorrahadrich/Desktop/output.txt", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
panic(err)
}
defer f.Close()
ParseJson(m, f, err)
}
func ParseJson(m interface{}, f *os.File, err error) {
switch v := m.(interface{}).(type){
case map[string]interface{}:
ParseMap (m.(map[string]interface{}),f,err)
fmt.Println(v)
case []interface{}:
ParseArray (m.([]interface{}),f,err)
fmt.Println(v)
default:
}
}
func ParseMap(aMap map[string]interface{}, f *os.File, err error) {
for key, val := range aMap {
switch val.(type) {
case map[string]interface{}:
if _, err = f.WriteString(key + "={\n"); err != nil {
panic(err)
}
ParseMap(val.(map[string]interface{}), f, err)
//Close brackets
if _, err = f.WriteString("};\n"); err != nil {
panic(err)
}
case []interface{}:
//Write to file
if _, err = f.WriteString(key + "={\n"); err != nil {
panic(err)
}
ParseArray(val.([]interface{}), f, err)
//Close brackets
if _, err = f.WriteString("};\n"); err != nil {
panic(err)
}
default:
otherValues(key, val.(interface{}), f , err)
}
}
}
func ParseArray(anArray []interface{}, f *os.File, err error) {
for _, val := range anArray {
switch val.(type) {
case map[string]interface{}:
ParseMap(val.(map[string]interface{}), f, err)
case []interface{}:
ParseArray(val.([]interface{}), f, err)
default:
}
}
}
func otherValues(key string, other interface{}, f *os.File, err error) {
if _, err = f.WriteString(key); err != nil {
panic(err)
}
if _, err = f.WriteString("="); err != nil {
panic(err)
}
switch other.(interface{}).(type) {
case string:
if _, err = f.WriteString(other.(string)); err != nil {
panic(err)
}
case float64:
if _, err = f.WriteString(strconv.FormatFloat(other.(float64), 'f', -1, 64)); err != nil {
panic(err)
}
case bool:
if _, err = f.WriteString(strconv.FormatBool(other.(bool))); err != nil {
panic(err)
}
default:
}
}
The problem is that whenever a JSON contains a bool/int/float or any not string value the program panics saying that it fails converting an interface to the given type! Please note that the JSON is arbitrary so I don't have any idea about the keys nor the values, I can't unmrashal into an interface nor access the values giving a path.
The error says it all:
interface conversion: interface{} is bool/float64
when you are unmarshalling json the values for int and bool which are not of interface type. In your switch add case for bool/float64/string too. Since json is arbitrary unmarshal them using interface{}.
func otherValues(other interface{}, f *os.File, err error) {
switch bb := other.(interface{}).(type) {
case string:
fmt.Println("This is a string")
case float64:
fmt.Println("this is a float")
case bool:
fmt.Println("this is a boolean")
default:
fmt.Printf("Default value is of type %v", bb)
}
}
Use file.Write in place of file.WriteString
func (f *File) Write(b []byte) (n int, err error)
Write writes len(b) bytes to the File. It returns the number of bytes
written and an error, if any. Write returns a non-nil error when n !=
len(b).

interface conversion: interface {} is int64, not []uint8

I'm trying to implement a go program which can handle http requests and send response in nested JSON. When i run my code and call the URL, I'm getting a runtime error,what does it mean? how can i handle this?
panic serving 192.168.0.101:50760: interface conversion: interface {} is int64, not []uint8
goroutine 5 [running]
This is my function code which is called on hitting the url
func logInPass(res http.ResponseWriter, req *http.Request) {
type Resp struct {
Result []map[string]interface{} `json:"Result,omitempty"`
Status string `json:"Status"`
}
type AxleUser struct {
Mobile string `json:"Mobile"`
Password string `json:"Password"`
}
var Response Resp
Response.Status = "failed"
Result := make(map[string]interface{})
db, err := sql.Open("mysql", "root:chikkIbuddI57#tcp(127.0.0.1:3306)/b2b")
if err != nil {
panic(err.Error())
}
defer db.Close()
rnd := render.New()
b, err := ioutil.ReadAll(req.Body)
defer req.Body.Close()
if err != nil {
panic(err.Error())
}
// Unmarshal the request body
var msg AxleUser
err = json.Unmarshal(b, &msg)
if err != nil {
panic(err.Error())
}
// get shop id from emp table using mobile number and password
userrows, usererr := db.Query("SELECT b2b_emp_id,b2b_shop_id,b2b_shop_name,b2b_emp_name,b2b_emp_mobile_number FROM b2b_employee_tbl WHERE b2b_emp_mobile_number=? and b2b_password=?", msg.Mobile, msg.Password)
if usererr != nil {
panic(usererr.Error())
}
usercolumns, usererr := userrows.Columns()
if usererr != nil {
panic(usererr.Error())
}
usercount := len(usercolumns)
values := make([]interface{}, usercount)
scanArgs := make([]interface{}, usercount)
for i := range values {
scanArgs[i] = &values[i]
}
for userrows.Next() {
usererr := userrows.Scan(scanArgs...)
if usererr != nil {
panic(usererr.Error())
}
for i, v := range values {
if v != nil {
Result[usercolumns[i]] = fmt.Sprintf("%s", string(v.([]byte)))
}
}
Response.Result = append(Response.Result, Result)
Response.Status = "success"
}
res.Header().Set("Content-Type", "application/json")
rnd.JSON(res, http.StatusOK, Response)
}
Thanks in Advance!
I've changed this line
values := make([]interface{}, usercount)
To
values := make([]string, usercount)
And this line
Result[usercolumns[i]] = fmt.Sprintf("%s", string(v.([]byte)))
To
Result[usercolumns[i]] = v