Converting csv data ([]string) to float64 with strconv then summing the data - csv

Question

I am trying to sum data from a CSV file I created from a previous function. Here is a snippet of the file:
datetime,open,high,low,close,volume
2020-11-09 00:00,69.58,137.45,69.00,100.00,273517274.00
2020-11-10 00:00,104.65,128.80,101.75,107.00,141284399.00
2020-11-11 00:00,109.00,114.45,96.76,98.42,96648953.00
2020-11-12 00:00,95.98,106.60,89.15,90.00,149794913.00
[For context: this is historical price data for Rolls-Royce Holdings stock price from Yahoo finance. I plan to use up to 200 rows].
The problem I am facing is converting the []string data from the CSV to float64. The ParseFloat() function is trying to convert my headings and obviously can't as it is 'invalid syntax'. Here is the error code:
Error converting string: strconv.ParseFloat: parsing "open": invalid syntaxError converting string: strconv.ParseFloat: parsing "high": invalid syntaxError converting string: strconv.ParseFloat: parsing "low": invalid syntaxError converting string: strconv.ParseFloat: parsing "close": invalid syntaxError converting string: strconv.ParseFloat: parsing "volume": invalid syntax&{ 0 0 0 0 0}
My code is below for reference:
package main
import (
"encoding/csv"
"fmt"
"log"
"os"
"strconv"
)
const file = "./rr.csv"
// Data struct is the data from the csv file
type Data struct {
datetime string
open float64
high float64
low float64
close float64
volume float64
}
func readAmounts(r []string) (d *Data, err error) {
d = new(Data)
open := r[1]
d.open, err = strconv.ParseFloat(open, 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
high := r[2]
d.high, err = strconv.ParseFloat(high, 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
low := r[3]
d.low, err = strconv.ParseFloat(low, 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
close := r[4]
d.close, err = strconv.ParseFloat(close, 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
volume := r[5]
d.volume, err = strconv.ParseFloat(volume, 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
return d, nil
}
func main() {
csvFile, err := os.Open(file)
if err != nil {
log.Fatal(err)
}
r := csv.NewReader(csvFile)
lines, err := r.Read()
if err != nil {
log.Fatal(err)
}
data, err := readAmounts(lines)
if err != nil {
fmt.Printf("Error reading file: %v", err)
}
fmt.Println(data)
}
I am just printing the data to see if the ParseFloat() works and then I'll create a function to sum the columns.
So, what I'm asking is; how do I ignore the header line and just read through the numerical lines.
FYI: I've read other answers (eg: FieldsPerRecord) but they haven't worked for my specific problem, as I am then trying to sum the whole columns afterwards.

Answer 1

You can load the file into a CSV, then process row by row the file:
package main
import (
"bytes"
"encoding/csv"
"fmt"
"io/ioutil"
"strconv"
)
const file = "./data.csv"
// Data struct is the data from the csv file
type Data struct {
datetime string
open float64
high float64
low float64
close float64
volume float64
}
func main() {
f, err := ioutil.ReadFile(file)
if err != nil {
panic(err)
}
rawData, err := readCsv(f, ',')
if err != nil {
panic(err)
}
amounts, err := readAmounts(rawData[1:])
if err != nil {
panic(err)
}
fmt.Printf("%+v\n", amounts)
}
func readAmounts(r [][]string) ([]Data, error) {
var d []Data = make([]Data, len(r))
var err error
for i, row := range r {
d[i].datetime = row[0]
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
d[i].open, err = strconv.ParseFloat(row[1], 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
d[i].high, err = strconv.ParseFloat(row[2], 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
d[i].low, err = strconv.ParseFloat(row[3], 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
d[i].close, err = strconv.ParseFloat(row[4], 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
d[i].volume, err = strconv.ParseFloat(row[5], 64)
if err != nil {
fmt.Printf("Error converting string: %v", err)
}
}
return d, nil
}
func readCsv(data []byte, separator rune) ([][]string, error) {
csvReader := csv.NewReader(bytes.NewReader(data))
csvReader.Comma = separator
lines, err := csvReader.ReadAll()
if err != nil {
return nil, err
}
return lines, nil
}
Example of output:
[{datetime:2020-11-09 00:00 open:69.58 high:137.45 low:69 close:100 volume:2.73517274e+08} {datetime:2020-11-10 00:00 open:104.65 high:128.8 low:101.75 close:107 volume:1.41284399e+08} {datetime:2020-11-11 00:00 open:109 high:114.45 low:96.76 close:98.42 volume:9.6648953e+07} {datetime:2020-11-12 00:00 open:95.
98 high:106.6 low:89.15 close:90 volume:1.49794913e+08}]
NOTE:
You can find some example of code that work using the CSV library, you can view the following repositories: https://github.com/alessiosavi/GoSFTPtoS3

Answer 2

I've commented out the program so that's easy to understand. Basic idea is to ignore the header. Also, as you're indexing and getting the fields of the record; it's better to put a check for the number of fields that are present in the record (FieldsPerRecord).
package main
import (
"encoding/csv"
"errors"
"fmt"
"io"
"log"
"os"
"strconv"
)
// file stores the filepath
const file = "./rr.csv"
// Data store metadata
type Data struct {
datetime string
open float64
high float64
low float64
close float64
volume float64
}
// s2f converts string to float64
func s2f(str string) (float64, error) {
f, err := strconv.ParseFloat(str, 64)
if err != nil {
return 0, fmt.Errorf("Error converting string \"%v\" to float", err)
}
return f, nil
}
// ReadAmounts processes the fields from the record and stores them in Data
func ReadAmounts(r []string) (*Data, error) {
var (
dt = r[0]
open = r[1]
high = r[2]
low = r[3]
close = r[4]
volume = r[5]
d = new(Data)
err error
)
d.datetime = dt
d.open, err = s2f(open)
if err != nil {
return nil, err
}
d.high, err = s2f(high)
if err != nil {
return nil, err
}
d.low, err = s2f(low)
if err != nil {
return nil, err
}
d.close, err = s2f(close)
if err != nil {
return nil, err
}
d.volume, err = s2f(volume)
if err != nil {
return nil, err
}
return d, nil
}
func main() {
// Open the file
file, err := os.Open(file)
if err != nil {
log.Fatalln(err)
}
// CSV Reader
r := csv.NewReader(file)
// Set Options for the reader
{
r.Comma = ',' // Delimiter
r.TrimLeadingSpace = true // Trim the leading spaces
r.FieldsPerRecord = 0 // Rows should have same number of columns as header
r.ReuseRecord = true // Reuse the same backing array (Efficient)
}
// Alternatively, r.ReadAll() could be also used and slicing it using [1:] ignores
// the header as well.
// Ignore header
_, _ = r.Read()
for {
// Read record (one by one)
record, err := r.Read()
if err != nil {
// Exit out. Done!
if errors.Is(err, io.EOF) {
break
}
// Log and continue
log.Printf("Error reading record: %v\n", err)
continue
}
// Process
data, err := ReadAmounts(record)
if err != nil {
// Log and continue
fmt.Printf("Error reading record: %v\n", err)
continue
}
// Print the filled Data struct
fmt.Printf("Record: %+v\n", *data)
}
}
Output:
Record: {datetime:2020-11-09 00:00 open:69.58 high:137.45 low:69 close:100 volume:2.73517274e+08}
Record: {datetime:2020-11-10 00:00 open:104.65 high:128.8 low:101.75 close:107 volume:1.41284399e+08}
Record: {datetime:2020-11-11 00:00 open:109 high:114.45 low:96.76 close:98.42 volume:9.6648953e+07}
Record: {datetime:2020-11-12 00:00 open:95.98 high:106.6 low:89.15 close:90 volume:1.49794913e+08}

Answer 3

Some different options:
Skip parsing the first line. This assumes every file starts with a header.
Skip lines that have parsing errors. The easiest method, but hard to debug when things go wrong.
If the first line has parsing errors, skip it, because it is probably a header row.
On a side note, you should handle errors properly in your code, which you are not currently doing.