Go Concurrent Write to csv: "slice bounds out of range" - csv

Question

I'm trying to extract the certificates informations from a list of websites and write it to a csv.
I keep getting into the same error but not always at the same time and on different domains.
I get the error at line 63: err := writer.Write(data)
main.analyzeDomains(0xc0000840c0, 0xc0000126c0)
/root/BreakCert/SSLCert/src/main.go:95 +0x5f
created by main.main
/root/BreakCert/SSLCert/src/main.go:113 +0x1bf
panic: runtime error: slice bounds out of range
goroutine 35 [running]:
bufio.(*Writer).Flush(0xc000024140, 0x400002400, 0x0)
/usr/local/go/src/bufio/bufio.go:590 +0x1c0
bufio.(*Writer).WriteByte(0xc000024140, 0xc0000aa92c, 0xc000452500, 0x4d1)
/usr/local/go/src/bufio/bufio.go:645 +0x96
bufio.(*Writer).WriteRune(0xc000024140, 0xc00000002c, 0x4d1, 0x4d1, 0x0)
/usr/local/go/src/bufio/bufio.go:657 +0x1aa
encoding/csv.(*Writer).Write(0xc0000126c0, 0xc00060a000, 0x5, 0x8, 0x2, 0x1a)
/usr/local/go/src/encoding/csv/writer.go:47 +0x4b8
main.storeCertificate(0xc00018cb00, 0xc0000126c0, 0xc000396380, 0x12)
/root/BreakCert/SSLCert/src/main.go:63 +0x3e9
main.analyzeDomain(0xc000396380, 0x12, 0xc0000126c0)
/root/BreakCert/SSLCert/src/main.go:88 +0x19d
main.analyzeDomains(0xc0000840c0, 0xc0000126c0)
/root/BreakCert/SSLCert/src/main.go:95 +0x5f
created by main.main
/root/BreakCert/SSLCert/src/main.go:113 +0x1bf
exit status 2
My code is
package main
import (
"bufio"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/csv"
"fmt"
"log"
"net"
"os"
"strconv"
"strings"
"sync"
"time"
)
type CsvWriter struct {
mutex *sync.Mutex
csvWriter *csv.Writer
}
func NewCsvWriter(fileName string) (*CsvWriter, error) {
csvFile, err := os.Create(fileName)
if err != nil {
return nil, err
}
w := csv.NewWriter(csvFile)
return &CsvWriter{csvWriter: w, mutex: &sync.Mutex{}}, nil
}
func (w *CsvWriter) Write(row []string) {
w.mutex.Lock()
w.csvWriter.Write(row)
w.mutex.Unlock()
}
func (w *CsvWriter) Flush() {
w.mutex.Lock()
w.csvWriter.Flush()
w.mutex.Unlock()
}
func storeCertificate(cert *x509.Certificate, writer *csv.Writer, domain string) {
if v := cert.PublicKeyAlgorithm.String(); v == "RSA" {
if len(cert.Issuer.Organization) != 0 {
var data []string
// Get Issuer Organization
data = append(data, domain[:len(domain)-4])
data = append(data, cert.Issuer.Organization[0])
rsaPublicKey := cert.PublicKey.(*rsa.PublicKey)
if rsaPublicKey != nil {
data = append(data, rsaPublicKey.N.String())
data = append(data, strconv.Itoa(rsaPublicKey.E))
data = append(data, strconv.Itoa(rsaPublicKey.Size()))
fmt.Println("Done: ", domain)
if 6 <= len(data) {
data = data[:5]
}
err := writer.Write(data)
if err != nil {
log.Fatal(err)
}
}
}
}
}
func analyzeDomain(domain string, writer *csv.Writer) {
//fmt.Println("analyzing", domain)
dialer := net.Dialer{}
dialer.Timeout = 10 * time.Second
conn, err := tls.DialWithDialer(&dialer, "tcp", domain, &tls.Config{
InsecureSkipVerify: true,
})
if err != nil {
fmt.Println(fmt.Sprintf("\x1b[31;1mfailed to connect to %s", domain), err, "\x1b[0m")
return
}
defer conn.Close()
for _, cert := range conn.ConnectionState().PeerCertificates {
storeCertificate(cert, writer, domain)
}
}
func analyzeDomains(queue chan string, writer *csv.Writer) {
for {
domain := <-queue
analyzeDomain(domain, writer)
}
}
func main() {
// Creates a channel
cs := make(chan string)
// Creates result.csv
file, err := os.Create("result.csv")
//Verifies that the file has been created
checkError("Cannot create file", err)
defer file.Close()
writer := csv.NewWriter(file)
for i := 0; i < 80; i++ {
go analyzeDomains(cs, writer)
}
writer.Flush()
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := scanner.Text()
if !strings.Contains(line, ":") {
line = line + ":443"
}
cs <- line
}
time.Sleep(2 * time.Second)
}
func checkError(message string, err error) {
if err != nil {
log.Fatal(message, err)
}
}
and is used like that
cat domains | go run main.go
domains contains one url per line.

Answer 1

this is one solution to OP problem
// echo -e "google.com\ncnn.com\nstackoverflow.com" | go run main.go
package main
import (
"bufio"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/csv"
"io"
"log"
"net"
"os"
"strconv"
"strings"
"sync"
"time"
)
func certToCSV(cert *x509.Certificate, domain string) []string {
var data []string
data = append(data, domain[:len(domain)-4])
var org string
if len(cert.Issuer.Organization) > 0 {
org = cert.Issuer.Organization[0]
}
data = append(data, org)
if cert.PublicKey != nil {
rsaPublicKey := cert.PublicKey.(*rsa.PublicKey)
data = append(data, rsaPublicKey.N.String())
data = append(data, strconv.Itoa(rsaPublicKey.E))
data = append(data, strconv.Itoa(rsaPublicKey.Size()))
}
return data
}
func getCerts(d string) ([]*x509.Certificate, error) {
out := []*x509.Certificate{}
dialer := net.Dialer{}
dialer.Timeout = 10 * time.Second
conn, err := tls.DialWithDialer(&dialer, "tcp", d, &tls.Config{
InsecureSkipVerify: true,
})
if err != nil {
return out, err
}
defer conn.Close()
for _, cert := range conn.ConnectionState().PeerCertificates {
if v := cert.PublicKeyAlgorithm.String(); v != "RSA" {
log.Printf("%q not using RSA algorithm but %q", d, cert.PublicKeyAlgorithm)
continue
}
if len(cert.Issuer.Organization) < 1 {
log.Printf("%q does not have organization", d)
continue
}
out = append(out, cert)
}
return out, err
}
func analyze(dst chan []string, src chan string, errs chan error) {
for domain := range src {
certs, err := getCerts(domain)
if err != nil {
errs <- err
continue
}
for _, cert := range certs {
record := certToCSV(cert, domain)
dst <- record
}
}
}
func readCSVFile(dst chan string, fp string) error {
file, err := os.Create(fp)
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := scanner.Text()
if !strings.Contains(line, ":") {
line = line + ":443"
}
dst <- line
}
return scanner.Err()
}
func readCSV(dst chan string, src io.Reader) error {
scanner := bufio.NewScanner(src)
for scanner.Scan() {
line := scanner.Text()
if !strings.Contains(line, ":") {
line = line + ":443"
}
dst <- line
}
return scanner.Err()
}
func writeCSV(dst io.Writer, src chan []string, errs chan error) {
w := csv.NewWriter(dst)
for record := range src {
if err := w.Write(record); err != nil {
errs <- err
}
w.Flush()
}
if err := w.Error(); err != nil {
errs <- err
}
}
func main() {
var wg sync.WaitGroup
errs := make(chan error)
src := make(chan string)
t1 := make(chan []string)
// synchronize all routines to close errs once
go func() {
wg.Wait()
close(errs)
}()
var wg2 sync.WaitGroup
// analyze multiple domains in //
for i := 0; i < 4; i++ {
wg.Add(1)
wg2.Add(1)
go func() {
defer wg.Done()
defer wg2.Done()
analyze(t1, src, errs)
}()
}
// synchronize with analyze routines to close t1
go func() {
wg2.Wait()
close(t1)
}()
// write the csv file
wg.Add(1)
go func() {
defer wg.Done()
writeCSV(os.Stdout, t1, errs)
}()
// read the csv, fail if an error occurs reading the source
wg.Add(1)
go func() {
defer wg.Done()
err := readCSV(src, os.Stdin)
if err != nil {
log.Fatal(err)
}
close(src)
}()
// read and print errors, adjust exit code
var exitCode int
for err := range errs {
log.Println(err)
exitCode = 1
}
os.Exit(exitCode)
}