I am writing a stats in a csv file for incoming diameter traffic in my golang server but the file contain a "" character at the start of rach row.
01.;34642222231118599998;21;6588283272|6588283272|300|0|46692|1582611861|,|2001|01.;34642222231118599998;21;6588283272|gytwocsdr.circles.asia|circles.asia|0|1|1582611861
****01.;34642222231118599998;22;6588080153|6588080153|300|0|46692|1582611861|,|2001|01.;34642222231118599998;22;6588080153|gytwocsdr.circles.asia|circles.asia|0|1|1582611861
****01.;34642222231118599998;23;6587508893|6587508893|300|0|46692|1582611861|,|2001|01.;34642222231118599998;23;6587508893|gytwocsdr.circles.asia|circles.asia|0|1|1582611861
Please guide me on how to fix this.
stats.go>>
package main
import (
"encoding/csv"
"os"
)
var (
filename = "stats.csv"
)
// write the request and response to csv file
func updateCSVFile(req string, resp string) error {
file, err := os.OpenFile("/home/gyuser/charging-control-engine/stats/stats.csv", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
return err
}
defer file.Close()
// solve the encoding problem
file.WriteString("\xEF\xBB\xBF")
writer := csv.NewWriter(file)
defer writer.Flush()
return writer.Write([]string{req, resp})
}
ccr.go >>>
package main
import (
"log"
"time"
"github.com/fiorix/go-diameter/v4/diam/sm"
"github.com/fiorix/go-diameter/v4/diam"
"github.com/fiorix/go-diameter/v4/diam/avp"
)
const (
// CCRInitial - ccr request type
CCRInitial = 1
// CCRUpdate - ccr request type
CCRUpdate = 2
// CCRTermination - ccr request type
CCRTermination = 3
// VM - flags VM-
VM = avp.Mbit | avp.Vbit
// M - flags M-
M = avp.Mbit
// TGPP - vendor id for TGPP
TGPP = 10415
)
// Decoder for command requests
type Decoder interface {
Decode() (*diam.Message, error)
WriteCSV() error
}
// CCRGxDecoder try to parse the gx requests with ccr
type CCRGxDecoder struct {
msg *diam.Message
gx *GxStruct
r *CCRGxRequest
q *CCRGxResponse
auth bool
start time.Time
}
// handle the Credit Control Request(CCR)
func handleCCR(s *sm.Settings, conf *Config) diam.HandlerFunc {
return func(c diam.Conn, m *diam.Message) {
var decoder Decoder
// application id for Gx and Gy
//if m.Header.ApplicationID == conf.Gx.ID {
// decoder = NewCCRGxDecoder(m, &conf.Gx, conf.Server.Auth)
//} else
if m.Header.ApplicationID == conf.Gy.ID {
decoder = NewCCRGyDecoder(m, &conf.Gy, conf.Server.Auth)
} else {
log.Printf("invalid application id: %v\n", m.Header.ApplicationID)
return
}
// decode the requests and make the answer message
nm, err := decoder.Decode()
if err != nil {
log.Printf("decode failed: %v, %v\n", err, m)
return
}
// write the message back to the peer
if _, err := nm.WriteTo(c); err != nil {
log.Printf("failed to write message: %v\n", err)
return
}
// update the request and response to csv file
if err := decoder.WriteCSV(); err != nil {
log.Printf("write csv: %v\n", err)
}
}
}
file.WriteString("\xEF\xBB\xBF")
Just remove this line from your code. This is the BOM in UTF-8 encoding, which is exactly the same as the <feff> you see.
Related
I was playing with go recently and stuck with a runtime error, I can't explain. These are my working functions.
type User struct {
Browsers []string `json:"browsers"`
Name string `json:"name"`
Email string `json:"email"`
}
func asyncUserProcJson(wg *sync.WaitGroup, users *[]User, ch chan []byte) {
for buf := range ch {
var mu sync.Mutex
var user User
mu.Lock()
err := json.Unmarshal(buf, &user)
mu.Unlock()
if err != nil {
fmt.Println("json:", err)
wg.Done()
continue
}
*users = append(*users, user)
wg.Done()
}
}
func userProcJson(buf []byte) (User, error) {
var user User
err := json.Unmarshal(buf, &user)
if err != nil {
return User{}, err
}
return user, nil
}
If I do a common - non-concurrent aproach, its works as expected. But if, try to use channel to pass bytes to goroutine... it fails.
type AsyncUserProc func(*sync.WaitGroup, *[]User, chan []byte)
type UserProc func(buf []byte) (User, error)
type SearchParams struct {
out io.Writer
asyncUserProc AsyncUserProc
userProc UserProc
}
func (sp SearchParams) AsyncSearch() []User {
file, err := os.Open(filePath)
if err != nil {
log.Fatalln(err)
}
var Users = make([]User, 0, 1024)
var ch = make(chan []byte)
var wg sync.WaitGroup
go sp.asyncUserProcess(&wg, &Users, ch)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
wg.Add(1)
ch <- scanner.Bytes()
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading standard input:", err)
}
close(ch)
wg.Wait()
return Users
}
func (sp SearchParams) Search() []User {
file, err := os.Open(filePath)
if err != nil {
log.Fatalln(err)
}
// json processor
var Users = make([]User, 0, 1024)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
u, err := sp.userProcess(scanner.Bytes())
if err != nil {
log.Panicln(err)
continue
}
Users = append(Users, u)
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading standard input:", err)
}
return Users
}
Workflow is the next one:
filePath contains a JSON chunks (each on new line)
Open for reading.
Create a line scanner
(AsyncSearch)
Pass line to channel.
return value of the line from range (blocking operation)
pass to json.Unmarshal
troubles
(Search)
Pass line directly to userProc func
Enjoy result
I am getting a lot (different) errors.
a lot of json unmarshaling error.
index out of range
JSON decoder out of sync - data changing underfoot?
as description of last error:
// phasePanicMsg is used as a panic message when we end up with something that
// shouldn't happen. It can indicate a bug in the JSON decoder, or that
// something is editing the data slice while the decoder executes.
So here is a question: How the bytes slice is modified?
I thought it was blocking operation. What am I missing in language mechanics?
Example of the errors (different each run)
json: invalid character 'i' looking for beginning of value
json: invalid character ':' after top-level value
json: invalid character 'r' looking for beginning of value
panic: runtime error: index out of range
----
json: invalid character '.' after top-level value
json: invalid character 'K' looking for beginning of value
panic: JSON decoder out of sync - data changing underfoot?
Package bufio
import "bufio"
func (*Scanner) Bytes
func (s *Scanner) Bytes() []byte
Bytes returns the most recent token generated by a call to Scan. The
underlying array may point to data that will be overwritten by a
subsequent call to Scan. It does no allocation.
The underlying array may point to data that will be overwritten by a subsequent call to Scan.
I've been trying to have a " working " file to which i save certain basic state of my application instead of having them in Ram since they would need to be saved everyday, i've decided on creating file per day, this part is working but i've stripped it from the code for more clarity.
Now i'm able to initialise my file with false value for the informations struct and then unmarshalling and reading from it.
The problem arise when i'm trying to update the "file" after it's been unmarshalled before i save it back to the text file.
The isImportStarted does work (when removing the erronous line obv ) but i can't seem to update the file properly i get this error :
./test.go:62:34: cannot assign to struct field
TheList[symbol].ImportStarted in map
./test.go:71:3: cannot take the address of
TheList[symbol].ImportStarted
./test.go:71:34: cannot assign to &TheList[symbol].ImportStarted
My code :
package main
import (
"encoding/json"
"fmt"
"os"
"io/ioutil"
"log"
)
type Informations struct {
ImportStarted bool
ImportDone bool
}
var MyList = map[string]*Informations{
"test": &Informations{ImportStarted: false,ImportDone:false},
"test2": &Informations{ImportStarted: false,ImportDone:false},
}
func ReadFile(filename string) []byte{
data, err := ioutil.ReadFile(filename)
if err != nil {
log.Panicf("failed reading data from file: %s", err)
}
return data
}
func writeFile(json string,filename string){
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
defer file.Close()
if err != nil {
fmt.Println(err)
}
_,err2 := file.WriteString(json)
fmt.Println(err2)
}
func main() {
isImportStarted("test")
ImportStart("test")
}
func ImportStart(symbol string){
filename := "test.txt"
_, err := os.Stat(filename)
if err != nil {
if os.IsNotExist(err) {
fmt.Println("File does not exist creating it...")
file, err := os.Create(filename)
jsonString, _ := json.Marshal(MyList)
writeFile(string(jsonString),filename)
if err != nil {
fmt.Println(err)
}
fmt.Println("reading from file"+filename )
x := ReadFile(filename)
var TheList = map[string]Informations{}
json.Unmarshal(x,&TheList )
TheList[symbol].ImportStarted = true
defer file.Close()
//wanting to save afterwards...
}
} else {
fmt.Println("reading from file "+ filename)
x := ReadFile(filename)
var TheList = map[string]Informations{}
json.Unmarshal(x,&TheList )
&TheList[symbol].ImportStarted = true
}
}
func isImportStarted(symbol string) bool{
filename := "test.txt"
x := ReadFile(filename)
var TheList = map[string]Informations{}
json.Unmarshal(x,&TheList )
return TheList[symbol].ImportStarted
}
I've tried the Why do I get a "cannot assign" error when setting value to a struct as a value in a map? question but it doesn't fit my use case at all as it would effectivly initialize all my structs with nil instead of {false,false}
Any ideas?
Try var TheList = map[string]*Informations{}, why you cannot assign a value in a map please refer to why-do-i-get-a-cannot-assing-error or access-struct-in-map-without-copying
Something tells me I'm not understanding json correctly. I'm trying to grab some data off http://api.bitcoincharts.com/v1/trades.csv?symbol=rockUSD, but my Unmarshal seems to not be able to read the json data. I'm a fresh beginner to golang (and json as well), and I'm wondering how I am able to skip that wrong character error I'm making.
My error:
invalid character ',' after top-level value
panic: invalid character ',' after top-level value
My code:
package main
import ("fmt"
"net/http"
"io/ioutil"
"encoding/json"
)
type Prices struct {
Data string
}
func main() {
url := "http://api.bitcoincharts.com/v1/trades.csv?symbol=rockUSD"
httpresp, err := http.Get(url)
if err != nil{
fmt.Println(err)
panic(err)
}
defer httpresp.Body.Close()
htmldata, err := ioutil.ReadAll(httpresp.Body)
if err != nil{
fmt.Println(err)
panic (err)
}
var jsonData []Prices
err = json.Unmarshal([]byte(htmldata), &jsonData)
if err != nil {
fmt.Println(err)
panic (err)
}
fmt.Println(jsonData)
}
That is NOT json data at all, you'd have to write a custom parser.
Example:
.........
data := readData(httpresp.Body)
........
func readData(r io.Reader) (out [][3]float64) {
br := bufio.NewScanner(r)
for br.Scan() {
parts := strings.Split(br.Text(), ",")
if len(parts) != 3 {
continue
}
var fparts [3]float64
for i, p := range parts {
// bad idea to ignore errors, but it's left as exercise for the reader.
fparts[i], _ = strconv.ParseFloat(p, 64)
}
out = append(out, fparts)
}
return
}
playground
I'm trying to read the content of a bucket on Google Cloud Storage using GO.
I'm able to do that, but is is very slow.
The content of the bucket is like this:
bucket name
-> folders with alphanumeric characters
----> 5 files into each of the folder
--------> each file has a json array inside
what I want to do is to inspect the content of the jsons files for all the folders in the bucket and look for a specific value. The following code work, but it is very slow:
package backend
import (
"encoding/json"
"fmt"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/appengine"
"google.golang.org/appengine/file"
"google.golang.org/appengine/urlfetch"
"google.golang.org/cloud"
"google.golang.org/cloud/storage"
"io"
"io/ioutil"
"net/http"
)
var bucket = "bucket_Name"
type jsonStruct struct {
Gender string `json:"gender"`
Age string `json:"age"`
ID string `json:"id"`
Done int `json:"done"`
}
type saveData struct {
c context.Context
r *http.Request //http response
w http.ResponseWriter //http writer
ctx context.Context
cleanUp []string // cleanUp is a list of filenames that need cleaning up at the end of the saving.
failed bool // failed indicates that one or more of the saving steps failed.
}
func init() {
http.HandleFunc("/", handleStatic)
http.HandleFunc("/listBuckets", listBuckets)
}
func handleStatic(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache")
http.ServeFile(w, r, "static/"+r.URL.Path)
}
func listBuckets(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
if bucket == "" {
var err error
if bucket, err = file.DefaultBucketName(c); err != nil {
// log.Errorf(c, "failed to get default GCS bucket name: %v", err)
return
}
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
// structure to holds information needed to run the various saving functions
d := &saveData{
c: c,
r: r,
w: w,
ctx: ctx,
}
d.listBucket(bucket)
}
func (d *saveData) errorf(format string, args ...interface{}) {
d.failed = true
// log.Errorf(d.c, format, args...)
}
func (d *saveData) listBucket(bucket string) {
io.WriteString(d.w, "\nListbucket result:\n")
query := &storage.Query{}
for query != nil {
objs, err := storage.ListObjects(d.ctx, bucket, query)
if err != nil {
d.errorf("listBucket: unable to list bucket %q: %v", bucket, err)
return
}
query = objs.Next
for _, obj := range objs.Results {
d.readFile(obj.Name)
}
}
}
func (d *saveData) readFile(fileName string) {
rc, err := storage.NewReader(d.ctx, bucket, fileName)
if err != nil {
d.errorf("readFile: unable to open file from bucket %q, file %q: %v", bucket, fileName, err)
return
}
defer rc.Close()
slurp, err := ioutil.ReadAll(rc)
if err != nil {
d.errorf("readFile: unable to read data from bucket %q, file %q: %v", bucket, fileName, err)
return
}
var userDetails jsonStruct
err1 := json.Unmarshal(slurp, &userDetails)
if err1 != nil {
d.errorf("readFile: %v", err1)
return
}
fmt.Fprintf(d.w, "done is: %v\n", userDetails.Done)
}
Basically now I'm reading the folder name from the bucket and then I read the content using the folder name. It would be possible to cache all the bucket content in a go variable and then work on that variable instead of read the bucket for each folder?
I really need this to be faster because I need to present the result back in real time.
Thanks a lot
See below a simple Go sample code to list the bucket content on Google Cloud Storage:
package main
import (
"context"
"fmt"
"log"
"os"
"cloud.google.com/go/storage"
"google.golang.org/api/iterator"
)
func main() {
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS",
"C:\\Users\\Shubham Snehi\\Downloads\\awacs-dev-160bf0e57dc1.json")
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// Sets the name for the new bucket.
bucketName := "balatestawacs"
// Creates a Bucket instance.
bucket := client.Bucket(bucketName)
it := bucket.Objects(ctx, nil)
for {
attrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
panic(err)
}
fmt.Println(attrs.Owner)
}
}
I have very recently started playing with GO and I am trying to unmarshal a JSON response from http://www.oref.org.il/WarningMessages/alerts.json.
For some reason that I can't understand the unmarshaling failed, the unmarshalled struct is empty (my guess is that it is somehow related to encoding).
Below is the code, any help is appreciated.
Thanks,
Itay
package main
import (
"fmt"
"io/ioutil"
"net/http"
"encoding/json"
)
const alertsUrl = "http://www.oref.org.il/WarningMessages/alerts.json"
type Record struct {
Id string `json:id`
Title string `json:title`
Data []string `json:data`
}
func main() {
client := &http.Client{}
req, err := http.NewRequest("GET", alertsUrl, nil)
perror(err)
req.Header.Add("Content-Type", "application/json; charset=utf-8")
res, err := client.Do(req)
perror(err)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
perror(err)
var record Record
json.Unmarshal(body, &record)
fmt.Println(record)
}
func perror(err error) {
if err != nil {
panic(err)
}
}
You're ignoring the error on JSON Unmarshal:
func Unmarshal(data []byte, v interface{}) error
See that it returns an error. Adding that in,
err = json.Unmarshal(body, &record)
perror(err)
It looks like this is a Unicode error — you need to decode the UTF-16 data.
How should you do this? Take a look at this answer. Basically once you read the body like body, err := ioutil.ReadAll(res.Body), you want to decode the UTF-16 bytes to a string. There is a lot going on there, but we can take some liberties: for instance, pulling up the URL in Chrome, the browser tells us that it is UTF-16 LE. So we can skip the ByteOrder detection. So the key here is this function:
func UTF16BytesToString(b []byte, o binary.ByteOrder) string {
utf := make([]uint16, (len(b)+(2-1))/2)
for i := 0; i+(2-1) < len(b); i += 2 {
utf[i/2] = o.Uint16(b[i:])
}
if len(b)/2 < len(utf) {
utf[len(utf)-1] = utf8.RuneError
}
return string(utf16.Decode(utf))
}
Knowing our byte order and passing it in, this will convert the naïve byte array to a string of UTF-16 characters. Thanks to user OneOfOne's comment, we can also detect the BOM easily.
The result:
package main
import (
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"unicode/utf16"
"unicode/utf8"
)
const alertsUrl = "http://www.oref.org.il/WarningMessages/alerts.json"
type Record struct {
Id string `json:id`
Title string `json:title`
Data []string `json:data`
}
// LazyUTF16BytesToString converts UTF-16 encoded bytes, in big or little endian byte order,
// to a UTF-8 encoded string.
func LazyUTF16BytesToString(b []byte) string {
if len(b)%2 != 0 {
panic("len(b) % 2 != 0")
}
var codec binary.ByteOrder = binary.LittleEndian
if b[0] == 0xFE && b[1] == 0xFF { //check and strip the BOM
b = b[2:]
codec = binary.BigEndian
} else if b[0] == 0xFF && b[1] == 0xFE {
b = b[2:]
}
utf := make([]uint16, (len(b)+(2-1))/2)
for i := 0; i+(2-1) < len(b); i += 2 {
utf[i/2] = codec.Uint16(b[i:])
}
if len(b)/2 < len(utf) {
utf[len(utf)-1] = utf8.RuneError
}
return string(utf16.Decode(utf))
}
func main() {
client := &http.Client{}
req, err := http.NewRequest("GET", alertsUrl, nil)
perror(err)
req.Header.Add("Content-Type", "application/json; charset=utf-8")
res, err := client.Do(req)
perror(err)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
perror(err)
bodyString := LazyUTF16BytesToString(body)
var record Record
err = json.Unmarshal([]byte(bodyString), &record)
perror(err)
fmt.Println(record)
}
func perror(err error) {
if err != nil {
panic(err)
}
}