I am wondering why my function does not return the lines. I'm using closures and my goal is to display each line from a decoded text. I was able to achieve this using Python.
Here's my Python code:
def get_line():
lines = base64_decode()
index = 0
def closure():
nonlocal index
def go_next():
nonlocal index
next_line = line[index]
index += 1
return next_line
if index != len(lines):
return go_next()
else:
index = 0
return go_next()
return closure
Here's my Go code:
package main
import (
"encoding/base64"
"fmt"
"log"
"strings"
)
func base64Decode() string {
str := "REDACTED"
data, err := base64.StdEncoding.DecodeString(str)
if err != nil {
log.Fatal("Error:", err)
}
return string(data)
}
func getLine(str string) func() string {
i := 0
lines := strings.Split(str, "\n")
return func() string {
if i != len(lines) {
nextLine := lines[i]
i++
return nextLine
}
return ""
}
}
func main() {
fmt.Println(getLine(base64Decode()))
}
What happens when I run this is it only prints: 0x1095850 instead of This is the first line from the text.
You have to invoke the function:
func main() {
fmt.Println(getLine(base64Decode())())
}
Related
I have a json array where it contains some flags as key and I have set the default values for those keys as false. this is my json array.
var flags = map[string]bool{
"terminationFlag": false,
"transferFlag": false,
"jrCancelledFlag": false,
"jrFilledFlag": false,
}
On performing an operation in a for loop, i have to update 1 field in the above json array as true. During the next iteration, it has to update the 2nd field in the json array as true. After all the fields in the json array is set to true, I have to return the json array.
the code i tried:
Keystrings := []string{"terminationReport - 2019-1","transferReport - 2019-1","jrCancelledReport - 2019-1","jrFilledReport - 2019-1"}
fmt.Println("Keystrings ", Keystrings)
for i,value := range Keystrings {
bytesread, err = stub.GetState(value)
var result []string
_ = json.Unmarshal(bytesread, &result)
fmt.Println("result ", result)
if result[0] == "yes"{
fmt.Println("result in if ", result)
flags[i] = true
}
}
Since it's very hard to understand from the question what is being asked, here's a simple attempt at working with similar data as the question, in the hope that you can take the right parts from this sample and adapt them to your issue. Follow the comments in the code to understand what's going on.
package main
import (
"encoding/json"
"fmt"
"log"
)
var jsonBlob = []byte(`["jrCancelledFlag", "yes"]`)
var flags = map[string]bool{
"terminationFlag": false,
"transferFlag": false,
"jrCancelledFlag": false,
"jrFilledFlag": false,
}
func main() {
// Parse jsonBlob into a slice of strings
var parsed []string
if err := json.Unmarshal(jsonBlob, &parsed); err != nil {
log.Fatalf("JSON unmarshal: %s", err)
}
// Expect the slice to be of length 2, first item flag name, second item
// yes/no.
if len(parsed) != 2 {
log.Fatalf("parsed len %d, expected 2", len(parsed))
}
// Assume parsed[0] actually appears in flags... otherwise more error checking
// is needed.
if parsed[1] == "yes" {
flags[parsed[0]] = true
}
// Emit updated flags as json
json, err := json.Marshal(flags)
if err != nil {
log.Fatalf("JSON marshal: %s", err)
}
fmt.Println(string(json))
}
This can be achieved cleaning by using the JSON interface to define your own unmarshaller
https://medium.com/#nate510/dynamic-json-umarshalling-in-go-88095561d6a0
package main
import (
"encoding/json"
"fmt"
"log"
)
var jsonBlob = []byte(`["jrCancelledFlag", "yes"]`)
// Flags ...
type Flags struct {
TerminationFlag bool `json:"terminationFlag,omitempty"`
TransferFlag bool `json:"transferFlag,omitempty"`
JRCancelledFlag bool `json:"jrCancelledFlag,omitempty"`
JRFilledFlag bool `json:"jrFilledFlag,omitempty"`
}
// UnmarshalJSON satisfies the JSON unmarshaller interface
func (f *Flags) UnmarshalJSON(data []byte) error {
var parsed []string
if err := json.Unmarshal(jsonBlob, &parsed); err != nil {
return err
}
if len(parsed)%2 != 0 {
return fmt.Errorf("expected string to be evenly paired")
}
for i := 0; i < len(parsed); i++ {
j := i + 1
if j < len(parsed) {
switch parsed[i] {
case "terminationFlag":
f.TerminationFlag = toBool(parsed[j])
case "transferFlag":
f.TransferFlag = toBool(parsed[j])
case "jrCancelledFlag":
f.JRCancelledFlag = toBool(parsed[j])
case "jrFilledFlag":
f.JRFilledFlag = toBool(parsed[j])
}
}
}
return nil
}
func toBool(s string) bool {
if s == "yes" {
return true
}
return false
}
func main() {
var flags Flags
err := json.Unmarshal(jsonBlob, &flags)
if err != nil {
log.Fatal(err)
}
b, _ := json.Marshal(flags)
fmt.Println(string(b))
}
First of all let me explain the problem.
I have a stream of JSON records coming into my Golang app. It basically forwards these to a data store (InfluxDB). There are some integer values in the JSON, and also some float values. It is essential that these get forwarded to the data store with the original data type. If they don't, there will be type conflicts and the write operation will fail.
The Ruby JSON parser has no trouble doing this:
require 'json'
obj = { "a" => 123, "b" => 12.3 }
parsed = JSON.parse(obj.to_json)
print parsed["a"].class # => Integer
print parsed["b"].class # => Float
The encoding/json package in Golang, does have some trouble (all numbers are parsed as floats):
package main
import "encoding/json"
import "fmt"
func main () {
str := "{\"a\":123,\"b\":12.3}"
var parsed map[string]interface{}
json.Unmarshal([]byte(str), &parsed)
for key, val := range parsed {
switch val.(type) {
case int:
fmt.Println("int type: ", key)
case float64:
fmt.Println("float type: ", key)
default:
fmt.Println("unknown type: ", key)
}
}
}
Which prints:
float type: a
float type: b
I need a way to parse ints as ints, and floats as floats, in the way the Ruby JSON parser does.
It is not feasible in this case to parse everything as strings and check whether or not there is a decimal. Certain values come in as strings such as "123" and I need to push those as strings.
I do not have structs for the parsed objects, nor is that an option. The golang app doesn't actually care about the schema and just forwards the input as it receives it.
I tried the approach outlined here: Other ways of verifying reflect.Type for int and float64 (using reflect) but it did not accurately identify the int:
t := reflect.TypeOf(parsed["a"])
k := t.Kind()
k == reflect.Int // false
k == reflect.Float64 // true
For example, Ruby JSON number types using the general Go mechanism for custom JSON values,
package main
import (
"encoding/json"
"fmt"
"strconv"
)
func main() {
str := `{"a":123,"b":12.3,"c":"123","d":"12.3","e":true}`
var raw map[string]json.RawMessage
err := json.Unmarshal([]byte(str), &raw)
if err != nil {
panic(err)
}
parsed := make(map[string]interface{}, len(raw))
for key, val := range raw {
s := string(val)
i, err := strconv.ParseInt(s, 10, 64)
if err == nil {
parsed[key] = i
continue
}
f, err := strconv.ParseFloat(s, 64)
if err == nil {
parsed[key] = f
continue
}
var v interface{}
err = json.Unmarshal(val, &v)
if err == nil {
parsed[key] = v
continue
}
parsed[key] = val
}
for key, val := range parsed {
fmt.Printf("%T: %v %v\n", val, key, val)
}
}
Playground: https://play.golang.org/p/VmG8IZV4CG_Y
Output:
int64: a 123
float64: b 12.3
string: c 123
string: d 12.3
bool: e true
Another example, Ruby JSON number types using the Go json.Number type,
package main
import (
"encoding/json"
"fmt"
"strings"
)
func main() {
str := `{"a":123,"b":12.3,"c":"123","d":"12.3","e":true}`
var parsed map[string]interface{}
d := json.NewDecoder(strings.NewReader(str))
d.UseNumber()
err := d.Decode(&parsed)
if err != nil {
panic(err)
}
for key, val := range parsed {
n, ok := val.(json.Number)
if !ok {
continue
}
if i, err := n.Int64(); err == nil {
parsed[key] = i
continue
}
if f, err := n.Float64(); err == nil {
parsed[key] = f
continue
}
}
for key, val := range parsed {
fmt.Printf("%T: %v %v\n", val, key, val)
}
}
Playground: https://play.golang.org/p/Hk_Wb0EM-aY
Output:
int64: a 123
float64: b 12.3
string: c 123
string: d 12.3
bool: e true
A working version of #ShudiptaSharma's suggestion.
There exists a type json.Number in json package which has 3 format functions String() string, Float64() (float64, error) and Int64() (int64, error). We can use them to parse integer and float type.
So, I handle json integer parsing in this way:
package main
import "encoding/json"
import "fmt"
import "strings"
import (
"reflect"
)
func main () {
str := "{\"a\":123,\"b\":12.3}"
var parsed map[string]interface{}
d := json.NewDecoder(strings.NewReader(str))
d.UseNumber()
fmt.Println(d.Decode(&parsed))
for key, val := range parsed {
fmt.Println(reflect.TypeOf(val))
fmt.Printf("decoded to %#v\n", val)
switch val.(type) {
case json.Number:
if n, err := val.(json.Number).Int64(); err == nil {
fmt.Println("int64 type: ", key, n)
} else if f, err := val.(json.Number).Float64(); err == nil {
fmt.Println("float64 type: ", key, f)
} else {
fmt.Println("string type: ", key, val)
}
default:
fmt.Println("unknown type: ", key, val)
}
fmt.Println("===============")
}
}
How to check if a given string is in form of multiple json string separated by spaces/newline?
For example,
given: "test" 123 {"Name": "mike"} (3 json concatenated with space)
return: true, since each of item ("test" 123 and {"Name": "mike"}) is a valid json.
In Go, I can write a O(N^2) function like:
// check given string is json or multiple json concatenated with space/newline
func validateJSON(str string) error {
// only one json string
if isJSON(str) {
return nil
}
// multiple json string concatenate with spaces
str = strings.TrimSpace(str)
arr := []rune(str)
start := 0
end := 0
for start < len(str) {
for end < len(str) && !unicode.IsSpace(arr[end]) {
end++
}
substr := str[start:end]
if isJSON(substr) {
for end < len(str) && unicode.IsSpace(arr[end]) {
end++
}
start = end
} else {
if end == len(str) {
return errors.New("error when parsing input: " + substr)
}
for end < len(str) && unicode.IsSpace(arr[end]) {
end++
}
}
}
return nil
}
func isJSON(str string) bool {
var js json.RawMessage
return json.Unmarshal([]byte(str), &js) == nil
}
But this won't work for large input.
There are two options. The simplest, from a coding standpoint, is going to be just to decode the JSON string normally. You can make this most efficient by decoding to an empty struct:
package main
import "encoding/json"
func main() {
input := []byte(`{"a":"b", "c": 123}`)
var x struct{}
if err := json.Unmarshal(input, &x); err != nil {
panic(err)
}
input = []byte(`{"a":"b", "c": 123}xxx`) // This one fails
if err := json.Unmarshal(input, &x); err != nil {
panic(err)
}
}
(playground link)
This method has a few potential drawbacks:
It only works with a single JSON object. That is, a list of objects (as requested in the question) will fail, without additional logic.
As pointed out by #icza in comments, it only works with JSON objects, so bare arrays, numbers, or strings will fail. To accomodate these types, interface{} must be used, which introduces the potential for some serious performance penalties.
The throw-away x value must still be allocated, and at least one reflection call is likely under the sheets, which may introduce a noticeable performance penalty for some workloads.
Given these limitations, my recommendation is to use the second option: loop through the entire JSON input, ignoring the actual contents. This is made simple with the standard library json.Decoder:
package main
import (
"bytes"
"encoding/json"
"io"
)
func main() {
input := []byte(`{"a":"b", "c": 123}`)
dec := json.NewDecoder(bytes.NewReader(input))
for {
_, err := dec.Token()
if err == io.EOF {
break // End of input, valid JSON
}
if err != nil {
panic(err) // Invalid input
}
}
input = []byte(`{"a":"b", "c": 123}xxx`) // This input fails
dec = json.NewDecoder(bytes.NewReader(input))
for {
_, err := dec.Token()
if err == io.EOF {
break // End of input, valid JSON
}
if err != nil {
panic(err) // Invalid input
}
}
}
(playground link)
As Volker mentioned in the comments, use a *json.Decoder to decode all json documents in your input successively:
package main
import (
"encoding/json"
"io"
"log"
"strings"
)
func main() {
input := `"test" 123 {"Name": "mike"}`
dec := json.NewDecoder(strings.NewReader(input))
for {
var x json.RawMessage
switch err := dec.Decode(&x); err {
case nil:
// not done yet
case io.EOF:
return // success
default:
log.Fatal(err)
}
}
}
Try it on the playground: https://play.golang.org/p/1OKOii9mRHn
Try fastjson.Scanner:
s := `"test" 123 {"Name": "mike"}`
var sc fastjson.Scanner
sc.Init(s)
// Iterate over a stream of json objects
for sc.Next() {}
if sc.Error() != nil {
fmt.Println("ok")
} else {
fmt.Println("false")
}
I have a JSON file with a single field that takes a huge amount of space when loaded into memory. The other fields are reasonable, but I'm trying to take care not to load that particular field unless I absolutely have to.
{
"Field1": "value1",
"Field2": "value2",
"Field3": "a very very long string that potentially takes a few GB of memory"
}
When reading that file into memory, I'd want to ignore Field3 (because loading it could crash my app). Here's some code that I would assume does that because it uses io streams rather than passing a []byte type to the Unmarshal command.
package main
import (
"encoding/json"
"os"
)
func main() {
type MyStruct struct {
Field1 string
Field2 string
}
fi, err := os.Open("myJSONFile.json")
if err != nil {
os.Exit(2)
}
// create an instance and populate
var mystruct MyStruct
err = json.NewDecoder(fi).Decode(&mystruct)
if err != nil {
os.Exit(2)
}
// do some other stuff
}
The issue is that the built-in json.Decoder type reads the entire file into memory on Decode before throwing away key-values that don't match the struct's fields (as has been pointed out on StackOverflow before: link).
Are there any ways of decoding JSON in Go without keeping the entire JSON object in memory?
You could write a custom io.Reader that you feed to json.Decoder and that will pre-read your json file and skip that specific field.
The other option is to write your own decoder, more complicated and messy.
//edit it seemed like a fun exercise, so here goes:
type IgnoreField struct {
io.Reader
Field string
buf bytes.Buffer
}
func NewIgnoreField(r io.Reader, field string) *IgnoreField {
return &IgnoreField{
Reader: r,
Field: field,
}
}
func (iF *IgnoreField) Read(p []byte) (n int, err error) {
if n, err = iF.Reader.Read(p); err != nil {
return
}
s := string(p)
fl := `"` + iF.Field + `"`
if i := strings.Index(s, fl); i != -1 {
l := strings.LastIndex(s[0:i], ",")
if l == -1 {
l = i
}
iF.buf.WriteString(s[0:l])
s = s[i+1+len(fl):]
i = strings.Index(s, `"`)
if i != -1 {
s = s[i+1:]
}
for {
i = strings.Index(s, `"`) //end quote
if i != -1 {
s = s[i+1:]
fmt.Println("Skipped")
break
} else {
if n, err = iF.Reader.Read(p); err != nil {
return
}
s = string(p)
}
}
iF.buf.WriteString(s)
}
ln := iF.buf.Len()
if ln >= len(p) {
tmp := iF.buf.Bytes()
iF.buf.Reset()
copy(p, tmp[0:len(p)])
iF.buf.Write(p[len(p):])
ln = len(p)
} else {
copy(p, iF.buf.Bytes())
iF.buf.Reset()
}
return ln, nil
}
func main() {
type MyStruct struct {
Field1 string
Field2 string
}
fi, err := os.Open("myJSONFile.json")
if err != nil {
os.Exit(2)
}
// create an instance and populate
var mystruct MyStruct
err := json.NewDecoder(NewIgnoreField(fi, "Field3")).Decode(&mystruct)
if err != nil {
fmt.Println(err)
}
fmt.Println(mystruct)
}
playground
The problem how to automatically deserialize/unmarshal record from CSV file into Go struct.
For example, I have
type Test struct {
Name string
Surname string
Age int
}
And CSV file contains records
John;Smith;42
Piter;Abel;50
Is there an easy way to unmarshal those records into struct except by using "encoding/csv" package for reading record and then doing something like
record, _ := reader.Read()
test := Test{record[0],record[1],atoi(record[2])}
There is gocarina/gocsv which handles custom struct in the same way encoding/json does.
You can also write custom marshaller and unmarshaller for specific types.
Example:
type Client struct {
Id string `csv:"client_id"` // .csv column headers
Name string `csv:"client_name"`
Age string `csv:"client_age"`
}
func main() {
in, err := os.Open("clients.csv")
if err != nil {
panic(err)
}
defer in.Close()
clients := []*Client{}
if err := gocsv.UnmarshalFile(in, &clients); err != nil {
panic(err)
}
for _, client := range clients {
fmt.Println("Hello, ", client.Name)
}
}
Seems I've done with automatic marshaling of CSV records into structs (limited to string and int). Hope this would be useful.
Here is a link to playground: http://play.golang.org/p/kwc32A5mJf
func Unmarshal(reader *csv.Reader, v interface{}) error {
record, err := reader.Read()
if err != nil {
return err
}
s := reflect.ValueOf(v).Elem()
if s.NumField() != len(record) {
return &FieldMismatch{s.NumField(), len(record)}
}
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
switch f.Type().String() {
case "string":
f.SetString(record[i])
case "int":
ival, err := strconv.ParseInt(record[i], 10, 0)
if err != nil {
return err
}
f.SetInt(ival)
default:
return &UnsupportedType{f.Type().String()}
}
}
return nil
}
I'll try to create github package is someone needs this implementation.
You could bake your own. Perhaps something like this:
package main
import (
"fmt"
"strconv"
"strings"
)
type Test struct {
Name string
Surname string
Age int
}
func (t Test) String() string {
return fmt.Sprintf("%s;%s;%d", t.Name, t.Surname, t.Age)
}
func (t *Test) Parse(in string) {
tmp := strings.Split(in, ";")
t.Name = tmp[0]
t.Surname = tmp[1]
t.Age, _ = strconv.Atoi(tmp[2])
}
func main() {
john := Test{"John", "Smith", 42}
fmt.Printf("john:%v\n", john)
johnString := john.String()
fmt.Printf("johnString:%s\n", johnString)
var rebornJohn Test
rebornJohn.Parse(johnString)
fmt.Printf("rebornJohn:%v\n", rebornJohn)
}
Using csvutil it is possible to give column header see example.
In your case, this could be :
package main
import (
"encoding/csv"
"fmt"
"io"
"os"
"github.com/jszwec/csvutil"
)
type Test struct {
Name string
Surname string
Age int
}
func main() {
csv_file, _ := os.Open("test.csv")
reader := csv.NewReader(csv_file)
reader.Comma = ';'
userHeader, _ := csvutil.Header(Test{}, "csv")
dec, _ := csvutil.NewDecoder(reader, userHeader...)
var users []Test
for {
var u Test
if err := dec.Decode(&u); err == io.EOF {
break
}
users = append(users, u)
}
fmt.Println(users)
}