Golang Encode/Decode base64 with json post doesn't work - json

I build a client and a server in golang both are using this functions to encrypt/decrypt
func encrypt(text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
b := base64.StdEncoding.EncodeToString(text)
ciphertext := make([]byte, aes.BlockSize+len(b))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
cfb := cipher.NewCFBEncrypter(block, iv)
cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
return ciphertext, nil
}
func decrypt(text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
if len(text) < aes.BlockSize {
return nil, errors.New("ciphertext too short")
}
iv := text[:aes.BlockSize]
text = text[aes.BlockSize:]
cfb := cipher.NewCFBDecrypter(block, iv)
cfb.XORKeyStream(text, text)
data, err := base64.StdEncoding.DecodeString(string(text))
if err != nil {
return nil, err
}
return data, nil
}
so yeah I make a normal post request
url := "https://"+configuration.Server+configuration.Port+"/get"
// TODO maybe bugs rest here
ciphertext, err := encrypt([]byte(*getUrl))
if err != nil {
fmt.Println("Error: " + err.Error())
}
fmt.Println(string(ciphertext))
values := map[string]interface{}{"url": *getUrl, "urlCrypted": ciphertext}
jsonValue, _ := json.Marshal(values)
jsonStr := bytes.NewBuffer(jsonValue)
req, err := http.NewRequest("POST", url, jsonStr)
and the servercode is as following
requestContent := getRequestContentFromRequest(req)
url := requestContent["url"].(string)
undecryptedUrl := requestContent["urlCrypted"].(string)
decryptedurl, err := decrypt([]byte(undecryptedUrl))
if err != nil {
fmt.Println("Error: " + err.Error())
}
fmt.Println(decryptedurl)
where getRequestContentFromRequest is as following
func getRequestContentFromRequest(req *http.Request)
map[string]interface{} {
buf := new(bytes.Buffer)
buf.ReadFrom(req.Body)
data := buf.Bytes()
var requestContent map[string]interface{}
err := json.Unmarshal(data, &requestContent)
if err != nil {
fmt.Println(err)
}
return requestContent
}
Now to the problem.
If I encrypt my string in the client and decrypt it direct after that everything is fine.
But, when I send the encrypted string to the server and try to decrypt it with literrally the same function as in the client, the decrypt function throws an error.
Error: illegal base64 data at input byte 0
I think the Problem is the unmarshalling of the JSON.
Thanks for help.
P.S.
Repos are
github.com/BelphegorPrime/goSafeClient and github.com/BelphegorPrime/goSafe
UPDATE
Example JSON
{"url":"facebook2.com","urlCrypted":"/}\ufffd\ufffd\ufffdgP\ufffdN뼞\ufffd\u0016\ufffd)\ufffd\ufffd\ufffdy\u001c\u000f\ufffd\ufffd\ufffdep\ufffd\rY\ufffd\ufffd$\ufffd\ufffd"}
UPDATE2
I made a playground here

The problem is that you encode in base64 twice. The first time in the encrypt function and the second time during the JSON marshalling. byte slices are automatically converted into base64 strings by the encoding/json marshaller.
The solution is to decode the base64 string before calling decrypt.
Example on the Go PlayGround
EDIT
Working solution here

Related

How can I use AES encryption on top of JSON marshalling using Go?

I tried to use the following tutorial:
https://golangdocs.com/aes-encryption-decryption-in-golang
In order to encrypt/decrypt text using AES256 with Go,
It seems to work with plain strings, but not with JSON encoded structure.
I don't understand why, because I thought JSON encoded data were strings as well.
The part of the code dealing with plain strings is commented with Using trings.
// Using strings
pt := "This is a secret"
c := EncryptAES([]byte(key), []byte(pt))
fmt.Printf("Initial string: %#v\n", pt)
fmt.Printf("Coded: %v\n", c)
decoded := DecryptAES([]byte(key), c)
fmt.Printf("Decoded: %s\n", decoded)
The part of the code after the comment Using JSON strings is the part which doesn't seem to word as expected.
// Using JSON strings
p2 := []record{{Name: "John", Age: 20}, {Name: "Jane", Age: 25}}
m2, _ := json.Marshal(p2)
fmt.Printf("m2 = %s\n", string(m2))
fmt.Printf("m2 = %#v\n", string(m2))
coded := EncryptAES([]byte(key), m2)
decoded = DecryptAES([]byte(key), coded)
fmt.Printf("Decoded: %s\n", decoded)
What am I doing wrong?
I'm using Go: go version go1.18 darwin/arm64
package main
import (
"crypto/aes"
"encoding/json"
"fmt"
)
func CheckError(err error) {
if err != nil {
panic(err)
}
}
type record struct {
Name string `json:"first_name"`
Age int `json:"age"`
}
func main() {
// cipher key
key := "thisis32bitlongpassphraseimusing"
fmt.Printf("len of key %d\n", len(key))
// Using strings
pt := "This is a secret"
c := EncryptAES([]byte(key), []byte(pt))
fmt.Printf("Initial string: %#v\n", pt)
fmt.Printf("Coded: %v\n", c)
decoded := DecryptAES([]byte(key), c)
fmt.Printf("Decoded: %s\n", decoded)
// Using JSON strings
p2 := []record{{Name: "John", Age: 20}, {Name: "Jane", Age: 25}}
m2, _ := json.Marshal(p2)
fmt.Printf("m2 = %s\n", string(m2))
fmt.Printf("m2 = %#v\n", string(m2))
coded := EncryptAES([]byte(key), m2)
decoded = DecryptAES([]byte(key), coded)
fmt.Printf("Decoded: %s\n", decoded)
}
func EncryptAES(key []byte, plaintext []byte) []byte {
c, err := aes.NewCipher(key)
CheckError(err)
out := make([]byte, len(plaintext))
c.Encrypt(out, []byte(plaintext))
return out
}
func DecryptAES(key []byte, ct []byte) []byte {
c, err := aes.NewCipher(key)
CheckError(err)
pt := make([]byte, len(ct))
c.Decrypt(pt, ct)
return pt
}
Here is a working implementation of the encryptFile and decryptFile functions:
(Based on: https://medium.com/#mertkimyonsen/encrypt-a-file-using-go-f1fe3bc7c635)
func encryptFile(key []byte, plainText []byte) []byte {
// Creating block of algorithm
block, err := aes.NewCipher(key)
if err != nil {
log.Fatalf("cipher err: %v", err.Error())
}
// Creating GCM mode
gcm, err := cipher.NewGCM(block)
if err != nil {
log.Fatalf("cipher GCM err: %v", err.Error())
}
// Generating random nonce
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
log.Fatalf("nonce err: %v", err.Error())
}
// Decrypt file
cipherText := gcm.Seal(nonce, nonce, plainText, nil)
return cipherText
}
func decryptFile(key []byte, cipherText []byte) []byte {
// Creating block of algorithm
block, err := aes.NewCipher(key)
if err != nil {
log.Fatalf("cipher err: %v", err.Error())
}
// Creating GCM mode
gcm, err := cipher.NewGCM(block)
if err != nil {
log.Fatalf("cipher GCM err: %v", err.Error())
}
// Deattached nonce and decrypt
nonce := cipherText[:gcm.NonceSize()]
cipherText = cipherText[gcm.NonceSize():]
plainText, err := gcm.Open(nil, nonce, cipherText, nil)
if err != nil {
log.Fatalf("decrypt file err: %v", err.Error())
}
return plainText
}

Parsing GeoJson data in golang

I want to parse a JSON file that contains a feature collection of a country's regions.
I am using this package https://github.com/tidwall/geojson
I opened the file like this:
jsonFile, err := os.Open("filename.json")
if err != nil {
return nil, err
}
defer jsonFile.Close()
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
And then I parse the file using this:
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
but it returns a single geojson.Object and I want a list of features
Can someone help me with this
Problem solved
// open json file
jsonFile, err := os.Open(filename)
if err != nil {
return nil, err
}
defer jsonFile.Close()
// read the file
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
// parse into a single geojson.Object
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
// typecast geojson.Object into geojson.FeatureCollection
fc, ok := obj.(*geojson.FeatureCollection)
if !ok {
return nil, errors.Newf(errors.Internal, nil, "cannot convert into feature collection")
}

Exporting JSON into single file from loop function

I wrote some code which hits one public API and saves the JSON output in a file. But the data is storing line by line into the file instead of a single JSON format.
For eg.
Current Output:
{"ip":"1.1.1.1", "Country":"US"}
{"ip":"8.8.8.8", "Country":"IN"}
Desired Output:
[
{"ip":"1.1.1.1", "Country":"US"},
{"ip":"8.8.8.8", "Country":"IN"}
]
I know this should be pretty simple and i am missing out something.
My Current Code is:
To read IP from file and hit the API one by one on each IP.
func readIPfromFile(filename string, outFile string, timeout int) {
data := jsonIn{}
//open input file
jsonFile, err := os.Open(filename) //open input file
...
...
jsonData := bufio.NewScanner(jsonFile)
for jsonData.Scan() {
// marshal json data & check for logs
if err := json.Unmarshal(jsonData.Bytes(), &data); err != nil {
log.Fatal(err)
}
//save to file
url := fmt.Sprintf("http://ipinfo.io/%s", data.Host)
GetGeoIP(url, outFile, timeout)
}
}
To make HTTP Request with custom request header and call write to file function.
func GetGeoIP(url string, outFile string, timeout int) {
geoClient := http.Client{
Timeout: time.Second * time.Duration(timeout), // Timeout after 5 seconds
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("accept", "application/json")
res, getErr := geoClient.Do(req)
if getErr != nil {
log.Fatal(getErr)
}
if res.Body != nil {
defer res.Body.Close()
}
body, readErr := ioutil.ReadAll(res.Body)
if readErr != nil {
log.Fatal(readErr)
}
jsonout := jsonOut{}
jsonErr := json.Unmarshal(body, &jsonout)
if jsonErr != nil {
log.Fatal(jsonErr)
}
file, _ := json.Marshal(jsonout)
write2file(outFile, file)
}
To Write data to file:
func write2file(outFile string, file []byte) {
f, err := os.OpenFile(outFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
log.Fatal(err)
}
defer f.Close()
if _, err = f.WriteString(string(file)); err != nil {
log.Fatal(err)
}
if _, err = f.WriteString("\n"); err != nil {
log.Fatal(err)
}
I know, i can edit f.WriteString("\n"); to f.WriteString(","); to add comma but still adding [] in the file is challenging for me.
First, please do not invent a new way of json marshaling, just use golang built-in encoding/json or other library on github.
Second, if you want to create a json string that represents an array of object, you need to create the array of objects in golang and marshal it into string (or more precisely, into array of bytes)
I create a simple as below, but please DIY if possible.
https://go.dev/play/p/RR_ok-fUTb_4

Print/Log full unstructured json from stream in Go

I inherited someone else's code for an API and since I'm not familiar with the requests that it's receiving I'm trying to print them or log them so I can see their structure. From what I've read about Go, jsons are decoded with Structs but since I don't know how the requests are received I cant write a struct.
I've tried the following on a basic API but they just print me out an empty map or nothing at all:
func createBook(w http.ResponseWriter, r *http.Request) {
var result map[string]interface{}
_ = json.NewDecoder(r.Body).Decode(&result)
fmt.Println(result)
func createBook(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var book Book
_ = json.NewDecoder(r.Body).Decode(&book)
buf := new(bytes.Buffer)
buf.ReadFrom(r.Body)
newStr := buf.String()
reader := strings.NewReader(newStr)
writter := os.Stdout
dec := json.NewDecoder(reader)
enc := json.NewEncoder(writter)
for {
var m map[string]interface{}
if err := dec.Decode(&m); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
if err := enc.Encode(&m); err != nil {
log.Println(err)
}
fmt.Println(m)
}
book.ID = strconv.Itoa(rand.Intn(100000000)) // Mock ID - not safe
books = append(books, book)
json.NewEncoder(w).Encode(book)
}
Is there any other way that it would print the received json without me knowing the strut beforehand?
Use json.Unmarshal function
import "bytes"
func createBook(w http.ResponseWriter, r *http.Request) {
var result map[string]interface{}
data :+ StreamToByte(r.Body)
err := json.Unmarshal(data, &result)
if err !=nil{
fmt.Println(err) //better to use log
}else
fmt.Println(result)
}
}
func StreamToByte(stream io.Reader) []byte {
buf := new(bytes.Buffer)
buf.ReadFrom(stream)
return buf.Bytes()
}
Refer :
https://appdividend.com/2020/02/28/golang-how-to-convert-json-to-map-in-go/
https://gist.github.com/dixudx/3989284b142414e10352fde9def5c771

Set headers in JSON get request

I'm getting JSON resonse from an external API with the following way:
func Request(url string, contentType string) []byte {
resp, err := http.Get(url)
resp.Header.Set("Content-Type", contentType)
if err != nil {
log.Fatal(err)
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
log.Fatal(err)
}
return body
}
url := fmt.Sprintf("https://example.com/api/category/%s", category)
contentType := "application/json"
body := Request(url, contentType)
res := &JSONRespStruct{}
err := json.Unmarshal([]byte(body), res)
if err != nil {
log.Fatal(err)
}
The problem if I start to benchmark my site with go-wrk, the server crashes with the following error message:
2018/01/02 12:13:35 invalid character '<' looking for beginning of value
I think the code try to parse the JSON response as HTML. How I can force to get the response as a JSON?
You probably want to set the header on the request. Setting the header on the response has no impact.
func Request(url string, contentType string) []byte {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", contentType)
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
return body
}