Set headers in JSON get request - json

I'm getting JSON resonse from an external API with the following way:
func Request(url string, contentType string) []byte {
resp, err := http.Get(url)
resp.Header.Set("Content-Type", contentType)
if err != nil {
log.Fatal(err)
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
log.Fatal(err)
}
return body
}
url := fmt.Sprintf("https://example.com/api/category/%s", category)
contentType := "application/json"
body := Request(url, contentType)
res := &JSONRespStruct{}
err := json.Unmarshal([]byte(body), res)
if err != nil {
log.Fatal(err)
}
The problem if I start to benchmark my site with go-wrk, the server crashes with the following error message:
2018/01/02 12:13:35 invalid character '<' looking for beginning of value
I think the code try to parse the JSON response as HTML. How I can force to get the response as a JSON?

You probably want to set the header on the request. Setting the header on the response has no impact.
func Request(url string, contentType string) []byte {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", contentType)
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
return body
}

Related

Parsing GeoJson data in golang

I want to parse a JSON file that contains a feature collection of a country's regions.
I am using this package https://github.com/tidwall/geojson
I opened the file like this:
jsonFile, err := os.Open("filename.json")
if err != nil {
return nil, err
}
defer jsonFile.Close()
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
And then I parse the file using this:
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
but it returns a single geojson.Object and I want a list of features
Can someone help me with this
Problem solved
// open json file
jsonFile, err := os.Open(filename)
if err != nil {
return nil, err
}
defer jsonFile.Close()
// read the file
data, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, err
}
// parse into a single geojson.Object
obj, err := geojson.Parse(string(data), geojson.DefaultParseOptions)
if err != nil {
return nil, err
}
// typecast geojson.Object into geojson.FeatureCollection
fc, ok := obj.(*geojson.FeatureCollection)
if !ok {
return nil, errors.Newf(errors.Internal, nil, "cannot convert into feature collection")
}

golang, post with json fails, but multipart succeed

When I tried to send post with json,
401 error (unauthorized messages) showed.
But with multipart, then succeeded.
In the following code,
I could get the token with only requestTokenByBuff which includes multipart.
Everytime I send post with json, I got failed.
I tried to print json body, and there was no error.
No header with body also made same result(401 error).
I did not test http.Post because Header should be inputed.
Why does post with json fail?
type AuthConfig struct {
ClientId string `json:"client_id"`
ClientSecret string `json:"client_secret"`
Code string `json:"data"`
GrantType string `json:"grant_type"`
}
var host string
func initAuthConfig(a *AuthConfig) {
host = "https://xxxxxxx"
a.ClientId = "xxxxxxx"
a.ClientSecret = "xxxxxxx"
a.Code = "xxxxx"
a.GrantType = "xxxx"
}
func initAuthConfig2() {
host = "https://xxxxxxx"
}
func requestToken(auth *AuthConfig) {
authbytes, _ := json.Marshal(auth)
authbuff := bytes.NewBuffer(authbytes)
client := &http.Client{}
req, err := http.NewRequest("POST", host, authbuff)
if err != nil {
log.Fatalln(err)
}
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.xxxxxx")
req.Header.Add("Cookie", "SCOUTER=xxxxxxx")
req.Header.Add("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
data := string(body)
fmt.Println(data)
}
func requestTokenByBuff(buff *bytes.Buffer) {
writer := multipart.NewWriter(buff)
_ = writer.WriteField("client_id", "xxxxxxxxxx")
_ = writer.WriteField("client_secret", "xxxxxxxx")
_ = writer.WriteField("data", "xxxxxx")
_ = writer.WriteField("grant_type", "xxxxxxx")
err := writer.Close()
if err != nil {
log.Fatalln(err)
return
}
client := &http.Client{}
req, err := http.NewRequest("POST", host, buff)
if err != nil {
log.Fatalln(err)
}
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.xxxx")
req.Header.Add("Cookie", "SCOUTER=xxxxxxxx")
req.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
data := string(body)
fmt.Println(data)
}
func main() {
// config := new(AuthConfig)
// initAuthConfig(config)
// requestToken(config)
buff := &bytes.Buffer{}
initAuthConfig2()
requestTokenByBuff(buff)
}

Get proxy response body as clean string in golang

I read this solution for resolve body data from a proxy.
Golang: how to read response body of ReverseProxy?
But I cannot read the body as a plain string, maybe the encoding is not right or other cryption.
My question is how to encode or transform the body to readable html string?
Currently I get:
n8�������♠�♠�A��b:J���g>-��ˤ���[���.....
Code example:
reverseProxy := httputil.NewSingleHostReverseProxy(url)
reverseProxy.ModifyResponse = func (resp *http.Response) (err error) {
var contentType = resp.Header.Get("Content-Type")
if strings.HasPrefix(contentType, "text/html") {
b, err := ioutil.ReadAll(resp.Body) //Read html
if err != nil {
return err
}
err = resp.Body.Close()
if err != nil {
return err
}
body := ioutil.NopCloser(bytes.NewReader(b))
resp.Body = body
resp.ContentLength = int64(len(b))
log.Printf(string(b))
}
return nil
}

Multipart request upload file and JSON message

How to implement a curl request in a go program
curl -XPOST -H 'Authorization: Bearer ***API*KEY***' http://127.0.0.1:9000/api/case/task/AVqqeXc9yQ6w1DNC8aDj/log -F '_json={"message": "Screenshot of fake site"};type=application/json' -F 'attachment=#screenshot1.png;type=image/png'
I tried to do something similar, but my knowledge is probably not enough
func CreateAttachedTask(caseid string, filename string, tasktitle string, taskdescription string, baseURL string, token string) {
fileDir, _ := os.Getwd()
filePath := path.Join(fileDir, filename)
file, err := os.Open(filePath)
defer file.Close()
buffer := new(bytes.Buffer)
writer := multipart.NewWriter(buffer)
part, _ := writer.CreateFormFile("filename", filepath.Base(file.Name()))
io.Copy(part, file)
writer.WriteField("_json", `{"message":"Screenshot of fake site"}`)
defer writer.Close()
req, err = http.NewRequest("POST", "http://127.0.0.1:9000/api/case/task/AVqqeXc9yQ6w1DNC8aDj/log", buffer)
if err != nil {
log.Fatal(err.Error())
}
req.Header.Set("Authorization", token)
req.Header.Set("Content-Type", writer.FormDataContentType())
client = &http.Client{Transport: defaultTtransportObs}
resp, err = client.Do(req)
if err != nil {
panic(err)
}
fmt.Println("Log response Status:", resp.Status)
defer resp.Body.Close()
}
Here is the solution
var b bytes.Buffer
w := multipart.NewWriter(&b)
file, err := os.Open(filename)
req.Header.Set("Authorization", token)
h := make(textproto.MIMEHeader)
h.Set("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="attachment"; filename="%s"`, filename))
fileWriter, err := w.CreatePart(h)
if err != nil {
fmt.Println(err)
}
io.Copy(fileWriter, file)
w.WriteField("_json", `{"message":"Attach:"}`)
w.Close()
req, err = http.NewRequest("POST", "http://127.0.0.1:9000/api/case/task/AVqqeXc9yQ6w1DNC8aDj/log", &b)
if err != nil {
log.Fatal(err.Error())
}
req.Header.Set("Content-Type", w.FormDataContentType())
req.Header.Set("Authorization", token)
res, err := client.Do(req)
if err != nil {
log.Fatal(err.Error())
}
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}

Golang Encode/Decode base64 with json post doesn't work

I build a client and a server in golang both are using this functions to encrypt/decrypt
func encrypt(text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
b := base64.StdEncoding.EncodeToString(text)
ciphertext := make([]byte, aes.BlockSize+len(b))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
cfb := cipher.NewCFBEncrypter(block, iv)
cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
return ciphertext, nil
}
func decrypt(text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
if len(text) < aes.BlockSize {
return nil, errors.New("ciphertext too short")
}
iv := text[:aes.BlockSize]
text = text[aes.BlockSize:]
cfb := cipher.NewCFBDecrypter(block, iv)
cfb.XORKeyStream(text, text)
data, err := base64.StdEncoding.DecodeString(string(text))
if err != nil {
return nil, err
}
return data, nil
}
so yeah I make a normal post request
url := "https://"+configuration.Server+configuration.Port+"/get"
// TODO maybe bugs rest here
ciphertext, err := encrypt([]byte(*getUrl))
if err != nil {
fmt.Println("Error: " + err.Error())
}
fmt.Println(string(ciphertext))
values := map[string]interface{}{"url": *getUrl, "urlCrypted": ciphertext}
jsonValue, _ := json.Marshal(values)
jsonStr := bytes.NewBuffer(jsonValue)
req, err := http.NewRequest("POST", url, jsonStr)
and the servercode is as following
requestContent := getRequestContentFromRequest(req)
url := requestContent["url"].(string)
undecryptedUrl := requestContent["urlCrypted"].(string)
decryptedurl, err := decrypt([]byte(undecryptedUrl))
if err != nil {
fmt.Println("Error: " + err.Error())
}
fmt.Println(decryptedurl)
where getRequestContentFromRequest is as following
func getRequestContentFromRequest(req *http.Request)
map[string]interface{} {
buf := new(bytes.Buffer)
buf.ReadFrom(req.Body)
data := buf.Bytes()
var requestContent map[string]interface{}
err := json.Unmarshal(data, &requestContent)
if err != nil {
fmt.Println(err)
}
return requestContent
}
Now to the problem.
If I encrypt my string in the client and decrypt it direct after that everything is fine.
But, when I send the encrypted string to the server and try to decrypt it with literrally the same function as in the client, the decrypt function throws an error.
Error: illegal base64 data at input byte 0
I think the Problem is the unmarshalling of the JSON.
Thanks for help.
P.S.
Repos are
github.com/BelphegorPrime/goSafeClient and github.com/BelphegorPrime/goSafe
UPDATE
Example JSON
{"url":"facebook2.com","urlCrypted":"/}\ufffd\ufffd\ufffdgP\ufffdN뼞\ufffd\u0016\ufffd)\ufffd\ufffd\ufffdy\u001c\u000f\ufffd\ufffd\ufffdep\ufffd\rY\ufffd\ufffd$\ufffd\ufffd"}
UPDATE2
I made a playground here
The problem is that you encode in base64 twice. The first time in the encrypt function and the second time during the JSON marshalling. byte slices are automatically converted into base64 strings by the encoding/json marshaller.
The solution is to decode the base64 string before calling decrypt.
Example on the Go PlayGround
EDIT
Working solution here