parse complex json from http response - json

Question

I'm doing a http.PostForm against an API and the result is a json.
The Json structure is like this:
{
"sites": [
{
"site_id": 456,
"status": "pending-dns-changes",
"domain": "blabla",
"account_id": 123,
"acceleration_level": "advanced",
"site_creation_date": 1515455285000,
"ips": [
"1.2.3.4"
],
"dns": [
{
"dns_record_name": "something.bla.com",
"set_type_to": "CNAME",
"set_data_to": [
"something.x.incapdns.net"
]
}
],
"original_dns": [
{
"dns_record_name": "blabla2.com",
"set_type_to": "A",
"set_data_to": [
""
]
},
{
"dns_record_name": "blabla3.something.com",
"set_type_to": "A",
"set_data_to": [
"1.2.4.5"
]
},
],
"warnings": [],
"active": "bypass",
"support_all_tls_versions": false,
"use_wildcard_san_instead_of_full_domain_san": true,
"add_naked_domain_san": true,
"additionalErrors": [],
"display_name": "something.blablabla2.com",
"security": {
"waf": {
"rules": [
{
"action": "api.threats.action.alert",
"action_text": "Alert Only",
"id": "api.threats.sql_injection",
"name": "SQL Injection"
},
{
"action": "api.threats.action.alert",
"action_text": "Alert Only",
"id": "api.threats.cross_site_scripting",
"name": "Cross Site Scripting"
},
{
"action": "api.threats.action.alert",
"action_text": "Alert Only",
"id": "api.threats.illegal_resource_access",
"name": "Illegal Resource Access"
},
{
"block_bad_bots": true,
"challenge_suspected_bots": false,
"id": "api.threats.bot_access_control",
"name": "Bot Access Control"
},
{
"activation_mode": "api.threats.ddos.activation_mode.auto",
"activation_mode_text": "Auto",
"ddos_traffic_threshold": 1000,
"id": "api.threats.ddos",
"name": "DDoS"
},
{
"action": "api.threats.action.quarantine_url",
"action_text": "Auto-Quarantine",
"exceptions": [
{
"values": [
{
"urls": [
{
"value": "/neverbogus.pvr",
"pattern": "EQUALS"
}
],
"id": "api.rule_exception_type.url",
"name": "URL"
}
],
"id": 1618746719
},
{
"values": [
{
"urls": [
{
"value": "/doubleneverbogus.pvr",
"pattern": "EQUALS"
},
{
"value": "/ddoubleneverbogus.pvr",
"pattern": "EQUALS"
}
],
"id": "api.rule_exception_type.url",
"name": "URL"
}
],
"id": 856301271
},
{
"values": [
{
"client_apps": [
"537"
],
"id": "api.rule_exception_type.client_app_id",
"name": "Client app ID"
}
],
"id": 58318563
},
{
"values": [
{
"ips": [
"192.168.66.66"
],
"id": "api.rule_exception_type.client_ip",
"name": "IP"
}
],
"id": 707083378
},
{
"values": [
{
"geo": {
"countries": [
"BF"
]
},
"id": "api.rule_exception_type.country",
"name": "Country"
}
],
"id": 1432086237
},
{
"values": [
{
"user_agents": [
"gasdfafafdfasdfadsfadsffads"
],
"id": "api.rule_exception_type.user_agent",
"name": "User agent"
}
],
"id": 1876871261
},
{
"values": [
{
"parameters": [
"bogusparamnamehere234"
],
"id": "api.rule_exception_type.http_parameter",
"name": "Http parameter"
}
],
"id": 1338747790
}
],
"id": "api.threats.backdoor",
"name": "Backdoor Protect"
},
{
"action": "api.threats.action.alert",
"action_text": "Alert Only",
"id": "api.threats.remote_file_inclusion",
"name": "Remote File Inclusion"
},
{
"action": "api.threats.action.disabled",
"action_text": "Ignore",
"id": "api.threats.customRule",
"name": "IncapRules"
}
]
}
},
"sealLocation": {
"id": "api.seal_location.none",
"name": "No seal "
},
"ssl": {
"origin_server": {
"detected": true,
"detectionStatus": "ok"
},
"custom_certificate": {
"active": false
},
"generated_certificate": {
"ca": "GS",
"validation_method": "dns",
"validation_data": [
{
"dns_record_name": "blablablasomething2",
"set_type_to": "TXT",
"set_data_to": [
"somethingtexty"
]
}
],
"san": [
"*.blabla2.com"
],
"validation_status": "done"
}
},
"siteDualFactorSettings": {
"enabled": false,
"customAreas": [],
"customAreasExceptions": [],
"allowAllUsers": true,
"shouldSuggestApplicatons": true,
"allowedMedia": [
"ga",
"sms"
],
"shouldSendLoginNotifications": true,
"version": 0
},
"login_protect": {
"enabled": false,
"specific_users_list": [],
"send_lp_notifications": true,
"allow_all_users": true,
"authentication_methods": [
"ga",
"sms"
],
"urls": [],
"url_patterns": []
},
"performance_configuration": {
"advanced_caching_rules": {
"never_cache_resources": [],
"always_cache_resources": []
},
"acceleration_level": "advanced",
"async_validation": true,
"minify_javascript": true,
"minify_css": true,
"minify_static_html": true,
"compress_jpeg": true,
"compress_jepg": true,
"progressive_image_rendering": false,
"aggressive_compression": false,
"compress_png": true,
"on_the_fly_compression": true,
"tcp_pre_pooling": true,
"comply_no_cache": false,
"comply_vary": false,
"use_shortest_caching": false,
"perfer_last_modified": false,
"prefer_last_modified": false,
"disable_client_side_caching": false,
"cache300x": false,
"cache_headers": []
},
"extended_ddos": 1000000,
"log_level": "security",
"incap_rules": [
{
"id": 51589,
"name": "Wordpress",
"action": "api.rule_action_type.rule_action_alert",
"rule": "(URL contains \"/xmlrpc.php$\" | URL contains \"/wp-login.php$\") & (User-Agent == \"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" | User-Agent == \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1\")",
"creation_date": 1518810206000
},
{
"id": 52179,
"name": "No Browser",
"action": "api.rule_action_type.rule_action_alert",
"rule": "(ClientType != Browser & ClientType != SpamBot & ClientType != DDoSBot & ClientType != SiteHelper) & (User-Agent != \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b\" & User-Agent != \"Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)\")",
"creation_date": 1519432068000
}
],
"res": 0,
"res_message": "OK",
"debug_info": {
"id-info": "13019"
}
},
API Docs: https://docs.imperva.com/bundle/cloud-application-security/page/api/sites-api.htm#List
This seems to be very painful for me to understand in go as I'm just starting..
From what I've been reading I should just define an object of type struct with my expected format.
Any way I can tackle this a lot easier? I'm interested in extracted for each individual site a few items like site_id / ips / domain.
Unsure how I should tackle this. I've been trying to get my head around https://github.com/buger/jsonparser but I'm failing to actually understand how to use it.
So far what I've been doing is:
func main() {
fmt.Println("Starting this..")
formData := url.Values{
"api_id": {keyid},
"api_key": {apikey},
"page_size": {"100"},
"page_num": {"0"},
}
response, err := http.PostForm("https://my.imperva.com/api/prov/v1/sites/list", formData)
if nil != err {
fmt.Println("Ooops..", err)
}
log.Println(response.Status)
bodyBytes, err := ioutil.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
}
var results map[string]interface{}
fmt.Println(string(bodyBytes))
json.Unmarshal([]byte(bodyBytes), &results)
defer response.Body.Close()
}

Answer 1

If you only need to extract a couple of fields, you don't really need to define an elaborate struct type to describe the whole structure. You can use the stdlib to unmarshal the JSON into a map[string]interface{} and then use something like this function to look for interesting keys:
// findNested looks for a key named s in map m. If values in m map to other
// maps, findNested looks into them recursively. Returns true if found, and
// the value found.
func findNested(m map[string]interface{}, s string) (bool, interface{}) {
// Try to find key s at this level
for k, v := range m {
if k == s {
return true, v
}
}
// Not found on this level, so try to find it nested
for _, v := range m {
nm, ok := v.(map[string]interface{})
if ok {
found, val := findNested(nm, s)
if found {
return found, val
}
}
}
// Not found recursively
return false, nil
}
This code deals only with nested maps. You can adjust it slightly to see through slices, etc.

Answer 2

You can define a single, complex go struct that reflects the entire json object structure, and annotate all of the elements with the json object/element names.
You can define simple structs for each of the nested objects, and annotate these structs with the json object/element names; and then build the complex struct by composing these structs.
Either of these are a ton of effort. You do not need the entire struct, you only need a few elements.
You can either define only the parts of the struct that you need to extract, and annotate those, or you can use the map[string]interface approach you have used. Both approaches work well.
package main
import (
"encoding/json"
"fmt"
)
func main() {
// your http query/response here
// Declare an empty interface
var results map[string]interface{}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(body))
// Unmarshal or Decode the JSON to the interface.
json.Unmarshal([]byte(body), &results)
defer response.Body.Close()
json.Unmarshal([]byte(empJson), &result)
//Reading each value by its key
fmt.Println("site_id:", result["site_id"],
"\ndomain:", result["domain"],
"\nip:", result["ips"][0],
// etc
)
}
See also:
How to unmarshal JSON in to array of interface and use