how can read a json nested in multiple levels - json

{
"result": [
{
"id": "140964",
"label": " 3688-RELT-PRD-LOGIN",
"location": "St. Louis, MO",
"startdate": "September 1, 2019 00:00:00",
"enddate": "September 12, 2019 04:57:41",
"statushistory": [
{
"status": "DOWN",
"statustype": "Content error",
"starttime": "09-01-2019 00:00",
"endtime": "09-11-2019 23:57",
"duration": "950261"
}
]
},
{
"id": "142285",
"label": " 4316_Rebar Messaging_PROD",
"location": "Chicago, IL",
"startdate": "September 1, 2019 00:00:00",
"enddate": "September 12, 2019 04:57:41",
"statushistory": [
{
"status": "UP",
"statustype": "OK",
"starttime": "09-01-2019 00:00",
"endtime": "09-11-2019 23:57",
"duration": "950261"
}
]
},
{
"id": "153272",
"label": "10002-Self Service SQL Failover",
"location": "Miami, FL",
"startdate": "September 1, 2019 00:00:00",
"enddate": "September 12, 2019 04:57:41",
"statushistory": [
{
"status": "UP",
"statustype": "OK",
"starttime": "09-01-2019 00:00",
"endtime": "09-11-2019 23:57",
"duration": "950261"
}
]
},
{
"id": "156764",
"label": "10054-SMARTSUPPLY-PRD",
"location": "Bangalore, India",
"startdate": "September 1, 2019 00:00:00",
"enddate": "September 12, 2019 04:57:41",
"statushistory": [
{
"status": "UP",
"statustype": "OK",
"starttime": "09-01-2019 00:00",
"endtime": "09-11-2019 23:57",
"duration": "950261"
}
]
},
{
"id": "156764",
"label": "10054-SMARTSUPPLY-PRD",
"location": "Mumbai, India",
"startdate": "September 1, 2019 00:00:00",
"enddate": "September 12, 2019 04:57:41",
"statushistory": [
{
"status": "UP",
"statustype": "OK",
"starttime": "09-08-2019 10:35",
"endtime": "09-11-2019 23:57",
"duration": "307316"
},
{
"status": "DOWN",
"statustype": "Timeout warning",
"starttime": "09-08-2019 10:31",
"endtime": "09-08-2019 10:35",
"duration": "274"
},
{
"status": "UP",
"statustype": "OK",
"starttime": "09-01-2019 00:00",
"endtime": "09-08-2019 10:31",
"duration": "642670"
}
]
}
]
}
i have this above json file .
which holds multiple data set this is just an example with 5 .
i want to pull the status value from the above json .
We can usual do it by result['statushistory'][0]['status']
But it pulls only the 5 values . It is returning only the first value of
the last statushistory .
How can i write a code which returns all the value of statushistory ?
And also if the statushistory doesnt have any status how can i mange that in the code ?(i.e return status only if statushistory has status present in it)

Something like:
statushistory_list = [result["statushistory"] for result in json_data["result"]]
status_list = [
[item["status"] for item in statushistory if "status" in item]
for statushistory in statushistory_list
]
this will get you:
[['DOWN'], ['UP'], ['UP'], ['UP'], ['UP', 'DOWN', 'UP']]
If it's not what you are looking for then, please show an example of the output you would like.

Related

Json parsing losgstash

i can't parse this json with logstash... someone could help me?
seems like the way it is parsed can't be readed by logstash.
there is a ruby code to parse this?
I cannot extract the fields nested in the square brackets
[
{
"capacity": 0,
"created_at": "2021-04-06T16:18:34+02:00",
"decisions": [
{
"duration": "22h16m4.141220361s",
"id": 842,
"origin": "CAPI",
"scenario": "crowdsecurity/http-bad-user-agent",
"scope": "ip",
"simulated": false,
"type": "ban",
"value": "3.214.184.223/32"
},
.
.
.
{
"duration": "22h16m4.195897491s",
"id": 904,
"origin": "CAPI",
"scenario": "crowdsecurity/http-backdoors-attempts",
"scope": "ip",
"simulated": false,
"type": "ban",
"value": "51.68.11.195/32"
}
],
"events": null,
"events_count": 0,
"id": 12,
"labels": null,
"leakspeed": "",
"machine_id": "N/A",
"message": "",
"scenario": "update : +63/-0 IPs",
"scenario_hash": "",
"scenario_version": "",
"simulated": false,
"source": {
"scope": "Community blocklist",
"value": ""
},
"start_at": "2021-04-06 16:18:34.750588276 +0200 +0200",
"stop_at": "2021-04-06 16:18:34.750588717 +0200 +0200"
}
]
Require JSON
JSON.parse(yourString)
Would likely be what you're looking for.
The module is described here

Nested json - store values in csv

I am trying to convert a nested json file into csv. It's data from a darts API and the structure is always the same. Nevertheless I got some problems flattening and storing the values in a csv because of the nested structure.
json:
{
"summaries": [{
"sport_event": {
"id": "sr:sport_event:12967512",
"start_time": "2017-11-11T13:15:00+00:00",
"start_time_confirmed": true,
"sport_event_context": {
"sport": {
"id": "sr:sport:22",
"name": "Darts"
},
"category": {
"id": "sr:category:104",
"name": "International"
},
"competition": {
"id": "sr:competition:597",
"name": "Grand Slam of Darts"
},
"season": {
"id": "sr:season:47332",
"name": "Grand Slam of Darts 2017",
"start_date": "2017-11-11",
"end_date": "2017-11-20",
"year": "2017",
"competition_id": "sr:competition:597"
},
"stage": {
"order": 1,
"type": "league",
"phase": "stage_1",
"start_date": "2017-11-11",
"end_date": "2017-11-15",
"year": "2017"
},
"round": {
"number": 1
},
"groups": [{
"id": "sr:league:29766",
"name": "Grand Slam of Darts 2017, Group G",
"group_name": "G"
}]
},
"coverage": {
"live": true
},
"competitors": [{
"id": "sr:competitor:35936",
"name": "Smith, Michael",
"abbreviation": "SMI",
"qualifier": "home"
}, {
"id": "sr:competitor:83895",
"name": "Wilson, James",
"abbreviation": "WIL",
"qualifier": "away"
}]
},
"sport_event_status": {
"status": "closed",
"match_status": "ended",
"home_score": 5,
"away_score": 3,
"winner_id": "sr:competitor:35936"
}
}, {
"sport_event": {
"id": "sr:sport_event:12967508",
"start_time": "2017-11-11T13:40:00+00:00",
"start_time_confirmed": true,
"sport_event_context": {
"sport": {
"id": "sr:sport:22",
"name": "Darts"
},
"category": {
"id": "sr:category:104",
"name": "International"
},
"competition": {
"id": "sr:competition:597",
"name": "Grand Slam of Darts"
},
"season": {
"id": "sr:season:47332",
"name": "Grand Slam of Darts 2017",
"start_date": "2017-11-11",
"end_date": "2017-11-20",
"year": "2017",
"competition_id": "sr:competition:597"
},
"stage": {
"order": 1,
"type": "league",
"phase": "stage_1",
"start_date": "2017-11-11",
"end_date": "2017-11-15",
"year": "2017"
},
"round": {
"number": 1
},
"groups": [{
"id": "sr:league:29764",
"name": "Grand Slam of Darts 2017, Group F",
"group_name": "F"
}]
},
"coverage": {
"live": true
},
"competitors": [{
"id": "sr:competitor:70916",
"name": "Bunting, Stephen",
"abbreviation": "BUN",
"qualifier": "home"
}, {
"id": "sr:competitor:191262",
"name": "de Zwaan, Jeffrey",
"abbreviation": "DEZ",
"qualifier": "away"
}]
},
"sport_event_status": {
"status": "closed",
"match_status": "ended",
"home_score": 5,
"away_score": 4,
"winner_id": "sr:competitor:70916"
}
}
So for each sport_event I would like to store the variables:
"start_time"
from "season" the variable "name"
from "competitors" both "id" and "name"
from "sport_event_status" the "winner_id"
I have already tried to flatten the json file with this code:
import json
f = open(r'path of file.json')
data = json.load(f)
def flatten(data):
for key,value in data.items():
print (str(key)+'->'+str(value))
if type(value) == type(dict()):
flatten(value)
elif type(value) == type(list()):
for val in value:
if type(val) == type(str()):
pass
elif type(val) == type(list()):
pass
else:
flatten(val)
flatten(data)
print(data)
This actually prints out the following:
id->sr:season:47332
name->Grand Slam of Darts 2017
start_date->2017-11-11
end_date->2017-11-20
year->2017
competition_id->sr:competition:597
Now my question is how to store the values I mentioned above in a csv file.
Thanks in advance for your support.
Using jq, you basically just have to transcribe your specification, adding a bit of context and taking care of an embedded array:
.summaries[]
| .sport_event # Your specification:
| [.start_time, # start_time
.sport_event_context.season.name] # from "season" the variable "name"
+ [.competitors[] | .id, .name] # from "competitors" both "id" and "name"
+ [.sport_event_status.winner_id] # from "sport_event_status" the "winner_id"
| #csv
Invocation
E.g.
jq -rf program.jq my.json

Python 3 - Extracting value from key in nested dictionary

Hoping for some pointers here as I'm striking out with my attempts. Am working with python 3.8.5.
I'm querying a Car Park booking system that returns a json list of availability. The result has lots of nested dictionaries and I'm struggling to extract just the values i want.
This is what I've been doing:
import requests
import json
enquiry = requests.post(url.....) #queries api, this works fine
results = enquiry.text #extracts response data
dictionary = json.loads(results) #convert response to python dict
If there is one slot available, I get this output (apologies for length). If there are multiple slots available, i get the same output repeated:
{
"data": {
"services": [{
"id": null,
"name": null,
"services": [{
"id": null,
"name": "Car Park",
"met": true,
"filterCount": 1,
"primary": true,
"options": [{
"id": "9",
"images": [],
"available": true,
"calendarId": "AAAA",
"templateId": "BBBB",
"capacity": 0,
"name": "Car Park Slot 3",
"sessionId": "CCCC",
"functions": null,
"startDate": "2020-08-18T13:30:00Z", <--this is what I want to extract
"endDate": "2020-08-18T14:30:00Z",
"geo": {
"lat": 0.0,
"lng": 0.0
},
"selected": false,
"linkedServices": [],
"tiers": null
}]
}],
"currentBookingId": null,
"startDate": {
"ms": 1597757400000,
"year": 2020,
"month": 8,
"day": 18,
"dayOfWeek": 2,
"time": {
"seconds": 0,
"minutes": 30,
"hours": 14,
"days": 0
}
},
"endDate": {
"ms": 1597761000000,
"year": 2020,
"month": 8,
"day": 18,
"dayOfWeek": 2,
"time": {
"seconds": 0,
"minutes": 30,
"hours": 15,
"days": 0
}
},
"sessionId": "2222222",
"chargeType": 1,
"hasPrimaryBookable": false,
"hasBookable": false,
"hasDiscounts": false,
"hasMultipleTiers": false,
"isPreferred": false,
"primaryServiceAvailable": true,
"primaryServiceId": null,
"primaryServiceType": "undefined",
"unavailableAttendees": []
}],
"bookingLimit": null
},
"success": true,
"suppress": false,
"version": "2.3.293",
"message": null,
"result": null,
"errors": null,
"code": null,
"flags": 0,
"redirect": null
}
I want to extract:
"startDate": "2020-08-18T13:30:00Z"
from each key, value pair in any of the returned slots. However, I cant work it out.
Extracting the whole of this nested dictionary would also contain the same data, but would then involve more work to tidy it up after.
"startDate": {
"ms": 1597757400000,
"year": 2020,
"month": 8,
"day": 18,
"dayOfWeek": 2,
"time": {
"seconds": 0,
"minutes": 30,
"hours": 14,
"days": 0
}
I've tried loads of dictionary.get and dictionary.items variations, but cant seem to get anywhere.
I tried something like
key = ('startDate')
availability = dictionary.get(key)
print(availability)
this just returns 'none', so think im way off
Any pointers?
Thanks in advance!
Thanks for the full data. It makes testing easier :)
I had to replace null -> None, true -> True, false -> False
slot = {
"data": {
"services": [{
"id": None,
"name": None,
"services": [{
"id": None,
"name": "Car Park",
"met": True,
"filterCount": 1,
"primary": True,
"options": [{
"id": "9",
"images": [],
"available": True,
"calendarId": "AAAA",
"templateId": "BBBB",
"capacity": 0,
"name": "Car Park Slot 3",
"sessionId": "CCCC",
"functions": None,
"startDate": "2020-08-18T13:30:00Z", # <--this is what I want to extract
................
print("Start Date:", slot['data']['services'][0]['services'][0]['options'][0]['startDate'])
Output
Start Date: 2020-08-18T13:30:00Z

return specific data from JSON file

I have a project I’m working on but I know very little about Javascript/Jquery/Ajax and am having trouble figuring out exactly what I need to do and don’t really have anyone around me to ask. Using the JSON info below, I need to include an Ajax call to populate current 'condition.temp’, the returned 'temperature' unit, and set the weather to be linked to 'link' key in the returned response. I’d like to return it to the console. I have it set up to return all of the JSON data to the console, but am stuck on how to return just the particular data items I need.
{
"success": true,
"data": [
{
"units": {
"distance": "mi",
"speed": "mph",
"pressure": "in",
"temperature": "F"
},
"language": "en-us",
"description": "Yahoo! Weather for Paducah, KY, US",
"astronomy": {
"sunrise": "5:45 am",
"sunset": "7:57 pm"
},
"item": {
"forecast": [
{
"day": "Mon",
"text": "Mostly Cloudy",
"date": "16 May 2016",
"high": 61,
"low": 47,
"code": 28
},
{
"day": "Tue",
"text": "Scattered Showers",
"date": "17 May 2016",
"high": 62,
"low": 51,
"code": 39
},
{
"day": "Wed",
"text": "Scattered Showers",
"date": "18 May 2016",
"high": 66,
"low": 50,
"code": 39
},
{
"day": "Thu",
"text": "Mostly Cloudy",
"date": "19 May 2016",
"high": 71,
"low": 51,
"code": 28
},
{
"day": "Fri",
"text": "Showers",
"date": "20 May 2016",
"high": 73,
"low": 58,
"code": 11
},
{
"day": "Sat",
"text": "Scattered Thunderstorms",
"date": "21 May 2016”,
"high": 74,
"low": 61,
"code": 47
},
{
"day": "Sun",
"text": "Partly Cloudy",
"date": "22 May 2016”,
"high": 76,
"low": 59,
"code": 30
},
{
"day": "Mon",
"text": "Partly Cloudy",
"date": "23 May 2016”,
"high": 81,
"low": 61,
"code": 30
},
{
"day": "Tue",
"text": "Thunderstorms",
"date": "24 May 2016",
"high": 81,
"low": 64,
"code": 4
},
{
"day": "Wed",
"text": "Rain",
"date": "25 May 2016",
"high": 82,
"low": 67,
"code": 12
}
],
"lat": 37.030182,
"guid": {
"isPermaLink": false
},
"condition": {
"text": "Cloudy",
"date": "Mon, 16 May 2016 03:00 PM CDT",
"temp": 60,
"code": 26
},
"description": "<![CDATA[<img src=\"http://l.yimg.com/a/i/us/we/52/26.gif\"/>\n<BR />\n<b>Current Conditions:</b>\n<BR />Cloudy\n<BR />\n<BR />\n<b>Forecast:</b>\n<BR /> Mon - Mostly Cloudy. High: 61Low: 47\n<BR /> Tue - Scattered Showers. High: 62Low: 51\n<BR /> Wed - Scattered Showers. High: 66Low: 50\n<BR /> Thu - Mostly Cloudy. High: 71Low: 51\n<BR /> Fri - Showers. High: 73Low: 58\n<BR />\n<BR />\nFull Forecast at Yahoo! Weather\n<BR />\n<BR />\n(provided by <a href=\"http://www.weather.com\" >The Weather Channel</a>)\n<BR />\n]]>",
"link": "http://us.rd.yahoo.com/dailynews/rss/weather/Country__Country/*https://weather.yahoo.com/country/state/city-12775789/",
"pubDate": "Mon, 16 May 2016 03:00 PM CDT",
"long": -88.712601,
"title": "Conditions for Paducah, KY, US at 03:00 PM CDT"
},
"image": {
"link": "http://weather.yahoo.com",
"height": 18,
"url": "http://l.yimg.com/a/i/brand/purplelogo//uh/us/news-wea.gif",
"width": 142,
"title": "Yahoo! Weather"
},
"ttl": 60,
"atmosphere": {
"visibility": 16.1,
"humidity": 60,
"pressure": 1007,
"rising": 0
},
"location": {
"country": "United States",
"city": "Paducah",
"region": " KY"
},
"link": "http://us.rd.yahoo.com/dailynews/rss/weather/Country__Country/*https://weather.yahoo.com/country/state/city-12775789/",
"lastBuildDate": "Mon, 16 May 2016 04:35 PM CDT",
"wind": {
"direction": 105,
"chill": 59,
"speed": 7
},
"title": "Yahoo! Weather - Paducah, KY, US"
}
]
}
Assuming that you're asking how to extract data from JSON...
If you use the node console, you can dig into it and see the inner workings.
first, assign the json data to a variable, ie
var test = {
"success": true,
"data": [
{
"units": {
"distance": "mi",
"speed": "mph",
"pressure": "in",
"temperature": "F"
},
"language": "en-us", etc....
then start digging down the data using console.log:
> console.log(test);
{ success: true,
data:
[ { units: [Object],
language: 'en-us',
description: 'Yahoo! Weather for Paducah, KY, US',
So, how do we access "success"? This worked:
> console.log(test['success']);
true
But there is a better way:
> console.log(test.success);
true
Now, how do we get the "data" attributes?
> console.log(test.data);
[ { units: { distance: 'mi', speed: 'mph', pressure: 'in', temperature: 'F' },
language: 'en-us',
description: 'Yahoo! Weather for Paducah, KY, US',
astronomy: { sunrise: '5:45 am', sunset: '7:57 pm' },
item:
{ forecast: [Object],
lat: 37.030182,
we could try this:
> console.log(test.data.units);
undefined
Uh, oh... what happened? Short answer, data is an array ( [ )
> console.log(test.data[0].units);
{ distance: 'mi', speed: 'mph', pressure: 'in', temperature: 'F' }
Let's go one more:
> console.log(test.data[0].units.speed);
mph
Using this method you can extract any information out of JSON.

Map Reduce/Sum using Underscorejs

I am trying to convert the following dataset:
var data = {
"csrf": "token",
"items": [
{"category": "product1", "image": "image1.jpg", "cost": "$6", "item_totals": [
{"category": "discount", "amount": "-$1.0"}]
},
{"category": "product2", "image": "image2.jpg", "cost": "$8", "item_totals": [
{"category": "discount", "amount": "-$1.2"}]
}],
"totals": [
{"category": "shipping", "amount": "$0.00", "name": "Shipping", "is_zero": true},
{"category": "taxes", "amount": "$0.00", "name": "Taxes", "is_zero": true}
],
"total": "$1234", "subtotal": "$1234", "id": abc123, "currency_code": "USD"}
into a single value, being the sum of all 'amount' values in the item_totals array(s). I cannot seem to do this whilst accounting a varying amount of records in 'items' & for empty 'item_total' arrays. Any help would be great!
var data = {
"csrf": "token",
"items": [{
"category": "product1",
"image": "image1.jpg",
"cost": "$6",
"item_totals": [{
"category": "discount",
"amount": "-$1.0"
}]
},
{
"category": "product2",
"image": "image2.jpg",
"cost": "$8",
"item_totals": [ // multiple item_totals
{
"category": "discount",
"amount": "-$1.2"
},
{
"category": "discount",
"amount": "-$1.2"
}
]
},
{
"category": "product2",
"image": "image2.jpg",
"cost": "$8",
"item_totals": [] // empty item_totals
},
{
"category": "product2",
"image": "image2.jpg",
"cost": "$8"
// absent item_totals
}
],
"totals": [{
"category": "shipping",
"amount": "$0.00",
"name": "Shipping",
"is_zero": true
},
{
"category": "taxes",
"amount": "$0.00",
"name": "Taxes",
"is_zero": true
}
],
"total": "$1234",
"subtotal": "$1234",
"id": "abc123",
"currency_code": "USD"
}
var totalAmount = _.chain(data.items)
.map(item => item.item_totals || []) // emit totals
.tap(console.log)
.reduce((memo, bucket) => memo.concat(bucket), []) // collect (flatten)
.tap(console.log)
.map(total => parseFloat(total.amount.replace('$', ''), 10)) // emit amounts
.tap(console.log)
.reduce((ret, amount) => ret + amount, 0) // collect (sum)
.value();
console.log(totalAmount);
<script src="http://underscorejs.org/underscore.js"></script>