Reading sertain set of key-value pair in JSON using python - json

suppose I have below json data, my requirement is I only want to parse few key-value pair to be print. Like I want name,description,start & end key-value pair will be printed just by calling print once. All other key-value pair should be skipped while printing. Only the asked above information is needed for my work, so I don't want to keep all other key_value pair.
'events': [
{
'name': {
'text': 'Sample Event',
'html': 'Sample Event'
},
'description': {
'text': 'This is a test event',
'html': '<P>This is a test event</P>'
},
'start': {
'timezone': 'Asia/Kolkata',
'local': '2018-10-12T19:00:00',
'utc': '2018-10-12T13:30:00Z'
},
'end': {
'timezone': 'Asia/Kolkata',
'local': '2018-10-12T22:00:00',
'utc': '2018-10-12T16:30:00Z'
},
'organization_id': '269994560152',
'created': '2018-09-02T15:48:49Z',
'changed': '2018-09-02T15:49:00Z',
'capacity': 1,
'capacity_is_custom': False,
'status': 'live',
'currency': 'USD',
'listed': True,
'shareable': True,
'invite_only': False,
'online_event': False,
'show_remaining': True,
'tx_time_limit': 480,
'hide_start_date': False,
'hide_end_date': False,
'locale': 'en_US',
'is_locked': False,
'privacy_setting': 'unlocked',
'is_series': False,
'is_series_parent': False,
'is_reserved_seating': False,
'show_pick_a_seat': False,
'show_seatmap_thumbnail': False,
'show_colors_in_seatmap_thumbnail': False,
'source': 'create_2.0',
'is_free': True,
'version': '3.0.0',
'logo_id': None,
'category_id': '199',
'subcategory_id': None,
'format_id': '16',
'logo': None
}

Related

Json decode error while importing from a csv file

I am writing a python program that load a json string and decode from a .csv file. The .csv file includs the title and one entry below for reference.
key,labels,raw_tweet
2017_Q3_270,"[0, 0]","{'in_reply_to_screen_name': None, 'user': {'profile_banner_url': 'https://pbs.twimg.com/profile_banners/148491006/1494299074', 'follow_request_sent': None, 'name': 'Vanessa', 'verified': False, 'profile_sidebar_fill_color': 'FFFFFF', 'profile_background_color': '352726', 'is_translator': False, 'profile_background_image_url_https': 'https://pbs.twimg.com/profile_background_images/578700342637895680/j-o_FCwY.png', 'id': 148491006, 'geo_enabled': True, 'profile_background_image_url': 'http://pbs.twimg.com/profile_background_images/578700342637895680/j-o_FCwY.png', 'default_profile': False, 'contributors_enabled': False, 'default_profile_image': False, 'location': 'everywhere', 'profile_background_tile': True, 'notifications': None, 'listed_count': 9, 'profile_link_color': '7FDBB6', 'protected': False, 'profile_image_url': 'http://pbs.twimg.com/profile_images/891824958225215488/h__HMMlC_normal.jpg', 'profile_image_url_https': 'https://pbs.twimg.com/profile_images/891824958225215488/h__HMMlC_normal.jpg', 'following': None, 'time_zone': 'Eastern Time (US & Canada)', 'friends_count': 588, 'url': 'https://Instagram.com/vmanks/', 'profile_text_color': '333333', 'followers_count': 541, 'utc_offset': -14400, 'id_str': '148491006', 'description': 'from the bronx, studying at cornell, slowly but surely finding solace', 'created_at': 'Wed May 26 21:01:46 +0000 2010', 'screen_name': 'vmankss', 'favourites_count': 19781, 'profile_use_background_image': True, 'profile_sidebar_border_color': 'FFFFFF', 'statuses_count': 50506, 'lang': 'en'}, 'retweet_count': 0, 'is_quote_status': False, 'in_reply_to_user_id': None, 'id': 901132409508421632, 'coordinates': None, 'entities': {'symbols': [], 'urls': [], 'user_mentions': [], 'hashtags': []}, 'text': ""I basically just go to financial aid to take candy from the candy bowl, y'all are unhelpful"", 'in_reply_to_status_id_str': None, 'in_reply_to_status_id': None, 'geo': None, 'favorited': False, 'place': {'country_code': 'US', 'bounding_box': {'type': 'Polygon', 'coordinates': [[[-76.547738, 42.41815], [-76.547738, 42.480827], [-76.469987, 42.480827], [-76.469987, 42.41815]]]}, 'attributes': {}, 'country': 'United States', 'url': 'https://api.twitter.com/1.1/geo/id/ae76bffcaf2bf545.json', 'full_name': 'Ithaca, NY', 'name': 'Ithaca', 'id': 'ae76bffcaf2bf545', 'place_type': 'city'}, 'favorite_count': 0, 'retweeted': False, 'timestamp_ms': '1503681683314', 'truncated': False, 'id_str': '901132409508421632', 'created_at': 'Fri Aug 25 17:21:23 +0000 2017', 'in_reply_to_user_id_str': None, 'contributors': None, 'source': 'Twitter for iPhone', 'lang': 'en', 'filter_level': 'low'}"
2015_Q1_494,"[0, 0]","{'in_reply_to_user_id_str': None, 'id_str': '577090329658175488', 'timestamp_ms': '1426424031067', 'in_reply_to_status_id_str': None, 'lang': 'en', 'favorited': False, 'retweeted': False, 'in_reply_to_status_id': None, 'id': 577090329658175488, 'filter_level': 'low', 'created_at': 'Sun Mar 15 12:53:51 +0000 2015', 'in_reply_to_user_id': None, 'place': {'country': 'United States', 'url': 'https://api.twitter.com/1.1/geo/id/a307591cd0413588.json', 'id': 'a307591cd0413588', 'country_code': 'US', 'place_type': 'city', 'attributes': {}, 'full_name': 'Buffalo, NY', 'bounding_box': {'type': 'Polygon', 'coordinates': [[[-78.912276, 42.826008], [-78.912276, 42.966451], [-78.79485, 42.966451], [-78.79485, 42.826008]]]}, 'name': 'Buffalo'}, 'truncated': False, 'entities': {'user_mentions': [], 'hashtags': [], 'symbols': [], 'trends': [], 'urls': []}, 'text': '""He licked coke off an encyclopedia"" only in south buffalo', 'retweet_count': 0, 'source': 'Twitter for iPhone', 'in_reply_to_screen_name': None, 'user': {'id_str': '480575646', 'friends_count': 367, 'profile_image_url': 'http://pbs.twimg.com/profile_images/571759767896629250/C-94okMM_normal.jpeg', 'profile_banner_url': 'https://pbs.twimg.com/profile_banners/480575646/1402863912', 'listed_count': 2, 'screen_name': 'MichaelaFeeney', 'lang': 'en', 'notifications': None, 'profile_text_color': '333333', 'verified': False, 'favourites_count': 3995, 'name': 'Michæla...', 'protected': False, 'statuses_count': 2666, 'id': 480575646, 'profile_sidebar_border_color': 'C0DEED', 'profile_use_background_image': True, 'profile_sidebar_fill_color': 'DDEEF6', 'is_translator': False, 'time_zone': None, 'profile_link_color': '0084B4', 'created_at': 'Wed Feb 01 17:11:27 +0000 2012', 'geo_enabled': True, 'url': None, 'contributors_enabled': False, 'following': None, 'default_profile_image': False, 'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png', 'description': 'They call me Lông Isländ. Brockport2018✌', 'utc_offset': None, 'location': '', 'profile_image_url_https': 'https://pbs.twimg.com/profile_images/571759767896629250/C-94okMM_normal.jpeg', 'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png', 'profile_background_tile': False, 'default_profile': True, 'followers_count': 221, 'follow_request_sent': None, 'profile_background_color': 'C0DEED'}, 'coordinates': {'type': 'Point', 'coordinates': [-78.805803, 42.869134]}, 'possibly_sensitive': False, 'geo': {'type': 'Point', 'coordinates': [42.869134, -78.805803]}, 'favorite_count': 0, 'contributors': None}"
2017_Q4_280,"[0, 0]","{'in_reply_to_screen_name': None, 'user': {'profile_banner_url': 'https://pbs.twimg.com/profile_banners/2812396208/1425183203', 'follow_request_sent': None, 'name': 'HunnyBon', 'verified': False, 'profile_sidebar_fill_color': '000000', 'profile_background_color': '000000', 'notifications': None, 'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png', 'id': 2812396208, 'geo_enabled': True, 'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png', 'default_profile': False, 'contributors_enabled': False, 'default_profile_image': False, 'location': 'New York, NY', 'profile_background_tile': False, 'translator_type': 'none', 'listed_count': 5, 'profile_link_color': '666666', 'protected': False, 'profile_image_url': 'http://pbs.twimg.com/profile_images/572570217272713216/rzw1Bbqs_normal.png', 'profile_image_url_https': 'https://pbs.twimg.com/profile_images/572570217272713216/rzw1Bbqs_normal.png', 'following': None, 'time_zone': None, 'friends_count': 68, 'url': 'http://www.hunnybon.com', 'profile_text_color': '000000', 'followers_count': 66, 'utc_offset': None, 'id_str': '2812396208', 'description': ""A Healthier Candy Store..organic, vegan, and nonGMO. Indulge your sweet tooth without the guilt. Chocolates, gummies, caramels...what's your indulgence?"", 'created_at': 'Tue Sep 16 03:56:36 +0000 2014', 'screen_name': 'HunnyBonSweets', 'favourites_count': 53, 'profile_use_background_image': False, 'profile_sidebar_border_color': '000000', 'lang': 'en', 'statuses_count': 252, 'is_translator': False}, 'retweet_count': 0, 'is_quote_status': False, 'in_reply_to_user_id': None, 'id': 925755798147313664, 'coordinates': {'type': 'Point', 'coordinates': [-74.0064, 40.7142]}, 'entities': {'symbols': [], 'urls': [{'expanded_url': '', 'display_url': 'instagram.com/p/Ba9WuoQlYuk/', 'url': '', 'indices': [98, 121]}], 'user_mentions': [], 'hashtags': []}, 'text': '🍫Hello November, and hello to our new Chocolate Matcha Truffles! 🍫RAW dark chocolate, CREAMY NUT… ', 'in_reply_to_status_id_str': None, 'in_reply_to_status_id': None, 'geo': {'type': 'Point', 'coordinates': [40.7142, -74.0064]}, 'favorited': False, 'reply_count': 0, 'place': {'country_code': 'US', 'bounding_box': {'type': 'Polygon', 'coordinates': [[[-74.026675, 40.683935], [-74.026675, 40.877483], [-73.910408, 40.877483], [-73.910408, 40.683935]]]}, 'attributes': {}, 'country': 'United States', 'url': '', 'full_name': 'Manhattan, NY', 'name': 'Manhattan', 'id': '01a9a39529b27f36', 'place_type': 'city'}, 'favorite_count': 0, 'retweeted': False, 'timestamp_ms': '1509552356646', 'possibly_sensitive': False, 'truncated': False, 'id_str': '925755798147313664', 'created_at': 'Wed Nov 01 16:05:56 +0000 2017', 'quote_count': 0, 'in_reply_to_user_id_str': None, 'contributors': None, 'source': '', 'lang': 'en', 'filter_level': 'low'}"
I am trying to load raw_tweet, which is a json object as a string and decode it into a json object. I keep getting errors regardless of how I decode the string.
import csv
import json
with open('testfile.csv','r', encoding='utf-8', newline='') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
jobj = row['raw_tweet'].replace("\'", "\"")
jobj = jobj.replace("None", "\"\"")
json.loads(jobj)
How I load the csv file. When I run the program, I get the following error. I also trying using panda dataframe to load and decode it into json object. I failed. Please suggest where I did wrong.
Traceback (most recent call last):
File "/Sandbox/csvfile.py", line 9, in <module>
json.loads(jobj)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 184 (char 183)
So in your csv file within the column raw tweet there are instances of False without any quotes. Also replacing the single quotes to double quotes has major break condition like your json already has strings like y'all which inherently uses single quote. So we only need to replace quotes for the keys and actual values and not quotes that occur within the string. So there are a lot of conditions to be replaced.
So I would rather suggest a different way of evaluating the csv and dumping jsons of the raw_tweet column.
import pandas as pd
data = pd.read_csv("test.csv").to_dict('records')
for d in data:
raw_tweet_dict = eval(d['raw_tweet'])
with open("json_dump.json", "w") as fp:
json.dump(raw_tweet_dict, fp)
You can use the raw_tweet_dict as a dictionary if this needs further transformation.
Alternatively you can also use your approach but you have add a lot of condition which I have added for now, it should work on your csv sample.
with open("test.csv", "r") as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
jobj = row['raw_tweet'].replace('"', "'")
jobj = jobj.replace("None", "''")
jobj = jobj.replace("False", "'False'").replace("True", "'True'")
jobj = jobj.replace("':", '\":').replace(": '", ': \"').replace("',", '\",').replace(", '", ', \"').replace("{'", '{\"').replace("'}", '\"}')
json.loads(jobj)

'Reader' object has no attribute 'tell' error with jsonlines and cloud storage

I'm writing a cloud function to extract data from an API in the following format:
[{'_id': '123qasd', 'description': 'stuff here', 'userId': 'xxxx', 'billable': False, 'taskId': 'asddf1234', 'projectId': '5e11gg55', 'timeInterval': {'start': '2020-05-16T09:00:00+02:00', 'end': '2020-05-16T13:00:00+02:00', 'duration': 14400}, 'taskName': 'Production', 'tags': [], 'isLocked': True, 'customFields': [], 'userName': 'Dupond', 'userEmail': 'stuff#mail.com', 'projectName': 'my project', 'projectColor': '#607D8B', 'clientName': 'my client', 'clientId': 'xxxxxxxx'}, {'_id': '123qasd', 'description': 'stuff here', 'userId': 'xxxx', 'billable': False, 'taskId': 'asddf1234', 'projectId': '5e11gg55', 'timeInterval': {'start': '2020-05-16T09:00:00+02:00', 'end': '2020-05-16T13:00:00+02:00', 'duration': 14400}, 'taskName': 'Production', 'tags': [], 'isLocked': True, 'customFields': [], 'userName': 'Dupond', 'userEmail': 'stuff#mail.com', 'projectName': 'my project', 'projectColor': '#607D8B', 'clientName': 'my client', 'clientId': 'xxxxxxxx'}, {'_id': '123qasd', 'description': 'stuff here', 'userId': 'xxxx', 'billable': False, 'taskId': 'asddf1234', 'projectId': '5e11gg55', 'timeInterval': {'start': '2020-05-16T09:00:00+02:00', 'end': '2020-05-16T13:00:00+02:00', 'duration': 14400}, 'taskName': 'Production', 'tags': [], 'isLocked': True, 'customFields': [], 'userName': 'Dupond', 'userEmail': 'stuff#mail.com', 'projectName': 'my project', 'projectColor': '#607D8B', 'clientName': 'my client', 'clientId': 'xxxxxxxx'}]
I want to send this data to Cloud Storage and then, read them with BigQuery. In order to do this I need to convert json to ndjson.
I'm using jsonlines to handle the conversion. Here is an extract of my function:
with jsonlines.open(f'/tmp/output.jsonl', "w") as writer:
writer.write(data)
date = datetime.today().strftime('%Y-%m-%d %H:%M:%S')
bucket = storage_client.bucket(bucket_name)
destination_blob_name = '{}'.format(date)
blob = bucket.blob(destination_blob_name)
with jsonlines.open('/tmp/output.jsonl') as reader:
blob.upload_from_file(reader)
I have the following error 'Reader' object has no attribute 'tell'
What's wrong?
I managed to get it to work! use just normal file open.
with open('output.jsonl', 'rb') as f:
blob.upload_from_file(f)

How do I get an element that is nested within a JSON dictionary?

I got a JSON response using the Spotify API and I'm trying to access the element called 'name' (one says '3 Doors Down' and the other starts with 'Bret Michaels') that seems to be inside the 'items' element but I can't seem to find the solution.
This is how I loaded the data:
search_results = requests.get(search_url + 'q=' + query + '&type=artist', headers=granted_headers).json()
Here is my JSON data:
{
'artists': {
'href': 'https://api.spotify.com/v1/search?query=3+doors+down&type=artist&offset=0&limit=20',
'items': [
{
'external_urls': {
'spotify': 'https://open.spotify.com/artist/2RTUTCvo6onsAnheUk3aL9'
},
'followers': {
'href': None,
'total': 2631330
},
'genres': [
'alternative metal',
'nu metal',
'pop rock',
'post-grunge'
],
'href': 'https://api.spotify.com/v1/artists/2RTUTCvo6onsAnheUk3aL9',
'id': '2RTUTCvo6onsAnheUk3aL9',
'images': [
{
'height': 640,
'url': 'https://i.scdn.co/image/ead4e883a59d30d8c157385aa531d3fe8e688fc0',
'width': 640
},
{
'height': 320,
'url': 'https://i.scdn.co/image/611a4fd8aaf2637c5894acf65f12e79d75926329',
'width': 320
},
{
'height': 160,
'url': 'https://i.scdn.co/image/f1a1a2c37f2f6d242b1ab7ae3f4d893bf5822095',
'width': 160
}
],
'name': '3 Doors Down',
'popularity': 72,
'type': 'artist',
'uri': 'spotify:artist:2RTUTCvo6onsAnheUk3aL9'
},
{
'external_urls': {
'spotify': 'https://open.spotify.com/artist/2kPbQDZvnasPcCuXbq6YQx'
},
'followers': {
'href': None,
'total': 156
},
'genres': [
],
'href': 'https://api.spotify.com/v1/artists/2kPbQDZvnasPcCuXbq6YQx',
'id': '2kPbQDZvnasPcCuXbq6YQx',
'images': [
],
'name': 'Bret Michaels (Featuring Brad Arnold of 3 Doors Down, Chris Cagle, Mark Wills)',
'popularity': 4,
'type': 'artist',
'uri': 'spotify:artist:2kPbQDZvnasPcCuXbq6YQx'
}
],
'limit': 20,
'next': None,
'offset': 0,
'previous': None,
'total': 4
}
}
artists = search_results['artists']['items'] # This is a list
for artist in artists: # artist is each object in the list
print(artist['name'])
Output:
> '3 Doors Down'
> 'Bret Michaels (Featuring Brad Arnold of 3 Doors Down, Chris Cagle, Mark Wills)'

How to return Nested TreeView JSON in Web API

I'm new to web API. I need some help to generate the JSON like following.
[
{
'id': 66,
'text': 'This is the first comment.',
'creator': {
'id': 52,
'display_name': 'Ben'
},
'respondsto': null,
'created_at': '2014-08-14T13:19:59.751Z',
'responses': [
{
'id': 71,
'text': 'This is a response to the first comment.',
'creator': {
'id': 14,
'display_name': 'Daniel',
},
'respondsto': 66,
'created_at': '2014-08-14T13:27:13.915Z',
'responses': [
{
'id': 87,
'text': 'This is a response to the response.',
'creator': {
'id': 52,
'display_name': 'Ben',
},
'respondsto': 71,
'created_at': '2014-08-14T13:27:38.046Z',
'responses': []
}
]
}
]
},
{
'id': 70,
'text': 'Đây là bình luận thứ hai.',
'creator': {
'id': 12,
'display_name': 'Nguyễn'
},
'respondsto': null,
'created_at': '2014-08-14T13:25:47.933Z',
'responses': []
}
];
My Intention is to give JSON Data for the Image.
I'm able to generate normal JSON data. I was struck how to create that responses inside responses until the empty response comes.
Any help would be appreciated.
UPDATE: I found the Answer

how to highlight certain rows in prime-ui datatable, depending on a condition (from the json data )

I am using the prime-ui datatable. I have to highlight certain rows if a condition is true. This condition depends on the Json data which we pass to the table.
How can we specify the condition in this case ?
$('#divInWhichTableIsRendered').puidatatable({
columns: [
{field:'f1', headerText: 'f1', sortable:true},
{field:'f2', headerText: 'f2', sortable:true},
{field:'f3', headerText: 'f3', sortable:true},
{field:'f4', headerText: 'f4', sortable:true},
{field:'f5', headerText: 'f5', sortable:true}
],
datasource: ourJson1,
});
Played around a amount of time, did come up with this 'solution'. Unfortunately it is not the exact solution you want. I'll explain the problem after the solution.
1.) We need to use the property content of the colum definitions.
content: A function that takes row data and expects a string or a jQuery object to customize the cell.
2.) Use content on every column:
{
field: 'vin',
headerText: 'Vin',
sortable: true,
content: function(rowData) {
return contentFunc(rowData, 'vin');
}
}
3.) Write a function that takes the rowData and checks if this element must be highlighted or not.
function contentFunc(rowData, prop) {
var result = $("<span />").html(rowData[prop]);
if (rowData.highlight) {
result.css('background', 'red');
}
return result;
}
Problem:
We can only 'highlight' the span we've created. Neither the td nor the tr. Why? Because the span gets inserted when the function returns the jQuery object. Before that, we do not have access to the td or tr. I don't know of we can add some callbacks to do it afterwards.
(A hack will be a mouseover which ranges over the whole viewport and calls a function, but that's a hack imho.)
Bring this all together:
<div id="tbllocal"></div>
var ourJson1 = [{
'highlight': false,
'brand': 'Volkswagen',
'year': 2012,
'color': 'White',
'vin': 'dsad231ff'
}, {
'highlight': true,
'brand': 'Audi',
'year': 2011,
'color': 'Black',
'vin': 'gwregre345'
}, {
'highlight': false,
'brand': 'Renault',
'year': 2005,
'color': 'Gray',
'vin': 'h354htr'
}, {
'highlight': false,
'brand': 'Bmw',
'year': 2003,
'color': 'Blue',
'vin': 'j6w54qgh'
}, {
'highlight': false,
'brand': 'Mercedes',
'year': 1995,
'color': 'White',
'vin': 'hrtwy34'
}, {
'highlight': false,
'brand': 'Opel',
'year': 2005,
'color': 'Black',
'vin': 'jejtyj'
}, {
'highlight': true,
'brand': 'Honda',
'year': 2012,
'color': 'Yellow',
'vin': 'g43gr'
}, {
'highlight': false,
'brand': 'Chevrolet',
'year': 2013,
'color': 'White',
'vin': 'greg34'
}, {
'highlight': false,
'brand': 'Opel',
'year': 2000,
'color': 'Black',
'vin': 'h54hw5'
}, {
'highlight': false,
'brand': 'Mazda',
'year': 2013,
'color': 'Red',
'vin': '245t2s'
}];
$('#tbllocal').puidatatable({
caption: 'Local Datasource',
columns: [{
field: 'vin',
headerText: 'Vin',
sortable: true,
content: function(rowData) {
return contentFunc(rowData, 'vin');
}
}, {
field: 'brand',
headerText: 'Brand',
sortable: true,
content: function(rowData) {
return contentFunc(rowData, 'brand');
}
}, {
field: 'year',
headerText: 'Year',
sortable: true,
content: function(rowData) {
return contentFunc(rowData, 'year');
}
}, {
field: 'color',
headerText: 'Color',
sortable: true,
content: function(rowData) {
return contentFunc(rowData, 'color');
}
}],
datasource: ourJson1,
content: contentFunc
});
function contentFunc(rowData, prop) {
var result = $("<span />").html(rowData[prop]);
if (rowData.highlight) {
result.css('background', 'red');
}
return result;
}
Here is the fiddle