How can my python code change all json values to 0.
Code
str(ctx.author.id): score
This is the result I want:
str(ctx.author.id): 4
str(ctx.author.id): 3
str(ctx.author.id) is It can change at any time.
str(ctx.author.id) is Consists of 18 digits
You want all JSON values to be 0? then why are you giving 3, 4 in example
Anyways here is how you will do it
import json
with open("filename.json") as f:
data = json.load(f)
for i in data:
data[i] = 0
with open("filename.json", "w") as f:
json.dump(data, f, indent=4)
Related
Objective: Match key and Override values in json element values on two json files
Input:
Source json file looks like:
{"CLIENT_CODE":"client-d",
"ARM_SUBSCRIPTION_ID":"abced-edfgh-dk6dkk-97dke",
"location":"easteurope"}
Target json file looks like:
{"CLIENT_CODE":"dummy",
"ARM_SUBSCRIPTION_ID":"dummy",
"prefix":"orp",
"location":"westeurope",
"address_space":"10.0.0.0/16",
"aad_login_name":"abcd.onmicrosoft.com",
"aad_login_object_id":"dummy",
"aad_login_tenant_id":"dummy",
"bastion_allocation_method":"Static",
"bastion_sku_type":"premium",
"kv_sku_name":"premium",
"storage_account_tier":"Standard",
"storage_account_replication_type":"LRS",
"storage_account_kind":"StorageV2",
"sql_pool_sku_name":"DW100C",
"node_size_family":"MemoryOptimized"}
Output Expected:
{"CLIENT_CODE":"client-d",
"ARM_SUBSCRIPTION_ID":"abced-edfgh-dk6dkk-97dke",
"prefix":"orp",
"location":"easteurope",
"address_space":"10.0.0.0/16",
"aad_login_name":"abcd.onmicrosoft.com",
"aad_login_object_id":"dummy",
"aad_login_tenant_id":"dummy",
"bastion_allocation_method":"Static",
"bastion_sku_type":"premium",
"kv_sku_name":"premium",
"storage_account_tier":"Standard",
"storage_account_replication_type":"LRS",
"storage_account_kind":"StorageV2"}
What I tried:
import json
with open("D:\ABTest\source.json", encoding='utf-8') as f:
dataset1 = json.loads(f.read())
#print(dataset1)
with open("D:\ABTest\\target.json", encoding='utf-8') as f:
dataset2 = json.loads(f.read())
#print(dataset2)
if dataset1.keys() == dataset2.keys():
dataset2.update(dataset1)
print(dataset2)
But I am not getting any output
Update1 : Now I am able to write it in 3rd file. But not able to update same 2nd file which is target.json
import json
with open("D:\ABTest\source.json", encoding='utf-8') as f:
d1 = json.loads(f.read())
with open("D:\ABTest\\target.json", encoding='utf-8') as f:
d2 = json.loads(f.read())
for key in d1.keys():
if key in d2.keys():
d2[key] = d1[key]
print(d2)
with open('D:\ABTest\combined.json', 'w', ) as f:
json.dump(d2, f, ensure_ascii=False, indent=4)
Update 2:
I made it work. Updated working code in answer section.
Ok, I have now have following worked. It can help someone looking at similar issue
import json
# input file for d1
with open("D:\ABTest\source.json", encoding='utf-8') as f:
d1 = json.load(f)
# input file for d2
with open("D:\ABTest\\target.json", encoding='utf-8') as f:
d2 = json.loads(f)
# output file
with open('D:\ABTest\\target.json', 'w', ) as f:
# update values in d2 with values from d1
for key in d2:
try:
# raise an KeyError if d1 doesn't have the key
d2[key] = d1[key]
except KeyError:
pass
json.dump(d2, f, ensure_ascii=False, indent=4)
print(d2)
Eliminate the if statement and replace it with a try...except... block make the code more pythonic and performant.
The format in the file looks like this
{ 'match' : 'a', 'score' : '2'},{......}
I've tried pd.DataFrame and I've also tried reading it by line but it gives me everything in one cell
I'm new to python
Thanks in advance
Expected result is a pandas dataframe
Try use json_normalize() function
Example:
from pandas.io.json import json_normalize
values = [{'match': 'a', 'score': '2'}, {'match': 'b', 'score': '3'}, {'match': 'c', 'score': '4'}]
df = json_normalize(values)
print(df)
Output:
If one line of your file corresponds to one JSON object, you can do the following:
# import library for working with JSON and pandas
import json
import pandas as pd
# make an empty list
data = []
# open your file and add every row as a dict to the list with data
with open("/path/to/your/file", "r") as file:
for line in file:
data.append(json.loads(line))
# make a pandas data frame
df = pd.DataFrame(data)
If there is more than only one JSON object on one row of your file, then you should find those JSON objects, for example here are two possible options. The solution with the second option would look like this:
# import all you will need
import pandas as pd
import json
from json import JSONDecoder
# define function
def extract_json_objects(text, decoder=JSONDecoder()):
pos = 0
while True:
match = text.find('{', pos)
if match == -1:
break
try:
result, index = decoder.raw_decode(text[match:])
yield result
pos = match + index
except ValueError:
pos = match + 1
# make an empty list
data = []
# open your file and add every JSON object as a dict to the list with data
with open("/path/to/your/file", "r") as file:
for line in file:
for item in extract_json_objects(line):
data.append(item)
# make a pandas data frame
df = pd.DataFrame(data)
I am using the following code in Python 3 to convert ~30,000 json files to a csv.
with open('out.csv', 'w') as f:
for fname in glob("*.json"): # Reads all json from the current directory
with open(fname) as j:
f.write(str(json.load(j)))
f.write('\n')
The json files are timestamps and values, for example {"1501005600":956170,"1501048800":970046,...
The output currently is
.
How can I put each in their own respective cells so the output is ?
I have tried many approaches with csv.writer but I cannot figure this out.
UPDATE
with open('out.csv', 'w') as f:
for fname in glob("*.json"):
with open(fname) as j:
values = json.load(j)
for k, v in values.items():
f.write("{},{},".format(str(k), str(v)))
Parsing is correct but each .json file is on one row now.
A friend helped me get to the bottom of this, hope this may help others.
with open('[insert].csv', 'w') as f:
for fname in glob("*.json"):
with open(fname) as j:
values = json.load(j)
for k, v in values.items():
f.write("{},{},".format(str(k), str(v)))
f.write('\n')
I have an text file, where each line I have cleansed up to be of a json format. I can read each line, clean them, and convert them into a panda dataframe.
My problem is that I want to add/combine them all into one dataframe, but there are more than 200k lines.
I am reading each line in as 'd' = '{"test1":"test2","data":{"key":{"isin":"test3"},"creationTimeStamp":1541491884194,"signal":0,"hPreds":[0,0,0,0],"bidPrice":6.413000,"preferredBidSize":1,"offerPrice":6.415000,"preferredOfferSize":1,"averageTradeSize":1029,"averageTradePrice":0.065252,"changedValues":true,"test4":10,"snapshot":false}}'
Assume I am able to convert each line into a panda... is there a way to append each line into the panda dataframe, such that it is very fast. Right now, with >200k lines, it takes hours to append... reading the file itself takes less than 5 min...
file ='fileName.txt'
with open(file) as f:
content = f.readlines()
content = [x.strip() for x in content]
data = pd.DataFrame()
count = 0
for line in content:
line = line.replace('{"string1','')
z = line.splitlines()
z[0] = z[0][:-1]
z = pd.read_json('[%s]' % ','.join(z))
data = data.append(z)
You may check with Series
pd.Series(d)
Out[154]:
averageTradePrice 0.065
averageTradeSize 109
bidPrice 6.13
changedValues True
creationTimeStamp 15414994
Preds [0, 0, 0, 0]
key {'epic': 'XXX'}
dataLevel 10
offerPrice 3.333
dtype: object
Preds and key's value are list and dict , that is why when you pass it to DataFrame it flag as :
ValueError: arrays must all be same length
Since you mention json
from pandas.io.json import json_normalize
json_normalize(d)
Out[157]:
Preds averageTradePrice ... key.epic offerPrice
0 [0, 0, 0, 0] 0.065 ... XXX 3.333
[1 rows x 9 columns]
I have a csv file which has many columns. Now my requirement is to find all possible value that are present for that specific column.
Is there any built in function in python that helps me to get these values.
You can us pandas.
Example file many_cols.csv:
col1,col2,col3
1,10,100
1,20,100
2,10,100
3,30,100
Find unique values per column:
>>> import pandas as pd
>>> df = pd.read_csv('many_cols.csv')
>>> df.col1.drop_duplicates().tolist()
[1, 2, 3]
>>> df['col2'].drop_duplicates().tolist()
[10, 20, 30]
>>> df['col3'].drop_duplicates().tolist()
[100]
For all columns:
import pandas as pd
df = pd.read_csv('many_cols.csv')
for col in df.columns:
print(col, df[col].drop_duplicates().tolist())
Output:
col1 [1, 2, 3]
col2 [10, 20, 30]
col3 [100]
I would use a set() for this.
Lets say the csv file is this and we want only unique values from second column.
foo,1,bar
baz,2,foo
red,3,blue
git,3,foo
Here is the code that would accomplish this. I am simply printing out the unique values to test that it worked.
import csv
def parse_csv_file(rawCSVFile):
fileLineList = []
with open(rawCSVFile, newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
fileLineList.append(row)
return fileLineList
def main():
uniqueColumnValues = set()
fileLineList = parse_csv_file('sample.csv')
for row in fileLineList:
uniqueColumnValues.add(row[1]) # Selecting 2nd column here.
print(uniqueColumnValues)
if __name__ == '__main__':
main()
Overly "clever" approach to figuring out unique values for all the rows at once (assumes all columns are the same size, though it ignores empty lines seamlessly):
# Assumes somefile was opened properly earlier
csvin = filter(None, csv.reader(somefile))
for i, vals in enumerate(map(sorted, map(set, zip(*csvin)))):
print("Unique values for column", i)
print(vals)
It uses zip(*csvin) to do a table rotation (converting the normal one row at a time output to one column at a time), then uniquifies each column with set, and (for nice output) sorts it.