json errors when appending data with Python - json

Good day.
I have a small password generator program and I want to save the created passwords into a json file (append each time) so I can add them to an SQLITE3 database.
Just trying to do the append functionality I receive several errors that I don't understand.
Here are the errors I receive and below that is the code itself.
I'm quite new to Python so additional details are welcomed.
Traceback (most recent call last):
File "C:\Users\whitmech\OneDrive - Six Continents Hotels, Inc\04 - Python\02_Mosh_Python_Course\Py_Projects\PWGenerator.py", line 32, in
data = json.load(file)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json_init_.py", line 293, in load
return loads(fp.read(),
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json_init_.py", line 346, in loads
return _default_decoder.decode(s)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
import random
import string
import sqlite3
import json
from pathlib import Path
print('hello, Welcome to Password generator!')
# input the length of password
length = int(input('\nEnter the length of password: '))
# define data
lower = string.ascii_lowercase
upper = string.ascii_uppercase
num = string.digits
symbols = string.punctuation
# string.ascii_letters
# combine the data
all = lower + upper + num + symbols
# use random
temp = random.sample(all, length)
# create the password
password = "".join(temp)
filename = 'saved.json'
entry = {password}
with open(filename, "r+") as file:
data = json.load(file)
data.append(entry)
file.seek(0)
json.dump(data, file)
# print the password
print(password)
Update: I've changed the JSON code as directed and it works but when trying to do the SQLite3 code I'm knowing receiving a typeerror
Code:
with open(filename, "r+") as file:
try:
data = json.load(file)
data.append(entry)
except json.decoder.JSONDecodeError as e:
data = entry
file.seek(0)
json.dump(data, file)
# print the password
print(password)
store = input('Would you like to store the password? ')
if store == "Yes":
pwStored = json.loads(Path("saved.json").read_text())
with sqlite3.connect("db.pws") as conn:
command = "INSERT INTO Passwords VALUES (?)"
for i in pwStored:
conn.execute(command, tuple(i.values)) # Error with this code
conn.commit()
else:
exit()
Error:
AttributeError: 'str' object has no attribute 'values'

The error is because
Your json file is empty, you need to update the following block
entry = [password]
with open(filename, "r+") as file:
try:
data = json.load(file)
data.extend(entry)
except json.decoder.JSONDecodeError as e:
data = entry
file.seek(0)
json.dump(data, file)
Also you are adding password in a set ie., entry, and it will again throw you an error TypeError: Object of type set is not JSON serializable
So you need to convert that to either a list or dict
Note: Here I have used entry as a list

Related

How to resolve Json decode error in Ubuntu

I am running python script on Window 10.
In the python script, I am using json library.
When I run the same script on Ubuntu 20.04(running on VMware), I do see json decode error happening.
This behaviour I dont see when I run in Windows 10.
The following is the error I do get when I run the script in Ubuntu
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "uiControl.py", line 83, in getTcpData
self.taskObj = json.loads(data.decode('utf-8'))
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 5 column 2 (char 73)
In function on_message, I am printing the data received.
The following is the data I receive :
b'{"code":"101","user":"ssmr","evNumber":"TS15EC1100"}'
I call the function addToTaskQueue() to store the received data and then try to parse the data using function BackendParser()
def on_message(self,client, userdata, msg):
print(msg.payload)
self.taskObj = json.loads(msg.payload )
self.taskObj["commType"]= "mqtt"
self.taskObj["transactionType"]= "rx"
taskScheduler.addToTaskQueue(self.taskObj)
def BackendParser(msg):
if(msg["code"] == "101"):
Backend.userName = msg["user"]
Backend.evNumber = msg["evNumber"]
Backend.evChargeControl = "On"
if(Backend.requestStatus == ""):
Backend.requestStatus = "new"
class taskScheduler():
global qTaskList
qTaskList = queue.Queue()
def __init__(self):
super().__init__()
self.tcpCon = tcpServerClient("client")
self.mqttCon = mqttComm()
print("Initiated Task Scheduler class")
#staticmethod
def addToTaskQueue(item):
if not qTaskList.full():
#print("Task added")
qTaskList.put(item)
def executeFromTaskQueue(self):
if not qTaskList.empty():
item = qTaskList.get()
if("mqtt" == item["commType"]):
if("tx" == item["transactionType"]):
pubTopic = item["topic"]
del item["commType"]
del item["transactionType"]
del item["topic"]
self.mqttCon.mqttSend(item,pubTopic)
elif("rx" == item["transactionType"]):
BackendParser(item)
elif("tcp" == item["commType"]):
if("tx" == item["transactionType"]):
del item["commType"]
del item["transactionType"]
tcpServerClient.sendTcpData(item)
elif("rx" == item["transactionType"]):
BackendParser(item)
I figured out the error
I was using the following function getTcpData to receive the data.
I tried printing the data as received and noticed that there were \n characters in the message received.This was not issue when the script was executed in Windows 10. I now added the routine to remove the \n character and now it works fine in Ubuntu.
def getTcpData(self):
print("Waiting for tcp data")
while True:
if(tcpServerClient.clientsocket != None):
data=tcpServerClient.clientsocket.recv(1024)
if data:
print(data)
self.taskObj = json.loads(data.decode('utf-8'))
self.taskObj["commType"]= "tcp"
self.taskObj["transactionType"]= "rx"
taskScheduler.addToTaskQueue(self.taskObj)

'NoneType' object has no attribute 'read' when reading from JSON file

I am making a script for a school project that requires that I receive a JSON file that tells me if a license plate is visible in a picture. Right now the code sends a POST with an image to an API that then gives me a JSON in return, that JSON data is sent to the file "lastResponse.json."
The code that is giving out the error
with open('lastResponse.json', 'r+') as fp:
f = json.dump(r.json(), fp, sort_keys=True, indent=4) # Where the response data is sent to the JSON
data = json.load(f) # Line that triggers the error
print(data["results"]) # Debug code
print("------------------") # Debug code
print(data) # Debug code
# This statement just checks if a license plate is visible
if data["results"]["plate"] is None:
print("No car detected!")
else:
print("Car with plate number '" + data["results"]["plate"] + "' has been detected")
The Error
Traceback (most recent call last):
File "DetectionFinished.py", line 19, in <module>
data = json.load(f)
File "/usr/lib/python3.7/json/__init__.py", line 293, in load
return loads(fp.read(),
AttributeError: 'NoneType' object has no attribute 'read'
I am not very experienced in Python so I would appreciate explanations!
It turns out, after rereading the API's documentation and using their examples I was able to fix my issues
import requests
from pprint import pprint
regions = ['gb', 'it']
with open('/path/to/car.jpg', 'rb') as fp:
response = requests.post(
'https://api.platerecognizer.com/v1/plate-reader/',
data=dict(regions=regions), # Optional
files=dict(upload=fp),
headers={'Authorization': 'Token API_TOKEN'})
pprint(response.json())

How to use mock_open with json.load()?

I'm trying to get a unit test working that validates a function that reads credentials from a JSON-encoded file. Since the credentials themselves aren't fixed, the unit test needs to provide some and then test that they are correctly retrieved.
Here is the credentials function:
def read_credentials():
basedir = os.path.dirname(__file__)
with open(os.path.join(basedir, "authentication.json")) as f:
data = json.load(f)
return data["bot_name"], data["bot_password"]
and here is the test:
def test_credentials(self):
with patch("builtins.open", mock_open(
read_data='{"bot_name": "name", "bot_password": "password"}\n'
)):
name, password = shared.read_credentials()
self.assertEqual(name, "name")
self.assertEqual(password, "password")
However, when I run the test, the json code blows up with a decode error. Looking at the json code itself, I'm struggling to see why the mock test is failing because json.load(f) simply calls f.read() then calls json.loads().
Indeed, if I change my authentication function to the following, the unit test works:
def read_credentials():
# Read the authentication file from the current directory and create a
# HTTPBasicAuth object that can then be used for future calls.
basedir = os.path.dirname(__file__)
with open(os.path.join(basedir, "authentication.json")) as f:
content = f.read()
data = json.loads(content)
return data["bot_name"], data["bot_password"]
I don't necessarily mind leaving my code in this form, but I'd like to understand if I've got something wrong in my test that would allow me to keep my function in its original form.
Stack trace:
Traceback (most recent call last):
File "test_shared.py", line 56, in test_credentials
shared.read_credentials()
File "shared.py", line 60, in read_credentials
data = json.loads(content)
File "/home/philip/.local/share/virtualenvs/atlassian-webhook-basic-3gOncDp4/lib/python3.6/site-packages/flask/json/__init__.py", line 205, in loads
return _json.loads(s, **kwargs)
File "/usr/lib/python3.6/json/__init__.py", line 367, in loads
return cls(**kw).decode(s)
File "/usr/lib/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.6/json/decoder.py", line 357, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I had the same issue and got around it by mocking json.load and builtins.open:
import json
from unittest.mock import patch, MagicMock
# I don't care about the actual open
p1 = patch( "builtins.open", MagicMock() )
m = MagicMock( side_effect = [ { "foo": "bar" } ] )
p2 = patch( "json.load", m )
with p1 as p_open:
with p2 as p_json_load:
f = open( "filename" )
print( json.load( f ) )
Result:
{'foo': 'bar'}
I had the exact same issue and solved it. Full code below, first the function to test, then the test itself.
The original function I want to test loads a json file that is structured like a dictionary, and checks to see if there's a specific key-value pair in it:
def check_if_file_has_real_data(filepath):
with open(filepath, "r") as f:
data = json.load(f)
if "fake" in data["the_data"]:
return False
else:
return True
But I want to test this without loading any actual file, exactly as you describe. Here's how I solved it:
from my_module import check_if_file_has_real_data
import mock
#mock.patch("my_module.json.load")
#mock.patch("my_module.open")
def test_check_if_file_has_real_data(mock_open, mock_json_load):
mock_json_load.return_value = dict({"the_data": "This is fake data"})
assert check_if_file_has_real_data("filepath") == False
mock_json_load.return_value = dict({"the_data": "This is real data"})
assert check_if_file_has_real_data("filepath") == True
The mock_open object isn't called explicitly in the test function, but if you don't include that decorator and argument you get a filepath error when the with open part of the check_if_file_has_real_data function tries to run using the actual open function rather than the MagicMock object that's been passed into it.
Then you overwrite the response provided by the json.load mock with whatever you want to test.

Serialize in JSON a base64 encoded data

I'm writing a script to automate data generation for a demo and I need to serialize in a JSON some data. Part of this data is an image, so I encoded it in base64, but when I try to run my script I get:
Traceback (most recent call last):
File "lazyAutomationScript.py", line 113, in <module>
json.dump(out_dict, outfile)
File "/usr/lib/python3.4/json/__init__.py", line 178, in dump
for chunk in iterable:
File "/usr/lib/python3.4/json/encoder.py", line 422, in _iterencode
yield from _iterencode_dict(o, _current_indent_level)
File "/usr/lib/python3.4/json/encoder.py", line 396, in _iterencode_dict
yield from chunks
File "/usr/lib/python3.4/json/encoder.py", line 396, in _iterencode_dict
yield from chunks
File "/usr/lib/python3.4/json/encoder.py", line 429, in _iterencode
o = _default(o)
File "/usr/lib/python3.4/json/encoder.py", line 173, in default
raise TypeError(repr(o) + " is not JSON serializable")
TypeError: b'iVBORw0KGgoAAAANSUhEUgAADWcAABRACAYAAABf7ZytAAAABGdB...
...
BF2jhLaJNmRwAAAAAElFTkSuQmCC' is not JSON serializable
As far as I know, a base64-encoded-whatever (a PNG image, in this case) is just a string, so it should pose to problem to serializating. What am I missing?
You must be careful about the datatypes.
If you read a binary image, you get bytes.
If you encode these bytes in base64, you get ... bytes again! (see documentation on b64encode)
json can't handle raw bytes, that's why you get the error.
I have just written some example, with comments, I hope it helps:
from base64 import b64encode
from json import dumps
ENCODING = 'utf-8'
IMAGE_NAME = 'spam.jpg'
JSON_NAME = 'output.json'
# first: reading the binary stuff
# note the 'rb' flag
# result: bytes
with open(IMAGE_NAME, 'rb') as open_file:
byte_content = open_file.read()
# second: base64 encode read data
# result: bytes (again)
base64_bytes = b64encode(byte_content)
# third: decode these bytes to text
# result: string (in utf-8)
base64_string = base64_bytes.decode(ENCODING)
# optional: doing stuff with the data
# result here: some dict
raw_data = {IMAGE_NAME: base64_string}
# now: encoding the data to json
# result: string
json_data = dumps(raw_data, indent=2)
# finally: writing the json string to disk
# note the 'w' flag, no 'b' needed as we deal with text here
with open(JSON_NAME, 'w') as another_open_file:
another_open_file.write(json_data)
Alternative solution would be encoding stuff on the fly with a custom encoder:
import json
from base64 import b64encode
class Base64Encoder(json.JSONEncoder):
# pylint: disable=method-hidden
def default(self, o):
if isinstance(o, bytes):
return b64encode(o).decode()
return json.JSONEncoder.default(self, o)
Having that defined you can do:
m = {'key': b'\x9c\x13\xff\x00'}
json.dumps(m, cls=Base64Encoder)
It will produce:
'{"key": "nBP/AA=="}'
What am I missing?
The error is yelling that a binary is not JSON serializable.
from base64 import b64encode
# *binary representation* of the base64 string
assert b64encode(b"binary content") == b'YmluYXJ5IGNvbnRlbnQ='
# base64 string
assert b64encode(b"binary content").decode('utf-8') == 'YmluYXJ5IGNvbnRlbnQ='
The latter is definitely "JSON serializable" because is the base64 string representation of the binary b"binary content".

Handling Django form.save() ValueError

I am a newbie to Django error handling and I spent a week trying to figure out what exactly goes wrong when trying to update a MySQL table using Django forms. I end up with ValueError: invalid literal for int() with base 10: '\x01' error. I tried to surround the erroneous code with try catch block trapping valueError and printing the row sql query, and here's what I get.
The code:
def updateTask(request, task_id):
#cur_usr_sale_point = PersonUnique.objects.filter(employees__employeeuser__auth_user = request.user.id).values_list('agreementemployees__agreement_unique__sale_point_id',flat=True)
selected_task = Tasks.objects.get(id=task_id)
responsible_people = TaskResponsiblePeople.objects.get(task_id = task_id)
task_table = Tasks. objects.all()
if request.method == 'POST':
task_form = TaskForm(request.POST,instance=selected_task)
responsible_people_form = TaskResponsiblePeopleForm(request.POST, instance = responsible_people)
if task_form.is_valid() and responsible_people_form.is_valid():
responsible_people_instance = responsible_people_form.save(commit=False)
try:
responsible_people_instance.task = task_form.save()
responsible_people_form.save()
except ValueError:
from django.db import connection
print connection.queries[-1]
return HttpResponseRedirect(reverse('task_list'))
Print gives me an absolutely valid MySQL Select-statement (to my surpise, I expected an update-statement).
The traceback without try-catch block:
Internal Server Error: /task_list/update_task/200/
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\django\core\handlers\base.py", line 149, in get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Python27\lib\site-packages\django\core\handlers\base.py", line 147, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "\\10.8.0.1\share\djprj\djprj\djprj\task\views.py", line 101, in updateTask
task_form.save();
File "C:\Python27\lib\site-packages\django\forms\models.py", line 451, in save
self.instance.save()
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 700, in save
force_update=force_update, update_fields=update_fields)
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 728, in save_base
updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 793, in _save_table
forced_update)
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 843, in _do_update
return filtered._update(values) > 0
File "C:\Python27\lib\site-packages\django\db\models\query.py", line 645, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "C:\Python27\lib\site-packages\django\db\models\sql\compiler.py", line 1149, in execute_sql
cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
File "C:\Python27\lib\site-packages\django\db\models\sql\compiler.py", line 837, in execute_sql
sql, params = self.as_sql()
File "C:\Python27\lib\site-packages\django\db\models\sql\compiler.py", line 1117, in as_sql
val = field.get_db_prep_save(val, connection=self.connection)
File "C:\Python27\lib\site-packages\django\db\models\fields\__init__.py", line 728, in get_db_prep_save
prepared=False)
File "C:\Python27\lib\site-packages\django\db\models\fields\__init__.py", line 720, in get_db_prep_value
value = self.get_prep_value(value)
File "C:\Python27\lib\site-packages\django\db\models\fields\__init__.py", line 1853, in get_prep_value
return int(value)
ValueError: invalid literal for int() with base 10: '\x01'
[10/Apr/2016 11:15:46] "POST /task_list/update_task/200/ HTTP/1.1" 500 126245
Help me out please !!!
EDIT: added is_valid method
You need to first use the form's is_valid method:
A Form instance has an is_valid() method, which runs validation routines for all its fields. When this method is called, if all fields contain valid data.
from django.shortcuts import render
from django.http import HttpResponseRedirect
from .forms import NameForm
def get_name(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = NameForm(request.POST)
# check whether it's valid:
if form.is_valid():
# process the data in form.cleaned_data as required
# ...
# redirect to a new URL:
return HttpResponseRedirect('/thanks/')
# if a GET (or any other method) we'll create a blank form
else:
form = NameForm()
return render(request, 'name.html', {'form': form})
Source: Django Docs: Working with forms
I would recommend that you read the above documentation.