How to resolve Json decode error in Ubuntu - json

I am running python script on Window 10.
In the python script, I am using json library.
When I run the same script on Ubuntu 20.04(running on VMware), I do see json decode error happening.
This behaviour I dont see when I run in Windows 10.
The following is the error I do get when I run the script in Ubuntu
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "uiControl.py", line 83, in getTcpData
self.taskObj = json.loads(data.decode('utf-8'))
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 5 column 2 (char 73)
In function on_message, I am printing the data received.
The following is the data I receive :
b'{"code":"101","user":"ssmr","evNumber":"TS15EC1100"}'
I call the function addToTaskQueue() to store the received data and then try to parse the data using function BackendParser()
def on_message(self,client, userdata, msg):
print(msg.payload)
self.taskObj = json.loads(msg.payload )
self.taskObj["commType"]= "mqtt"
self.taskObj["transactionType"]= "rx"
taskScheduler.addToTaskQueue(self.taskObj)
def BackendParser(msg):
if(msg["code"] == "101"):
Backend.userName = msg["user"]
Backend.evNumber = msg["evNumber"]
Backend.evChargeControl = "On"
if(Backend.requestStatus == ""):
Backend.requestStatus = "new"
class taskScheduler():
global qTaskList
qTaskList = queue.Queue()
def __init__(self):
super().__init__()
self.tcpCon = tcpServerClient("client")
self.mqttCon = mqttComm()
print("Initiated Task Scheduler class")
#staticmethod
def addToTaskQueue(item):
if not qTaskList.full():
#print("Task added")
qTaskList.put(item)
def executeFromTaskQueue(self):
if not qTaskList.empty():
item = qTaskList.get()
if("mqtt" == item["commType"]):
if("tx" == item["transactionType"]):
pubTopic = item["topic"]
del item["commType"]
del item["transactionType"]
del item["topic"]
self.mqttCon.mqttSend(item,pubTopic)
elif("rx" == item["transactionType"]):
BackendParser(item)
elif("tcp" == item["commType"]):
if("tx" == item["transactionType"]):
del item["commType"]
del item["transactionType"]
tcpServerClient.sendTcpData(item)
elif("rx" == item["transactionType"]):
BackendParser(item)

I figured out the error
I was using the following function getTcpData to receive the data.
I tried printing the data as received and noticed that there were \n characters in the message received.This was not issue when the script was executed in Windows 10. I now added the routine to remove the \n character and now it works fine in Ubuntu.
def getTcpData(self):
print("Waiting for tcp data")
while True:
if(tcpServerClient.clientsocket != None):
data=tcpServerClient.clientsocket.recv(1024)
if data:
print(data)
self.taskObj = json.loads(data.decode('utf-8'))
self.taskObj["commType"]= "tcp"
self.taskObj["transactionType"]= "rx"
taskScheduler.addToTaskQueue(self.taskObj)

Related

json errors when appending data with Python

Good day.
I have a small password generator program and I want to save the created passwords into a json file (append each time) so I can add them to an SQLITE3 database.
Just trying to do the append functionality I receive several errors that I don't understand.
Here are the errors I receive and below that is the code itself.
I'm quite new to Python so additional details are welcomed.
Traceback (most recent call last):
File "C:\Users\whitmech\OneDrive - Six Continents Hotels, Inc\04 - Python\02_Mosh_Python_Course\Py_Projects\PWGenerator.py", line 32, in
data = json.load(file)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json_init_.py", line 293, in load
return loads(fp.read(),
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json_init_.py", line 346, in loads
return _default_decoder.decode(s)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1264.0_x64__qbz5n2kfra8p0\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
import random
import string
import sqlite3
import json
from pathlib import Path
print('hello, Welcome to Password generator!')
# input the length of password
length = int(input('\nEnter the length of password: '))
# define data
lower = string.ascii_lowercase
upper = string.ascii_uppercase
num = string.digits
symbols = string.punctuation
# string.ascii_letters
# combine the data
all = lower + upper + num + symbols
# use random
temp = random.sample(all, length)
# create the password
password = "".join(temp)
filename = 'saved.json'
entry = {password}
with open(filename, "r+") as file:
data = json.load(file)
data.append(entry)
file.seek(0)
json.dump(data, file)
# print the password
print(password)
Update: I've changed the JSON code as directed and it works but when trying to do the SQLite3 code I'm knowing receiving a typeerror
Code:
with open(filename, "r+") as file:
try:
data = json.load(file)
data.append(entry)
except json.decoder.JSONDecodeError as e:
data = entry
file.seek(0)
json.dump(data, file)
# print the password
print(password)
store = input('Would you like to store the password? ')
if store == "Yes":
pwStored = json.loads(Path("saved.json").read_text())
with sqlite3.connect("db.pws") as conn:
command = "INSERT INTO Passwords VALUES (?)"
for i in pwStored:
conn.execute(command, tuple(i.values)) # Error with this code
conn.commit()
else:
exit()
Error:
AttributeError: 'str' object has no attribute 'values'
The error is because
Your json file is empty, you need to update the following block
entry = [password]
with open(filename, "r+") as file:
try:
data = json.load(file)
data.extend(entry)
except json.decoder.JSONDecodeError as e:
data = entry
file.seek(0)
json.dump(data, file)
Also you are adding password in a set ie., entry, and it will again throw you an error TypeError: Object of type set is not JSON serializable
So you need to convert that to either a list or dict
Note: Here I have used entry as a list

Randomatically select from a json discord.py

I'm trying to create a command where a random meme is shown from a json.
Unfortunately I can't do it ... I can do it using random.choice normally but since I'm a lazy person to update the code I always created a command that would allow me to add them in a json. So I have to be able to randomly extract the main key of my json. How could I do?
code:
#client.command(aliases=["Meme","MEME"])
async def meme(ctx):
await ctx.message.delete()
meme_data_dict = meme_data.json
url = random.choice(list(meme_data_dict.keys))
print(url)
author = meme_data_dict[url]["autore"]
title = meme_data_dict[url]["titolo"]
channel = client.get_channel(638016878589116416)
if ctx.channel.id == channel.id:
embed = discord.Embed(
title=f'{title}',
color=0x003399
)
embed.set_author(name=f'fatto da {author}')
embed.set_footer(text=f'Richiesto da {ctx.author.name}')
embed.set_image(url=url)
await ctx.send(embed=embed)
else:
embed = discord.Embed(
color=0xa61022
)
embed.set_author(
name=f"Per questo comando devi usare {channel.name}!",
)
await ctx.send(embed=embed, delete_after=10.0)
return
Json (your name is meme_data.json):
{"https://cdn.discordapp.com/avatars/491769129318088714/f23fd300d377ab133db1d6ac7db5d10b.webp?size=1024": {"autore": "10", "titolo": "sesso pazzoide"}, "https://cdn.discordapp.com/attachments/638016878589116416/839977460384923659/soviet_soldier_2.jpg": {"autore": "pene", "titolo": "ciaociao"}}
This is how my json is active:
#client.event
async def on_ready():
global meme_data
try:
with open('meme_data.json') as f:
meme_data = json.load(f)
except FileNotFoundError:
print("Impossibile caricare meme_data.json")
meme_data = {}
error:
Ignoring exception in command meme:
Traceback (most recent call last):
File "C:\Users\PC GIUSEPPE\PycharmProjects\LMIIBot Development\venv\lib\site-packages\discord\ext\commands\core.py", line 85, in wrapped
ret = await coro(*args, **kwargs)
File "C:/Users/PC GIUSEPPE/PycharmProjects/LMIIBot Development/LMIIBot Development.py", line 2235, in meme
meme_data_dict = meme_data.json
AttributeError: 'dict' object has no attribute 'json'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\PC GIUSEPPE\PycharmProjects\LMIIBot Development\venv\lib\site-packages\discord\ext\commands\bot.py", line 903, in invoke
await ctx.command.invoke(ctx)
File "C:\Users\PC GIUSEPPE\PycharmProjects\LMIIBot Development\venv\lib\site-packages\discord\ext\commands\core.py", line 859, in invoke
await injected(*ctx.args, **ctx.kwargs)
File "C:\Users\PC GIUSEPPE\PycharmProjects\LMIIBot Development\venv\lib\site-packages\discord\ext\commands\core.py", line 94, in wrapped
raise CommandInvokeError(exc) from exc
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: AttributeError: 'dict' object has no attribute 'json'
To get a random object of a dictionary (imported json file), you use url = random.choice(list(meme_data_dict.keys)). That in your code would be:
#client.command(aliases=["Meme","MEME"])
async def meme(ctx):
await ctx.message.delete()
meme_data_dict = meme_data #you have to rename your variable meme, for example to meme_data, because there is already the function called meme
url = random.choice(list(meme_data_dict.keys()))
print(url)
author = meme_data_dict[url]["autore"]
title = meme_data_dict[url]["titolo"]
#the rest of your code can stay the same
If this doesn't answer your question, clarify more what the problem is.

How to use mock_open with json.load()?

I'm trying to get a unit test working that validates a function that reads credentials from a JSON-encoded file. Since the credentials themselves aren't fixed, the unit test needs to provide some and then test that they are correctly retrieved.
Here is the credentials function:
def read_credentials():
basedir = os.path.dirname(__file__)
with open(os.path.join(basedir, "authentication.json")) as f:
data = json.load(f)
return data["bot_name"], data["bot_password"]
and here is the test:
def test_credentials(self):
with patch("builtins.open", mock_open(
read_data='{"bot_name": "name", "bot_password": "password"}\n'
)):
name, password = shared.read_credentials()
self.assertEqual(name, "name")
self.assertEqual(password, "password")
However, when I run the test, the json code blows up with a decode error. Looking at the json code itself, I'm struggling to see why the mock test is failing because json.load(f) simply calls f.read() then calls json.loads().
Indeed, if I change my authentication function to the following, the unit test works:
def read_credentials():
# Read the authentication file from the current directory and create a
# HTTPBasicAuth object that can then be used for future calls.
basedir = os.path.dirname(__file__)
with open(os.path.join(basedir, "authentication.json")) as f:
content = f.read()
data = json.loads(content)
return data["bot_name"], data["bot_password"]
I don't necessarily mind leaving my code in this form, but I'd like to understand if I've got something wrong in my test that would allow me to keep my function in its original form.
Stack trace:
Traceback (most recent call last):
File "test_shared.py", line 56, in test_credentials
shared.read_credentials()
File "shared.py", line 60, in read_credentials
data = json.loads(content)
File "/home/philip/.local/share/virtualenvs/atlassian-webhook-basic-3gOncDp4/lib/python3.6/site-packages/flask/json/__init__.py", line 205, in loads
return _json.loads(s, **kwargs)
File "/usr/lib/python3.6/json/__init__.py", line 367, in loads
return cls(**kw).decode(s)
File "/usr/lib/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.6/json/decoder.py", line 357, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I had the same issue and got around it by mocking json.load and builtins.open:
import json
from unittest.mock import patch, MagicMock
# I don't care about the actual open
p1 = patch( "builtins.open", MagicMock() )
m = MagicMock( side_effect = [ { "foo": "bar" } ] )
p2 = patch( "json.load", m )
with p1 as p_open:
with p2 as p_json_load:
f = open( "filename" )
print( json.load( f ) )
Result:
{'foo': 'bar'}
I had the exact same issue and solved it. Full code below, first the function to test, then the test itself.
The original function I want to test loads a json file that is structured like a dictionary, and checks to see if there's a specific key-value pair in it:
def check_if_file_has_real_data(filepath):
with open(filepath, "r") as f:
data = json.load(f)
if "fake" in data["the_data"]:
return False
else:
return True
But I want to test this without loading any actual file, exactly as you describe. Here's how I solved it:
from my_module import check_if_file_has_real_data
import mock
#mock.patch("my_module.json.load")
#mock.patch("my_module.open")
def test_check_if_file_has_real_data(mock_open, mock_json_load):
mock_json_load.return_value = dict({"the_data": "This is fake data"})
assert check_if_file_has_real_data("filepath") == False
mock_json_load.return_value = dict({"the_data": "This is real data"})
assert check_if_file_has_real_data("filepath") == True
The mock_open object isn't called explicitly in the test function, but if you don't include that decorator and argument you get a filepath error when the with open part of the check_if_file_has_real_data function tries to run using the actual open function rather than the MagicMock object that's been passed into it.
Then you overwrite the response provided by the json.load mock with whatever you want to test.

Handling Django form.save() ValueError

I am a newbie to Django error handling and I spent a week trying to figure out what exactly goes wrong when trying to update a MySQL table using Django forms. I end up with ValueError: invalid literal for int() with base 10: '\x01' error. I tried to surround the erroneous code with try catch block trapping valueError and printing the row sql query, and here's what I get.
The code:
def updateTask(request, task_id):
#cur_usr_sale_point = PersonUnique.objects.filter(employees__employeeuser__auth_user = request.user.id).values_list('agreementemployees__agreement_unique__sale_point_id',flat=True)
selected_task = Tasks.objects.get(id=task_id)
responsible_people = TaskResponsiblePeople.objects.get(task_id = task_id)
task_table = Tasks. objects.all()
if request.method == 'POST':
task_form = TaskForm(request.POST,instance=selected_task)
responsible_people_form = TaskResponsiblePeopleForm(request.POST, instance = responsible_people)
if task_form.is_valid() and responsible_people_form.is_valid():
responsible_people_instance = responsible_people_form.save(commit=False)
try:
responsible_people_instance.task = task_form.save()
responsible_people_form.save()
except ValueError:
from django.db import connection
print connection.queries[-1]
return HttpResponseRedirect(reverse('task_list'))
Print gives me an absolutely valid MySQL Select-statement (to my surpise, I expected an update-statement).
The traceback without try-catch block:
Internal Server Error: /task_list/update_task/200/
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\django\core\handlers\base.py", line 149, in get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Python27\lib\site-packages\django\core\handlers\base.py", line 147, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "\\10.8.0.1\share\djprj\djprj\djprj\task\views.py", line 101, in updateTask
task_form.save();
File "C:\Python27\lib\site-packages\django\forms\models.py", line 451, in save
self.instance.save()
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 700, in save
force_update=force_update, update_fields=update_fields)
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 728, in save_base
updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 793, in _save_table
forced_update)
File "C:\Python27\lib\site-packages\django\db\models\base.py", line 843, in _do_update
return filtered._update(values) > 0
File "C:\Python27\lib\site-packages\django\db\models\query.py", line 645, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "C:\Python27\lib\site-packages\django\db\models\sql\compiler.py", line 1149, in execute_sql
cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
File "C:\Python27\lib\site-packages\django\db\models\sql\compiler.py", line 837, in execute_sql
sql, params = self.as_sql()
File "C:\Python27\lib\site-packages\django\db\models\sql\compiler.py", line 1117, in as_sql
val = field.get_db_prep_save(val, connection=self.connection)
File "C:\Python27\lib\site-packages\django\db\models\fields\__init__.py", line 728, in get_db_prep_save
prepared=False)
File "C:\Python27\lib\site-packages\django\db\models\fields\__init__.py", line 720, in get_db_prep_value
value = self.get_prep_value(value)
File "C:\Python27\lib\site-packages\django\db\models\fields\__init__.py", line 1853, in get_prep_value
return int(value)
ValueError: invalid literal for int() with base 10: '\x01'
[10/Apr/2016 11:15:46] "POST /task_list/update_task/200/ HTTP/1.1" 500 126245
Help me out please !!!
EDIT: added is_valid method
You need to first use the form's is_valid method:
A Form instance has an is_valid() method, which runs validation routines for all its fields. When this method is called, if all fields contain valid data.
from django.shortcuts import render
from django.http import HttpResponseRedirect
from .forms import NameForm
def get_name(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = NameForm(request.POST)
# check whether it's valid:
if form.is_valid():
# process the data in form.cleaned_data as required
# ...
# redirect to a new URL:
return HttpResponseRedirect('/thanks/')
# if a GET (or any other method) we'll create a blank form
else:
form = NameForm()
return render(request, 'name.html', {'form': form})
Source: Django Docs: Working with forms
I would recommend that you read the above documentation.

error loading json using topsy

When i load single record json is created just fine when i try to load multiple records i get this error. Sorry i am new to python http://tny.cz/ce1baaba
Traceback (most recent call last):
File "TweetGraber.py", line 26, in <module>
get_tweets_by_query(topic)
File "TweetGraber.py", line 15, in get_tweets_by_query
json_tree = json.loads(source)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.py", line 338, in loads
return _default_decoder.decode(s)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py", line 368, in decode
raise ValueError(errmsg("Extra data", s, end, len(s)))
ValueError: Extra data: line 2 column 1 - line 11 column 1 (char 2380 - 46974)
Here is my code
def get_tweets_by_query(query, count=10):
"""A function that gets the tweets by a query."""
Tweets=[]
queryEncoded=urllib.quote(query)
api_key = "xxxxx"
source=urllib.urlopen("http://api.topsy.com/v2/content/bulktweets.json?q=%s&type=tweet&offset=0&perpage=%s&window=realtime&apikey=%s" % (queryEncoded, count, api_key)).read()
json_tree = json.loads(source)
pprint(json_tree)
topic = raw_input("Please enter a topic: ")
get_tweets_by_query(topic)
Thanks Timusan I was able to correct my json The problem with the original it was missing the root element "[" which showed we are expecting array and there "," was missing after end of each object. So here is fixed code.
So here is the code
def get_tweets_by_query(query, count=10):
"""A function that gets the tweets by a query."""
Tweets=[]
queryEncoded=urllib.quote(query)
api_key = "xx"
source=urllib.urlopen("http://api.topsy.com/v2/content/bulktweets.json?q=%s&type=tweet&offset=0&perpage=%s&window=realtime&apikey=%s" % (queryEncoded, count, api_key)).read()
source="["+source+"]"
source=source.replace("}\n{","},{")
json_tree = json.loads(source)
pprint(json_tree)