I'm inserting some data to a database and most of the queries are inserted correctly but I keep getting at least one random query error.
I'm using Python 3, MySQL 5.6.17 and MySQL python connector 2.1.3 (upgraded after having the same problem with 2.0.2).
The queries are run in a Multiprocessing Pool map_async().
multiprocessing.pool.RemoteTraceback: bytearray index out of range
Traceback (most recent call last):
File "./../../../my-python-script.py", line 930, in insert_into_database
mysql_query(mysql_script, values) # <-- My mysql wrapper function
File "./../../../my-python-script.py", line 297, in mysql_query
for row in results:
File "./../../../mysql/connector/cursor.py", line 450, in _execute_iter
result = next(query_iter)
File "./../../../mysql/connector/connection.py", line 520, in cmd_query_iter
yield self._handle_result(self._send_cmd(ServerCmd.QUERY, statements))
File "./../../../mysql/connector/connection.py", line 405, in _handle_result
self._socket.recv(), self.python_charset)
File "./../../../mysql/connector/protocol.py", line 238, in parse_column
(packet, _) = utils.read_lc_string(packet[4:]) # catalog
File "./../../../mysql/connector/utils.py", line 199, in read_lc_string
if buf[0] == 251: # \xfb
IndexError: bytearray index out of range
The above exception was the direct cause of the following exception:
IndexError: bytearray index out of range
Or sometimes I get (from the "for row in results" line)
File "./../../../mysql/connector/cursor.py", line 450, in _execute_iter
result = next(query_iter)
File "./../../../mysql/connector/connection.py", line 520, in cmd_query_iter
yield self._handle_result(self._send_cmd(ServerCmd.QUERY, statements))
File "./../../../mysql/connector/connection.py", line 384, in _handle_result
elif packet[4] == 0:
IndexError: bytearray index out of range
The above exception was the direct cause of the following exception:
IndexError: bytearray index out of range
My setup is something like
class InsertData:
def __init__(self):
with(multiprocessing.Pool(2) as Pool:
Pool.map_async(self.insert_into_database(),set(1,2,3.....))
Pool.close()
Pool.join()
def insert_into_database(self,values):
# use the values to do some calculations then insert to database
mysql_query(mysql_script, values)
def mysql_query(script, values):
cursor.execute(query, values, multi = True)
And the sql script
'INSERT INTO table1 ( column1 ) VALUES ( "x" ); '
'SET #table1 = LAST_INSERT_ID(); '
'INSERT INTO table2 ( column1, column2 ) VALUES ( "y", #table1 ); '
'SET #table2 = LAST_INSERT_ID(); '
...
I'm currently looking at the connector.py and the utils code trying to figure what's happening. But this is too advanced for me.
https://github.com/mysql/mysql-connector-python/blob/master/lib/mysql/connector/connection.py#L368
https://github.com/mysql/mysql-connector-python/blob/master/lib/mysql/connector/utils.py#L167
In a desperate attempt I've tried setting the buffered to True https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursorbuffered.html
I need to read up on bytearrays but I suspect that my query script is causing the problem because I didn't have (I think?) this problem when I ran the queries one connector at a time cursor.execute(query, values, multi = False)
When I send the database connection as described in Accessing a MySQL connection pool from Python multiprocessing the problem disappears.
Something like
mysql_conn = None
def db_conn():
global mysql_conn
mysql_conn = connector.connect(...)
class InsertData:
def __init__(self):
with(multiprocessing.Pool(2, initializer = db_conn) as Pool:
Pool.map_async(self.insert_into_database(),set(1,2,3.....))
Pool.close()
Pool.join()
def insert_into_database(self,values):
# use the values to do some calculations then insert to database
self.mysql_query(mysql_script, values)
def mysql_query(script, values):
cursor = mysql_conn.cursor()
cursor.execute(query, values, multi = True)
Related
i am trying to fetch the list of sql query running more than 3600 sec and kill those id's using python below is the code
import json
import mysql.connector
import pymysql
def main():
# TODO implement
connection = pymysql.connect(user='', password='',
host='',
port=3306,
database='');
cursor = connection.cursor() # get the cursor
# cursor.execute('SHOW PROCESSLIST;')
# extracted_data = cursor.fetchall();
# for i in extracted_data:
# print(i)
with connection.cursor() as cursor:
print(cursor.execute('SHOW PROCESSLIST'))
for item in cursor.fetchall():
if item.get('Time') > 3600 and item.get('command') == 'query':
_id = item.get('Id')
print('kill %s' % item)
cursor.execute('kill %s', _id)
connection.close()
main()
below is the error i am getting
"C:\drive c\pyfile\venv\Scripts\python.exe" "C:/drive c/pyfile/sqlnew2.py"
Traceback (most recent call last):
File "C:\drive c\pyfile\sqlnew2.py", line 23, in <module>
main()
File "C:\drive c\pyfile\sqlnew2.py", line 18, in main
if item.get('Time') > 3600 and item.get('command') == 'query':
AttributeError: 'tuple' object has no attribute 'get'
The .fetchall() method returns a tuple, not a dictionary. Therefore you should access the elements using the numerical indexes, for example item[0], item[1], etc
As an alternative, if you want to fetch the results as a dictionary, you can use a DictCursor
First import it:
import pymysql.cursors
Then modify the cursor line like that:
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
...
I am having an issue getting my UPDATE Query to work. I am using python 3.7 and building a gui using tkinter. I have an entry box that a user can select an Item by ID number, that auto-populates the following entry boxes and the user can then modify the entries. My error occurs when it comes time to save the changes. I have a button that calls a function save when clicked. see below
def save():
conn = mysql.connect(
host="localhost",
user="XXXX",
passwd="XXXX",
database="inventory")
c = conn.cursor()
a0 = selectent.get()
a1 = item.get()
a2 = asset_tag.get()
a3 = amount.get()
a4 = notes.get()
c.execute(""" UPDATE items SET
item = 'a1',
asset_tag = 'a2',
amount = 'a3',
notes = 'a4'
WHERE id = 'a0' """)
conn.commit()
conn.close()
I am self-teaching myself mySQL, and from what I have read about this error it appears that the error occurs when trying to compare a number and string in a WHERE clause. This makes sense since my traceback takes me to that line in my code.
Traceback (most recent call last):
File "C:\Users\mbrow\AppData\Local\Programs\Python\Python37\lib\tkinter\__init__.py", line 1705, in __call__
return self.func(*args)
File "C:\Users\mbrow\AppData\Local\Programs\Python\Python37\Inventory3\editmodule.py", line 61, in save
WHERE id = 'a0' """)
File "C:\Users\mbrow\AppData\Local\Programs\Python\Python37\lib\site-packages\mysql\connector\cursor_cext.py", line 266, in execute
raw_as_string=self._raw_as_string)
File "C:\Users\mbrow\AppData\Local\Programs\Python\Python37\lib\site-packages\mysql\connector\connection_cext.py", line 475, in cmd_query
sqlstate=exc.sqlstate)
mysql.connector.errors.DataError: 1292 (22007): Truncated incorrect DOUBLE value: 'a0'
Can someone explain this to me so I better understand what is going on?
I know amount is spelled wrong.
Use parameterized query and prepared statement
c.execute(""" UPDATE items SET
item = %s,
asset_tag = %s,
amount = %s,
notes = %s
WHERE id = %s """,(a1,a2,a3,a4,a0))
I am trying to do encrypt some JSON data with AES-256, using a password hashed with pbkdf2_sha256 as the key. I want to store the data in a file, be able to load it up, decrypt it, alter it, encrypt it, store it, and repeat.
I am using the passlib and pycryptodome libraries with python 3.8. The following test occurs inside a docker container and throws an error I haven't been able to correct
Does anyone have any clues on how I can improve my code (and knowledge)?
Test.py:
import os, json
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES
from passlib.hash import pbkdf2_sha256
def setJsonData(jsonData, jsonFileName):
with open(jsonFileName, 'wb') as jsonFile:
password = 'd'
key = pbkdf2_sha256.hash(password)[-16:]
data = json.dumps(jsonData).encode("utf8")
cipher = AES.new(key.encode("utf8"), AES.MODE_EAX)
ciphertext, tag = cipher.encrypt_and_digest(data)
[ jsonFile.write(x) for x in (cipher.nonce, tag, ciphertext) ]
def getJsonData(jsonFileName):
with open(jsonFileName, 'rb') as jsonFile:
password = 'd'
key = pbkdf2_sha256.hash(password)[-16:]
nonce, tag, ciphertext = [ jsonFile.read(x) for x in (16, 16, -1) ]
cipher = AES.new(key.encode("utf8"), AES.MODE_EAX, nonce)
data = cipher.decrypt_and_verify(ciphertext, tag)
return json.loads(data)
dictTest = {}
dictTest['test'] = 1
print(str(dictTest))
setJsonData(dictTest, "test")
dictTest = getJsonData("test")
print(str(dictTest))
Output:
{'test': 1}
Traceback (most recent call last):
File "test.py", line 37, in <module>
dictTest = getJsonData("test")
File "test.py", line 24, in getJsonData
data = cipher.decrypt_and_verify(ciphertext, tag)
File "/usr/local/lib/python3.8/site-packages/Crypto/Cipher/_mode_eax.py", line 368, in decrypt_and_verify
self.verify(received_mac_tag)
File "/usr/local/lib/python3.8/site-packages/Crypto/Cipher/_mode_eax.py", line 309, in verify
raise ValueError("MAC check failed")
ValueError: MAC check failed
Research:
Looked into this answer, but I believe my verify() call is in
the right place
I noted that in the python docs, it says:
loads(dumps(x)) != x if x has non-string keys.
but, when I re-run the test with dictTest['test'] = 'a' I have the same error.
I suspected the problem was the json formatting, so I did the same test with a string and didn't make the json.loads and json.dumps calls, but I have the same error
The problem here is that key = pbkdf2_sha256.hash(password)[-16:] hashes the key with a new salt each call. Therefore, the cipher used to encrypt and decrypt the cipher text is going to be different, yielding different data, and thus failing the integrity check.
I changed my key derivation function to the following:
h = SHA3_256.new()
h.update(password.encode("utf-8"))
key = h.digest()
I'm having a heck of a time getting the mysql.connector module to work. I'd really like to find some accurate documentation on it. By hit and by miss, I have arrived here.
Traceback (most recent call last):
File "update_civicrm_address.py", line 80, in <module>
cursor.execute(mysql_select_query, address_id)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor.py", line 1210, in execute
msg="Incorrect number of arguments " \
mysql.connector.errors.ProgrammingError: 1210: Incorrect number of arguments executing prepared statement
Here is the program (it's a bit messy because I have tried so many things to get it to work). Aside from the fact that the update is not working at all, what is causing the error? There is only one parameter and it is accounted for.
import sys
import mysql.connector
import csv
import os
from mysql.connector import Error
from mysql.connector import errorcode
#Specify the import file
try:
inputCSV = 'geocoded_rhode_island_export.csv'
#Open the file and give it a handle
csvFile = open(inputCSV, 'r')
#Create a reader object for the input file
reader = csv.reader(csvFile, delimiter = ',')
except IOError as e:
print("The input file ", inputCSV, " was not found", e)
exit()
try:
conn = mysql.connector.connect(host='localhost',
database='wordpress',
user='wp_user',
password='secret!',
use_pure=True)
cursor = conn.cursor(prepared=True)
except mysql.connector.Error as error:
print( "Failed to connect to database: {}".format(error))
exit()
try:
record_count = 0
for row in reader:
contact_id,address_id,last_name, first_name, middle_name, longitude, latitude = row
print(row)
#Update single record now
print(address_id)
cursor.execute(
"""
update civicrm_address
set
geo_code_1 = %s,
geo_code_2 = %s
where
id = %s
and
location_type_id = %s
""",
(longitude, latitude, address_id, 6)
)
conn.commit
print(cursor.rowcount)
print("Record updated successfully")
mysql_select_query = """
select
id,
geo_code_1,
geo_code_2
from
civicrm_address
where
id = %s
"""
input = (address_id)
cursor.execute(mysql_select_query, address_id)
record = cursor.fetchone()
print(record)
record_count = record_count + 1
finally:
print(record_count, " records updated")
#closing database connection.
if(conn.is_connected()):
conn.close()
print("connection is closed")
The is an error in the code
conn.commit
should be
conn.commit()
I'm trying to run a function (f) every x seconds (in my case 60) which will close an active database connection if one exists and upon completion opens it again.
I am using threading.timer although I'm having trouble passing the connection into the function, and in some situations the function runs repeatedly with nothing else running.
The function needs to return the connection to globals after it completes and I'm finding it hard to pass the connection to the function and assign the return globally from within the function which is how I believe the threading.timer works:
enter code from socketIO_client import SocketIO
import logging
import json
import MySQLdb as mdb
import os
import threading
con = mdb.connect('localhost','username','password','databaseName')
cur = con.cursor()
def f(con):
if 'con' in globals():
con.close()
print ("Connection closed")
os.system('php -f /home/ubuntu/grab.php')
con = mdb.connect('localhost','username','password','databaseName')
cur = con.cursor()
print ("DB Connection opened")
con = mdb.connect('localhost','username','password','databaseName')
cur = con.cursor()
threading.Timer(60,f,con).start(); ######PROBLEM LINE
return con
def on_connect():
print "Connecting to database"
areas = ['EH','BE']
socketIO.emit('subscribe_areas', areas)
def on_message(answer):
print("\nNew message received")
array = (json.loads(answer))
print (array)
runningIdentity = array["value"]
berthID = array["to"]
area = array["area"]
if berthID:
query = ("SELECT crs FROM signalBerth WHERE signalBerth=\'%s\';"%(berthID))
cur.execute(("%s")%(query))
reply = cur.fetchall()
for row in reply:
crs= row[0]
query = "UPDATE service SET lastSeen = \'%s\' WHERE runningIdentity=\'%s"%(crs,runningIdentity)+"\';" #berthID == crs, need to alter
print (("%s")%(query))
cur.execute(("%s")%(query))
con.commit()
print("affected rows = {}".format(cur.rowcount))
socketIO = SocketIO('http://www.realtimetrains.co.uk', 41280) #opens connection
socketIO.on('connect', on_connect) #sends subscription
socketIO.on('message', on_message) #reads data, creates mysql and executes it
con = f(con) ######FIRST CALL TO FUNCTION
socketIO.wait() #Keeps connection openhere
Error:
Traceback (most recent call last): File "input.py", line 49, in
socketIO.wait() #Keeps connection open File "build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 175,
in wait File
"build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 194,
in _process_events File
"build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 202,
in _process_packet File
"build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 327,
in _on_event File "input.py", line 36, in on_message
cur.execute(("%s")%(query)) File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 155, in
execute
charset = db.character_set_name()
_mysql_exceptions.InterfaceError: (0, '') Exception in thread Thread-1: Traceback (most recent call last): File
"/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run() File "/usr/lib/python2.7/threading.py", line 1082, in run
self.function(*self.args, **self.kwargs) TypeError: f() argument after * must be a sequence, not Connection
Perhaps there is a more suited method to my needs, however the important bit it that the connection is closed, the function run and the connection opened again every minute or so. Thought about a cron job, but I'd rather keep my code doing everything.
According to Timer object, its third parameter is args. It is a list, but you pass only the con instead.
You need to replace your problem line with:
threading.Timer(60, f, (con,)).start()