Python3 after cursor.execute it stopped? - mysql

Question

After much trying on python3 (as of still new in this language), the line whereby cursor.execute will prevent the for loop to continue when condition met. However when I comment cursor.execute line, the looping able to continue until the end. How can I made it continue till the last result of the loop?
Objective to achieve: -
I am trying to filter a bunch of data from CFC_xxxx table, process and put it back into SENSOR_TREEHUGGERS table.
Line where the show stopper happened: -
cursor.execute(sqlInsert,
(xxGatewayId,qqqGatewayId,treeDiameter,temperature,recordTime,dateTime,treeHuggerID))
Python3 Code: -
import base64
import struct
import pymysql.cursors
import sys
import datetime
from contextlib import closing
connection = pymysql.connect(host='localhost',
user='xxx',
password='xxx',
db='xxx',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
with closing(connection.cursor()) as cursor:
sql = "SELECT * FROM CFC_xxxx"
cursor.execute(sql)
for row in cursor:
check = struct.unpack('>15x2f4B1L1x', base64.b64decode(row['Value']))
if check[6] > 20000 or check[6] < 10000:
continue
else:
xxGatewayId = int(row['Node_ID'])
qqqGatewayId = int(row['Gateway_ID'])
treeDiameter = int(check[0])
temperature = int(check[1])
recordTime = str(row['Timestamp'])
year = datetime.datetime.fromtimestamp(row['Timestamp']).strftime('%Y')
if check[2] == 0:
hours = '00'
else:
hours = str(check[2])
if check[3] == 0:
minute = '00'
else:
minute = str(check[3])
if check[4] < 10:
day = '0'+str(check[4])
else:
day = str(check[4])
if check[5] < 10:
month = '0'+str(check[5])
else:
month = str(check[5])
dateTime = str(year + '-' + month + '-' + day + ' ' + hours + ':' + minute + ':00')
treeHuggerID = int(check[6])
sqlInsert = "INSERT INTO SENSOR_TREEHUGGERS(`xx_Gateway_Id`,`qqq_Gateway_Id`,`treeDiameter`,`temperature`,`recordTime`,`dateTime`,`TreeHuggerId`) VALUES (%s,%s,%s,%s,%s,%s,%s)"
cursor.execute(sqlInsert, (xxGatewayId,qqqGatewayId,treeDiameter,temperature,recordTime,dateTime,treeHuggerID))
connection.commit()

Answer 1

Below is how I get my trouble solved. Using array to append all the processed data and use executemany to save them at once. In beforehand, have to modify mysql config max_allowed_packet = 500M
A pain but valuable lesson.
Answer: -
import base64
import struct
import pymysql.cursors
import sys
import datetime
from contextlib import closing
collectData = []
connection = pymysql.connect(host='localhost',
user='xxx',
password='xxx',
db='xxx',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
with closing(connection.cursor()) as cursor:
sql = "SELECT * FROM CFC_xxxx"
cursor.execute(sql)
for row in cursor:
check = struct.unpack('>15x2f4B1L1x', base64.b64decode(row['Value']))
if check[6] > 20000 or check[6] < 10000:
continue
else:
xxGatewayId = int(row['Node_ID'])
qqqGatewayId = int(row['Gateway_ID'])
treeDiameter = int(check[0])
temperature = int(check[1])
recordTime = str(row['Timestamp'])
year = datetime.datetime.fromtimestamp(row['Timestamp']).strftime('%Y')
if check[2] == 0:
hours = '00'
else:
hours = str(check[2])
if check[3] == 0:
minute = '00'
else:
minute = str(check[3])
if check[4] < 10:
day = '0'+str(check[4])
else:
day = str(check[4])
if check[5] < 10:
month = '0'+str(check[5])
else:
month = str(check[5])
dateTime = str(year + '-' + month + '-' + day + ' ' + hours + ':' + minute + ':00')
treeHuggerID = int(check[6])
collectData.append(xxGatewayId,qqqGatewayId,treeDiameter,temperature,recordTime,dateTime,treeHuggerID)
c1 = connection.cursor()
sqlInsert = "INSERT INTO SENSOR_TREEHUGGERS(`xx_Gateway_Id`,`qqq_Gateway_Id`,`treeDiameter`,`temperature`,`recordTime`,`dateTime`,`TreeHuggerId`) VALUES (%s,%s,%s,%s,%s,%s,%s)"
c1.executemany(sqlInsert, collectData)
connection.commit()