i am trying to fetch the list of sql query running more than 3600 sec and kill those id's using python below is the code
import json
import mysql.connector
import pymysql
def main():
# TODO implement
connection = pymysql.connect(user='', password='',
host='',
port=3306,
database='');
cursor = connection.cursor() # get the cursor
# cursor.execute('SHOW PROCESSLIST;')
# extracted_data = cursor.fetchall();
# for i in extracted_data:
# print(i)
with connection.cursor() as cursor:
print(cursor.execute('SHOW PROCESSLIST'))
for item in cursor.fetchall():
if item.get('Time') > 3600 and item.get('command') == 'query':
_id = item.get('Id')
print('kill %s' % item)
cursor.execute('kill %s', _id)
connection.close()
main()
below is the error i am getting
"C:\drive c\pyfile\venv\Scripts\python.exe" "C:/drive c/pyfile/sqlnew2.py"
Traceback (most recent call last):
File "C:\drive c\pyfile\sqlnew2.py", line 23, in <module>
main()
File "C:\drive c\pyfile\sqlnew2.py", line 18, in main
if item.get('Time') > 3600 and item.get('command') == 'query':
AttributeError: 'tuple' object has no attribute 'get'
The .fetchall() method returns a tuple, not a dictionary. Therefore you should access the elements using the numerical indexes, for example item[0], item[1], etc
As an alternative, if you want to fetch the results as a dictionary, you can use a DictCursor
First import it:
import pymysql.cursors
Then modify the cursor line like that:
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
...
Related
I am a novice when it comes to Python and I am trying to import a .csv file into an already existing MySQL table. I have tried it several different ways but I cannot get anything to work. Below is my latest attempt (not the best syntax I'm sure). I originally tried using ‘%s’ instead of ‘?’, but that did not work. Then I saw an example of the question mark but that clearly isn’t working either. What am I doing wrong?
import mysql.connector
import pandas as pd
db = mysql.connector.connect(**Login Info**)
mycursor = db.cursor()
df = pd.read_csv("CSV_Test_5.csv")
insert_data = (
"INSERT INTO company_calculations.bs_import_test(ticker, date_updated, bs_section, yr_0, yr_1, yr_2, yr_3, yr_4, yr_5, yr_6, yr_7, yr_8, yr_9, yr_10, yr_11, yr_12, yr_13, yr_14, yr_15)"
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
)
for row in df.itertuples():
data_inputs = (row.ticker, row.date_updated, row.bs_section, row.yr_0, row.yr_1, row.yr_2, row.yr_3, row.yr_4, row.yr_5, row.yr_6, row.yr_7, row.yr_8, row.yr_9, row.yr_10, row.yr_11, row.yr_12, row.yr_13, row.yr_14, row.yr_15)
mycursor.execute(insert_data, data_inputs)
db.commit()
Error Message:
> Traceback (most recent call last): File
> "C:\...\Python_Test\Excel_Test_v1.py",
> line 33, in <module>
> mycursor.execute(insert_data, data_inputs) File "C:\...\mysql\connector\cursor_cext.py",
> line 325, in execute
> raise ProgrammingError( mysql.connector.errors.ProgrammingError: Not all parameters were used in the SQL statement
MySQL Connector/Python supports named parameters (which includes also printf style parameters (format)).
>>> import mysql.connector
>>> mysql.connector.paramstyle
'pyformat'
According to PEP-249 (DB API level 2.0) the definition of pyformat is:
pyformat: Python extended format codes, e.g. ...WHERE name=%(name)s
Example:
>>> cursor.execute("SELECT %s", ("foo", ))
>>> cursor.fetchall()
[('foo',)]
>>> cursor.execute("SELECT %(var)s", {"var" : "foo"})
>>> cursor.fetchall()
[('foo',)]
Afaik the qmark paramstyle (using question mark as a place holder) is only supported by MariaDB Connector/Python.
i was messing around with the sqlalchemy ORM functionality
i was able to make it work on my main app but when i created a separate file test.py to test something, i kept getting event loop closed errors:
Exception ignored in: <function Connection.__del__ at 0x7f7041c07310>
Traceback (most recent call last):
File "/home/krypt/Documents/Projects/app/env/lib/python3.9/site-packages/aiomysql/connection.py", line 1072, in __del__
File "/home/krypt/Documents/Projects/app/env/lib/python3.9/site-packages/aiomysql/connection.py", line 298, in close
File "/usr/lib/python3.9/asyncio/selector_events.py", line 700, in close
File "/usr/lib/python3.9/asyncio/base_events.py", line 746, in call_soon
File "/usr/lib/python3.9/asyncio/base_events.py", line 510, in _check_closed
RuntimeError: Event loop is closed
here is the code for test.py:
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.orm import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import Table, Column, Integer, String
from sqlalchemy.future import select
from sqlalchemy import delete
import asyncio
Base = declarative_base()
class Table(Base):
__tablename__ = 'Table'
id = Column(Integer, primary_key=True)
string = Column(String(30))
prefix = Column(String(1), default = "!")
async def main():
engine = create_async_engine("mariadb+aiomysql://user:password#127.0.0.1:3306/dbname")
session = AsyncSession(engine)
stmt = select(Table).where(Table.prefix == "!")
res = await session.execute(stmt)
row = res.scalars().first()
print(row)
asyncio.run(main())
The problem seems to be that aiomysql is trying to close its connection after the event loop has closed. I could make the code in the question work by ensuring that the session was closed and the engine disposed.
async def main():
engine = create_async_engine("mariadb+aiomysql://user:password#127.0.0.1:3306/dbname")
async with AsyncSession(engine) as session:
stmt = select(Table).where(Table.prefix == "!")
res = await session.execute(stmt)
row = res.scalars().first()
print(row)
await engine.dispose()
There's some discussion about this here (towards the end); explicitly closing and disposing is the recommended workaround to prevent the connection's __del__ method executing after the event loop has closed.
I'm having a heck of a time getting the mysql.connector module to work. I'd really like to find some accurate documentation on it. By hit and by miss, I have arrived here.
Traceback (most recent call last):
File "update_civicrm_address.py", line 80, in <module>
cursor.execute(mysql_select_query, address_id)
File "/home/ubuntu/.local/lib/python3.6/site-packages/mysql/connector/cursor.py", line 1210, in execute
msg="Incorrect number of arguments " \
mysql.connector.errors.ProgrammingError: 1210: Incorrect number of arguments executing prepared statement
Here is the program (it's a bit messy because I have tried so many things to get it to work). Aside from the fact that the update is not working at all, what is causing the error? There is only one parameter and it is accounted for.
import sys
import mysql.connector
import csv
import os
from mysql.connector import Error
from mysql.connector import errorcode
#Specify the import file
try:
inputCSV = 'geocoded_rhode_island_export.csv'
#Open the file and give it a handle
csvFile = open(inputCSV, 'r')
#Create a reader object for the input file
reader = csv.reader(csvFile, delimiter = ',')
except IOError as e:
print("The input file ", inputCSV, " was not found", e)
exit()
try:
conn = mysql.connector.connect(host='localhost',
database='wordpress',
user='wp_user',
password='secret!',
use_pure=True)
cursor = conn.cursor(prepared=True)
except mysql.connector.Error as error:
print( "Failed to connect to database: {}".format(error))
exit()
try:
record_count = 0
for row in reader:
contact_id,address_id,last_name, first_name, middle_name, longitude, latitude = row
print(row)
#Update single record now
print(address_id)
cursor.execute(
"""
update civicrm_address
set
geo_code_1 = %s,
geo_code_2 = %s
where
id = %s
and
location_type_id = %s
""",
(longitude, latitude, address_id, 6)
)
conn.commit
print(cursor.rowcount)
print("Record updated successfully")
mysql_select_query = """
select
id,
geo_code_1,
geo_code_2
from
civicrm_address
where
id = %s
"""
input = (address_id)
cursor.execute(mysql_select_query, address_id)
record = cursor.fetchone()
print(record)
record_count = record_count + 1
finally:
print(record_count, " records updated")
#closing database connection.
if(conn.is_connected()):
conn.close()
print("connection is closed")
The is an error in the code
conn.commit
should be
conn.commit()
I've got a program that sends a lot of requests to a website using RoboBrowser and gets the answers, but now I need to filter these answers to only the ones that don't have this string " Case Status Not Available " I tried to use beautifulsoup for it, but it is returning an error.
Here's the code so far:
import shlex
import subprocess
import os
import platform
from bs4 import BeautifulSoup
import re
import csv
import pickle
import requests
from robobrowser import RoboBrowser
def rename_files():
file_list = os.listdir(r"C:\\PROJECT\\pdfs")
print(file_list)
saved_path = os.getcwd()
print('Current working directory is '+saved_path)
os.chdir(r'C:\\PROJECT\\pdfs')
for file_name in file_list:
os.rename(file_name, file_name.translate(None, " "))
os.chdir(saved_path)
rename_files()
def run(command):
if platform.system() != 'Windows':
args = shlex.split(command)
else:
args = command
s = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, errors = s.communicate()
return s.returncode == 0, output, errors
# Change this to your PDF file base directory
base_directory = 'C:\\PROJECT\\pdfs'
if not os.path.isdir(base_directory):
print "%s is not a directory" % base_directory
exit(1)
# Change this to your pdf2htmlEX executable location
bin_path = 'C:\\Python27\\pdfminer-20140328\\tools\\pdf2txt.py'
if not os.path.isfile(bin_path):
print "Could not find %s" % bin_path
exit(1)
for dir_path, dir_name_list, file_name_list in os.walk(base_directory):
for file_name in file_name_list:
# If this is not a PDF file
if not file_name.endswith('.pdf'):
# Skip it
continue
file_path = os.path.join(dir_path, file_name)
# Convert your PDF to HTML here
args = (bin_path, file_name, file_path)
success, output, errors = run("python %s -o %s.html %s " %args)
if not success:
print "Could not convert %s to HTML" % file_path
print "%s" % errors
htmls_path = 'C:\\PROJECT'
with open ('score.csv', 'w') as f:
writer = csv.writer(f)
for dir_path, dir_name_list, file_name_list in os.walk(htmls_path):
for file_name in file_name_list:
if not file_name.endswith('.html'):
continue
with open(file_name) as markup:
soup = BeautifulSoup(markup.read())
text = soup.get_text()
match = re.findall("PA/(\S*)", text)#To remove the names that appear, just remove the last (\S*), to add them is just add the (\S*), before it there was a \s*
print(match)
writer.writerow(match)
for item in match:
data = item.split('/')
case_number = data[0]
case_year = data[1]
browser = RoboBrowser()
browser.open('http://www.pa.org.mt/page.aspx?n=63C70E73&CaseType=PA')
form = browser.get_forms()[0] # Get the first form on the page
form['ctl00$PageContent$ContentControl$ctl00$txtCaseNo'].value = case_number
form['ctl00$PageContent$ContentControl$ctl00$txtCaseYear'].value = case_year
browser.submit_form(form, submit=form['ctl00$PageContent$ContentControl$ctl00$btnSubmit'])
# Use BeautifulSoup to parse this data
print(browser.response.text)
souptwo = BeautifulSoup(browser.response.text)
texttwo = soup.get_text()
matchtwo = soup.findall('<td class="fieldData">Case Status Not Available</TD>')
if not matchtwo:
soupthree = BeautifulSoup(browser.response.text)
print soupthree
The error that returns is:
Traceback (most recent call last):
File "C:\PROJECT\pdfs\converterpluspa.py", line 87, in <module>
matchtwo = soup.findall('<td class="fieldData">Case Status Not Available</TD>')
TypeError: 'NoneType' object is not callable
Line 87 includes an attempt to call the method findall of soup. soup was defined in line 65 where BeautifulSoup was called to parse the contents of a file. Since the error diagnostic says that soup is None this means that BeautifulSoup was unable to parse that file.
I'm trying to run a function (f) every x seconds (in my case 60) which will close an active database connection if one exists and upon completion opens it again.
I am using threading.timer although I'm having trouble passing the connection into the function, and in some situations the function runs repeatedly with nothing else running.
The function needs to return the connection to globals after it completes and I'm finding it hard to pass the connection to the function and assign the return globally from within the function which is how I believe the threading.timer works:
enter code from socketIO_client import SocketIO
import logging
import json
import MySQLdb as mdb
import os
import threading
con = mdb.connect('localhost','username','password','databaseName')
cur = con.cursor()
def f(con):
if 'con' in globals():
con.close()
print ("Connection closed")
os.system('php -f /home/ubuntu/grab.php')
con = mdb.connect('localhost','username','password','databaseName')
cur = con.cursor()
print ("DB Connection opened")
con = mdb.connect('localhost','username','password','databaseName')
cur = con.cursor()
threading.Timer(60,f,con).start(); ######PROBLEM LINE
return con
def on_connect():
print "Connecting to database"
areas = ['EH','BE']
socketIO.emit('subscribe_areas', areas)
def on_message(answer):
print("\nNew message received")
array = (json.loads(answer))
print (array)
runningIdentity = array["value"]
berthID = array["to"]
area = array["area"]
if berthID:
query = ("SELECT crs FROM signalBerth WHERE signalBerth=\'%s\';"%(berthID))
cur.execute(("%s")%(query))
reply = cur.fetchall()
for row in reply:
crs= row[0]
query = "UPDATE service SET lastSeen = \'%s\' WHERE runningIdentity=\'%s"%(crs,runningIdentity)+"\';" #berthID == crs, need to alter
print (("%s")%(query))
cur.execute(("%s")%(query))
con.commit()
print("affected rows = {}".format(cur.rowcount))
socketIO = SocketIO('http://www.realtimetrains.co.uk', 41280) #opens connection
socketIO.on('connect', on_connect) #sends subscription
socketIO.on('message', on_message) #reads data, creates mysql and executes it
con = f(con) ######FIRST CALL TO FUNCTION
socketIO.wait() #Keeps connection openhere
Error:
Traceback (most recent call last): File "input.py", line 49, in
socketIO.wait() #Keeps connection open File "build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 175,
in wait File
"build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 194,
in _process_events File
"build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 202,
in _process_packet File
"build/bdist.linux-x86_64/egg/socketIO_client/init.py", line 327,
in _on_event File "input.py", line 36, in on_message
cur.execute(("%s")%(query)) File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 155, in
execute
charset = db.character_set_name()
_mysql_exceptions.InterfaceError: (0, '') Exception in thread Thread-1: Traceback (most recent call last): File
"/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run() File "/usr/lib/python2.7/threading.py", line 1082, in run
self.function(*self.args, **self.kwargs) TypeError: f() argument after * must be a sequence, not Connection
Perhaps there is a more suited method to my needs, however the important bit it that the connection is closed, the function run and the connection opened again every minute or so. Thought about a cron job, but I'd rather keep my code doing everything.
According to Timer object, its third parameter is args. It is a list, but you pass only the con instead.
You need to replace your problem line with:
threading.Timer(60, f, (con,)).start()