I am getting very weird behavior while using PyMySql.
My Connection class is as follows.
class MySqlConnection:
def __init__(self):
self.host = conf.DB_HOST
self.port = conf.DB_PORT
self.user = conf.DB_USER
self.passwd = conf.DB_PASSWD
self.db = conf.DB_DEFAULT
self.connection = None
self.cursor = None
#self.__connect()
def connect(self):
self.connection = pymysql.connect(host=self.host,
user=self.user,
password=self.passwd,
db=self.db)
self.cursor = self.connection.cursor()
And there is a wrapper on top of it, which executes the query.
class QueryExecutor:
def __init__(self, table_name_for_query, table_attributes):
self.conn = MySqlConnection()
self.table_for_query = table_name_for_query
self.table_attributes = table_attributes
def create_connection(self):
self.conn.connect()
def close_connection(self):
self.conn.close()
And then there is a table class.
class SupportedTables:
def __init__(self, table_name, attributes):
self.query_executor = QueryExecutor(table_name, attributes)
def save(self):
'''
Inserts query for saving data in DB.
'''
self.query_executor.create_connection()
value_list = []
for attrib in self.attributes:
if 'AUTO_INCREMENT' in attrib[1]:
continue
col_name = attrib[0]
value_list.append(getattr(self, col_name))
insert_data = [value_list]
if len(value_list) != 0:
self.query_executor.insert_data(data_to_insert=insert_data)
self.query_executor.close_connection()
Now, when I execute tests for SupportedTables and it's derived classes, the code works fine. I do not get any issue.
However, when I try executing the code out of test class, I get an error like:
self.query_executor.create_connection()
"/home/priyesh/db/QueryExecutor_v2.py", line 17,
in create_connection
self.conn.connect() File "/home/priyesh/db/connection_handler.py", line 24, in connect
db=self.db) File "/home/priyesh/virtualenvs/virtualenv_python3_ibsync/lib/python3.6/site-packages/pymysql/__init__.py",
line 94, in Connect
return Connection(*args, **kwargs) File "/home/priyesh/virtualenvs/virtualenv_python3_ibsync/lib/python3.6/site-packages/pymysql/connections.py",
line 327, in init
self.connect() File "/home/priyesh/virtualenvs/virtualenv_python3_ibsync/lib/python3.6/site-packages/pymysql/connections.py",
line 598, in connect
self._request_authentication()
File "/home/priyesh/virtualenvs/virtualenv_python3_ibsync/lib/python3.6/site-packages/pymysql/connections.py",
line 849, in _request_authentication
data += struct.pack('B', len(connect_attrs)) + connect_attrs struct.error: ubyte format requires 0 <= number <= 255
Any idea why this might be happening? The code is pretty simple and works from Test Class but not from Main execution script.
Related
I've been trying to solve the pong atari with a DQN. I'm using OpenAI gym for the pong environment.
I've made a custom ObservationWrapper but I'm unable to figure out whats the problem with the reset() method I've overriden.
Error:
Traceback (most recent call last):
File "C:\Users\berna\Documents\Pytorch Experiment\Torching the Dead Grass\DeepQLearning\training.py", line 123, in <module>
agent = Agent(env, buffer)
File "C:\Users\berna\Documents\Pytorch Experiment\Torching the Dead Grass\DeepQLearning\training.py", line 56, in __init__
self._reset()
File "C:\Users\berna\Documents\Pytorch Experiment\Torching the Dead Grass\DeepQLearning\training.py", line 59, in _reset
self.state = env.reset()
File "C:\Users\berna\AppData\Local\Programs\Python\Python310\lib\site-packages\gym\core.py", line 379, in reset
obs, info = self.env.reset(**kwargs)
File "C:\Users\berna\Documents\Pytorch Experiment\Torching the Dead Grass\DeepQLearning\wrappers.py", line 106, in reset
return self.observation(self.env.reset())
File "C:\Users\berna\AppData\Local\Programs\Python\Python310\lib\site-packages\gym\core.py", line 379, in reset
obs, info = self.env.reset(**kwargs)
File "C:\Users\berna\AppData\Local\Programs\Python\Python310\lib\site-packages\gym\core.py", line 379, in reset
obs, info = self.env.reset(**kwargs)
ValueError: too many values to unpack (expected 2)
Process finished with exit code 1
and the code:
Agent:
class Agent:
def __init__(self, env, exp_buffer):
self.env = env
self.exp_buffer = exp_buffer
self._reset()
def _reset(self):
self.state = env.reset()
self.total_reward = 0.0
wrapper:
class BufferWrapper(gym.ObservationWrapper):
def __init__(self, env, n_steps, dtype=np.float32):
super(BufferWrapper, self).__init__(env)
self.dtype = dtype
old_space = env.observation_space
self.observation_space = gym.spaces.Box(old_space.low.repeat(n_steps, axis=0),
old_space.high.repeat(n_steps, axis=0), dtype=dtype)
def reset(self):
self.buffer = np.zeros_like(self.observation_space.low, dtype=self.dtype)
return self.observation(self.env.reset())
def observation(self, observation):
self.buffer[:-1] = self.buffer[1:]
self.buffer[-1] = observation
return self.buffer
Can someone helping me understand why I'm receiving that error?
In given below if Medicine table consists medicine_name then query execute fine, but when medicine name doesn't exist on the Corresponding table then error is occured.
Matching Query Doesn't Exists
views.py Code
#api_view(['POST'])
def addPeople(request):
m = People()
m.bp_no = request.POST['bp_no']
m.name = request.POST['name']
m.corporation_name = request.POST['corporation_name']
m.medicine_name = request.POST['medicine_name']
m.no_of_medicine = request.POST['no_of_medicine']
existing = Medicine.objects.get(medicine_name=m.medicine_name).no_of_medicine - int(m.no_of_medicine)
p_key = Medicine.objects.get(medicine_name=m.medicine_name).id
if Medicine.objects.filter(medicine_name=m.medicine_name).exists():
if existing > 0:
m.save()
Medicine.objects.filter(id=p_key).update(no_of_medicine=existing)
return Response({"message": "Successfully Recorded"})
else:
return Response({"message": "Not much Medicine Stored"})
else:
return Response({"message": "Medicine is not Stored"})
models.py
class People(models.Model):
bp_no = models.IntegerField(blank=False,null=False)
name = models.CharField(blank=False,null=False,max_length=200)
corporation_name = models.CharField(blank=False,null=False,max_length=200)
medicine_name = models.CharField(blank=False,null=False,max_length=200)
no_of_medicine = models.IntegerField()
class Medicine(models.Model):
medicine_name = models.CharField(null=False,blank=False,max_length=200)
no_of_medicine = models.IntegerField(null=False,blank=False)
def __str__(self):
return self.medicine_name
Error Traceback: When Medicine table doesn't contains the corresponding filter name then This error will be shown
Internal Server Error: /api/add-people/
Traceback (most recent call last):
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\django\views\decorators\csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\django\views\generic\base.py", line 70, in view
return self.dispatch(request, *args, **kwargs)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\rest_framework\views.py", line 509, in dispatch
response = self.handle_exception(exc)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\rest_framework\views.py", line 469, in handle_exception
self.raise_uncaught_exception(exc)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\rest_framework\views.py", line 480, in raise_uncaught_exception
raise exc
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\rest_framework\views.py", line 506, in dispatch
response = handler(request, *args, **kwargs)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\rest_framework\decorators.py", line 50, in handler
return func(*args, **kwargs)
File "E:\django\backend\medirecords\api\views.py", line 37, in addPeople
existing = Medicine.objects.get(medicine_name=m.medicine_name).no_of_medicine - int(m.no_of_medicine)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\MonirHossain\AppData\Roaming\Python\Python39\site-packages\django\db\models\query.py", line 435, in get
raise self.model.DoesNotExist(
api.models.Medicine.DoesNotExist: Medicine matching query does not exist.
I haven't tested this, but purely in terms of handling objects not found, I have a suggestion you can try to simplify the conditions / flow.
.get(): Wrap in try / except
get(*args, **kwargs) raises Model.DoesNotExist when not found.
When there's a not found, you need to determine how you want to handle it. For example, you can have a default fallback value, or raise an error.
In this case the code would fail if the existing medicine wasn't found, so I will have it raise.
#api_view(["POST"])
def addPeople(request):
medicine_name = request.POST["medicine_name"]
try:
medicine = Medicine.objects.get(medicine_name=medicine_name)
except Medicine.DoesNotExist:
return Response({"message": "Not much Medicine Stored"})
m = People()
m.bp_no = request.POST["bp_no"]
m.name = request.POST["name"]
m.corporation_name = request.POST["corporation_name"]
m.medicine_name = medicine_name
m.no_of_medicine = request.POST["no_of_medicine"]
existing = medicine.no_of_medicine - int(m.no_of_medicine)
p_key = medicine.id
if medicine.exists():
if existing > 0:
m.save()
medicine.num_of_medicine = existing
medicine.save(update_fields=["num_of_medicine"])
return Response({"message": "Successfully Recorded"})
else:
return Response({"message": "Not much Medicine Stored"})
else:
return Response({"message": "Medicine is not Stored"})
Move object check earlier
Since the request relies on Medicine, this is should be caught early as possible.
Uses try/catch as .get() will always raise when object is not found
The alternative is to do .exists() or filter(*args, **kwargs) + .first()
Update dependent model via Model.save() and limit field updated to num_of_medicine via update_fields.
get_object_or_404()
get_object_or_404(klass, *args, **kwargs) can also be used in lieu of the earlier try/catch block:
from django.shortcuts import get_object_or_404
medicine = get_object_or_404(Medicine.objects.all(), medicine_name=m.medicine_name)
existing = medicine.no_of_medicine - int(m.no_of_medicine)
Final note
I'm assuming this is an example, but it also may be good to validate request.POST["no_of_medicine"] in the POST data against a backend source of truth to make sure it's not faked.
This is my code:-
from flask import Flask, render_template, request
from flask_sqlalchemy import SQLAlchemy
import json
import datetime
with open ("config.json", "r", errors="ignore") as c:
parameters = json.load(c)["parameters"]
local_server = True
app = Flask(__name__)
if local_server:
app.config['SQLALCHEMY_DATABASE_URI'] = parameters["local_uri"]
else:
app.config['SQLALCHEMY_DATABASE_URI'] = parameters["prod_uri"]
db = SQLAlchemy(app)
class Contacts(db.Model):
'''
sno, name, email, message, date
'''
sno = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(80), nullable=False)
message = db.Column(db.String(120), nullable=False)
date = db.Column(db.String(12), nullable=True)
email = db.Column(db.String(20), nullable=False)
#app.route("/")
def index ():
return render_template('index.html', parameters=parameters)
#app.route("/about/")
def about():
return render_template('about.html', parameters=parameters)
#app.route("/contact/", methods = ['GET', 'POST'])
def contact():
if(request.method=='POST'):
'''Add entry to the database'''
name = request.form.get('name')
email = request.form.get('email')
message = request.form.get('message')
entry = Contacts(name=name, message=message, date=datetime.date.today(), email=email )
db.session.add(entry)
db.session.commit()
return render_template('contact.html', parameters=parameters)
app.run(debug=True)
and this is my json file saved as config.json:-
{
"parameters":
{
"local_server": "True",
"local_uri": "mysql://root:#localhost/coderoad",
"prod_uri": "mysql://root:#localhost/coderoad",
"git_url": "github.com/road2code"
}
}
This is my error:-
PS C:\Users\shomi\OneDrive\Desktop\Flask> & C:/Users/shomi/AppData/Local/Programs/Python/Python38/python.exe c:/Users/shomi/OneDrive/Desktop/Flask/flask3.py
Traceback (most recent call last):
File "c:/Users/shomi/OneDrive/Desktop/Flask/flask3.py", line 7, in <module>
parameters = json.load(c)["parameters"]
File "C:\Users\shomi\AppData\Local\Programs\Python\Python38\lib\json\__init__.py", line 293, in load
return loads(fp.read(),
File "C:\Users\shomi\AppData\Local\Programs\Python\Python38\lib\json\__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "C:\Users\shomi\AppData\Local\Programs\Python\Python38\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\shomi\AppData\Local\Programs\Python\Python38\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I'm working on a bootstrap template, even after banging my head around google I am unable to find a solution, a little help would be much appreciated.
You may want to specify path to config.json, perhaps './config.json'; also, try loading it in repl (e.g. ipython)
I don't know where your config.json resides relative to your flask app. If relative path doesn't work, try using complete path.
with open ("./config.json", "r", errors="ignore") as c:
parameters = json.load(c)["parameters"]
update:
try this
with open('foo.bar', 'w') as dummy:
dummy.write('dummy')
put this before your the code that reads the config.json . Here, i'm not specifying the path to 'foo.bar', just like you didn't when you were trying to read config.json. After you run your app again, the location of foo.bar will be the location where flask is trying to read config.json from.
I'm writing an application where I've moved all the MySQL connection setup and teardown to a class, initializing within individual function calls with a With statement.
Now that the development is all done, I'm optimizing and would like to set up connection pooling - but I can't for the life of me figure out how - if I initialize the pool when I set up the object in enter, won't that set up a new pool for each object?
If I put the pool setup in the global of the module, then how do I ensure I set up the pool before I start creating DB objects?
My DB code looks somewhat like this:
# Setting up details for connecting to a local MariaDB/MySQL instance
# replace with suitable code/module when porting to cloud/production
import sys
import mysql.connector
"""Module for abstracting database connectivity
Import this module and then call run_query(), run_query_vals() or run_query_no_return() """
__all__ = ['UseDatabase', 'CredentialsError', 'ConnectionError', 'SQLError']
class ConnectionError(Exception):
pass
class CredentialsError(Exception):
pass
class SQLError(Exception):
pass
dbconfig = { 'host': '127.0.0.1', 'user' : 'statdev', 'password' : 'statdev', 'database': 'stat',}
# Just so we remember. This also doubles as default server details while doing unit testing.
class UseDatabase:
# myconfig = {'host': '127.0.0.1', 'user': 'statdev', 'password': 'statdev', 'database': 'stat', }
config = None
def __init__(self, config: dict):
self.config = config
def __enter__(self) -> 'self':
try:
self.conn = mysql.connector.connect(**self.config)
self.cursor = self.conn.cursor(dictionary=True)
return self
except mysql.connector.InterfaceError as err:
print('Can\'t connect to Database - is it available? \nError: ', str(err))
raise ConnectionError(err)
except mysql.connector.ProgrammingError as err:
print('Invalid credentials - please check ID/Password. \nError: ', str(err))
raise CredentialsError(err)
except mysql.connector.IntegrityError as err:
print("Error: {}".format(err))
except Exception as err:
print('Something else went wrong:', str(err))
return err
def __exit__(self, exc_type, exc_value, exc_traceback):
self.conn.commit()
self.cursor.close()
self.conn.close()
if exc_type is mysql.connector.errors.ProgrammingError:
print('Error in SQL Code - please check the query. \nError: ', str(exc_type))
raise SQLError(exc_value)
elif exc_type:
print('Something else went wrong\n', str(exc_type))
raise exc_type(exc_value)
def run_query(self,query_str) -> 'cursor':
"""query function that takes """
self.cursor.execute(query_str, None)
return self.cursor
def run_query_vals(self, query_str, tupleval) -> 'cursor':
# print("\n\n %s " % query_str)
self.cursor.execute(query_str, tupleval)
return self.cursor
def run_query_no_return(self,query_str) -> 'cursor':
"""query function that takes """
self.cursor.execute(query_str)
return self.cursor
def test():
# dbconfig = {'host': '127.0.0.1', 'user': 'statdev', 'password': 'statdev', 'database': 'stat', }
with UseDatabase(dbconfig) as db:
# result = db.run_query("Select NULL from dual")
result = db.run_query_vals('Select NULL from dual', None)
res = result.fetchone()
if res == {'NULL': None}:
print("DB Module Test was successful! \n"
"Queries return values in dictionaries."
"\nTest query \'Select NULL from dual\' returned result: %s" % str(res))
if __name__ == '__main__':
test()
This has worked for me but I am not sure it's a perfect solution as, for example, trying to do multiple inserts via a for loop results in a 'Failed getting connection; pool exhausted' error. I did not have this problem when I was using a function-based (non class-based) connection pool. Anyway, to avoid this problem I just simply use 'cursor.executemany' in one go.
Hope this helps someone!
from mysql.connector.pooling import MySQLConnectionPool
from mysql.connector.errors import ProgrammingError, InterfaceError
from settings import config
# Database connection pool
dbconfig = config.dbconfig
dbconfig_pool = config.dbconfig_pool
#The following is my 'class DBasePool' content:
def __init__(self, dbconfig, dbconfig_pool):
self.dbconfig = dbconfig
self.pool_name = dbconfig_pool['pool_name']
self.pool_size = dbconfig_pool['pool_size']
try:
self.cnxpool = self.create_pool(pool_name=self.pool_name, pool_size=self.pool_size)
self.cnx = self.cnxpool.get_connection()
self.cursor = self.cnx.cursor(buffered=True)
except InterfaceError as e:
logger.error(e)
raise ConnectionError(e)
except ProgrammingError as e:
logger.error(e)
raise CredentialsError(e)
except Exception as e:
logger.error(e)
raise
def create_pool(self, pool_name, pool_size):
return MySQLConnectionPool(pool_name=pool_name, pool_size= pool_size, **self.dbconfig)
def close(self, cnx, cursor):
cursor.close()
cnx.close()
def execute(self, sql, data=None):
# Get connection form connection pool instead of creating one
cnx = self.cnxpool.get_connection()
cursor = cnx.cursor(buffered=True)
cursor.execute(sql, data)
if cursor.rowcount:
cnx.commit()
rowcount = cursor.rowcount
self.close(cnx, cursor)
return rowcount
else:
print('Could not insert record(s): {}, {}'.format(sql, data))
return 0
I'm using Tweepy for the first time. Currently getting this error
---------------------------------------------------------------------------
IOError Traceback (most recent call last)
<ipython-input-11-cdd7ebe0c00f> in <module>()
----> 1 data_json = io.open('raw_tweets.json', mode='r', encoding='utf-8').read() #reads in the JSON file
2 data_python = json.loads(data_json)
3
4 csv_out = io.open('tweets_out_utf8.csv', mode='w', encoding='utf-8') #opens csv file
IOError: [Errno 2] No such file or directory: 'raw_tweets.json'
I've got a feeling that the code I've got isn't working. For example print(status) doesn't print anything. Also I see no saved CSV or JSON file in the directory.
I'm a newbie so any help/documentation you can offer would be great!
import time
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import os
import json
import csv
import io
from pymongo import MongoClient
ckey = 'blah'
consumer_secret = 'blah'
access_token_key = 'blah'
access_token_secret = 'blah'
#start_time = time.time() #grabs the system time
keyword_list = ['keyword'] #track list
#Listener Class Override
class listener(StreamListener):
def __init__(self, start_time, time_limit=60):
self.time = start_time
self.limit = time_limit
self.tweet_data = []
def on_data(self, data):
saveFile = io.open('raw_tweets.json', 'a', encoding='utf-8')
while (time.time() - self.time) < self.limit:
try:
self.tweet_data.append(data)
return True
except BaseException, e:
print 'failed ondata,', str(e)
time.sleep(5)
pass
saveFile = io.open('raw_tweets.json', 'w', encoding='utf-8')
saveFile.write(u'[\n')
saveFile.write(','.join(self.tweet_data))
saveFile.write(u'\n]')
saveFile.close()
exit()
def on_error(self, status):
print status
class listener(StreamListener):
def __init__(self, start_time, time_limit=10):
self.time = start_time
self.limit = time_limit
def on_data(self, data):
while (time.time() - self.time) < self.limit:
print(data)
try:
client = MongoClient('blah', 27017)
db = client['blah']
collection = db['blah']
tweet = json.loads(data)
collection.insert(tweet)
return True
except BaseException as e:
print('failed ondata,')
print(str(e))
time.sleep(5)
pass
exit()
def on_error(self, status):
print(status)
data_json = io.open('raw_tweets.json', mode='r', encoding='utf-8').read() #reads in the JSON file
data_python = json.loads(data_json)
csv_out = io.open('tweets_out_utf8.csv', mode='w', encoding='utf-8') #opens csv file
UPDATED: Creates file but file is empty
import tweepy
import datetime
auth = tweepy.OAuthHandler('xxx', 'xxx')
auth.set_access_token('xxx', 'xxx')
class listener(tweepy.StreamListener):
def __init__(self, timeout, file_name, *args, **kwargs):
super(listener, self).__init__(*args, **kwargs)
self.start_time = None
self.timeout = timeout
self.file_name = file_name
self.tweet_data = []
def on_data(self, data):
if self.start_time is None:
self.start_time = datetime.datetime.now()
while (datetime.datetime.now() - self.start_time).seconds < self.timeout:
with open(self.file_name, 'a') as data_file:
data_file.write('\n')
data_file.write(data)
def on_error(self, status):
print status
l = listener(60, 'stack_raw_tweets.json')
mstream = tweepy.Stream(auth=auth, listener=l)
mstream.filter(track=['python'], async=True)
You are not creating a Stream for the listener. The last but one line of the code below does that. Followed by that you have to start the Stream, which is the last line. I must warn you that storing this in mongodb is the right thing to do as the file that I am storing it seems to grow easily to several GB. Also the file is not exactly a json. Each line in the file is a json. You must tweak it to your needs.
import tweepy
import datetime
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
class listener(tweepy.StreamListener):
def __init__(self, timeout, file_name, *args, **kwargs):
super(listener, self).__init__(*args, **kwargs)
self.start_time = None
self.timeout = timeout
self.file_name = file_name
self.tweet_data = []
def on_data(self, data):
if self.start_time is None:
self.start_time = datetime.datetime.now()
while (datetime.datetime.now() - self.start_time).seconds < self.timeout:
with open(self.file_name, 'a') as data_file:
data_file.write('\n')
data_file.write(data)
def on_error(self, status):
print status
l = listener(60, 'raw_tweets.json')
mstream = tweepy.Stream(auth=auth, listener=l)
mstream.filter(track=['python'], async=True)