I'm trying to debug my aiopg connection settings in production environment using aiohttp logger and gunicorn.
I'm trying to log my database credentials:
models.py:
async def init_pg(app):
logger = logging.getLogger('aiohttp.access')
logger.error("POSTGRES_USER = %s" % app['settings'].POSTGRES_USER)
logger.error("POSTGRES_DATABASE = %s" % app['settings'].POSTGRES_DATABASE)
logger.error("POSTGRES_HOST = %s" % app['settings'].POSTGRES_HOST)
logger.error("POSTGRES_PASSWORD = %s" % app['settings'].POSTGRES_PASSWORD)
app['engine'] = await create_engine(
user=app['settings'].POSTGRES_USER,
database=app['settings'].POSTGRES_DATABASE,
host=app['settings'].POSTGRES_HOST,
password=app['settings'].POSTGRES_PASSWORD
)
This doesn't add any output to /var/log/gunicorn/error_log, although it is expected to.
Here is how I start gunicorn:
/usr/local/bin/gunicorn producer.main:app --daemon --bind 0.0.0.0:8002 --worker-class aiohttp.worker.GunicornWebWorker --access-logfile /var/log/gunicorn/access_log --error-logfile /var/log/gunicorn/error_log --env ENVIRONMENT=PRODUCTION --timeout 120
Here is how I create aiohttp app:
main.py:
import logging
import aiohttp_jinja2
import jinja2
from aiojobs.aiohttp import setup as setup_aiojobs
from aiohttp_swagger import setup_swagger
from aiohttp import web, web_middlewares
from . import settings
from .models import init_pg
from .urls import setup_routes
"""
Run either of the following commands from the parent of current directory:
adev runserver producer --livereload
python3 -m producer.main
"""
def create_app():
logging.basicConfig(level=logging.DEBUG)
app = web.Application(middlewares=[
web_middlewares.normalize_path_middleware(append_slash=True),
], client_max_size=2048**2)
app.update(name='producer', settings=settings)
# setup Jinja2 template renderer
aiohttp_jinja2.setup(app, loader=jinja2.PackageLoader('producer', 'templates'))
# create db connection on startup, shutdown on exit
app.on_startup.append(init_pg)
# app.on_cleanup.append(close_pg)
# setup views and routes
setup_routes(app)
# setup middlewares
# setup_middlewares(app)
# setup aiojobs scheduler
setup_aiojobs(app)
# setup swagger documentation
setup_swagger(app, swagger_url="api/doc")
return app
if __name__ == '__main__':
app = create_app()
web.run_app(app, host=app['settings'].HOST, port=app['settings'].PORT)
Related
I'd like my application to be "plug-and-play", so I need to automatically create the database on the first run. I use docker with docker-compose
My attempt is to connect without specifying the database name and run a custom command before running the server:
command:
sh -c "python manage.py create_db &&
python manage.py runserver 0.0.0.0:8000"
And the command itself:
class Command(BaseCommand):
"""Django command to create DB"""
def handle(self, *args, **options):
con = connections['default']
db_name = os.environ.get('DB_NAME')
db_up = False
while not db_up:
try:
cursor = con.cursor()
cursor.execute(f'CREATE DATABASE IF NOT EXISTS {db_name}')
cursor.execute(f'USE {db_name}')
db_up = True
except Exception as err:
self.stdout.write('Database unavailable, waiting 1 second...')
self.stdout.write(str(err))
time.sleep(1)
self.stdout.write(self.style.SUCCESS('Database available!'))
If this is the right way, then now I just need to update the connection to use the newly created database, but I don't know how. The line cursor.execute(f'USE {db_name}') of course doesn't work.
Is it the right way to create the database?
If so, how to update the connection?
If not, how to do it?
Thanks!
EDIT
After hints from Nealium, I created an independent script (not a Django command) which I run before running the server.
import os
import time
from MySQLdb import _mysql
import os
db_host=os.environ.get('DB_HOST')
db_user=os.environ.get('DB_USER')
db_password=os.environ.get('DB_PASS')
db_name = os.environ.get('DB_NAME')
db_up = False
while not db_up:
try:
db = _mysql.connect(
host=db_host,
user=db_user,
password=db_password
)
db_up = True
db.query(f'CREATE DATABASE IF NOT EXISTS {db_name}')
db.close()
except Exception as err:
print('Database unavailable, waiting 1 second...')
time.sleep(1)
print('Database available!')
This what my management command generally looks like
call_command() basically does python manage.py {command}
Updated dothing command
from django.core.management.base import BaseCommand
from django.core.management import call_command
def create_db():
import mysql.connector
try:
mydb = mysql.connector.connect(
host=os.environ.get('DB_HOST'),
user=os.environ.get('DB_USER'),
password=os.environ.get('DB_PASS')
)
mycursor = mydb.cursor()
mycursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(os.environ.get('DB_NAME')))
return True
except mysql.connector.Error as err:
print('Something went wrong: {}'.format(err))
except Exception as ex:
message("Exception: {}".format(ex))
return False
class Command(BaseCommand):
help = 'does thing'
def add_arguments(self, parser):
# Named (optional) arguments
parser.add_argument(
'--import',
action='store_true',
help='Skips Import',
)
def handle(self, *args, **kwargs):
print("Doing Thing")
# connect to db + create if it doesn't exist
status = create_db()
if status:
# create migrations
call_command('makemigrations') # (Django Command)
# can also pass arguemnts like a specific app
# call_command('makemigrations', 'app1')
# This create db **if** it doesn't exist
# + checks that migrations are up to date
call_command('migrate') # (Django Command)
if kwargs['import']:
# another management command to handle importing
# I've just a csv reader and a loop
call_command('importDb') # (Custom Command)
# Collect Static (+ don't ask for confirmation)
call_command('collectstatic', interactive=False) # (Django Command)
print('Thing has been Done')
else:
print('Thing not Done')
So with that, I just run:
python manage.py dothing (python manage.py dothing --import if I want db to be imported)
and then:
python manage.py runserver and it's good to go!
Edit
Just do something like this and pull the options from the settings:
https://www.w3schools.com/python/python_mysql_create_db.asp
Edit 2
It should work; From my testing Django doesn't actually connect to the db until its told to run a query (filter/get/create/delete)
You can generally test this with a basic management command and bad db settings:
Set db name in settings to 'invalid_db'
Test command below
from django.core.management.base import BaseCommand
from django.core.management import call_command
import os
class Command(BaseCommand):
help = 'testing '
def handle(self, *args, **kwargs):
print('Command is Running!')
print('doing the things')
print('about to migrate / use db / crash')
print('*'*100)
call_command('migrate')
So in theory, as long as you aren't using Django commands you should be able to do whatever you want
So I'm trying to deploy my Django project using lambda, with zappa. I'm using MySQL for DB engine. Now after doing some research, I realized that I needed to create a custom Django command to create DB, since I'm using MySQL. So I created crate_db command, zappa updated, then ran zappa manage dev create_db. Then I got this error: 2004 (HY000): Can't create TCP/IP socket (97)
below is my create_db.py file, for your information.
import sys
import logging
import mysql.connector
import os
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
rds_host = os.environ.get("MY HOST")
db_name = os.environ.get("")
user_name = os.environ.get("MY USERNAME")
password = os.environ.get("MY PASSWORD")
port = os.environ.get("3306")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
class Command(BaseCommand):
help = 'Creates the initial database'
def handle(self, *args, **options):
print('Starting db creation')
try:
db = mysql.connector.connect(host=rds_host, user=user_name,
password=password, db="mysql", connect_timeout=10)
c = db.cursor()
print("connected to db server")
c.execute("""CREATE DATABASE bookcake_db;""")
c.execute("""GRANT ALL ON bookcake_db.* TO 'ryan'#'%'""")
c.close()
print("closed db connection")
except mysql.connector.Error as err:
logger.error("Something went wrong: {}".format(err))
sys.exit()
Any ideas? Thanks.
I'm trying to use MySQL cursor to interact with remote database:
from flask import Flask
from flask_mysqldb import MySQL
app = Flask(__name__)
app.config['MYSQL_USER'] = 'sql7368254' # it's a testing database. Nothing to exploit, really.
app.config['MYSQL_PASSWORD'] = 'YnCZ8j4jbi'
app.config['MYSQL_HOST'] = 'sql7.freemysqlhosting.net'
app.config['MYSQL_DB'] = 'sql7368254'
app.config['MYSQL_CURSORCLASS'] = 'DictCursor'
db = MySQL(app)
#app.route('/')
def index():
cur = db.connection.cursor()
cur.execute('''CREATE TABLE users (id INTEGER, email VARCHAR(30), password VARCHAR(255))''')
return 'Done'
if __name__ == '__main__':
app.run(debug=True)
When I spin this off I get:
* Serving Flask app "server.py"
* Environment: production
WARNING: This is a development server. Do not use it in a production deployment.
Use a production WSGI server instead.
* Debug mode: off
Segmentation fault (core dumped)
"Segmentation fault" isn't telling me what is wrong. What might be an issue ?
The import was a problem:
from flask_mysqldb import MySQL
As this topic describes. Flask-MySQLdb doesn't work nicely with Python 3.
Using Python MySQL connector instead is advised.
I am working on a Spark Application and I want to create a rest API in Django, below is my code
from django.shortcuts import render
from django.http import Http404
from rest_framework.views import APIView
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from django.http import JsonResponse
from django.core import serializers
from django.conf import settings
import json
from pyspark import SparkContext, SparkConf, SQLContext
sc = SparkContext()
sql = SQLContext(sc)
df = Sql.read.format("jdbc").options(
url = "jdbc:mysql://127.0.0.1:3306/demo",
driver = "com.mysql.cj.jdbc.Driver",
dbtable = "tablename",
user = "xyz",
password = "abc"
).load()
totalrecords = df.count()
# Create your views here.
#api_view(["GET"])
def Demo(self):
try:
a = str(totalrecords)
return JsonResponse(a,safe=False)
except ValueError as e:
return Response(e.args[0],status.HTTP_400_BAD_REQUEST)
I want to know how will I run this code, as I have directly tried "python manage.py runserver" which is not working, so how to run this spark and django with django api and spark-submit with all required spark jar file?
To run this code you have to use spark submit only,
spark-submit --jars mysql.jar manage.py runserver 0.0.0.0:8000
or
spark-submit manage.py runserver
I am new to flask framework. I want to connect with a MySQL database
and my code in the __init__.py is
app = Flask(__name__)
app.config.from_object(Config)
db = SQLAlchemy(app)
migrate =Migrate(app,db)
but I am getting this error
Authentication plugin '{0}' is not supported".format(plugin_name))
sqlalchemy.exc.NotSupportedError:
(mysql.connector.errors.NotSupportedError) Authentication plugin
'caching_sha2_password' is not supported
(Background on this error at: http://sqlalche.me/e/tw8g)
Can anyone please help me?
Please install the following requirement using pip:
pip install flask-mysql
I perform my MySQL connection with Flask using similar code (tested now):
from flask import Flask
from flaskext.mysql import MySQL
app = Flask(__name__)
app.config['MYSQL_DATABASE_USER'] = 'youruser'
app.config['MYSQL_DATABASE_PASSWORD'] = 'yourpassword'
app.config['MYSQL_DATABASE_DB'] = 'yourdb'
app.config['MYSQL_DATABASE_HOST'] = 'yourhost'
mysql = MySQL(app)
mysql.init_app(app)
#app.route("/")
def hello_db():
conn = mysql.connect()
cursor =conn.cursor()
cursor.execute('''SELECT * from yourtable''')
data = cursor.fetchall()
return str(data)
if __name__ == "__main__":
app.run()
Please change the variables with your data (user/password etc) and try the connection.