I'd like my application to be "plug-and-play", so I need to automatically create the database on the first run. I use docker with docker-compose
My attempt is to connect without specifying the database name and run a custom command before running the server:
command:
sh -c "python manage.py create_db &&
python manage.py runserver 0.0.0.0:8000"
And the command itself:
class Command(BaseCommand):
"""Django command to create DB"""
def handle(self, *args, **options):
con = connections['default']
db_name = os.environ.get('DB_NAME')
db_up = False
while not db_up:
try:
cursor = con.cursor()
cursor.execute(f'CREATE DATABASE IF NOT EXISTS {db_name}')
cursor.execute(f'USE {db_name}')
db_up = True
except Exception as err:
self.stdout.write('Database unavailable, waiting 1 second...')
self.stdout.write(str(err))
time.sleep(1)
self.stdout.write(self.style.SUCCESS('Database available!'))
If this is the right way, then now I just need to update the connection to use the newly created database, but I don't know how. The line cursor.execute(f'USE {db_name}') of course doesn't work.
Is it the right way to create the database?
If so, how to update the connection?
If not, how to do it?
Thanks!
EDIT
After hints from Nealium, I created an independent script (not a Django command) which I run before running the server.
import os
import time
from MySQLdb import _mysql
import os
db_host=os.environ.get('DB_HOST')
db_user=os.environ.get('DB_USER')
db_password=os.environ.get('DB_PASS')
db_name = os.environ.get('DB_NAME')
db_up = False
while not db_up:
try:
db = _mysql.connect(
host=db_host,
user=db_user,
password=db_password
)
db_up = True
db.query(f'CREATE DATABASE IF NOT EXISTS {db_name}')
db.close()
except Exception as err:
print('Database unavailable, waiting 1 second...')
time.sleep(1)
print('Database available!')
This what my management command generally looks like
call_command() basically does python manage.py {command}
Updated dothing command
from django.core.management.base import BaseCommand
from django.core.management import call_command
def create_db():
import mysql.connector
try:
mydb = mysql.connector.connect(
host=os.environ.get('DB_HOST'),
user=os.environ.get('DB_USER'),
password=os.environ.get('DB_PASS')
)
mycursor = mydb.cursor()
mycursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(os.environ.get('DB_NAME')))
return True
except mysql.connector.Error as err:
print('Something went wrong: {}'.format(err))
except Exception as ex:
message("Exception: {}".format(ex))
return False
class Command(BaseCommand):
help = 'does thing'
def add_arguments(self, parser):
# Named (optional) arguments
parser.add_argument(
'--import',
action='store_true',
help='Skips Import',
)
def handle(self, *args, **kwargs):
print("Doing Thing")
# connect to db + create if it doesn't exist
status = create_db()
if status:
# create migrations
call_command('makemigrations') # (Django Command)
# can also pass arguemnts like a specific app
# call_command('makemigrations', 'app1')
# This create db **if** it doesn't exist
# + checks that migrations are up to date
call_command('migrate') # (Django Command)
if kwargs['import']:
# another management command to handle importing
# I've just a csv reader and a loop
call_command('importDb') # (Custom Command)
# Collect Static (+ don't ask for confirmation)
call_command('collectstatic', interactive=False) # (Django Command)
print('Thing has been Done')
else:
print('Thing not Done')
So with that, I just run:
python manage.py dothing (python manage.py dothing --import if I want db to be imported)
and then:
python manage.py runserver and it's good to go!
Edit
Just do something like this and pull the options from the settings:
https://www.w3schools.com/python/python_mysql_create_db.asp
Edit 2
It should work; From my testing Django doesn't actually connect to the db until its told to run a query (filter/get/create/delete)
You can generally test this with a basic management command and bad db settings:
Set db name in settings to 'invalid_db'
Test command below
from django.core.management.base import BaseCommand
from django.core.management import call_command
import os
class Command(BaseCommand):
help = 'testing '
def handle(self, *args, **kwargs):
print('Command is Running!')
print('doing the things')
print('about to migrate / use db / crash')
print('*'*100)
call_command('migrate')
So in theory, as long as you aren't using Django commands you should be able to do whatever you want
Related
So I'm trying to deploy my Django project using lambda, with zappa. I'm using MySQL for DB engine. Now after doing some research, I realized that I needed to create a custom Django command to create DB, since I'm using MySQL. So I created crate_db command, zappa updated, then ran zappa manage dev create_db. Then I got this error: 2004 (HY000): Can't create TCP/IP socket (97)
below is my create_db.py file, for your information.
import sys
import logging
import mysql.connector
import os
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
rds_host = os.environ.get("MY HOST")
db_name = os.environ.get("")
user_name = os.environ.get("MY USERNAME")
password = os.environ.get("MY PASSWORD")
port = os.environ.get("3306")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
class Command(BaseCommand):
help = 'Creates the initial database'
def handle(self, *args, **options):
print('Starting db creation')
try:
db = mysql.connector.connect(host=rds_host, user=user_name,
password=password, db="mysql", connect_timeout=10)
c = db.cursor()
print("connected to db server")
c.execute("""CREATE DATABASE bookcake_db;""")
c.execute("""GRANT ALL ON bookcake_db.* TO 'ryan'#'%'""")
c.close()
print("closed db connection")
except mysql.connector.Error as err:
logger.error("Something went wrong: {}".format(err))
sys.exit()
Any ideas? Thanks.
I'm trying to use MySQL cursor to interact with remote database:
from flask import Flask
from flask_mysqldb import MySQL
app = Flask(__name__)
app.config['MYSQL_USER'] = 'sql7368254' # it's a testing database. Nothing to exploit, really.
app.config['MYSQL_PASSWORD'] = 'YnCZ8j4jbi'
app.config['MYSQL_HOST'] = 'sql7.freemysqlhosting.net'
app.config['MYSQL_DB'] = 'sql7368254'
app.config['MYSQL_CURSORCLASS'] = 'DictCursor'
db = MySQL(app)
#app.route('/')
def index():
cur = db.connection.cursor()
cur.execute('''CREATE TABLE users (id INTEGER, email VARCHAR(30), password VARCHAR(255))''')
return 'Done'
if __name__ == '__main__':
app.run(debug=True)
When I spin this off I get:
* Serving Flask app "server.py"
* Environment: production
WARNING: This is a development server. Do not use it in a production deployment.
Use a production WSGI server instead.
* Debug mode: off
Segmentation fault (core dumped)
"Segmentation fault" isn't telling me what is wrong. What might be an issue ?
The import was a problem:
from flask_mysqldb import MySQL
As this topic describes. Flask-MySQLdb doesn't work nicely with Python 3.
Using Python MySQL connector instead is advised.
I have a Python Flask Server setup in an Ubuntu Machine and a MySQL from XAMPP as backend for the same.
How ever when I try to access the database tables from my python program it shows as
pymysql.err.InternalError: (1109, "Unknown table 'ALL_PLUGINS' in information_schema")
but i can access the database directly in MySQL admin page
the sample program I used to access the data.
from flaskext.mysql import MySQL
from flask import (Flask, request, session, g, redirect, url_for, abort, render_template, flash, Response)
import os
from werkzeug.utils import secure_filename
mysql = MySQL()
app = Flask(__name__)
app.config['MYSQL_DATABASE_USER'] = 'root'
app.config['MYSQL_DATABASE_PASSWORD'] = 'root'
app.config['MYSQL_DATABASE_DB'] = 'information_schema'
app.config['MYSQL_DATABASE_HOST'] = 'localhost'
mysql.init_app(app)
#app.route('/')
def insert_student():
qry = "SELECT * FROM ALL_PLUGINS "
conn = mysql.connect()
cursor = conn.cursor()
cursor.execute(qry)
data = cursor.fetchall()
print(data)
conn.commit()
return "Sucess"
if __name__ == '__main__':
app.secret_key = 'super secret key'
app.debug = True
app.run()
instead of normal running I ge the following
the screen shot
Mysql does not have an all_plugins table in information schema. The plugins table (well, view) is called plugins.
So, your query should be:
SELECT * FROM PLUGINS
Based on the comment from #snakecharmerb:
Mariadb, on the other hand, does have all_plugins table, which presumably is the cause of the confusion.
I created a .bat script to import several .csv in my DB for a Laravel project.
At first, I was using python and each time it took an eternity to restore long files, so I decided to back up those tables and restore them with MySQL.
old .bat file
echo.
echo - Rebuilding database
php artisan migrate:fresh
echo.
echo - Importing animals data
cd py_animalimporter
python importer.py
cd ..
echo.
echo - Importing colors data
cd py_colorimporter
python importer.py
cd ..
echo.
echo - Rebuilding database
php artisan db: seed
echo.
echo - Importing places data
cd py_placeimporter
python importer.py
cd ..
echo.
echo - Starting local server
php artisan serve
New .bat file
echo.
echo - Rebuilding database
php artisan migrate:fresh
echo.
echo - Restoring sql backup
mysql -u username -p test_local < backup.sql
password
echo.
echo - Rebuilding database
php artisan db: seed
echo.
echo - Importing places data
cd py_placeimporter
python importer.py
cd ..
echo.
echo - Starting local server
php artisan serve
My python scripts read MySQL credentials from my laravel.env file (thanks to dotenv library), unfortunately, I can't figure how to do anything similar from the windows terminal.
.env file
DB_HOST=127.0.0.1
DB_PORT=3308
DB_DATABASE=test_local
DB_USERNAME=username
DB_PASSWORD=password
.py files example
from dotenv import load_dotenv
from pathlib import Path
import os
import mysql.connector
from mysql.connector import errorcode
def connectDb():
# Retrieve db credentials from .env
env_path = '../.env'
load_dotenv(dotenv_path=env_path)
db_host = os.getenv("DB_HOST")
db_port = os.getenv("DB_PORT")
db_database = os.getenv("DB_DATABASE")
db_username = os.getenv("DB_USERNAME")
db_password = os.getenv("DB_PASSWORD")
if db_password is None:
db_password = ''
return mysql.connector.connect(user=db_username, password=db_password,
host=db_host,
port=db_port,
database=db_database)
def insertPrimaryColour(hex,color):
try:
cnx = connectDb()
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
print("] Wrong Credentials")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
print("] No Existing Database")
else:
print("] " + err)
else:
cursor = cnx.cursor()
query = f"INSERT INTO dom_colors(`order`,hex,id_translation) VALUES(0,'{hex}','{color}');"
cursor.execute(query)
insert_id = cursor.lastrowid
cnx.commit()
cnx.close()
return insert_id
Alternately, I could use python to restore the DB but everything I tried didn't work!
If you want to make ad os indipendent solution you can try to use subprocess.
You can use in this way:
mysqlLogin = [...]
process = subprocess.Popen(mysqlLogin, shell=True, stdout=subprocess.PIPE)
process.wait()
This code line will run the command contained in mysqlLogin and wait for his termination.
You can also configure standard output redirection with stdout paramters.
Here is the docs: https://docs.python.org/3/library/subprocess.html
I finally found a solution with Python sending code to the Windows Terminal!
Here's the Python Script that now I call from the batch file
from dotenv import load_dotenv
from pathlib import Path
import os
def restoreDB():
# Retrieve DB credentials from .env
env_path = '.env'
load_dotenv(dotenv_path=env_path)
#db_host = os.getenv("DB_HOST")
#db_port = os.getenv("DB_PORT")
db_database = os.getenv("DB_DATABASE")
db_username = os.getenv("DB_USERNAME")
db_password = os.getenv("DB_PASSWORD")
if db_password is "":
mysqlLogin = "mysql -u "+db_username+" "+db_database+" < backup.sql"
else:
mysqlLogin = "mysql -u "+db_username+" --password='"+db_password+"' "+db_database+" < backup.sql"
os.system('cmd /c "%s"' % mysqlLogin)
restoreDB()
I'm trying to debug my aiopg connection settings in production environment using aiohttp logger and gunicorn.
I'm trying to log my database credentials:
models.py:
async def init_pg(app):
logger = logging.getLogger('aiohttp.access')
logger.error("POSTGRES_USER = %s" % app['settings'].POSTGRES_USER)
logger.error("POSTGRES_DATABASE = %s" % app['settings'].POSTGRES_DATABASE)
logger.error("POSTGRES_HOST = %s" % app['settings'].POSTGRES_HOST)
logger.error("POSTGRES_PASSWORD = %s" % app['settings'].POSTGRES_PASSWORD)
app['engine'] = await create_engine(
user=app['settings'].POSTGRES_USER,
database=app['settings'].POSTGRES_DATABASE,
host=app['settings'].POSTGRES_HOST,
password=app['settings'].POSTGRES_PASSWORD
)
This doesn't add any output to /var/log/gunicorn/error_log, although it is expected to.
Here is how I start gunicorn:
/usr/local/bin/gunicorn producer.main:app --daemon --bind 0.0.0.0:8002 --worker-class aiohttp.worker.GunicornWebWorker --access-logfile /var/log/gunicorn/access_log --error-logfile /var/log/gunicorn/error_log --env ENVIRONMENT=PRODUCTION --timeout 120
Here is how I create aiohttp app:
main.py:
import logging
import aiohttp_jinja2
import jinja2
from aiojobs.aiohttp import setup as setup_aiojobs
from aiohttp_swagger import setup_swagger
from aiohttp import web, web_middlewares
from . import settings
from .models import init_pg
from .urls import setup_routes
"""
Run either of the following commands from the parent of current directory:
adev runserver producer --livereload
python3 -m producer.main
"""
def create_app():
logging.basicConfig(level=logging.DEBUG)
app = web.Application(middlewares=[
web_middlewares.normalize_path_middleware(append_slash=True),
], client_max_size=2048**2)
app.update(name='producer', settings=settings)
# setup Jinja2 template renderer
aiohttp_jinja2.setup(app, loader=jinja2.PackageLoader('producer', 'templates'))
# create db connection on startup, shutdown on exit
app.on_startup.append(init_pg)
# app.on_cleanup.append(close_pg)
# setup views and routes
setup_routes(app)
# setup middlewares
# setup_middlewares(app)
# setup aiojobs scheduler
setup_aiojobs(app)
# setup swagger documentation
setup_swagger(app, swagger_url="api/doc")
return app
if __name__ == '__main__':
app = create_app()
web.run_app(app, host=app['settings'].HOST, port=app['settings'].PORT)