Airflow initdb with mysql as backend is not working - mysql

Question

when I initialize airflow using mysql as backend I see below error
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py",
line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib/python3.7/site-packages/MySQLdb/init.py", line 80,
in Connect
from MySQLdb.connections import Connection
File "/usr/local/lib/python3.7/site-packages/MySQLdb/connections.py", line
36
raise errorclass, errorvalue
^
SyntaxError: invalid syntax
I tried re-installing mysql-python and pymysql still its same
airflow initdb
/usr/local/lib/python3.7/site-packages/airflow/configuration.py:627: DeprecationWarning: You have two airflow.cfg files: /home/airflow/airflow/airflow.cfg and /home/airflow/airflow.cfg. Airflow used to look at ~/airflow/airflow.cfg, even when AIRFLOW_HOME was set to a different value. Airflow will now only read /home/airflow/airflow.cfg, and you should remove the other file
category=DeprecationWarning,
[2019-09-24 12:10:32,310] {settings.py:213} INFO - settings.configure_orm(): Using pool settings. pool_size=5, max_overflow=10, pool_recycle=1800, pid=15403
[2019-09-24 12:10:32,508] {__init__.py:51} INFO - Using executor LocalExecutor
DB: mysql://airdba:***#host/airflow_main
[2019-09-24 12:10:32,820] {db.py:369} INFO - Creating tables
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 32, in <module>
args.func(args)
File "/usr/local/lib/python3.7/site-packages/airflow/bin/cli.py", line 1102, in initdb
db.initdb(settings.RBAC)
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 106, in initdb
upgradedb()
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 377, in upgradedb
command.upgrade(config, 'heads')
File "/usr/local/lib/python3.7/site-packages/alembic/command.py", line 298, in upgrade
script.run_env()
File "/usr/local/lib/python3.7/site-packages/alembic/script/base.py", line 489, in run_env
util.load_python_file(self.dir, "env.py")
File "/usr/local/lib/python3.7/site-packages/alembic/util/pyfiles.py", line 98, in load_python_file
module = load_module_py(module_id, path)
File "/usr/local/lib/python3.7/site-packages/alembic/util/compat.py", line 173, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/usr/local/lib/python3.7/site-packages/airflow/migrations/env.py", line 92, in <module>
run_migrations_online()
File "/usr/local/lib/python3.7/site-packages/airflow/migrations/env.py", line 78, in run_migrations_online
with connectable.connect() as connection:
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2206, in connect
return self._connection_cls(self, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 103, in __init__
else engine.raw_connection()
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2306, in raw_connection
self.pool.unique_connection, _connection
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2275, in _wrap_pool_connect
return fn()
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 303, in unique_connection
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 492, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/impl.py", line 139, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/impl.py", line 136, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib/python3.7/site-packages/MySQLdb/__init__.py", line 80, in Connect
from MySQLdb.connections import Connection
File "/usr/local/lib/python3.7/site-packages/MySQLdb/connections.py", line 36
raise errorclass, errorvalue
^
SyntaxError: invalid syntax
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib/python3.7/site-packages/MySQLdb/__init__.py", line 80, in Connect
from MySQLdb.connections import Connection
File "/usr/local/lib/python3.7/site-packages/MySQLdb/connections.py", line 36
raise errorclass, errorvalue
^
SyntaxError: invalid syntax

Answer 1

removed mysql-python module and forced airflow to use pymysql.
changed connection string in airflow.cfg
sql_alchemy_conn = mysql+pymysql://user:password#server/dbname