Related
How can I switch TPU version for TPU VM architechture?
When attempting to switch software version for TPU(TPU VM architechture switching from tpu-vm-tf-2.6.0-pod to tpu-vm-base) using instructions found here, I get Connection Refused exception with traceback:
Traceback (most recent call last):
File "/usr/lib/python3.8/urllib/request.py", line 1354, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/usr/lib/python3.8/http/client.py", line 1256, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1011, in _send_output
self.send(msg)
File "/usr/lib/python3.8/http/client.py", line 951, in send
self.connect()
File "/usr/lib/python3.8/http/client.py", line 922, in connect
self.sock = self._create_connection(
File "/usr/lib/python3.8/socket.py", line 808, in create_connection
raise err
File "/usr/lib/python3.8/socket.py", line 796, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "switch.py", line 20, in <module>
c.configure_tpu_version(args.target_version, restart_type="ifNeeded")
File "/usr/local/lib/python3.8/dist-packages/cloud_tpu_client/client.py", line 391, in configure_tpu_version
for result in results:
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
yield fs.pop().result()
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 444, in result
return self.__get_result()
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/dist-packages/cloud_tpu_client/client.py", line 375, in configure_worker
request.urlopen(req)
File "/usr/lib/python3.8/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.8/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/usr/lib/python3.8/urllib/request.py", line 542, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/usr/lib/python3.8/urllib/request.py", line 1383, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "/usr/lib/python3.8/urllib/request.py", line 1357, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
The command is run on cloud TPU VM with cloud-tpu-client version 0.10
When running the same command from my PC I get Connection timed out after a long pause with traceback:
Traceback (most recent call last):
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 1350, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/http/client.py", line 1281, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/http/client.py", line 1327, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/http/client.py", line 1276, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/http/client.py", line 1036, in _send_output
self.send(msg)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/http/client.py", line 976, in send
self.connect()
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/http/client.py", line 948, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/socket.py", line 728, in create_connection
raise err
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/socket.py", line 716, in create_connection
sock.connect(sa)
TimeoutError: [Errno 110] Connection timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "tpu_version.py", line 19, in <module>
c.configure_tpu_version(args.target_version, restart_type="ifNeeded")
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/site-packages/cloud_tpu_client/client.py", line 392, in configure_tpu_version
for result in results:
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/concurrent/futures/_base.py", line 598, in result_iterator
yield fs.pop().result()
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/concurrent/futures/_base.py", line 435, in result
return self.__get_result()
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/site-packages/cloud_tpu_client/client.py", line 376, in configure_worker
request.urlopen(req)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 543, in _open
'_open', req)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 503, in _call_chain
result = func(*args)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 1378, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "/home/nevus/anaconda3/envs/imageGen/lib/python3.7/urllib/request.py", line 1352, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 110] Connection timed out>
This feature is not supported by the TPU VM architecture.
Best way to change the TPU VM's version would be to delete it and recreate a new one with the desired version.
I am running a Flask application on local and production server. I have no issues with the local, I am facing 'MySQL connection not available' for every second database request on production server. After reloading or after performing rollback operation, it is getting executed, but the issue repeats. I tried major solutions available on the internet by changing, pool_recycle, 'wait_timeout' and other timeouts to same value of 1600, but didn't work. I finally destroyed the application and reinstalled everything on the server but the issue is still on. Please help in this, thank you.
My production MySQL config:
class ProductionConfig(Config):
SECRET_KEY = 'secret_key'
DEBUG=False
SQLALCHEMY_DATABASE_URI = "mysql+pymysql://{username}:
{password}#{hostname}/{databasename}".format(
username="myusername",
password="password",
hostname="myhostname",
databasename="mydatabase",
)
SQLALCHEMY_POOL_RECYCLE = 299
SQLALCHEMY_TRACK_MODIFICATIONS = False
Error Message:
Exception on /add_questions/1/2 [POST]
Traceback (most recent call last):
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 713, in _write_bytes
self._sock.sendall(data)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context
cursor, statement, parameters, context
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 590, in do_execute
cursor.execute(statement, parameters)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 516, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 771, in _execute_command
self._write_bytes(packet)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 718, in _write_bytes
"MySQL server has gone away (%r)" % (e,))
pymysql.err.OperationalError: (2006, "MySQL server has gone away (ConnectionResetError(104, 'Connection reset by peer'))")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask_login/utils.py", line 270, in decorated_view
elif not current_user.is_authenticated:
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/werkzeug/local.py", line 347, in __getattr__
return getattr(self._get_current_object(), name)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/werkzeug/local.py", line 306, in _get_current_object
return self.__local()
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask_login/utils.py", line 26, in <lambda>
current_user = LocalProxy(lambda: _get_user())
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask_login/utils.py", line 346, in _get_user
current_app.login_manager._load_user()
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/flask_login/login_manager.py", line 318, in _load_user
user = self._user_callback(user_id)
File "/home/ulznrcvr/jaihindpro/application/auth/views.py", line 26, in load_user
return User.query.get(int(user_id))
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 1004, in get
return self._get_impl(ident, loading.load_on_pk_identity)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 1121, in _get_impl
return db_load_fn(self, primary_key_identity)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/loading.py", line 287, in load_on_pk_identity
return q.one()
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3360, in one
ret = self.one_or_none()
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3329, in one_or_none
ret = list(self)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3405, in __iter__
return self._execute_and_instances(context)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3430, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 984, in execute
return meth(self, multiparams, params)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1103, in _execute_clauseelement
distilled_params,
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1288, in _execute_context
e, statement, parameters, cursor, context
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1482, in _handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context
cursor, statement, parameters, context
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 590, in do_execute
cursor.execute(statement, parameters)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 516, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 771, in _execute_command
self._write_bytes(packet)
File "/home/ulznrcvr/virtualenv/jaihindpro/3.6/lib/python3.6/site-packages/pymysql/connections.py", line 718, in _write_bytes
"MySQL server has gone away (%r)" % (e,))
sqlalchemy.exc.OperationalError: (pymysql.err.OperationalError) (2006, "MySQL server has gone away (ConnectionResetError(104, 'Connection reset by peer'))")
[SQL: SELECT user.id AS user_id, user.student_id AS user_student_id, user.role_id AS user_role_id, user.course_id AS user_course_id, user.sub_course_id AS user_sub_course_id, user.batch_id AS user_batch_id, user.full_name AS user_full_name, user.email AS user_email, user.phone_number AS user_phone_number, user.password AS user_password, user.active AS user_active, user.confirmed AS user_confirmed, user.confirmed_date AS user_confirmed_date, user.subscribed AS user_subscribed, user.start_date AS user_start_date, user.end_date AS user_end_date, user.re_url AS user_re_url, user.online AS user_online
FROM user
WHERE user.id = %(param_1)s]
[parameters: {'param_1': 2}]
(Background on this error at: http://sqlalche.me/e/e3q8)
I figured it out myself, I flagged "pool_pre_ping" attribute to "True" in engine settings.
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_pre_ping": True,
"pool_recycle": 300,
}
Ref: https://medium.com/#heyjcmc/controlling-the-flask-sqlalchemy-engine-a0f8fae15b47
I have an airflow instance on EC2 that is running the webserver/scheduler. I want to hook up a MySQL RDS instance as the backend metadata database as opposed to the native SQLite. I replaced the one line in Airflow.cfg that connects to the database via sql_alchemy to connect to RDS with a pymysql driver:
#sql_alchemy_conn = sqlite:////home/cloud-user/airflow/airflow.db
sql_alchemy_conn = mysql+pymysql://admin:<PASSWORD>#airflow-db.xxxxxxxxxxxx.us-east-1.rds.amazonaws.com:3306/airflow
The connection seems to work fine and I am able to get into the RDS instance and query tables via a MySQL client set up on my EC2 instance.
When I toggle a DAG on or off, I get this nasty python stack trace in my shell:
[2019-09-30 14:00:51,774] {app.py:1891} ERROR - Exception on /admin/airflow/paused [POST]
Traceback (most recent call last):
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask/app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask/app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask/app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/home/cloud-user/.local/lib/python2.7/site-packages/flask_login/utils.py", line 258, in decorated_view
return func(*args, **kwargs)
File "/home/cloud-user/.local/lib/python2.7/site-packages/airflow/www/utils.py", line 279, in wrapper
session.commit()
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1027, in commit
self.transaction.commit()
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 494, in commit
self._prepare_impl()
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 473, in _prepare_impl
self.session.flush()
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2459, in flush
self._flush(objects)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2597, in _flush
transaction.rollback(_capture_exception=True)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2557, in _flush
flush_context.execute()
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
uow,
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1138, in _emit_insert_statements
statement, params
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute
return meth(self, multiparams, params)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
distilled_params,
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1253, in _execute_context
e, statement, parameters, cursor, context
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1473, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1249, in _execute_context
cursor, statement, parameters, context
File "/home/cloud-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
cursor.execute(statement, parameters)
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/home/cloud-user/.local/lib/python2.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
ProgrammingError: (pymysql.err.ProgrammingError) (1064, u"You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '[(\\'is_paused\\', u\\'true\\'), (\\'dag_id\\', u\\'test\\')]'')' at line 1")
[SQL: INSERT INTO log (dttm, dag_id, task_id, event, execution_date, owner, extra) VALUES (%(dttm)s, %(dag_id)s, %(task_id)s, %(event)s, %(execution_date)s, %(owner)s, %(extra)s)]
[parameters: {'task_id': None, 'extra': "[('is_paused', u'true'), ('dag_id', u'test')]", 'execution_date': None, 'event': 'paused', 'owner': 'anonymous', 'dttm': datetime.datetime(2019, 9, 30, 18, 0, 51, 768073, tzinfo=<Timezone [UTC]>), 'dag_id': u'test'}]
(Background on this error at: http://sqlalche.me/e/f405)
From what I saw on the Airflow documentation, I'm making the change correctly, and the 'airflow initdb / resetdb' commands execute without error.
I already spent a fair amount of time googling this error, but there are no clear answers to this problem. I'm really not sure if I'm missing a prerequisite or if I should be using a different connector?
EDIT: I'm using python 2.7, as seen in the stack trace. Airflow claims compatibility in the short-term, but I see another SO user's problems went away after upgrading to python 3.6: Link to other solution. I'll try this and update if it seems to work.
It looks like the solution is indeed to upgrade to python 3.6, leveraging a virtual environment due to the required duality of python 2.x and 3.y with Linux systems and Airflow. Specifically, I followed this guide and my DAGs seem to be executing successfully.
I used Djano to build a simple restful API server. SSL has been introduced to the communication between the server and the remote database to enforce security. When the API server tried to write to the database, I got the following error:
File "/app/anaconda/lib/python2.7/site-packages/django/core/handlers/exception.py", line 39, in inner
response = get_response(request)
File "/app/anaconda/lib/python2.7/site-packages/django/core/handlers/base.py", line 187, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/app/anaconda/lib/python2.7/site-packages/django/core/handlers/base.py", line 185, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/app/anaconda/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
return view_func(*args, **kwargs)
File "/app/anaconda/lib/python2.7/site-packages/django/views/generic/base.py", line 68, in view
return self.dispatch(request, *args, **kwargs)
File "/app/anaconda/lib/python2.7/site-packages/rest_framework/views.py", line 474, in dispatch
response = self.handle_exception(exc)
File "/app/anaconda/lib/python2.7/site-packages/rest_framework/views.py", line 434, in handle_exception
self.raise_uncaught_exception(exc)
File "/app/anaconda/lib/python2.7/site-packages/rest_framework/views.py", line 471, in dispatch
response = handler(request, *args, **kwargs)
File "/app/anaconda/lib/python2.7/site-packages/rest_framework/decorators.py", line 52, in handler
return func(*args, **kwargs)
File "/app/mobile/mpq_backup_response/mpq_backup_response/backup_response/views.py", line 39, in sms
q.save()
File "/app/anaconda/lib/python2.7/site-packages/django/db/models/base.py", line 796, in save
force_update=force_update, update_fields=update_fields)
File "/app/anaconda/lib/python2.7/site-packages/django/db/models/base.py", line 821, in save_base
with transaction.atomic(using=using, savepoint=False):
File "/app/anaconda/lib/python2.7/site-packages/django/db/transaction.py", line 158, in __enter__
if not connection.get_autocommit():
File "/app/anaconda/lib/python2.7/site-packages/django/db/backends/base/base.py", line 365, in get_autocommit
self.ensure_connection()
File "/app/anaconda/lib/python2.7/site-packages/django/db/backends/base/base.py", line 199, in ensure_connection
self.connect()
File "/app/anaconda/lib/python2.7/site-packages/django/db/backends/base/base.py", line 171, in connect
self.connection = self.get_new_connection(conn_params)
File "/app/anaconda/lib/python2.7/site-packages/django/db/backends/mysql/base.py", line 265, in get_new_connection
conn = Database.connect(**conn_params)
File "/app/anaconda/lib/python2.7/site-packages/pymysql/__init__.py", line 90, in Connect
return Connection(*args, **kwargs)
File "/app/anaconda/lib/python2.7/site-packages/pymysql/connections.py", line 688, in __init__
self.connect()
File "/app/anaconda/lib/python2.7/site-packages/pymysql/connections.py", line 906, in connect
self._request_authentication()
File "/app/anaconda/lib/python2.7/site-packages/pymysql/connections.py", line 1086, in _request_authentication
self._sock = self.ctx.wrap_socket(self._sock, server_hostname=self.host)
File "/app/anaconda/lib/python2.7/ssl.py", line 352, in wrap_socket
_context=self)
TypeError: __init__() got an unexpected keyword argument 'server_hostname'
However, if I went into Djangon's shell interactive environment and wrote directly into the database, everything went smoothly. I don't why this problem happens.
This issue seems to exist in OS X with python 2.7.10 and according to the link below, you will most likely run into this with CentOS 7 and Python ver 2.7.9 or above.
https://github.com/gevent/gevent/issues/477
Updating gevent to 1.2.1 seems to solve the problem.
You could run "pip install --upgrade gevent" on mac to fix this.
I am using Django and celery to run some tasks. i have 6 workers with concurrency 1, and different tasks are sent to different workers. one of witch raises a Error in timer: OperationalError(2006, 'MySQL server has gone away').
system runs normally until something happens and worker stops running tasks until manually restarted.
I have:
Django==1.6
celery==3.1.5
django-celery==3.1.1
MySQL-python==1.2.4
kombu==3.0.6
Traceback:
[2015-12-10 09:50:43,300: ERROR/MainProcess] Error in timer: OperationalError(2006, 'MySQL server has gone away')
Traceback (most recent call last):
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/kombu/async/timer.py", line 171, in apply_entry
entry()
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/kombu/async/timer.py", line 64, in __call__
return self.fun(*self.args, **self.kwargs)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/kombu/async/timer.py", line 132, in _reschedules
return fun(*args, **kwargs)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/celery/events/snapshot.py", line 73, in capture
self.state.freeze_while(self.shutter, clear_after=self.clear_after)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/celery/events/state.py", line 428, in freeze_while
return fun(*args, **kwargs)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/celery/events/snapshot.py", line 70, in shutter
self.on_shutter(self.state)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/djcelery/snapshot.py", line 139, in on_shutter
_handle_tasks()
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/djcelery/snapshot.py", line 135, in _handle_tasks
self.handle_task(task)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/djcelery/snapshot.py", line 82, in handle_task
(task.worker.hostname, task.worker),
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/djcelery/snapshot.py", line 72, in handle_worker
defaults={'last_heartbeat': self.get_heartbeat(worker)},
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/djcelery/managers.py", line 87, in update_or_create
return get_queryset(self).update_or_create(**kwargs)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/djcelery/managers.py", line 70, in update_or_create
obj, created = self.get_or_create(**kwargs)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/query.py", line 373, in get_or_create
return self.get(**lookup), False
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/query.py", line 301, in get
num = len(clone)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/query.py", line 77, in __len__
self._fetch_all()
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/query.py", line 854, in _fetch_all
self._result_cache = list(self.iterator())
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/query.py", line 220, in iterator
for row in compiler.results_iter():
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 710, in results_iter
for rows in self.execute_sql(MULTI):
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 781, in execute_sql
cursor.execute(sql, params)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/backends/util.py", line 53, in execute
return self.cursor.execute(sql, params)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/utils.py", line 99, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/backends/util.py", line 53, in execute
return self.cursor.execute(sql, params)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/django/db/backends/mysql/base.py", line 124, in execute
return self.cursor.execute(query, args)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "/opt/virtualenvs/simocEnv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
OperationalError: (2006, 'MySQL server has gone away')
if you use flask, you can use db.session.close() before some CRUD operator.
Reference:
OperationalError: (2006, 'MySQL server has gone away after upgrading from celery 4.1.0 to 4.2.0')
OperationalError: (2006, 'MySQL server has gone away')