Snowflake sqlalchemy python cant connect because of getaddrinfo failed inside virtual machine of company network - sqlalchemy
I need to read and write some data to a snowflake databse. I have the credentials and everything is working fine on my local pc. But in the virtual machine of the company I am doing this I get this
I think its a proxy problem but i dont know what to do and how to fix. On the virtual machine i can access the snowflake urls flawlessly and everything works in google chrome for example. but why is this request not working in python??
Why?
How can i fix it?
please help :(
error message `Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connection.py", line 174, in _new_conn
File "snowflake\connector\vendored\urllib3\util\connection.py", line 72, in create_connection
File "socket.py", line 954, in getaddrinfo
socket.gaierror: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 703, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 386, in _make_request
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 1042, in _validate_conn
File "snowflake\connector\vendored\urllib3\connection.py", line 358, in connect
File "snowflake\connector\vendored\urllib3\connection.py", line 186, in _new_conn
snowflake.connector.vendored.urllib3.exceptions.NewConnectionError: <snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002073B6C6C10>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\requests\adapters.py", line 489, in send
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 815, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 787, in urlopen
File "snowflake\connector\vendored\urllib3\util\retry.py", line 592, in increment
snowflake.connector.vendored.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=d395ab35-5f2a-4fb2-a83b-48458979f2c9&databaseName=project_database&schemaName=project_schema&request_guid=19c8e3a5-33ad-48c2-8124-fc69a1fa2af9 (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002073B6C6C10>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\network.py", line 1018, in _request_exec
File "snowflake\connector\vendored\requests\sessions.py", line 587, in request
File "snowflake\connector\vendored\requests\sessions.py", line 701, in send
File "snowflake\connector\vendored\requests\adapters.py", line 565, in send
snowflake.connector.vendored.requests.exceptions.ConnectionError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=d395ab35-5f2a-4fb2-a83b-48458979f2c9&databaseName=project_database&schemaName=project_schema&request_guid=19c8e3a5-33ad-48c2-8124-fc69a1fa2af9 (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002073B6C6C10>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\connection.py", line 1072, in __authenticate
File "snowflake\connector\auth.py", line 257, in authenticate
File "snowflake\connector\network.py", line 704, in _post_request
File "snowflake\connector\network.py", line 794, in fetch
File "snowflake\connector\network.py", line 917, in _request_exec_wrapper
File "snowflake\connector\network.py", line 837, in _request_exec_wrapper
File "snowflake\connector\network.py", line 1095, in _request_exec
snowflake.connector.errors.OperationalError: 251011: 251011: ConnectionTimeout occurred. Will be handled by authenticator
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "sqlalchemy\engine\base.py", line 3361, in _wrap_pool_connect
File "sqlalchemy\pool\base.py", line 327, in connect
File "sqlalchemy\pool\base.py", line 894, in _checkout
File "sqlalchemy\pool\base.py", line 493, in checkout
File "sqlalchemy\pool\impl.py", line 146, in _do_get
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\impl.py", line 143, in _do_get
File "sqlalchemy\pool\base.py", line 273, in _create_connection
File "sqlalchemy\pool\base.py", line 388, in __init__
File "sqlalchemy\pool\base.py", line 691, in __connect
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\base.py", line 686, in __connect
File "sqlalchemy\engine\create.py", line 578, in connect
File "sqlalchemy\engine\default.py", line 598, in connect
File "snowflake\connector\__init__.py", line 51, in Connect
File "snowflake\connector\connection.py", line 297, in __init__
File "snowflake\connector\connection.py", line 550, in connect
File "snowflake\connector\connection.py", line 789, in __open_connection
File "snowflake\connector\connection.py", line 1052, in _authenticate
File "snowflake\connector\connection.py", line 1117, in __authenticate
File "snowflake\connector\connection.py", line 1094, in __authenticate
File "snowflake\connector\auth_by_plugin.py", line 117, in handle_timeout
snowflake.connector.errors.OperationalError: 250001: 250001: Could not connect to Snowflake backend after 0 attempt(s).Aborting
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "src\snowflakesqlalchemy.py", line 19, in <module>
File "sqlalchemy\engine\base.py", line 3315, in connect
File "sqlalchemy\engine\base.py", line 96, in __init__
File "sqlalchemy\engine\base.py", line 3394, in raw_connection
File "sqlalchemy\engine\base.py", line 3364, in _wrap_pool_connect
File "sqlalchemy\engine\base.py", line 2198, in _handle_dbapi_exception_noconnection
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\engine\base.py", line 3361, in _wrap_pool_connect
File "sqlalchemy\pool\base.py", line 327, in connect
File "sqlalchemy\pool\base.py", line 894, in _checkout
File "sqlalchemy\pool\base.py", line 493, in checkout
File "sqlalchemy\pool\impl.py", line 146, in _do_get
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\impl.py", line 143, in _do_get
File "sqlalchemy\pool\base.py", line 273, in _create_connection
File "sqlalchemy\pool\base.py", line 388, in __init__
File "sqlalchemy\pool\base.py", line 691, in __connect
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\base.py", line 686, in __connect
File "sqlalchemy\engine\create.py", line 578, in connect
File "sqlalchemy\engine\default.py", line 598, in connect
File "snowflake\connector\__init__.py", line 51, in Connect
File "snowflake\connector\connection.py", line 297, in __init__
File "snowflake\connector\connection.py", line 550, in connect
File "snowflake\connector\connection.py", line 789, in __open_connection
File "snowflake\connector\connection.py", line 1052, in _authenticate
File "snowflake\connector\connection.py", line 1117, in __authenticate
File "snowflake\connector\connection.py", line 1094, in __authenticate
File "snowflake\connector\auth_by_plugin.py", line 117, in handle_timeout
sqlalchemy.exc.OperationalError: (snowflake.connector.errors.OperationalError) 250001: 250001: Could not connect to Snowflake backend after 0 attempt(s).Aborting
(Background on this error at: https://sqlalche.me/e/14/e3q8)
[44952] Failed to execute script 'snowflakesqlalchemy' due to unhandled exception!`
I also printed these logs:
`2023-01-10 11:17:18,250 - MainThread connection.py:275 - __init__() - INFO - Snowflake Connector for Python Version: 2.8.3, Python Version: 3.9.13, Platform: Windows-10-10.0.17763-SP0
2023-01-10 11:17:18,250 - MainThread connection.py:520 - connect() - DEBUG - connect
2023-01-10 11:17:18,250 - MainThread connection.py:810 - __config() - DEBUG - __config
2023-01-10 11:17:18,250 - MainThread connection.py:934 - __config() - INFO - This connection is in OCSP Fail Open Mode. TLS Certificates would be checked for validity and revocation status. Any other Certificate Revocation related exceptions or OCSP Responder failures would be disregarded in favor of connectivity.
2023-01-10 11:17:18,251 - MainThread connection.py:952 - __config() - INFO - Setting use_openssl_only mode to False
2023-01-10 11:17:18,251 - MainThread converter.py:145 - __init__() - DEBUG - use_numpy: False
2023-01-10 11:17:18,251 - MainThread converter_issue23517.py:27 - __init__() - DEBUG - initialized
2023-01-10 11:17:18,251 - MainThread connection.py:713 - __open_connection() - DEBUG - REST API object was created: kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:18,252 - MainThread auth.py:170 - authenticate() - DEBUG - authenticate
2023-01-10 11:17:18,252 - MainThread auth.py:200 - authenticate() - DEBUG - assertion content: *********
2023-01-10 11:17:18,252 - MainThread auth.py:203 - authenticate() - DEBUG - account=kzhwbsi-gb82213, user=karlpd4c, database=project_database, schema=project_schema, warehouse=None, role=None, request_id=10256208-cea8-4269-a480-820a1c55e4a3
2023-01-10 11:17:18,252 - MainThread auth.py:236 - authenticate() - DEBUG - body['data']: {'CLIENT_APP_ID': 'PythonConnector', 'CLIENT_APP_VERSION': '2.8.3', 'SVN_REVISION': None, 'ACCOUNT_NAME': 'kzhwbsi-gb82213', 'LOGIN_NAME': 'karlpd4c', 'CLIENT_ENVIRONMENT': {'APPLICATION': 'PythonConnector', 'OS': 'Windows', 'OS_VERSION': 'Windows-10-10.0.17763-SP0', 'PYTHON_VERSION': '3.9.13', 'PYTHON_RUNTIME': 'CPython', 'PYTHON_COMPILER': 'MSC v.1929 64 bit (AMD64)', 'OCSP_MODE': 'FAIL_OPEN', 'TRACING': 10, 'LOGIN_TIMEOUT': 120, 'NETWORK_TIMEOUT': None}, 'SESSION_PARAMETERS': {'AUTOCOMMIT': False, 'CLIENT_PREFETCH_THREADS': 4}}
2023-01-10 11:17:18,252 - MainThread auth.py:254 - authenticate() - DEBUG - Timeout set to 120
2023-01-10 11:17:18,253 - MainThread retry.py:351 - from_int() - DEBUG - Converted retries value: 1 -> Retry(total=1, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:18,253 - MainThread retry.py:351 - from_int() - DEBUG - Converted retries value: 1 -> Retry(total=1, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:18,253 - MainThread network.py:1147 - _use_requests_session() - DEBUG - Session status for SessionPool 'kzhwbsi-gb82213.snowflakecomputing.com', SessionPool 1/1 active sessions
2023-01-10 11:17:18,253 - MainThread network.py:827 - _request_exec_wrapper() - DEBUG - remaining request timeout: 120, retry cnt: 1
2023-01-10 11:17:18,254 - MainThread network.py:808 - add_request_guid() - DEBUG - Request guid: 3bcec6fe-8f7d-4c05-9203-626636c975ea
2023-01-10 11:17:18,254 - MainThread network.py:1006 - _request_exec() - DEBUG - socket timeout: 60
2023-01-10 11:17:18,257 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (1): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:19,295 - MainThread retry.py:594 - increment() - DEBUG - Incremented Retry for (url='/session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea'): Retry(total=0, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:19,295 - MainThread connectionpool.py:812 - urlopen() - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9A30>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed')': /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea
2023-01-10 11:17:19,296 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (2): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:19,297 - MainThread network.py:1090 - _request_exec() - DEBUG - Hit a timeout error while logging in. Will be handled by authenticator. Ignore the following. Error stack: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connection.py", line 174, in _new_conn
File "snowflake\connector\vendored\urllib3\util\connection.py", line 72, in create_connection
File "socket.py", line 954, in getaddrinfo
socket.gaierror: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 703, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 386, in _make_request
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 1042, in _validate_conn
File "snowflake\connector\vendored\urllib3\connection.py", line 358, in connect
File "snowflake\connector\vendored\urllib3\connection.py", line 186, in _new_conn
snowflake.connector.vendored.urllib3.exceptions.NewConnectionError: <snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\requests\adapters.py", line 489, in send
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 815, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 787, in urlopen
File "snowflake\connector\vendored\urllib3\util\retry.py", line 592, in increment
snowflake.connector.vendored.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\network.py", line 1018, in _request_exec
File "snowflake\connector\vendored\requests\sessions.py", line 587, in request
File "snowflake\connector\vendored\requests\sessions.py", line 701, in send
File "snowflake\connector\vendored\requests\adapters.py", line 565, in send
snowflake.connector.vendored.requests.exceptions.ConnectionError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
2023-01-10 11:17:19,305 - MainThread network.py:1152 - _use_requests_session() - DEBUG - Session status for SessionPool 'kzhwbsi-gb82213.snowflakecomputing.com', SessionPool 0/1 active sessions
2023-01-10 11:17:19,305 - MainThread connection.py:1087 - __authenticate() - DEBUG - Operational Error raised at authenticationfor authenticator: AuthByDefault
2023-01-10 11:17:19,305 - MainThread auth_by_plugin.py:114 - handle_timeout() - DEBUG - Default timeout handler invoked for authenticator
2023-01-10 11:17:19,305 - MainThread auth_by_plugin.py:123 - handle_timeout() - DEBUG - Hit connection timeout, attempt number 0. Will retry in a bit...
2023-01-10 11:17:19,305 - MainThread auth_by_plugin.py:56 - next_sleep_duration() - DEBUG - Sleeping for 2 seconds
2023-01-10 11:17:21,306 - MainThread auth.py:170 - authenticate() - DEBUG - authenticate
2023-01-10 11:17:21,306 - MainThread auth.py:200 - authenticate() - DEBUG - assertion content: *********
2023-01-10 11:17:21,306 - MainThread auth.py:203 - authenticate() - DEBUG - account=kzhwbsi-gb82213, user=karlpd4c, database=project_database, schema=project_schema, warehouse=None, role=None, request_id=f8210470-5260-46c4-b7a5-1458f5dc318a
2023-01-10 11:17:21,306 - MainThread auth.py:236 - authenticate() - DEBUG - body['data']: {'CLIENT_APP_ID': 'PythonConnector', 'CLIENT_APP_VERSION': '2.8.3', 'SVN_REVISION': None, 'ACCOUNT_NAME': 'kzhwbsi-gb82213', 'LOGIN_NAME': 'karlpd4c', 'CLIENT_ENVIRONMENT': {'APPLICATION': 'PythonConnector', 'OS': 'Windows', 'OS_VERSION': 'Windows-10-10.0.17763-SP0', 'PYTHON_VERSION': '3.9.13', 'PYTHON_RUNTIME': 'CPython', 'PYTHON_COMPILER': 'MSC v.1929 64 bit (AMD64)', 'OCSP_MODE': 'FAIL_OPEN', 'TRACING': 10, 'LOGIN_TIMEOUT': 120, 'NETWORK_TIMEOUT': None}, 'SESSION_PARAMETERS': {'AUTOCOMMIT': False, 'CLIENT_PREFETCH_THREADS': 4}}
2023-01-10 11:17:21,307 - MainThread auth.py:254 - authenticate() - DEBUG - Timeout set to 120
2023-01-10 11:17:21,307 - MainThread network.py:1147 - _use_requests_session() - DEBUG - Session status for SessionPool 'kzhwbsi-gb82213.snowflakecomputing.com', SessionPool 1/1 active sessions
2023-01-10 11:17:21,307 - MainThread network.py:827 - _request_exec_wrapper() - DEBUG - remaining request timeout: 120, retry cnt: 1
2023-01-10 11:17:21,307 - MainThread network.py:808 - add_request_guid() - DEBUG - Request guid: dbe010e1-6776-46ed-bc5a-9979d617bee4
2023-01-10 11:17:21,307 - MainThread network.py:1006 - _request_exec() - DEBUG - socket timeout: 60
2023-01-10 11:17:21,311 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (3): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:21,312 - MainThread retry.py:594 - increment() - DEBUG - Incremented Retry for (url='/session/v1/login-request?request_id=f8210470-5260-46c4-b7a5-1458f5dc318a&databaseName=project_database&schemaName=project_schema&request_guid=dbe010e1-6776-46ed-bc5a-9979d617bee4'): Retry(total=0, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:21,312 - MainThread connectionpool.py:812 - urlopen() - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002999382B790>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed')': /session/v1/login-request?request_id=f8210470-5260-46c4-b7a5-1458f5dc318a&databaseName=project_database&schemaName=project_schema&request_guid=dbe010e1-6776-46ed-bc5a-9979d617bee4
2023-01-10 11:17:21,312 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (4): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:21,314 - MainThread network.py:1090 - _request_exec() - DEBUG - Hit a timeout error while logging in. Will be handled by authenticator. Ignore the following. Error stack: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=f8210470-5260-46c4-b7a5-1458f5dc318a&databaseName=project_database&schemaName=project_schema&request_guid=dbe010e1-6776-46ed-bc5a-9979d617bee4 (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002999382BA00>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connection.py", line 174, in _new_conn
File "snowflake\connector\vendored\urllib3\util\connection.py", line 72, in create_connection
File "socket.py", line 954, in getaddrinfo
socket.gaierror: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
...
This is my python code:
#!/usr/bin/env python
from snowflake.sqlalchemy import URL
from sqlalchemy import create_engine
import pandas as pd
import logging
import os
engine = create_engine('snowflake://user:password#myaccount(e.g.asdfhjk-jh45567)/project_database/project_schema')
os.environ['NO_PROXY'] = 'snowflakecomputing.com'
path = input('logpath:\n')
for logger_name in ['snowflake','botocore']:
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
ch = logging.FileHandler(path+'python_connector.log')
ch.setLevel(logging.DEBUG)
ch.setFormatter(logging.Formatter('%(asctime)s - %(threadName)s %(filename)s:%(lineno)d - %(funcName)s() - %(levelname)s - %(message)s'))
logger.addHandler(ch)
try:
connection = engine.connect()
sql = "SELECT * FROM project_comments"
df = pd.read_sql_query(sql,connection)
print(df)
df.to_csv("./snowflakedata.csv",index=False)
finally:
engine.dispose()
connection.close()
Besides working with sqlalchemy i tried the snowflake connector:
import pandas as pd
import snowflake.connector
import sys
import os
us = input("user: \n")
pw = input("password: \n")
acc = input("account: \n")
cnn = snowflake.connector.connect(
user=us,
password=pw,
account=acc,
)
cs = cnn.cursor()
wh = input("warehouse: \n")
db = input("database: \n")
schema = input("schema: \n")
table = input("table: \n")
path = input("output directory: \n").replace("\\", "/")
try:
sql = "USE WAREHOUSE " + wh
cs.execute(sql)
sql = "USE DATABASE "+ db
cs.execute(sql)
sql = "USE SCHEMA " + schema
cs.execute(sql)
sql = "SELECT * FROM " + table
df = pd.read_sql_query(sql,cnn)
print(df)
df.to_csv(path + "snowflakedata.csv",index=False)
input("snowflake has been read, press any key to close")
finally:
cs.close()
cnn.close()
same problem occurs. On local pc its working fine and on the virtual machine in the on premise network it does not work.
and i tried to deactivate proxy in windows with:
set NO_PROXY=snowflakecomputing.com
Do I need the ip of the snowflake db? How do I get it? Is there something wrong with the dns configuration in sqlalchemy? how do i fix it? do i need another connector or database engine?
I figured out the answer:
If you are behind a coorperate proxy, you need to configure the proxy one in the environment variables as new variable then add these two:
HTTP_PROXY
http://companyuser:companypw#proxy.companydomain.companyname.com:8080
and
HTTPS_PROXY
http://companyuser:companypw#proxy.companydomain.companyname.com:8080
.Then you also need to configre those http and https proxy at application level, meaning in your python program at the beginning:
os.environ["http_proxy"] = "http://companyuser:companypw#proxy.companydomain.companyname.com:8080"
os.environ["https_proxy"] ="http://companyuser:companypw#proxy.companydomain.companyname.com:8080"
in my case the companyuser is the user from my windows machine.
Related
Elastic Beanstalk deployment fail cfnbootstrap.construction_errors.ToolError
My eb is on Amazon linux server: without this command in config file my file deployed successfully commands: brewCommands: command: brew install antiword but when i added this, it is throwing error cfnbootstrap.construction_errors.ToolError: here is my cfn-init.log file error Unhandled exception during build: Command 001_install failed Traceback (most recent call last): File "/opt/aws/bin/cfn-init", line 176, in <module> worklog.build(metadata, configSets) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 137, in build Contractor(metadata).build(configSets, self) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 564, in build self.run_config(config, worklog) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 576, in run_config CloudFormationCarpenter(config, self._auth_config).build(worklog) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 276, in build self._config.commands) File "/usr/lib/python3.7/site-packages/cfnbootstrap/command_tool.py", line 127, in apply raise ToolError(u"Command %s failed" % name) cfnbootstrap.construction_errors.ToolError: Command 001_install failed
peewee.OperationalError - MySQL connection refused
I'm using peewee for a Python ORM to a MySQL database and Flask to serve a web site from my local desktop running Ubuntu. I can connect to the database from the command line (mysql test -u user -p), but am getting a connection refused error when trying from Python. This code has worked on my previous Ubuntu PC, and I dumped the database from there, imported into the new PC and granted permissions, so I'm really not sure why it's giving me this. Model.py: from peewee import MySQLDatabase, Model, AutoField, CharField, IntegerField, \ ForeignKeyField, BooleanField, TextField, DateTimeField from flask_security import UserMixin, RoleMixin class BaseModel(Model): """ Base model for the model classes; prevents having to define Meta class with the database for each Model class. Must call BaseModel._meta.database.init() before connecting - should be able to just call once on BaseModel before using derived model classes. """ class Meta: database = MySQLDatabase(None) #init before calling connect() #Additional code to define various tables Trying to connect in Python command line: import Model as mmm db = mmm.BaseModel._meta.database db.init('test', host = 'localhost', user = 'mike', passwd = password, port = 3306) db.connect() Error stack trace: Traceback (most recent call last): File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 569, in connect sock = socket.create_connection( File "/usr/lib/python3.8/socket.py", line 808, in create_connection raise err File "/usr/lib/python3.8/socket.py", line 796, in create_connection sock.connect(sa) ConnectionRefusedError: [Errno 111] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3035, in connect self._state.set_connection(self._connect()) File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3933, in _connect conn = mysql.connect(db=self.database, **self.connect_params) File "/home/mike/.local/lib/python3.8/site-packages/pymysql/__init__.py", line 94, in Connect return Connection(*args, **kwargs) File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 327, in __init__ self.connect() File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 619, in connect raise exc pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on 'aspire' ([Errno 111] Connection refused)") During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<stdin>", line 1, in <module> File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3038, in connect self._initialize_connection(self._state.conn) File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 2873, in __exit__ reraise(new_type, new_type(exc_value, *exc_args), traceback) File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 183, in reraise raise value.with_traceback(tb) File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3035, in connect self._state.set_connection(self._connect()) File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3933, in _connect conn = mysql.connect(db=self.database, **self.connect_params) File "/home/mike/.local/lib/python3.8/site-packages/pymysql/__init__.py", line 94, in Connect return Connection(*args, **kwargs) File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 327, in __init__ self.connect() File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 619, in connect raise exc peewee.OperationalError: (2003, "Can't connect to MySQL server on 'aspire' ([Errno 111] Connection refused)")
Cannot connect to MySQL database in TensorFlow Extended
I am having problems using a MySQL database as a metadata database in a TensorFlow Extended pipeline. I used the penguin template to set up a very simple pipeline and also set up a MySQL database locally. The only thing I changed in the code was using tfx.orchestration.metadata.mysql_metadata_connection_config as the metadata_connection_config input to the pipeline instead of tfx.orchestration.metadata.sqlite_metadata_connection_config from tfx import v1 as tfx # [...] Add a simple CsvExampleGen tfx.dsl.Pipeline( pipeline_name=pipeline_name, pipeline_root=pipeline_root, components=components, metadata_connection_config=tfx.orchestration.metadata .mysql_metadata_connection_config( host="localhost", port=3306, database="ml_metadata", username="root", password="password"), beam_pipeline_args=beam_pipeline_args, ) Running this code results in the following error message: [2021-11-17 12:02:16,948] {taskinstance.py:1270} INFO - Marking task as FAILED. dag_id=penguin_pipeline, task_id=CsvExampleGen, execution_date=20211117T110208, start_date=20211117T110212, end_date=20211117T110216 [2021-11-17 12:02:16,966] {standard_task_runner.py:88} ERROR - Failed to execute job 101 for task CsvExampleGen Traceback (most recent call last): File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/task/task_runner/standard_task_runner.py", line 85, in _start_by_fork args.func(args, dag=self.dag) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/cli_parser.py", line 48, in command return func(*args, **kwargs) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/utils/cli.py", line 92, in wrapper return f(*args, **kwargs) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/commands/task_command.py", line 292, in task_run _run_task_by_selected_method(args, dag, ti) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/commands/task_command.py", line 107, in _run_task_by_selected_method _run_raw_task(args, ti) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/commands/task_command.py", line 180, in _run_raw_task ti._run_raw_task( File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/utils/session.py", line 70, in wrapper return func(*args, session=session, **kwargs) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1332, in _run_raw_task self._execute_task_with_callbacks(context) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1458, in _execute_task_with_callbacks result = self._execute_task(context, self.task) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1514, in _execute_task result = execute_callable(context=context) File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/operators/python.py", line 151, in execute return_value = self.execute_callable() File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/operators/python.py", line 162, in execute_callable return self.python_callable(*self.op_args, **self.op_kwargs) File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/airflow/airflow_component.py", line 76, in _airflow_component_launcher launcher.launch() File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/launcher/base_component_launcher.py", line 191, in launch execution_decision = self._run_driver(self._input_dict, self._output_dict, File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/launcher/base_component_launcher.py", line 152, in _run_driver with self._metadata_connection as m: File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/metadata.py", line 152, in __enter__ raise RuntimeError( RuntimeError: Failed to establish connection to Metadata storage with error: mysql_real_connect failed: errno: , error: In this log I was using AirFlow, but the same exception message is shown if I run a LocalDagRunner: Exception has occurred: RuntimeError Failed to establish connection to Metadata storage with error: mysql_real_connect failed: errno: , error: I have tried changing the host to "127.0.0.1", but this didn't change anything. Anyone had a similar issue or maybe see an apparent error in my approach? The MySQL server is version 8.0 and I am using TensorFlow Extended version 1.4.0
Google Cloud Function error "OperationError: code=3, message=Function failed on loading user code"
I get an error from time to time when deploying nodejs10 cloud functions to GCP. The error seems to go away on it's own, I just redeploy the same thing a few times. Anyone know what causes it? He's the log: command: gcloud beta functions deploy exchangeIcon --verbosity debug --runtime nodejs10 --memory 128 --region europe-west1 --timeout 5 --trigger-http --set-env-vars=FUNCTION_REGION=europe-west1,BUILD_DATE=2019-05-09T10:01:05.497Z --entry-point app DEBUG: Running [gcloud.beta.functions.deploy] with arguments: [--entry-point: "app", --memory: "134217728", --region: "europe-west1", --runtime: "nodejs10", --set-env-vars: "OrderedDict([(u'FUNCTION_REGION', u'europe-west1'), (u'BUILD_DATE', u'2019-05-09T10:01:05.497Z')])", --timeout: "5", --trigger-http: "True", --verbosity: "debug", NAME: "exchangeIcon"] INFO: Not using a .gcloudignore file. INFO: Not using a .gcloudignore file. Deploying function (may take a while - up to 2 minutes)... ..........................................................................failed. DEBUG: (gcloud.beta.functions.deploy) OperationError: code=3, message=Function failed on loading user code. Error message: Traceback (most recent call last): File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/calliope/cli.py", line 985, in Execute resources = calliope_command.Run(cli=self, args=args) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/calliope/backend.py", line 795, in Run resources = command_instance.Run(args) File "/Users/me/Downloads/google-cloud-sdk/lib/surface/functions/deploy.py", line 231, in Run enable_vpc_connector=True) File "/Users/me/Downloads/google-cloud-sdk/lib/surface/functions/deploy.py", line 175, in _Run return api_util.PatchFunction(function, updated_fields) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/util.py", line 300, in CatchHTTPErrorRaiseHTTPExceptionFn return func(*args, **kwargs) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/util.py", line 356, in PatchFunction operations.Wait(op, messages, client, _DEPLOY_WAIT_NOTICE) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/operations.py", line 126, in Wait _WaitForOperation(client, request, notice) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/operations.py", line 101, in _WaitForOperation sleep_ms=SLEEP_MS) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/core/util/retry.py", line 219, in RetryOnResult result = func(*args, **kwargs) File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/operations.py", line 65, in _GetOperationStatus raise exceptions.FunctionsError(OperationErrorToString(op.error)) FunctionsError: OperationError: code=3, message=Function failed on loading user code. Error message: ERROR: (gcloud.beta.functions.deploy) OperationError: code=3, message=Function failed on loading user code. In my Stackdriver Logging I just see INVALID_ARGUMENT, but nothing else.
The problem stems from your terminal commands not being properly formatted. --verbosity=debug is the proper way to type this. Also same thing with your runtime.
mysql.connector.errors.InterfaceError: Failed parsing EOF packet
I run into such problems when I running my Django project(a week ago, the project work properly, today find this problem): My Django version is 1.10.2 with python version 3.5.2, MySQL version is 5.5 on ubuntu 14.0. /Users/deja/Virtualenv/python3.5/bin/python3.5 /Applications/PyCharm.app/Contents/helpers/pydev/pydevd.py --multiproc --qt-support --client 127.0.0.1 --port 60850 --file /Users/mozat/project/crawler_management_system/manage.py runserver 0.0.0.0:6380 pydev debugger: process 81527 is connecting Connected to pydev debugger (build 145.1504) pydev debugger: process 81528 is connecting Performing system checks... System check identified no issues (0 silenced). You have 13 unapplied migration(s). Your project may not work properly until you apply the migrations for app(s): admin, auth, contenttypes, sessions. Run 'python manage.py migrate' to apply them. January 03, 2017 - 07:55:47 Django version 1.10.2, using settings 'crawler_management_system.settings' Starting development server at http://0.0.0.0:6380/ Quit the server with CONTROL-C. Internal Server Error: / Traceback (most recent call last): File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/network.py", line 226, in recv_plain chunk = self.sock.recv(4 - packet_len) socket.timeout: timed out During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/mozat/project/crawler_management_system/crawler_management_system/mysql_utility.py", line 32, in open_db raise err File "/Users/mozat/project/crawler_management_system/crawler_management_system/mysql_utility.py", line 28, in open_db yield cursor File "/Users/mozat/project/crawler_management_system/crawler_management_system/views.py", line 40, in select_batch_records cursor.execute(sql.format(table=self.table,times=times)) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/cursor.py", line 515, in execute self._handle_result(self._connection.cmd_query(stmt)) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 488, in cmd_query result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query)) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 267, in _send_cmd return self._socket.recv() File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/network.py", line 255, in recv_plain errno=2055, values=(self.get_address(), _strioerror(err))) mysql.connector.errors.OperationalError: 2055: Lost connection to MySQL server at '127.0.0.1:10189', system error: timed out During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 710, in reset_session self.cmd_reset_connection() File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 1046, in cmd_reset_connection raise errors.NotSupportedError("MySQL version 5.7.2 and " mysql.connector.errors.NotSupportedError: MySQL version 5.7.2 and earlier does not support COM_RESET_CONNECTION. During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/protocol.py", line 267, in parse_eof unpacked = struct_unpack('<xxxBBHH', packet) struct.error: unpack requires a bytes object of length 9 During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/exception.py", line 39, in inner response = get_response(request) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/base.py", line 249, in _legacy_get_response response = self._get_response(request) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/base.py", line 187, in _get_response response = self.process_exception_by_middleware(e, request) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/base.py", line 185, in _get_response response = wrapped_callback(request, *callback_args, **callback_kwargs) File "/Users/mozat/project/crawler_management_system/crawler_management_system/views.py", line 99, in mainpage result = batch_record_repo.select_batch_records(times) File "/Users/mozat/project/crawler_management_system/crawler_management_system/views.py", line 41, in select_batch_records return cursor.fetchall() File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/contextlib.py", line 77, in __exit__ self.gen.throw(type, value, traceback) File "/Users/mozat/project/crawler_management_system/crawler_management_system/mysql_utility.py", line 40, in open_db connection.close() File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/pooling.py", line 117, in close cnx.reset_session() File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 713, in reset_session self._database, self._charset_id) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 661, in cmd_change_user self._post_connection() File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/abstracts.py", line 695, in _post_connection self.set_charset_collation(self._charset_id) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/abstracts.py", line 654, in set_charset_collation charset_name, collation_name)) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 869, in _execute_query self.cmd_query(query) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 488, in cmd_query result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query)) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 393, in _handle_result return self._handle_eof(packet) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 344, in _handle_eof eof = self._protocol.parse_eof(packet) File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/protocol.py", line 269, in parse_eof raise errors.InterfaceError(err_msg) mysql.connector.errors.InterfaceError: Failed parsing EOF packet. [03/Jan/2017 07:56:40] "GET / HTTP/1.1" 500 172448 anyone happen to see same problem?Could you please give me some suggestion.
all ,nothing wrong with mysql, and Django. It's for I use a complex SQL to access mysql: SELECT * FROM (SELECT *, #batch_rank := IF(#current_batch = spider , #batch_rank + 1, 1) AS batch_rank, #current_batch := spider FROM deja_crawler.crawl_batch_record where spider in (select distinct spider from deja_crawler.crawl_batch_record where spider like '%_new' or spider like '%_update') ORDER BY spider, create_time DESC) ranked WHERE batch_rank <= 3 the tables used to have not too much data, so it works, later, the table became huge, and the access time cost is almost 30s, but the connection time out is set as 20s, so it will raise time out error.