Snowflake sqlalchemy python cant connect because of getaddrinfo failed inside virtual machine of company network - sqlalchemy

I need to read and write some data to a snowflake databse. I have the credentials and everything is working fine on my local pc. But in the virtual machine of the company I am doing this I get this
I think its a proxy problem but i dont know what to do and how to fix. On the virtual machine i can access the snowflake urls flawlessly and everything works in google chrome for example. but why is this request not working in python??
Why?
How can i fix it?
please help :(
error message `Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connection.py", line 174, in _new_conn
File "snowflake\connector\vendored\urllib3\util\connection.py", line 72, in create_connection
File "socket.py", line 954, in getaddrinfo
socket.gaierror: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 703, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 386, in _make_request
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 1042, in _validate_conn
File "snowflake\connector\vendored\urllib3\connection.py", line 358, in connect
File "snowflake\connector\vendored\urllib3\connection.py", line 186, in _new_conn
snowflake.connector.vendored.urllib3.exceptions.NewConnectionError: <snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002073B6C6C10>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\requests\adapters.py", line 489, in send
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 815, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 787, in urlopen
File "snowflake\connector\vendored\urllib3\util\retry.py", line 592, in increment
snowflake.connector.vendored.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=d395ab35-5f2a-4fb2-a83b-48458979f2c9&databaseName=project_database&schemaName=project_schema&request_guid=19c8e3a5-33ad-48c2-8124-fc69a1fa2af9 (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002073B6C6C10>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\network.py", line 1018, in _request_exec
File "snowflake\connector\vendored\requests\sessions.py", line 587, in request
File "snowflake\connector\vendored\requests\sessions.py", line 701, in send
File "snowflake\connector\vendored\requests\adapters.py", line 565, in send
snowflake.connector.vendored.requests.exceptions.ConnectionError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=d395ab35-5f2a-4fb2-a83b-48458979f2c9&databaseName=project_database&schemaName=project_schema&request_guid=19c8e3a5-33ad-48c2-8124-fc69a1fa2af9 (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002073B6C6C10>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\connection.py", line 1072, in __authenticate
File "snowflake\connector\auth.py", line 257, in authenticate
File "snowflake\connector\network.py", line 704, in _post_request
File "snowflake\connector\network.py", line 794, in fetch
File "snowflake\connector\network.py", line 917, in _request_exec_wrapper
File "snowflake\connector\network.py", line 837, in _request_exec_wrapper
File "snowflake\connector\network.py", line 1095, in _request_exec
snowflake.connector.errors.OperationalError: 251011: 251011: ConnectionTimeout occurred. Will be handled by authenticator
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "sqlalchemy\engine\base.py", line 3361, in _wrap_pool_connect
File "sqlalchemy\pool\base.py", line 327, in connect
File "sqlalchemy\pool\base.py", line 894, in _checkout
File "sqlalchemy\pool\base.py", line 493, in checkout
File "sqlalchemy\pool\impl.py", line 146, in _do_get
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\impl.py", line 143, in _do_get
File "sqlalchemy\pool\base.py", line 273, in _create_connection
File "sqlalchemy\pool\base.py", line 388, in __init__
File "sqlalchemy\pool\base.py", line 691, in __connect
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\base.py", line 686, in __connect
File "sqlalchemy\engine\create.py", line 578, in connect
File "sqlalchemy\engine\default.py", line 598, in connect
File "snowflake\connector\__init__.py", line 51, in Connect
File "snowflake\connector\connection.py", line 297, in __init__
File "snowflake\connector\connection.py", line 550, in connect
File "snowflake\connector\connection.py", line 789, in __open_connection
File "snowflake\connector\connection.py", line 1052, in _authenticate
File "snowflake\connector\connection.py", line 1117, in __authenticate
File "snowflake\connector\connection.py", line 1094, in __authenticate
File "snowflake\connector\auth_by_plugin.py", line 117, in handle_timeout
snowflake.connector.errors.OperationalError: 250001: 250001: Could not connect to Snowflake backend after 0 attempt(s).Aborting
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "src\snowflakesqlalchemy.py", line 19, in <module>
File "sqlalchemy\engine\base.py", line 3315, in connect
File "sqlalchemy\engine\base.py", line 96, in __init__
File "sqlalchemy\engine\base.py", line 3394, in raw_connection
File "sqlalchemy\engine\base.py", line 3364, in _wrap_pool_connect
File "sqlalchemy\engine\base.py", line 2198, in _handle_dbapi_exception_noconnection
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\engine\base.py", line 3361, in _wrap_pool_connect
File "sqlalchemy\pool\base.py", line 327, in connect
File "sqlalchemy\pool\base.py", line 894, in _checkout
File "sqlalchemy\pool\base.py", line 493, in checkout
File "sqlalchemy\pool\impl.py", line 146, in _do_get
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\impl.py", line 143, in _do_get
File "sqlalchemy\pool\base.py", line 273, in _create_connection
File "sqlalchemy\pool\base.py", line 388, in __init__
File "sqlalchemy\pool\base.py", line 691, in __connect
File "sqlalchemy\util\langhelpers.py", line 70, in __exit__
File "sqlalchemy\util\compat.py", line 211, in raise_
File "sqlalchemy\pool\base.py", line 686, in __connect
File "sqlalchemy\engine\create.py", line 578, in connect
File "sqlalchemy\engine\default.py", line 598, in connect
File "snowflake\connector\__init__.py", line 51, in Connect
File "snowflake\connector\connection.py", line 297, in __init__
File "snowflake\connector\connection.py", line 550, in connect
File "snowflake\connector\connection.py", line 789, in __open_connection
File "snowflake\connector\connection.py", line 1052, in _authenticate
File "snowflake\connector\connection.py", line 1117, in __authenticate
File "snowflake\connector\connection.py", line 1094, in __authenticate
File "snowflake\connector\auth_by_plugin.py", line 117, in handle_timeout
sqlalchemy.exc.OperationalError: (snowflake.connector.errors.OperationalError) 250001: 250001: Could not connect to Snowflake backend after 0 attempt(s).Aborting
(Background on this error at: https://sqlalche.me/e/14/e3q8)
[44952] Failed to execute script 'snowflakesqlalchemy' due to unhandled exception!`
I also printed these logs:
`2023-01-10 11:17:18,250 - MainThread connection.py:275 - __init__() - INFO - Snowflake Connector for Python Version: 2.8.3, Python Version: 3.9.13, Platform: Windows-10-10.0.17763-SP0
2023-01-10 11:17:18,250 - MainThread connection.py:520 - connect() - DEBUG - connect
2023-01-10 11:17:18,250 - MainThread connection.py:810 - __config() - DEBUG - __config
2023-01-10 11:17:18,250 - MainThread connection.py:934 - __config() - INFO - This connection is in OCSP Fail Open Mode. TLS Certificates would be checked for validity and revocation status. Any other Certificate Revocation related exceptions or OCSP Responder failures would be disregarded in favor of connectivity.
2023-01-10 11:17:18,251 - MainThread connection.py:952 - __config() - INFO - Setting use_openssl_only mode to False
2023-01-10 11:17:18,251 - MainThread converter.py:145 - __init__() - DEBUG - use_numpy: False
2023-01-10 11:17:18,251 - MainThread converter_issue23517.py:27 - __init__() - DEBUG - initialized
2023-01-10 11:17:18,251 - MainThread connection.py:713 - __open_connection() - DEBUG - REST API object was created: kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:18,252 - MainThread auth.py:170 - authenticate() - DEBUG - authenticate
2023-01-10 11:17:18,252 - MainThread auth.py:200 - authenticate() - DEBUG - assertion content: *********
2023-01-10 11:17:18,252 - MainThread auth.py:203 - authenticate() - DEBUG - account=kzhwbsi-gb82213, user=karlpd4c, database=project_database, schema=project_schema, warehouse=None, role=None, request_id=10256208-cea8-4269-a480-820a1c55e4a3
2023-01-10 11:17:18,252 - MainThread auth.py:236 - authenticate() - DEBUG - body['data']: {'CLIENT_APP_ID': 'PythonConnector', 'CLIENT_APP_VERSION': '2.8.3', 'SVN_REVISION': None, 'ACCOUNT_NAME': 'kzhwbsi-gb82213', 'LOGIN_NAME': 'karlpd4c', 'CLIENT_ENVIRONMENT': {'APPLICATION': 'PythonConnector', 'OS': 'Windows', 'OS_VERSION': 'Windows-10-10.0.17763-SP0', 'PYTHON_VERSION': '3.9.13', 'PYTHON_RUNTIME': 'CPython', 'PYTHON_COMPILER': 'MSC v.1929 64 bit (AMD64)', 'OCSP_MODE': 'FAIL_OPEN', 'TRACING': 10, 'LOGIN_TIMEOUT': 120, 'NETWORK_TIMEOUT': None}, 'SESSION_PARAMETERS': {'AUTOCOMMIT': False, 'CLIENT_PREFETCH_THREADS': 4}}
2023-01-10 11:17:18,252 - MainThread auth.py:254 - authenticate() - DEBUG - Timeout set to 120
2023-01-10 11:17:18,253 - MainThread retry.py:351 - from_int() - DEBUG - Converted retries value: 1 -> Retry(total=1, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:18,253 - MainThread retry.py:351 - from_int() - DEBUG - Converted retries value: 1 -> Retry(total=1, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:18,253 - MainThread network.py:1147 - _use_requests_session() - DEBUG - Session status for SessionPool 'kzhwbsi-gb82213.snowflakecomputing.com', SessionPool 1/1 active sessions
2023-01-10 11:17:18,253 - MainThread network.py:827 - _request_exec_wrapper() - DEBUG - remaining request timeout: 120, retry cnt: 1
2023-01-10 11:17:18,254 - MainThread network.py:808 - add_request_guid() - DEBUG - Request guid: 3bcec6fe-8f7d-4c05-9203-626636c975ea
2023-01-10 11:17:18,254 - MainThread network.py:1006 - _request_exec() - DEBUG - socket timeout: 60
2023-01-10 11:17:18,257 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (1): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:19,295 - MainThread retry.py:594 - increment() - DEBUG - Incremented Retry for (url='/session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea'): Retry(total=0, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:19,295 - MainThread connectionpool.py:812 - urlopen() - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9A30>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed')': /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea
2023-01-10 11:17:19,296 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (2): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:19,297 - MainThread network.py:1090 - _request_exec() - DEBUG - Hit a timeout error while logging in. Will be handled by authenticator. Ignore the following. Error stack: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connection.py", line 174, in _new_conn
File "snowflake\connector\vendored\urllib3\util\connection.py", line 72, in create_connection
File "socket.py", line 954, in getaddrinfo
socket.gaierror: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 703, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 386, in _make_request
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 1042, in _validate_conn
File "snowflake\connector\vendored\urllib3\connection.py", line 358, in connect
File "snowflake\connector\vendored\urllib3\connection.py", line 186, in _new_conn
snowflake.connector.vendored.urllib3.exceptions.NewConnectionError: <snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\vendored\requests\adapters.py", line 489, in send
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 815, in urlopen
File "snowflake\connector\vendored\urllib3\connectionpool.py", line 787, in urlopen
File "snowflake\connector\vendored\urllib3\util\retry.py", line 592, in increment
snowflake.connector.vendored.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "snowflake\connector\network.py", line 1018, in _request_exec
File "snowflake\connector\vendored\requests\sessions.py", line 587, in request
File "snowflake\connector\vendored\requests\sessions.py", line 701, in send
File "snowflake\connector\vendored\requests\adapters.py", line 565, in send
snowflake.connector.vendored.requests.exceptions.ConnectionError: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=10256208-cea8-4269-a480-820a1c55e4a3&databaseName=project_database&schemaName=project_schema&request_guid=3bcec6fe-8f7d-4c05-9203-626636c975ea (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x00000299937F9BE0>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
2023-01-10 11:17:19,305 - MainThread network.py:1152 - _use_requests_session() - DEBUG - Session status for SessionPool 'kzhwbsi-gb82213.snowflakecomputing.com', SessionPool 0/1 active sessions
2023-01-10 11:17:19,305 - MainThread connection.py:1087 - __authenticate() - DEBUG - Operational Error raised at authenticationfor authenticator: AuthByDefault
2023-01-10 11:17:19,305 - MainThread auth_by_plugin.py:114 - handle_timeout() - DEBUG - Default timeout handler invoked for authenticator
2023-01-10 11:17:19,305 - MainThread auth_by_plugin.py:123 - handle_timeout() - DEBUG - Hit connection timeout, attempt number 0. Will retry in a bit...
2023-01-10 11:17:19,305 - MainThread auth_by_plugin.py:56 - next_sleep_duration() - DEBUG - Sleeping for 2 seconds
2023-01-10 11:17:21,306 - MainThread auth.py:170 - authenticate() - DEBUG - authenticate
2023-01-10 11:17:21,306 - MainThread auth.py:200 - authenticate() - DEBUG - assertion content: *********
2023-01-10 11:17:21,306 - MainThread auth.py:203 - authenticate() - DEBUG - account=kzhwbsi-gb82213, user=karlpd4c, database=project_database, schema=project_schema, warehouse=None, role=None, request_id=f8210470-5260-46c4-b7a5-1458f5dc318a
2023-01-10 11:17:21,306 - MainThread auth.py:236 - authenticate() - DEBUG - body['data']: {'CLIENT_APP_ID': 'PythonConnector', 'CLIENT_APP_VERSION': '2.8.3', 'SVN_REVISION': None, 'ACCOUNT_NAME': 'kzhwbsi-gb82213', 'LOGIN_NAME': 'karlpd4c', 'CLIENT_ENVIRONMENT': {'APPLICATION': 'PythonConnector', 'OS': 'Windows', 'OS_VERSION': 'Windows-10-10.0.17763-SP0', 'PYTHON_VERSION': '3.9.13', 'PYTHON_RUNTIME': 'CPython', 'PYTHON_COMPILER': 'MSC v.1929 64 bit (AMD64)', 'OCSP_MODE': 'FAIL_OPEN', 'TRACING': 10, 'LOGIN_TIMEOUT': 120, 'NETWORK_TIMEOUT': None}, 'SESSION_PARAMETERS': {'AUTOCOMMIT': False, 'CLIENT_PREFETCH_THREADS': 4}}
2023-01-10 11:17:21,307 - MainThread auth.py:254 - authenticate() - DEBUG - Timeout set to 120
2023-01-10 11:17:21,307 - MainThread network.py:1147 - _use_requests_session() - DEBUG - Session status for SessionPool 'kzhwbsi-gb82213.snowflakecomputing.com', SessionPool 1/1 active sessions
2023-01-10 11:17:21,307 - MainThread network.py:827 - _request_exec_wrapper() - DEBUG - remaining request timeout: 120, retry cnt: 1
2023-01-10 11:17:21,307 - MainThread network.py:808 - add_request_guid() - DEBUG - Request guid: dbe010e1-6776-46ed-bc5a-9979d617bee4
2023-01-10 11:17:21,307 - MainThread network.py:1006 - _request_exec() - DEBUG - socket timeout: 60
2023-01-10 11:17:21,311 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (3): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:21,312 - MainThread retry.py:594 - increment() - DEBUG - Incremented Retry for (url='/session/v1/login-request?request_id=f8210470-5260-46c4-b7a5-1458f5dc318a&databaseName=project_database&schemaName=project_schema&request_guid=dbe010e1-6776-46ed-bc5a-9979d617bee4'): Retry(total=0, connect=None, read=None, redirect=None, status=None)
2023-01-10 11:17:21,312 - MainThread connectionpool.py:812 - urlopen() - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002999382B790>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed')': /session/v1/login-request?request_id=f8210470-5260-46c4-b7a5-1458f5dc318a&databaseName=project_database&schemaName=project_schema&request_guid=dbe010e1-6776-46ed-bc5a-9979d617bee4
2023-01-10 11:17:21,312 - MainThread connectionpool.py:1003 - _new_conn() - DEBUG - Starting new HTTPS connection (4): kzhwbsi-gb82213.snowflakecomputing.com:443
2023-01-10 11:17:21,314 - MainThread network.py:1090 - _request_exec() - DEBUG - Hit a timeout error while logging in. Will be handled by authenticator. Ignore the following. Error stack: HTTPSConnectionPool(host='kzhwbsi-gb82213.snowflakecomputing.com', port=443): Max retries exceeded with url: /session/v1/login-request?request_id=f8210470-5260-46c4-b7a5-1458f5dc318a&databaseName=project_database&schemaName=project_schema&request_guid=dbe010e1-6776-46ed-bc5a-9979d617bee4 (Caused by NewConnectionError('<snowflake.connector.vendored.urllib3.connection.HTTPSConnection object at 0x000002999382BA00>: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'))
Traceback (most recent call last):
File "snowflake\connector\vendored\urllib3\connection.py", line 174, in _new_conn
File "snowflake\connector\vendored\urllib3\util\connection.py", line 72, in create_connection
File "socket.py", line 954, in getaddrinfo
socket.gaierror: [Errno 11002] getaddrinfo failed
During handling of the above exception, another exception occurred:
...
This is my python code:
#!/usr/bin/env python
from snowflake.sqlalchemy import URL
from sqlalchemy import create_engine
import pandas as pd
import logging
import os
engine = create_engine('snowflake://user:password#myaccount(e.g.asdfhjk-jh45567)/project_database/project_schema')
os.environ['NO_PROXY'] = 'snowflakecomputing.com'
path = input('logpath:\n')
for logger_name in ['snowflake','botocore']:
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
ch = logging.FileHandler(path+'python_connector.log')
ch.setLevel(logging.DEBUG)
ch.setFormatter(logging.Formatter('%(asctime)s - %(threadName)s %(filename)s:%(lineno)d - %(funcName)s() - %(levelname)s - %(message)s'))
logger.addHandler(ch)
try:
connection = engine.connect()
sql = "SELECT * FROM project_comments"
df = pd.read_sql_query(sql,connection)
print(df)
df.to_csv("./snowflakedata.csv",index=False)
finally:
engine.dispose()
connection.close()
Besides working with sqlalchemy i tried the snowflake connector:
import pandas as pd
import snowflake.connector
import sys
import os
us = input("user: \n")
pw = input("password: \n")
acc = input("account: \n")
cnn = snowflake.connector.connect(
user=us,
password=pw,
account=acc,
)
cs = cnn.cursor()
wh = input("warehouse: \n")
db = input("database: \n")
schema = input("schema: \n")
table = input("table: \n")
path = input("output directory: \n").replace("\\", "/")
try:
sql = "USE WAREHOUSE " + wh
cs.execute(sql)
sql = "USE DATABASE "+ db
cs.execute(sql)
sql = "USE SCHEMA " + schema
cs.execute(sql)
sql = "SELECT * FROM " + table
df = pd.read_sql_query(sql,cnn)
print(df)
df.to_csv(path + "snowflakedata.csv",index=False)
input("snowflake has been read, press any key to close")
finally:
cs.close()
cnn.close()
same problem occurs. On local pc its working fine and on the virtual machine in the on premise network it does not work.
and i tried to deactivate proxy in windows with:
set NO_PROXY=snowflakecomputing.com
Do I need the ip of the snowflake db? How do I get it? Is there something wrong with the dns configuration in sqlalchemy? how do i fix it? do i need another connector or database engine?

I figured out the answer:
If you are behind a coorperate proxy, you need to configure the proxy one in the environment variables as new variable then add these two:
HTTP_PROXY
http://companyuser:companypw#proxy.companydomain.companyname.com:8080
and
HTTPS_PROXY
http://companyuser:companypw#proxy.companydomain.companyname.com:8080
.Then you also need to configre those http and https proxy at application level, meaning in your python program at the beginning:
os.environ["http_proxy"] = "http://companyuser:companypw#proxy.companydomain.companyname.com:8080"
os.environ["https_proxy"] ="http://companyuser:companypw#proxy.companydomain.companyname.com:8080"
in my case the companyuser is the user from my windows machine.

Related

Elastic Beanstalk deployment fail cfnbootstrap.construction_errors.ToolError

My eb is on Amazon linux server:
without this command in config file my file deployed successfully
commands:
brewCommands:
command: brew install antiword
but when i added this, it is throwing error
cfnbootstrap.construction_errors.ToolError:
here is my cfn-init.log file error
Unhandled exception during build: Command 001_install failed Traceback (most recent call last):
File "/opt/aws/bin/cfn-init", line 176, in <module>
worklog.build(metadata, configSets) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 137, in build
Contractor(metadata).build(configSets, self) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 564, in build
self.run_config(config, worklog) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 576, in run_config
CloudFormationCarpenter(config, self._auth_config).build(worklog) File "/usr/lib/python3.7/site-packages/cfnbootstrap/construction.py", line 276, in build
self._config.commands) File "/usr/lib/python3.7/site-packages/cfnbootstrap/command_tool.py", line 127, in apply
raise ToolError(u"Command %s failed" % name)
cfnbootstrap.construction_errors.ToolError: Command 001_install failed

peewee.OperationalError - MySQL connection refused

I'm using peewee for a Python ORM to a MySQL database and Flask to serve a web site from my local desktop running Ubuntu. I can connect to the database from the command line (mysql test -u user -p), but am getting a connection refused error when trying from Python. This code has worked on my previous Ubuntu PC, and I dumped the database from there, imported into the new PC and granted permissions, so I'm really not sure why it's giving me this.
Model.py:
from peewee import MySQLDatabase, Model, AutoField, CharField, IntegerField, \
ForeignKeyField, BooleanField, TextField, DateTimeField
from flask_security import UserMixin, RoleMixin
class BaseModel(Model):
"""
Base model for the model classes; prevents having to define Meta class with the
database for each Model class. Must call BaseModel._meta.database.init() before
connecting - should be able to just call once on BaseModel before
using derived model classes.
"""
class Meta:
database = MySQLDatabase(None) #init before calling connect()
#Additional code to define various tables
Trying to connect in Python command line:
import Model as mmm
db = mmm.BaseModel._meta.database
db.init('test', host = 'localhost', user = 'mike', passwd = password, port = 3306)
db.connect()
Error stack trace:
Traceback (most recent call last):
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 569, in connect
sock = socket.create_connection(
File "/usr/lib/python3.8/socket.py", line 808, in create_connection
raise err
File "/usr/lib/python3.8/socket.py", line 796, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3035, in connect
self._state.set_connection(self._connect())
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3933, in _connect
conn = mysql.connect(db=self.database, **self.connect_params)
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/__init__.py", line 94, in Connect
return Connection(*args, **kwargs)
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 327, in __init__
self.connect()
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 619, in connect
raise exc
pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on 'aspire' ([Errno 111] Connection refused)")
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3038, in connect
self._initialize_connection(self._state.conn)
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 2873, in __exit__
reraise(new_type, new_type(exc_value, *exc_args), traceback)
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 183, in reraise
raise value.with_traceback(tb)
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3035, in connect
self._state.set_connection(self._connect())
File "/home/mike/.local/lib/python3.8/site-packages/peewee.py", line 3933, in _connect
conn = mysql.connect(db=self.database, **self.connect_params)
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/__init__.py", line 94, in Connect
return Connection(*args, **kwargs)
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 327, in __init__
self.connect()
File "/home/mike/.local/lib/python3.8/site-packages/pymysql/connections.py", line 619, in connect
raise exc
peewee.OperationalError: (2003, "Can't connect to MySQL server on 'aspire' ([Errno 111] Connection refused)")

Cannot connect to MySQL database in TensorFlow Extended

I am having problems using a MySQL database as a metadata database in a TensorFlow Extended pipeline.
I used the penguin template to set up a very simple pipeline and also set up a MySQL database locally.
The only thing I changed in the code was using
tfx.orchestration.metadata.mysql_metadata_connection_config as the metadata_connection_config input to the pipeline instead of tfx.orchestration.metadata.sqlite_metadata_connection_config
from tfx import v1 as tfx
# [...] Add a simple CsvExampleGen
tfx.dsl.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=components,
metadata_connection_config=tfx.orchestration.metadata
.mysql_metadata_connection_config(
host="localhost",
port=3306,
database="ml_metadata",
username="root",
password="password"),
beam_pipeline_args=beam_pipeline_args,
)
Running this code results in the following error message:
[2021-11-17 12:02:16,948] {taskinstance.py:1270} INFO - Marking task as FAILED. dag_id=penguin_pipeline, task_id=CsvExampleGen, execution_date=20211117T110208, start_date=20211117T110212, end_date=20211117T110216
[2021-11-17 12:02:16,966] {standard_task_runner.py:88} ERROR - Failed to execute job 101 for task CsvExampleGen
Traceback (most recent call last):
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/task/task_runner/standard_task_runner.py", line 85, in _start_by_fork
args.func(args, dag=self.dag)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/cli_parser.py", line 48, in command
return func(*args, **kwargs)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/utils/cli.py", line 92, in wrapper
return f(*args, **kwargs)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/commands/task_command.py", line 292, in task_run
_run_task_by_selected_method(args, dag, ti)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/commands/task_command.py", line 107, in _run_task_by_selected_method
_run_raw_task(args, ti)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/cli/commands/task_command.py", line 180, in _run_raw_task
ti._run_raw_task(
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/utils/session.py", line 70, in wrapper
return func(*args, session=session, **kwargs)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1332, in _run_raw_task
self._execute_task_with_callbacks(context)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1458, in _execute_task_with_callbacks
result = self._execute_task(context, self.task)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1514, in _execute_task
result = execute_callable(context=context)
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/operators/python.py", line 151, in execute
return_value = self.execute_callable()
File "/home/user/anaconda3/lib/python3.8/site-packages/airflow/operators/python.py", line 162, in execute_callable
return self.python_callable(*self.op_args, **self.op_kwargs)
File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/airflow/airflow_component.py", line 76, in _airflow_component_launcher
launcher.launch()
File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/launcher/base_component_launcher.py", line 191, in launch
execution_decision = self._run_driver(self._input_dict, self._output_dict,
File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/launcher/base_component_launcher.py", line 152, in _run_driver
with self._metadata_connection as m:
File "/home/user/anaconda3/lib/python3.8/site-packages/tfx/orchestration/metadata.py", line 152, in __enter__
raise RuntimeError(
RuntimeError: Failed to establish connection to Metadata storage with error: mysql_real_connect failed: errno: , error:
In this log I was using AirFlow, but the same exception message is shown if I run a LocalDagRunner:
Exception has occurred: RuntimeError
Failed to establish connection to Metadata storage with error: mysql_real_connect failed: errno: , error:
I have tried changing the host to "127.0.0.1", but this didn't change anything. Anyone had a similar issue or maybe see an apparent error in my approach?
The MySQL server is version 8.0 and I am using TensorFlow Extended version 1.4.0

Google Cloud Function error "OperationError: code=3, message=Function failed on loading user code"

I get an error from time to time when deploying nodejs10 cloud functions to GCP. The error seems to go away on it's own, I just redeploy the same thing a few times. Anyone know what causes it? He's the log:
command: gcloud beta functions deploy exchangeIcon --verbosity debug --runtime nodejs10 --memory 128 --region europe-west1 --timeout 5 --trigger-http --set-env-vars=FUNCTION_REGION=europe-west1,BUILD_DATE=2019-05-09T10:01:05.497Z --entry-point app
DEBUG: Running [gcloud.beta.functions.deploy] with arguments: [--entry-point: "app", --memory: "134217728", --region: "europe-west1", --runtime: "nodejs10", --set-env-vars: "OrderedDict([(u'FUNCTION_REGION', u'europe-west1'), (u'BUILD_DATE', u'2019-05-09T10:01:05.497Z')])", --timeout: "5", --trigger-http: "True", --verbosity: "debug", NAME: "exchangeIcon"]
INFO: Not using a .gcloudignore file.
INFO: Not using a .gcloudignore file.
Deploying function (may take a while - up to 2 minutes)...
..........................................................................failed.
DEBUG: (gcloud.beta.functions.deploy) OperationError: code=3, message=Function failed on loading user code. Error message:
Traceback (most recent call last):
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/calliope/cli.py", line 985, in Execute
resources = calliope_command.Run(cli=self, args=args)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/calliope/backend.py", line 795, in Run
resources = command_instance.Run(args)
File "/Users/me/Downloads/google-cloud-sdk/lib/surface/functions/deploy.py", line 231, in Run
enable_vpc_connector=True)
File "/Users/me/Downloads/google-cloud-sdk/lib/surface/functions/deploy.py", line 175, in _Run
return api_util.PatchFunction(function, updated_fields)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/util.py", line 300, in CatchHTTPErrorRaiseHTTPExceptionFn
return func(*args, **kwargs)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/util.py", line 356, in PatchFunction
operations.Wait(op, messages, client, _DEPLOY_WAIT_NOTICE)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/operations.py", line 126, in Wait
_WaitForOperation(client, request, notice)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/operations.py", line 101, in _WaitForOperation
sleep_ms=SLEEP_MS)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/core/util/retry.py", line 219, in RetryOnResult
result = func(*args, **kwargs)
File "/Users/me/Downloads/google-cloud-sdk/lib/googlecloudsdk/api_lib/functions/operations.py", line 65, in _GetOperationStatus
raise exceptions.FunctionsError(OperationErrorToString(op.error))
FunctionsError: OperationError: code=3, message=Function failed on loading user code. Error message:
ERROR: (gcloud.beta.functions.deploy) OperationError: code=3, message=Function failed on loading user code.
In my Stackdriver Logging I just see INVALID_ARGUMENT, but nothing else.
The problem stems from your terminal commands not being properly formatted.
--verbosity=debug
is the proper way to type this. Also same thing with your runtime.

mysql.connector.errors.InterfaceError: Failed parsing EOF packet

I run into such problems when I running my Django project(a week ago, the project work properly, today find this problem):
My Django version is 1.10.2 with python version 3.5.2, MySQL version is 5.5 on ubuntu 14.0.
/Users/deja/Virtualenv/python3.5/bin/python3.5 /Applications/PyCharm.app/Contents/helpers/pydev/pydevd.py --multiproc --qt-support --client 127.0.0.1 --port 60850 --file /Users/mozat/project/crawler_management_system/manage.py runserver 0.0.0.0:6380
pydev debugger: process 81527 is connecting
Connected to pydev debugger (build 145.1504)
pydev debugger: process 81528 is connecting
Performing system checks...
System check identified no issues (0 silenced).
You have 13 unapplied migration(s). Your project may not work properly until you apply the migrations for app(s): admin, auth, contenttypes, sessions.
Run 'python manage.py migrate' to apply them.
January 03, 2017 - 07:55:47
Django version 1.10.2, using settings 'crawler_management_system.settings'
Starting development server at http://0.0.0.0:6380/
Quit the server with CONTROL-C.
Internal Server Error: /
Traceback (most recent call last):
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/network.py", line 226, in recv_plain
chunk = self.sock.recv(4 - packet_len)
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/mozat/project/crawler_management_system/crawler_management_system/mysql_utility.py", line 32, in open_db
raise err
File "/Users/mozat/project/crawler_management_system/crawler_management_system/mysql_utility.py", line 28, in open_db
yield cursor
File "/Users/mozat/project/crawler_management_system/crawler_management_system/views.py", line 40, in select_batch_records
cursor.execute(sql.format(table=self.table,times=times))
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/cursor.py", line 515, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 488, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 267, in _send_cmd
return self._socket.recv()
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/network.py", line 255, in recv_plain
errno=2055, values=(self.get_address(), _strioerror(err)))
mysql.connector.errors.OperationalError: 2055: Lost connection to MySQL server at '127.0.0.1:10189', system error: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 710, in reset_session
self.cmd_reset_connection()
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 1046, in cmd_reset_connection
raise errors.NotSupportedError("MySQL version 5.7.2 and "
mysql.connector.errors.NotSupportedError: MySQL version 5.7.2 and earlier does not support COM_RESET_CONNECTION.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/protocol.py", line 267, in parse_eof
unpacked = struct_unpack('<xxxBBHH', packet)
struct.error: unpack requires a bytes object of length 9
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/exception.py", line 39, in inner
response = get_response(request)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/base.py", line 249, in _legacy_get_response
response = self._get_response(request)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/base.py", line 187, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/django/core/handlers/base.py", line 185, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/Users/mozat/project/crawler_management_system/crawler_management_system/views.py", line 99, in mainpage
result = batch_record_repo.select_batch_records(times)
File "/Users/mozat/project/crawler_management_system/crawler_management_system/views.py", line 41, in select_batch_records
return cursor.fetchall()
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/mozat/project/crawler_management_system/crawler_management_system/mysql_utility.py", line 40, in open_db
connection.close()
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/pooling.py", line 117, in close
cnx.reset_session()
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 713, in reset_session
self._database, self._charset_id)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 661, in cmd_change_user
self._post_connection()
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/abstracts.py", line 695, in _post_connection
self.set_charset_collation(self._charset_id)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/abstracts.py", line 654, in set_charset_collation
charset_name, collation_name))
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 869, in _execute_query
self.cmd_query(query)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 488, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 393, in _handle_result
return self._handle_eof(packet)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/connection.py", line 344, in _handle_eof
eof = self._protocol.parse_eof(packet)
File "/Users/deja/Virtualenv/python3.5/lib/python3.5/site-packages/mysql/connector/protocol.py", line 269, in parse_eof
raise errors.InterfaceError(err_msg)
mysql.connector.errors.InterfaceError: Failed parsing EOF packet.
[03/Jan/2017 07:56:40] "GET / HTTP/1.1" 500 172448
anyone happen to see same problem?Could you please give me some suggestion.
all ,nothing wrong with mysql, and Django. It's for I use a complex SQL to access mysql:
SELECT *
FROM
(SELECT *,
#batch_rank := IF(#current_batch = spider , #batch_rank + 1, 1) AS batch_rank,
#current_batch := spider
FROM deja_crawler.crawl_batch_record
where spider in (select distinct spider from deja_crawler.crawl_batch_record where spider like '%_new' or spider like '%_update')
ORDER BY spider, create_time DESC)
ranked
WHERE batch_rank <= 3
the tables used to have not too much data, so it works, later, the table became huge, and the access time cost is almost 30s, but the connection time out is set as 20s, so it will raise time out error.