How can I connect to a moto mysql instance? - mysql

I created an RDS DB Instance using a mocked boto3 rds client. Here's how I set it up in my conftest.py
#pytest.fixture
def aws_credentials():
"""Mocked AWS Credentials for moto."""
os.environ["AWS_ACCESS_KEY_ID"] = "testing"
os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
os.environ["AWS_SECURITY_TOKEN"] = "testing"
os.environ["AWS_SESSION_TOKEN"] = "testing"
#pytest.fixture
def rds_client(aws_credentials, aws_region):
with mock_rds():
client = boto3.client("rds", region_name=aws_region)
yield client
Following the example here (https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.Connecting.Python.html) I set up my mysql connector like this:
db_instance = rds_client.create_db_instance(DBInstanceIdentifier="TestDBInstanceIdentifier",
DBInstanceClass="db.m4.large", Engine="mysql",
MasterUsername="root", DBName="TestDBName")
print("RDS Instance-----------------------------------------------------")
print(db_instance)
host = db_instance['DBInstance']['Endpoint']['Address']
port = db_instance['DBInstance']['Endpoint']['Port']
user = db_instance['DBInstance']['MasterUsername']
dbname = db_instance['DBInstance']['DBName']
print("Starting connection")
token = rds_client.generate_db_auth_token(DBHostname=host, Port=port, DBUsername=user, Region=aws_region)
mydb = mysql.connector.connect(host=host, database=dbname, user=user, passwd=token, port=port)
However, the connector can't find the DB:
FAILED test_read_rds_db - mysql.connector.errors.DatabaseError: 2005 (HY000): Unknown MySQL server host 'TestDBInstanceIdentifier.aaaaaaaaaa.us-east-1.rds.amazonaws.com' (8)
Has someone been able to set this up before?

Moto does not offer this functionality. It mocks the AWS API, but does not expose any RDBMS-functionality.
You could look into Localstack instead. It uses Moto in the background to mock calls to AWS, but offers features on top of that such as the ability to connect to an RDS instance.
See the docs here: https://docs.localstack.cloud/aws/rds/

Related

My Lambda is not able to access MySQL RDS. What to do?

I have an architecture where the lambda would run when a irs data file is put on the S3 bucket, I can easily connect to my RDS on my local machine but for some very weird reason the Lambda is not able to access it and giving error:
"errorMessage": "2022-11-15T22:22:51.919Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Task timed out after 60.06 seconds"
2-11-15T22:21:53.402Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 URI updated to: https://irs-data.s3.amazonaws.com/?prefix=index&encoding-type=url
[DEBUG] 2022-11-15T22:21:53.402Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Calculating signature using v4 auth.
[DEBUG] 2022-11-15T22:21:53.402Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 CanonicalRequest:
GET
/
encoding-type=url&prefix=index
host:irs-data.s3.amazonaws.com
x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
x-amz-date:20221115T222153Z
x-amz-security-token:IQoJb3JpZ2luX2VjEP///////////wEaCXVzLWVhc3QtMSJHMEUCIQC9x2awzo/kQIantnRem2kmylKVHw5fBV+ylz/PQeP0DwIgQHovdX5Jv9/cpe/PAaWDTBZGcc3TxXGUALQRJCh1XMsq7QII9///////////ARAAGgw5NzU4MjIzMjkxNDIiDDegGWv5Wxk3ihIEdCrBAlqSbCaW/e4tIn2SK5gAcePArZf5Ij7o1qhoqEyG2boXivxftDkd7vM3RGg9lK2YaMEx9ku3mCBFpS03T5zlbr2EnaQjRuvEZzdHBKY79qqbUOCqcITmYkQQK+GSCoAyfnckjbjY0yORD41/7OS6wRa9pRKzu0ib8V/aE8Uln5Eem9ylYSn7LdyNWanD2I0CNfYNMV+Xx0bduAhVyXP6HjXikjTG5e2gqlA61xQmq4NMXyRixxINUk47R1FWBqPnYVqQWOIPW1HKcbj26qlW+JJyh530ML1RK3qqkssnH7c0LGu8rJz9Ag9wldHcRlODljZcaOmX7OlErdwIImGoeb99ngcVKVrCc+QnegTQolsoAhU3AG68LrZrmY/zRborttAslMzeUpiZ4fkA86QKJJDdpJEL/sZc/ZXzBMCj2x/ZozD+odCbBjqeAVPiKRQMCuBUqK8LlnALW2ki6RwMyS8WmGFpSoDjUYcyFDhMkHSa8TnTa+0gdertafyc4c4NPfsWFBYTLavdkgmACCkug75ENt3LWAgpGvBMxp6f2hiZKjJzqQnOE6VofIUXU8PLycB+L9uaJuYplLuMoRmjURtHFj5whMZrGclS0+V9/eH2ep8x9SAiFIJ1yOimmox6FTw2DhvpuE8U
host;x-amz-content-sha256;x-amz-date;x-amz-security-token
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
[DEBUG] 2022-11-15T22:21:53.439Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 StringToSign:
AWS4-HMAC-SHA256
20221115T222153Z
20221115/us-east-1/s3/aws4_request
5d61ffe01d9d6dd6aee4b1faeecbf21721efb8696f94f969389c93b05579847c
[DEBUG] 2022-11-15T22:21:53.439Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Signature:
d6e70d2c6350adfa7231bd7b2a63e5ac2fd83583f5dde1dfada2b08854d493d2
[DEBUG] 2022-11-15T22:21:53.439Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Sending http request: <AWSPreparedRequest stream_output=False, method=GET, url=https://irs-data.s3.amazonaws.com/?prefix=index&encoding-type=url, headers={'User-Agent': b'Boto3/1.19.10 Python/3.9.13 Linux/4.14.255-285-225.501.amzn2.x86_64 exec-env/AWS_Lambda_python3.9 Botocore/1.22.12 Resource', 'X-Amz-Date': b'20221115T222153Z', 'X-Amz-Security-Token': b'IQoJb3JpZ2luX2VjEP///////////wEaCXVzLWVhc3QtMSJHMEUCIQC9x2awzo/kQIantnRem2kmylKVHw5fBV+ylz/PQeP0DwIgQHovdX5Jv9/cpe/PAaWDTBZGcc3TxXGUALQRJCh1XMsq7QII9///////////ARAAGgw5NzU4MjIzMjkxNDIiDDegGWv5Wxk3ihIEdCrBAlqSbCaW/e4tIn2SK5gAcePArZf5Ij7o1qhoqEyG2boXivxftDkd7vM3RGg9lK2YaMEx9ku3mCBFpS03T5zlbr2EnaQjRuvEZzdHBKY79qqbUOCqcITmYkQQK+GSCoAyfnckjbjY0yORD41/7OS6wRa9pRKzu0ib8V/aE8Uln5Eem9ylYSn7LdyNWanD2I0CNfYNMV+Xx0bduAhVyXP6HjXikjTG5e2gqlA61xQmq4NMXyRixxINUk47R1FWBqPnYVqQWOIPW1HKcbj26qlW+JJyh530ML1RK3qqkssnH7c0LGu8rJz9Ag9wldHcRlODljZcaOmX7OlErdwIImGoeb99ngcVKVrCc+QnegTQolsoAhU3AG68LrZrmY/zRborttAslMzeUpiZ4fkA86QKJJDdpJEL/sZc/ZXzBMCj2x/ZozD+odCbBjqeAVPiKRQMCuBUqK8LlnALW2ki6RwMyS8WmGFpSoDjUYcyFDhMkHSa8TnTa+0gdertafyc4c4NPfsWFBYTLavdkgmACCkug75ENt3LWAgpGvBMxp6f2hiZKjJzqQnOE6VofIUXU8PLycB+L9uaJuYplLuMoRmjURtHFj5whMZrGclS0+V9/eH2ep8x9SAiFIJ1yOimmox6FTw2DhvpuE8U', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=ASIA6GM4LCU3GISECH6S/20221115/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=d6e70d2c6350adfa7231bd7b2a63e5ac2fd83583f5dde1dfada2b08854d493d2'}>
[DEBUG] 2022-11-15T22:21:53.459Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Certificate path: /var/task/botocore/cacert.pem
[DEBUG] 2022-11-15T22:21:53.459Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Starting new HTTPS connection (1): irs-data.s3.amazonaws.com:443
2022-11-15T22:22:51.919Z 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Task timed out after 60.06 seconds
END RequestId: 9f20c035-5a47-4c6f-be9f-407b4a43aee6
REPORT RequestId: 9f20c035-5a47-4c6f-be9f-407b4a43aee6 Duration: 60061.89 ms Billed Duration: 60000 ms Memory Size: 128 MB Max Memory Used: 116 MB Init Duration: 1040.64 ms
Lambda Code with S3 data:
import pandas as pd
import boto3
import os
from dotenv import load_dotenv
import logging
import sys
import time
import datetime as dt
import io
import pymysql
####### LOADING ENVIRONMENT VARIABLES #######
load_dotenv()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
BUCKET = os.getenv('BUCKET')
BUCKET_PREFIX = os.getenv('BUCKET_PREFIX')
# Credentials to database connection
hostname= os.getenv('HOSTNAME')
dbname= os.getenv('DATABASE')
uname= os.getenv('USERNAME')
pwd= os.getenv('PASSWORD')
def lambda_handler(event, context):
try:
logger.info("TEST")
logger.info(BUCKET)
s3 = boto3.resource('s3')
# assigning the bucket:
my_bucket = s3.Bucket(BUCKET)
data_list = []
for my_bucket_object in my_bucket.objects.filter(Prefix=BUCKET_PREFIX):
if my_bucket_object.key.endswith(".csv"):
key=my_bucket_object.key
body=my_bucket_object.get()['Body'].read()
temp_data = pd.read_csv(io.BytesIO(body))
data_list.append(temp_data)
# concatenating all the files together:
df = pd.concat(data_list)
# Connect to MySQL Database
connection = pymysql.connect(host=hostname,user=uname,password=pwd,database=dbname)
cursor = connection.cursor()
# Truncate the table everytime before an ETL:
sql_trunc = "TRUNCATE TABLE `irs990`"
cursor.execute(sql_trunc)
# commit the results
connection.commit()
# creating columns from the dataframe:
cols = "`,`".join([str(i) for i in df.columns.tolist()])
# adding dataframe to mysql RDS
for i,row in df.iterrows():
sql = "INSERT INTO `irs990` (`" +cols + "`) VALUES (" + "%s,"*(len(row)-1) + "%s)"
cursor.execute(sql, tuple(row))
connection.commit()
# checking if data was successfully written:
sql = "SELECT * FROM `irs990`"
cursor.execute(sql)
result = cursor.fetchall()
for i in result:
print(i)
# closing MySQL connection:
connection.close()
except Exception as e:
logging.error(e)
My Lambda VPC details:
2
My RDS details:
3
4
5
Can somebody please help me what to do? I am assigning the lambda the same VPC as the RDS, I tried using the same security group as well and making sure the outbound IP address of lambda is in the inbound rules for RDS. But nothing :(
The proper security configuration should be:
A Security Group on the AWS Lambda function (Lambda-SG) that permits All outbound access (which is the default configuration)
A Security Group on the Amazon RDS database (DB-SG) that permits inbound connections on port 3306 from Lambda-SG
That is, DB-SG should specifically reference Lambda-SG. This will then permit the incoming connection from the Lambda function.
Merely putting the Lambda function and the RDS database "in the same Security Group" is insufficient because security groups apply to each resource individually. Unless the security group allows a connection from 'itself', this will not permit the desired access. Much better to use two security groups as described above.

Failing to connect to MySQL in Dlang with vibe-d and mysql-native

I have simple vibe-D program which is trying to connect to SQL:
import std.stdio;
import mysql;
import vibe.d;
void main()
{
MySQLPool db_pool = new MySQLPool("localhost","root","","dbname",3306);
Connection db = db_pool.lockConnection();
// same thing happens with:
// string connectionStr = "host=localhost;port=3306;user=root;db=dbname";
// db = new Connection(connectionStr);
}
(I deleted everything else for simplification)
Dependencies:
"dependencies": {
"mysql-native": "~>3.2.0",
"vibe-d": "~>0.9.4"
}
And it fails to connect with:
object.Exception#../../../.dub/packages/vibe-core-1.22.4/vibe-core/source/vibe/core/net.d(256): Failed to connect to [0:0:0:0:0:0:0:1]:3306: refused
When I try it without vibe-d in the dub project (using phobos sockets) it connects with no problem. What am I doing wrong?
that's an ipv6 address.... is your mysql listening on that interface? might help trying 127.0.0.1 instead of localhost and seeing what happens.
can also consider reconfiguring mysql to listen on all interfaces too, including the ipv6

Why do I get TCP/IP error when trying to create DB in my Lambda?

So I'm trying to deploy my Django project using lambda, with zappa. I'm using MySQL for DB engine. Now after doing some research, I realized that I needed to create a custom Django command to create DB, since I'm using MySQL. So I created crate_db command, zappa updated, then ran zappa manage dev create_db. Then I got this error: 2004 (HY000): Can't create TCP/IP socket (97)
below is my create_db.py file, for your information.
import sys
import logging
import mysql.connector
import os
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
rds_host = os.environ.get("MY HOST")
db_name = os.environ.get("")
user_name = os.environ.get("MY USERNAME")
password = os.environ.get("MY PASSWORD")
port = os.environ.get("3306")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
class Command(BaseCommand):
help = 'Creates the initial database'
def handle(self, *args, **options):
print('Starting db creation')
try:
db = mysql.connector.connect(host=rds_host, user=user_name,
password=password, db="mysql", connect_timeout=10)
c = db.cursor()
print("connected to db server")
c.execute("""CREATE DATABASE bookcake_db;""")
c.execute("""GRANT ALL ON bookcake_db.* TO 'ryan'#'%'""")
c.close()
print("closed db connection")
except mysql.connector.Error as err:
logger.error("Something went wrong: {}".format(err))
sys.exit()
Any ideas? Thanks.

How to connect to AWS MySQL database from Java?

I want to connect to AWS MySQL database instance. Here is my code:
val hikari = HikariConfig().run {
driverClassName = "com.mysql.jdbc.Driver"
jdbcUrl = "jdbc:mysql://${mainConfig.databaseHost}:${mainConfig.databasePort}" +
"?user=username&password=password"
username = Config.DATABASE_USER
password = Config.DATABASE_PASSWORD
isAutoCommit = false
transactionIsolation = "TRANSACTION_REPEATABLE_READ"
return HikariDataSource(this)
}
Database.connect(hikari(config))
And my mainConfig.databaseHost looks like: mydatabase.xyz.region.rds.amazonaws.com.
So, this connection is working but I can't to exec any SQL statements because:
java.sql.SQLException: No database selected
I've tried to specify database name in my jdbc url but it causes exception.
So how can I connect to specified AWS database?
After research I found that DB identifier and DB name are different, so I specified DB name in jdbc url and now it working!

Connecting MySQL to Scala using slick

I am trying to connect MySQL database do scala application using slick 3.1
In demo which is provided by Typesafe, this code is used as application.conf :
h2mem1 = {
url = "jdbc:h2:mem:test1"
driver = org.h2.Driver
connectionPool = disabled
keepAliveConnection = true
}
I am running MySQL database on ampps apache server and now I am confused regarding what to enter as url and driver so that my application can connect to database?
Should my url be "localhost"?
Something like this:
database = {
url = "jdbc:mysql://localhost/test"
user = "user"
password = "pwd"
driver = com.mysql.jdbc.Driver
connectionPool = disabled
keepAliveConnection = true
}