AWS Lambda and RDS working example (need it to work with Sequelize) - mysql

Here's a working example of AWS Lambda and MySQL, but I'd like it to work with Sequelize. How do I initialize Sequelize to work with AWS Lambda? I have the authenticated IAM role working too.
https://dzone.com/articles/passwordless-database-authentication-for-aws-lambd
'use strict';
const mysql = require('mysql2');
const AWS = require('aws-sdk');
// TODO use the details of your database connection
const region = 'eu-west-1';
const dbPort = 3306;
const dbUsername = 'lambda'; // the name of the database user you created in step 2
const dbName = 'lambda_test'; // the name of the database your database user is granted access to
const dbEndpoint = 'lambdatest-cluster-1.cluster-c8o7oze6xoxs.eu-west-1.rds.amazonaws.com';
module.exports.handler = (event, context, cb) => {
var signer = new AWS.RDS.Signer();
signer.getAuthToken({ // uses the IAM role access keys to create an authentication token
region: region,
hostname: dbEndpoint,
port: dbPort,
username: dbUsername
}, function(err, token) {
if (err) {
console.log(`could not get auth token: ${err}`);
cb(err);
} else {
var connection = mysql.createConnection({
host: dbEndpoint,
port: dbPort,
user: dbUsername,
password: token,
database: dbName,
ssl: 'Amazon RDS',
authSwitchHandler: function (data, cb) { // modifies the authentication handler
if (data.pluginName === 'mysql_clear_password') { // authentication token is sent in clear text but connection uses SSL encryption
cb(null, Buffer.from(token + '\0'));
}
}
});
connection.connect();
// TODO replace with your SQL query
connection.query('SELECT * FROM lambda_test.test', function (err, results, fields) {
connection.end();
if (err) {
console.log(`could not execute query: ${err}`);
cb(err);
} else {
cb(undefined, results);
}
});
}
});
};

Instead of using mysql.createConnection() and use your RDS Signer token:
var sequelize = require('sequelize')
const Sequelize = new sequelize(
process.env.database_name,
process.env.databse_user,
token,
{
dialect: 'mysql',
dialectOptions: {
ssl: 'Amazon RDS',
authPlugins: { // authSwitchHandler is deprecated
mysql_clear_password: () => () => {
return token
}
}
},
host: process.env.db_proxy_endpoint,
port: process.env.db_port,
pool: {
min: 0, //default
max: 5, // default
idle: 3600000
},
define: {
charset: 'utf8mb4'
}
}
// then return your models (defined in separate files usually)
await Sequelize.authenticate() // this just does a SELECT 1+1 as result;
await Sequelize.sync() // DO NOT use this in production, this tries to create tables defined by your models. Consider using sequelize migrations instead of using sync()
Also it's a good idea to keep your database connection parameters in a config file so no one can see them. (process.env)

We are working with Sequelize and Lambda, but you will need to reserve more resources, in our case we need at least 1GB to run a lambda with Sequelize. Without it, just with mysql2 it runs just with 128MB.
But if you really wanna use Sequelize just replace your createConnection for something like what you will find in sequelize doc
Probably you will use the context.callbackWaitsForEmptyEventLoop=true because you may have some issues when you call the callback function and you get nothing because your Event Loop probably will never be empty.

Related

Connecting Cypress V10++ into sql database [duplicate]

I get an error when running the integration tests:
0 passing (17s)
1 failure
1) Registration page
register new users allowed and update status in the database:
TypeError: Net.connect is not a function
at new Connection (webpack:///./node_modules/mysql2/lib/connection.js:50:0)
at ./node_modules/mysql2/index.js.exports.createConnection (webpack:///./node_modules/mysql2/index.js:10:0)
at Context.eval (webpack:///./cypress/integration/registration.spec.js:23:34)
Here is my environment:
MySQL Workbench
MySQL Server 8.0.29
I raised local backendless, I have access to the database. Here is my code:
const mysql2 = require('mysql2');
describe('Registration page', () => {
beforeEach(() => {
// visit the registration page
cy.visit('http://localhost:3000/registration');
});
it('register new users allowed and update status in the database', () => {
// fill out the registration form
cy.get('input[name="fullName"]').type("Nazar Dmytryshyn")
cy.get('input[type="email"]').type('testuser#example.com');
cy.get('input[name="pwd"]').type('testpassword');
cy.get('input[name="confirmPassword"]').type('testpassword');
// submit the form
cy.get('button[class="btn btn-success"]').click();
// check that the user is redirected to the login page
cy.url().should('include', '/login');
// create a connection to the test database
const connection = mysql2.createConnection({
host: '127.0.0.1:3306',
user: 'root',
password: 'rootpassword',
database: 'local1'
});
// open the connection
connection.connect();
// update the developer status in the database
connection.query(
'UPDATE `main_backendless`.`Developer` SET `developerStatusId` = "1" WHERE (`email` = "testuser#example.com")',
(error, results) => {
if (error) throw error;
expect(results.affectedRows).to.equal(1);
}
);
// close the connection
connection.end();
});
});
I checked this data 10 times, it is correct and I can connect to the database through MySQL WorkBench
host: '127.0.0.1:3306',
user: 'root',
password: 'rootpassword',
database: 'main_backendless'
I will be grateful for any ideas that can be achieved!
I recommend using the cypress-mysql, which hides a lot of the implementation details for you.
If you try to roll your own task, you may end up with an undefined return value.
Install
npm install cypress-mysql
//or
yarn add cypress-mysql
Configure
The release notes are out of date, here is the configuration for Cypress 10+
// cypress.config.js
const { defineConfig } = require("cypress");
const mysql = require('cypress-mysql');
module.exports = defineConfig({
// ...
e2e: {
setupNodeEvents(on, config) {
mysql.configurePlugin(on);
},
"env": {
"db": {
"host": "localhost",
"user": "user",
"password": "password",
"database": "database"
}
}
})
// cypress/support/e2e.js
const mysql = require('cypress-mysql');
mysql.addCommands();
Test
const sql = 'UPDATE "main_backendless.Developer" SET "developerStatusId" = "1" WHERE ("email" = "testuser#example.com")'
cy.query(sql).then(res => {
expect(res.affectedRows).to.equal(1)
});
If you want to use a task to call the mySql library, you must return a Promise from the task.
This is because the mysql calls are asynchronous, and the only way Cypress knows to wait for them is to get a promise returned from your code.
cypress.config.js
const { defineConfig } = require("cypress")
const mysql2 = require('mysql2')
const connection = mysql2.createConnection({
host: '127.0.0.1:3306',
user: 'root',
password: 'rootpassword',
database: 'local1'
})
module.exports = defineConfig({
e2e: {
setupNodeEvents(on, config) {
on('task', {
mySql: (sql) => {
return new Promise((resolve, reject) => {
connection.query(sql, (error, results) => {
if (error) {
reject(error)
} else {
resolve(results.affectedRows)
})
})
})
}
})
},
})
it('tests with mysql', () => {
cy.task('mySql', 'sql staement here')
.then(result => {
expect(result).to.equal(1);
})
})
With Promise-wrapper
Alternatively, mysql2 provides a promise-wrapper that can simplify your code:
const { defineConfig } = require("cypress")
const mysql = require('mysql2/promise') // different import here
const connection = mysql2.createConnection({
host: '127.0.0.1:3306',
user: 'root',
password: 'rootpassword',
database: 'local1'
})
module.exports = defineConfig({
e2e: {
setupNodeEvents(on, config) {
on('task', {
mySql: async (sql) => { // async here
const result = await connection.execute(sql) // await here
return result;
}
})
},
})
The issue is that you're using a nodejs library with Cypress. Cypress tests execute inside a browser and cannot directly utilize nodejs libraries within a test.
To do what you want to need to create a cy.task() to be able to execute code in nodejs.
Assuming you're using js, create a file with a function to use the sql connector
// runSql.js
const mysql2 = require('mysql2');
function runSql(sql) {
const connection = mysql2.createConnection({
host: '127.0.0.1:3306',
user: 'root',
password: 'rootpassword',
database: 'local1'
});
connection.connect();
let rows;
connection.query(sql, (error, results) => {
if (error) throw error;
rows = results.affectedRows
});
connection.end();
return rows;
}
module.exports = runSql;
Then in the cypress.config.js file
const runSql = require('./runSql.js');
module.exports = defineConfig({
// ...
e2e: {
setupNodeEvents(on, config) {
on('task', {
runSql
});
},
});
Now to call the task in a test
describe('Registration page', () => {
it('test', () => {
const sql = 'UPDATE `main_backendless`.`Developer` SET `developerStatusId` = "1" WHERE (`email` = "testuser#example.com")';
cy.task('runSql', sql).then((rows) => {
expect(rows).to.equal(1);
});
});
});

Dynamically change the name of mySql database in pool in node.js

connection.js
var mysql = require("mysql");
const { createPool } = require("mysql");
const pool = createPool({
port: process.env.DB_PORT,
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.MYSQL_DB,
connectionLimit: 10,
});
module.exports = pool;
product.service.js
const pool = require("../../../config/database");
module.exports = {
viewVariate: (callBack) => {
pool.query(
`SELECT varId , varName FROM variations WHERE varIsActive='1'`,
(errors, results, fields) => {
if (errors) {
return callBack(errors);
}
return callBack(null, results);
}
);
},
}
I have to maintain more than 100 users. so each person has own database in that node app. each person database name call via API , so I need to set given database name to above pool dynamically , I referred may ways. it did not work. connection.js file is my database connection with pool. then it call to service files to make connection queries. for that pool data obtains form env file. it doesn't matter when config mentioned requirement will be done.

How do I connect mysql with cypress through ssh tunneling?

Currently cypress supports mysql connection without using ssh as seen in the link below
https://docs.cypress.io/api/commands/task#Allows-a-single-argument-only
But I am trying to connect cypress to mysql through an ssh tunneling.
I am using the npm package mysql-ssh to establish the connection.
I am able to achieve this directly using node.js but I am facing issues while implementing through cypress. Here's the snippet I tried in node.js.
const mysqlssh = require('mysql-ssh');
const fs = require('fs');
mysqlssh.connect(
{
host: 'x.x.x.x',
user: 'xyz',
privateKey: fs.readFileSync('filePath') //this is the ssh filePath
},
{
host: 'HOST_NAME',
user: 'USER_NAME',
password: 'xxxx',
database: 'DB_NAME'
}
)
.then(client => {
client.query('select * from TABLE_NAME', function (err, results, fields) {
if (err)
{
console.log(err)
}
console.log(results);
mysqlssh.close()
})
})
.catch(err => {
console.log(err)
})
I want to do this either through the cypress/plugins/index.js file or directly in cypress/integration. Is there a simple way to do this?
I have found the solution. Here is my code for cypress/plugins/index.js file:
const dotenvPlugin = require('cypress-dotenv');
const mysqlssh = require('mysql-ssh');
const fs = require('fs');
module.exports = (on, config) => {
// `config` is the resolved Cypress config
config = dotenvPlugin(config);
on('task', {
executeSql (sql, ...args) {
return new Promise(async (resolve, reject) => {
try {
let connection = await mysqlssh.connect( {
host: process.env.SSH_HOST,
user: process.env.SSH_USER,
privateKey: fs.readFileSync(process.env.HOME + '/.ssh/id_rsa_old')
},
{
host: process.env.MYSQL_HOST,
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
database: process.env.MYSQL_DB
});
let result = await connection.promise().query(sql, args);
mysqlssh.close();
resolve(result[0][0]);
} catch (err) {
reject(err);
}
});
}
})
return config
}
So this connection has to be established in this file b/c cypress does not communicate with node process supplied by the host. So we need to use cypress tasks to run a Node code. see docs here - https://docs.cypress.io/api/commands/task#Examples
And in a test file example, I used it like so:
describe('Db Test', () => {
it('Query Test', () => {
cy.task('executeSql', 'SELECT count(id) as cnt FROM table_name').then(result => {
expect(result.cnt, 'Does not equal to 8').to.equal(2000);
})
})
})
P.S. Additional cypress-dotenv package is just used to load env vars from .env file.

errno: 1203, code: 'ER_TOO_MANY_USER_CONNECTIONS' in nodejs mysql code?

Hello Everyone,
I'm a beginner in Node.js Mysql. I have connected to Node.js with mysql. While starting the Node.js server, I got the error like " code: 'ER_TOO_MANY_USER_CONNECTIONS', " further I will attach the mysql db connection code below. Any type of help would be appreciated. Thanks in advance...
var db = mysql.createPool({
host: 'xxxxxxxxxxxxxxxxx',
port: 'xxx',
user: 'xxxx',
password: 'xxx',
database: 'xxx'
});
db.getConnection((err, tempConn) => {
if (err) {
console.log(err);
}
else {
tempConn.release();
console.log('Mysql Connected');
}
});
module.exports={db};
If you're creating a pool you don't need to use getConnection. There is a shortcut that allows you to use it directly. If you do use getConnection you must follow it with a query, then you may release the connection. Your example is missing a query.
Here is a helpful template for using a pool config:
// in your application initialization file such as app.js
//
// other require items here as well like express maybe?
//
const mysql = require('mysql');
const connection = mysql.createPool({
connectionLimit: 10,
host: process.env.DB_HOST || '127.0.0.1',
user: process.env.DB_USER || 'local_user',
password: process.env.DB_PASSWORD || 'local_password',
database: process.env.DB_NAME || 'local_database',
multipleStatements: true,
charset: 'utf8mb4' // necessary if you might need support for emoji characters
});
connection.on('connection', function (connection) {
// handy for testing
console.log('Pool id %d connected', connection.threadId);
});
connection.on('enqueue', function () {
// handy for testing
console.log('Waiting for available connection slot');
});
global.db = connection;
//
// other app setup stuff here like app.set, app.engine, app.use, module.exports = app and all that good stuff
//
// later…
// everywhere else in your app, use the global db variable when running queries
// ../new_users.js or similar maybe?
const _create_user = (user_payload) => {
db.query(
'INSERT INTO users SET ?', user_payload, function(error, results, fields) {
if (error) throw error;
console.log(results);
});
}
// maybe we are in a module that has access to
// the request object so we can use something
// that has come via POST
//
// here is a manual object as a placeholder…
let new_user = {
first_name: 'John',
last_name: 'Smith',
email: 'j.smith#example.com',
password: 'keyboard_cat'
}
_create_user(new_user);

Node.js: how to apply util.promisify to mysql pool in its simplest way?

I saw another thread and and the post Create a MySQL Database Middleware with Node.js 8 and Async/Await, but right now I just want to use the util.promisify in the most simplest and straight-forward way that I can think, applying it to mysql connection-pool. Regarding the Node.js documentation, bellow is my code snipet:
exports.genAdminCfg = function (app) {
let sql = 'SELECT * FROM nav_tree;';
let pool = require('mysql').createPool({
host: 'localhost',
user: 'root',
password: 'mysql',
database: 'n4_ctrl',
connectionLimit: 4,
multipleStatements: true
});
/* --- Works fine without "promisify":
pool.query(sql, function (err, rows) {
if (err) {
console.log(err);
return err;
} else {
app.locals.adminCfg = genCfg(rows);
app.locals.adminCfg.dbConnectionPool = pool;
}
});
*/
/* --- Does not worke with "promisify":
TypeError: Cannot read property 'connectionConfig' of undefined
at Pool.query (/usr/home/zipper/node/n4.live/node_modules/mysql/lib/Pool.js:194:33)
*/
require('util').promisify(pool.query)(sql).then(function (rows) {
app.locals.adminCfg = genCfg(rows);
app.locals.adminCfg.dbConnectionPool = pool;
}, function (error) {
console.log(error);
});
};
The code I commented-out works fine without promisify. But the similar code next to it with promisify does not work and shows TypeError: Cannot read property 'connectionConfig' of undefined. What's wrong with the code?
node version = v8.1.4
It always should be expected that a method relies on the context, unless proven otherwise.
Since pool.query is a method, it should be bound to correct context:
const query = promisify(pool.query).bind(pool);
This may be unneeded because there are promise-based alternatives for most popular libraries that could make use of promises, including mysql.
Before sharing my example, here are greater details about bind method and async function.
const mysql = require('mysql')
const { promisify } = require('util')
const databaseConfig = {
connectionLimit : 10,
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME
}
const pool = mysql.createPool(databaseConfig)
const promiseQuery = promisify(pool.query).bind(pool)
const promisePoolEnd = promisify(pool.end).bind(pool)
const query = `select * from mock_table limit 1;`
const result = await promiseQuery(query) // use in async function
promisePoolEnd()