I am using mysql2 with nodejs and currently using simple connection like the following:
mysql.ts
const Connect = () => {
if (!connection)
return mysql.createConnection(params);
return connection;
};
someQuery.ts
const connection = Connect();
const asyncQuery = util.promisify(connection.query).bind(connection);
Now, after a while that the server is running the app just crashes because of the connection loss, I read some online and understood that in order to solve it and also as a better practice I should use a connection pool.
The thing is that I haven't really found anything that works like the above with async-await.
Does anyone have a solution for this?
await asyncQuery('query here');
i had a project last year, where i used the mysql2/promise library.
connector.js
const mysql = require('mysql2/promise');
const con = mysql.createPool({
host: "localhost",
user: "root",
password: "pw",
database: "database",
waitForConnections: true,
connectionLimit: 10,
queueLimit: 0
});
module.exports = con;
queries.js
const db = require('../helpers/mysql/connector');
getAnything: async () => {
try {
let anything = await db.execute("SELECT * FROM anything");
return colos[0];
} catch (e) {
console.log(e);
return null;
}
},
Related
Hello Everyone,
I'm a beginner in Node.js Mysql. I have connected to Node.js with mysql. While starting the Node.js server, I got the error like " code: 'ER_TOO_MANY_USER_CONNECTIONS', " further I will attach the mysql db connection code below. Any type of help would be appreciated. Thanks in advance...
var db = mysql.createPool({
host: 'xxxxxxxxxxxxxxxxx',
port: 'xxx',
user: 'xxxx',
password: 'xxx',
database: 'xxx'
});
db.getConnection((err, tempConn) => {
if (err) {
console.log(err);
}
else {
tempConn.release();
console.log('Mysql Connected');
}
});
module.exports={db};
If you're creating a pool you don't need to use getConnection. There is a shortcut that allows you to use it directly. If you do use getConnection you must follow it with a query, then you may release the connection. Your example is missing a query.
Here is a helpful template for using a pool config:
// in your application initialization file such as app.js
//
// other require items here as well like express maybe?
//
const mysql = require('mysql');
const connection = mysql.createPool({
connectionLimit: 10,
host: process.env.DB_HOST || '127.0.0.1',
user: process.env.DB_USER || 'local_user',
password: process.env.DB_PASSWORD || 'local_password',
database: process.env.DB_NAME || 'local_database',
multipleStatements: true,
charset: 'utf8mb4' // necessary if you might need support for emoji characters
});
connection.on('connection', function (connection) {
// handy for testing
console.log('Pool id %d connected', connection.threadId);
});
connection.on('enqueue', function () {
// handy for testing
console.log('Waiting for available connection slot');
});
global.db = connection;
//
// other app setup stuff here like app.set, app.engine, app.use, module.exports = app and all that good stuff
//
// later…
// everywhere else in your app, use the global db variable when running queries
// ../new_users.js or similar maybe?
const _create_user = (user_payload) => {
db.query(
'INSERT INTO users SET ?', user_payload, function(error, results, fields) {
if (error) throw error;
console.log(results);
});
}
// maybe we are in a module that has access to
// the request object so we can use something
// that has come via POST
//
// here is a manual object as a placeholder…
let new_user = {
first_name: 'John',
last_name: 'Smith',
email: 'j.smith#example.com',
password: 'keyboard_cat'
}
_create_user(new_user);
My very simple Node.js code doesn't seem like its connection pool work as it's supposed to do. _connectionQueue of Pool object just gets longer and longer infinitely, and app dies. I mean it does make a pool and there are pre-made connections already, but they are not reusable or insert requests are too many and fast? I'm not sure..
I've tried to put some more connectionLimit like following :
let state = { pool: null }
export const connect = () => {
state.pool = mysql.createPool({
connectionLimit: 200,
host: process.env.DATABASE_HOST || 'localhost',
user: process.env.DATABASE_USER || 'root',
password: process.env.DATABASE_PASSWORD || 'password',
database: process.env.DATABASE_NAME || 'database'
})
}
export const get = () => state.pool
Mostly given job of this server is subscription and insertion. It subscribes several MQTT topics and just tries to insert messages into RDB. About 100 messages arrives every second, and that code looks like this.
mqttClient.on('message', function (topic, message) {
if(topic.includes('sensor')){
try {
const data = JSON.parse(message.toString())
if(validate(data.uuid)){
const params = [data.a, data.b, data.c, ...]
sensor.setStatus(params)
}
} catch(err){
console.error(err)
}
}
}
export const setStatus = (params) => {
const SQL = `INSERT INTO ...`
db.get().query(SQL, params, (err, result) => {
if (err) console.error(err)
})
}
Then, I see this through chrome-devtools
Object
pool: Pool
config: PoolConfig {acquireTimeout: 10000, connectionConfig: ConnectionConfig, waitForConnections: true, connectionLimit: 200, queueLimit: 0}
domain: null
_acquiringConnections: []
_allConnections: (200) [PoolConnection, PoolConnection, …]
_closed: false
_connectionQueue: (11561) [ƒ, ƒ, ƒ, ƒ, …]
_events: {}
_eventsCount: 0
_freeConnections: []
_maxListeners: undefined
__proto__: EventEmitter
__proto__: Object
I've put console.log into setStatus like following :
export const setStatus = (params) => {
const SQL = `INSERT INTO ...`
console.log(`allConnections=${db.get()._allConnections.length}, connectionQueue=${db.get()._connectionQueue.length}`)
db.get().query(SQL, params, (err, result) => {
if (err) console.error(err)
})
}
, and got these.
allConnections=200, connectionQueue=29
allConnections=200, connectionQueue=30
allConnections=200, connectionQueue=31
allConnections=200, connectionQueue=32
allConnections=200, connectionQueue=33
allConnections=200, connectionQueue=34
...
It seems like server created a connection pool very well, but not using those connections. Instead, trying to create a new connection more and more all the time and those requests just get stuck in _connectionQueue.
It appears you are creating a new pool every time you'd like to make a query. The common model is to create a pool once when the application starts, then use connections from that pool as needed (one pool, many connections).
Also if you're using a simple DB model you can simplify access to the pool by making it global. Below is an alternate to your code you might try:
app.js
const mysql = require('mysql');
const connection = mysql.createPool({
host: process.env.DB_HOST || '127.0.0.1',
user: process.env.DB_USER || 'local_user',
password: process.env.DB_PASSWORD || 'local_password',
database: process.env.DB_NAME || 'local_database'
});
global.db = connection;
modules.js
export const setStatus = (params) => {
let SQL = `INSERT INTO ...`
db.query(SQL, params, (err, result) => {
if (err) console.error(err)
console.log(result)
})
}
Documentation for further reference :: https://github.com/mysqljs/mysql#pooling-connections
Edit 1 - Log pool events
db.on('acquire', function (connection) {
console.log('Connection %d acquired', connection.threadId);
});
db.on('connection', function (connection) {
console.log('Pool id %d connected', connection.threadId);
});
db.on('enqueue', function () {
console.log('Waiting for available connection slot');
});
db.on('release', function (connection) {
console.log('Connection %d released', connection.threadId);
});
I saw another thread and and the post Create a MySQL Database Middleware with Node.js 8 and Async/Await, but right now I just want to use the util.promisify in the most simplest and straight-forward way that I can think, applying it to mysql connection-pool. Regarding the Node.js documentation, bellow is my code snipet:
exports.genAdminCfg = function (app) {
let sql = 'SELECT * FROM nav_tree;';
let pool = require('mysql').createPool({
host: 'localhost',
user: 'root',
password: 'mysql',
database: 'n4_ctrl',
connectionLimit: 4,
multipleStatements: true
});
/* --- Works fine without "promisify":
pool.query(sql, function (err, rows) {
if (err) {
console.log(err);
return err;
} else {
app.locals.adminCfg = genCfg(rows);
app.locals.adminCfg.dbConnectionPool = pool;
}
});
*/
/* --- Does not worke with "promisify":
TypeError: Cannot read property 'connectionConfig' of undefined
at Pool.query (/usr/home/zipper/node/n4.live/node_modules/mysql/lib/Pool.js:194:33)
*/
require('util').promisify(pool.query)(sql).then(function (rows) {
app.locals.adminCfg = genCfg(rows);
app.locals.adminCfg.dbConnectionPool = pool;
}, function (error) {
console.log(error);
});
};
The code I commented-out works fine without promisify. But the similar code next to it with promisify does not work and shows TypeError: Cannot read property 'connectionConfig' of undefined. What's wrong with the code?
node version = v8.1.4
It always should be expected that a method relies on the context, unless proven otherwise.
Since pool.query is a method, it should be bound to correct context:
const query = promisify(pool.query).bind(pool);
This may be unneeded because there are promise-based alternatives for most popular libraries that could make use of promises, including mysql.
Before sharing my example, here are greater details about bind method and async function.
const mysql = require('mysql')
const { promisify } = require('util')
const databaseConfig = {
connectionLimit : 10,
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME
}
const pool = mysql.createPool(databaseConfig)
const promiseQuery = promisify(pool.query).bind(pool)
const promisePoolEnd = promisify(pool.end).bind(pool)
const query = `select * from mock_table limit 1;`
const result = await promiseQuery(query) // use in async function
promisePoolEnd()
I'm getting confused. All the tutorials I see with mySql end up with something like this:
in models/dbconnection.js
var mysql = require('mysql');
port = process.env.PORT || 3333;
if (port == 3333) {
var connection = mysql.createConnection({
host: 'localhost',
port: 3306,
user: 'root',
password: 'root',
database: 'nameDataBase',
insecureAuth: true
});
} else {
console.log("Error");
}
connection.connect();
module.exports = connection;
And then in routes/user.js
...
router.delete("/:id", verifyToken, (req, res) => {
const newLocal = "DELETE FROM login_user WHERE id = ?";
connection.query(newLocal, [req.params.id], (err,rows,fields) => {
if (err) {
res.sendStatus(500);
return;
}
console.log(rows.affectedRows);
res.status(200).send({delete: rows});
});
});
module.exports = router;
model and controller aren't getting mixed here? If tomorrow I want to change the type of database, I have to make changes in the model and in the routes. Shouldn't I make functions such as getAllUsersBlaBla(params) in something like models/user.js and then call it from routes/user.js ?
I agree. There shouldn't be any database queries in the router, which is considered part of the controller in MVC.
The model should provide wrapper functions around database queries that can be called from the controller.
A lot of node apps (and probably tutorials) will choose simplicity rather than modularity, that's why you would see code like that.
Here's a working example of AWS Lambda and MySQL, but I'd like it to work with Sequelize. How do I initialize Sequelize to work with AWS Lambda? I have the authenticated IAM role working too.
https://dzone.com/articles/passwordless-database-authentication-for-aws-lambd
'use strict';
const mysql = require('mysql2');
const AWS = require('aws-sdk');
// TODO use the details of your database connection
const region = 'eu-west-1';
const dbPort = 3306;
const dbUsername = 'lambda'; // the name of the database user you created in step 2
const dbName = 'lambda_test'; // the name of the database your database user is granted access to
const dbEndpoint = 'lambdatest-cluster-1.cluster-c8o7oze6xoxs.eu-west-1.rds.amazonaws.com';
module.exports.handler = (event, context, cb) => {
var signer = new AWS.RDS.Signer();
signer.getAuthToken({ // uses the IAM role access keys to create an authentication token
region: region,
hostname: dbEndpoint,
port: dbPort,
username: dbUsername
}, function(err, token) {
if (err) {
console.log(`could not get auth token: ${err}`);
cb(err);
} else {
var connection = mysql.createConnection({
host: dbEndpoint,
port: dbPort,
user: dbUsername,
password: token,
database: dbName,
ssl: 'Amazon RDS',
authSwitchHandler: function (data, cb) { // modifies the authentication handler
if (data.pluginName === 'mysql_clear_password') { // authentication token is sent in clear text but connection uses SSL encryption
cb(null, Buffer.from(token + '\0'));
}
}
});
connection.connect();
// TODO replace with your SQL query
connection.query('SELECT * FROM lambda_test.test', function (err, results, fields) {
connection.end();
if (err) {
console.log(`could not execute query: ${err}`);
cb(err);
} else {
cb(undefined, results);
}
});
}
});
};
Instead of using mysql.createConnection() and use your RDS Signer token:
var sequelize = require('sequelize')
const Sequelize = new sequelize(
process.env.database_name,
process.env.databse_user,
token,
{
dialect: 'mysql',
dialectOptions: {
ssl: 'Amazon RDS',
authPlugins: { // authSwitchHandler is deprecated
mysql_clear_password: () => () => {
return token
}
}
},
host: process.env.db_proxy_endpoint,
port: process.env.db_port,
pool: {
min: 0, //default
max: 5, // default
idle: 3600000
},
define: {
charset: 'utf8mb4'
}
}
// then return your models (defined in separate files usually)
await Sequelize.authenticate() // this just does a SELECT 1+1 as result;
await Sequelize.sync() // DO NOT use this in production, this tries to create tables defined by your models. Consider using sequelize migrations instead of using sync()
Also it's a good idea to keep your database connection parameters in a config file so no one can see them. (process.env)
We are working with Sequelize and Lambda, but you will need to reserve more resources, in our case we need at least 1GB to run a lambda with Sequelize. Without it, just with mysql2 it runs just with 128MB.
But if you really wanna use Sequelize just replace your createConnection for something like what you will find in sequelize doc
Probably you will use the context.callbackWaitsForEmptyEventLoop=true because you may have some issues when you call the callback function and you get nothing because your Event Loop probably will never be empty.