node.js mysql pool connection with async/ await - mysql

Is there a way to use pool.getConnection() taken from the mysqljs/mysql lib with the async/ await syntax?
The idea is to have a method which returns one connection which can be passed around amongst write queries with various foreign key constraints (sequential queries) before releasing it and at the same time potentially get further connections from the pool for the purpose of various read queries (parallel).

Share my working example:
I use this Promisified MySQL middleware for Node.js
read this article Create a MySQL Database Middleware with Node.js 8 and Async/Await
here is my database.js
var mysql = require('mysql');
// node -v must > 8.x
var util = require('util');
// !!!!! for node version < 8.x only !!!!!
// npm install util.promisify
//require('util.promisify').shim();
// -v < 8.x has problem with async await so upgrade -v to v9.6.1 for this to work.
// connection pool https://github.com/mysqljs/mysql [1]
var pool = mysql.createPool({
connectionLimit : process.env.mysql_connection_pool_Limit, // default:10
host : process.env.mysql_host,
user : process.env.mysql_user,
password : process.env.mysql_password,
database : process.env.mysql_database
})
// Ping database to check for common exception errors.
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
// Promisify for Node.js async/await.
pool.query = util.promisify(pool.query)
module.exports = pool
You must upgrade node -v > 8.x
you must use async function to be able to use await.
example:
var pool = require('./database')
// node -v must > 8.x, --> async / await
router.get('/:template', async function(req, res, next)
{
...
try {
var _sql_rest_url = 'SELECT * FROM arcgis_viewer.rest_url WHERE id='+ _url_id;
var rows = await pool.query(_sql_rest_url)
_url = rows[0].rest_url // first record, property name is 'rest_url'
if (_center_lat == null) {_center_lat = rows[0].center_lat }
if (_center_long == null) {_center_long= rows[0].center_long }
if (_center_zoom == null) {_center_zoom= rows[0].center_zoom }
_place = rows[0].place
} catch(err) {
throw new Error(err)
}

Mates. I don't know why but I tried all the day long but couldn't get it to work. By the help of your comments I tried again and it of course does work.
db.js:
const pool = mysql.createPool(config);
exports.getConnection = () => {
return new Promise((resolve, reject) => {
pool.getConnection(function (err, connection) {
if (err) {
return reject(err);
}
resolve(connection);
});
});
};
someWhereElse.js:
const db = require('./db');
const wrappingFunction = async () => {
const connection = await db.getConnection();
console.log(connection);
};
wrappingFunction();

Seems like implementing promises manually is a better option.
Just sharing what I have used in my code -
const mysql = require('mysql');
const config = require('config');
const pool = mysql.createPool(config.get('db.mysql'));
module.exports = {
checkConnection: () => {
return new Promise((resolve, reject) => {
pool.getConnection((err, conn) => {
if (err) {
return reject(err);
}
resolve(conn.release());
});
});
},
pool,
closeConnection: () => pool.end(),
};

Previous answers (with util.promisify) did not work for me, and only implementing Promise manually works:
Function:
async function removeItem (id) {
return new Promise( (resolve) => {
pool.query('DELETE FROM table_name WHERE id=' + id, (error) => {
resolve ({result: !error});
});
});
}
Usage:
const app = express();
const mysql = require('mysql');
const pool = mysql.createPool({
connectionLimit: 10,
host: 'localhost',
user: 'login',
password: 'pass',
database: 'dbname'
});
app.post("/:id", async (req, res) => {
const answer = await itemRemove(id);
res.send(answer);
});

Sure, you would have to promisify it first, which you can do since node 8.0.0 now:
const util = require('util');
async function doSomething() {
const getConnectionAsync = util.promisify(pool.getConnection);
try {
const result = await getConnectionAsync('MASTER');
}catch(err) {
console.log('Oh no');
}
}
If for some reason you can't use node 8 or above, there are other ways to promisify it, like http://bluebirdjs.com/docs/api/promise.promisify.html

Just sharing what I've always use in my code:
//Filename: MySQL.js
module.exports = {
connect: function ()
{
return new Promise((resolve, reject) => {
let pool = Mysql.createPool({ //require configfile.js or just put connection detail here
connectionLimit: config.mysql.connectionLimit,
host: config.mysql.host,
user: config.mysql.user,
password: config.mysql.password,
database: config.mysql.database
});
pool.getConnection((err, connection) =>
{
try
{
if (connection)
{
resolve({"status":"success", "data":"MySQL connected.", "con":pool});
connection.release();
}
}
catch (err)
{
reject({"status":"failed", "error":`MySQL error. ${err}`});
}
resolve({"status":"failed", "error":"Error connecting to MySQL."});
});
});
}
}
Then whenever you need to call the connection to MySQL
//Filename: somefile.js
const useMySQL = require('./path/to/MySQL');
module.exports = {
getSomething: function () {
return new Promise(async (resolve) => {
try
{
let connection = await useMySQL.connect();
con = connection.con;
//Do some query here, then
resolve(`Send some result/handle error`);
}
catch (err)
{
//Handle error if any, log, etc, and eventually
resolve(err);
}
});
}
Hope this helps.

Related

How can I resolve a promised mysql query in express.js?

I'm trying to use the npm package promise-mysql and return json data (or a string doesn't matter) but I'm having issues following the promise chain with await/async.
With the current code i'm receiving Promise { undefined } in the console.log I have right before the response to the user. The response just sends nothing to the user and closes it. Can anyone point in the right direction of how to debug this?
index.js
app.get("/", async (req, res) => {
console.log( Promise.resolve(await getLogs()) )
res.send(await getLogs());
});
mysql.js
const mysql = require("promise-mysql");
let pool;
async function startDatabasePool() {
pool = await mysql.createPool({
connectionLimit: 10,
host: "xxx",
user: "xxx",
password: "xxx",
database: "xxx"
});
}
async function getDatabasePool() {
if (!pool) await startDatabasePool();
return pool;
}
module.exports = {
getDatabasePool,
startDatabasePool
};
users.js
const { getDatabasePool } = require("./mysql");
async function getLogs() {
let pool = await getDatabasePool();
pool.query("SELECT * from logs order by logdate desc", function(
error,
results,
fields
) {
if (error) throw error;
return JSON.stringify(results);
});
}
module.exports = {
getLogs
};
index.js
app.get("/", async (req, res) => {
const result = await getLogs();
res.send(result);
});
mysql.js
const mysql = require("promise-mysql");
let pool;
module.exports.startDatabasePool = async () => {
pool = await mysql.createPool({
connectionLimit: 10,
host: "xxx",
user: "xxx",
password: "xxx",
database: "xxx"
});
}
module.exports.getDatabasePool = async () => {
if (!pool) await startDatabasePool();
return pool;
}
// convert function as promise
module.exports.executeQuery = async(params) => {
return new Promise((resolve, reject) => {
pool.query(params, function (error, result, fields) {
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
};
users.js
const { executeQuery } = require("./mysql");
module.exports.getLogs = async () => {
return await executeQuery("SELECT * from logs order by logdate desc");
}
First I'd try it like:
app.get("/", async (req, res) => {
let logs = await getLogs()
console.log(logs)
res.send(logs);
});
I hope it helps!

node js Calling stored procedure in mysql using connection pool and multiple parameters

db.js
const mysql = require('mysql');
const connection = mysql.createPool({
host: 'localhost',
user : 'xxxxx',
password : 'xxxxx',
database : 'xxxx'
})
module.exports = connection;
customer.js
const db = require("../../db");
in customer.js i need to call a SP of mysql which will take 10 input parameters as it will insert a record in table.
What will be the best way to call SP from customer.js using db.js function
I will create db class like this
let pool;
async function setConnectionPool() {
if (!pool) {
pool = await sql.connect("connectionstring here");
}
sql.on("error", err => {
logger.error("An error has occured while setting connection pool", err);
throw error;
});
}
async function save(entity) {
try {
await pool.request()
.input("SomeId1", sql.VarChar(50), entity.field1)
.execute("SomeProcedureName");
} catch (error) {
console.log("error", error);
}
}
module.exports = save;
From your customer.js you could simply call the method save
Hope this helps.

How to avoid deadlock in nodejs mysql with a lot of queries?

I have a lot of urls, for every url I call the function load(url), this function parse the html, extracts the needed data and builds a bulk insert query as you can see in my test.js code. The problem is, if I have to many urls (like 100+), I get a Error: ER_LOCK_DEADLOCK from mysql. I tried to use async.queue but this is somehow not working (I don't know why, maybe I am using is wrongly). How can I run many urls + queries one after another, avoiding parallel execution which I think resulted in a deadlock? Even using async.queue results to a DEADLOCK (not always).
test.js
const request = require('request');
const async = require('async');
const pool = require('./database');
const urls = [
'https://www.quora.com/What-is-the-best-way-to-have-delayed-job-queue-with-node-js',
'https://de.wikipedia.org/wiki/Reinhardt-Zimmermann-L%C3%B6sung',
'https://towardsdatascience.com/the-5-clustering-algorithms-data-scientists-need-to-know-a36d136ef68'
]
let load = function(url) {
request({url: url}, function(error, response, html) {
if(!error) {
console.log(html);
/**
* 1. Parse HTML
* 2. Create Array of Values
* 3. Call pool.query(sql, [values], function(error) { ... })
*/
let data = [{}];
let sql = "INSERT IGNORE INTO tbl_test (title, content) VALUES ?";
let values = [];
data.forEach((item) => { values.push(item) });
pool.query(sql, [values], function(error) {
if(error) throw error;
})
} else {
console.log("handle error...");
}
})
}
let jobs = []
/*urls.forEach((url) => {
//jobs.push(load(url)); // --> Works but fails if the urls list is to big -> mysql deadlock error!
jobs.push(function(callback) { callback(load(url)) });
})*/
let q = async.queue(function(task, callback) {
console.log("Task:", task.uri);
callback();
})
q.drain = function() {
console.log('all task completed');
pool.end();
}
urls.forEach((url) => {
q.push({uri: url}, function(err) {
console.log('finished processing ...')
});
});
databse.js
require('dotenv').config();
const mysql = require('mysql');
let pool = mysql.createPool(
{
connectionLimit: 10,
host: process.env.DB_HOST,
port: process.env.DB_PORT,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME
}
);
pool.getConnection((err, connection) => {
if(err) {
if(err.code === 'PROTOCOL_CONNECTION_LOST') {
console.log('Database connection lost.')
}
if(err.code === 'ER_CON_COUNT_ERROR') {
console.log('Database has too many connections.')
}
if(err.code === 'ECONNREFUSED') {
console.log('Database connection refused.')
}
if(err.code === 'POOL_CLOSED') {
console.log('Pool is closed.')
}
}
if(connection) {
connection.release()
}
return;
});
module.exports = pool;
I have changed the code to use async.series instead of async.queue, beacuse the tasks would run in parallel in queue (see: https://caolan.github.io/async/docs.html#queue).
test.js
...
let tasks = [];
context.forEach((ctx) => {
tasks.push(function(callback) { load(ctx, callback) });
});
async.series(tasks, function(err) {
if(err) return next(err);
});

Serverless with nodejs and mysql => Process exited before completing request

I'm developing some functions with serverless with the nodejs template. I have a service that connects to a mysql database and retrieves some data. Everything is fine when I make the first call, but when I repeat it I receive a "Process exited before completing request" error message.
If I try that same call again, I receive data. So the service is doing right on the odd calls and it's returning the error on the even calls (funny right?). This is the code of the handler function:
module.exports.getAll = (event, context, done) => {
deviceRepository.getAllDevices().then((response) => {
done(null, { response });
}).catch((error) => {
done(error);
});
};
and this is the code of the repository function:
const mysql = require('mysql');
const when = require('when');
const config = require('./config');
const conn = mysql.createConnection({
host: config.RDSHOST,
user: config.RDSUSER,
password: config.RDSPASS,
database: config.RDSDB,
port: config.RDSPORT
});
module.exports.getAllDevices = () => {
const deferred = when.defer();
conn.connect();
conn.query('SELECT * FROM device', (err, rows) => {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(rows);
}
conn.end();
});
return deferred.promise;
};
As you can see I use promises with the 'when' library. I call the 'done' callback in the handler, and there should be a response from the promise in every possible situation.
I can't see what is wrong with this and why is making the odd requests wrong. Anyone can help?
Thanks in advance!
Solved by myself...
The problem is that I was making the createConnection outside of the handler (when I declared the conn constant).
Moving the createConnection declaration inside the handler function works as expected in every call.
const mysql = require('mysql');
const when = require('when');
const config = require('./config');
module.exports.getAllDevices = () => {
const conn = mysql.createConnection({
host: config.RDSHOST,
user: config.RDSUSER,
password: config.RDSPASS,
database: config.RDSDB,
port: config.RDSPORT
});
const deferred = when.defer();
conn.connect();
conn.query('SELECT * FROM device', (err, rows) => {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(rows);
}
conn.end();
});
return deferred.promise;
};
Hope this helps. Thanks!

How to provide a mysql database connection in single file in nodejs

I need to provide the mysql connection for modules. I have a code like this.
var express = require('express'),
app = express(),
server = require('http').createServer(app);
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database : 'chat'
});
connection.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
});
app.get('/save', function(req,res){
var post = {from:'me', to:'you', msg:'hi'};
var query = connection.query('INSERT INTO messages SET ?', post, function(err, result) {
if (err) throw err;
});
});
server.listen(3000);
But how we provide one time mysql connection for all the modules.
You could create a db wrapper then require it. node's require returns the same instance of a module every time, so you can perform your connection and return a handler. From the Node.js docs:
every call to require('foo') will get exactly the same object returned, if it would resolve to the same file.
You could create db.js:
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database : 'chat'
});
connection.connect(function(err) {
if (err) throw err;
});
module.exports = connection;
Then in your app.js, you would simply require it.
var express = require('express');
var app = express();
var db = require('./db');
app.get('/save',function(req,res){
var post = {from:'me', to:'you', msg:'hi'};
db.query('INSERT INTO messages SET ?', post, function(err, result) {
if (err) throw err;
});
});
server.listen(3000);
This approach allows you to abstract any connection details, wrap anything else you want to expose and require db throughout your application while maintaining one connection to your db thanks to how node require works :)
I took a similar approach as Sean3z but instead I have the connection closed everytime i make a query.
His way works if it's only executed on the entry point of your app, but let's say you have controllers that you want to do a var db = require('./db'). You can't because otherwise everytime you access that controller you will be creating a new connection.
To avoid that, i think it's safer, in my opinion, to open and close the connection everytime.
here is a snippet of my code.
mysq_query.js
// Dependencies
var mysql = require('mysql'),
config = require("../config");
/*
* #sqlConnection
* Creates the connection, makes the query and close it to avoid concurrency conflicts.
*/
var sqlConnection = function sqlConnection(sql, values, next) {
// It means that the values hasnt been passed
if (arguments.length === 2) {
next = values;
values = null;
}
var connection = mysql.createConnection(config.db);
connection.connect(function(err) {
if (err !== null) {
console.log("[MYSQL] Error connecting to mysql:" + err+'\n');
}
});
connection.query(sql, values, function(err) {
connection.end(); // close the connection
if (err) {
throw err;
}
// Execute the callback
next.apply(this, arguments);
});
}
module.exports = sqlConnection;
Than you can use it anywhere just doing like
var mysql_query = require('path/to/your/mysql_query');
mysql_query('SELECT * from your_table where ?', {id: '1'}, function(err, rows) {
console.log(rows);
});
UPDATED:
config.json looks like
{
"db": {
"user" : "USERNAME",
"password" : "PASSWORD",
"database" : "DATABASE_NAME",
"socketPath": "/tmp/mysql.sock"
}
}
Hope this helps.
I think that you should use a connection pool instead of share a single connection. A connection pool would provide a much better performance, as you can check here.
As stated in the library documentation, it occurs because the MySQL protocol is sequential (this means that you need multiple connections to execute queries in parallel).
Connection Pool Docs
From the node.js documentation, "To have a module execute code multiple times, export a function, and call that function", you could use node.js module.export and have a single file to manage the db connections.You can find more at Node.js documentation. Let's say db.js file be like:
const mysql = require('mysql');
var connection;
module.exports = {
dbConnection: function () {
connection = mysql.createConnection({
host: "127.0.0.1",
user: "Your_user",
password: "Your_password",
database: 'Your_bd'
});
connection.connect();
return connection;
}
};
Then, the file where you are going to use the connection could be like useDb.js:
const dbConnection = require('./db');
var connection;
function callDb() {
try {
connection = dbConnectionManager.dbConnection();
connection.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (!error) {
let response = "The solution is: " + results[0].solution;
console.log(response);
} else {
console.log(error);
}
});
connection.end();
} catch (err) {
console.log(err);
}
}
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'yourip',
port : 'yourport',
user : 'dbusername',
password : 'dbpwd',
database : 'database schema name',
dateStrings: true,
multipleStatements: true
});
// TODO - if any pool issues need to try this link for connection management
// https://stackoverflow.com/questions/18496540/node-js-mysql-connection-pooling
module.exports = function(qry, qrytype, msg, callback) {
if(qrytype != 'S') {
console.log(qry);
}
pool.getConnection(function(err, connection) {
if(err) {
if(connection)
connection.release();
throw err;
}
// Use the connection
connection.query(qry, function (err, results, fields) {
connection.release();
if(err) {
callback('E#connection.query-Error occurred.#'+ err.sqlMessage);
return;
}
if(qrytype==='S') {
//for Select statement
// setTimeout(function() {
callback(results);
// }, 500);
} else if(qrytype==='N'){
let resarr = results[results.length-1];
let newid= '';
if(resarr.length)
newid = resarr[0]['#eid'];
callback(msg + newid);
} else if(qrytype==='U'){
//let ret = 'I#' + entity + ' updated#Updated rows count: ' + results[1].changedRows;
callback(msg);
} else if(qrytype==='D'){
//let resarr = results[1].affectedRows;
callback(msg);
}
});
connection.on('error', function (err) {
connection.release();
callback('E#connection.on-Error occurred.#'+ err.sqlMessage);
return;
});
});
}
try this
var express = require('express');
var mysql = require('mysql');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var users = require('./routes/users');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
// uncomment after placing your favicon in /public
//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
console.log(app);
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "admin123",
database: "sitepoint"
});
con.connect(function(err){
if(err){
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
module.exports = app;
you can create a global variable and then access that variable in other files.
here is my code, I have created a separate file for MySQL database connection called db.js
const mysql = require('mysql');
var conn = mysql.createConnection({
host: "localhost",
user: "root",
password: "xxxxx",
database: "test"
});
conn.connect((err) => {
if (err) throw err;
console.log('Connected to the MySql DB');
});
module.exports = conn;
Then in the app.js file
const express = require('express');
const router = express.Router();
// MySql Db connection and set in globally
global.db = require('../config/db');
Now you can use it in any other file
const express = require('express');
const router = express.Router();
router.post('/signin', (req, res) => {
try {
var param = req.body;
var sql = `select * from user`;
// db is global variable
db.query(sql, (err, data) => {
if (err) throw new SyntaxError(err);
res.status(200).json({ 'auth': true, 'data': data });
});
} catch (err) {
res.status(400).json({ 'auth': false, 'data': err.message });
}
});