I have a node/express/mysql2 web app that accesses a mySql DB through a connection pool object and I often run into the following issue: I leave the code for a while then when i come back and access pages that run queries I'll get
Error in foo: Error: connect ETIMEDOUT
Error in bar: Error: read ECONNRESET
I guess that on the other side mysql sees idle connections and close them, the client app doesn't know that, get those connections from the pool and then run into those issues, fine. But I was under the impression that this was automatically handled by mysql2 ?
This is roughly how i organised the db code
sqlConnectionPool.js
const dbParam = require('./dbParam.js');
const sqlPool = require('mysql2/promise').createPool(dbParam.connection.prod);
module.exports = sqlPool;
dummyQuery.js
const sqlPool = require('./sqlConnectionPool.js');
module.exports.updatefoo = async (ID, sqlConnection = undefined) => {
let connection;
try {
connection = sqlConnection === undefined ? await sqlPool.getConnection() : await sqlConnection;
const [updateResult] = await connection.query('update foo set barID=?', [ID]);
if (updateResult.affectedRows !== 1) {
throw (new Error(`error on ID ${ID}`));
}
return undefined;
} catch (err) {
console.error(`Error in updatefoo: ${err}`);
return err;
} finally {
if (sqlConnection === undefined) {
connection.release();
}
}
};
Is there something I'm missing to have those errors automatically handled, or simply not run into them ? I guess the mysql2 library needs to close the connection when they get connreset or conntimeout error and return them to the pool...
Thanks !
I think you should use something like setTimeout() to jab good the DB.
Related
I'm fairly new to how database connections work using nodejs, and I'm having issues with database connections that aren't being closed properly. I've asked a few questions on here before about it, and it seems like everyone is telling me to use pool instead of the way I have been doing it. The only problem is that when I search online about using pool from promise-mysql, everyone seems to use a very simple and generic approach, but I'm using it within a complex application using sockets. So I'm wondering how I can switch my old approach using createConnection() to using pool instead, in hopes of clearing up these connection issues.
Each time I call a socket it makes a connection to the database and then releases it after it is complete, or so it seems. It sounds like this is not a very scalable approach, and that using pool will help run multiple queries in parallel.
db.js:
import mysql from 'promise-mysql';
import env from '../../../env.config.json';
const db = async (sql, descriptor, serializedParameters = []) => {
return new Promise( async (resolve, reject) => {
try {
const connection = await mysql.createConnection({
host: env.DB.HOST,
user: env.DB.USER,
password: env.DB.PASSWORD,
database: env.DB.NAME,
port: env.DB.PORT
})
if (connection && env.ENV === "development") {
//console.log(/*"There is a connection to the db for: ", descriptor*/);
}
let result;
if(serializedParameters.length > 0) {
result = await connection.query(sql, serializedParameters)
} else result = await connection.query(sql);
connection.end();
resolve(result);
} catch (e) {
console.log("ERROR pool.db: " + e);
reject(e);
};
});
}
export default db;
This is an example of how I would create a connection to query the db
inventory.js:
import db from '../API/db';
export const selectAllFromBuildItems = () => {
return new Promise( async (resolve, reject) => {
try {
const getAllBuildItems = "SELECT * FROM mydb.build_items;"
const response = await db(getAllBuildItems, "AllBuildItems");
resolve(response);
} catch (e) {
console.log("ERROR inventory.selectAllFromBuildItems: " + e);
reject(e);
}
});
};
How can I change my code so that I use a pool instead. I have a lot of different queries that can be called from our application so I'm not quite sure what the right approach for this would be. I saw some people say that I should create the pool once and then use it throughout the application, but I don't know where that would go. If anyone has any suggestions on how I can make this switch, that would help me out a lot. Thanks!
Create the pool. Better if you create once when you run your application.
If it is in different file then you have export here and import in required file.
var pool = mysql.createPool({
host: env.DB.HOST,
user: env.DB.USER,
password: env.DB.PASSWORD,
database: env.DB.NAME,
connectionLimit: 10
});
I had to create this prototype function as the library had a bug of close connection was not returning the connection to the pool.
pool.prototype.releaseConnection = function releaseConnection(connection) {
return this.pool.releaseConnection(connection.connection);
};
Funtion for getting connection from the pool that is created earlier.
If you want you can call pool.getConnection() in all your query functions.
function connect() {
return pool.getConnection().then(function(connection) {
return connection
}).catch(function(e) {
console.log("Error Creating Connection");
throw e;
});
}
Now this is your query function to get data from dd.
function selectAllFromBuildItems() {
var sql_query = `SELECT * FROM mydb.build_items`;
return connect().then(function(conn) {
return conn.query(sql_query).then(function(rows) {
pool.releaseConnection(conn);
return rows;
});
}).catch(function(e) {
console.log("ERROR inventory.selectAllFromBuildItems: " + e);
throw e;
});
}
Update: Descriptions are added. Hope this helps you.
running into some issues trying to figure out an Azure Function (node.js-based) can connect to our mysql database (also hosted on Azure). We're using mysql2 and following tutorials pretty much exactly (https://learn.microsoft.com/en-us/azure/mysql/connect-nodejs, and similar) Here's the meat of the call:
const mysql = require('mysql2');
const fs = require('fs');
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
if (req.query.fname || (req.body && req.body.fname)) {
context.log('start');
var config = {
host:process.env['mysql_host'],
user: process.env['mysql_user'],
password: process.env['mysql_password'],
port:3306,
database:'database_name',
ssl:{
ca : fs.readFileSync(__dirname + '\\certs\\cacert.pem')
},
connectTimeout:5000
};
const conn = mysql.createConnection(config);
/*context.log(conn);*/
conn.connect(function (err) {
context.log('here');
if (err) {
context.error('error connecting: ' + err.stack);
context.log("shit is broke");
throw err;
}
console.log("Connection established.");
});
context.log('mid');
conn.query('SELECT 1+1',function(error,results,fields) {
context.log('here');
context.log(error);
context.log(results);
context.log(fields);
});
Basically, running into an issue where the conn.connect(function(err)... doesn't return anything - no error message, no logs, etc. conn.query works similarly.
Everything seems set up properly, but I don't even know where to look next to resolve the issue. Has anyone come across this before or have advice on how to handle?
Thanks!!
Ben
I believe the link that Baskar shared covers debugging your function locally
As for your function, you can make some changes to improve performance.
Create the connection to the DB outside the function code otherwise it will create a new instance and connect every time. Also, you can enable pooling to reuse connections and not cross the 300 limit that the sandbox in which Azure Functions run has.
Use the Promises along with async/await
You basically can update your code to something like this
const mysql = require('mysql2/promise');
const fs = require('fs');
var config = {
host: process.env['mysql_host'],
user: process.env['mysql_user'],
password: process.env['mysql_password'],
port: 3306,
database: 'database_name',
ssl: {
ca: fs.readFileSync(__dirname + '\\certs\\cacert.pem')
},
connectTimeout: 5000,
connectionLimit: 250,
queueLimit: 0
};
const pool = mysql.createPool(config);
module.exports = async function(context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
if (req.query.fname || (req.body && req.body.fname)) {
context.log('start');
const conn = await pool.getConnection();
context.log('mid');
await conn.query('SELECT 1+1', function(error, results, fields) {
context.log('here');
context.log(error);
context.log(results);
context.log(fields);
});
conn.release();
}
};
PS: I haven't test this code as such but I believe something like this should work
Debugging on serverless is challenging for obvious reasons. You can try one of the hacky solutions to debug locally (like Serverless Framework), but that won't necessarily help you if your issue is to do with a connection to a DB. You might see different behaviour locally.
Another option is to see if you can step debug using Rookout, which should let you catch the full stack at different points in the code execution and give you a good sense of what's failing and why.
It seems that MySQL pool isn't releasing connection from my NodeJS app. When looking at the server processes from mysql, connections are established but not released.
The app is crashing with the following error: TypeError: Cannot read property 'query' of undefined. From my understanding, this is because i've exceeded connections limit.
To note, I'm using Heroku MySQL plugin and have a limit of 10 connections for testing. I've added connectionLimit to end of my .env DB_URL to try to limit my connections in the pool.
In my .env file, the link is formatted as such:
DEV_DB_URL=mysql://user:pass#host:3306/db_name?connectionLimit=5
The db.js file:
var mysql = require('mysql');
var dbURL = process.env.PRODUCTION === 'true'
? process.env.LIVE_DB_URL
: process.env.DEV_DB_URL;
var dbConnection = function dbConnection( sql, values, next){
// It means that the values hasn't been passed
if (arguments.length === 2){
next = values;
values = null;
}
var pool = mysql.createPool(dbURL);
pool.getConnection(function(err, connection){
connection.query(sql, values, function(err) {
connection.release();
if (err) {
throw err;
}
// Execute the callback
next.apply(this, arguments);
});
});
};
module.exports = dbConnection;
My controller that uses the db.js file looks like this:
var DB = require('../services/db');
exports.updateReporting = function(req, res){
var body = req.body;
var sentStatus = body.edc;
if( sentStatus === 'pass'){
res.status(200)
.send({
message: 'EDC is all good.'
})
}
if( sentStatus === 'fail'){
var dateTime = require('node-datetime');
var dt = dateTime.create();
var curTimestamp = dt.format('Y-m-d H:M:S');
var storeId = body.store_id
.substring('Addr1='.length);
var fileTimestamp = body.file_ts;
var data = {
status: sentStatus,
store_id: storeId,
file_ts: fileTimestamp,
current_ts: curTimestamp
};
DB('INSERT INTO edc_reporting SET ?', data, function( err, row ){
if (err) throw err;
});
res.status(200)
.send({
message: 'Message recorded.'
})
}
};
As i'm new to Node, i'm not sure if my connection.release() is in the wrong place and not executed because of call back?
Any pointers would be appreciated.
This was really silly mistake.
In the db.js file, I had var pool = mysql.createPool(dbURL); inside the exported function; this is wrong.
Every time there was a request, the connection would be re-created and a new pool started. To solve the issue, I moved the creation of the pool outside the dbConnection function, hence, only 1 pool was created and connecting was started.
Despite declaring the pool outside the export function and releasing connection with connection.release() it didnot work for me. Using pool.query instead of connection.query worked perfectly fine it releases connection automatically everytime.
I am very new to Node.js development and I am working on an app that requires me to pull users from a mysql database. I am using the promise-mysql library to query a mysql database. I am trying to use a connection pool like this:
var pool = mysql.createPool({
host: hgh.host,
user: hgh.user,
password: hgh.pw,
database: hgh.name,
connectionLimit: 10
});
As a global variable in my module.
I then have the above function to return a connection from the pool.
function connect() {
return pool.getConnection().then(function(connection) {
return connection
}).catch(function(error) {
console.log("Connect failed");
throw ErrorModel.generateErrorObject(error, 500);
});
}
Below is a function I am using to query the database with:
function getUser(username) {
var sql_query = `SELECT * FROM userstable WHERE userName = ` + `'` + username + `'`;
return connect().then(function(conn) {
return conn.query(sql_query).then(function(rows) {
console.log("HGH getUser Then");
console.log(pool);
conn.release();
return rows;
});
}).catch(function(error) {
console.log("HGH getUser Catch");
console.log(error);
throw ErrorModel.generateErrorObject(error, 500);
});
}
I am getting this issue:
conn.release is not a function when trying to release my current connection into the pool. Is my logic wrong here? My goal is to have a pool with a bunch of connections (up to a certain number) and if a user needs to query, the getConnection() function just grabs them either a free connection from the pool, or creates them one. Problem is I cannot for the life of me release it back to the pool as free..Every time I make a request with conn.end() instead of release the connection remains in the _allConnections array of the pool when I console it out, and there are absolutely no connections in the _freeConnections array.
Anyone know how I can make connections free again??
Looking at the module's code I found the function for releasing a connection from a pool:
pool.prototype.releaseConnection = function releaseConnection(connection) {
//Use the underlying connection from the mysql-module here:
return this.pool.releaseConnection(connection.connection);
};
So if all of these functions live in the same file you could do the following in the getUser function:
replace
conn.release();
with
pool.releaseConnection(conn);
Looking at the code, promise-mysql wraps the original connection object, which doesn't expose the release method. However, the original is exposed as a property called connection, so this works:
conn.connection.release();
A few random thoughts:
You should probably escape your query input:
var sql_query = `SELECT * FROM userstable WHERE userName = ${ pool.escape(username) }`;
Your code doesn't release connections when an error occurs (because the code in the .then() callback wouldn't get called); it's better to use .finally() to do the releasing, as that will get called for both resolved and rejected cases:
function connect() {
var conn = null;
return pool.getConnection().then(function(connection) {
conn = connection;
return connection;
}).catch(function(error) {
console.log("Connect failed", error);
}).finally(function() {
if (conn) {
conn.connection.release();
}
});
}
G'day all,
I'm trying to convert some old php code over to Node, and part of the journey has been trying to figure out the best way to perform sql queries against my database (I'm using SQL so I can port the existing database over).
I've got them working, but have encountered the "Pyramid of Doom" problem, and it's subsequent scope issues (i.e. the returned values not baing available to subsequent "then"s).
An example of the sort of code I have here is: (dbPool.queryOPromise returns a query wrapped in a promise)
dbPool.queryOPromise(query)
.then(function(result){
console.log(result);
var query = {
sql:"INSERT INTO newusers (newuserid, ipaddress, email) VALUES (?,?,?)",
values: [newuserid, ipAddress, email]
};
dbPool.queryOPromise(query)
.then(function(value){
console.log(value);
if(value.code==200) {
res.status(200).json({code:200, status:"New User Created"});
} else {
res.status(400).json({code:value.code, status:"Error creating new user: ".value.status});
}
})
})
Does anyone have a view on the best way to attack this situation?
Thanks!
You're supposed to return the subsequent promises instead of calling .then on them
dbPool.queryOPromise(query)
.then(function(result) {
console.log(result);
var query = {
sql: "INSERT INTO newusers (newuserid, ipaddress, email) VALUES (?,?,?)",
values: [newuserid, ipAddress, email]
};
// RETURN the second promise,
return dbPool.queryOPromise(query);
})
.then(function(value) {
console.log(value);
if (value.code == 200) {
res.status(200).json({code: 200, status: "New User Created"});
} else {
res.status(400).json({code: value.code, status: "Error creating new user: ".value.status });
}
})
.catch(console.error); // and always catch the errors at the end.
It's a #1 rookie mistake in using promises. Checkout this wonderfully written article addressing issues exactly like this
For node -v > 8.x only,
Share my working example:
I use this Promisified MySQL middleware for Node.js
read this article Create a MySQL Database Middleware with Node.js 8 and Async/Await
here is my database.js
var mysql = require('mysql');
// node -v must > 8.x
var util = require('util');
// !!!!! for node version < 8.x only !!!!!
// npm install util.promisify
//require('util.promisify').shim();
// -v < 8.x has problem with async await so upgrade -v to v9.6.1 for this to work.
// connection pool https://github.com/mysqljs/mysql [1]
var pool = mysql.createPool({
connectionLimit : process.env.mysql_connection_pool_Limit, // default:10
host : process.env.mysql_host,
user : process.env.mysql_user,
password : process.env.mysql_password,
database : process.env.mysql_database
})
// Ping database to check for common exception errors.
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
// Promisify for Node.js async/await.
pool.query = util.promisify(pool.query)
module.exports = pool
You must upgrade node -v > 8.x
you must use async function to be able to use await.
example:
var pool = require('./database')
// node -v must > 8.x, --> async / await
router.get('/:template', async function(req, res, next)
{
...
try {
var _sql_rest_url = 'SELECT * FROM arcgis_viewer.rest_url WHERE id='+ _url_id;
var rows = await pool.query(_sql_rest_url)
_url = rows[0].rest_url // first record, property name is 'rest_url'
if (_center_lat == null) {_center_lat = rows[0].center_lat }
if (_center_long == null) {_center_long= rows[0].center_long }
if (_center_zoom == null) {_center_zoom= rows[0].center_zoom }
_place = rows[0].place
} catch(err) {
throw new Error(err)
}