I'm using MySQL on Nodejs. I'm using mysql pool to create the connection:
Var mysqk = require ('mysql');
Var pool = mysql.createPool ({my configs});
My question is:
Where in the app will I use pool.end() as reported in the documentation?
For example: in my www file, I created a code to release other things as I have pointed in de code. Should I use pool.end() there?
var app = require('../app');
var debug = require('debug')('cancela:server');
var http = require('http');
var port = normalizePort(process.env.PORT || '4000');
app.set('port', port);
var server = http.createServer(app);
// my app codes...
..
.
.
process.once('SIGTERM', end);
process.once('SIGINT', end);
function end() {
server.close(function(err){
if(err) throw err();
// Should I end the Pool connection here? <<<<<<===================================
// pool.end(function (err) {
// // all connections in the pool have ended
// });
console.log('Server endded!');
/* exit gracefully */
process.exit();
});
}
Sometimes you want to manually release the connection pool for whatever reasons so you can use pool.end() before eventually recreating the pool.
Most of times you want to release it when your server is about to shutdown so you can do what you are thinking. Pay attention to the location of the function call process.exit(). It must be called after the connection release, otherwise the pool won't finish to release.
function end() {
server.close(function (err) {
if (err) throw err;
console.log('Server endded!');
pool.end(function (err) {
if (err) throw err;
process.exit();
});
});
}
Related
It seems that MySQL pool isn't releasing connection from my NodeJS app. When looking at the server processes from mysql, connections are established but not released.
The app is crashing with the following error: TypeError: Cannot read property 'query' of undefined. From my understanding, this is because i've exceeded connections limit.
To note, I'm using Heroku MySQL plugin and have a limit of 10 connections for testing. I've added connectionLimit to end of my .env DB_URL to try to limit my connections in the pool.
In my .env file, the link is formatted as such:
DEV_DB_URL=mysql://user:pass#host:3306/db_name?connectionLimit=5
The db.js file:
var mysql = require('mysql');
var dbURL = process.env.PRODUCTION === 'true'
? process.env.LIVE_DB_URL
: process.env.DEV_DB_URL;
var dbConnection = function dbConnection( sql, values, next){
// It means that the values hasn't been passed
if (arguments.length === 2){
next = values;
values = null;
}
var pool = mysql.createPool(dbURL);
pool.getConnection(function(err, connection){
connection.query(sql, values, function(err) {
connection.release();
if (err) {
throw err;
}
// Execute the callback
next.apply(this, arguments);
});
});
};
module.exports = dbConnection;
My controller that uses the db.js file looks like this:
var DB = require('../services/db');
exports.updateReporting = function(req, res){
var body = req.body;
var sentStatus = body.edc;
if( sentStatus === 'pass'){
res.status(200)
.send({
message: 'EDC is all good.'
})
}
if( sentStatus === 'fail'){
var dateTime = require('node-datetime');
var dt = dateTime.create();
var curTimestamp = dt.format('Y-m-d H:M:S');
var storeId = body.store_id
.substring('Addr1='.length);
var fileTimestamp = body.file_ts;
var data = {
status: sentStatus,
store_id: storeId,
file_ts: fileTimestamp,
current_ts: curTimestamp
};
DB('INSERT INTO edc_reporting SET ?', data, function( err, row ){
if (err) throw err;
});
res.status(200)
.send({
message: 'Message recorded.'
})
}
};
As i'm new to Node, i'm not sure if my connection.release() is in the wrong place and not executed because of call back?
Any pointers would be appreciated.
This was really silly mistake.
In the db.js file, I had var pool = mysql.createPool(dbURL); inside the exported function; this is wrong.
Every time there was a request, the connection would be re-created and a new pool started. To solve the issue, I moved the creation of the pool outside the dbConnection function, hence, only 1 pool was created and connecting was started.
Despite declaring the pool outside the export function and releasing connection with connection.release() it didnot work for me. Using pool.query instead of connection.query worked perfectly fine it releases connection automatically everytime.
I use node js with mysql and want to avoid that the app crash on connection errors.At the moment i use this :
function mysql_handleDisconnect() {
mysql_connection = mysql.createConnection(mysql_config_obj); // Recreate the connection, since
// the old one cannot be reused.
mysql_connection.connect(function(err) { // The server is either down
if(err) { // or restarting (takes a while sometimes).
console.log('error when connecting to db:', err);
mysql_handleDisconnect(); // We introduce a delay before attempting to reconnect,
} // to avoid a hot loop, and to allow our node script to
}); // process asynchronous requests in the meantime.
// If you're also serving http, display a 503 error.
mysql_connection.on('error', function(err) {
console.log('db error', err);
if(err.code === 'PROTOCOL_CONNECTION_LOST') { // Connection to the MySQL server is usually
mysql_handleDisconnect(); // lost due to either server restart, or a
} else { // connnection idle timeout (the wait_timeout
throw err; // server variable configures this)
}
});
}
mysql_handleDisconnect(mysql_connection);
so this is blocking because it leads to a hot loop if the connection is closed.my problem is, if i add a setTimeout to reestablish connection just every 2 seconds i could get an fatal error when i do a query with "mysql_connection.query('SELECT ...')".in this case the app crashes.
So my question is,if there's a possibility to check the connection before i do a query?
Try using below code in every microservice before doing anything:
if(connection.state === 'disconnected'){
return respond(null, { status: 'fail', message: 'server down'});
}
State of connection to DB could fall in 2 states:
disconnected (when due to DB server down or wrong config use for DB connection is wrong)
authenticated (when DB connection is successfully created to DB server).
So either check state == 'disconnected' or state == 'authenticated'
I know this is an old question but I have found connection.ping( (err) => {...}) to be very useful for health-checks made from load balancers and whatnot.
Every time, while I'm pushing my code in production, the mysql connection is lost. It is a very common problem in production, or local.
My solution is that At every query established the db connection and remove connection after completing the db query.
My solution is to establish the db connection before every query, and then remove the connection after completing the db query.
Step1: Here is the code for dbConnection.js
//this code is for conenct to db
const mysql = require('mysql2');
require('dotenv').config();
module.exports.stablishedConnection = ()=>{
return new Promise((resolve,reject)=>{
const con = mysql.createConnection( {
host: process.env.DB_HOST||localhost,
user: process.env.DB_USER_NAME||myUserName ,
password: process.env.DB_PASSWORD||mypassword,
database: process.env.DB_NAME||mydb
});
con.connect((err) => {
if(err){
reject(err);
}
resolve(con);
});
})
}
module.exports.closeDbConnection =(con)=> {
con.destroy();
}
Step2: For Router.js I am import the db connection and handle the promise
const router = require('express').Router();
const {stablishedConnection,closeDbConnection} =require('../db/dbConnection');
router.get('/user/:sId/:userId',function(req,res){
stablishedConnection()
.then((db)=>{
console.log("Db connection stablished");
db.query(`select * from user WHERE sent_id=${req.params.sId} AND user_id=${req.params.userId}`, null, function (err,data) {
if (!data) {
res.status(200).json({sucess:false,err});
}else{
res.status(200).json({sucess:true,data});
closeDbConnection(db);
console.log("Db Connection close Successfully");
}
})
}).catch((error)=>{
console.log("Db not connected successfully",error);
});
});
router.get('/sen/:userId',function(req,res){
stablishedConnection()
.then((db)=>{
console.log("Db connection stablished");
db.query(`select * from sen WHERE user_id=${req.params.userId}`, null, function (err,data) {
if (!data) {
res.status(200).json({sucess:false,err});
}else{
res.status(200).json({sucess:true,data});
closeDbConnection(db);
console.log("Db Connection close Successfully");
}
})
}).catch((error)=>{
console.log("Db not connected successfully",error);
});
});
router.get('/language',(req,res)=>{
stablishedConnection()
.then((db)=>{
console.log("Db connection stablished");
db.query("select * from language", null, function (err,data) {
if (!data) {
res.status(200).json({sucess:false,err});
}else{
res.status(200).json({sucess:true,data});
closeDbConnection(db);
console.log("Db Connection close Successfully")
}
})
}).catch((error)=>{
console.log("Db not connected successfully",error);
});
})
module.exports = router;
This is perfectly run If you want to create and close connection at every query ..
I solved this problem like this:
let connection = mysql.createConnection(DB_CONFIG);
function runDBQuery() {
const disconnected = await new Promise(resolve => {
connection.ping(err => {
resolve(err);
});
});
if (disconnected) {
connection = mysql.createConnection(DB_CONFIG);
}
... use actual connection
}
I've got this script for reading a file and then insert the data into mysql tables. The script works, but it hangs, so I have to press CTRL-C to stop the script.
But the script should stop normally, what do I have to change?
var fs = require('fs');
var filename;
var myGID;
filename = "data/insertUser1_next.json";
function get_line(filename, line_no, callback) {
fs.readFile(filename, function (err, data) {
if (err) throw err;
// Data is a buffer that we need to convert to a string
// Improvement: loop over the buffer and stop when the line is reached
var lines = data.toString('utf-8').split("\n");
if(+line_no > lines.length){
return callback('File end reached without finding line', null);
}
// lines
callback(null, lines[0], lines[1], lines[2], lines[3]);
});
}
get_line(filename, 0, function(err, line, line2, line3, line4){
line = line.replace(/(\r\n|\n|\r)/gm,"");
line2 = line2.replace(/(\r\n|\n|\r)/gm,"");
line3 = line3.replace(/(\r\n|\n|\r)/gm,"");
/*line4 = line4.replace(/(\r\n|\n|\r)/gm,"");*/
console.log('The line: ' + line);
console.log('The line2: ' + line2);
console.log('The line3: ' + line3);
console.log('The line4: ' + line4);
var post = {gid: line, uid: line2};
var post2 = {uid: line2, displayname: line3, password: line4};
var mysql = require('mysql');
var db_config = {
host : '123.456.789.012',
user : 'user',
password : 'password',
database : 'maindata'
};
var con = mysql.createPool(db_config);
con.getConnection(function(err){
if (err) {
console.log(err);
return;
}
con.query('INSERT INTO group_user SET ?', post, function(err, result) {
if (err) {
console.log(err);
return;
}
});
con.query('INSERT INTO users SET ?', post2, function(err, result) {
if (err) {
console.log(err);
return;
}
});
});
});
Here you can see what happened:
When you are done using the pool, you have to end all the connections or the Node.js event loop will stay active until the connections are closed by the MySQL server. This is typically done if the pool is used in a script or when trying to gracefully shutdown a server. To end all the connections in the pool, use the end method on the pool:
pool.end(function (err) {
// all connections in the pool have ended
});
So, if you place con.end() after your queries are done, the script will terminate normally
The following statement will close the connection ensuring that all the queries in the queue are processed. Please note that this is having a callback function.
connection.end(function(err){
// Do something after the connection is gracefully terminated.
});
The following statement will terminate the assigned socket and close the connection immediately. Also there is no more callbacks or events triggered for the connection.
connection.destroy();
hey I suggest to install forever and start node servers.js with forever you dont need any terminal open.
And you need to close you mysql connection at the end to stop you hangs problem, i think.
npm install -g forever
npm install forever
//FOR your Problem
con.end(function(err){
// Do something after the connection is gracefully terminated.
});
con.destroy();
The following statement will close the connection ensuring that all the queries in the queue are processed. Please note that this is having a callback function.
connection.end(function(err){
// Do something after the connection is gracefully terminated.
});
The following statement will terminate the assigned socket and close the connection immediately. Also there is no more callbacks or events triggered for the connection.
connection.destroy();
I need a persistent MySQL connection for my Node web app. The problem is that this happens about a few times a day:
Error: Connection lost: The server closed the connection.
at Protocol.end (/var/www/n/node_modules/mysql/lib/protocol/Protocol.js:73:13)
at Socket.onend (stream.js:79:10)
at Socket.EventEmitter.emit (events.js:117:20)
at _stream_readable.js:895:16
at process._tickCallback (node.js:415:13)
error: Forever detected script exited with code: 8
error: Forever restarting script for 2 time
info: socket.io started
Here is my connection code:
// Yes I know multipleStatements can be dangerous in the wrong hands.
var sql = mysql.createConnection({
host: 'localhost',
user: 'my_username',
password: 'my_password',
database: 'my_database',
multipleStatements: true
});
sql.connect();
function handleDisconnect(connection) {
connection.on('error', function(err) {
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
console.log('Re-connecting lost connection: ' + err.stack);
sql = mysql.createConnection(connection.config);
handleDisconnect(sql);
sql.connect();
});
}
handleDisconnect(sql);
As you can see, the handleDisconnect code does not work..
Use the mysql connection pool. It will reconnect when a connection dies and you get the added benefit of being able to make multiple sql queries at the same time. If you don't use the database pool, your app will block database requests while waiting for currently running database requests to finish.
I usually define a database module where I keep my queries separate from my routes. It looks something like this...
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
exports.getUsers = function(callback) {
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
var sql = "SELECT id,name FROM users";
connection.query(sql, [], function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err) {
console.log(err);
callback(true);
return;
}
callback(false, results);
});
});
});
I know this is super delayed, but I've written a solution to this that I think might be a bit more generic and usable. I had written an app entirely dependent on connection.query() and switching to a pool broke those calls.
Here's my solution:
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'user',
password : 'secret',
database : 'test',
port : 3306
});
module.exports = {
query: function(){
var sql_args = [];
var args = [];
for(var i=0; i<arguments.length; i++){
args.push(arguments[i]);
}
var callback = args[args.length-1]; //last arg is callback
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
return callback(err);
}
if(args.length > 2){
sql_args = args[1];
}
connection.query(args[0], sql_args, function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err){
console.log(err);
return callback(err);
}
callback(null, results);
});
});
}
};
This instantiates the pool once, then exports a method named query. Now, when connection.query() is called anywhere, it calls this method, which first grabs a connection from the pool, then passes the arguments to the connection. It has the added effect of grabbing the callback first, so it can callback any errors in grabbing a connection from the pool.
To use this, simply require it as module in place of mysql. Example:
var connection = require('../middleware/db');
function get_active_sessions(){
connection.query('Select * from `sessions` where `Active`=1 and Expires>?;', [~~(new Date()/1000)], function(err, results){
if(err){
console.log(err);
}
else{
console.log(results);
}
});
}
This looks just like the normal query, but actually opens a pool and grabs a connection from the pool in the background.
In response to #gladsocc question:
Is there a way to use pools without refactoring everything? I have
dozens of SQL queries in the app.
This is what I ended up building. It's a wrapper for the query function. It will grab the connection, do the query, then release the connection.
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
And I use it like I would normally.
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});
I am writing a nodejs application and want to use connection pooling.
However, the following application does not terminate - although I would expect it to terminate after the call to connection.end()
Application works just fine, if I use one connection instead of the pool. Do I need to terminate the pool in some way?
Library used: https://github.com/felixge/node-mysql
node.js version: 0.10.4 on Ubuntu
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'myuser',
password : 'youbet',
database : 'notrevealingdetails',
insecureAuth: true
});
function getCampaignData(callback)
{
pool.getConnection(function(err, connection) {
if(err) throw err;
connection.query(
'SELECT cam.id, cam.name AS campaign_name, cam.subdomain, usr.email, usr.display_name AS user_displayname ' +
'FROM campaigns AS cam INNER JOIN users AS usr ON usr.id = cam.user_id ' +
'WHERE cam.state=2',
function(err, rows) {
callback(err, rows,connection);
//console.log('called end()');
}); // callback function for connection.query
}); // end pool.GetConnection
}
getCampaignData(function(err, rows, connection) {
if (err) throw err;
connection.end();
console.log("I expect my app to terminate");
});
I was having the very same problem, but looking at the source code
https://github.com/felixge/node-mysql/blob/master/lib/Pool.js
I found that the pool, at least in its current implementation, has an end() method that is turns call end() on all connections.
It also accept a callback function to be called after all connections are actually ended (or whenever an error occur).
pool.end(function (err) {
if (err) console.error("An error occurred: " + err);
else console.log("My app terminated");
});
I would use
getCampaignData(function(err, rows, connection)
{
if (err) throw err;
connection.release();
console.log("I expect my app to terminate");
});