Reproduce MySQL error: The server closed the connection (node.js) - mysql

Question

I'm trying to reproduce a MySQL error I'm seeing in my node.js app on EC2 with the node mysql library:
Connection lost: The server closed the connection.
I am unable to reproduce the error locally- killing the database is handled just fine by my code- it just rechecks every few seconds and reconnects to the db once it is restarted. On EC2, it happens around 4am Pacific, but the db is still up and running fine.
I'd like to
Reproduce the crash with my local mysql
Add whatever logic I need in my mysql helper module to handle this
Here's the error in my node.js app:
2012-10-22T08:45:40.518Z - error: uncaughtException date=Mon Oct 22
2012 08:45:40 GMT+0000 (UTC), pid=14184, uid=0, gid=0,
cwd=/home/ec2-user/my-app, execPath=/usr/bin/nodejs,
version=v0.6.18, argv=[/usr/local/bin/node,
/home/ec2-user/my-app/app.js, --my-app], rss=15310848,
heapTotal=6311392, heapUsed=5123292, loadavg=[0.0029296875,
0.0146484375, 0.04541015625], uptime=3238343.511107486, trace=[column=13,
file=/home/ec2-user/my-app/node_modules/mysql/lib/protocol/Protocol.js,
function=Protocol.end, line=63, method=end, native=false, column=10,
file=stream.js, function=Socket.onend, line=80, method=onend,
native=false, column=20, file=events.js, function=Socket.emit,
line=88, method=emit, native=false, column=51, file=net.js,
function=TCP.onread, line=388, method=onread, native=false],
stack=[Error: Connection lost: The server closed the connection.,
at Protocol.end
(/home/ec2-user/my-app/node_modules/mysql/lib/protocol/Protocol.js:63:13), at Socket.onend (stream.js:80:10), at Socket.emit
(events.js:88:20), at TCP.onread (net.js:388:51)]
Here's my code (mysql helper module):
module.exports = function (conf,logger) {
var mysql = require('mysql');
var connectionState = false;
var connection = mysql.createConnection({
host: conf.db.hostname,
user: conf.db.user,
password: conf.db.pass,
database: conf.db.schema,
insecureAuth: true
});
function attemptConnection(connection) {
if(!connectionState){
connection = mysql.createConnection(connection.config);
connection.connect(function (err) {
// connected! (unless `err` is set)
if (err) {
logger.error('mysql db unable to connect: ' + err);
connectionState = false;
} else {
logger.info('mysql connect!');
connectionState = true;
}
});
connection.on('close', function (err) {
logger.error('mysqldb conn close');
connectionState = false;
});
connection.on('error', function (err) {
logger.error('mysqldb error: ' + err);
connectionState = false;
/*
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
*/
});
}
}
attemptConnection(connection);
var dbConnChecker = setInterval(function(){
if(!connectionState){
logger.info('not connected, attempting reconnect');
attemptConnection(connection);
}
}, conf.db.checkInterval);
return connection;
};

Answer 1

Check out mysql pool feature in node-mysql
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
pool.getConnection(function(err, connection) {
// connected! (unless `err` is set)
connection.end();
});

Answer 2

I was having similar problems and created a getConnection() wrapper function that checks the health of the mysql connection before returning it to the caller and re-establishes the connection as necessary. In my testing it has handled fatal and non-fatal connection issues transparently for the application. If the connection simply timed out, the application recovers without experiencing any errors. If there is a transient but fatal database connection problem, the application will resume functioning automatically as soon as database connectivity is available again.
As far as reproducing the problem for testing, add the following two lines to the my.ini or my.cnf file under the [mysqld] block:
interactive_timeout=30
wait_timeout=30
Here is the contents of a file I have named "database.js":
var mysql = require("mysql");
var CONFIG = require(__dirname + "/configuration");
module.exports.getConnection = function() {
// Test connection health before returning it to caller.
if ((module.exports.connection) && (module.exports.connection._socket)
&& (module.exports.connection._socket.readable)
&& (module.exports.connection._socket.writable)) {
return module.exports.connection;
}
console.log(((module.exports.connection) ?
"UNHEALTHY SQL CONNECTION; RE" : "") + "CONNECTING TO SQL.");
var connection = mysql.createConnection({
host : CONFIG.db.host,
user : CONFIG.db.user,
password : CONFIG.db.password,
database : CONFIG.db.database,
port : CONFIG.db.port
});
connection.connect(function(err) {
if (err) {
console.log("SQL CONNECT ERROR: " + err);
} else {
console.log("SQL CONNECT SUCCESSFUL.");
}
});
connection.on("close", function (err) {
console.log("SQL CONNECTION CLOSED.");
});
connection.on("error", function (err) {
console.log("SQL CONNECTION ERROR: " + err);
});
module.exports.connection = connection;
return module.exports.connection;
}
// Open a connection automatically at app startup.
module.exports.getConnection();
// If you've saved this file as database.js, then get and use the
// connection as in the following example:
// var database = require(__dirname + "/database");
// var connection = database.getConnection();
// connection.query(query, function(err, results) { ....

Answer 3

Here's what I ended up using, and it worked pretty well. On the occasional connection lost/restart it recovered nicely. I have a database.js file which establishes connections and checks them periodically.
To make a request:
var conn = require('./database');
var sql = 'SELECT foo FROM bar;';
conn.query(sql, [userId, plugId], function (err, rows) {
// logic
}
Here's my databbase.js
var mysql = require('mysql');
var Common = require('./common');
var conf = Common.conf;
var logger = Common.logger;
var connectionState = false;
var connection = mysql.createConnection({
host: conf.db.hostname,
user: conf.db.user,
password: conf.db.pass,
database: conf.db.schema,
insecureAuth: true
});
connection.on('close', function (err) {
logger.error('mysqldb conn close');
connectionState = false;
});
connection.on('error', function (err) {
logger.error('mysqldb error: ' + err);
connectionState = false;
});
function attemptConnection(connection) {
if(!connectionState){
connection = mysql.createConnection(connection.config);
connection.connect(function (err) {
// connected! (unless `err` is set)
if (err) {
logger.error('mysql db unable to connect: ' + err);
connectionState = false;
} else {
logger.info('mysql connect!');
connectionState = true;
}
});
connection.on('close', function (err) {
logger.error('mysqldb conn close');
connectionState = false;
});
connection.on('error', function (err) {
logger.error('mysqldb error: ' + err);
if (!err.fatal) {
//throw err;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
//throw err;
} else {
connectionState = false;
}
});
}
}
attemptConnection(connection);
var dbConnChecker = setInterval(function(){
if(!connectionState){
logger.info('not connected, attempting reconnect');
attemptConnection(connection);
}
}, conf.db.checkInterval);
// Mysql query wrapper. Gives us timeout and db conn refreshal!
var queryTimeout = conf.db.queryTimeout;
var query = function(sql,params,callback){
if(connectionState) {
// 1. Set timeout
var timedOut = false;
var timeout = setTimeout(function () {
timedOut = true;
callback('MySQL timeout', null);
}, queryTimeout);
// 2. Make query
connection.query(sql, params, function (err, rows) {
clearTimeout(timeout);
if(!timedOut) callback(err,rows);
});
} else {
// 3. Fail if no mysql conn (obviously)
callback('MySQL not connected', null);
}
}
// And we present the same interface as the node-mysql library!
// NOTE: The escape may be a trickier for other libraries to emulate because it looks synchronous
exports.query = query;
exports.escape = connection.escape;

Answer 4

Using generic-pool, I wrote something that works locally. I guess I'll test it and see if it doesn't crash in bizarre manner on the server side.
// Test node connection pool stuff
// Create a MySQL connection pool with
// a max of 10 connections, a min of 2, and a 30 second max idle time
var poolModule = require('generic-pool');
var pool = poolModule.Pool({
name : 'mysql',
create : function(callback) {
var Client = require('mysql').Client; // use node-mysql library in all it's dubious glory
var c = new Client();
c.user = 'root';
c.password = 'xxx';
c.database = 'test';
c.on('close', function (err) {
console.log('mysqldb conn close');
});
c.on('error', function (err) {
console.log('mysqldb error: ' + err);
});
// parameter order: err, resource
// new in 1.0.6
callback(null, c);
},
destroy : function(client) { client.end(); },
max : 10,
// optional. if you set this, make sure to drain() (see step 3)
min : 2,
// specifies how long a resource can stay idle in pool before being removed
idleTimeoutMillis : 30000,
// if true, logs via console.log - can also be a function
log : true
});
var http = require('http');
http.createServer(function (req, res) {
// Get db conn
pool.acquire(function(err, client) {
if (err) {
// handle error - this is generally the err from your
// factory.create function
console.log('pool.acquire err: ' + err);
res.writeHead(500, {'Content-Type': 'application/json'});
out = {
err: err
}
res.end(JSON.stringify(out));
}
else {
client.query("select * from foo", [], function(err, results) {
if(err){
res.writeHead(500, {'Content-Type': 'application/json'});
out = {
err: err
}
res.end(JSON.stringify(out));
} else {
res.writeHead(500, {'Content-Type': 'application/json'});
out = {
results: results
}
res.end(JSON.stringify(out));
}
// return object back to pool
pool.release(client);
});
}
});
}).listen(9615);
Pretty please don't die at 4am for no apparent reason!

Answer 5

The solution is use pooling connection !
You can wrote code to handle connection manually, it works.
However pooling is design for this, use pooling connection solved connection drop error.
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 10,
host : 'example.org',
user : 'bob',
password : 'secret',
database : 'my_db'
});
pool.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
pooling mysql connection