Code running in wrong order after added connection pool (async/await needed?) - mysql

I have a NodeJS API which was using 'normal connections' to a MYSQL DB and running well execpt I was having disconnect issues.
I have implemented pooling now and that is working however it seems some code is now running out of order due to the new nature of the connections.
I've never used async/await as I'm very new to coding but I've tried to do it here to get the response I need.
Pretty much I have put in console logs to see the order it is running and instead of gettting "1,2,3", I'm getting "2,3,1" which obviously is going to give me the wrong results as I need to get query data before proceeding.
Can someone show how to get this await system working?
The first section gets the MAC and ID from the user and then checks if this exists in my DB already. If it does then it updates the dateTime of recent login. If it Doesnt then it adds the MAC.
The issue is that I'm not getting my response regarding if the mac exists, so it always is "false" therefore the mac keeps getting added over and over as the it isnt waiting for the very first query result!
router.post('/updateComp/',verify,async (req,res) => {
//console.log(req.params.MAC)
var sqlString = "SET #chemistID = ?; SET #MAC = ?; Call checkMAC(#chemistID,#MAC)";
try{
const MAC = req.body.MAC;;
const compName = req.body.compName;
var compCount = 0;
var MACExists = false;
console.log(MAC + " " + compName);
await connection.query(sqlString,[req.user._id,MAC], (err,rows,fields)=>{
console.log("Check 1");
if(!err){
rows.forEach(element => {
if(element.constructor == Array){
compCount = element[0].compCount;
MACExists = element[0].MACExists;
console.log(compCount);
console.log(MACExists);
}
else{
//array not returned?
return res.status(500);
}
})
}else{
//sql con error?
return res.status(500);
}
console.log("comcount = " + compCount);
})
console.log("Check 2");
if(compCount == 0 || (compCount < 7 && MACExists == false)){
//Insert new comp
var sqlString = "INSERT INTO tblLicense (chemistID,compName,MAC,lastAccess) VALUES (?,?,?,current_timestamp());";
console.log("Check 3");
connection.query(sqlString,[req.user._id,compName,MAC], (err,rows,fields)=>{
if(!err){
console.log("New terminal added for " + req.user._id);
return res.status(200).json({
Result: true,
compAdded: true
})
}else{
console.log("Failed to add new computer to sub " + req.user._id);
return res.status(500).json({
Result: false,
compAdded: false,
Comment: "Failed to add new computer to sub"
})
}
})
}else{
if (compCount == 7){
if(MACExists){
return res.status(200).json({
Result: true
})
}else{
return res.status(200).json({
Result: false,
compAdded: false,
Comment: compCount
})
}
}else{
//Update time of current comp access
var sqlString = "UPDATE tblLicense SET lastAccess = current_timestamp() WHERE MAC = ? AND chemistID = ?;";
connection.query(sqlString,[MAC,req.user._id], (err,rows,fields)=>{
if(!err){
return res.status(200).json({
Result: true,
compAdded: false
})
}
else
{
return res.status(500).json({
Result: false,
compAdded: false
})
}
})
}
}
} catch (e) {
// this catches any exception in this scope or await rejection
console.log(e);
res.status(500).json({ Result: e });
}
});
Connection Config:
const mysql = require('mysql');
var pool = mysql.createPool({
host:'localhost',
user: '1234',
password: '1234',
database : '1234',
multipleStatements: true,
connectionLimit: 10
});
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
module.exports ={
connection : pool
}

Checking https://github.com/mysqljs/mysql doesn't seem connection.query ever returns a Promise.
You can still use async/await by wrapping connection.query inside a promise.
Here's a post that explains how to do it https://medium.com/wenchin-rolls-around/example-of-using-transactions-with-async-await-via-mysql-connection-pool-9a37092f226f
And looks like another option is to use Bluebird How to promisify a MySql function using bluebird?

Related

Node.js MySQL connection with timer and error handling

I have a basic node.js app running as a service on RaspPi.
There should be two important issues:
Check MySQL database every 3 secs and get Status
Never throw even if internet connection is lost or MySQL server stops etc.
This code runs well but app closes and throw an error if MySQL server stops. It needs to be run even if there is an any kind of error. (No need to log and get error type)
And also; is this every 3 secs connection algorithm correct for performance?
var mysql = require('mysql');
var con = mysql.createConnection({
host: "192.168.1.100",
user: "test",
password: "test",
database: "test"
});
var Gpio = require('onoff').Gpio;
var LED = new Gpio(4, 'out'); //use GPIO pin 4
var getStatus = setInterval(checkStatus, 3000);
function checkStatus() {
try
{
con.connect(function(err) {
//if (err) throw err;
con.query("SELECT device_status FROM Device_Status WHERE
device_id='device01'", function (err, result, fields) {
if (err)
{
//console.log(err);
}
else
{
//console.log(result[0].device_status);
if(result[0].device_status == "1")
{
LED.writeSync(1);
}
else
{
LED.writeSync(0);
}
}
});
});
}
catch(err){ //console.log(err);
}
}
Have you thought about pooling connections? Then you would connect on every query.
connect() {
return new Promise((resolve, reject) => {
pool = mysql.createPool({
connectionLimit: 10,
host : this.host,
user : this.user,
password : this.password,
database : this.database
});
resolve(pool);
});
}
Then you would search like this:
search(collection, searchStr) {
return new Promise((resolve, reject) => {
pool.getConnection((err, connection) => {
if (err) {
resolve(err);
} else {
var sql = "SELECT * FROM " + collection
if (searchStr)
sql += " WHERE " + searchStr;
connection.query(sql, (err, result) => {
if (err) reject(err);
resolve(result);
connection.release();
});
}
});
});
}
Can you check this code from my repo https://github.com/MathewJohn1414/node-mysql-quickstart
. It is uses the MySQL connection pool and the connection errors are also handled.

How to avoid deadlock in nodejs mysql with a lot of queries?

I have a lot of urls, for every url I call the function load(url), this function parse the html, extracts the needed data and builds a bulk insert query as you can see in my test.js code. The problem is, if I have to many urls (like 100+), I get a Error: ER_LOCK_DEADLOCK from mysql. I tried to use async.queue but this is somehow not working (I don't know why, maybe I am using is wrongly). How can I run many urls + queries one after another, avoiding parallel execution which I think resulted in a deadlock? Even using async.queue results to a DEADLOCK (not always).
test.js
const request = require('request');
const async = require('async');
const pool = require('./database');
const urls = [
'https://www.quora.com/What-is-the-best-way-to-have-delayed-job-queue-with-node-js',
'https://de.wikipedia.org/wiki/Reinhardt-Zimmermann-L%C3%B6sung',
'https://towardsdatascience.com/the-5-clustering-algorithms-data-scientists-need-to-know-a36d136ef68'
]
let load = function(url) {
request({url: url}, function(error, response, html) {
if(!error) {
console.log(html);
/**
* 1. Parse HTML
* 2. Create Array of Values
* 3. Call pool.query(sql, [values], function(error) { ... })
*/
let data = [{}];
let sql = "INSERT IGNORE INTO tbl_test (title, content) VALUES ?";
let values = [];
data.forEach((item) => { values.push(item) });
pool.query(sql, [values], function(error) {
if(error) throw error;
})
} else {
console.log("handle error...");
}
})
}
let jobs = []
/*urls.forEach((url) => {
//jobs.push(load(url)); // --> Works but fails if the urls list is to big -> mysql deadlock error!
jobs.push(function(callback) { callback(load(url)) });
})*/
let q = async.queue(function(task, callback) {
console.log("Task:", task.uri);
callback();
})
q.drain = function() {
console.log('all task completed');
pool.end();
}
urls.forEach((url) => {
q.push({uri: url}, function(err) {
console.log('finished processing ...')
});
});
databse.js
require('dotenv').config();
const mysql = require('mysql');
let pool = mysql.createPool(
{
connectionLimit: 10,
host: process.env.DB_HOST,
port: process.env.DB_PORT,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME
}
);
pool.getConnection((err, connection) => {
if(err) {
if(err.code === 'PROTOCOL_CONNECTION_LOST') {
console.log('Database connection lost.')
}
if(err.code === 'ER_CON_COUNT_ERROR') {
console.log('Database has too many connections.')
}
if(err.code === 'ECONNREFUSED') {
console.log('Database connection refused.')
}
if(err.code === 'POOL_CLOSED') {
console.log('Pool is closed.')
}
}
if(connection) {
connection.release()
}
return;
});
module.exports = pool;
I have changed the code to use async.series instead of async.queue, beacuse the tasks would run in parallel in queue (see: https://caolan.github.io/async/docs.html#queue).
test.js
...
let tasks = [];
context.forEach((ctx) => {
tasks.push(function(callback) { load(ctx, callback) });
});
async.series(tasks, function(err) {
if(err) return next(err);
});

Simple server handling routes and giving error nodejs mysql

I'm trying to write a simple server using nodejs and have the server ship back different queries and/or custom headers/responses based on the routes. However, in the getUsers() function the error keeps getting hit and printing the 'Error querying' to the console instead of printing the email rows. I know the server is connected fine, because I can return a query when I just use the db and return a query with createConnection only using the second example. Any help spotting the error is greatly appreciated. Thanks.
What I'm trying to get done:
var http = require('http');
var mysql = require('mysql');
var url = require('url');
var util = require('util');
var db = mysql.createConnection({
host : "*********",
user : "*********",
password : "*********",
port : '****',
database : '*********'
});
db.connect(function(err) {
console.log('connected');
if (err)
console.error('Error connecting to db' + err.stack);
});
function getUsers() {
db.query('SELECT * FROM users', function(err, rows, fields) {
if (err)
// changed console.error('Error querying');
console.error(err);
if (rows)
console.log('Rows not null');
for (var i in rows) {
console.log(rows[i].email)
}
});
}
var server = http.createServer(function(req, res) {
console.log(req.url);
if (req.url == '/signup') {
console.log("User signing up");
} else if (req.url == '/signin') {
console.log("User signing in");
} else if (req.url == '/new') {
console.log("User request new game");
getUsers();
}
//res.writeHead(200);
//res.end('Hello Http');
});
server.listen(3000);
// changed and commented out db.end();
What does work with querying the db:
var connection = mysql.createConnection({
host : "********",
user : "********",
password : "********",
port : '****',
database : '********'
});
connection.connect();
var queryString = 'SELECT * FROM users';
connection.query(queryString, function(err, rows, fields) {
if (err) throw err;
for (var i in rows) {
console.log('Users: ', rows[i].email);
}
});
connection.end();
The code has been updated with the changes, and the problem was I was closing the database. After changing the error logs as was suggested in the comments, this was the error received.
{ [Error: Cannot enqueue Query after invoking quit.] code: 'PROTOCOL_ENQUEUE_AFTER_QUIT', fatal: false }
I then commented out the
db.end()
and the queries were returned fine.
Thanks for the help.

node.js + mysql connection pooling

I'm trying to figure out how to structure my application to use MySQL most efficent way. I'm using node-mysql module. Other threads here suggested to use connection pooling so i set up a little module mysql.js
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'root',
password : 'root',
database : 'guess'
});
exports.pool = pool;
Now whenever I want to query mysql I require this module and then query the databse
var mysql = require('../db/mysql').pool;
var test = function(req, res) {
mysql.getConnection(function(err, conn){
conn.query("select * from users", function(err, rows) {
res.json(rows);
})
})
}
Is this good approach? I couldn't really find too much examples of using mysql connections besides very simple one where everything is done in main app.js script so I don't really know what the convention / best practices are.
Should I always use connection.end() after each query? What if I forget about it somewhere?
How to rewrite the exports part of my mysql module to return just a connection so I don't have to write getConnection() every time?
It's a good approach.
If you just want to get a connection add the following code to your module where the pool is in:
var getConnection = function(callback) {
pool.getConnection(function(err, connection) {
callback(err, connection);
});
};
module.exports = getConnection;
You still have to write getConnection every time. But you could save the connection in the module the first time you get it.
Don't forget to end the connection when you are done using it:
connection.release();
You should avoid using pool.getConnection() if you can. If you call pool.getConnection(), you must call connection.release() when you are done using the connection. Otherwise, your application will get stuck waiting forever for connections to be returned to the pool once you hit the connection limit.
For simple queries, you can use pool.query(). This shorthand will automatically call connection.release() for you—even in error conditions.
function doSomething(cb) {
pool.query('SELECT 2*2 "value"', (ex, rows) => {
if (ex) {
cb(ex);
} else {
cb(null, rows[0].value);
}
});
}
However, in some cases you must use pool.getConnection(). These cases include:
Making multiple queries within a transaction.
Sharing data objects such as temporary tables between subsequent queries.
If you must use pool.getConnection(), ensure you call connection.release() using a pattern similar to below:
function doSomething(cb) {
pool.getConnection((ex, connection) => {
if (ex) {
cb(ex);
} else {
// Ensure that any call to cb releases the connection
// by wrapping it.
cb = (cb => {
return function () {
connection.release();
cb.apply(this, arguments);
};
})(cb);
connection.beginTransaction(ex => {
if (ex) {
cb(ex);
} else {
connection.query('INSERT INTO table1 ("value") VALUES (\'my value\');', ex => {
if (ex) {
cb(ex);
} else {
connection.query('INSERT INTO table2 ("value") VALUES (\'my other value\')', ex => {
if (ex) {
cb(ex);
} else {
connection.commit(ex => {
cb(ex);
});
}
});
}
});
}
});
}
});
}
I personally prefer to use Promises and the useAsync() pattern. This pattern combined with async/await makes it a lot harder to accidentally forget to release() the connection because it turns your lexical scoping into an automatic call to .release():
async function usePooledConnectionAsync(actionAsync) {
const connection = await new Promise((resolve, reject) => {
pool.getConnection((ex, connection) => {
if (ex) {
reject(ex);
} else {
resolve(connection);
}
});
});
try {
return await actionAsync(connection);
} finally {
connection.release();
}
}
async function doSomethingElse() {
// Usage example:
const result = await usePooledConnectionAsync(async connection => {
const rows = await new Promise((resolve, reject) => {
connection.query('SELECT 2*4 "value"', (ex, rows) => {
if (ex) {
reject(ex);
} else {
resolve(rows);
}
});
});
return rows[0].value;
});
console.log(`result=${result}`);
}
You will find this wrapper usefull :)
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
Require it, use it like this:
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});
I am using this base class connection with mysql:
"base.js"
var mysql = require("mysql");
var pool = mysql.createPool({
connectionLimit : 10,
host: Config.appSettings().database.host,
user: Config.appSettings().database.username,
password: Config.appSettings().database.password,
database: Config.appSettings().database.database
});
var DB = (function () {
function _query(query, params, callback) {
pool.getConnection(function (err, connection) {
if (err) {
connection.release();
callback(null, err);
throw err;
}
connection.query(query, params, function (err, rows) {
connection.release();
if (!err) {
callback(rows);
}
else {
callback(null, err);
}
});
connection.on('error', function (err) {
connection.release();
callback(null, err);
throw err;
});
});
};
return {
query: _query
};
})();
module.exports = DB;
Just use it like that:
var DB = require('../dal/base.js');
DB.query("select * from tasks", null, function (data, error) {
callback(data, error);
});
When you are done with a connection, just call connection.release() and the connection will return to the pool, ready to be used again by someone else.
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
// Use the connection
connection.query('SELECT something FROM sometable', function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
// Don't use the connection here, it has been returned to the pool.
});
});
If you would like to close the connection and remove it from the pool, use connection.destroy() instead. The pool will create a new connection the next time one is needed.
Source: https://github.com/mysqljs/mysql
You can use this format as I used
const mysql = require('mysql');
const { HOST, USERNAME, PASSWORD, DBNAME, PORT } = process.env;
console.log();
const conn = mysql.createPool({
host: HOST,
user: USERNAME,
password: PASSWORD,
database: DBNAME
}, { debug: true });
conn.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('Db is connected - The solution is: ', results[0].solution);
});
module.exports = conn;
Using the standard mysql.createPool(), connections are lazily created by the pool. If you configure the pool to allow up to 100 connections, but only ever use 5 simultaneously, only 5 connections will be made. However if you configure it for 500 connections and use all 500 they will remain open for the durations of the process, even if they are idle!
This means if your MySQL Server max_connections is 510 your system will only have 10 mySQL connections available until your MySQL Server closes them (depends on what you have set your wait_timeout to) or your application closes! The only way to free them up is to manually close the connections via the pool instance or close the pool.
mysql-connection-pool-manager module was created to fix this issue and automatically scale the number of connections dependant on the load. Inactive connections are closed and idle connection pools are eventually closed if there has not been any activity.
// Load modules
const PoolManager = require('mysql-connection-pool-manager');
// Options
const options = {
...example settings
}
// Initialising the instance
const mySQL = PoolManager(options);
// Accessing mySQL directly
var connection = mySQL.raw.createConnection({
host : 'localhost',
user : 'me',
password : 'secret',
database : 'my_db'
});
// Initialising connection
connection.connect();
// Performing query
connection.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
// Ending connection
connection.end();
Ref: https://www.npmjs.com/package/mysql-connection-pool-manager
i always use connection.relase(); after pool.getconnetion like
pool.getConnection(function (err, connection) {
connection.release();
if (!err)
{
console.log('*** Mysql Connection established with ', config.database, ' and connected as id ' + connection.threadId);
//CHECKING USERNAME EXISTENCE
email = receivedValues.email
connection.query('SELECT * FROM users WHERE email = ?', [email],
function (err, rows) {
if (!err)
{
if (rows.length == 1)
{
if (bcrypt.compareSync(req.body.password, rows[0].password))
{
var alldata = rows;
var userid = rows[0].id;
var tokendata = (receivedValues, userid);
var token = jwt.sign(receivedValues, config.secret, {
expiresIn: 1440 * 60 * 30 // expires in 1440 minutes
});
console.log("*** Authorised User");
res.json({
"code": 200,
"status": "Success",
"token": token,
"userData": alldata,
"message": "Authorised User!"
});
logger.info('url=', URL.url, 'Responce=', 'User Signin, username', req.body.email, 'User Id=', rows[0].id);
return;
}
else
{
console.log("*** Redirecting: Unauthorised User");
res.json({"code": 200, "status": "Fail", "message": "Unauthorised User!"});
logger.error('*** Redirecting: Unauthorised User');
return;
}
}
else
{
console.error("*** Redirecting: No User found with provided name");
res.json({
"code": 200,
"status": "Error",
"message": "No User found with provided name"
});
logger.error('url=', URL.url, 'No User found with provided name');
return;
}
}
else
{
console.log("*** Redirecting: Error for selecting user");
res.json({"code": 200, "status": "Error", "message": "Error for selecting user"});
logger.error('url=', URL.url, 'Error for selecting user', req.body.email);
return;
}
});
connection.on('error', function (err) {
console.log('*** Redirecting: Error Creating User...');
res.json({"code": 200, "status": "Error", "message": "Error Checking Username Duplicate"});
return;
});
}
else
{
Errors.Connection_Error(res);
}
});

Reproduce MySQL error: The server closed the connection (node.js)

I'm trying to reproduce a MySQL error I'm seeing in my node.js app on EC2 with the node mysql library:
Connection lost: The server closed the connection.
I am unable to reproduce the error locally- killing the database is handled just fine by my code- it just rechecks every few seconds and reconnects to the db once it is restarted. On EC2, it happens around 4am Pacific, but the db is still up and running fine.
I'd like to
Reproduce the crash with my local mysql
Add whatever logic I need in my mysql helper module to handle this
Here's the error in my node.js app:
2012-10-22T08:45:40.518Z - error: uncaughtException date=Mon Oct 22
2012 08:45:40 GMT+0000 (UTC), pid=14184, uid=0, gid=0,
cwd=/home/ec2-user/my-app, execPath=/usr/bin/nodejs,
version=v0.6.18, argv=[/usr/local/bin/node,
/home/ec2-user/my-app/app.js, --my-app], rss=15310848,
heapTotal=6311392, heapUsed=5123292, loadavg=[0.0029296875,
0.0146484375, 0.04541015625], uptime=3238343.511107486, trace=[column=13,
file=/home/ec2-user/my-app/node_modules/mysql/lib/protocol/Protocol.js,
function=Protocol.end, line=63, method=end, native=false, column=10,
file=stream.js, function=Socket.onend, line=80, method=onend,
native=false, column=20, file=events.js, function=Socket.emit,
line=88, method=emit, native=false, column=51, file=net.js,
function=TCP.onread, line=388, method=onread, native=false],
stack=[Error: Connection lost: The server closed the connection.,
at Protocol.end
(/home/ec2-user/my-app/node_modules/mysql/lib/protocol/Protocol.js:63:13), at Socket.onend (stream.js:80:10), at Socket.emit
(events.js:88:20), at TCP.onread (net.js:388:51)]
Here's my code (mysql helper module):
module.exports = function (conf,logger) {
var mysql = require('mysql');
var connectionState = false;
var connection = mysql.createConnection({
host: conf.db.hostname,
user: conf.db.user,
password: conf.db.pass,
database: conf.db.schema,
insecureAuth: true
});
function attemptConnection(connection) {
if(!connectionState){
connection = mysql.createConnection(connection.config);
connection.connect(function (err) {
// connected! (unless `err` is set)
if (err) {
logger.error('mysql db unable to connect: ' + err);
connectionState = false;
} else {
logger.info('mysql connect!');
connectionState = true;
}
});
connection.on('close', function (err) {
logger.error('mysqldb conn close');
connectionState = false;
});
connection.on('error', function (err) {
logger.error('mysqldb error: ' + err);
connectionState = false;
/*
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
*/
});
}
}
attemptConnection(connection);
var dbConnChecker = setInterval(function(){
if(!connectionState){
logger.info('not connected, attempting reconnect');
attemptConnection(connection);
}
}, conf.db.checkInterval);
return connection;
};
Check out mysql pool feature in node-mysql
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
pool.getConnection(function(err, connection) {
// connected! (unless `err` is set)
connection.end();
});
I was having similar problems and created a getConnection() wrapper function that checks the health of the mysql connection before returning it to the caller and re-establishes the connection as necessary. In my testing it has handled fatal and non-fatal connection issues transparently for the application. If the connection simply timed out, the application recovers without experiencing any errors. If there is a transient but fatal database connection problem, the application will resume functioning automatically as soon as database connectivity is available again.
As far as reproducing the problem for testing, add the following two lines to the my.ini or my.cnf file under the [mysqld] block:
interactive_timeout=30
wait_timeout=30
Here is the contents of a file I have named "database.js":
var mysql = require("mysql");
var CONFIG = require(__dirname + "/configuration");
module.exports.getConnection = function() {
// Test connection health before returning it to caller.
if ((module.exports.connection) && (module.exports.connection._socket)
&& (module.exports.connection._socket.readable)
&& (module.exports.connection._socket.writable)) {
return module.exports.connection;
}
console.log(((module.exports.connection) ?
"UNHEALTHY SQL CONNECTION; RE" : "") + "CONNECTING TO SQL.");
var connection = mysql.createConnection({
host : CONFIG.db.host,
user : CONFIG.db.user,
password : CONFIG.db.password,
database : CONFIG.db.database,
port : CONFIG.db.port
});
connection.connect(function(err) {
if (err) {
console.log("SQL CONNECT ERROR: " + err);
} else {
console.log("SQL CONNECT SUCCESSFUL.");
}
});
connection.on("close", function (err) {
console.log("SQL CONNECTION CLOSED.");
});
connection.on("error", function (err) {
console.log("SQL CONNECTION ERROR: " + err);
});
module.exports.connection = connection;
return module.exports.connection;
}
// Open a connection automatically at app startup.
module.exports.getConnection();
// If you've saved this file as database.js, then get and use the
// connection as in the following example:
// var database = require(__dirname + "/database");
// var connection = database.getConnection();
// connection.query(query, function(err, results) { ....
Here's what I ended up using, and it worked pretty well. On the occasional connection lost/restart it recovered nicely. I have a database.js file which establishes connections and checks them periodically.
To make a request:
var conn = require('./database');
var sql = 'SELECT foo FROM bar;';
conn.query(sql, [userId, plugId], function (err, rows) {
// logic
}
Here's my databbase.js
var mysql = require('mysql');
var Common = require('./common');
var conf = Common.conf;
var logger = Common.logger;
var connectionState = false;
var connection = mysql.createConnection({
host: conf.db.hostname,
user: conf.db.user,
password: conf.db.pass,
database: conf.db.schema,
insecureAuth: true
});
connection.on('close', function (err) {
logger.error('mysqldb conn close');
connectionState = false;
});
connection.on('error', function (err) {
logger.error('mysqldb error: ' + err);
connectionState = false;
});
function attemptConnection(connection) {
if(!connectionState){
connection = mysql.createConnection(connection.config);
connection.connect(function (err) {
// connected! (unless `err` is set)
if (err) {
logger.error('mysql db unable to connect: ' + err);
connectionState = false;
} else {
logger.info('mysql connect!');
connectionState = true;
}
});
connection.on('close', function (err) {
logger.error('mysqldb conn close');
connectionState = false;
});
connection.on('error', function (err) {
logger.error('mysqldb error: ' + err);
if (!err.fatal) {
//throw err;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
//throw err;
} else {
connectionState = false;
}
});
}
}
attemptConnection(connection);
var dbConnChecker = setInterval(function(){
if(!connectionState){
logger.info('not connected, attempting reconnect');
attemptConnection(connection);
}
}, conf.db.checkInterval);
// Mysql query wrapper. Gives us timeout and db conn refreshal!
var queryTimeout = conf.db.queryTimeout;
var query = function(sql,params,callback){
if(connectionState) {
// 1. Set timeout
var timedOut = false;
var timeout = setTimeout(function () {
timedOut = true;
callback('MySQL timeout', null);
}, queryTimeout);
// 2. Make query
connection.query(sql, params, function (err, rows) {
clearTimeout(timeout);
if(!timedOut) callback(err,rows);
});
} else {
// 3. Fail if no mysql conn (obviously)
callback('MySQL not connected', null);
}
}
// And we present the same interface as the node-mysql library!
// NOTE: The escape may be a trickier for other libraries to emulate because it looks synchronous
exports.query = query;
exports.escape = connection.escape;
Using generic-pool, I wrote something that works locally. I guess I'll test it and see if it doesn't crash in bizarre manner on the server side.
// Test node connection pool stuff
// Create a MySQL connection pool with
// a max of 10 connections, a min of 2, and a 30 second max idle time
var poolModule = require('generic-pool');
var pool = poolModule.Pool({
name : 'mysql',
create : function(callback) {
var Client = require('mysql').Client; // use node-mysql library in all it's dubious glory
var c = new Client();
c.user = 'root';
c.password = 'xxx';
c.database = 'test';
c.on('close', function (err) {
console.log('mysqldb conn close');
});
c.on('error', function (err) {
console.log('mysqldb error: ' + err);
});
// parameter order: err, resource
// new in 1.0.6
callback(null, c);
},
destroy : function(client) { client.end(); },
max : 10,
// optional. if you set this, make sure to drain() (see step 3)
min : 2,
// specifies how long a resource can stay idle in pool before being removed
idleTimeoutMillis : 30000,
// if true, logs via console.log - can also be a function
log : true
});
var http = require('http');
http.createServer(function (req, res) {
// Get db conn
pool.acquire(function(err, client) {
if (err) {
// handle error - this is generally the err from your
// factory.create function
console.log('pool.acquire err: ' + err);
res.writeHead(500, {'Content-Type': 'application/json'});
out = {
err: err
}
res.end(JSON.stringify(out));
}
else {
client.query("select * from foo", [], function(err, results) {
if(err){
res.writeHead(500, {'Content-Type': 'application/json'});
out = {
err: err
}
res.end(JSON.stringify(out));
} else {
res.writeHead(500, {'Content-Type': 'application/json'});
out = {
results: results
}
res.end(JSON.stringify(out));
}
// return object back to pool
pool.release(client);
});
}
});
}).listen(9615);
Pretty please don't die at 4am for no apparent reason!
The solution is use pooling connection !
You can wrote code to handle connection manually, it works.
However pooling is design for this, use pooling connection solved connection drop error.
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 10,
host : 'example.org',
user : 'bob',
password : 'secret',
database : 'my_db'
});
pool.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
pooling mysql connection