Writing mysql queries in nodejs with pooling connections - mysql

Would like to know the best practice in writing mysql queries in nodejs with connection pool. Found some related threads. But none of them answered the exact question. So starting a new one.
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
// insert into first table
connection.query( 'Insert into table values(....)', function(err, rows) {
//get the auto incremented value from first insert and use it in the second insert
connection.query("insert into table2 values(rows.insertID,..)",function(err,rows){
//release the connection to pool after performing all the inserts
connection.release();
});
});
}); `
Doubts :
Is this a correct method to write these queries
While using pool is there a need to use end() . What I understood is that when we use the release() function the connection will go back to the pool and we can reuse it. So will there be a need to use end() any where.

Your code looks fine (IE opening a pool, multiple queries while grabbing the insertID of the previous query). With a pool, connection.release() will 'close' the session and return to the pool. connection.end() is not needed.
Additional:
IMO, connection pools are used if you have operations that would result in blocking other users (mostly large reads, or large writes). Since you are inserting data (assuming this would be relatively fast), you could try using Transactions.
Transactions provide methods to fail safe your queries (increase integrity of your database). If you use a transaction, you can check if EVERYTHING was called without errors before committing to the DB (you have to option to rollback changes).
var mysql = require('mysql');
var connection = mysql.createConnection({...});
// Open it up
connection.connect();
connection.beginTransaction(function(err){
if(err) { throw err; }
// Use connection.escape() to sanitize user input
var insertQuery = 'Insert into table values(....)';
connection.query(insertQuery, function(err, result) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
//get the auto incremented value from first insert and use it in the second insert
var rowId = result.insertId;
var insertQuery2 = "insert into table2 values(" + rowId + ",..)";
connection.query(insertQuery2, function(err, result){
if(err) {
return connection.rollback(function(){
throw err;
});
}
connection.commit(function(err){
if (err) {
return connection.rollback(function() {
throw err;
});
}
// Close it up
connection.end();
});
});
});
});

Related

Store mysql query rows in variable for later use

I'm doing a monitoring system project in which I have Arduino sensors data being sent to a node.js server (thru GET requests) and then stored in a MySQL DB.
Whenvever I successfully send data to the server, it connects to the MySQL DB and queries the last 5 received records to do some processing.
Therefore, I need to store the rows of those 5 records in a variable for later use. Meaning that I have to get rows from a connection.query in a variable.
I read that the fact that I'm not able to do this is because node.js being async. So my questions are:
Is it possible to do the described tasks the way I'm trying?
If not, is there any other way to do so?
I'm not putting the whole code here but I'm running a separated test that also doesn't run properly. Here it is:
var mysql = require('mysql');
var con = mysql.createConnection({
host : "127.0.0.1",
user : "root",
password: "xxxx",
database: "mydb",
port : 3306
});
var queryString = "SELECT id, temp1, temp2, temp3, temp4, level_ice_bank, flow FROM tempdata ORDER BY id DESC LIMIT 5";
con.connect(function(err) {
if (err) throw err;
});
var result_arr = [];
function setValue (value) {
result_arr = value;
}
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
//console.log(rows);
setValue(rows);
}
});
console.log(result_arr);
It logs:
[]
But if I uncomment console.log(rows); it logs what I need to store in the variable result_arr.
Thanks in advance to all.
You're seeing this behaviour because con.query(...) is an asynchronous function. That means that:
console.log(result_arr);
Runs before:
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
//console.log(rows);
setValue(rows);
}
});
(Specifically, the setValue(rows) call)
To fix this in your example, you can just do:
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
setValue(rows);
console.log(result_arr);
}
});
If you want to do more than just log the data, then you can call a function which depends on result_arr from the con.query callback, like this:
con.query(queryString, function (err, rows, fields) {
if (err) throw err;
else {
setValue(rows);
doCleverStuffWithData();
}
});
function doCleverStuffWithData() {
// Do something with result_arr
}

has exceeded the 'max_user_connections' resource

I have a MySQL, Express, Angular, NodeJS application and sometimes when I log in I get the following error in my node console:
TypeError: Cannot read property 'query' of undefined
The error occurs in my passport.local.js file, this is the line:
connection.query('SELECT * FROM users WHERE username LIKE ?', [username], function (err, user) {
This is the passport function
passport.use(new LocalStrategy(
function(username, password, done) {
console.log('app.js');
pool.getConnection(function(err, connection) {
console.log('err: ' + err);
console.log(connection);
connection.query('SELECT * FROM users WHERE username LIKE ?', [username], function (err, user) {
if (err) throw err;
for (var i = user.length - 1; i >= 0; i--) {
var current = user[i];
}
if(current){
if(bcrypt.compareSync(password, current.password)){
return done(null, user);
} else {
return done(null, false);
}
} else {
console.log('no user');
return done(null, false);
}
});
connection.release();
});
}
));
I require my pool in the top of my file
var pool = require('../../config/connection');
When the error occurs the:
console.log(connection);
Gets:
undefined
I also log the error:
console.log('err: ' + err);
Shows:
err: Error: ER_USER_LIMIT_REACHED: User 'bfe4a8980ede74' has exceeded the 'max_user_connections' resource (current value: 10)
I am assuming that your max_user_connections is set to 10. Please increase the max_user_connection value.
show global variables like '%connections%';
Will help you in giving the no of connections you have set. Increase the number of connections If its less than 25 or 50. Max connections can be more than 16000 I guess, it all depends on your cpu, no of threads It can handle etc.
SET GLOBAL max_user_connections=100;
max_user_connections is a dynamic variable, which means you can directly run this query. You donot have to shutdown mysql.
The error you're getting is stating the issue: your MySQL server is only allowing 10 connection per user, and that limit has been reached.
The default for the mysql connection pool also happens to be 10, which is cutting it really close. If any other MySQL client besides your Express app is connected to the database with the same user credentials, you may run into that particular error. I would suggest increasing max_user_connections in the MySQL configuration.
Aside from that, there's another issue with your code: it's releasing the connection before the query has finished, which may lead to unexpected behaviour. Move the call to connection.release() to inside the callback:
pool.getConnection(function(err, connection) {
...
connection.query('SELECT * FROM users WHERE username LIKE ?', [username], function (err, user) {
connection.release();
...
});
});
If this is a common way you're using MySQL (get a connection, perform a query, release the connection), you can make life a bit easier by using pool.query() instead. See this example.
And finally, if you're working with async code, don't throw errors but pass them to the callback (and make sure that you actually handle them, because you're not handling any errors from pool.getConnection now, besides logging them):
pool.getConnection(function(err, connection) {
if (err) return done(err);
...
});

Insert data into mysql with node.js works, but script hangs

I've got this script for reading a file and then insert the data into mysql tables. The script works, but it hangs, so I have to press CTRL-C to stop the script.
But the script should stop normally, what do I have to change?
var fs = require('fs');
var filename;
var myGID;
filename = "data/insertUser1_next.json";
function get_line(filename, line_no, callback) {
fs.readFile(filename, function (err, data) {
if (err) throw err;
// Data is a buffer that we need to convert to a string
// Improvement: loop over the buffer and stop when the line is reached
var lines = data.toString('utf-8').split("\n");
if(+line_no > lines.length){
return callback('File end reached without finding line', null);
}
// lines
callback(null, lines[0], lines[1], lines[2], lines[3]);
});
}
get_line(filename, 0, function(err, line, line2, line3, line4){
line = line.replace(/(\r\n|\n|\r)/gm,"");
line2 = line2.replace(/(\r\n|\n|\r)/gm,"");
line3 = line3.replace(/(\r\n|\n|\r)/gm,"");
/*line4 = line4.replace(/(\r\n|\n|\r)/gm,"");*/
console.log('The line: ' + line);
console.log('The line2: ' + line2);
console.log('The line3: ' + line3);
console.log('The line4: ' + line4);
var post = {gid: line, uid: line2};
var post2 = {uid: line2, displayname: line3, password: line4};
var mysql = require('mysql');
var db_config = {
host : '123.456.789.012',
user : 'user',
password : 'password',
database : 'maindata'
};
var con = mysql.createPool(db_config);
con.getConnection(function(err){
if (err) {
console.log(err);
return;
}
con.query('INSERT INTO group_user SET ?', post, function(err, result) {
if (err) {
console.log(err);
return;
}
});
con.query('INSERT INTO users SET ?', post2, function(err, result) {
if (err) {
console.log(err);
return;
}
});
});
});
Here you can see what happened:
When you are done using the pool, you have to end all the connections or the Node.js event loop will stay active until the connections are closed by the MySQL server. This is typically done if the pool is used in a script or when trying to gracefully shutdown a server. To end all the connections in the pool, use the end method on the pool:
pool.end(function (err) {
// all connections in the pool have ended
});
So, if you place con.end() after your queries are done, the script will terminate normally
The following statement will close the connection ensuring that all the queries in the queue are processed. Please note that this is having a callback function.
connection.end(function(err){
// Do something after the connection is gracefully terminated.
});
The following statement will terminate the assigned socket and close the connection immediately. Also there is no more callbacks or events triggered for the connection.
connection.destroy();
hey I suggest to install forever and start node servers.js with forever you dont need any terminal open.
And you need to close you mysql connection at the end to stop you hangs problem, i think.
npm install -g forever
npm install forever
//FOR your Problem
con.end(function(err){
// Do something after the connection is gracefully terminated.
});
con.destroy();
The following statement will close the connection ensuring that all the queries in the queue are processed. Please note that this is having a callback function.
connection.end(function(err){
// Do something after the connection is gracefully terminated.
});
The following statement will terminate the assigned socket and close the connection immediately. Also there is no more callbacks or events triggered for the connection.
connection.destroy();

Node Mysql Cannot Enqueue a query after calling quit

where do i close the mysql connection?
I need to run queries in sequence. I am writing code that looks like this at present:
var sqlFindMobile = "select * from user_mobiles where mobile=?";
var sqlNewUser = "insert into users (password) values (?)";
//var sqlUserId = "select last_insert_id() as user_id";
var sqlNewMobile = "insert into user_mobiles (user_id, mobile) values (?,?)";
connection.connect(function(err){});
var query = connection.query(sqlFindMobile, [req.body.mobile], function(err, results) {
if(err) throw err;
console.log("mobile query");
if(results.length==0) {
var query = connection.query(sqlNewUser, [req.body.password], function(err, results) {
if(err) throw err;
console.log("added user");
var user_id = results.insertId;
var query = connection.query(sqlNewMobile, [user_id, req.body.mobile], function(err, results) {
if(err) throw err;
console.log("added mobile");
//connection.end();
});
});
}
});
//connection.end();
(I am a beginner with node, npm-express and npm-mysql. I have tried searching SO for "express mysql cannot enqueue" to find related questions and have not found them.)
I fixed this problem use this method:
connection.end() in your connection.query function
The fixed code is here
If you're using the node-mysql module by felixge then you can call connection.end() at any point after you've made all of the connection.query() calls, since it will wait for all of the queries to finish before it terminates the connection.
See the example here for more information.
If you're wanting to run lots of queries in series, you should look into the async module, it's great for dealing with a series of asynchronous functions (i.e. those that have a callback).
Maybe the problem is that the mySQL query is executed after the connection is already closed, due to the asynchronous nature of Node. Try using this code to call connection.end() right before the thread exits:
function exitHandler(options, err) {
connection.end();
if (options.cleanup)
console.log('clean');
if (err)
console.log(err.stack);
if (options.exit)
process.exit();
}
//do something when app is closing
process.on('exit', exitHandler.bind(null, {cleanup: true}));
Code adapted from #Emil Condrea, doing a cleanup action just before node.js exits
In my case connection.end was being called in a spot that was hard to notice, so an errant call to connection.end could be the problem with this error

How to structure Node.js script with MySQL so that connection doesn't close prematurely

So I am making my first attempt with Node and I can't really wrap my head around how to work with the MySQL connection. The script is somewhat simplified like this
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '192.168.40.1',
user : 'user',
password : 'password',
database : 'database'
});
function DoSomething(connection, item, callback) {
connection.query(
'SELECT COUNT(*) AS count FROM another_table WHERE field=?',
item.field,
function (err, results) {
if (err) throw err;
if (results.length > 0 && results[0].count >= 1) {
callback(err, connection, item, 'Found something')
}
});
}
function DoSomethingElse(connection, item, callback) {
// Similar to DoSomething()
}
function StoreResult(err, connection, item, reason) {
if (err) throw err;
connection.query(
'INSERT INTO result (item_id, reason) VALUES (?, ?)',
[item.id, reason],
function (err, results) {
if (err) throw err;
});
}
connection.query('SELECT * FROM table WHERE deleted=?', [0], function (err, results)
{
if (err) throw err;
results.forEach(function (item, index) {
DoSomething(connection, item, StoreResult);
DoSomethingElse(connection, item, StoreResult);
});
});
connection.end();
What I am having trouble with (as far as I can tell) is that since DoSomething() it seems that connection.end() is called before all of the DoSomething()'s have finished causing errors that queries can't be performed when the connection is closed.
I tried playing around with the async library, but I haven't gotten anywhere so far. Anyone with some good advice on how to do this?
The problem with your code is that you're closing the connection synchronously while an asynchronous request is still being handled. You should call connection.end() only after all query callbacks have been called.
Since you are doing multiple queries, this means using some way to wait for all their results. The simplest way is to nest every next call into the callback of the previous one, but that way leads to the pyramid of doom. There are a number of popular libraries that solve this, but my own preference is for async.
Using async I would rewrite your code as follows:
async.waterfall([function(next) {
connection.query('SELECT * FROM table WHERE deleted=?', [0], next); // note the callback
},
function(results, next) {
// asynchronously handle each results. If they should be in order, use forEachSeries
async.forEach(results, function(item, next) {
// handle in parallel
async.parallel([function(done) {
DoSomething(connection, item, function(err, connection, item, reason) {
// This is a hack, StoreResult should have a callback which is called
// after it's done, because the callback is now being called too soon
StoreResult(err, connection, item, reason);
callback(err);
});
}, function(done) {
DoSomethingElse(connection, item, function(err, connection, item, reason) {
// This is a hack, StoreResult should have a callback which is called
// after it's done, because the callback is now being called too soon
StoreResult(err, connection, item, reason);
callback(err);
}], function(err) {
// this callback is called with an error as soon as it occurs
// or after all callbacks are called without error
next(err);
});
}, function(err) {
// idem
next(err);
});
}], function(err, results) {
// idem
// Do whatever you want to do with the final error here
connection.end();
});
This also allows you to solve a possible issue with the order of your queries in the forEach: They are started in order, but are not guaranteed to finish in order due to their asynchronous nature.
Close your connection after you have done everything you want in the script.
When programming in asynchronous language, keep in mind that the real ending point of your script is the last asynchronous callback, instead of the last line like other scripts (e.g. PHP).
Note that you don't want to simply ignore the connection.end(); as the underlying MySQL driver will keep the connection alive and your script will stuck in the last line forever until you kill it.
This is the modified version of your code.
connection.query('SELECT * FROM table WHERE deleted=?', [0], function (err, results)
{
if (err) throw err;
results.forEach(function (item, index) {
DoSomething(connection, item, StoreResult);
DoSomethingElse(connection, item, StoreResult);
});
// End your connection here
connection.end();
});