Insert data into mysql with node.js works, but script hangs - mysql

I've got this script for reading a file and then insert the data into mysql tables. The script works, but it hangs, so I have to press CTRL-C to stop the script.
But the script should stop normally, what do I have to change?
var fs = require('fs');
var filename;
var myGID;
filename = "data/insertUser1_next.json";
function get_line(filename, line_no, callback) {
fs.readFile(filename, function (err, data) {
if (err) throw err;
// Data is a buffer that we need to convert to a string
// Improvement: loop over the buffer and stop when the line is reached
var lines = data.toString('utf-8').split("\n");
if(+line_no > lines.length){
return callback('File end reached without finding line', null);
}
// lines
callback(null, lines[0], lines[1], lines[2], lines[3]);
});
}
get_line(filename, 0, function(err, line, line2, line3, line4){
line = line.replace(/(\r\n|\n|\r)/gm,"");
line2 = line2.replace(/(\r\n|\n|\r)/gm,"");
line3 = line3.replace(/(\r\n|\n|\r)/gm,"");
/*line4 = line4.replace(/(\r\n|\n|\r)/gm,"");*/
console.log('The line: ' + line);
console.log('The line2: ' + line2);
console.log('The line3: ' + line3);
console.log('The line4: ' + line4);
var post = {gid: line, uid: line2};
var post2 = {uid: line2, displayname: line3, password: line4};
var mysql = require('mysql');
var db_config = {
host : '123.456.789.012',
user : 'user',
password : 'password',
database : 'maindata'
};
var con = mysql.createPool(db_config);
con.getConnection(function(err){
if (err) {
console.log(err);
return;
}
con.query('INSERT INTO group_user SET ?', post, function(err, result) {
if (err) {
console.log(err);
return;
}
});
con.query('INSERT INTO users SET ?', post2, function(err, result) {
if (err) {
console.log(err);
return;
}
});
});
});

Here you can see what happened:
When you are done using the pool, you have to end all the connections or the Node.js event loop will stay active until the connections are closed by the MySQL server. This is typically done if the pool is used in a script or when trying to gracefully shutdown a server. To end all the connections in the pool, use the end method on the pool:
pool.end(function (err) {
// all connections in the pool have ended
});
So, if you place con.end() after your queries are done, the script will terminate normally

The following statement will close the connection ensuring that all the queries in the queue are processed. Please note that this is having a callback function.
connection.end(function(err){
// Do something after the connection is gracefully terminated.
});
The following statement will terminate the assigned socket and close the connection immediately. Also there is no more callbacks or events triggered for the connection.
connection.destroy();

hey I suggest to install forever and start node servers.js with forever you dont need any terminal open.
And you need to close you mysql connection at the end to stop you hangs problem, i think.
npm install -g forever
npm install forever
//FOR your Problem
con.end(function(err){
// Do something after the connection is gracefully terminated.
});
con.destroy();
The following statement will close the connection ensuring that all the queries in the queue are processed. Please note that this is having a callback function.
connection.end(function(err){
// Do something after the connection is gracefully terminated.
});
The following statement will terminate the assigned socket and close the connection immediately. Also there is no more callbacks or events triggered for the connection.
connection.destroy();

Related

How to return a promise as well as close the mysql connection in the same code in NodeJS

We want our function to return a promise as well as close the database connection. We put the promise in try and connection closing command in finally to which it gives the following errors -
conPool.close() //It says Error - conPool.close() is not a function
conPool.release() // Error - conPool.release() is not a function
conPool.releaseConnection(); // Error - Cannot read property '_pool'
of undefined
Please help me close the mySQL connection. I want to return my data . But after return nothing works so the connection doesn't close. I am afraid i may loose out on maximum limit. I have even set maximum connections in mySQL but still. And there is a possible solution that i may declare conPool outside the function and make all the functions use this single conPool but this is also not working.
code -
function viewChoice() {
var sqlQuery;
sqlQuery = "select * from TRANSACTION_PAYLOAD where INTERFACE_NAME
= 'Vehicle RC' AND (STATUS ='INPUT_ERROR' OR STATUS ='ERROR')";
}
var deferred = Q.defer();
var host = config.host;
var user = config.user;
var password = config.password;
var database = config.database;
var conPool = mysql.createPool({
host: host,
user: user,
password: password,
database: database
});
try{
conPool.getConnection(function (err) {
console.log("Inside getConnection ")
if (err)
deferred.reject(err.name + ': ' + err.message);
conPool.query(sqlQuery,
function (err, result, fields) {
if (err) deferred.reject(err.name + ': ' + err.message);
console.log(result);
deferred.resolve(result);
});
});
return deferred.promise;
}
finally{
console.log("inside Finally")
conPool.releaseConnection();
}
}
From MySQL connector docs :
Closing all the connections in a pool
When you are done using the pool, you have to end all the connections or the Node.js event loop will stay active until the connections are closed by the MySQL server. This is typically done if the pool is used in a script or when trying to gracefully shutdown a server. To end all the connections in the pool, use the end method on the pool:
pool.end(function (err) {
// all connections in the pool have ended
});
The end method takes an optional callback that you can use to know when all the connections are ended.
Once pool.end is called, pool.getConnection and other operations can no longer be performed. Wait until all connections in the pool are released before calling pool.end. If you use the shortcut method pool.query, in place of pool.getConnection → connection.query → connection.release, wait until it completes.
pool.end calls connection.end on every active connection in the pool. This queues a QUIT packet on the connection and sets a flag to prevent pool.getConnection from creating new connections. All commands / queries already in progress will complete, but new commands won't execute.
Plus you may move your return deferred.promise; inside finally block

Where to end the mysql pool connection on nodejs?

I'm using MySQL on Nodejs. I'm using mysql pool to create the connection:
Var mysqk = require ('mysql');
Var pool = mysql.createPool ({my configs});
My question is:
Where in the app will I use pool.end() as reported in the documentation?
For example: in my www file, I created a code to release other things as I have pointed in de code. Should I use pool.end() there?
var app = require('../app');
var debug = require('debug')('cancela:server');
var http = require('http');
var port = normalizePort(process.env.PORT || '4000');
app.set('port', port);
var server = http.createServer(app);
// my app codes...
..
.
.
process.once('SIGTERM', end);
process.once('SIGINT', end);
function end() {
server.close(function(err){
if(err) throw err();
// Should I end the Pool connection here? <<<<<<===================================
// pool.end(function (err) {
// // all connections in the pool have ended
// });
console.log('Server endded!');
/* exit gracefully */
process.exit();
});
}
Sometimes you want to manually release the connection pool for whatever reasons so you can use pool.end() before eventually recreating the pool.
Most of times you want to release it when your server is about to shutdown so you can do what you are thinking. Pay attention to the location of the function call process.exit(). It must be called after the connection release, otherwise the pool won't finish to release.
function end() {
server.close(function (err) {
if (err) throw err;
console.log('Server endded!');
pool.end(function (err) {
if (err) throw err;
process.exit();
});
});
}

Node.JS and MySQL - queries lock up and execute extremely slowly

I am getting strange behavior using Node.JS and MySQL with this driver - https://github.com/mysqljs/mysql
Essentially, I have a button on the frontend that triggers an app.get that makes a query in the database and I can happily use the results in my backend.
This works nicely, until I press the button 4-5 times in a second, where as the queries lock up and I have to wait for 2-3 minutes until they continue executing. I have a similar write function that behaves the same way.
Is it possible this is a problem, because I'm trying to execute the exact same query asynchronously? I.e. do I have to limit this from the front end or is it a backend problem?
Any ideas on how to debug what exactly is going on?
// database.js
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit: 100,
host : 'localhost',
user : 'secret',
password : 'secret',
database : 'mydb'
});
exports.getConnection = function(callback) {
pool.getConnection(function(err, connection) {
callback(err, connection);
});
};
// dbrw.js
var con = require('../config/database');
function read(id, done) {
con.getConnection(function(err, connection){
if(!err){
connection.query("SELECT * FROM users WHERE id = ?",[id], function(err, rows) {
connection.release();
if (err)
done(err);
if (rows.length) {
console.log("rows " + JSON.stringify(rows));
done(rows[0].progress);
};
});
}
else {
console.log(err);
}
});
}
exports.read = read;
// routes.js
var dbrw = require('./dbrw.js');
app.get('/read', isLoggedIn, function(req, res) {
dbrw.read(req.user.id, function(result) {
console.log(result);
});
});
// Frontend - angular app.js
$scope.tryread = function() {
$http.get('/read');
}
Thanks in advance for any input.
I see a few issues:
function read(id, done) {
con.getConnection(function(id, connection){...}
}
Notice how you overwrite the id passed to read by giving that same name to an argument of the callback to getConnection.
Also, your Express route doesn't actually end the request by sending back a response, which will make your browser time out the connection. At some point, it will even refuse to send more requests because too many are still pending.
So make sure to end the request:
app.get('/read', isLoggedIn, function(req, res) {
dbrw.read(req.user.id, function(result) {
console.log(result);
res.end(); // or `res.send(result)`
});
});
And a tip: you should use the callback calling convertion for Node, where the first argument represents an error (if there is any) and the second argument represents the return value.

No hint of errors when trying to connect to a database

I've been struggling to get MySQL working with node for a while. When I run the following code no errors are thrown, but simultaneously none of the console messages are being printed (except for the obvious one).
var app = require('express')();
var http = require('http').Server(app);
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '******',
database : 'blogDB'
});
connection.connect(function(err) {
if(err) {
console.log('error when connecting to database:', err);
}
console.log('Connected to the database');
});
var queryString = 'SELECT * FROM blogs';
connection.query(queryString, function(err, rows, fields) {
if (err) throw err;
for (var i in rows) {
console.log('Post: ', rows[i].id);
}
});
connection.end();
http.listen(3306, function(){
console.log('listening on *:3306');
});
Output:listening on *:3306
On top of this, when I go to "localhost:3306" in the browser, a download is immediately started and nothing appears on the web page. The download is a file with no extensions, but contained the following:
J
5.6.19 tscvKP3M ÿ÷ € g?F!q6X:Y2*z mysql_native_password ! ÿ„#08S01Got packets out of order
I am not sure if that is relevant, but it certainly was not happening when I was not running MySQL. I have no idea how to troubleshoot this. Any ideas what could be going wrong?
The error here is you're coding node.js as if it were procedural. It's not.
connection.connect(function(err) {
if(err) {
console.log('error when connecting to database:', err);
}
console.log('Connected to the database');
var queryString = 'SELECT * FROM blogs';
//change from connection to "this" because you're inside the context of the connection object now
this.query(queryString, function(err, rows, fields) {
if (err) throw err;
for (var i in rows) {
console.log('Post Titles: ', rows[i].id);
}
});
});
Node.js uses a series of callbacks that run when a task is completed. So when you want to do something AFTER you're connected to the DB, you run that code inside the callback.
What your code is doing is attempting to connect to the database, then while attempting to connect to the database you're querying a database you're not connected to, and so on and so forth.
For sake of illustrating the principle a little more, node functions use the following general methodology.
//1
myObj.myFunc( function( err , foo , bar ) {
//A
});
//2
myObj.myOtherFunc( function( err , someVar ) {
//B
});
1 will always run before 2. A and B may run in either order depending on when 1 and 2 finish executing. A will always run after 1 is done. B will always run after 2 is done.
Hopefully that helps clear things up ;)
As it turns out, MySQL and the app were running using the same port (3306). Changing the app's port to 3307 did the trick.

Application does not terminate with MySQL pool

I am writing a nodejs application and want to use connection pooling.
However, the following application does not terminate - although I would expect it to terminate after the call to connection.end()
Application works just fine, if I use one connection instead of the pool. Do I need to terminate the pool in some way?
Library used: https://github.com/felixge/node-mysql
node.js version: 0.10.4 on Ubuntu
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'myuser',
password : 'youbet',
database : 'notrevealingdetails',
insecureAuth: true
});
function getCampaignData(callback)
{
pool.getConnection(function(err, connection) {
if(err) throw err;
connection.query(
'SELECT cam.id, cam.name AS campaign_name, cam.subdomain, usr.email, usr.display_name AS user_displayname ' +
'FROM campaigns AS cam INNER JOIN users AS usr ON usr.id = cam.user_id ' +
'WHERE cam.state=2',
function(err, rows) {
callback(err, rows,connection);
//console.log('called end()');
}); // callback function for connection.query
}); // end pool.GetConnection
}
getCampaignData(function(err, rows, connection) {
if (err) throw err;
connection.end();
console.log("I expect my app to terminate");
});
I was having the very same problem, but looking at the source code
https://github.com/felixge/node-mysql/blob/master/lib/Pool.js
I found that the pool, at least in its current implementation, has an end() method that is turns call end() on all connections.
It also accept a callback function to be called after all connections are actually ended (or whenever an error occur).
pool.end(function (err) {
if (err) console.error("An error occurred: " + err);
else console.log("My app terminated");
});
I would use
getCampaignData(function(err, rows, connection)
{
if (err) throw err;
connection.release();
console.log("I expect my app to terminate");
});