My Node app is freezing when running a function I've written. The code will stop executing at a specific line on the 11th loop, with no error message. No further requests can be made to the app, and it must be restarted. The data is changing every time it's run, and it is always the 11th iteration.
The function it is calling is relatively complex, so I can't put the code here. My question is how do you approach diagnosing a piece of code when you see no failure notices? I assume the problem is mysql-related, as the last thing logged in the simplified version of the code below is 'Org request start 11' - no error message ever follows. But I don't understand how the code can stop executing here without entering the callback.
Example code is here:
async function processDataset(datasets){
for (let i = 0; i < datasets.length; i++) {
const dataset= datasets[i];
const orgData = await getOrgData(dataset.id);
const parsedDataset = await processDataset(dataset, orgData);
}
}
function getOrgData(datasetId) {
return new Promise(function (resolve, reject) {
console.log("Org request start", datasetId);
connection.getConnection(function (err, connection) {
console.log("got connection");
if (err) {
console.log(err);
reject(err);
}
connection.query("select * from orgs;", function (error, rows, fields) {
console.log("query returned");
connection.release();
if (error) {
console.log(error);
reject(error);
}
resolve(rows);
});
});
});
}
Running Node v12 / Express, mysql 5.5
So I found the problem was that elsewhere I had a query running that was missing a line to release the mysql connection in the second function called. Adding this resolved the bug. I don't understand why this doesn't trigger an error, or how I would have gone about diagnosing this with a more systematic approach, so would like to hear any comments.
Related
been building a website with a mysql database using the nodejs mysql package.
Stopped working on it for a couple weeks.
With no changes to code, come back and realize that the end function in mysql package appears to no longer be working.
I use an async function like so:
app.get('/', (req, res) => {
async function homeResponse(){
try{
var db = createConnection();
await conquerie.connect(db);
// get stuff from db
await conquerie.end(db);
}catch(err){
console.log(err);
res.render('error');
}
}
homeResponse();
});
conquerie js mentioned above uses:
function connect(db){
let connectpromise = new Promise(function(resolve, reject){
db.connect((err) => {
if(err){
console.log('ERROR COULD NOT CONNECT NERD');
reject(err);
}
console.log('Connected to the DB!!!');
resolve();
});
});
return connectpromise;
}
function end(db){
let endpromise = new Promise(function(resolve, reject){
db.end((err) =>{
if(err){
console.log('cant end connecty');
reject(err);
}
console.log('DB Connection ended!!!');
resolve();
});
});
return endpromise;
}
DB Connect works fine, Info pulls from DB with no issues, end does not work.
It catches an error on db.end and returns 'can't end connecty'.
Connection does not end. If I remove that catch, I get a PROTOCOL_SEQUENCE_TIMEOUT error.
I have tried async and sync, no change.
Have latest version of mysql package, and no one else online is mentioning this so It's gotta be something I did I guess.
Any Advice is much appreciated. Thanks!~
I'm trying to write some external data into some local tables. We'll be looping through an array, writing most of the data in each array element to the main table and the rest to related tables, replacing all the data each time.
I've stripped the code down to the bare bones to show the problem I'm having. The DELETE runs fine, but the INSERT runs only once, and doesn't even return.
I have a screenshot of the output at https://imgur.com/a/zA6Hz8g .
In it, you can see that the code for the DELETE runs fine (ComQueryPacket sent, OkPacket returned) but when it gets to the INSERT, the ComQueryPacket is sent but nothing is returned. And then the code just falls through.
This results in the first row writing successfully, but no subsequent rows get written.
I've tried changing the connection to use pools, but that didn't help either.
Any ideas?
var mysql = require('mysql');
var promise = require('promise');
const con = mysql.createConnection({
<connectionInfo>,
debug: true
});
function connectToDB() {
return new promise((resolve, reject) => {
console.log("IN connectToDB");
con.connect(function(err) {
if (err) {
console.log("ERROR: Could not connect -- " + err);
reject;
}
console.log("Connected!");
resolve();
});
});
}
function deleteExistingMainRow() {
return new promise((resolve, reject) => {
var query = "DELETE FROM my_table";
con.query(query, [],
function(err, result) {
if (err) {
console.log("ERROR in deleteExistingMainRow: " + err);
reject;
}
else {
console.log("DEBUG: Successful delete of main row");
resolve();
}
});
});
}
function writeMainRow(data_row) {
return new promise((resolve, reject) => {
console.log("IN writeMainRow");
var query = 'INSERT INTO my_table SET id = ?';
con.query(query, [data_row.id],
function(err, result) {
console.log("YES we tried to query");
if (err) {
console.log("ERROR in writeMainRow: " + err);
reject(err);
}
else {
console.log("DEBUG: Successful write of main row");
resolve();
}
});
});
}
exports.handler = function(event, context) {
connectToDB().then(function(script) {
deleteExistingMainRow().then(function(script) {
var data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
data.forEach(data_row => {
writeMainRow(data_row).then(function(script) {
console.log("DEBUG: Main row written in forEach");
},
function(err) {
if (err) { console.log("ERR"); } process.exit(0);
}());
});
console.log("DEBUG: Hey we're exiting now");
con.commit;
con.end(function(err) {
console.log("Error on con end: " + err);
});
context.done(null, "DONE");
process.exit(0);
});
});
};
Just a few moths ago AWS made Node.js v 8.10 runtime available in lambda.
Which means, you can use async/await and Promises. So, we can rearrange code to something like this:
exports.handler = async (event, context) => {
const dbConnection = await connectToDB();
await deleteExistingMainRow();
const data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
// use here for...of loop to keep working with async/await behaviour
for(const data_row of data){
await writeMainRow(data_row);
}
}
Also, you can rewrite your code to use native Promises or async/await functions.
And of course, cover logic on try/catch block, I've skipped them for simplicity.
The reason why your code is not behaving as you expect is because of the asynchronous nature of NodeJS.
Your for_each loop spawns several threads that are going to INSERT the data in your database.
As soon as these threads are started, the rest of the code will execute, starting with console.log("DEBUG: Hey we're exiting now");
So the commit happens before all the INSERT calls are done and, more importantly, you're calling Process.exit() in your code. This terminates the runtime, even before the INSERT can finish.
Call callback() instead as per https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
Handling your multiple asynchronous writes can be done differently. First, as grynets commented before me, I would strongly suggest to rewrite your code using async/await to make the call easier to read.
Then, you have to understand that each call to writeMainRow will return its own Promise and your code must wait for ALL promises to complete before to commit() and to callback()
Promise.all(...) will do that for you. See the doc at https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
try using
INSERT INTO table_name(id) VALUES (?);
I know both your query and the above query works the same. Just give it a Try.
And just make sure your for loop is working properly sending values to the writeMainRow(function). It wouldnt show an error you if pass an empty value and make sure you are not passing the same values in the for loop. And i think you have to pass writeMainRow(data_row.id) rather than writeMainRow(data_row).
Hope this helps.
And one more suggestion if you are updating multiple rows there are options in mysql node library like transactions. Using those functions will be more effective and you can roll back the result if you face error. Other option is to write procedures, in which case you mysql server will bear the computation.
I'm using node.js and express, also mysql.
I use a connection pool to request connections and create a promise on it, to limit callback nightmare, the following snippet is set in a file that I import later, note that that I set an handler on error to not terminate the application in case of anything going really wrong
exports.getConnection = () => {
return new Promise((resolve, reject) => {
pool.getConnection((err, connection) => {
if (err) {
reject(`Could not obtain the connection from the pool: ${err}`);
}
connection.on('error', err => {
console.log(`SQL error (code: ${err.code}, message: ${err.sqlMessage}) while executing query: ${err.sql}`);
});
resolve(connection);
});
});
};
And here is an example of usecase (the idea is to get the connection, chain the query in the then, and if a non fatal error happen I will throw it and handle the connection release in the catch handler
// Exception handler that release the connection then call the callback
function releaseConnectionHandler(err, connection, callback) {
connection.release();
callback(err, null);
}
exports.someRequest = function(ID, callback) {
sqlPool.getConnection().then(connection => {
connection.query("SELECT * from tableNotExists",
(err, result) => {
if (err) {
throw ({ err, connection, callback });
}
connection.release();
callback(null, result);
});
}).catch(({ err, connection, callback}) => releaseConnectionHandler(err, connection, callback));
};
The query will fail, but I see that the handler is not even called (I put some trace in it...) and the application terminates on
node_modules/mysql/lib/protocol/Parser.js:80
throw err; // Rethrow non-MySQL errors
Correct querie yoeld no troubles...Any ideas what I did wrong on the error handling ?
You're re-throwing the error passed to the callback of your query, which the library you're using then re-throws as well, and finally isn't properly caught and handled anywhere and results in a failure. You're not in the context of the Promise when you throw, but the context of the callback function called from the mysql module.
You're also unnecessarily mixing promises and callbacks, in particular the function you're exporting. Your question indicates that you want to move away from callbacks, so I'm going to base this answer on that indication.
To solve the main issue, don't throw the error. Instead, pass it up to the callee:
const promisify = require("util").promisify;
exports.someRequest = function (ID) {
return sqlPool.getConnection().then(connection => {
return promisify(connection.query)("select * from tableNotExist")
.finally(connection.release);
});
};
The connection will always be released back to the pool, whether successful or on error. You can then call the method with:
yourModule.someRequest(id).then((results) => {
// Do something with the result set
}).catch((e) => {
// Handle error. Can be either a pool connection error or a query error.
});
If you have the possibility to use async/await, the code can be rewritten:
const promisify = require("util").promisify;
exports.someRequest = async function (ID) {
let connection = await sqlPool.getConnection();
try {
return await promisify(connection.query)("select * from tableNotExist");
} finally {
connection.release();
}
};
I also recommend using node-mysql2 since they have a Promise-based API in addition to their callback-style API and in my experience better performance as well. Then you don't have to write these tedious wrappers and instead just require('mysql2/promise') and be good to go.
where do i close the mysql connection?
I need to run queries in sequence. I am writing code that looks like this at present:
var sqlFindMobile = "select * from user_mobiles where mobile=?";
var sqlNewUser = "insert into users (password) values (?)";
//var sqlUserId = "select last_insert_id() as user_id";
var sqlNewMobile = "insert into user_mobiles (user_id, mobile) values (?,?)";
connection.connect(function(err){});
var query = connection.query(sqlFindMobile, [req.body.mobile], function(err, results) {
if(err) throw err;
console.log("mobile query");
if(results.length==0) {
var query = connection.query(sqlNewUser, [req.body.password], function(err, results) {
if(err) throw err;
console.log("added user");
var user_id = results.insertId;
var query = connection.query(sqlNewMobile, [user_id, req.body.mobile], function(err, results) {
if(err) throw err;
console.log("added mobile");
//connection.end();
});
});
}
});
//connection.end();
(I am a beginner with node, npm-express and npm-mysql. I have tried searching SO for "express mysql cannot enqueue" to find related questions and have not found them.)
I fixed this problem use this method:
connection.end() in your connection.query function
The fixed code is here
If you're using the node-mysql module by felixge then you can call connection.end() at any point after you've made all of the connection.query() calls, since it will wait for all of the queries to finish before it terminates the connection.
See the example here for more information.
If you're wanting to run lots of queries in series, you should look into the async module, it's great for dealing with a series of asynchronous functions (i.e. those that have a callback).
Maybe the problem is that the mySQL query is executed after the connection is already closed, due to the asynchronous nature of Node. Try using this code to call connection.end() right before the thread exits:
function exitHandler(options, err) {
connection.end();
if (options.cleanup)
console.log('clean');
if (err)
console.log(err.stack);
if (options.exit)
process.exit();
}
//do something when app is closing
process.on('exit', exitHandler.bind(null, {cleanup: true}));
Code adapted from #Emil Condrea, doing a cleanup action just before node.js exits
In my case connection.end was being called in a spot that was hard to notice, so an errant call to connection.end could be the problem with this error
So I am making my first attempt with Node and I can't really wrap my head around how to work with the MySQL connection. The script is somewhat simplified like this
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '192.168.40.1',
user : 'user',
password : 'password',
database : 'database'
});
function DoSomething(connection, item, callback) {
connection.query(
'SELECT COUNT(*) AS count FROM another_table WHERE field=?',
item.field,
function (err, results) {
if (err) throw err;
if (results.length > 0 && results[0].count >= 1) {
callback(err, connection, item, 'Found something')
}
});
}
function DoSomethingElse(connection, item, callback) {
// Similar to DoSomething()
}
function StoreResult(err, connection, item, reason) {
if (err) throw err;
connection.query(
'INSERT INTO result (item_id, reason) VALUES (?, ?)',
[item.id, reason],
function (err, results) {
if (err) throw err;
});
}
connection.query('SELECT * FROM table WHERE deleted=?', [0], function (err, results)
{
if (err) throw err;
results.forEach(function (item, index) {
DoSomething(connection, item, StoreResult);
DoSomethingElse(connection, item, StoreResult);
});
});
connection.end();
What I am having trouble with (as far as I can tell) is that since DoSomething() it seems that connection.end() is called before all of the DoSomething()'s have finished causing errors that queries can't be performed when the connection is closed.
I tried playing around with the async library, but I haven't gotten anywhere so far. Anyone with some good advice on how to do this?
The problem with your code is that you're closing the connection synchronously while an asynchronous request is still being handled. You should call connection.end() only after all query callbacks have been called.
Since you are doing multiple queries, this means using some way to wait for all their results. The simplest way is to nest every next call into the callback of the previous one, but that way leads to the pyramid of doom. There are a number of popular libraries that solve this, but my own preference is for async.
Using async I would rewrite your code as follows:
async.waterfall([function(next) {
connection.query('SELECT * FROM table WHERE deleted=?', [0], next); // note the callback
},
function(results, next) {
// asynchronously handle each results. If they should be in order, use forEachSeries
async.forEach(results, function(item, next) {
// handle in parallel
async.parallel([function(done) {
DoSomething(connection, item, function(err, connection, item, reason) {
// This is a hack, StoreResult should have a callback which is called
// after it's done, because the callback is now being called too soon
StoreResult(err, connection, item, reason);
callback(err);
});
}, function(done) {
DoSomethingElse(connection, item, function(err, connection, item, reason) {
// This is a hack, StoreResult should have a callback which is called
// after it's done, because the callback is now being called too soon
StoreResult(err, connection, item, reason);
callback(err);
}], function(err) {
// this callback is called with an error as soon as it occurs
// or after all callbacks are called without error
next(err);
});
}, function(err) {
// idem
next(err);
});
}], function(err, results) {
// idem
// Do whatever you want to do with the final error here
connection.end();
});
This also allows you to solve a possible issue with the order of your queries in the forEach: They are started in order, but are not guaranteed to finish in order due to their asynchronous nature.
Close your connection after you have done everything you want in the script.
When programming in asynchronous language, keep in mind that the real ending point of your script is the last asynchronous callback, instead of the last line like other scripts (e.g. PHP).
Note that you don't want to simply ignore the connection.end(); as the underlying MySQL driver will keep the connection alive and your script will stuck in the last line forever until you kill it.
This is the modified version of your code.
connection.query('SELECT * FROM table WHERE deleted=?', [0], function (err, results)
{
if (err) throw err;
results.forEach(function (item, index) {
DoSomething(connection, item, StoreResult);
DoSomethingElse(connection, item, StoreResult);
});
// End your connection here
connection.end();
});