NodeJS: How to stop code execution till create operation to MySQL completes - mysql

I have a very basic scenario, I am making a create operation call to MySQL in my NodeJS application. Once I get result of create operation (success or failure) I have to execute some code.
But now due to asynchronous behavior of NodeJS my code which is dependent on result MySQL create operation is getting executed before MySQL create operation sends results back.
Here is my code
calculation.js
var mysql = require("mysql");
var methods = {};
// Creating connection
methods.executeQuery = function(selectQuery, values){
var result;
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "*********",
database: "******"
});
// getting connection
con.connect(function(err){
if(err){
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
con.query(selectQuery, values, function(err,rows){
if(err) throw err;
console.log(rows);
result = rows;
console.log(result);
return result;
});
}
module.exports = methods;
client.js
var execute = require("./calculate.js");
var selectQuery = 'INSERT INTO users (username,password) VALUES (?,?)';
var values = ['sohamsoham12','sohamsoham12'];
var insertedRowInfo = execute.executeQuery(selectQuery, values);
if(insertedRowInfo){
console.log("true");
}else{
console.log("false");
}

I don't know if I correctly understand your question (what is the "create operation", for example?). But...
You can try this solution: execute the query inside the connect success callback:
// getting connection and executing query
con.connect(function(err){
if(err){
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
con.query(selectQuery, values, function(err,rows){
if(err) throw err;
console.log(rows);
result = rows;
console.log(result);
return result; // this can't work... you should invoke a callback function, here...
});
});
UPDATE:
After OP comment, I now fully understand the question... (sorry :-().
You just miss a bit of async behavior... :-)
You should simply change methods.executeQuery from
function(selectQuery, values) {
...
}
to
function(selectQuery, values, callback) {
...
}
Then, instead using
return result;
You should simply use
callback(err, result); // if any error occurred
or
callback(null, result); // if no error occurred
Then, in client.js, when calling the executeQuery method, instead of
var insertedRowInfo = execute.executeQuery(selectQuery, values);
You should simply do
execute.executeQuery(selectQuery, values, function(err, insertedRowInfo) {
if (err) {
// handle error
} else {
// handle success, using insertedRowInfo...
}
});

Related

Node.js executing mysql query after receiving message from mqtt broker

I have a node.js file that subscribes to a topic and upon receiving a published message scans a local mysql db for the most recent entry in a variable named "command". Command values will trigger various responses, but I have left this portion out since my issue is before this.
My mysql query appears to be giving me errors. I am trying to look for the most recent entry of the command column and assign the value to a var command. I thought this code would do the trick:
var sql = 'SELECT command FROM motoron2 ORDER BY id DESC LIMIT 1';
con.query(sql, function (err, result) {
if (err) throw err;
});
console.log(result);
var command = result[1];
console.log(command);
But I am getting the following response which seems to indicate an error in the mysql query:
user#server.domain [bin]# node motorlistener.js
Connected to MYSQL!
Connected to Broker!
{"pulse":1}
1
/home/user/etc/domain/bin/motorlistener.js:62
console.log(result);
^
ReferenceError: result is not defined
at MqttClient.<anonymous> (/home/user/etc/domain/bin/motorlistener.js:62:17)
at MqttClient.emit (events.js:314:20)
at MqttClient._handlePublish (/home/user/node_modules/mqtt/lib/client.js:1277:12)
at MqttClient._handlePacket (/home/user/node_modules/mqtt/lib/client.js:410:12)
at work (/home/user/node_modules/mqtt/lib/client.js:321:12)
at Writable.writable._write (/home/user/node_modules/mqtt/lib/client.js:335:5)
at doWrite (/home/user/node_modules/readable-stream/lib/_stream_writable.js:409:139)
at writeOrBuffer (/home/user/node_modules/readable-stream/lib/_stream_writable.js:398:5)
at Writable.write (/home/user/node_modules/readable-stream/lib/_stream_writable.js:307:11)
at TLSSocket.ondata (_stream_readable.js:718:22)
The full code is below, but does anyone know what is causing this error?
////////////////////////////////////////////////////////////////////////////////
//setup
var mqtt = require('mqtt'); //for client use
const fs = require('fs');
var caFile = fs.readFileSync("/home/user/etc/domain/bin/ca.crt");
var topic = "heartbeat";
var mysql = require('mysql');
var con = mysql.createConnection({
host : 'localhost',
user : 'myuser',
password : 'mypass',
database : 'mydb'
});
var options={
port:8883,
clientId:"yo",
username:"myuser2",
password:"mypassw",
protocol: 'mqtts',
clean:true,
rejectUnauthorized: false,
retain:false,
ca:caFile
};
var client = mqtt.connect("http://dns.org",options);
//mqtt connection dialog
client.on("connect",function(){
console.log("Connected to Broker!");
client.subscribe(topic, {qos:1});
});
//mqtt connection handle errors
client.on("error",function(error){
console.log("Broker Connection Error");
process.exit(1);
});
//database connection
con.connect(function(err) {
if (err) throw err;
console.log("Connected to MYSQL!");
});
//handle incoming messages from broker
client.on('message',function(topic, message, packet){
var raw = ""+message;
console.log(raw);
var obj = JSON.parse(raw);
var pulse = obj.pulse;
console.log(pulse);
var sql = 'SELECT command FROM motoron2 ORDER BY id DESC LIMIT 1';
con.query(sql, function (err, result) {
if (err) throw err;
});
console.log(result);
var command = result[1];
console.log(command);
if (command == 1) {
console.log("command=1");
}
else {
console.log("command not equal to 0");
}
});
I am getting the following response which seems to indicate an error in the mysql query
That's not an error in your MySQL query. It's a null reference error because you're trying to use result outside the callback.
Changing your code to this will work:
var sql = 'SELECT command FROM motoron2 ORDER BY id DESC LIMIT 1';
con.query(sql, function (err, result) {
if (err) {
throw err;
}
// access result inside the callback
console.log(result);
var command = result[0];
console.log(command);
});
Depending on your environment you may be able to re-write your code using promises and async/await to reduce the nested scopes.
To do so, you'd need to turn your callback into a promise and then you can await it, like so:
let sql = 'SELECT command FROM motoron2 ORDER BY id DESC LIMIT 1';
// 1 -- we turn the query into a promise
const queryPromise = new Promise((resolve, reject) => {
con.query(sql, function (queryError, queryResult) {
if (queryError) {
reject(queryError);
}
resolve(queryResult);
});
});
try {
// 2 -- we can now await the promise; note the await
let result = await queryPromise;
// 3 -- now we can use the result as if it executed synchronously
console.log(result);
let command = result[0];
console.log(command);
} catch(err) {
// we can catch query errors and handle them here
}
Putting it all together, you should be able to change the on message event handler to an async function in order to take advantage of the async/await pattern as shown above:
client.on('message', async function(topic, message, packet) {
/* .. you can use await here .. */
});
All above code from #Mike Dinescu works perfectly fine. Just dont forget on the end to close the connection!
Else the runner will hangs after tests have finished.
the full solution:
async function mySqlConnect(dbquery) {
const conn = mysql.createPool({
host: 'localhost',
port: 3306,
user: 'test',
password: 'test',
database: 'test'
}, { debug: true });
// 1 -- we turn the query into a promise
const queryPromise = new Promise((resolve, reject) => {
conn.query(dbquery, function (queryError, queryResult) {
if (queryError) {
reject(queryError);
}
resolve(queryResult);
});
});
try {
// 2 -- we can now await the promise; note the await
let result = await queryPromise;
// 3 -- now we can use the result as if it executed synchronously
//console.log(result);
let command = await result[0];
//console.log(command);
return command;
} catch(err) {
}
finally{
conn.end(function(err) {
if (err) {
return console.log('error:' + err.message);
}
//console.log('Close the database connection.');
});
}
}

Connecting to MySql database from AWS Lambda function using Node.js, no connect callback

I am trying to connect an external (not AWS) MySql server from an AWS Lambda function written in Node.js using nodejs14.x environment, but the connect() callback is not called.
I am been struggling with this problem since days, there are a lot of references to similar issues but I really tried all possible permutations of solutions I found.
I am deploying with SAM and testing both on local machine and on real AWS.
Here is the sample code of the lambda helper
const mysql = require('mysql');
exports.helloFromLambdaHandler = async () => {
const message = 'Hello from Lambda!';
console.info(`${message}`);
var sql = "SELECT 1+? AS sum";
var values = [1];
console.log("Doing createConnection");
const connection = mysql.createConnection({
/* my connection data */
});
console.log("Doing connect");
connection.connect( (err) => {
console.log("Inside connection callback");
console.log('connected as id ' + connection.threadId);
if(!err) {
console.log("DB connected, thread id is " + connection.threadId);
console.log("Doing query");
connection.query(sql, values, (err, result, values) => {
console.log("Inside query callback");
if(!err) {
console.log("Query ok!");
console.log(result);
connection.end();
} else {
console.log("Error executing query: " + err.message);
}
});
} else {
console.log("Error connecting db: "+ err.message);
}
});
console.log ("Returning...");
return message;
}
The log is
Hello from Lambda!
Doing createConnection
Doing connect
Returning...
The expected behaviour is that after "Returning..." I should see the log "Inside connection callback" then "Inside query callback" and then "Query ok!".
Instead the callback of connect() appears not invoked.
I know that I can call query() directly skipping connect() but also doing so I encounter same issue.
Any clue?
Thank you!
SOLUTION
As suggested by the accepted answer, returning a promise is the solution to let Node complete all the queue. Unfortunately it's not possible to complete the Lambda and leave it running in background in a safe manner, for what I understand.
I am investigating alternative solutions such as:
mysql2 library which supports promises natively
serverless-mysql npm package which handles shared db connections
Below the running demo code
const mysql = require('mysql');
exports.helloFromLambdaHandler = async (event, context) => {
const message = 'Hello from Lambda!';
console.info(`${message}`);
var sql = "SELECT 1+? AS sum";
var values = [1];
console.log("Doing createConnection");
const connection = mysql.createConnection({
/* my connection data */
});
console.log("Doing query");
const promise = new Promise( (resolve, reject) => {
connection.query(sql, values, (err, result, values) => {
console.log("Inside query callback");
if(!err) {
console.log("Query ok!");
console.log(result);
connection.end();
resolve(message);
} else {
console.log("Error executing query: " + err.message);
reject(err);
}
});
});
console.log ("Returning...");
return promise;
}
You are using async handler, thus your function probably completes before your connect() has a chance to execute.
To try to overcome the issue, you can use Promise as shown in AWS docs.

can't set headers after they are sent return res.json

exports.saveUserInterfaceConfig = function(req,res){
var body = req.body;
console.log('body:['+JSON.stringify(body)+']');
var mysql = require('mysql');
var UiConfigId = [];
var connection = getDBConnection();
if(body && connection){
connection.beginTransaction(function(err){
if (err) {
/*var errorObj = {error:{code:0, text:'backend error'}};
return res.json(200, errorObj);*/
throw err;
}
var companyId = body.companyId;
var moduleId = body.moduleId;
var submoduleId = body.submoduleId;
var formfieldsId = body.formfieldsId;
for(var index3 in formfieldsId){
var UIConfigInfo = {Company_CompanyId: companyId, Modules_ModuleId: moduleId, SubModule_SubModuleId: submoduleId, SubmoduleFieldConfig_SubmoduleFieldConfigId: formfieldsId[index3]};
var saveUIConfigQuery = 'INSERT INTO ui_config SET ?';
connection.query(saveUIConfigQuery, UIConfigInfo, function(err, result) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
UiConfigId.push(result.insertId);
console.log('result:['+JSON.stringify(result)+']');
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
connection.end(function(err) {
// The connection is terminated now
});
throw err;
});
} else {
connection.end(function(err) {
// The connection is terminated now
});
}
return res.json(200,{UiConfigId: UiConfigId});
console.log('UiConfigId:['+JSON.stringify(UiConfigId)+']');
console.log('success!');
// connection.release();
});
})
}
})
}
}
I have the above in my Node API. I have to execute same query in loop more than once . but im facing an issue placing the return statement for which im getting the below error.
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:335:11)
How do I fix it?
you are calling res.json multiple times in a loop, that is the reason you are getting that error..
In Simple Words., This type of error will get when you pass statements or something after sending response.
for example:
res.send("something response");
console.log("jhgfjhgsdhgfsdf");
console.log("sdgsdfhdgfdhgsdf");
res.send("sopmething response");
it generates, what the error u got.!! Beccoz once the response have been sent, the following res.send Will not be executed..because, we can send response only once per a request.!!
for this you need to use the callbacks.
Good Luck
The reason you are getting that error is because you are calling res.json multiple times in a loop.
First of all, you should be using a callback mechanism to execute the query in a loop. For'ing over it can mess up by executing multiple queries before even the others are finished.
And coming to the response, it also should be done through a callback based on a condition. Condition can be to check whether you have finished all the queries successfully.
Here is a page with good info on exactly what you need:
https://mostafa-samir.github.io/async-iterative-patterns-pt1/

nodejs + mysql: how can i reopen mysql after end()?

code as follows:
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
port :"3306",
database :"mydb",
user : 'root',
password : '007007',
});
var isCon = true;
connection.connect(
function (err) {
console.log(isCon);
if (err) {
isCon=false;
console.error("error connecting :"+err);
return;
};
}
);
if(isCon){
connection.query('select * from tb_items', function(err, result) {
if (err) throw err;
console.log('The solution is: ', result);
console.log('The typeof solution is ',typeof(result));
debugger;
});
connection.end();
}
connection.connect(
function (err) {
console.log(isCon);
if (err) {
isCon=false;
console.error("error connecting :"+err);
return;
};
}
);
if(isCon){
connection.query('select * from tb_items', function(err, result) {
if (err) throw err;
console.log('The solution is: ', result);
console.log('The typeof solution is ',typeof(result));
debugger;
});
connection.end();
}
i just open()-->connect()-->query()-->end(),then did it again,but second time , there is a error : Error:Could not enqueue Handshake after invoking quiting .
question : maybe i can't reopen it after end().but i just wanna kwon ,if i end(),how can i reopen it?
no, you can't, just create another one. All state in the connection class only relevant for current connection, so having "reopenWithSameConfig" method is possible, but you should really do this from outside.
Also, you don't need to close connection on each query - just continue to reuse it without calling .end()
If you want multiple connections and automatic disposal of dead connections you should use Pool class.
One more note: your isCon check is incorrect, connection.query is called before connect() callback so it's always true. It's safe to just check error in query callback itself. If connection was not successful error is propagated to a queued command

Node.js MySQL Needing Persistent Connection

I need a persistent MySQL connection for my Node web app. The problem is that this happens about a few times a day:
Error: Connection lost: The server closed the connection.
at Protocol.end (/var/www/n/node_modules/mysql/lib/protocol/Protocol.js:73:13)
at Socket.onend (stream.js:79:10)
at Socket.EventEmitter.emit (events.js:117:20)
at _stream_readable.js:895:16
at process._tickCallback (node.js:415:13)
error: Forever detected script exited with code: 8
error: Forever restarting script for 2 time
info: socket.io started
Here is my connection code:
// Yes I know multipleStatements can be dangerous in the wrong hands.
var sql = mysql.createConnection({
host: 'localhost',
user: 'my_username',
password: 'my_password',
database: 'my_database',
multipleStatements: true
});
sql.connect();
function handleDisconnect(connection) {
connection.on('error', function(err) {
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
console.log('Re-connecting lost connection: ' + err.stack);
sql = mysql.createConnection(connection.config);
handleDisconnect(sql);
sql.connect();
});
}
handleDisconnect(sql);
As you can see, the handleDisconnect code does not work..
Use the mysql connection pool. It will reconnect when a connection dies and you get the added benefit of being able to make multiple sql queries at the same time. If you don't use the database pool, your app will block database requests while waiting for currently running database requests to finish.
I usually define a database module where I keep my queries separate from my routes. It looks something like this...
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
exports.getUsers = function(callback) {
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
var sql = "SELECT id,name FROM users";
connection.query(sql, [], function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err) {
console.log(err);
callback(true);
return;
}
callback(false, results);
});
});
});
I know this is super delayed, but I've written a solution to this that I think might be a bit more generic and usable. I had written an app entirely dependent on connection.query() and switching to a pool broke those calls.
Here's my solution:
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'user',
password : 'secret',
database : 'test',
port : 3306
});
module.exports = {
query: function(){
var sql_args = [];
var args = [];
for(var i=0; i<arguments.length; i++){
args.push(arguments[i]);
}
var callback = args[args.length-1]; //last arg is callback
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
return callback(err);
}
if(args.length > 2){
sql_args = args[1];
}
connection.query(args[0], sql_args, function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err){
console.log(err);
return callback(err);
}
callback(null, results);
});
});
}
};
This instantiates the pool once, then exports a method named query. Now, when connection.query() is called anywhere, it calls this method, which first grabs a connection from the pool, then passes the arguments to the connection. It has the added effect of grabbing the callback first, so it can callback any errors in grabbing a connection from the pool.
To use this, simply require it as module in place of mysql. Example:
var connection = require('../middleware/db');
function get_active_sessions(){
connection.query('Select * from `sessions` where `Active`=1 and Expires>?;', [~~(new Date()/1000)], function(err, results){
if(err){
console.log(err);
}
else{
console.log(results);
}
});
}
This looks just like the normal query, but actually opens a pool and grabs a connection from the pool in the background.
In response to #gladsocc question:
Is there a way to use pools without refactoring everything? I have
dozens of SQL queries in the app.
This is what I ended up building. It's a wrapper for the query function. It will grab the connection, do the query, then release the connection.
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
And I use it like I would normally.
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});