I have a function to retrieve a list of UserID's from a mysql database.
function GetUsers(callback) {
UpdateLogFile('Function Call: GetUsers()')
var users = []
Database.execute( connectionStr,
database => database.query('select UserID from Users')
.then( rows => {
for (let i = 0; i < rows.length; i++){
users.push(rows[i].UserID)
}
return callback(users)
})
).catch( err => {
console.log(err)
})
}
For Reference:
Database class which came from here
const mysql = require( 'mysql' )
class Database {
constructor( config ) {
this.connection = mysql.createConnection( config )
}
query( sql, args ) {
return new Promise( ( resolve, reject ) => {
this.connection.query( sql, args, ( err, rows ) => {
if ( err )
return reject( err )
resolve( rows )
})
})
}
close() {
return new Promise( ( resolve, reject ) => {
this.connection.end( err => {
if ( err )
return reject( err )
resolve()
})
})
}
}
Database.execute = function( config, callback ) {
const database = new Database( config )
return callback( database ).then(
result => database.close().then( () => result ),
err => database.close().then( () => { throw err } )
)
}
After hours of learning about promises and callbacks, I was finally able to get GetUsers() to at least work and return what I'm looking for. However, I seem to only be able to use it as such:
GetUsers(function(result){
// Do something with result
})
But I would really like to be able to have a traditional return statement in the function so that I could use it like this: var users = GetUsers(). I have seen posts saying that this is impossible due to the nature of asynchronous functions but I am still hopeful since I would really like to be able to avoid callback hell. I tried the code below but "users" just results as undefined after execution. So, my main goal is to be able to get the return value from GetUsers() without chaining callbacks together since I have other functions that behave similarly. Is this possible?
var users
GetUsers(function(result){
users = result
})
console.log(users)
This is a very confusing topic, and it took me a while to really understand why what you are asking simply is not possible (at least, in the exact way you are asking). For the examples I will using python Django and Node.js to compare.
Sync
def synchronous():
print('foo') //this will always print first
print('bar')
def getUsers():
with connection.cursor() as cursor:
cursor.execute('SELECT * FROM USERS') //this query is executed
users = cursor.fetchall()
print('foo') //this doesn't trigger until your server gets a response from the db, and users is defined
print(users)
Async
function asynchronous() {
console.log('foo'); //this will also always print first
console.log('bar');
}
function getUsers() {
var connection = mysql.createConnection(config);
connection.query('SELECT * FROM USERS', function(error, users) { //this is a "callback"
console.log(users); //this will print
//everything inside of here will be postponed until your server gets a response from the db
});
console.log('foo') //this will print before the console.log above
console.log(users); //this will print undefined
//this is executed before the query results are in and will be undefined since the "users" object doesn't exist yet.
}
A callback is simply the function that your server is supposed to run once you get a response. We typically use the actual word "callback" like this:
function getUsers(callback) {
var connection = mysql.createConnection(config);
connection.query('SELECT * FROM USERS', function(error, users) {
if (error) throw error; //always do your error handling on the same page as your query. Its much cleaner that way
callback(users) //server asks what to do with the "users" object you requested
});
}
Now on somewhere else on your server:
getUsers(function(users) {// the callback gets called here
console.log(users); //do what you want with users here
});
The getUsers function takes some other function (ie a callback) as its argument and executes that function after you perform your query. If you want to do the same thing without using the word "callback", you can use an await/async function like fsociety, or you explicitly write out your code and not make functions that take other functions as their arguments.
This is functionality identical to the code from above:
var connection = mysql.createConnection(config);
connection.query('SELECT * FROM USERS', function(error, users) {
if (error) throw error;
console.log(users);
});
Callback hell is inevitable, but it really Isn't too bad once you get the hang of it.
Use an async-await function instead.
async function GetUsers(callback) {
try {
UpdateLogFile('Function Call: GetUsers()')
var users = []
let rows = await Database.execute( connectionStr,
database => database.query('select UserID from Users')
for (let i = 0; i < rows.length; i++){
users.push(rows[i].UserID)
}
return callback(users)
} catch(err) {
console.log(err)
}
}
Hope this helps!
I'm trying to write some external data into some local tables. We'll be looping through an array, writing most of the data in each array element to the main table and the rest to related tables, replacing all the data each time.
I've stripped the code down to the bare bones to show the problem I'm having. The DELETE runs fine, but the INSERT runs only once, and doesn't even return.
I have a screenshot of the output at https://imgur.com/a/zA6Hz8g .
In it, you can see that the code for the DELETE runs fine (ComQueryPacket sent, OkPacket returned) but when it gets to the INSERT, the ComQueryPacket is sent but nothing is returned. And then the code just falls through.
This results in the first row writing successfully, but no subsequent rows get written.
I've tried changing the connection to use pools, but that didn't help either.
Any ideas?
var mysql = require('mysql');
var promise = require('promise');
const con = mysql.createConnection({
<connectionInfo>,
debug: true
});
function connectToDB() {
return new promise((resolve, reject) => {
console.log("IN connectToDB");
con.connect(function(err) {
if (err) {
console.log("ERROR: Could not connect -- " + err);
reject;
}
console.log("Connected!");
resolve();
});
});
}
function deleteExistingMainRow() {
return new promise((resolve, reject) => {
var query = "DELETE FROM my_table";
con.query(query, [],
function(err, result) {
if (err) {
console.log("ERROR in deleteExistingMainRow: " + err);
reject;
}
else {
console.log("DEBUG: Successful delete of main row");
resolve();
}
});
});
}
function writeMainRow(data_row) {
return new promise((resolve, reject) => {
console.log("IN writeMainRow");
var query = 'INSERT INTO my_table SET id = ?';
con.query(query, [data_row.id],
function(err, result) {
console.log("YES we tried to query");
if (err) {
console.log("ERROR in writeMainRow: " + err);
reject(err);
}
else {
console.log("DEBUG: Successful write of main row");
resolve();
}
});
});
}
exports.handler = function(event, context) {
connectToDB().then(function(script) {
deleteExistingMainRow().then(function(script) {
var data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
data.forEach(data_row => {
writeMainRow(data_row).then(function(script) {
console.log("DEBUG: Main row written in forEach");
},
function(err) {
if (err) { console.log("ERR"); } process.exit(0);
}());
});
console.log("DEBUG: Hey we're exiting now");
con.commit;
con.end(function(err) {
console.log("Error on con end: " + err);
});
context.done(null, "DONE");
process.exit(0);
});
});
};
Just a few moths ago AWS made Node.js v 8.10 runtime available in lambda.
Which means, you can use async/await and Promises. So, we can rearrange code to something like this:
exports.handler = async (event, context) => {
const dbConnection = await connectToDB();
await deleteExistingMainRow();
const data = [{ "id": 1 }, { "id": 2 }, { "id": 3 }];
// use here for...of loop to keep working with async/await behaviour
for(const data_row of data){
await writeMainRow(data_row);
}
}
Also, you can rewrite your code to use native Promises or async/await functions.
And of course, cover logic on try/catch block, I've skipped them for simplicity.
The reason why your code is not behaving as you expect is because of the asynchronous nature of NodeJS.
Your for_each loop spawns several threads that are going to INSERT the data in your database.
As soon as these threads are started, the rest of the code will execute, starting with console.log("DEBUG: Hey we're exiting now");
So the commit happens before all the INSERT calls are done and, more importantly, you're calling Process.exit() in your code. This terminates the runtime, even before the INSERT can finish.
Call callback() instead as per https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
Handling your multiple asynchronous writes can be done differently. First, as grynets commented before me, I would strongly suggest to rewrite your code using async/await to make the call easier to read.
Then, you have to understand that each call to writeMainRow will return its own Promise and your code must wait for ALL promises to complete before to commit() and to callback()
Promise.all(...) will do that for you. See the doc at https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
try using
INSERT INTO table_name(id) VALUES (?);
I know both your query and the above query works the same. Just give it a Try.
And just make sure your for loop is working properly sending values to the writeMainRow(function). It wouldnt show an error you if pass an empty value and make sure you are not passing the same values in the for loop. And i think you have to pass writeMainRow(data_row.id) rather than writeMainRow(data_row).
Hope this helps.
And one more suggestion if you are updating multiple rows there are options in mysql node library like transactions. Using those functions will be more effective and you can roll back the result if you face error. Other option is to write procedures, in which case you mysql server will bear the computation.
exports.saveUserInterfaceConfig = function(req,res){
var body = req.body;
console.log('body:['+JSON.stringify(body)+']');
var mysql = require('mysql');
var UiConfigId = [];
var connection = getDBConnection();
if(body && connection){
connection.beginTransaction(function(err){
if (err) {
/*var errorObj = {error:{code:0, text:'backend error'}};
return res.json(200, errorObj);*/
throw err;
}
var companyId = body.companyId;
var moduleId = body.moduleId;
var submoduleId = body.submoduleId;
var formfieldsId = body.formfieldsId;
for(var index3 in formfieldsId){
var UIConfigInfo = {Company_CompanyId: companyId, Modules_ModuleId: moduleId, SubModule_SubModuleId: submoduleId, SubmoduleFieldConfig_SubmoduleFieldConfigId: formfieldsId[index3]};
var saveUIConfigQuery = 'INSERT INTO ui_config SET ?';
connection.query(saveUIConfigQuery, UIConfigInfo, function(err, result) {
if (err) {
return connection.rollback(function() {
throw err;
});
}
UiConfigId.push(result.insertId);
console.log('result:['+JSON.stringify(result)+']');
connection.commit(function(err) {
if (err) {
return connection.rollback(function() {
connection.end(function(err) {
// The connection is terminated now
});
throw err;
});
} else {
connection.end(function(err) {
// The connection is terminated now
});
}
return res.json(200,{UiConfigId: UiConfigId});
console.log('UiConfigId:['+JSON.stringify(UiConfigId)+']');
console.log('success!');
// connection.release();
});
})
}
})
}
}
I have the above in my Node API. I have to execute same query in loop more than once . but im facing an issue placing the return statement for which im getting the below error.
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:335:11)
How do I fix it?
you are calling res.json multiple times in a loop, that is the reason you are getting that error..
In Simple Words., This type of error will get when you pass statements or something after sending response.
for example:
res.send("something response");
console.log("jhgfjhgsdhgfsdf");
console.log("sdgsdfhdgfdhgsdf");
res.send("sopmething response");
it generates, what the error u got.!! Beccoz once the response have been sent, the following res.send Will not be executed..because, we can send response only once per a request.!!
for this you need to use the callbacks.
Good Luck
The reason you are getting that error is because you are calling res.json multiple times in a loop.
First of all, you should be using a callback mechanism to execute the query in a loop. For'ing over it can mess up by executing multiple queries before even the others are finished.
And coming to the response, it also should be done through a callback based on a condition. Condition can be to check whether you have finished all the queries successfully.
Here is a page with good info on exactly what you need:
https://mostafa-samir.github.io/async-iterative-patterns-pt1/
I have a python background and is currently migrating to node.js. I have problem adjusting to node.js due to its asynchronous nature.
For example, I am trying to return a value from a MySQL function.
function getLastRecord(name)
{
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
console.log(err);
logger.info(err);
}
else {
//console.log(rows);
return rows;
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
}
var rows = getLastRecord('name_record');
console.log(rows);
After some reading up, I realize the above code cannot work and I need to return a promise due to node.js's asynchronous nature. I cannot write node.js code like python. How do I convert getLastRecord() to return a promise and how do I handle the returned value?
In fact, what I want to do is something like this;
if (getLastRecord() > 20)
{
console.log("action");
}
How can this be done in node.js in a readable way?
I would like to see how promises can be implemented in this case using bluebird.
This is gonna be a little scattered, forgive me.
First, assuming this code uses the mysql driver API correctly, here's one way you could wrap it to work with a native promise:
function getLastRecord(name)
{
return new Promise(function(resolve, reject) {
// The Promise constructor should catch any errors thrown on
// this tick. Alternately, try/catch and reject(err) on catch.
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
connection.query(query_str, query_var, function (err, rows, fields) {
// Call reject on error states,
// call resolve with results
if (err) {
return reject(err);
}
resolve(rows);
});
});
}
getLastRecord('name_record').then(function(rows) {
// now you have your rows, you can see if there are <20 of them
}).catch((err) => setImmediate(() => { throw err; })); // Throw async to escape the promise chain
So one thing: You still have callbacks. Callbacks are just functions that you hand to something to call at some point in the future with arguments of its choosing. So the function arguments in xs.map(fn), the (err, result) functions seen in node and the promise result and error handlers are all callbacks. This is somewhat confused by people referring to a specific kind of callback as "callbacks," the ones of (err, result) used in node core in what's called "continuation-passing style", sometimes called "nodebacks" by people that don't really like them.
For now, at least (async/await is coming eventually), you're pretty much stuck with callbacks, regardless of whether you adopt promises or not.
Also, I'll note that promises aren't immediately, obviously helpful here, as you still have a callback. Promises only really shine when you combine them with Promise.all and promise accumulators a la Array.prototype.reduce. But they do shine sometimes, and they are worth learning.
I have modified your code to use Q(NPM module) promises.
I Assumed your 'getLastRecord()' function that you specified in above snippet works correctly.
You can refer following link to get hold of Q module
Click here : Q documentation
var q = require('q');
function getLastRecord(name)
{
var deferred = q.defer(); // Use Q
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
deferred.reject(err);
}
else {
//console.log(rows);
deferred.resolve(rows);
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
return deferred.promise;
}
// Call the method like this
getLastRecord('name_record')
.then(function(rows){
// This function get called, when success
console.log(rows);
},function(error){
// This function get called, when error
console.log(error);
});
I am new to Node.js and promises. I was searching for a while for something that will meet my needs and this is what I ended up using after combining several examples I found. I wanted the ability to acquire connection per query and release it right after the query finishes (querySql), or to get a connection from pool and use it within Promise.using scope, or release it whenever I would like it (getSqlConnection).
Using this method you can concat several queries one after another without nesting them.
db.js
var mysql = require('mysql');
var Promise = require("bluebird");
Promise.promisifyAll(mysql);
Promise.promisifyAll(require("mysql/lib/Connection").prototype);
Promise.promisifyAll(require("mysql/lib/Pool").prototype);
var pool = mysql.createPool({
host: 'my_aws_host',
port: '3306',
user: 'my_user',
password: 'my_password',
database: 'db_name'
});
function getSqlConnection() {
return pool.getConnectionAsync().disposer(function (connection) {
console.log("Releasing connection back to pool")
connection.release();
});
}
function querySql (query, params) {
return Promise.using(getSqlConnection(), function (connection) {
console.log("Got connection from pool");
if (typeof params !== 'undefined'){
return connection.queryAsync(query, params);
} else {
return connection.queryAsync(query);
}
});
};
module.exports = {
getSqlConnection : getSqlConnection,
querySql : querySql
};
usage_route.js
var express = require('express');
var router = express.Router();
var dateFormat = require('dateformat');
var db = require('../my_modules/db');
var getSqlConnection = db.getSqlConnection;
var querySql = db.querySql;
var Promise = require("bluebird");
function retrieveUser(token) {
var userQuery = "select id, email from users where token = ?";
return querySql(userQuery, [token])
.then(function(rows){
if (rows.length == 0) {
return Promise.reject("did not find user");
}
var user = rows[0];
return user;
});
}
router.post('/', function (req, res, next) {
Promise.resolve().then(function () {
return retrieveUser(req.body.token);
})
.then(function (user){
email = user.email;
res.status(200).json({ "code": 0, "message": "success", "email": email});
})
.catch(function (err) {
console.error("got error: " + err);
if (err instanceof Error) {
res.status(400).send("General error");
} else {
res.status(200).json({ "code": 1000, "message": err });
}
});
});
module.exports = router;
I am still a bit new to node, so maybe I missed something let me know how it works out. Instead of triggering async node just forces it on you, so you have to think ahead and plan it.
const mysql = require('mysql');
const db = mysql.createConnection({
host: 'localhost',
user: 'user', password: 'password',
database: 'database',
});
db.connect((err) => {
// you should probably add reject instead of throwing error
// reject(new Error());
if(err){throw err;}
console.log('Mysql: Connected');
});
db.promise = (sql) => {
return new Promise((resolve, reject) => {
db.query(sql, (err, result) => {
if(err){reject(new Error());}
else{resolve(result);}
});
});
};
Here I am using the mysql module like normal, but instead I created a new function to handle the promise ahead of time, by adding it to the db const. (you see this as "connection" in a lot of node examples.
Now lets call a mysql query using the promise.
db.promise("SELECT * FROM users WHERE username='john doe' LIMIT 1;")
.then((result)=>{
console.log(result);
}).catch((err)=>{
console.log(err);
});
What I have found this useful for is when you need to do a second query based on the first query.
db.promise("SELECT * FROM users WHERE username='john doe' LIMIT 1;")
.then((result)=>{
console.log(result);
var sql = "SELECT * FROM friends WHERE username='";
sql = result[0];
sql = "';"
return db.promise(sql);
}).then((result)=>{
console.log(result);
}).catch((err)=>{
console.log(err);
});
You should actually use the mysql variables, but this should at least give you an example of using promises with mysql module.
Also with above you can still continue to use the db.query the normal way anytime within these promises, they just work like normal.
Hope this helps with the triangle of death.
You don't need to use promises, you can use a callback function, something like that:
function getLastRecord(name, next)
{
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
console.log(err);
logger.info(err);
next(err);
}
else {
//console.log(rows);
next(null, rows);
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
}
getLastRecord('name_record', function(err, data) {
if(err) {
// handle the error
} else {
// handle your data
}
});
Using the package promise-mysql the logic would be to chain promises using then(function(response){your code})
and
catch(function(response){your code}) to catch errors from the "then" blocks preceeding the catch block.
Following this logic, you will pass query results in objects or arrays using return at the end of the block. The return will help passing the query results to the next block. Then, the result will be found in the function argument (here it is test1). Using this logic you can chain several MySql queries and the code that is required to manipulate the result and do whatever you want.
the Connection object is created to be global because every object and variable created in every block are only local. Don't forget that you can chain more "then" blocks.
var config = {
host : 'host',
user : 'user',
password : 'pass',
database : 'database',
};
var mysql = require('promise-mysql');
var connection;
let thename =""; // which can also be an argument if you embed this code in a function
mysql.createConnection(config
).then(function(conn){
connection = conn;
let test = connection.query('select name from records WHERE name=? LIMIT 1',[thename]);
return test;
}).then(function(test1){
console.log("test1"+JSON.stringify(test1)); // result of previous block
var result = connection.query('select * from users'); // A second query if you want
connection.end();
connection = {};
return result;
}).catch(function(error){
if (connection && connection.end) connection.end();
//logs out the error from the previous block (if there is any issue add a second catch behind this one)
console.log(error);
});
To answer your initial question: How can this be done in node.js in a readable way?
There is a library called co, which gives you the possibility to write async code in a synchronous workflow. Just have a look and npm install co.
The problem you face very often with that approach, is, that you do not get Promise back from all the libraries you like to use. So you have either wrap it yourself (see answer from #Joshua Holbrook) or look for a wrapper (for example: npm install mysql-promise)
(Btw: its on the roadmap for ES7 to have native support for this type of workflow with the keywords async await, but its not yet in node: node feature list.)
This can be achieved quite simply, for example with bluebird, as you asked:
var Promise = require('bluebird');
function getLastRecord(name)
{
return new Promise(function(resolve, reject){
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
console.log(err);
logger.info(err);
reject(err);
}
else {
resolve(rows);
//console.log(rows);
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
});
}
getLastRecord('name_record')
.then(function(rows){
if (rows > 20) {
console.log("action");
}
})
.error(function(e){console.log("Error handler " + e)})
.catch(function(e){console.log("Catch handler " + e)});
May be helpful for others, extending #Dillon Burnett answer
Using async/await and params
db.promise = (sql, params) => {
return new Promise((resolve, reject) => {
db.query(sql,params, (err, result) => {
if(err){reject(new Error());}
else{resolve(result);}
});
});
};
module.exports = db;
async connection(){
const result = await db.promise("SELECT * FROM users WHERE username=?",[username]);
return result;
}
I'm playing with a Node.JS app for the first time, and so far I love it, but I'm having a problem...
I have written a wrapper function on the client.query function that simply allows me to pass a parameter and it gets the query from an array of allowed queries...
var runProcedure = function(proc, vars){
var params;
if(typeof vars != 'undefined'){
params.concat(eval('(' + vars + ')'));
}
this._client.query(queries[proc], params, function(err, results, fields){
if(err) { throw err; }
if(results){
return(results);
}else{
return "No data found.";
}
});
}
The function works correctly and if I console.log results, the data I want is there.. however, it's not getting returned to where I called it...
var data = runProcedure(procedureName, parameters);
console.log(data); // undefined
While troubleshooting, it seems that the query function is run asynchronously.... but this causes me a big problem. The runProcedure function is being called from within an http request handler.. so I need to be able to access the response variable. I guess I could pass it all the way down as a parameter... but that seems clumsy. What is the best code pattern to handle this? Should I set the response as a global var? can I run the mysql synchronously?
Cheers,
whiteatom
just pass your data to callback instead of returning with return
var runProcedure = function(proc, vars, callback) {
var params;
if(typeof vars != 'undefined'){
params.concat(eval('(' + vars + ')'));
}
this._client.query(queries[proc], params, function(err, results, fields){
if(err) { throw err; }
if(results){
callback(results);
}else{
callback('No data found.');
}
});
}
Now in http request handler:
// ...
function httpRequestHandler(req, resp)
{
//...
runProcedure(proc, vars, function(dbResp) {
if (dbResp === 'No data found.')
{
// handle error
} else {
// do something with result
}
})
}