I'm using nodejs with mysql and fs to query a database and modify the results by adding a text file to each row. I realize I need to use a promise here but I can't quite get it. I can't figure out what I'm doing wrong. Here is my code so far:
db.query(sql, params, function(err, rows) {
if(err) {
res.status(400).json({"error": err.message});
return;
}
var data = {};
var p = new Promise(function(resolve, reject) {
rows.forEach(function(row) {
var filePath = `${contentdir}/${row.strTicketNumber}-${row.strTicketRevision}.txt`;
row.fileTxt = fs.readFile(filePath, 'utf8', function(error, content) { if(error) return ""; return content; });
data[row.intSerial] = row;
});
resolve(data);
});
p.then(function() {
res.json( data );
})
});
You can save yourself using a Promise here if you struggle with them by using synchronous Node APIs:
db.query(sql, params, function(err, rows) {
if(err) {
res.status(400).json({"error": err.message});
return;
}
var data = {};
rows.forEach(function(row) {
var filePath = `${contentdir}/${row.strTicketNumber}-${row.strTicketRevision}.txt`;
row.fileTxt = fs.readFileSync(filePath, 'utf8');
data[row.intSerial] = row;
});
res.json( data );
});
Note readFileSync instead of readFile
Or, if you really want to use Promises, you can do it like so:
db.query(sql, params, function(err, rows) {
if(err) {
res.status(400).json({"error": err.message});
return;
}
var data = {};
var promises = []
rows.forEach(function(row) {
var filePath = `${contentdir}/${row.strTicketNumber}-${row.strTicketRevision}.txt`;
promises.push(new Promise((resolve, reject) => {
fs.readFile(filePath, 'utf8', (err, content) => {
if (err) reject(err)
row.fileTxt = content
data[row.intSerial] = row
resolve()
});
}))
});
Promise.all(promises).then(() => {
res.json( data );
})
});
The main difference between these snippets is that no. 1 reads each text file one-by-one synchronously, while no. 2 reads them "at the same time" (with a few asterisks :p)
Whether this matters to you will depend on how many files you'll be reading, as well as whether you find it significantly easier to read & maintain the first example until you are more comfortable with Promises.
I want the result on other file but not getting how to return it
function getUsers(){
console.log("Fetching all user data");
const connection = getConnection();
const sql = "SELECT * FROM users";
var result = connection.query(sql,(err, rows, fields) =>{
if(err){
console.log("Failed to get users data");
res.sendStatus(500);
throw err;
}
console.log("Fetched Users Successfully");
return rows;
})
return result;
}
This is a standard asynchronous function problem.
return rows will not assign rows to the result variable. Second argument of query() function is a callback function and it indicates a block of code that will be executed after rows are fetched. It means that result of the query is only visible inside callback block.
You can fix it in 2 ways:
Write all your code that uses SQL result inside callback function block (You can call another function at that point and pass result as argument for example).
This method is simpler if you are new to javascript programming. I see that you want to use function getUsers() in another place. If you choose this approach, best way would probably be to pass a callback to the getUsers() function and then invoke it in the callback of the query() method.
Example:
getUsers(callback){
const connection = getConnection();
const sql = "SELECT * FROM users";
connection.query(sql,(err, rows, fields) => callback(rows))
}
mainFunction(req,res){
getUsers(doSomethingWithUsers)
}
doSomethingWithUsers(users){
...
}
This is of course, oversimplified pattern, but the idea should be clear.
Research about promises and how they handle async functions. This will require you to "wrap" you function body into Promise object type.
They will allow you to write something like this:
mainFunction(){
getUsers().then(result => ....);
// or even
const users = await getUsers();
}
EDIT:
To wrap "return value of promise", or as we say "to resolve a promise", you can wrap function the following way:
function getUsers(){
console.log("Fetching all user data");
const connection = getConnection();
const sql = "SELECT * FROM users";
return new Promise( (resolve,reject) => {
var result = connection.query(sql,(err, rows, fields) =>{
if(err){
// this will cause promise to "fail"
reject(err);
}
console.log("Fetched Users Successfully");
// this will tell javascript that promise is finished
// and rows are accesable in then()
resolve(rows);
})
})
}
function mainFunction(){
const result = getUsers()
.then(rows =>{/** this will happen if resolve(rows) is called */})
.catch(error => {/** this will be executed if reject(err) happens
*/});
}
// or... (Try/Catch is required since "await" silently fails
async function asyncMainFunction(){
try{
const rows = await getUsers();
// You can use rows from resolve() here;
}catch(error){
// here, reject is called
}
}
Here is a link where you can read more about promises:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
You are not getting the result because the code is asynchronous. So, you have to use async/await to get the result. Your code should look like this:-
async function getUsers(){
console.log("Fetching all user data");
const connection = await getConnection();
const sql = "SELECT * FROM users";
try {
var result = await connection.query(sql);
console.log("Fetched Users Successfully");
return result;
} catch (err) {
console.log("Failed to get users data");
res.sendStatus(500);
throw err;
}
}
Finally, call this function like this:-
var output = await getUsers();
But make sure that you call this line in an async function.
Hope this helps!
create file for function : getUsers.js
module.exports = async getUsers(){
const connection = getConnection();
const sql = "SELECT * FROM users";
var myPromise = () => {
return new Promise((resolve, reject) => {
connection.query(sql, query.params, function (error, results, fields) {
error ? reject(error) : resolve(results);
})
});
}
var result = await (myPromise());
return result;
}
require this file and call function :
var getUsers = require('..path to getUsers');
var data = getUsers().then((data)=>{
}).catch((error)=>{
console.log('error',error)
return error;
});
using callback :
or using callback function :
module.exports = function getUsers(callback){
const connection = getConnection();
const sql = "SELECT * FROM users";
connection.query(sql, query.params, function (error, results, fields) {
if(error){ callback(error,null)}
if(!error && results){
callback(null,results)
}
});
}
require file :
var getUsers = require('..path to getUsers');
call it this way :
getUsers(err,data)=>{
if(err){
return res.send(err);
}
if(!err && data){
return res.send(data);
}
}
function getUsers(callback) {
console.log("Fetching all user data");
const connection = getConnection();
const sql = "SELECT * FROM users";
connection.query(sql, (err, rows, fields) => {
if (err) {
console.log("Failed to get users data");
res.sendStatus(500);
callback(err, null);
}
console.log("Fetched Users Successfully");
callback(null, rows)
})
}
// USE ABOVE FUNCTION AS BELOW...
getUsers(function(err, rows) {
if(err){
console.error(err);
}
console.log(rows);
})
I have written this piece of code that retrieves a list of contacts and in response show a JSON everything works fine but the code doesn't show any output what should I have done? this is my code:
export const ContactListPost = (req, res) => {
//const userHeader = req.get('userid');
const contacts = req.body;
console.log(contacts);
let users = [];
const delay = contact => new Promise(function (resolve, reject) {
let sql = "select user.userid, user.fullname, user.phone,
photos.name as photo_link " +
"from user left outer join photos on photos.uploader_id = user.userid where user.phone = ?";
console.log("contact " + contact);
con.query(sql, [contact], function (err, result) {
if (err){
console.log(err);
reject(err);
} else{
if (result.length > 0) {
users.push(result);
console.log(result);
resolve(result);
}else{
console.log("empty number");
//reject(result);
}
}
})
});
(async function loop() {
for (let i = 0; i< contacts.length ; i++){
let result = await delay(contacts[i].number);
users.push(result);
console.log("result " + JSON.stringify(result));
console.log("i " + i);
}
console.log("users " + JSON.stringify(users));
res.json(JSON.stringify(users));
})();
};
this piece of code doesn't show
console.log("users " + JSON.stringify(users));
res.json(JSON.stringify(users));
thanks for advice
If the promise is rejected, the await expression throws the rejected value.
You need to handle rejected promises. If one of the calls to delay returns a rejected promise, the rejected value is thrown and the execution of the function will stop and control will be passed to the first catch block in the call stack.
await operator
throw statement
I Have two mysql queries that runs with promise.
The first one is updates information on a mysql table and then resolves the issue and calls the next mysql query. The problem is that, when it calls the next mysql query the promise returns UNDEFINED and I am not sure why. When I console.log it out in my node js server post request, it gives undefined. I documented on the code which areas are problems.
UpdateUserPath = (data) => new Promise((resolve,reject)=>{
data.UPDATE_DT = getDateTime();
db.query('UPDATE path UPDATE_DT = ? where Owner = ?',
[data.UPDATE_DT, data.Owner], function(err,results,fields){
if(err){
reject('Could not update user path');
}else{
if(results.affectedRows > 0){
data.ID = null;
data.UPDATE_DT = null;
// The problem is here, when this gets resolved it calls the other function SaveUserPath
resolve(saveUserPath(data));
}else{
reject('Could not update user path');
}
}
});
});
saveUserPath = (data) => new Promise((resolve, reject) => {
db.query('INSERT INTO path SET ?', data, function (error, results, fields) {
if (error) {
reject('Could not insert path');
}else{
var Id = results.insertId;
db.query('UPDATE path SET ORIG_ID = ? where ID = ?',[Id, Id], function(err,results,fields){
if(err){
reject('Could not insert row to path table - saveuserpath');
}else{
if(results.affectedRows > 0){
// THIS INFORMATION HERE IS UNDEFINED
return resolve(results[0]);
}else{
reject('Could not update path');
}
}
});
}
});
});
In the server it gets called like this.
getUserPath(req.session.userid).then((path_data)=>{
path_data.status = 1;
UpdateUserPath(path_data).then((result)=>{
console.log(result); // THIS IS UNDEFINED
});
});
I am wondering if resolve(saveUserPath(data)); is the right way to call another promise which is not outside in the server.
I was thinking of just doing it this way.
UpdateUserPath(path_data).then((result)=>{
saveUserPath(result).then((result_save) => {
console.log(result_save); // THIS MIGHT WORK
});
});
But why is the normal way wrong.
I have several guesses why it isn't working, but there are a number of things wrong such that it's better to just clean up the code to a much better design.
When combining multiple asynchronous callback-driven operations in an otherwise promise-based interface, you really want to promisify the underlying functions at their lowest level and then you can implement all your control flow and error handling using the benefits of promises. I think that will also make your problem go away and probably fix a couple other bugs too.
// promisify db.query()
// if a promisified interface is built into your database, use that one instead
db.queryP = function(q, d) {
return new Promise((resolve, reject) {
db.query(q, d, (err, results, fields) => {
if (err) {
reject(err);
} else {
resolve(results);
}
});
});
}
UpdateUserPath = function(data) {
data.UPDATE_DT = getDateTime();
let q = 'UPDATE path UPDATE_DT = ? where Owner = ?';
return db.queryP(q, [data.UPDATE_DT, data.Owner]).then(results => {
if (results.affectedRows > 0) {
data.ID = null;
data.UPDATE_DT = null;
return saveUserPath(data);
} else {
throw new Error('Could not update user path');
}
});
}
saveUserPath = function(data) {
let q = 'INSERT INTO path SET ?'
return db.queryP(q, data).then(results => {
let q2 = 'UPDATE path SET ORIG_ID = ? where ID = ?';
var Id = results.insertId;
return db.queryP(q2, [Id, Id]).then(results2 => {
if (results2.affectedRows > 0) {
return results2[0];
} else {
throw new Error('Could not update path');
}
});
});
}
getUserPath(req.session.userid).then(path_data => {
path_data.status = 1;
return UpdateUserPath(path_data);
}).then(result => {
// process result here
}).catch(err => {
// process error here
});
I have a python background and is currently migrating to node.js. I have problem adjusting to node.js due to its asynchronous nature.
For example, I am trying to return a value from a MySQL function.
function getLastRecord(name)
{
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
console.log(err);
logger.info(err);
}
else {
//console.log(rows);
return rows;
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
}
var rows = getLastRecord('name_record');
console.log(rows);
After some reading up, I realize the above code cannot work and I need to return a promise due to node.js's asynchronous nature. I cannot write node.js code like python. How do I convert getLastRecord() to return a promise and how do I handle the returned value?
In fact, what I want to do is something like this;
if (getLastRecord() > 20)
{
console.log("action");
}
How can this be done in node.js in a readable way?
I would like to see how promises can be implemented in this case using bluebird.
This is gonna be a little scattered, forgive me.
First, assuming this code uses the mysql driver API correctly, here's one way you could wrap it to work with a native promise:
function getLastRecord(name)
{
return new Promise(function(resolve, reject) {
// The Promise constructor should catch any errors thrown on
// this tick. Alternately, try/catch and reject(err) on catch.
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
connection.query(query_str, query_var, function (err, rows, fields) {
// Call reject on error states,
// call resolve with results
if (err) {
return reject(err);
}
resolve(rows);
});
});
}
getLastRecord('name_record').then(function(rows) {
// now you have your rows, you can see if there are <20 of them
}).catch((err) => setImmediate(() => { throw err; })); // Throw async to escape the promise chain
So one thing: You still have callbacks. Callbacks are just functions that you hand to something to call at some point in the future with arguments of its choosing. So the function arguments in xs.map(fn), the (err, result) functions seen in node and the promise result and error handlers are all callbacks. This is somewhat confused by people referring to a specific kind of callback as "callbacks," the ones of (err, result) used in node core in what's called "continuation-passing style", sometimes called "nodebacks" by people that don't really like them.
For now, at least (async/await is coming eventually), you're pretty much stuck with callbacks, regardless of whether you adopt promises or not.
Also, I'll note that promises aren't immediately, obviously helpful here, as you still have a callback. Promises only really shine when you combine them with Promise.all and promise accumulators a la Array.prototype.reduce. But they do shine sometimes, and they are worth learning.
I have modified your code to use Q(NPM module) promises.
I Assumed your 'getLastRecord()' function that you specified in above snippet works correctly.
You can refer following link to get hold of Q module
Click here : Q documentation
var q = require('q');
function getLastRecord(name)
{
var deferred = q.defer(); // Use Q
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
deferred.reject(err);
}
else {
//console.log(rows);
deferred.resolve(rows);
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
return deferred.promise;
}
// Call the method like this
getLastRecord('name_record')
.then(function(rows){
// This function get called, when success
console.log(rows);
},function(error){
// This function get called, when error
console.log(error);
});
I am new to Node.js and promises. I was searching for a while for something that will meet my needs and this is what I ended up using after combining several examples I found. I wanted the ability to acquire connection per query and release it right after the query finishes (querySql), or to get a connection from pool and use it within Promise.using scope, or release it whenever I would like it (getSqlConnection).
Using this method you can concat several queries one after another without nesting them.
db.js
var mysql = require('mysql');
var Promise = require("bluebird");
Promise.promisifyAll(mysql);
Promise.promisifyAll(require("mysql/lib/Connection").prototype);
Promise.promisifyAll(require("mysql/lib/Pool").prototype);
var pool = mysql.createPool({
host: 'my_aws_host',
port: '3306',
user: 'my_user',
password: 'my_password',
database: 'db_name'
});
function getSqlConnection() {
return pool.getConnectionAsync().disposer(function (connection) {
console.log("Releasing connection back to pool")
connection.release();
});
}
function querySql (query, params) {
return Promise.using(getSqlConnection(), function (connection) {
console.log("Got connection from pool");
if (typeof params !== 'undefined'){
return connection.queryAsync(query, params);
} else {
return connection.queryAsync(query);
}
});
};
module.exports = {
getSqlConnection : getSqlConnection,
querySql : querySql
};
usage_route.js
var express = require('express');
var router = express.Router();
var dateFormat = require('dateformat');
var db = require('../my_modules/db');
var getSqlConnection = db.getSqlConnection;
var querySql = db.querySql;
var Promise = require("bluebird");
function retrieveUser(token) {
var userQuery = "select id, email from users where token = ?";
return querySql(userQuery, [token])
.then(function(rows){
if (rows.length == 0) {
return Promise.reject("did not find user");
}
var user = rows[0];
return user;
});
}
router.post('/', function (req, res, next) {
Promise.resolve().then(function () {
return retrieveUser(req.body.token);
})
.then(function (user){
email = user.email;
res.status(200).json({ "code": 0, "message": "success", "email": email});
})
.catch(function (err) {
console.error("got error: " + err);
if (err instanceof Error) {
res.status(400).send("General error");
} else {
res.status(200).json({ "code": 1000, "message": err });
}
});
});
module.exports = router;
I am still a bit new to node, so maybe I missed something let me know how it works out. Instead of triggering async node just forces it on you, so you have to think ahead and plan it.
const mysql = require('mysql');
const db = mysql.createConnection({
host: 'localhost',
user: 'user', password: 'password',
database: 'database',
});
db.connect((err) => {
// you should probably add reject instead of throwing error
// reject(new Error());
if(err){throw err;}
console.log('Mysql: Connected');
});
db.promise = (sql) => {
return new Promise((resolve, reject) => {
db.query(sql, (err, result) => {
if(err){reject(new Error());}
else{resolve(result);}
});
});
};
Here I am using the mysql module like normal, but instead I created a new function to handle the promise ahead of time, by adding it to the db const. (you see this as "connection" in a lot of node examples.
Now lets call a mysql query using the promise.
db.promise("SELECT * FROM users WHERE username='john doe' LIMIT 1;")
.then((result)=>{
console.log(result);
}).catch((err)=>{
console.log(err);
});
What I have found this useful for is when you need to do a second query based on the first query.
db.promise("SELECT * FROM users WHERE username='john doe' LIMIT 1;")
.then((result)=>{
console.log(result);
var sql = "SELECT * FROM friends WHERE username='";
sql = result[0];
sql = "';"
return db.promise(sql);
}).then((result)=>{
console.log(result);
}).catch((err)=>{
console.log(err);
});
You should actually use the mysql variables, but this should at least give you an example of using promises with mysql module.
Also with above you can still continue to use the db.query the normal way anytime within these promises, they just work like normal.
Hope this helps with the triangle of death.
You don't need to use promises, you can use a callback function, something like that:
function getLastRecord(name, next)
{
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
console.log(err);
logger.info(err);
next(err);
}
else {
//console.log(rows);
next(null, rows);
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
}
getLastRecord('name_record', function(err, data) {
if(err) {
// handle the error
} else {
// handle your data
}
});
Using the package promise-mysql the logic would be to chain promises using then(function(response){your code})
and
catch(function(response){your code}) to catch errors from the "then" blocks preceeding the catch block.
Following this logic, you will pass query results in objects or arrays using return at the end of the block. The return will help passing the query results to the next block. Then, the result will be found in the function argument (here it is test1). Using this logic you can chain several MySql queries and the code that is required to manipulate the result and do whatever you want.
the Connection object is created to be global because every object and variable created in every block are only local. Don't forget that you can chain more "then" blocks.
var config = {
host : 'host',
user : 'user',
password : 'pass',
database : 'database',
};
var mysql = require('promise-mysql');
var connection;
let thename =""; // which can also be an argument if you embed this code in a function
mysql.createConnection(config
).then(function(conn){
connection = conn;
let test = connection.query('select name from records WHERE name=? LIMIT 1',[thename]);
return test;
}).then(function(test1){
console.log("test1"+JSON.stringify(test1)); // result of previous block
var result = connection.query('select * from users'); // A second query if you want
connection.end();
connection = {};
return result;
}).catch(function(error){
if (connection && connection.end) connection.end();
//logs out the error from the previous block (if there is any issue add a second catch behind this one)
console.log(error);
});
To answer your initial question: How can this be done in node.js in a readable way?
There is a library called co, which gives you the possibility to write async code in a synchronous workflow. Just have a look and npm install co.
The problem you face very often with that approach, is, that you do not get Promise back from all the libraries you like to use. So you have either wrap it yourself (see answer from #Joshua Holbrook) or look for a wrapper (for example: npm install mysql-promise)
(Btw: its on the roadmap for ES7 to have native support for this type of workflow with the keywords async await, but its not yet in node: node feature list.)
This can be achieved quite simply, for example with bluebird, as you asked:
var Promise = require('bluebird');
function getLastRecord(name)
{
return new Promise(function(resolve, reject){
var connection = getMySQL_connection();
var query_str =
"SELECT name, " +
"FROM records " +
"WHERE (name = ?) " +
"LIMIT 1 ";
var query_var = [name];
var query = connection.query(query_str, query_var, function (err, rows, fields) {
//if (err) throw err;
if (err) {
//throw err;
console.log(err);
logger.info(err);
reject(err);
}
else {
resolve(rows);
//console.log(rows);
}
}); //var query = connection.query(query_str, function (err, rows, fields) {
});
}
getLastRecord('name_record')
.then(function(rows){
if (rows > 20) {
console.log("action");
}
})
.error(function(e){console.log("Error handler " + e)})
.catch(function(e){console.log("Catch handler " + e)});
May be helpful for others, extending #Dillon Burnett answer
Using async/await and params
db.promise = (sql, params) => {
return new Promise((resolve, reject) => {
db.query(sql,params, (err, result) => {
if(err){reject(new Error());}
else{resolve(result);}
});
});
};
module.exports = db;
async connection(){
const result = await db.promise("SELECT * FROM users WHERE username=?",[username]);
return result;
}