Node.js Promise concept during if else condition in then block - mysql

I have a scenario where the webservice needs to check for existense of key in redis if present give it as a response else get it from mysql, store in redis and then give it as response.
So i am using promises concept where first time when i call return new Set_Data(); it doesn't go to next then block it just stays idle. But next time since data already exists the return new Set_Data();
is not executed which is correct.
But why is that i am getting problem for first time when i call return new Set_Data(); which is not going for next then block.
Below is my code
constants.js file
var Promise = require('bluebird');
module.exports =
{
getRedisConnection: function ()
{
return require("redis").createClient(6379, 'path', { auth_pass: 'key' });
},
getMySqlConnection: function ()
{
var conObj = { host: "localhost", user: "root", password: "", database: "deccan" };
var connection = require("mysql").createConnection(conObj);
return new Promise(function (resolve, reject)
{
connection.connect(function (error)
{
if (error)
reject(error);
else
resolve(connection);
});
});
}
};
webservicefile.js
var Promise = require('bluebird');
var express = require('express');
var router = express.Router();
var constants = require("../constants");
function getSettings(request, response)
{
var client = constants.getRedisConnection();
get_Data();
function get_Data()
{
return new Promise(function (resolve, reject)
{
client.get("url", function (error, reply)
{
if (error)
reject(error);
else
{
if (reply == null)
resolve(); // Key not present so create
else
resolve(reply);
}
});
}).
catch(function (e)
{
console.log("Error at : " + new Date().toString() + ", => " + e);
}).
then(function (urlResult)
{
return new Promise(function (resolve, reject)
{
if (urlResult == undefined || urlResult == null)
{
return new Set_Data();
}
else
{
client.quit();
return resolve(urlResult);
}
});
}).
then(function (urlResult)
{
if (urlResult)
response.status(200).send({ url : urlResult });
else
response.status(500).send();
})
}
function Set_Data()
{
constants.getMySqlConnection().then(function (connection)
{
return new Promise(function (resolve, reject)
{
connection.query("select url from table where id = 1", function (error, results)
{
connection.end();
if (error)
reject(error);
else
resolve(results);
});
});
}).
then(function (url)
{
return new Promise(function (resolve, reject)
{
client.set('url', url, function (err, reply)
{
if (err)
reject(err);
else
resolve(url);
});
});
});
}
}

A couple of changes should do the trick, first Set_Data() doesn't return a promise like you think it does, add a return:
function Set_Data() {
return constants.getMySqlConnection().then(function (connection).then()
// ...
}
Inside this callback, you don't have a resolve() in the if so the promise is never resolved, returning something doesn't resolve:
// your code
then(function (urlResult) {
return new Promise(function (resolve, reject) {
if (urlResult == undefined || urlResult == null) {
return new Set_Data();
} else {
client.quit();
return resolve(urlResult);
}
});
}).
Return Set_Data() which is now a promise or the url:
then(function (urlResult) {
if (urlResult == undefined || urlResult == null) {
return new Set_Data();
} else {
client.quit();
return urlResult;
}
}).
On a side note, don't format your js code like C#, { shouldn't be on a new line.

You can return new Set_Data() directly if no urlResult, or return urlResult otherwise - inside then any (non Promise) value returned is a resolved Promise - so the .then chain will continue as required
function get_Data() {
// ...
.then(function (urlResult) {
// you don't need a "new Promise" here
if (urlResult == undefined || urlResult == null) {
return new Set_Data();
} else {
client.quit();
return urlResult;
}
})
// ...
}
One thing I noticed in your code is a .catch in the middle of your .then chain which will effectively turn an error into a fulfilled promise - not sure if that's the behaviour you are looking for. Something else to look out for

Related

How to wait for a MYSQL query to finish before executing another using Node server?

I am building an Express server to receive request (a dict with 10 items) from my React front end and then save the data to database. Below is my code.
I found that the query may crash during the insertion e.g. 2 queries got the same id by last_insert_id(). I have tried to use setTimeout() to wrap the getConnection function but the issue still exists. How to better solve the problem?
The request data:
{{.....}, {.....}, {.....}, {.....}, {.....}} #10 item
Code:
router.post('/fruit', (req, res) => {
const dict = req.body;
let itemCount = 0;
var err_list = [];
Object.keys(dict).forEach(function(r){
let query = "call sp_insert_fruit();"
setTimeout(function() {
getConnection(function(err, conn){
if (err) {
return res.json({ success: false, error: err })
} else {
conn.query(query, function (err, result, fields) {
if (err) {
err_list.push({'errno':err.errno, 'sql_message':err.sqlMessage});
}
itemCount ++;
if (itemCount === Object.keys(dict).length) {
conn.release()
console.log('released', err_list)
if (err_list .length === 0) {
return res.json({ success: true});
} else {
return res.json({ success: false, error: err_list});
}
}
});
}
});
}, 1000);
});
});
connection.js:
const p = mysql.createPool({
"connectionLimit" : 100,
"host": "example.org",
"user": "test",
"password": "test",
"database": "test",
"multipleStatements": true
});
const getConnection = function(callback) {
p.getConnection(function(err, connection) {
callback(err, connection)
})
};
module.exports = getConnection
You should replace callbacks with Promises and async/await to avoid callback hell. Using Promises, this problem should be easy to solve.
connection.js
const p = mysql.createPool({
"connectionLimit" : 100,
"host": "example.org",
"user": "test",
"password": "test",
"database": "test",
"multipleStatements": true
});
// wrap p.getConnection with Promise
function getConnection() {
return new Promise((resolve, reject) => {
p.getConnection((err, connection) => {
if (err) reject(err);
else resolve(connection);
});
});
};
module.exports = getConnection;
Router code
// wrap conn.query with Promise
function executeQuery(conn, query) {
return new Promise((resolve, reject) => {
conn.query(query, (err, result, fields) => {
if (err) reject(err);
else resolve({ result, fields });
});
});
}
router.post('/fruit', async (req, res) => {
const dict = req.body;
const errList = [];
const query = "call sp_insert_fruit();"
let conn = null;
try {
conn = await getConnection();
} catch (err) {
return res.json({
success: false,
error: err
});
}
for (const r of Object.keys(dict)) {
try {
const { result, fields } = await executeQuery(conn, query);
} catch (err) {
errList.push({
'errno': err.errno,
'sql_message': err.sqlMessage
});
}
}
conn.release();
console.log('released', errList);
// I don't know what err_imnt is, so I guess it's errList?
if (errList.length === 0) {
return res.json({
success: true
});
} else {
return res.json({
success: false,
error: errList
});
}
});

MySQL Transactions in Node

Before I commit anything to the database, I want all my update promises resolve; otherwise, I rollback. In other words, I want atomicity. I suppose I could handle the rollback by deleting out rows, but this has its own risks. I noticed if there is an error in any of the promises, the data still gets updated in database. What am I doing wrong?
I have written a simple program to illustrate the issue.
This is the main process:
const db = require('./db.js');
const async = require('async');
let insertList = [];
for (let i = 0; i<3; i++) {
insertList.push(i);
}
async function func1 () {
return new Promise((resolve, reject) => {
console.log("In Func1");
async.forEachOf(insertList, function(value, key, callback) {
console.log('>>>>' + value + '<<<<<<' + key );
db.insertOne('coll1', {value}).then(() => {
callback();
}).catch(err => {callback(err)})
}, function(err) {
// if any of the file processing produced an error, err would equal that error
if( err ) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('err:', err);
reject(err);
} else {
console.log('Col1 All inserts have been processed successfully');
resolve("Success");
}
});
})
}
function func2 () {
return new Promise((resolve, reject) => {
console.log("In Func2");
async.forEachOf(insertList, function(value, key, callback) {
console.log('>>>>' + value + '<<<<<<' + key );
db.insertOne('coll2', {value}).then(() => {
callback();
}).catch(err => {callback(err)})
}, function(err) {
// if any of the file processing produced an error, err would equal that error
if( err ) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('err:', err);
reject(err);
} else {
console.log('Col2 All inserts have been processed successfully');
resolve("Success");
}
});
})
}
function func3 () {
return new Promise((resolve, reject) => {
console.log("In Func3");
async.forEachOf(insertList, function(value, key, callback) {
console.log('>>>>' + value + '<<<<<<' + key );
if(key === 1) {
value = 'a';
}
db.insertOne('coll3', {value}).then(() => {
callback();
}).catch(err => {callback(err)})
}, function(err) {
// if any of the file processing produced an error, err would equal that error
if( err ) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('err:', err);
reject(err);
} else {
console.log('Col3 All inserts have been processed successfully');
resolve("Success");
}
});
})
}
db.connect().then((pool) => {
pool.getConnection((err, connection) =>{
if (err)
return console.error(err);
else {
}
connection.beginTransaction((err) => {
if (err) {
return console.error(err);
}
let func1Promise = new Promise((resolve, reject) => {func1().then(() => {
console.log("Func1 complete");
resolve("Func1 complete");
}).catch((err) => {
console.error("Func1 ERROR: ", err);
reject("Func1 ERROR: ", err);
})});
let func2Promise = new Promise((resolve, reject) => {func2().then(() => {
console.log("Func2 complete");
resolve("Func2 complete");
}).catch((err) => {
console.error("Func2 ERROR: ", err);
reject("Func2 ERROR: ", err);
})});
let func3Promise = new Promise((resolve, reject) => {func3().then(() => {
console.log("Func3 complete");
resolve("Func3 complete");
}).catch((err) => {
console.error("Func3 ERROR: ", err);
reject("Func3 ERROR: ", err);
})});
Promise.all([func1Promise, func2Promise, func3Promise])
.then(()=> {
console.log("All Processes completed successfully.");
connection.commit(err => {
if (err) {
connection.rollback(() => {
throw err;
});
}
console.log('Commit Complete.');
connection.release();
});
})
.catch((err)=> {
console.error(err);
console.error("An update process has failed.");
connection.rollback(() => {
console.error(err);
connection.release();
});
})
});
})
});
The db.js looks like this:
const mysql = require('mysql');
const config = {
db: {
host: 'localhost',
user: process.env.DBUSER,
password: process.env.DBPASSWORD,
database: 'test',
}
};
var pool;
class DB {
constructor(host, user, password, database) {
this.host = host;
this.user = user;
this.password = password;
this.database = database;
}
connect() {
return new Promise((resolve, reject) => {
pool = mysql.createPool({
connectionLimit: 10,
host : this.host,
user : this.user,
password : this.password,
database : this.database
});
resolve(pool);
});
}
objToArray(obj) {
let arr = obj instanceof Array;
return (arr ? obj : Object.keys(obj)).map((i) => {
let val = arr ? i : obj[i];
if(typeof val === 'object' && val !== null)
return this.objToArray(val);
else
return val;
});
}
insertOne(collection, insertObj) {
return new Promise((resolve, reject) => {
pool.getConnection((err, connection) => {
if (err) {
resolve(err);
} else {
let sql = "INSERT INTO " + collection + " VALUES (?)";
// Convert the array of objects into an array of arrays.
let responseJson = this.objToArray(insertObj);
// The query object expects an array of objects so you pass in 'responseJson' as is
console.log(responseJson);
connection.query(sql, [responseJson], (err, result) => {
if (err) {
console.error(err);
return reject(err);
}
//console.log(result);
resolve("SUCCESS: object inserted into database");
});
}
});
});
}
}
const db = new DB(config.db.host, config.db.user, config.db.password, config.db.database);
Object.freeze(db);
module.exports = db;
My database "test" is simple and consists of 3 tables, coll1, coll2, coll3 and each has on field which is type int. In the third function I replace the 1 with 'a' This causes an error and the code catches this error and attempts a rollback, which does not work. If I set a breakpoint after func1 is executed and check the database, the values are already in the database.
Here is the version of MySQL that I am running:
Variable_name,Value
innodb_version,8.0.11
protocol_version,10
slave_type_conversions,
tls_version,"TLSv1,TLSv1.1,TLSv1.2"
version,8.0.11
version_comment,"MySQL Community Server - GPL"
version_compile_machine,x86_64
version_compile_os,macos10.13
version_compile_zlib,1.2.11
I am using the following NPM packages in node:
"async": "^2.6.2",
"mysql": "^2.15.0"
You're creating a transaction on a connection created in your test program, but your db.js's insertOne is grabbing a new connection from the pool that does not have a transaction. You should be passing in the connection you created in the test program.

Mocking/Stubbing/unit testing mysql streaming query rows node js

I have a following function which uses streaming-query-rows of mysql node js module. How can i unit test the below function and also i want to mock the database behavior instead of connecting to database while unit test.
'processRow' and ''wirteCsvFile'' function both are synchronous task.
function executeTask(sql_connection,sql_query) {
let query = sql_connection.query(sql_query);
let showInfo = {};
let showids = [];
query
.on('error', (error) => {
console.error(`error executing query --> ${error}`);
})
.on('result', function (row) {
sql_connection.pause();
processRow(row, showInfo, showids, function () {
sql_connection.resume();
});
})
.on('end', function () {
showids.forEach(showid => {
if (showInfo[showid].faults.length === 0) {
delete showInfo[showid];
}
});
wirteCsvFile(showInfo, (error, done) => {
if (error) {
console.error(error);
} else {
console.log("done");
process.exit();
}
})
});
}
You can stub the query function to return whatever you want instead of making request to database:
sinon.stub(connection, "query").callsFake(() => /* whatever you want here */);
You should also break executeTask into smaller functions, for ex:
function errorHandler(error) {
console.error(`error executing query --> ${error}`);
}
function resultHandler(data, row) {
sql_connection.pause();
processRow(row, data.showInfo, data.showids, function() {
sql_connection.resume();
});
}
function endHandler(data) {
data.showids.forEach(showid => {
if (data.showInfo[showid].faults.length === 0) {
delete data.showInfo[showid];
}
});
wirteCsvFile(data.showInfo, (error, done) => {
if (error) {
console.error(error);
} else {
console.log("done");
process.exit();
}
})
}
function executeTask(sql_connection, sql_query) {
let query = sql_connection.query(sql_query);
let data = {
showInfo: {},
showids: [],
};
query.on('error', errorHandler)
.on('result', resultHandler.bind(null, data))
.on('end', endHandler.bind(null, data));
}
Now you can test errorHandler, resultHandler, endHandler separately
What I'm thinking is we can mock the sql_connection with a class of Event Emitter.
const sinon = require("sinon");
const assert = require('assert');
const EventEmitter = require('events');
const src = require('....'); // your source file that contain `executeTask`
// Create mock emitter
class QueryEmitter extends EventEmitter {}
describe('test execute task', function() {
const queryEmitter = new QueryEmitter();
// we build mock connection that contains all methods used as in `sql_connection`
const conn = {
query: sinon.stub().returns(queryEmitter),
pause: sinon.spy(),
resume: sinon.spy()
};
const query = 'SELECT *';
before(function() {
src.executeTask(conn, query);
});
it('calls query', function() {
assert(conn.query.calledWith(query));
});
it('on result', function() {
queryEmitter.emit('result');
assert(conn.pause.called);
// assert if processRow is called with correct arguments
// assert if conn.resume is called
});
it('on end', function() {
queryEmitter.emit('end');
// assert if writeCsvFile is called
});
// probably is not needed since you only call console.log here
it('on error', function() {
queryEmitter.emit('error');
});
});
Hope it helps

Node.js returning a promise from a function

I'm exploring the possibilities of promises and callbacks in node.js
I'm trying to find a way for this code to work. Currently the issue I'm facing is that when I'm calling a function and want to use the return value, it is not ready yet. I know what I have to do, but don't know how. Basically, I have to make that insertAddress() returns a promise (so I can use the .then() on it), or takes a callback as a param. To do this, I also think databaseWork() should return a promise. But I don't know where to add it.
The issue is located in the 'console.log(out)', that runs before out variable is set (because insertAddress is still running).
Here is my code
app.js
-----
const databaseWork = require('./db/mysql.js').databaseWork;
app.use('/test', (req, resp) => {
var address = {
country : "Country",
city : "Randomcity",
street : "Random",
number : 6,
postalcode : "A789",
province : "a province"
}
var out = insertAddress(address); //<== takes time to finish, is not ready when the next console.log finishes
console.log(out);
});
function insertAddress(address){
var rows
databaseWork(
//Following anonymous function contains the actual workload. That has to be done inside a transaction
async (connection) => {
rows = await insertAddressQuery(address,connection);
console.log(rows); //this one waits for insertAddressQuery to be complete
})
return rows; //this will run before insertAddressQuery is complete
}
function insertAddressQuery(address,connection) {
return new Promise( (resolve, reject) => {
//async job
connection.query('INSERT INTO address (country,city,Street,number,postalcode,province) VALUES(?,?,?,?,?,?)', [address.country,'4','5',6,'7','8'] , (err, rows) => {
if (err) {reject(err);}
resolve(rows);
});
});
};
/db/mysql.js
------------
var mysql = require('mysql');
var dbpool = mysql.createPool({
host: process.env.HOST_DB,
user: process.env.USER_DB,
password: process.env.PWD_DB,
database: process.env.DB
});
function databaseWork(workload){
dbpool.getConnection( async (err, connection) => {
await beginTransaction(connection);
await workload(connection);
await commitTransaction(connection)
connection.release();
});
}
function beginTransaction(connection){
return new Promise( (resolve, reject) => {
//async job
connection.beginTransaction( (err) => {
if (err) {reject(err);}
resolve();
});
});
};
function commitTransaction(connection) {
return new Promise( (resolve, reject) => {
//async job
connection.commit( (err) => {
if (err) {reject(err);}
resolve();
});
});
};
exports.databaseWork = databaseWork;
You would do that in your databaseWork:
function databaseWork(workload) {
return new Promise((resolve, reject) => {
dbpool.getConnection(async (err, connection) => {
try {
await beginTransaction(connection);
var result = await workload(connection);
await commitTransaction(connection)
resolve(result);
} catch( err ) {
reject(err)
} finally {
connection.release();
}
});
})
}
The Promise returned by databaseWork will be resolved by the result of workload. And now you can change insertAddress to this:
async function insertAddress(address){
return databaseWork(connection => {
return insertAddressQuery(address,connection);
})
}
You then need to change the route to this:
app.use('/test', async (req, resp) => {
var address = {
country: "Country",
city: "Randomcity",
street: "Random",
number: 6,
postalcode: "A789",
province: "a province"
}
var out = await insertAddress(address); // use await here to wait for insertAddress to be finished
console.log(out);
});
*UPDATE code with an getConnection function that returns a Promise:
function getConnection() {
return new Promise((resolve, reject) => {
dbpool.getConnection((err, connection) => {
if (err) {
reject(err)
} else {
resolve(connection);
}
})
});
}
async function databaseWork(workload) {
var connection = await getConnection();
var result;
try {
await beginTransaction(connection)
result = await workload(connection)
await commitTransaction(connection)
} catch (err) {
// a rollback might be neccesaary at that place
throw err
} finally {
connection.release();
}
return result;
}
One way you can do this is by using async await.
var example = async (req, res) => {
var response = await myAsyncTask();
// this will get logged once the async task finished running.
console.log(response)
}
// Use async await to get response
var myAsyncTask = async () => {
try {
var response = await asyncTaskINeedDataFrom()
return response;
}
catch(err) {
return console.log(err);
}
}
Here's the npm module: https://www.npmjs.com/package/async

$http.post within a $http.post, return response is not updated directly

I have a function, it has a $http.post for login purpose. If success, another $http.post will call a php file that fetches data from database. The problem is that, when I am trying to load the data from localStorage it returns me null. Why is it so?
$scope.loginUser = function ()
{
var data =
{
username: $scope.loginInfo.username,
password: $scope.loginInfo.password
}
$http.post("endpoints/login.php", data).success(function(response)
{
if(response==="ERROR")
{
//DONT DO ANYTHING
}
else
{
localStorage.setItem("token", JSON.stringify(response));
console.log("loginController: name is this " + localStorage);
fetchDataFunction(data);
$state.go("application");
//$state.go("application", result);
}
}).error(function(error)
{
console.error(error);
});
}
fetchDataFunction = function(data)
{
$http.post("endpoints/fetchData.php", data).success(function(response)
{
localStorage.setItem("data", JSON.stringify(response));
}).error(function(error)
{
console.error(error);
});
}
You can return the $http.post, which will return a promise, and then all your code will work in the correct order:
$scope.loginUser = function () {
login($scope.loginInfo).then(function (response) {
localStorage.setItem("token", JSON.stringify(response));
console.log("loginController: name is this " + localStorage.getItem("token"));
fetchDataFunction(data).then(function () {
localStorage.setItem("data", JSON.stringify(response));
console.log(localStorage.getItem("data"));
$state.go("application");
}).catch(function (error) {
console.error(error);
});
}).catch(function (response) {
console.error(error);
});
};
var login = function (user) {
return post("endpoints/login.php", user);
};
var fetchDataFunction = function (data) {
return post("endpoints/fetchData.php", data);
};
var post = function (url, data) {
var deferred = $q.defer;
$http.post(url, data).then(function (response) {
if (response === "ERROR") {
deferred.reject(response);
}
else {
deferred.resolve(response);
}
}).catch(function (error) {
deferred.reject(error);
});
return deferred;
};
Notes:
You will need to make sure you inject $q into your controller along with $http
You should use localStorage.getItem() when recalling information from the global object
You should use then/catch instead of success/error, as these are now depreciated: https://docs.angularjs.org/api/ng/service/$http