Query inside foreach Node.js Promis - mysql

i put query inside for each on promise. I am trying to query a mysql database twice, the second time, multiple times for each result from the first time but I am unable to work out how to wait for the result from the second query before continuing
i want the output like this :
{
"data":[
{
"name":"Title result",
"images":[
{
"id":1,
"place_id":705,
"path_image":"http://3.bp.blogspot.com/-iwF-ImFpzvk/T6fKhC6F7YI/AAAAAAAAARA/FyKpNcDsP8M/s1600/asd2e1.jpg"
},
{
"id":2,
"place_id":705,
"path_image":"https://asrt.bp.com/data/photo/2014/07/22/sddrfr2.jpg",
}
]
}
]
}
but i get only like this :
{
"data":[
{
"name":"Title result",
"images":[]
}
and this is my code:
return new Promise((resolve, reject) => {
const { connection, errorHandler } = deps;
let arrayData = [];
let imageData = [];
connection.query(
"SELECT * FROM places WHERE id = 705",
(error, rows, results) => {
rows.forEach((row) => {
connection.query(
"SELECT * FROM place_gallery WHERE place_id = 705",
(error, rows, results) => {
imageData = rows;
}
)
arrayData.push({ name: row.title, images: imageData })
});
if (error) {
errorHandler(error, "failed", reject);
return false;
}
resolve({ data: arrayData });
}
);
})
},
how to solve this?

try this, another way instated of creating dbcall function you can convert the query callback to promise using util.promisify()
const dbcall = (query) => {
return new Promise((resolve, reject) => {
connection.query(
query,
(error, rows, results) => {
if (error) return reject(error);
return resolve(rows);
});
});
};
const somefunc = async () => {
const {
connection,
errorHandler
} = deps;
let arrayData = [];
try {
const rows = await dbcall("SELECT * FROM places WHERE id = 705");
rows.forEach(async (row) => {
const imageData = await dbcall("SELECT * FROM place_gallery WHERE place_id = 705");
arrayData.push({
name: row.title,
images: imageData
});
});
} catch (error) {
console.log(error);
}
return arrayData;
}

Related

How to wait for a MYSQL query to finish before executing another using Node server?

I am building an Express server to receive request (a dict with 10 items) from my React front end and then save the data to database. Below is my code.
I found that the query may crash during the insertion e.g. 2 queries got the same id by last_insert_id(). I have tried to use setTimeout() to wrap the getConnection function but the issue still exists. How to better solve the problem?
The request data:
{{.....}, {.....}, {.....}, {.....}, {.....}} #10 item
Code:
router.post('/fruit', (req, res) => {
const dict = req.body;
let itemCount = 0;
var err_list = [];
Object.keys(dict).forEach(function(r){
let query = "call sp_insert_fruit();"
setTimeout(function() {
getConnection(function(err, conn){
if (err) {
return res.json({ success: false, error: err })
} else {
conn.query(query, function (err, result, fields) {
if (err) {
err_list.push({'errno':err.errno, 'sql_message':err.sqlMessage});
}
itemCount ++;
if (itemCount === Object.keys(dict).length) {
conn.release()
console.log('released', err_list)
if (err_list .length === 0) {
return res.json({ success: true});
} else {
return res.json({ success: false, error: err_list});
}
}
});
}
});
}, 1000);
});
});
connection.js:
const p = mysql.createPool({
"connectionLimit" : 100,
"host": "example.org",
"user": "test",
"password": "test",
"database": "test",
"multipleStatements": true
});
const getConnection = function(callback) {
p.getConnection(function(err, connection) {
callback(err, connection)
})
};
module.exports = getConnection
You should replace callbacks with Promises and async/await to avoid callback hell. Using Promises, this problem should be easy to solve.
connection.js
const p = mysql.createPool({
"connectionLimit" : 100,
"host": "example.org",
"user": "test",
"password": "test",
"database": "test",
"multipleStatements": true
});
// wrap p.getConnection with Promise
function getConnection() {
return new Promise((resolve, reject) => {
p.getConnection((err, connection) => {
if (err) reject(err);
else resolve(connection);
});
});
};
module.exports = getConnection;
Router code
// wrap conn.query with Promise
function executeQuery(conn, query) {
return new Promise((resolve, reject) => {
conn.query(query, (err, result, fields) => {
if (err) reject(err);
else resolve({ result, fields });
});
});
}
router.post('/fruit', async (req, res) => {
const dict = req.body;
const errList = [];
const query = "call sp_insert_fruit();"
let conn = null;
try {
conn = await getConnection();
} catch (err) {
return res.json({
success: false,
error: err
});
}
for (const r of Object.keys(dict)) {
try {
const { result, fields } = await executeQuery(conn, query);
} catch (err) {
errList.push({
'errno': err.errno,
'sql_message': err.sqlMessage
});
}
}
conn.release();
console.log('released', errList);
// I don't know what err_imnt is, so I guess it's errList?
if (errList.length === 0) {
return res.json({
success: true
});
} else {
return res.json({
success: false,
error: errList
});
}
});

How to get return values from Async/await function when fetching the data from mySQL in Nodejs

I am fetching some exchange data from DB, then extracting the name of distinct exchanges and passing again into MYSQL query to fetch data from a different table.
The problem I am facing is that async await does not return the value rather just return Promise { }.
Below is the code that I am trying, wondering where I am going wrong.
//Function that fetches the exchanges from DB
const getExchange = () => {
return new Promise((resolve, reject) => {
db.connection.query(`
SELECT *
FROM,
(
SELECT
exchange,
COUNT(pair) as noOfMarkets
FROM ticker_data
) as t
`, (err, resp) => {
if (!err) {
resolve(resp)
} else {
reject(err)
}
})
})
}
// push unique exchanges to an array.
const getExchangesData = async () => {
const allExchanges = await getExchanges();
let exchanges = []
allExchanges.forEach(item => {
let exchange = {
exchange: item.exchange
}
exchanges.push(exchange)
})
return await exchanges
}
// mapping through an array of exchanges and passing to DB query to get data from the DB.
const getSingleExchange = async () => {
const exchanges = await getExchangesData()
await Promise.all(exchanges.map(async (item) => {
db.connection.query(`
SELECT
exchange_rank,
name
volume24hUSD
(
SELECT
volume24hUSD as tradingVolYesterday
FROM exchanges
WHERE name = '${item.exchange}'
AND createdAt >= now() -interval 1 day
AND createdAt < now() -interval 1 day + interval 120 second
LIMIT 1
) volumeDay1
FROM exchanges
WHERE name = '${item.exchange}'
`, (err, resp) => {
if (!err) {
console.log(resp) // getting all the values
let volData = {
name: resp[0].name,
exchange_rank: resp[0].exchange_rank,
icon: resp[0].icon
}
return volData
}
})
}))
}
const data = getSingleExchange()
console.log(data) // returning Promise { <pending> }
Edit
After making changes suggested in an answer, I still have an issue:
//Function that fetches the exchanges from DB
const getExchange = () => {
return new Promise((resolve, reject) => {
db.connection.query(`
SELECT *
FROM,
(
SELECT
exchange,
COUNT(pair) as noOfMarkets
FROM ticker_data
) as t
`, (err, resp) => {
if (!err) {
resolve(resp)
} else {
reject(err)
}
})
})
}
// push unique exchanges to an array.
const getExchangesData = async () => {
const allExchanges = await getExchanges();
let exchanges = []
allExchanges.forEach(item => {
let exchange = {
exchange: item.exchange
}
exchanges.push(exchange)
})
return await exchanges
}
// mapping through an array of exchanges and passing to DB query to get data from the DB.
const getSingleExchange = async () => {
const exchanges = await getExchangesData()
await Promise.all(exchanges.map((item) => {
return new Promise((resolve, reject) => {
db.connection.query(`...`, (err, resp) => {
if (!err) {
resolve(resp)
} else {
reject(err)
}
}).then(resp => {
console.log(resp)
let volData = {
name: resp[0].name,
exchange_rank: resp[0].exchange_rank,
icon: resp[0].icon
}
return volData
})
})
}))
}
getSingleExchange().then(data => {
console.log(data)
});
I now get this error:
(node:30583) UnhandledPromiseRejectionWarning: TypeError: db.connection.query(...).then is not a function
at Promise (/getExchanges.js:217:16)
at new Promise ()
at Promise.all.exchanges.map (/getExchanges.js:145:16)
at Array.map ()
at getSingleExchange (/getExchanges.js:144:33)
The main issue is in this part:
await Promise.all(exchanges.map(async (item) => {
That map callback is not returning anything, and it has no await, so using async makes no sense.
Instead remove async:
await Promise.all(exchanges.map((item) => {
... and return a promise in the callback function, much like you had done in the first function:
return new Promise((resolve, reject) => {
db.connection.query(`...`), (err, resp) => {
if (!err) {
resolve(resp)
} else {
reject(err)
}
})
}).then(resp => {
console.log(resp)
let volData = {
name: resp[0].name,
exchange_rank: resp[0].exchange_rank,
icon: resp[0].icon
}
return volData
});
You would benefit from writing one generic function that promisifies query, so that you don't have to do that new Promise-thing for every single query you need.
Finally, you cannot expect to get an asynchronous result synchronously: async functions do not return the asynchronous result synchronously, but return a promise for it. So your last lines (main code) should still await. So either do this:
(async () => {
const data = await getSingleExchange()
console.log(data)
})(); // immediately executing (async) function expression
Or:
getSingleExchange().then(data => {
console.log(data)
});
NB: doing return await exchanges in the second function makes no sense (exchanges is not a promise), so you can just do return exchanges.

How can I execute the SQL query first then the rest of the code?

static listFunc() {
let funclist = [];
const queryList = "SELECT * FROM func";
mysqlModule.queryDB(database, queryList, (err, result) => {
console.log(result[0].id);
if (err) {
res.status(500).json({
"status_code": 500,
"status_message": "internal server error"
});
} else {
for (var i = 0; i < result.length; i++) {
let func = {
'id': result[i].id,
'psw': result[i].senha,
'nome': result[i].nome,
'DoB': result[i].dataNascimento,
'sexo': result[i].genero,
'morada': result[i].morada,
'permissoes': result[i].permissoes
}
funclist.push(func);
}
return funclist;
}
});
}
I created a function to give me all the workers from my DataBase and then store them inside funclist array.
The problem is the for loop is running before the query.
How can I run the loop only after query as finished?
Pass a callback function into listFunc:
static listFunc(callback){...}
Instead of returning the list just invoke the callback:
callback(funclist);
static listFunc() {
return new Promise((resolve,reject)=>{
let funclist = [];
mysqlModule.queryDB(database,"SELECT * FROM func", (err, result) => {
if (err) throw err;
result.forEach((result) => {
let func = {
'id': result.id,
'psw': result.senha,
'nome': result.nome,
'DoB': result.dataNascimento,
'sexo': result.genero,
'morada': result.morada,
'permissoes': result.permissoes
}
funclist.push(func);
});
resolve(funclist);
});
});
}
First i changed the "for" loop to a "each" loop and i used the promise to give me the data only after i get the query and the loop finished.
function clistFunc(req, res){
Func.listFunc().then((data)=>{
res.render('admin/adminListFunc', { funclist: data});
console.log(data);
}).catch(()=>{
console.log('Error');
});
}
Then i just rendered the jade only after my listFunc() return the pretended data.

Node.js returning a promise from a function

I'm exploring the possibilities of promises and callbacks in node.js
I'm trying to find a way for this code to work. Currently the issue I'm facing is that when I'm calling a function and want to use the return value, it is not ready yet. I know what I have to do, but don't know how. Basically, I have to make that insertAddress() returns a promise (so I can use the .then() on it), or takes a callback as a param. To do this, I also think databaseWork() should return a promise. But I don't know where to add it.
The issue is located in the 'console.log(out)', that runs before out variable is set (because insertAddress is still running).
Here is my code
app.js
-----
const databaseWork = require('./db/mysql.js').databaseWork;
app.use('/test', (req, resp) => {
var address = {
country : "Country",
city : "Randomcity",
street : "Random",
number : 6,
postalcode : "A789",
province : "a province"
}
var out = insertAddress(address); //<== takes time to finish, is not ready when the next console.log finishes
console.log(out);
});
function insertAddress(address){
var rows
databaseWork(
//Following anonymous function contains the actual workload. That has to be done inside a transaction
async (connection) => {
rows = await insertAddressQuery(address,connection);
console.log(rows); //this one waits for insertAddressQuery to be complete
})
return rows; //this will run before insertAddressQuery is complete
}
function insertAddressQuery(address,connection) {
return new Promise( (resolve, reject) => {
//async job
connection.query('INSERT INTO address (country,city,Street,number,postalcode,province) VALUES(?,?,?,?,?,?)', [address.country,'4','5',6,'7','8'] , (err, rows) => {
if (err) {reject(err);}
resolve(rows);
});
});
};
/db/mysql.js
------------
var mysql = require('mysql');
var dbpool = mysql.createPool({
host: process.env.HOST_DB,
user: process.env.USER_DB,
password: process.env.PWD_DB,
database: process.env.DB
});
function databaseWork(workload){
dbpool.getConnection( async (err, connection) => {
await beginTransaction(connection);
await workload(connection);
await commitTransaction(connection)
connection.release();
});
}
function beginTransaction(connection){
return new Promise( (resolve, reject) => {
//async job
connection.beginTransaction( (err) => {
if (err) {reject(err);}
resolve();
});
});
};
function commitTransaction(connection) {
return new Promise( (resolve, reject) => {
//async job
connection.commit( (err) => {
if (err) {reject(err);}
resolve();
});
});
};
exports.databaseWork = databaseWork;
You would do that in your databaseWork:
function databaseWork(workload) {
return new Promise((resolve, reject) => {
dbpool.getConnection(async (err, connection) => {
try {
await beginTransaction(connection);
var result = await workload(connection);
await commitTransaction(connection)
resolve(result);
} catch( err ) {
reject(err)
} finally {
connection.release();
}
});
})
}
The Promise returned by databaseWork will be resolved by the result of workload. And now you can change insertAddress to this:
async function insertAddress(address){
return databaseWork(connection => {
return insertAddressQuery(address,connection);
})
}
You then need to change the route to this:
app.use('/test', async (req, resp) => {
var address = {
country: "Country",
city: "Randomcity",
street: "Random",
number: 6,
postalcode: "A789",
province: "a province"
}
var out = await insertAddress(address); // use await here to wait for insertAddress to be finished
console.log(out);
});
*UPDATE code with an getConnection function that returns a Promise:
function getConnection() {
return new Promise((resolve, reject) => {
dbpool.getConnection((err, connection) => {
if (err) {
reject(err)
} else {
resolve(connection);
}
})
});
}
async function databaseWork(workload) {
var connection = await getConnection();
var result;
try {
await beginTransaction(connection)
result = await workload(connection)
await commitTransaction(connection)
} catch (err) {
// a rollback might be neccesaary at that place
throw err
} finally {
connection.release();
}
return result;
}
One way you can do this is by using async await.
var example = async (req, res) => {
var response = await myAsyncTask();
// this will get logged once the async task finished running.
console.log(response)
}
// Use async await to get response
var myAsyncTask = async () => {
try {
var response = await asyncTaskINeedDataFrom()
return response;
}
catch(err) {
return console.log(err);
}
}
Here's the npm module: https://www.npmjs.com/package/async

mysql node: can't set headers after they are sent

I am trying to get a list of movies in a directory, parse titles, get movie information on TMDB than check if movie info is stored in mysql database and if not stored, insert info into the database.
I am using NodeJS/Express and mysql.
Here is my code so far:
exports.checkForMovies = function (req, res, next) {
const testFolder = './test/';
var movieList = [];
var movieResultsPromise = [];
var movieResults = [];
fs.readdirSync(testFolder).forEach(file => {
movieList.push(tnp(file));
});
movieList.forEach(movie => {
var waitPromise = searchTMDB(movie.title);
movieResultsPromise.push(waitPromise);
});
Promise.all(movieResultsPromise).then(result => {
movieResults = result;
movieResults.forEach(movie => {
checkMoviesInDB(movie.id, (err, data) => {
if (err) {
console.log(err)
}
if (data && data.update === true) {
var movieObj = {
m_tmdb_id: movie.id
};
insertMoviesToDB(movieObj, (resp, err) => {
if (err) {
console.log(err);
} else {
return res.json(resp);
}
});
} else {
return res.json(data);
}
});
});
});
}
function checkMoviesInDB(id, cb) {
var sql = "SELECT * FROM ?? WHERE m_tmdb_id = ?"
var table = ['movie', id];
sql = mysql.format(sql, table);
connection.query(sql, function (err, rows) {
if (err) {
return cb(err);
}
if (rows.length > 0) {
return cb(null, {
success: true,
update: false,
message: 'Movies up to date!'
})
} else {
return cb(null, {
update: true,
message: 'Updating database!'
})
}
});
}
function insertMoviesToDB(movie, cb) {
var sql = "INSERT INTO ?? SET ?";
var table = ['movie', movie];
sql = mysql.format(sql, table);
connection.query(sql, function (err, rows) {
if (err) {
return cb(err);
} else {
return cb(null, {
success: true,
message: 'Movie database updated!'
})
}
});
}
function searchTMDB(title) {
return new Promise((resolve, reject) => {
https.get(config.tmdbURL + title, response => {
var body = "";
response.setEncoding("utf8");
response.on("data", data => {
body += data;
});
response.on("end", () => {
body = JSON.parse(body);
resolve(body.results[0]);
});
response.on("error", (err) => {
reject(err);
});
});
});
}
After code execution it inserts movie info in the database or responses with "Movies up to date" but I am getting this error and NodeJS crashes:
Error: Can't set headers after they are sent.
Any help is appreciated, thanks!
EDIT!
This is the new code and I am still getting the same error...
exports.checkForMovies = function (req, res) {
const testFolder = './test/';
var movieList = [];
var movieResults = [];
fs.readdirSync(testFolder).forEach(file => {
movieList.push(tnp(file));
});
var movieObj = movieList.map(movie => {
var tmp = [];
return searchTMDB(movie.title).then(data => {
tmp.push(data);
return tmp
});
});
var checkDB = Promise.all(movieObj).then(moviesData => {
moviesData.map(movieData => {
checkMoviesInDB(movieData[0]).then(checkResponse => {
if (!checkResponse.movieToInsert) {
res.json(checkResponse);
} else {
var insertArray = checkResponse.movieToInsert;
var inserting = insertArray.map(movie => {
var movieObject = {
m_tmdb_id: movie.id,
m_name: movie.title,
m_year: movie.release_date,
m_desc: movie.overview,
m_genre: undefined,
m_poster: movie.poster_path,
m_watched: 0
};
insertMoviesToDB(movieObject).then(insertResponse => {
res.json(insertResponse);
});
});
}
});
});
});
}
function checkMoviesInDB(movie) {
var moviesToInsert = [];
return new Promise((resolve, reject) => {
var sql = "SELECT * FROM ?? WHERE m_tmdb_id = ?"
var table = ['movie', movie.id];
sql = mysql.format(sql, table);
connection.query(sql, function (err, rows) {
if (err) {
return reject(err);
}
if (rows.length === 0) {
moviesToInsert.push(movie);
resolve({
success: true,
movieToInsert: moviesToInsert
});
} else {
resolve({
success: true,
message: 'No movie to insert'
});
}
});
});
}
function insertMoviesToDB(movie) {
return new Promise((resolve, reject) => {
var sql = "INSERT INTO ?? SET ?";
var table = ['movie', movie];
sql = mysql.format(sql, table);
connection.query(sql, function (err, rows) {
if (err) {
return reject(err);
} else {
resolve({
success: true,
message: 'Movie added!'
});
}
});
});
}
function searchTMDB(title) {
return new Promise((resolve, reject) => {
https.get(config.tmdbURL + title, response => {
var body = "";
response.setEncoding("utf8");
response.on("data", data => {
body += data;
});
response.on("end", () => {
body = JSON.parse(body);
resolve(body.results[0]);
});
response.on("error", (err) => {
reject(err);
});
});
});
}
Auth.js
const config = require('./config');
const jwt = require('jsonwebtoken');
module.exports = function (req, res, next) {
var token = req.body.token || req.params.token || req.headers['x-access-token'];
if (token) {
jwt.verify(token, config.secret, function (err, decoded) {
if (err) {
return res.json({
success: false,
message: 'Failed to authenticate token.'
});
} else {
req.decoded = decoded;
next();
}
});
} else {
return res.status(403).send({
success: false,
message: 'Please login in to countinue!'
});
}
};
Hope this helps:
// Bad Way
const checkForMovies = (req, res) => {
const movieList = ['Braveheart', 'Highlander', 'Logan'];
movieList.forEach(movie => {
res.json(movie); // Will get Error on second loop: Can't set headers after they are sent.
})
}
// Good Way
const checkForMovies = (req, res) => {
const movieList = ['Braveheart', 'Highlander', 'Logan'];
const payload = { data: { movieList: [] } };
movieList.forEach(movie => {
payload.data.movieList.push(movie);
});
// send res once after the loop with aggregated data
res.json(payload);
}
/* GET home page. */
router.get('/', checkForMovies);