Fill array of objects with data from multiple mysql querys - mysql

Is there a way to pause iteration of map function so I can get response from my query and store it in some object.
So my data is an array of objects and I want to add property "receivers" for every single object in array so I use map funtion to loop through my array, problem is in callback :D ,my object reveive data after my express sends response so I had to dely response with Timeout, I know its not the right solution but it work for now because I have only 40-50 records in database.
Here is the code:
data.map((x) => {
let sql = `
SELECT username
FROM message
JOIN message2user ON message.message_id = message2user.message_id
JOIN user on message2user.receiver_id = user.user_id
where message.message_id = ?;
`;
testDB.query(sql, x.message_id, (err, dataReceivers) => {
if (err) console.log(err);
x["receivers"] = dataReceivers;
})
});
setTimeout(() => {
res.json({ success: true, data: data });
}, 1000);
I am using express for my API routes and MySQL module for node, maybe I sould use Promises or async functions from node, I dont know.
Do you have any ideas??

Here's an approach using Promises if you don't want to refactor your SQL query:
const sqlTemplateString = `
SELECT username
FROM message
JOIN message2user ON message.message_id = message2user.message_id
JOIN user on message2user.receiver_id = user.user_id
where message.message_id = ?;
`;
Promise.all(data.map((x) => new Promise((resolve, reject) => {
testDB.query(sqlTemplateString, x.message_id, (err, dataReceivers) => {
if (err) {
reject(err);
} else {
x.receivers = dataReceivers;
resolve(x);
}
});
}))).then((data) => {
res.json({
success: true,
data
});
}).catch((error) => {
console.log(error);
res.json({
success: false,
error
});
});
Or something like that.

Related

Bulk insert with mysql2 and NodeJs throws 500

I have a method which I want to bulk insert into mysql. I am using NodeJS and mysql2.
My method:
createWorklog = async ({ sqlArray }) => {
const sql = `INSERT INTO ${this.tableName}
(project_id, user_id, date, duration, task, description) VALUES ?`
const result = await query(sql, [sqlArray])
const affectedRows = result ? result.affectedRows : 0;
return affectedRows;
}
Where sqlArray is an array of arrays where all the children arrays are the same length.
And the query method that is called in this method is the next one:
query = async (sql, values) => {
return new Promise((resolve, reject) => {
const callback = (error, result) => {
if (error) {
reject(error);
return;
}
resolve(result);
}
// execute will internally call prepare and query
this.db.execute(sql, values, callback);
}).catch(err => {
const mysqlErrorList = Object.keys(HttpStatusCodes);
// convert mysql errors which in the mysqlErrorList list to http status code
err.status = mysqlErrorList.includes(err.code) ? HttpStatusCodes[err.code] : err.status;
throw err;
});
}
}
My problem is that the body parameters are ok (as I said, array of arrays) but the method throws 500.
Can this be possible because of execute command that is present in mysql2? Or is another mistake?
Thank you for your time!
EDIT
I changed my method from using 'execute' to 'query' Based on #Gaurav’s answer and it's working well.
This is a known issue with execute and query method in mysql2
I've found a working alternative.
createWorklog = async ({ sqlArray }) => {
const sql = `INSERT INTO ${this.tableName}
(project_id, user_id, date, duration, task, description) VALUES ?`
const result = await query(sql, [sqlArray], true) // adding true for multiple insert
const affectedRows = result ? result.affectedRows : 0;
return affectedRows;
}
Then query can be written as below
return new Promise((resolve, reject) => {
const callback = ...
if (multiple) this.db.query(sql, values, callback);
else this.db.execute(sql, values, callback);
}).catch(err => {
...
...
});
}
}
More info regarding this issue can be found here https://github.com/sidorares/node-mysql2/issues/830

Check if a user is present in the database for a parameter other than ID

I created 3 functions: findOne, create and update. Respectively the methods are GET, POST, PUT.
I changed my API path, it used to be /api/users/:id, now it's /api/users/:sub.
routes.js:
module.exports = app => {
const users = require("../controllers/user.controller.js");
const router = require("express").Router();
// Create a new User
router.post("/", users.create);
// Retrieve a single User with sub
router.get("/:sub", users.findOne);
// Update a User with sub
router.put("/:sub", users.update);
// Delete a User with sub
router.delete("/:sub", users.delete);
app.use('/api/users', router);
};
controller.js:
// Save User in the database
User.create(user)
.then(data => {
res.send(data);
})
.catch(err => {
res.status(500).send({
message:
err.message || "Some error occurred while creating the Users."
});
});
};
// Find a single User with an id and sub
exports.findOne = (req, res) => {
const sub = req.params.sub;
User.findOne({sub})
.then(data => {
res.send(data);
})
.catch(err => {
res.status(500).send({
message: "Error retrieving User with id=" +sub
});
});
};
// Update a User by the sub in the request
exports.update = (req, res) => {
const sub = req.params.sub;
User.update(req.body, {
where: { sub }
})
.then(num => {
if (sub) {
res.send({
message: "User was updated successfully."
});
} else {
res.send({
message: `Cannot update User with sub=. Maybe User was not found or req.body is empty!`
});
}
})
.catch(err => {
res.status(500).send({
message: "Error updating User with sub="
});
});
};
What I wanted to do was: check if the user_id provided by the authentication provider was present in my database.
If yes, update user data with that user_id.
If not, create a new user record
This is the front-end part involved:
//INFO SAVE AND UPDATE CONDITION
const userExist = InfoDataService.get(data.sub)
.then((response) => {
console.log('find', response.data);
return true;
});
if ( userExist ) {
InfoDataService.create(data)
.then((response) => {
console.log('create', response.data);
setInfo({
id: response.data.id,
sub: response.data.sub,
email: response.data.email,
firstname: response.data.firstname,
lastname: response.data.lastname,
});
})
} else {
InfoDataService.update(sub, data)
.then((response) => {
console.log(response.data);
})
.catch((e) => {
console.error(e);
});
}
};
I thought userInDatabase could only give true or false, so I used it as an argument in the if statement. It does not work and just updates.
If you need any other information, please ask, I have just started and I hope I have given the necessary info.
EDIT
Through findOne I can find the entire object in my database, but I thought that putting the function as an if condition could give me true if it found the object with its sub; false if he found nothing.
This is not the case, in fact in the code I just updated, although findOne works correctly, it continues to execute always and only create.

User entries not updating in database

I am using postman to send a request and I see Success message but in the database, it's not updated at all.
PostMAN request
database Snap shot
update services object: from this file I have used a database query to insert data in the database and set callBack funtion
const pool = require('../../config/database')
module.exports = {
updateUser: (data, callBack) => {
pool.query(
`UPDATE users SET firstName=?,email=?,password=?,lastName=?,phoneNumber=?, sex=? WHERE id=?`, [
data.firstName,
data.email,
data.password,
data.lastName,
data.phoneNumber,
data.sex,
data.id
], (error, results, fields) => {
if (error) {
return callBack(error)
}
return callBack(null, results)
}
)
}
}
update user controller here I have added a controller to update the user details which receive the data from update user services.
const {
create,
getUserbyID,
getUsers,
updateUser,
deleteUser,
getUserByEmail
} = require('./userService')
const {genSaltSync, hashSync, compareSync} = require('bcrypt')
const { sign } = require('jsonwebtoken')
module.exports ={
updateUser: (req, res) => {
const body = req.body;
const salt = genSaltSync(10);
body.password = hashSync(body.password, salt);
updateUser(body, (err, results) => {
if (err) {
console.log(err)
return false;
} // added
console.log("this is the body: "+JSON.stringify(req.body))
console.log("this is the results: "+ JSON.stringify(results))
if (!results) {
return res.json({
success:0,
message: "failed to update user"
})
}
return res.json({
success: 1,
message: "Updated Sucessfully"
})
})
},
}
router.js
router.patch('/update',checkToken, updateUser)
ADDED console.log
this is the body: {"Id":15,"firstName":"joey","email":"joey.chandler357#gmail.com","password":"$2b$10$ZBnRppSKAfQ1TrzGvs/wqOrVx/shb6ESJ7emXnC7IlWRN3VUGgfK2","lastName":"chandler","phoneNumber":"9860316634","sex":"Male"}
this is the results: {"fieldCount":0,"affectedRows":0,"insertId":0,"serverStatus":2,"warningCount":0,"message":"","protocol41":true,"changedRows":0}
I can see your console.log message
this is the results: {"fieldCount":0,"affectedRows":0,"insertId":0,"serverStatus":2,"warningCount":0,"message":"","protocol41":true,"changedRows":0}
Here you can notice affectedRows: 0 it means no row updated this happens when condition is not matched with any of the records. In postman you are passing "Id" I is in capital format but at the time of accessing this in service you are using "data.id" id is small latter so this is creating problem
we can handle this
instead of
if (!results) {
return res.json({
success:0,
message: "failed to update user"
})
}
use
if (!results.affectedRows) {
return res.json({
success:0,
message: "failed to update user"
})
}
this will be much better then previous check
I think you need to use an "insert" to add the db record. It's using an update... so it's looking for a pre-existing record.
Try two things:
wrap “users” in quotes on your update query. I’ve seen this w Postgres where some words are reserved in raw queries.
Examine the database response from your update. See what is console logged.

Cannot pause pool when streaming data with mysql-node

I use node and the mysql package to stream data from node to client.
The idea is,
define a pool, and queries based on the pool.
Then pass the streaming rows to an array.
If that array's length reaches a length, pause the stream, process the rows, send them to client via websockets.
Resume stream. Repeat until no other rows are left.
I am following the examples on the mysql npm page but I get pool.pause is not a function
Here is the code
var pool = mysql.createPool({
connectionLimit : 100,
host : config.host,
user : config.user,
password : config.password,
database : config.database
});
//turn simple queries to promises
const query = (str, ar) => {
return new Promise(function(resolve, reject) {
pool.query(str, ar, function (error, results, fields) {
if (error) {
return reject(error);
}
resolve({results, fields});
});
})//promise
}
const userdetails = (ws, data) => {
//do a check, unrelated to streaming
query('SELECT COUNT(id) as countrows FROM users WHERE category = ? ', [data.category])
.then((data)=>{
if(data.results[0].countrows > 5000){
// if more than 5000, we stream
// the following is based on the mysql code found in their page
// it has no relation to the promise-based query above
var query = pool.query('SELECT id, name, address, sale, preexisting, amount FROM users WHERE category = ? ', [data.category])
query.on('result', row => {
rowsToProcess.push(row);
if (rowsToProcess.length >= 100) {
pool.pause();
processRows();
}
});
query.on('end', () => {
processRows();
});
const processRows = (done) => {
//process some data
//send them back using websockets
ws.send(JSON.stringify({ data }));
pool.resume();
}
}
})
}
I dont know if this is related to making a simple query , a promise or using the pool, or anything else. This gives the TypeError: pool.pause is not a function and I cannot fix it. Please advice.
Thanks
You can try this solution,
I have used this many times:
const mysqlStreamQueryPromise = (queryString, params) => {
return new Promise((resolve, reject) => {
let streamData = connection.query(queryString,params).stream();
let data = [];
streamData.on('data', item => {
streamData.pause();
data.push(item);
streamData.resume();
});
streamData.on('end', end => {
return resolve(data);
});
streamData.on('error', error => {
return reject(error);
});
});
}
Use this
var pool = mysql.createPool({
connectionLimit : 100,
host : config.host,
user : config.user,
password : config.password,
database : config.database
});
//turn simple queries to promises
const query = (str, ar) => {
return new Promise(function(resolve, reject) {
pool.query(str, ar, function (error, results, fields) {
if (error) {
return reject(error);
}
resolve({results, fields});
});
})//promise
}
const userdetails = (ws, data) => {
//do a check, unrelated to streaming
query('SELECT COUNT(id) as countrows FROM users WHERE category = ? ', [data.category])
.then((data)=>{
if(data.results[0].countrows > 5000){
// if more than 5000, we stream
// the following is based on the mysql code found in their page
// it has no relation to the promise-based query above
var query = pool.query('SELECT id, name, address, sale, preexisting, amount FROM users WHERE category = ? ', [data.category]).stream();
query.on('result', row => {
rowsToProcess.push(row);
if (rowsToProcess.length >= 100) {
pool.pause();
processRows();
}
});
query.on('end', () => {
processRows();
});
const processRows = (done) => {
//process some data
//send them back using websockets
ws.send(JSON.stringify({ data }));
pool.resume();
}
}
})
}

Is there a way to pass a value from a mysql callback function to the outer function in express?

I'm using express and npm MySQL to develop an API.I have a json request in this format:
{
"payments":[
{
"PolicyNo": "ME3",
"PaymentDate": "2019-04-16T18:00:00.000Z",
},
{
"PolicyNo": "PIN001q",
"PaymentDate": "2019-04-16T18:00:00.000Z",
}]
}
I want to check the database if the policyNo exists before inserting. To avoid the common ERR_HTTP_HEADERS_SENT, I've looped through the payments querying the database with the PolicyNo. If it exists it's pushed into a success array if it doesn't it's pushed into a failed array.
This works perfectly but I can't access these arrays outside the callback.
Here's what I've tried:
router.post('/bla', (req, res)=>{
const values = []
const failedvalues = []
let sql = 'SELECT PolicyNo from pinclientinfo WHERE PolicyNo=?'
req.body.payments.forEach(element => {
connection.query(sql,element.PolicyNo,(err, rows) =>{
if(!err){
if(rows && rows.length > 0){
values.push(element.PolicyNo, element.PaymentDate)
}else{
failedvalues.push(element.PolicyNo)
}
}
})
})
res.json({
failed:failedvalues,
success:values
})
})
Here's the response I'm getting:
{
"failed": [],
"success": []
}
This has some major problems, mostly conceptually.
Firstly the forEach is synchronous will be called payments.length number of times, but the sql query is Asynchronous so it will complete in the future.
I think you are confused between synchronous and asynchronous functions and how they work.
But you can solve this (in your case) atleast two ways.
1) Use the IN syntax and get the array. Iterate over it and do stuff. "SELECT PolicyNo from pinclientinfo WHERE PolicyNo in (...)"
let sql = 'SELECT PolicyNo from pinclientinfo WHERE PolicyNo IN (' + Array(req.body.payments).fill('?').join(',') + ')'
const policies = req.body.payments.map(p => p.PolicyNo);
const values = [];
const failedvalues = [];
connection.query(sql, ...policies, (err, rows) => {
//now check each row..
rows.forEach(element => {
//Not optimized only depicts logic
///do stuff
/// like fill values and failedvalues
if(policies.indexOf(element.PolicyNo) > -1){
values.push(...)
}else{
failedvalues.push(...)
}
});
res.json({
failed: failedvalues,
success: values
})
})
Which will be 1 DB call.
2) The other approach is (not very good) doing multiple db calls and check for count.
let sql = 'SELECT PolicyNo from pinclientinfo WHERE PolicyNo=?'
let count = 0;
req.body.payments.forEach(element => {
connection.query(sql, element.PolicyNo, (err, rows) => {
if (!err) {
if (rows && rows.length > 0) {
values.push(element.PolicyNo, element.PaymentDate)
} else {
failedvalues.push(element.PolicyNo)
}
}
// check If all Complete
count+=1;
if(count === req.body.payments){
//all complete
res.json({
failed: failedvalues,
success: values
})
}
})
})
BUT SERIOUSLY, USE PROMISE. USE ASYNC/AWAIT USE THOSE SWEET LITTLE FEATURES ES6 GIVES YOU
Check out: this post
because connection.query is asynchronous, so return:
{
"failed": [],
"success": []
}
use promise and await you can synchronized resolve mysql data
use Promise.all() you can synchronized resolve list of promise
router.post("/bla", async (req, res) => {
let values = [];
let failedvalues;
let promises = [];
let sql = "SELECT PolicyNo from pinclientinfo WHERE PolicyNo=?";
req.body.payments.forEach(element => {
promises.push(
new Promise(function(resolve, reject) {
connection.query(sql, element.PolicyNo, (err, rows) => {
if (!err) {
if (rows && rows.length > 0) {
values.push(element.PolicyNo, element.PaymentDate);
} else {
failedvalues.push(element.PolicyNo);
}
}
resolve();
});
})
);
});
await Promise.all(promises);
res.json({
failed: failedvalues,
success: values
});
});