perform async multiple Mysql queries on Node - mysql

I'm using node with Mysql and here's my problem.
I'm trying to add new photos on my database and return it as an array
here is my function :
function addNewPhotos(_id, files) {
var deferred = Q.defer();
var new_photos = []
_.each(files, function (one) {
var data = [
one.path,
_id,
0
]
var sql = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)';
db.connection.query(sql, data, function (err, result) {
if (err)
deferred.reject(err.name + ': ' + err.message);
var sql = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?';
if (result){
db.connection.query(sql, [result.insertId], function(err, photo) {
if (err) deferred.reject(err.name + ': ' + err.message);
if (photo) {
new_photos.push(photo[0]);
}
});
}
})
})
deferred.resolve(Array.prototype.slice.call(new_photos));
return deferred.promise}
The Insert works well but i can't retrieve the results to send them back to the client. (my array is empty)
Thanks.

Always promisify at the lowest level, in this case db.connection.query().
if(!db.connection.queryAsync) {
db.connection.queryAsync = function(sql, data) {
return Q.Promise(function(resolve, reject) { // or possibly Q.promise (with lower case p), depending on version
db.connection.query(sql, data, function(err, result) {
if(err) {
reject(err);
} else {
resolve(result);
}
});
});
};
}
Now the higher level code becomes very simple :
function addNewPhotos(_id, files) {
var sql_1 = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)',
sql_2 = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?';
return Q.all(files.map(function(one) {
return db.connection.queryAsync(sql_1, [one.path, _id, 0]).then(function(result) {
return db.connection.queryAsync(sql_2, [result.insertId]);
});
}));
};
To prevent a single failure scuppering the whole thing, you might choose to catch individual errors and inject some kind of default ;
function addNewPhotos(_id, files) {
var sql_1 = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)',
sql_2 = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?',
defaultPhoto = /* whatever you want as a default string/object in case of error */;
return Q.all(files.map(function(one) {
return db.connection.queryAsync(sql_1, [one.path, _id, 0]).then(function(result) {
return db.connection.queryAsync(sql_2, [result.insertId]);
}).catch(function() {
return defaultPhoto;
});
}));
};

Do the return in your async loop function when all has been done
function addNewPhotos(_id, files) {
var deferred = Q.defer();
var new_photos = [];
var todo = files.length;
var done = 0;
_.each(files, function (one) {
var data = [
one.path,
_id,
0
]
var sql = 'INSERT INTO photos(photo_link, id_user, isProfil) VALUES (?, ?, ?)';
db.connection.query(sql, data, function (err, result) {
if (err)
deferred.reject(err.name + ': ' + err.message);
var sql = 'SELECT id_user, photo_link, isProfil FROM `photos` WHERE id = ?';
if (result){
db.connection.query(sql, [result.insertId], function(err, photo) {
if (err) deferred.reject(err.name + ': ' + err.message);
if (photo) {
new_photos.push(photo[0]);
}
if(++done >= todo){
deferred.resolve(Array.prototype.slice.call(new_photos));
return deferred.promise
}
});
}
else
{
if(++done >= todo){
deferred.resolve(Array.prototype.slice.call(new_photos));
return deferred.promise;
}
}
})
})
}

Related

How to eliminate all queries if one failed

app.post('/transfer', (req,res) => {
db.query('SELECT * FROM bank.user_info where acc_no = ?',
[req.body.from],(err, result) => {
if(err){
res.send({err: err,id: 0});
}else{
if(req.body.Amount>result[0].acc_bal){
res.send({err: "Insufficient Balance !!!",id: 1})
return 0;
}else{
const senderNew = {};
senderNew.amount = req.body.Amount;
senderNew.transaction = 'sent';
senderOld.push(senderNew);
const freshSend = JSON.stringify(senderOld);
const senderBal = result[0].acc_bal-req.body.Amount;
db.query('UPDATE user_info SET transactions = ?, acc_bal = ? WHERE acc_no = ?',
[freshSend, senderBal, req.body.from], (err,result) => {
if(err){
res.send({err: err,id: 2})
}else{
db.query('SELECT * from user_info WHERE acc_no = ?',
[req.body.to], (err, result) => {
if(err){
res.send({err: err,id: 3});
}else{
const receiverOld = JSON.parse(result[0].transactions);
const receiverNew = {};
receiverNew.amount = req.body.Amount;
receiverNew.transaction = 'received';
receiverOld.push(receiverNew);
const freshReceive = JSON.stringify(receiverOld);
const receiverBal = parseInt(result[0].acc_bal) + req.body.Amount;
db.query('UPDATE user_info SET transactions = ?, acc_bal = ? WHERE acc_no = ?',
[freshReceive, receiverBal, req.body.to], (err, result) => {
if(err){
res.send({err: err,id: 4});
}else{
db.query('SELECT * FROM user_info WHERE acc_no = ?',
[req.body.from], (err, final) => {
if(err){
res.send({err: err, id: 5});
}else{
res.send(final);
console.log(final);
}
})
}
})
}
})
}
})
}
}
})
});
Now in this code, if the initial block fails, the others don't execute but if the last ones fail, the initial block still gets passed, how can I improve the code so that if any query fails, the ithers fail too...
I've checked this but it didn't helped as I need a Node.js tutorial
How to stop all DB queries if one fails

Getting a function out of a query

I'm trying to get a variable out of a query as shown below, thanks in advance
var deutsch_meund_note = "";
connection.query(
'SELECT count(*) as sum FROM noten WHERE id = ? and fach = ?',
[id, "mathe"],
function(error, results, fields) {
if (error) {
console.log("no deutsch_schr for id: "+ id);
} else {
const deutsch_meund_viel = results[0].sum;
connection.query(
'SELECT SUM(note) as sum FROM noten WHERE id = ? and fach = ?',
[id, "deutsch_meund"],
function(error, results, fields) {
const deutsch_meund_insge = results[0].sum;
const deutsch_meund_note = deutsch_meund_insge / deutsch_meund_viel;
//this variable: **var deutsch_meund_note = deutsch_meund_note;**
});
}
});
I need to get the variable out of the "connection.query" function but when I try it like the example above it just says "undefined"

Error "too many connections" in Node and MYSQL

I have a script in Node that makes about 4 or 5 requests to MYSQL, but it seems that there is some overload, because after inserting 4 or 5 records I get: "Too many connections"
Below I leave my code
const puppeteer = require('puppeteer');
var fs = require('fs');
const mysql = require('mysql2');
const mysqlConOptions = {
connectionLimit : 10, // default = 10
host: "127.0.0.1",
user: "root",
password: "root123!",
database: "scraperpeli"
};
var con = mysql.createPool(mysqlConOptions);
con.getConnection(function(err) {
if (err) throw err;
console.log("Conectado!");
});
function insertarpost()
{
return new Promise(function(resolve, reject) {
var sql = "INSERT INTO `wp_posts` ( `post_author`, `post_date`, `post_date_gmt`, `post_content`, `post_title`,`post_excerpt`, `post_status`,`comment_status`, `ping_status`,`post_password`, `post_name`,`to_ping`,`pinged`, `post_modified`, `post_modified_gmt`,`post_content_filtered`,`post_parent`, `guid`,`menu_order`, `post_type`, `post_mime_type`,`comment_count`) VALUES (1, '"+fhoy+"', '"+fhoy+"', '"+description+"', '"+title+"','', 'publish', 'open', 'closed','','"+slug+"','','', '"+fhoy+"', '"+fhoy+"','', '0','','0', 'movies','' ,0);";
con.query(sql, function (err, result) {
if (err) throw err;
resolve(result);
console.log("1 registro insertado");
});
});
}
insertarpost().then(obtenerultimopostid);
function obtenerultimopostid() {
return new Promise(function(resolve, reject) {
con.query("SELECT ID FROM wp_posts ORDER BY ID DESC LIMIT 0,1;", function (err, result, fields) {
console.log(result);
var sql = "INSERT INTO wp_postmeta (post_id, meta_key, meta_value) VALUES ("+result[0].ID+",'runtime', '"+runtime+"'),("+result[0].ID+",'original_title','"+titleoriginal+"'),("+result[0].ID+",'Rated','"+cpgrated+"'),("+result[0].ID+",'Country', '"+country+"'),("+result[0].ID+",'date', '"+datemovie+"'),("+result[0].ID+",'imdbRating', '"+repimdb+"'),("+result[0].ID+",'vote_average', '"+reptmdb+"'),("+result[0].ID+",'imdbVotes', '"+quantimdb+"'),("+result[0].ID+",'vote_count', '"+quanttmdb+"'),("+result[0].ID+",'tagline', '"+tagline+"'),("+result[0].ID+",'dt_poster', '"+poster+"'),("+result[0].ID+",'dt_backdrop', '"+backdrop+"'),("+result[0].ID+",'imagenes', '"+backdrops+"'),("+result[0].ID+",'dt_cast', '"+textimgreparto+"'),("+result[0].ID+",'dt_dir', '"+textimgreparto2+"');";
con.query(sql, function (err, result) {
if (err) throw err;
resolve(result);
console.log("1 registro wpmeta insertado");
});
});
}).then(function() {
let c1='';
con.query("SELECT ID FROM wp_posts ORDER BY ID DESC LIMIT 0,1;", function(err, result, fields) {
console.log(result);
for (var i = 0; i < getData.length; i++) {
let source = getData[i]["source"];
let text = getData[i]["text"];
let quality = getData[i]["quality"];
let lang = getData[i]["language"];
new Promise(function(resolve, reject) {
let sql = "INSERT INTO `wp_posts` ( `post_author`, `post_date`, `post_date_gmt`, `post_content`, `post_title`,`post_excerpt`, `post_status`,`comment_status`, `ping_status`,`post_password`, `post_name`,`to_ping`,`pinged`, `post_modified`, `post_modified_gmt`,`post_content_filtered`,`post_parent`, `guid`,`menu_order`, `post_type`, `post_mime_type`,`comment_count`) VALUES (1, '" + fhoy + "', '" + fhoy + "', '', '" + make + "','', 'publish', 'closed', 'closed','','" + make + "','','', '" + fhoy + "', '" + fhoy + "','', '" + result[0].ID + "','','0', 'dt_links','' ,0);";
con.query(sql, function(err, result) {
if (err) throw err;
c1=result.insertId
resolve(result);
console.log("1 registro link insertado");
});
}).then(function() {
console.log(c1);
let sql = "INSERT INTO wp_postmeta (post_id, meta_key, meta_value) VALUES (" + c1 + ",'_dool_url', '" + source + "'),(" + c1 + ",'_dool_type','" + text + "'),(" + c1 + ",'_dool_quality','" + quality + "'),(" + c1 + ",'_dool_lang','" + lang + "');";
con.query(sql, function(err, result) {
if (err) throw err;
console.log("1 registro link meta insertado");
});
// });
});
}
});
}
I don't know why that error comes out, too many queries, am I doing something wrong? thanks.
I have tried increasing the connection limit but nothing, I hope you can guide me to the resolution of the problem

Node.js chaining promises in a loop with MySQL

I am trying to make a non-relational DB into a relational DB. So I am starting from data with no unique IDs.
I need to get the result from one SQL call loop through those rows, and for each one, do a SQL SELECT using part of the first result, then another SQL select using the next result, and then a write using IDs from the first and last queries.
I am using Node.js and ES6 promises to keep everything in order, but I seem to be missing something. I was actually trying to do an extra SQL call, and also use that result in the third query, but I am simplifying it to just get one call to feed into another.
Maybe some code will help show what I am trying to do.
Here is my query class that returns promises:
var mysql = require('mysql');
class Database {
constructor() {
this.connection = mysql.createConnection({
host: "localhost",
user: "root",
password: "root",
database: "pressfile"
});
}
query(sql, args) {
return new Promise((resolve, reject) => {
this.connection.query(sql, args, (err, result, fields) => {
if (err) return reject(err);
resolve (result);
});
});
}
close() {
return new Promise((resolve, reject) => {
this.connection.end(err => {
if (err) return reject (err);
resolve();
});
});
}
}
This was stolen pretty much as is from a tutorial site, and this part seems to work pretty well. Then here comes the loop, and the multiple queries:
var contactId;
var address1;
var orgName;
var database = new Database();
database.query("SELECT * FROM contact")
.then( result => {
for (var i = 0; i < result.length; i++) {
contactId = result[i].contactId;
orgName = result[i].org;
var sql2 = "SELECT * FROM organization WHERE (name = \"" + orgName + "\")";
console.log(sql2);
database.query(sql2)
.then(result2 => {
console.log(result2);
var orgId = result2[0].organizationId;
var sql3 = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
console.log(sql3);
return ""; //database.query(sql3);
}).then( result3 => {
console.log(result3);
});
}
}).catch((err) => {
console.log(err);
databse.close();
});
I know it is kind of unraveling at the end, but I'm not wanting to do the INSERT query until I know I can get it right. Right now in the console, I get a valid organization object, followed by:
`INSERT INTO contact_organization (contactId, organizationId) VALUES (17848, 29)'
17848 is the final contactId that is returned in the for loop. How can I get the contactId that is assigned before the second query. I know I am not doing this asynchronous stuff right.
Try something like this. Just a quick solution. (not tested).
const selectOrg = (result) => {
contactId = result[i].contactId;
orgName = result[i].org;
var sql = "SELECT * FROM organization WHERE (name = \"" + orgName + "\")";
return database.query(sql);
};
const insertOrg = (result) => {
var orgId = result[0].organizationId;
var sql = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
return database.query(sql);
};
database.query("SELECT * FROM contact")
.then(result => {
const promises = [];
for (var i = 0; i < result.length; i++) {
promises << selectOrg(result)
.then(insertOrg);
}
return Promise.all(promises);
})
.then(allResults => {
console.log(allResults);
})
.catch((err) => {
databse.close();
});
I found a way to do this, but it is kind of cheesy. I included the contactId as a constant in the SQL query to get the organization, so I could then pass the value to the .then, keeping everything in order.
My sql2 statement becomes:
var sql2 = "SELECT *, " + contactId + " AS contactId FROM organization WHERE (name = \"" + orgName + "\")";
Then when that query returns, I can just pull the correct contactId out as result[0].contactId, from the same result I get the organizationId from.
Here is the final code:
database.query("SELECT * FROM contact")
.then( result => {
for (var i = 0; i < result.length; i++) {
var contactId = result[i].contactId;
var orgName = result[i].org;
var sql2 = "SELECT *, " + contactId + " AS contactId FROM organization WHERE (name = \"" + orgName + "\")";
database.query(sql2)
.then(result2 => {
var orgId = result2[0].organizationId;
var contactId = result2[0].contactId;
var sql3 = "INSERT INTO contact_organization (contactId, organizationId) VALUES (" + contactId + ", " + orgId + ")";
console.log(sql3);
return database.query(sql3);
}).then( result3 => {
console.log(result3);
});
}
}).catch((err) => {
console.log(err);
databse.close();
});
The console.log(result3) returns a bunch of these:
OkPacket {
fieldCount: 0,
affectedRows: 1,
insertId: 0,
serverStatus: 2,
warningCount: 0,
message: '',
protocol41: true,
changedRows: 0 }
And I got one contact_organization inserted for every contact row returned from the first query.

Ignore update data if exist in mysql by node js

I'm going to check id of my data if exist in mysql update it and if not create it, but in select result it dosen't work and go to esle section and create action most of the times and got Duplicate entry error. at the end I run this file by infinite loop.
I should tell this when I have a lot data in redis this problem will happen.
I'll be glad if help me, thanks :)
Here is my code :
var
mysql = require('mysql'),
redis = require('redis'),
infiniteLoop = require('infinite-loop'),
client = redis.createClient(),
lop = new infiniteLoop,
config = require( __dirname + '/app/Config').config,
con = function() {
return mysql.createConnection({
host : config.db_host,
user : config.db_user,
password : config.db_pass,
database : config.db_name
});
},
insertDB = function( data ) {
var connection = con();
connection.connect( function( err ) {
connection.query( 'SELECT * FROM real_user WHERE id =' + data.id, function( err, res ) {
if ( err ) {
data = JSON.stringify(data);
client.lpush('aipi', data, function(err,reply) {
if ( err ) throw err;
});
}
console.log( "out", data.id, res.length, res );
if ( typeof res != 'undefined' && res.length > 0 ) {
console.log( "update", data.id );
connection.query( 'UPDATE real_user SET ? WHERE id =' + res[0].id, { request_count: data.request_count, updated_at: data.created_at },
function( err, res ) {
if ( err ) throw err;
});
} else {
console.log( "create", data.id );
connection.query( 'INSERT INTO real_user SET ?', data, function( err, res ) {
if ( err ) throw err;
});
}
connection.end();
});
});
},
run = function() {
var data;
client.brpop('aipi', 0, function(err, reply) {
if ( reply[1] ) {
data = JSON.parse(reply[1]);
insertDB(data);
}
});
};
lop.add(run, []).run();
Fixed it by :
ON DUPLICATE KEY UPDATE
connection.connect( function( err ) {
connection.query( "INSERT INTO real_user SET ? ON DUPLICATE KEY UPDATE updated_at = '" + data.created_at
+ "', request_count = '" + data.request_count + "'",
data, function( err, res ) {
connection.end();
});
});