Loop through MySQL rows and store results in array - mysql

I am trying to store details of affectedRows from a MySQL INSERT query using NodeJS. My mind is melting trying to comprehend callbacks and Promises. Being a single-man dev team I wanted to reach out and ask for the clearest explanation as to how a callback can be applied here in a foreach loop.
The goal should be clear from these few lines of code; store data in the affected_rows[] array.
var affected_rows = [];
asset_array.forEach(function(asset) { // Populate the asset table
var query_string = "INSERT IGNORE INTO " + asset_table + " SET symbol = '" + asset[0] + "', name = '" + asset[1] + "'";
connection.query(query_string, function(err, rows, fields) {
if (err) throw err;
if ( rows.affectedRows > 0 ) {
data_to_push = [asset_table, asset[0], asset[1]];
affected_rows.push(data_to_push);
}
});
});
console.log(affected_rows); // [] for obvious async reasons

One option would be to process the asset_array inside a function and pass a callback into it and when loops through asset_array check if the current index matches the asset_array length (-1). If so call the callback.
var affected_rows = [];
function processAssets(cb) {
var array_len = asset_array_len.length
asset_array.forEach(function(asset, index) {
var query_string = 'INSERT IGNORE INTO ' + asset_table + ' SET symbol = \'' + asset[0] + '\', name = \'' + asset[1] + '\'';
connection.query(query_string, function(err, rows, fields) {
if (err) throw err
if (rows.affectedRows > 0) {
data_to_push = [asset_table, asset[0], asset[1]];
affected_rows.push(data_to_push);
}
if (index === (array_len - 1)) cb()
});
});
}
processAssets(function() {
console.log(affected_rows)
})

Will suggest you to have a look at async Queue.
You can change your code like this to use it.
//2nd Step - Perform each task and then call callback() to move to next task
var q = async.queue(function(query_string, callback) {
connection.query(query_string, function(err, rows, fields) {
if (err) throw err;
if ( rows.affectedRows > 0 ) {
data_to_push = [asset_table, asset[0], asset[1]];
affected_rows.push(data_to_push);
}
callback(); //call next task
});
}, 2); //here 2 means concurrency ie 2 tasks will run in parallel
//Final Step - Drain gives you end of queue which means all tasks have finished processing
q.drain = function() {
//Do whatever you want after all tasks are finished
};
//1st Step - create a queue of all tasks that you need to perform
for (var i = 0; i < asset_array.length ; i++) {
var query_string = "INSERT IGNORE INTO " + asset_table + " SET symbol = '" + asset[0] + "', name = '" + asset[1] + "'";
q.push(query_string);
}

Related

NodeJS and MYSQL Issue

My code is supposed to retrieve the "DropsRemaining" for a user ID (which it does successfully) and then -1 from the number it retrieves. When retreiving the data it returns this string "[ RowDataPacket { DropsRemaining: 5 } }" however the end of the code does not -1 from the DropsRemaining it instead sets the DropsRemaining to -1. If anyone can help fix this I'd really appreciate it.
var sql = "SELECT DropsRemaining FROM UserData WHERE DiscordID LIKE " + message.author.id;
var DropCount = [];
connection.query(sql, function (err, result) {
if (!err)
setValue(result);
else
console.log("No Information For That User Found");
});
function setValue(value) {
DropCount = value;
console.log(DropCount);
};
//Remove drop from user
DropCount = DropCount - 1;
var sql = "UPDATE UserData SET DropsRemaining = " + DropCount + " WHERE DiscordID = " + message.author.id;
The problem is that the order you write your Javascript code is not exactly how it is performed in the end.
When you call the connection.query() function, the next line of code does not necessarily already have the result of that function.
I would recommend you to have a look on this book series, they have really good explanations about those characteristics.
Probably the following code will output the expected response. Notice that I nested the code, therefore I can control the flow properly.
var sql = "SELECT DropsRemaining FROM UserData WHERE DiscordID LIKE " + message.author.id;
// Get the DropsRemaining
connection.query(sql, function (err, result) {
if (!err) {
// No errors in the query, decrement the Drops
decrementDrop(result);
} else {
console.log("No Information For That User Found");
}
});
function decrementDrop(dropsAvailable) {
var dropsRemaining = dropsAvailable - 1;
var updateSql = "UPDATE UserData SET DropsRemaining = " + dropsRemaining + " WHERE DiscordID = " + message.author.id;
// Update the DropsRemaining column to the dropsRemaining, i.e., decrement the DropsRemaining column value
connection.query(updateSql, function (err, result) {
if (!err) {
console.log("DiscordID = " + message.author.id +" has " + dropsRemaining + " drops remaining")
} else {
console.log("Error!");
}
});
}

Nodejs multiple sql query loop

I am pretty new to nodejs and async worlds.
The case is, I have an array like var ids = [1, 2, 3, 4];
I need to update mytable according to sequence of the array element. So I do something like:
sort: function(ids, callback) {
// dont worry about this
this.create_connection();
this.connection.connect();
for(var i=0; i<ids.length;i++) {
var q = "UPDATE mytable SET sequence="+i+" where id="+ids[i]+"; ";
this.connection.query(q, function(err, result) {
// I am not sure about this
// callback(err);
});
}
// I need to return callback at the end
// return callback();
this.connection.end();
}
But yes.. it does not work because I have to return callback.. I think I need to do the query syncronously.. I am not sure. Please help thanks.
If you are new to async worlds, you should take a look at module 'async'.
You can then do something like this :
async.forEachOfSeries(ids, function(id,index,callback){
var q = "UPDATE mytable SET sequence="+index+" where id="+id+"; ";
this.connection.query(q, function(err, result) {
callback();
});
},function done(){
// whatever you want to do onces all the individual updates have been executed.
})
See my inline comments:
sort: function(ids, callback) {
this.create_connection();
this.connection.connect();
var q = "UPDATE mytable SET sequence CASE id ";
// Don't execute one query per index in ids - that's inefficient
// Instead, pack up all the queries and execute them at once
for(var i=0; i<ids.length;i++) {
q += "WHEN " + ids[i] + " THEN " + i + " ";
}
q += "ELSE sequence END;";
// The sort method will return the result of connection.query
return this.connection.query(q, function(err, result) {
// End the connection
this.connection.end();
if(err) {
// Handle any error here
return callback(err);
}
// Otherwise, process, then return the result
return callback(err, result);
});
}
And here's something slightly more elegant:
sort: function(ids, callback) {
this.create_connection();
this.connection.connect();
// Don't execute one query per index in ids - that's inefficient
// Instead, pack up all the queries and execute them at once
var q = ids.reduce(function(pv, cv, ci){
return pv + " WHEN " + cv + " THEN " + ci + " ";
}, "UPDATE mytable SET sequence CASE id ") + " ELSE sequence END;";
// The sort method will return the result of connection.query
return this.connection.query(q, function(err, result) {
// End the connection
this.connection.end();
if(err) {
// Handle any error here
return callback(err);
}
// Otherwise, process, then return the result
return callback(err, result);
});
}
And you can replace the .reduce in the previous example with the following, if you want to use ES6 arrow functions:
var q = ids.reduce((pv, cv, ci) => pv + " WHEN " + cv + " THEN " + ci + " ",
"UPDATE mytable SET sequence CASE id ") + " ELSE sequence END;";

Nested mysql query in each loop

So all i want to do is to make a query to get all id's from the database and then i want to do my method for the id and update the row in the database.
So i created this code. But all it is doing is to get the ids and do not the nested query with the insertString1.
var mysql = require('mysql');
var _ = require("underscore");
var connection = mysql.createConnection({
host : 'myHost.net',
user : 'myUser',
password : 'myPassword',
database : 'myDatabase'
});
connection.query('SELECT * FROM `idmarkethash`', function (error, results) {
_.each(results, function (row) {
// here i want to calculate the new values of the row
// This insert query is just an example
var insertString1 = "UPDATE idmarkethash SET " +
"Price=" + mysql.escape(2.00) + ", " +
"Gradient=" + mysql.escape(2.00) + ", " +
"LastUpdated=" + mysql.escape("2015-08-31 00:00:11") + " " +
"WHERE Id=" + mysql.escape(181) + ";";
connection.query(insertString1, function (error1, result1) {
console.log(result1);
});
});
});
I tried the same with the async package:
var async = require('async');
// Packages are all included.
connection.query('SELECT * FROM `idmarkethash`', function (error, results) {
async.each(results, function (row, callback) {
// here i want to calculate the new values of the row
// This insert query is just an example
var insertString1 = "UPDATE idmarkethash SET " +
"Price=" + mysql.escape(2.00) + ", " +
"Gradient=" + mysql.escape(2.00) + ", " +
"LastUpdated=" + mysql.escape("2015-08-31 00:00:11") + " " +
"WHERE Id=" + mysql.escape(181) + ";";
connection.query(insertString1, function (error1, result1) {
console.log(result1);
callback();
});
});
});
With debugging i could see that the declaration of insertString1 is called, but the second query is not fired / or the loop continues without recognizing the second query.

NodeJS and MySQL: getting array data inside query callback

I'm trying to run this piece of code:
setInterval(function () {
var params = {
QueueUrl: 'https://sqs.us-east-1.amazonaws.com/821808622769/Teste', // required
MaxNumberOfMessages: 10
};
sqs.receiveMessage(params, function (err, data) {
if (err)
console.log('Erro de SQS:' + err);
else {
var retorno = data.Messages;
for (var i in data.Messages) {
var queryString = 'SELECT now()'; //dummy query, just for tests
db_connection.query(queryString, null, function (err, rows, fields) {
if (err) {
console.log('Erro no BD:' + err);
return;
}
var date = new Date();
console.log(retorno[i].Body + ' ' + date.getTime().toString());
});
}
console.log();
}
});
}, 30000);
and I have 5 messages in AWS SQS, such as this:
but when I run my code, instead of having the retorno[i].Body of each message, sometimes I get a message repeated, as show in this image
My for loop is running from 1 to 5 queries, but how do I carry the retorno[i] inside the callback to the database query? I mean, how do I identify which message I was dealing with?
Using .bind() you can setting the internal version of i to be the same as the version of i in your loop.
.bind({i:i}) at the end of your callback and change the internal code to reference this.i
for (var i in data.Messages) {
var queryString = 'SELECT now()'; //dummy query, just for tests
db_connection.query(queryString, null, function(err, rows, fields) {
if (err) {
console.log('Erro no BD:' + err);
return;
}
var date = new Date();
console.log(retorno[this.i].Body + ' ' + date.getTime().toString());
}.bind({
i: i
});
}
}

multiple async mongo request generate messed up returns

I'm trying to build a JSON out of multiple requests on my mongodb.
since I'm not using DBRef, I have to build the "table joints" by myself, and that's how I ended up in this mess.
This is the code that is giving me the headaches from a couple of days now.
(the mongo part is done with mongoskin)
var getUserFeed = function(thelimit, out) {
userfeed = db.collection("userfeed");
apparel = db.collection("apparel");
store = db.collection("stores");
if(thelimit)
args = {limit:thelimit, sort: [['date',-1]]};
userfeed.find({},args).toArray(function(e, feed) {
if (e) console.log("error: ", e);
// gather aparel infos
var i=0;
var ret_feeds = [];
feed.forEach(function(cur_feed) {
var outfits=[];
console.log("beginning with: " + cur_feed.url);
var resfeed = "";
resfeed = cur_feed;
resfeed.url = baseurl + snapurl + resfeed.url + "_small.jpg";
i=0;
cur_feed.apparel_ids.forEach(function(item) {
/*>>*/ apparel.find({"_id": item},{limit:1}).toArray(function(e, results) {
console.log(">>>>>>>>>>> APPAREL_FIND { i:" + i + "}");
if (e) console.log("error: ", e);
results = results[0];
if(results.apparel_cat == 1)
url_subcat = "pants/";
else if(results.apparel_cat == 2)
url_subcat = "shirts/";
else if(results.apparel_cat == 2)
url_subcat = "tshirts/";
results.thumb = baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg";
results.size = "M"; ///// TOBE REAL VERY SOON
results.gallery = [
baseurl + outfiturl + url_subcat + results.apparel_id + "/model.jpg",
baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg"
];
outfits.push(results); // quick and dirty, 2 b refined..
i++;
if(i>=cur_feed.apparel_ids.length)
{
// pack it up
// resfeed.url = resfeed.url;
resfeed.outfits = outfits;
resfeed.fav = false;
resfeed.bough = false;
// retrieve store infos
/*>>>*/ store.find({"_id":resfeed.store_id}, {limit: 1}).toArray(function(e, resstore) {
console.log("\t############# STORE_FIND { i:" + i + "}");
if (e) console.log("error: ", e);
resfeed.store = resstore[0];
resfeed.store.class = "hem";
ret_feeds.push(resfeed);
if(ret_feeds.length >= feed.length)
{
console.log("\t\t######################calling return [ ret_feeds.length = " + ret_feeds.length + " feed.length = " + feed.length);
out.send(ret_feeds);
}
});
}
});
});
});
});
}
This code fails, because returns the json before finishing its task, so the next time that it tries to return another json it crashes miserably due to the fact the the headers have already been sent.
Now as you can see, I have 3 collections: userfeed, apparel and stores.
the goal of this function is to retrieve all the items in the userfeed collection, extract the outfits (based on the outfit_id array that is part of the userfeed collection), and also extract the store infos related in the same way to each userfeed entry, like so:
I know that async.js or equivalent is the way to go: I've red like a gazillion of other posts here on SO, but I still can't get my head around it, probably because the whole mechanism behind the async.js or flow control in general it's still out of focus in my mind.
I'm still a noob at node :)
UPDATE
I think I found the right path for understanding here: http://www.sebastianseilund.com/nodejs-async-in-practice
this guy made a terrific job in describing use-case by use-case all the ways to apply async.js to your code.
I'll post the solution as soon as I get around it.
UPDATE 2
Thanks to the above dude I could work out a working solution, below is the answer.
After so much struggling I have finally managed to get a solution.
async.js was the answer as I was (obviously) suspecting.
FYI here's the working code.
If you like to point out improvements or anything else, you are more than welcome
var getUserFeed = function(thelimit, out) {
userfeed = db.collection("userfeed");
apparel = db.collection("apparel");
store = db.collection("stores");
var args;
if(thelimit)
args = {limit:thelimit, sort: [['date',-1]]};
var outfits=[];
var feeds = array();
async.series([
// userfeed find
function(callback) {
userfeed.find({},args).toArray(function(e, feed) {
if(e) callback(e);
feeds = array(feed);
console.log(feeds.length + " retrieved. stepping in");
callback(null, null);
});
},
// join
function(callback) {
async.forEach(feeds, function(thefeed, callback) {
var i = feeds.indexOf(thefeed);
async.parallel([
// load apparel infos
function(callback) {
console.log("\t >>> analyzing thefeed id " + thefeed._id);
async.forEach(thefeed.apparel_ids, function(apparel_id, callback) {
apparel.find({"_id": apparel_id},{limit:1}).toArray(function(e, results) {
if (e) console.log("error: ", e);
results = results[0];
if(results.apparel_cat == 1)
url_subcat = "pants/";
else if(results.apparel_cat == 2)
url_subcat = "shirts/";
else if(results.apparel_cat == 2)
url_subcat = "tshirts/";
results.thumb = baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg";
results.size = "M"; ///// TOBE REAL VERY SOON
results.gallery = [
baseurl + outfiturl + url_subcat + results.apparel_id + "/model.jpg",
baseurl + outfiturl + url_subcat + results.apparel_id + "/front.jpg"
];
console.log("\t\t### pushing data into thefeed_index: " + i);
if(!util.isArray(feeds[i].oufits)) feeds[i].outfits = array();
feeds[i].outfits.push(results);
callback(null, null);
});
}, callback);
},
// load store infos
function(callback) {
store.find({"_id":thefeed.store_id}, {limit: 1}).toArray(function(e, resstore) {
console.log("\t### STORE_FIND");
if (e) console.log("error: ", e);
feeds[i].store = resstore[0];
feeds[i].store.class = "hem";
callback(null, null);
});
}
], callback);
}, callback);
}
// MAIN
], function(err, result) {
console.log("feed retrieval completed. stepping out");
if (err) return next(err);
out.send(feeds);
});
};