Using Q to return secondary query in node with express and mysql - mysql

New to node, As I am cycling through a roster of students, I need to check and see if a teacher has requested them for tutoring.
I realized I can't just do this:
var checkRequest = function(id){
var value = '';
roster.query('SELECT * FROM teacher_request WHERE student_id ='+id, function(err, row){
value = row.length;
}
return value;
}
After a bit of digging around promises looked like a great solution, but if I simply return the deferred.promise from the checkRequest function, all I get is an object that says [deferred promise] which I can't access the actual data from. (Or have not figured out how yet)
If I follow along with their api and use .then (as illustrated in the getRow) function, I am back in the same problem I was in before.
function checkRequest(id) {
console.log(id);
var deferred = Q.defer();
connection.query('SELECT * FROM teacher_request WHERE student_id ='+id, function(err, row){
deferred.resolve(row.length);
});
return deferred.promise;
}
var getRow = function(id){
checkRequest(id).then(function(val) {
console.log(val); // works great
return val; //back to the same problem
});
}
The roster needs to be able to be pulled from an external API which is why I am not bundling the request check with the original roster query.
Thanks in advance

From the stuff you posted, I assume you have not really understood the concept of promises. They allow you to queue up callbacks, that get executed, when the asynchronous operation has finished (by succeeding or failing).
So instead of somehow getting the results back to your synchronous workflow, you should convert that workflow to work asynchronous as well. So a small example for your current problem:
// your students' ids in here
var studentsArray = [ 1, 2, 5, 6, 9 ];
for( var i=0; i<studentsArray.length; i++ ) {
checkRequest( i )
.then( function( data ){
console.log( data.student_id );
// any other code related to a specific student in here
});
}
or another option, if you need all students' data at the same time:
// your students' ids in here
var studentsArray = [ 1, 2, 5, 6, 9 ];
// collect all promises
var reqs = [];
for( var i=0; i<studentsArray.length; i++ ) {
reqs.push( checkRequest( i ) );
}
Q.all( reqs )
.then( function(){
// code in here
// use `arguments` to access data
});

Related

node.js: put asynchronous returns in one object or array

I am missing something fundamental in terms of callbacks/async in the code below: why do I get:
[,,'[ {JSON1} ]']
[,,'[ {JSON2} ]']
(=2 console returns) instead of only one console return with one proper table, which is want I want and would look like:
[,'[ {JSON1} ]','[ {JSON2} ]']
or ideally:
[{JSON1},{JSON2}]
See my code below, getPTdata is a function I created to retrieve some JSON via a REST API (https request). I cannot get everything at once since the API I'm talking to has a limit, hence the limit and offset parameters of my calls.
offsets = [0,1]
res = []
function goGetData(callback) {
for(var a = 0; a < offsets.length; a++){
getPTdata('stories',
'?limit=1&offset='+offsets[a]+'&date_format=millis',
function(result){
//called once getPTdata is done
res[a] = result
callback(res)
});
}
}
goGetData(function(notgoingtowork){
//called once goGetData is done
console.log(res)
})
Solved like this:
offsets = [0,1]
res = []
function goGetData(callback) {
var nb_returns = 0
for(var a = 0; a < offsets.length; a++){
getPTdata('stories','?limit=1&offset='+offsets[a]+'&date_format=millis', function(result){
//note because of "loop closure" I cannot use a here anymore
//called once getPTdata is done, therefore we know result and can store it
nb_returns++
res.push(JSON.parse(result))
if (nb_returns == offsets.length) {
callback(res)
}
});
}
}
goGetData(function(consolidated){
//called once goGetData is done
console.log(consolidated)
})

how to ensure request is done sequentially?

I'd like to request one page after another, the following code seems request all the pages in the same time, is there a way to fetch one page after the previous one is done? thanks
var Promise = require("bluebird");
var request = Promise.promisifyAll(require('request'));
var URLS = ["http://sample.com/j1", "http://sample.com/j2"]
Promise.map(URLS, function (item) {
return request.postAsync({url: item}).spread(function (response,body) {
var items = JSON.parse(body)
return items
})
}).then(function (r) {
console.log(r.length)
})
You can set the concurrency level, which is specific to bluebird.
Promise.map(URLS, function (item) {
return request.postAsync({url: item}).spread(function (response,body) {
var items = JSON.parse(body)
return items
}, { concurrency: 1})
This will issue all the promises one at a time.

Yahoo-Finance Query Speed

I'm currently working on a project that involves querying yahoo-finance for many different ticker symbols. The bottleneck is acquiring the data from yahoo, so I was wondering if there is a way I might go about speeding this up.
If I used multiple machines to query and then aggregated the data, would that help? I only have one physical machine; how might I go about doing that?
Thanks!
EDIT: Currently, I'm using Node.js, yahoo-finance, and Q.deferred to ask yahoo for historical data. Then, once all the promises are fulfilled (for each ticker), I'm doing a Q.all() to persist the data.
var data = [];
tickers = ["goog", "aapl", ...];
...
Q.all(_.map(tickers, function(symbol) {
return getYahooPromise(symbol);
}))
.done( function() { persistData(data) });
getYahooPromise retrieves data for the ticker symbol and pushes it into the data array. Once all promises are resolved, the data is persisted in a MySQL database.
SECOND EDIT:
More code:
var sequentialCalls = [];
for ( var i = 0; i < tickers.length / chunkSize; i++ ) {
sequentialCalls.push( persistYahooChunk );
}
sequentialCalls.push( function(callback) {
connection.end();
callback();
});
async.series( sequentialCalls )
exports.persistYahooChunk = function(callback) {
console.log("Starting yahoo query");
var currentTickers = tickers.slice(currentTickerIndex,currentTickerIndex + chunkSize);
return yahooFinance.historical( {
symbols: currentTickers,
from: "2015-01-28",
to: "2015-02-05"
}).then( function(result) {
console.log("Query " + currentTickerIndex + "/" + tickers.length + "completed");
currentTickerIndex += chunkSize;
//add valid data
var toPersist = _.map(result, function(quotes, symbol) {
return [symbol, quotes.length != 0 ];
});
var query = "INSERT INTO `ticker` (`symbol`, `valid`) VALUES ?";
connection.query(query, [toPersist], function(err, result) {
if (err) {
console.log (err);
}
//console.log(result);
callback();
});
});
}
The bottleneck is because you are doing one query per ticker.
Depending on the data you need to pull, if you could do a single query that includes all your tickers it would be much faster.
Here is an example if you need to get all current prices for a list of tickers, with a single query :
http://finance.yahoo.com/webservice/v1/symbols/A,B,C,D,E/quote?format=json

Read records from firebase based on a previously saved value

I took an angularjs + firebase example and modified it for an app where I can register some kids for a small cross-country race.
I'm able to register kids (participants), races, locations, clubs etc. using a basic structure:
FIREBASE_URL/races
FIREBASE_URL/clubs
and so forth. When the active race is selected, I save the raceId and race json-object and can add participants to the active race.
Example:
FIREBASE_URL/active_race/-JI6H9VQewd444na_CQY
FIREBASE_URL/active_race/json-object
What I'd like to do is to get all the participants, if any, based on raceId:
FIREBASE_URL/races/-JI6H9VQewd444na_CQY/participants
I tried the following
'use strict';
app.factory('Race', function ($firebase, FIREBASE_URL, User) {
var ref = new Firebase(FIREBASE_URL + 'races');
var races = $firebase(ref);
var Race = {
all: races,
getParticipantsInRace: function () {
var fb = new Firebase(FIREBASE_URL);
fb.child('active_race/raceId').once('value', function (activeSnap) {
races.$child('/' + activeSnap.val() + '/participants');
});
}
};
return Race;
But I believe I'm doing it wrong. I tried to prepend return before races.$child and fb.child but it did not solve my problem.
I tried to hardcode the following json-array and this is shown on the webpage:
return [{name: 'Claus', born: '1967'}, {name: 'John', born: '1968'}];
How do I get all the participants into $scope.participantsInRace?
I believe I have a solution, but I'm not sure if it's wise to do it this way. But it may be that simple. Prepending $rootScope.participantsInRace = to put it into rootScope:
$rootScope.participantsInRace = races.$child('/' + activeSnap.val() + '/participants');
The code is already synchronizing all data in all races when it declares $firebase(URL+'races');. Additionally, you never assigned your races.$child(...) to anything, so it's not possible to reference that data later.
app.factory('Race', function ($firebase, FIREBASE_URL, User) {
var ref = new Firebase(FIREBASE_URL + 'races');
var races = $firebase(ref);
var Race = {
all: races,
getParticipantsInRace: function (raceId) {
return races[raceId]? races[raceId].participants || {};
}
};
return Race;
});
Keep in mind that the race data won't be available until races.$on('loaded') is invoked (when the data returns from the server).
Thank you for the input. I know a bit more about angularjs and javascript now so I did some refactoring and cleanup. Hardcoding raceId works:
getParticipantsInRace: function () {
return races.$child('-JIecmbdDa4kUT2L51iS').$child('participants');
}
When I wrap it in a call to Firebase I can't seem to return the desired data, probably due to my somewhat limited knowledge of javascript on how to return data. Example:
getParticipantsInRace: function () {
ref.child('activeRace').child('raceId').once('value', function (activeSnap) {
return races.$child(activeSnap.val()).$child('participants');
});
}
My idea is to get the raceId and then return all participants. I tried to prepend return to ref.child() but still no data was returned. So not really an answer.
Regards
Claus
This works. I changed $rootScope.participantsInRace to $scope.participantsInRace and the following:
getParticipantsInRace: function () {
if (User.signedIn()) {
var t = [];
var user = User.getCurrent();
var fb = new Firebase(FIREBASE_URL + 'users');
fb.child(user.username).child('activeRace/raceId').once('value', function (userSnap) {
t = races.$child(userSnap.val()).$child('participants');
});
return t;
}
},

How to loop through indexedDB tables synchronously?

I want to write a function in JS where I will loop through a tables in my indexed DB and get the maximum value of last modified of table and return that
function readData(){
var trans = '';
trans = idb.transaction(["tableName"],'readonly'); // Create the transaction
var request = trans.objectStore("tableName").openCursor();
request.onsuccess = function(e) {
var cursor = request.result || e.result;
if(cursor) {
// logic to and find maximum
} else {
return // max last modified
}
cursor.continue();
}
}
IMP--Since onsuccess method is asynchronous how can i make it synchronous? so that my method readData() will return only when max last modified record is found successfully. I can call this method(readData()) synchronously to get last modified record of 2-3 tables if I want.
The sync API is only available in a webworker. So this would be the first requirement. (As far as I know only IE10 supports this at the moment)
An other shot you can give is working with JS 1.7 and use the yield keyword. For more information about it look here
I would sugest to work with a callbakck method that you call when you reached the latest value.
function readData(callback){
var trans = '';
trans = idb.transaction(["tableName"],'readonly'); //Create the transaction
var request = trans.objectStore("tableName").openCursor();
var maxKey;
request.onsuccess = function(e) {
var cursor = request.result || e.result;
if(cursor.value){
//logic to and find maximum
maxKey = cursor.primaryKey
cursor.continue();
}
}
trans.oncomplete = function(e) {
callback(maxKey);
}
}
IndexedDB API in top frame is async. async cannot be synchronous. But you can read all tables in single transaction.