I'd like to request one page after another, the following code seems request all the pages in the same time, is there a way to fetch one page after the previous one is done? thanks
var Promise = require("bluebird");
var request = Promise.promisifyAll(require('request'));
var URLS = ["http://sample.com/j1", "http://sample.com/j2"]
Promise.map(URLS, function (item) {
return request.postAsync({url: item}).spread(function (response,body) {
var items = JSON.parse(body)
return items
})
}).then(function (r) {
console.log(r.length)
})
You can set the concurrency level, which is specific to bluebird.
Promise.map(URLS, function (item) {
return request.postAsync({url: item}).spread(function (response,body) {
var items = JSON.parse(body)
return items
}, { concurrency: 1})
This will issue all the promises one at a time.
Related
To set the scenario for the code, the database stores Documents, and each document has the potential to have Images associated with them.
I have been trying to write a route that queries the database for each Document that has Images related to them, storing this data in JSON which is returned to the ajax request when completed, so the data can be viewed on the page. The closest I have got so far is the below attempt (see code).
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
async.waterfall([
// First function is to get each document which has an image related
function getDocuments(callback){
connection.query(`SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added
FROM Document WHERE doc_type = 'image'`, function(err, results) {
if (err) {
callback(err, null);
return;
}
// The Object containing the array where the data from the db needs to be stored
var documents = {
'docs': []
};
// foreach to iterate through each result found from the first db query (getDocuments)
results.forEach(function(result) {
// New object to store each document
var document = {};
document.entry = result;
// This is the array where each image assciated with a document will be stored
document.entry.images = [];
// Push each document to the array (above)
documents.docs.push(document);
var doc_id = result.document_id;
})
// Returning the results as 'documents' to the next function
callback(null, documents);
})
},
function getImages(documents, callback){
// Variable assignement to the array of documents
var doc_array = documents.docs;
// Foreach of the objects within document array
async.forEachOf(doc_array, function(doc, key, callback){
// Foreach object do the following series of functions
async.waterfall([
function handleImages(callback){
// The id of the document to get the images for
var doc_id = doc.entry.document_id;
connection.query(`SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`, function(err, rows) {
if (err) {
callback(err, null);
return;
}
callback(null, rows);
})
},
// Function below to push each image to the document.entry.images array
//
function pushImages(rows, callback){
// If multiple images are found for that document, the loop iterates through each pushing to the images array
for (var j = 0; j < rows.length; j++) {
// Creating new object for each image found so the data can be stored within this object, then pushed into the images array
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image);
}
callback(null, doc_array);
}
], function(err, doc_array){
if (err) {
console.log('Error in second waterfall callback:')
callback(err);
return;
}
console.log(doc.entry);
// callback(null, doc_array);
})
}, function(err, doc_array){
if (err) {
callback(err);
return;
}
callback(null, doc_array);
});
callback(null, doc_array);
}
], function(err, doc_array) {
if (err){
console.log('Error is: '+err);
return;
}
// The response that should return each document with each related image in the JSON
res.send(doc_array);
})
});
At the moment the results returned are:
1:
{entry: {document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}}
entry:
{document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}
doc_date_added:"2016-10-24"
doc_name:"DocumentNameHere"
doc_version_no:1
document_id:1
images:[]
As can be seen above, the images array remains empty even though with testing, the images are being found (console.log).
I hope someone is able to assist with this, as I am struggling to find the problem with this complex one.
Thanks
There are several async operations going on here and each operation needs a callback. See revised code:
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
var getDocuments = function(next) {
// Function for getting documents from DB
var query = `SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added FROM Document WHERE doc_type = 'image'`; // Set the query
connection.query(query, function(err, results) {
// Run the query async
if(err) {
// If err end execution
next(err, null);
return;
}
var documentList = []; // Array for holding docs
for(var i=0; i<results.length; i++) {
// Loop over results, construct the document and push to an array
var documentEntry = results[i];
var documentObject = {};
documentObject.entry = documentEntry;
documentObject.entry.images = [];
documentObject.id = documentEntry.document_id;
documentList.push(documentObject);
}
next(null, documents); // Pass to next async operation
});
};
var getImages = function(documents, next) {
// Function for getting images from documents
var finalDocs = []; // Blank arry for final documents with images
for (var i=0; i<documents.length; i++) {
// Loop over each document and construct the query
var id = documents[i].id;
var query = `SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`;
connection.query(query, function(err, images) {
// Execute the query async
if(err) {
// Throw error to callback
next(err, null);
return;
}
var processedDoc = processImages(documents[i], images); // Call a helper function to process all images into the document object
finalDocs.push(processedDoc); // Push the processed doc
if(i === documents.length) {
// If there are no more documents move onto next async
next(null, finalDocs);
}
});
}
};
var processImages = function(doc, images) {
for (var i=0; i< images.length; i++) {
// Loop over each document image - construct object
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image); // Push image into document object
}
return doc; // Return processed doc
};
getDocuments(function(err, docs) {
if(err) {
// Your error handler
}
if(docs) {
getImages(docs, function(err, finalDocs) {
if(err) {
// Your error handler
}
if(finalDocs) {
console.log(finalDocs);
res.status(200).json(finalDocs); // Send response
}
});
}
});
});
First we create a function to get documents - This function accepts a callback as its parameter. We run our query and construct our doc list. Then we return the document list by executing our callback
Next we run a function to get our images for each document. This function accepts our document list and a callback as our parameters. It retrieves images for each document and calls a helper function (sync)
Our helper function processes the images into each document and returns the processed document.
We then finish our operations by returning an array of processed documents via the second callback.
Other notes
We could tide this up by structuring this procedural style code into a contained JSON object
The nesting of the function executions at the end of the document could be cleaned up further
I've avoided using the async libray as it helps better understanding of the callback model
Event emitters could be used to flatten the callbacks - see https://nodejs.org/dist/latest-v7.x/docs/api/events.html
Hope this helps
Dylan
New to node, As I am cycling through a roster of students, I need to check and see if a teacher has requested them for tutoring.
I realized I can't just do this:
var checkRequest = function(id){
var value = '';
roster.query('SELECT * FROM teacher_request WHERE student_id ='+id, function(err, row){
value = row.length;
}
return value;
}
After a bit of digging around promises looked like a great solution, but if I simply return the deferred.promise from the checkRequest function, all I get is an object that says [deferred promise] which I can't access the actual data from. (Or have not figured out how yet)
If I follow along with their api and use .then (as illustrated in the getRow) function, I am back in the same problem I was in before.
function checkRequest(id) {
console.log(id);
var deferred = Q.defer();
connection.query('SELECT * FROM teacher_request WHERE student_id ='+id, function(err, row){
deferred.resolve(row.length);
});
return deferred.promise;
}
var getRow = function(id){
checkRequest(id).then(function(val) {
console.log(val); // works great
return val; //back to the same problem
});
}
The roster needs to be able to be pulled from an external API which is why I am not bundling the request check with the original roster query.
Thanks in advance
From the stuff you posted, I assume you have not really understood the concept of promises. They allow you to queue up callbacks, that get executed, when the asynchronous operation has finished (by succeeding or failing).
So instead of somehow getting the results back to your synchronous workflow, you should convert that workflow to work asynchronous as well. So a small example for your current problem:
// your students' ids in here
var studentsArray = [ 1, 2, 5, 6, 9 ];
for( var i=0; i<studentsArray.length; i++ ) {
checkRequest( i )
.then( function( data ){
console.log( data.student_id );
// any other code related to a specific student in here
});
}
or another option, if you need all students' data at the same time:
// your students' ids in here
var studentsArray = [ 1, 2, 5, 6, 9 ];
// collect all promises
var reqs = [];
for( var i=0; i<studentsArray.length; i++ ) {
reqs.push( checkRequest( i ) );
}
Q.all( reqs )
.then( function(){
// code in here
// use `arguments` to access data
});
I am only getting back the first result in the array, and want to retrieve all the available results.
function runForm(){
$("#stock_news").html("");
var stockSymbol = $("input").val();
//very long str
var newsStocks = "http://query.yahooapis.com/v1/public/yql?
q=select%20*%20from%20html%20where%20url%3D'http%3A%2F%2Ffinance.yahoo.com
%2Fq%3Fs%3D"+stockSymbol+"'%20and%20xpath%3D'%2F%2Fdiv%5B%40id%3D%22yfi_headlines
%22%5D%2Fdiv%5B2%5D%2Ful%2Fli%2Fa'&format=json&diagnostics=true&callback=";
//getJSON
$.getJSON(newsStocks, function(data) {
var headlines = data.query.results.a[0];
//newsStr
newsStr = "<h3 style='text-decoration:underline'>Current Headlines</h3><p><ol>
<li><a href='"+headlines.href+"'>"+headlines.content+
"</a></li></ol></p>";
$("#stock_news").html(newsStr);
});
You haven't given enough information, but I could guess that this var headlines = data.query.results.a[0]; is your issue. It would seem like the other results are there, but you're filtering them out. console.log(data) and console.log(data.query.results) and look at what all is in there. Otherwise, update your question with more information.
Update:
This is a very basic ajax call (with jQuery) and a for loop to loop through and use each result.
$.get('path/here', function(data) {
//if your data is an array
for (var i=0; i<data.length; ++i) {
var item = data[i];
console.log(item);
}
});
Here's a live demo (click), but I'm not going to use the ajax data since the path is not valid.
Here's the same example, but using $.each. I prefer the for loop...why use jQuery for something so simple?
$.get('path/here', function(data) {
$.each(data, function(item, index) {
console.log(index);
});
});
[{"time":136803,"price":"1.4545","amount":"0.0885","ID":"112969"},
{"time":136804,"price":"2.5448","amount":"0.0568","ID":"5468489"},
{"time":136805,"price":"1.8948","amount":"0.0478","ID":"898489"}]
I have a large JSON file like the one above. It is a list of dictionaries. I want to choose a time and find the value assoaciated with that time. I will not know where in my list the time is located only the value for the time. Is there a way I can say, for time 136804, make x = to price? Or should I loop through each value? I also want to use this value (x) in a mathematical function.
My fist idea is to use brute force by going through each item and checking it for a matching time value in a loop.
Is this the best way?
Take a look at SpahQL http://danski.github.io/spahql/ which we use to query JSON in order to select values and subsequently change them as required.
I did something similar to this recently. JSON file I had to query had around 6000 lines and around 500 JSON objects. My query function given below loops through the each object to select the matching objects, but it can fetch any result within few milliseconds.
var data = '[{"time":136803,"price":"1.4545","amount":"0.0885","ID":"112969"},'+ '{"time":136804,"price":"2.5448","amount":"0.0568","ID":"5468489"},'+ '{"time":136805,"price":"1.8948","amount":"0.0478","ID":"898489"}]';
var data = JSON.parse(data);
var query = function(data, select, andwhere) {
var return_array = [];
$.each(data, function (i, obj) {
var temp_obj = {};
var where = true;
if (andwhere) {
$.each(andwhere, function(j, wh) {
if (obj[wh.col] !== wh.val) {
where = false;
}
});
}
if (where === false) {
return;
}
$.each(obj, function (j, elem) {
if (select.indexOf(j.trim())!==-1) {
temp_obj[j] = elem;
}
});
return_array.push(temp_obj);
});
return return_array;
};
var result = query(data, ['price','amount'],[{"col":"time","val":136804}]);
console.log(JSON.stringify(result));
http://jsfiddle.net/bejgy3sn/1/
I want to write a function in JS where I will loop through a tables in my indexed DB and get the maximum value of last modified of table and return that
function readData(){
var trans = '';
trans = idb.transaction(["tableName"],'readonly'); // Create the transaction
var request = trans.objectStore("tableName").openCursor();
request.onsuccess = function(e) {
var cursor = request.result || e.result;
if(cursor) {
// logic to and find maximum
} else {
return // max last modified
}
cursor.continue();
}
}
IMP--Since onsuccess method is asynchronous how can i make it synchronous? so that my method readData() will return only when max last modified record is found successfully. I can call this method(readData()) synchronously to get last modified record of 2-3 tables if I want.
The sync API is only available in a webworker. So this would be the first requirement. (As far as I know only IE10 supports this at the moment)
An other shot you can give is working with JS 1.7 and use the yield keyword. For more information about it look here
I would sugest to work with a callbakck method that you call when you reached the latest value.
function readData(callback){
var trans = '';
trans = idb.transaction(["tableName"],'readonly'); //Create the transaction
var request = trans.objectStore("tableName").openCursor();
var maxKey;
request.onsuccess = function(e) {
var cursor = request.result || e.result;
if(cursor.value){
//logic to and find maximum
maxKey = cursor.primaryKey
cursor.continue();
}
}
trans.oncomplete = function(e) {
callback(maxKey);
}
}
IndexedDB API in top frame is async. async cannot be synchronous. But you can read all tables in single transaction.