I have a for loop that calls the following function twice:
var getJSON = function (url, callback) {
var http = require('https');
http.get(url, function (res) {
var body = '';
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', function () {
//console.log("Got response: ", body);
var response = JSON.parse(body);
return callback(response);
});
}).on('error', function (e) {
console.log("Got error: ", e);
return callback(-1);
});
}
The url parameter is at first "https://api.bitok.com/open_api/btc_eur/ticker" and the second time "https://api.bitok.com/open_api/btc_usd/ticker". The callback parameter is just another function that the program should go. The problem is that if it only works the first time (no matter which of both endpoints), the second time is falling and not printing the error, not sure what to do.
EDIT
Here is the for loop where I call the function, I also can't understand what is wrong, "exch.pairs_list.length" is equal to 2.
for (var i = 0; i < exch.pairs_list.length; i++) {
getJSON(url, callback);
}
No error is been thrown, the problem is the callback function, it's only been called once, should be twice.
I'm sorry guys, the mistake was very silly, I was trying to print the result before I actually receive response from the API.
Related
To set the scenario for the code, the database stores Documents, and each document has the potential to have Images associated with them.
I have been trying to write a route that queries the database for each Document that has Images related to them, storing this data in JSON which is returned to the ajax request when completed, so the data can be viewed on the page. The closest I have got so far is the below attempt (see code).
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
async.waterfall([
// First function is to get each document which has an image related
function getDocuments(callback){
connection.query(`SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added
FROM Document WHERE doc_type = 'image'`, function(err, results) {
if (err) {
callback(err, null);
return;
}
// The Object containing the array where the data from the db needs to be stored
var documents = {
'docs': []
};
// foreach to iterate through each result found from the first db query (getDocuments)
results.forEach(function(result) {
// New object to store each document
var document = {};
document.entry = result;
// This is the array where each image assciated with a document will be stored
document.entry.images = [];
// Push each document to the array (above)
documents.docs.push(document);
var doc_id = result.document_id;
})
// Returning the results as 'documents' to the next function
callback(null, documents);
})
},
function getImages(documents, callback){
// Variable assignement to the array of documents
var doc_array = documents.docs;
// Foreach of the objects within document array
async.forEachOf(doc_array, function(doc, key, callback){
// Foreach object do the following series of functions
async.waterfall([
function handleImages(callback){
// The id of the document to get the images for
var doc_id = doc.entry.document_id;
connection.query(`SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`, function(err, rows) {
if (err) {
callback(err, null);
return;
}
callback(null, rows);
})
},
// Function below to push each image to the document.entry.images array
//
function pushImages(rows, callback){
// If multiple images are found for that document, the loop iterates through each pushing to the images array
for (var j = 0; j < rows.length; j++) {
// Creating new object for each image found so the data can be stored within this object, then pushed into the images array
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image);
}
callback(null, doc_array);
}
], function(err, doc_array){
if (err) {
console.log('Error in second waterfall callback:')
callback(err);
return;
}
console.log(doc.entry);
// callback(null, doc_array);
})
}, function(err, doc_array){
if (err) {
callback(err);
return;
}
callback(null, doc_array);
});
callback(null, doc_array);
}
], function(err, doc_array) {
if (err){
console.log('Error is: '+err);
return;
}
// The response that should return each document with each related image in the JSON
res.send(doc_array);
})
});
At the moment the results returned are:
1:
{entry: {document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}}
entry:
{document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}
doc_date_added:"2016-10-24"
doc_name:"DocumentNameHere"
doc_version_no:1
document_id:1
images:[]
As can be seen above, the images array remains empty even though with testing, the images are being found (console.log).
I hope someone is able to assist with this, as I am struggling to find the problem with this complex one.
Thanks
There are several async operations going on here and each operation needs a callback. See revised code:
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
var getDocuments = function(next) {
// Function for getting documents from DB
var query = `SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added FROM Document WHERE doc_type = 'image'`; // Set the query
connection.query(query, function(err, results) {
// Run the query async
if(err) {
// If err end execution
next(err, null);
return;
}
var documentList = []; // Array for holding docs
for(var i=0; i<results.length; i++) {
// Loop over results, construct the document and push to an array
var documentEntry = results[i];
var documentObject = {};
documentObject.entry = documentEntry;
documentObject.entry.images = [];
documentObject.id = documentEntry.document_id;
documentList.push(documentObject);
}
next(null, documents); // Pass to next async operation
});
};
var getImages = function(documents, next) {
// Function for getting images from documents
var finalDocs = []; // Blank arry for final documents with images
for (var i=0; i<documents.length; i++) {
// Loop over each document and construct the query
var id = documents[i].id;
var query = `SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`;
connection.query(query, function(err, images) {
// Execute the query async
if(err) {
// Throw error to callback
next(err, null);
return;
}
var processedDoc = processImages(documents[i], images); // Call a helper function to process all images into the document object
finalDocs.push(processedDoc); // Push the processed doc
if(i === documents.length) {
// If there are no more documents move onto next async
next(null, finalDocs);
}
});
}
};
var processImages = function(doc, images) {
for (var i=0; i< images.length; i++) {
// Loop over each document image - construct object
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image); // Push image into document object
}
return doc; // Return processed doc
};
getDocuments(function(err, docs) {
if(err) {
// Your error handler
}
if(docs) {
getImages(docs, function(err, finalDocs) {
if(err) {
// Your error handler
}
if(finalDocs) {
console.log(finalDocs);
res.status(200).json(finalDocs); // Send response
}
});
}
});
});
First we create a function to get documents - This function accepts a callback as its parameter. We run our query and construct our doc list. Then we return the document list by executing our callback
Next we run a function to get our images for each document. This function accepts our document list and a callback as our parameters. It retrieves images for each document and calls a helper function (sync)
Our helper function processes the images into each document and returns the processed document.
We then finish our operations by returning an array of processed documents via the second callback.
Other notes
We could tide this up by structuring this procedural style code into a contained JSON object
The nesting of the function executions at the end of the document could be cleaned up further
I've avoided using the async libray as it helps better understanding of the callback model
Event emitters could be used to flatten the callbacks - see https://nodejs.org/dist/latest-v7.x/docs/api/events.html
Hope this helps
Dylan
I am using Promise bluebird to process a json array objects from file. The problem arises if I want to store data in a json array (called list) and return this in the final process.
The list is empty/undefined after the return of list or even in the final process. Running the code, I always have 1 value that is not false which trigger the adding/push of the json in the list.
Can you help me with this issue? Below you will find my code.
Thanks in advance !!!
var Promise = require('bluebird');
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
fs.readdirAsync(dir).map(function (filename) {
return fs.readFileAsync(dir + "/" + filename, "utf8");
}).then(function(result){
var list=[];
result.map(function(row, index){
Promise.coroutine(function*() {
update(row, index).then(function(value){
if (value!=false){
var trade_update = new updated_Item(row.ID, row.Quantity, row.Price, row.Remark);
list.push(trade_update);
console.log(JSON.stringify(list)); <-- This works. It gives me data
}
return list;
})
})();
});
console.log('list: ' + JSON.stringify(list)); <-- output: list:[]
return list;
}).finally(function(result){
console.log('Final outcome: '+ ' ' + JSON.stringify(result)); <-- output: Final outcome: undefined
})
With the help of Samuel my code is now:
var Promise = require('bluebird');
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
function updateOrder(done){
fs.readdirAsync(dir).map(function (filename) {
return fs.readFileAsync(dir + "/" + filename, "utf8");
}).then(function(result){
var list=[];
result.map(function(row, index){
Promise.coroutine(function*() {
update(row, index).then(function(value){
if (value!=false){
var trade_update = new updated_Item(row.ID, row.Quantity, row.Price, row.Remark);
list.push(trade_update);
done(list);
}
})
})();
});
//done(list); <--if I put the done callback here, it will give me an empty list. I though once the result.map finished processing all the values give me the end result.
}
}
updateOrder(function(resultList){
console.log('List' + JSON.stringify(resultList));
})
This code give me whole resultList everytime the list has been updated (pushed) now.
I would to receive the resultList at the end once the function updateOrder is finished.
As noted in the comment. Promise.coroutine is asynchronous so this means that a result is not going to get return straight after your code reaches it. And this pretty much explains the phenomenon you are seeing where the latter print statements you got in the code is suggesting that list is undefined.
What you could do is wrap the entire code you got there in a function, then add a callback function as a parameter for the async functions to invoke when it has finished its duty, together returning the populated list back for later processing.
I have written a pseudo code for your case, unfortunately I couldn't test it on my IDE but the concept is there and it should work.
Consider my pseudo code:
var Promise = require('bluebird');
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
// Wrap everything you got into a function with a `done` parameter (callback fn)
function doStuff(done) {
fs.readdirAsync(dir).map(function (filename) {
return fs.readFileAsync(dir + "/" + filename, "utf8");
}).then(function(result){
var list=[];
result.map(function(row, index){
Promise.coroutine(function*() {
update(row, index).then(function(value){
if (value!=false){
var trade_update = new updated_Item(row.ID, row.Quantity, row.Price, row.Remark);
list.push(trade_update);
}
done(list);
})
})();
});
}).finally(function(result){
console.log('File read finish, but this doesnt mean I have finished doing everything!');
})
}
// call your function and provide a callback function for the async method to call
doStuff(function(resultList) {
console.log('list: ' + JSON.stringify(resultList));
// Continue processing the list data.
});
I'm setting up a web scraper using Node.js and want to grab some html from a url and save it as a variable. A stripped down version follows.
var request = require('request');
var get_html = function(){
var url = "http://www.google.com";
var html = '';
request.get(url,function(error, response, body){
html += body;
});
return html;
};
console.log(get_html());
It seems that the function returns before request can concatenate the html to the variable html. As far as I can see, request only allows me to manipulate the html within the callback function or pipe it to a file. Is there anyway to just return it as a variable?
request.get is asynchronous and it will return result in the callback function.
You need to adapt your code a little bit like this
var request = require('request');
// get_html receive callback to process result
var get_html = function(callback) {
var url = "http://www.google.com";
var html = '';
request.get(url,function(error, response, body){
return callback(body); // call callback and parse result to it
});
};
// call get_html function
// and log html result here
get_html(function (body) { console.log(body); });
Code with a lot of function callbacks looks not beautiful.
I prefer promise than callback.
If you wish to use promise, try 'request-promise' lib.
It appears that request.get is async, so you have to put return html; in the callback. Otherwise it's returning instantly, before request.get can finish running.
I wanna know if it's possible do multiple res.send/json in same method. I have a big problem with that because I wanna develop a function into a Web-Worker to call a put request each minute but in 2nd request I get "can't set headers after they are sent.". I know that it's not possible do it but I wanna know if exist some way to run.
exports.update = function (req, res) {
var feed = req.feed;
feed.title = req.body.title;
feed.apifeed = req.body.apifeed;
feed.apikey = req.body.apikey;
feed.active = req.body.active;
if(feed.apifeed && feed.apikey && feed.active){
var t = Threads.create();
t.eval(setInterval(
function(){
async.parallel([
function(callback, data){
var url = 'https://api.xively.com/v2/feeds/' + feed.apifeed + '.json?key=' + feed.apikey;
sensordata.getSensorData(url, function(data){
callback(null, data);
});
}
], function(err, data){
feed.content[0].value = data[0][0].value;
feed.content[0].date = new Date();
feed.save(function (err) {
if (err) {
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else {
res.json(feed);
}
});
});
}, 5000)
);
}
};
Thanks for supporting and I wait your responses. Greetings!
You cannot respond to a request more than once, that wouldn't make sense.
If you need to stream data, then you will need to use long polling, WebSockets, Server-sent Events, etc.
Also, assuming Threads.create() does what I think it may be doing, spawning an OS thread just to do what you're currently doing is a waste of resources. Nothing in that block of code is CPU-bound.
I've attempted to write a basic cron script to run and 'dump' a mysql database. For some reason, when it 'successfully saves the file', it does create the file, but it is empty. If instead of saving the file, I perform a console.log, it prints an empty string. Any thoughts on what I may be doing wrong?
Thanks in advance.
var mysql_backup = function(){
this.backup = '';
this.mysql = require('mysql'),
this.init = function(){
this.connection = this.mysql.createConnection({
user: 'root',
password: 'root',
database: 'test'
});
}
this.query = function(sql, callback) {
this.connection.query(sql, function (error, results, fields) {
if (error) {
throw error;
}
if (results.length > 0) {
callback(results);
}
});
}
this.get_tables = function(callback){
var me = this;
me.query('SHOW TABLES',
function(tables) {
for (var table in tables){
me.query(
'SHOW CREATE TABLE ' + tables[table].Tables_in_test,
function(r){
for (var t in r) {
me.backup += "DROP TABLE " + r[t].Table + "\n\n";
me.backup += r[t]["Create Table"] + "\n\n";
}
}
)
}
me.save_backup();
});
}
this.save_backup = function(){
var fs = require('fs');
fs.writeFile("./backup_test.txt", this.backup, function(err) {
if(err) {
console.log(err);
} else {
console.log("The file was saved!");
}
});
}
};
var db = new mysql_backup;
db.init();
db.get_tables();
db.connection.destroy();
The code as written didn't even get to a file saving for me. There seem like a few issues. Not sure if this is the actual code or some things got lost in the copy paste. However, based on what you've got:
A big one is that you never connect to the database in your code with connection.connect().
The code you want to run once connected should be inside the connection.connect() callback. e.g.
connection.connect(function (err, empty) {
if (err)
throw new Error ('Panic');
// if no error, we are off to the races...
}
However, even if you quickly refactor your code to wrap your last lines inside of that get connection callback, you'll still have problems, because you are destroying the connection before the various SQL calls are getting made, so you will want to move the code into some sort of final callback.
Even after you do that, you'll still have an empty file, because you're calling save_backup from your 'SHOW TABLES' callback rather than after you have actually populated it via the inner callback where you get the CREATE TABLE statement and populate the backup property.
This is the minimal rewriting of your code which will do what you are intending. An important thing to note is the "counter" which manages when to write the file and close the connection. I would make other changes if it were mine, including:
Using 'self' instead of 'me'
Using a numeric for loop rather than the for (... in ...) syntax
Having my own callbacks fall the node convention of (err, stuff)
A more substantial changes is that I would rewrite this to use promises, as doing so can spare you some grief with the confusion inherent with deeply nested callbacks. I personally like the Q library, but there are several options here.
Hope this helped.
var mysql_backup = function(){
this.backup = '';
this.mysql = require('mysql');
this.init = function(){
this.connection = this.mysql.createConnection({
user : 'root',
password : 'root',
database : 'test'
});
};
this.query = function(sql, callback) {
this.connection.query(sql, function (error, results, fields) {
if (error) {
throw error;
}
if (results.length > 0) {
callback(results);
}
});
};
this.get_tables = function(callback){
var counter = 0;
var me = this;
this.query('SHOW TABLES',
function(tables) {
for (table in tables){
counter++;
me.query(
'SHOW CREATE TABLE ' + tables[table].Tables_in_mvc,
function(r){
for (t in r) {
me.backup += "DROP TABLE " + r[t].Table + "\n\n";
me.backup += r[t]["Create Table"] + "\n\n";
}
counter--;
if (counter === 0){
me.save_backup();
me.connection.destroy();
}
}
)
}
});
};
this.save_backup = function(){
var fs = require('fs');
fs.writeFile("./backup_test.txt", this.backup, function(err) {
if(err) {
console.log(err);
} else {
console.log("The file was saved!");
}
});
}
};
var db = new mysql_backup;
db.init();
db.connection.connect(function (err){
if (err) console.log(err);
db.get_tables(function(x){;});
});
Update: If you are curious, here is a heavily-commented implementation using promises. Note that without the comments explaining the Q promise library functions, it is somewhat shorter than the original version and also offers more comprehensive error handling.
var MysqlBackup = function(connectionInfo, filename){
var Q = require('q');
var self = this;
this.backup = '';
// my personal preference is to simply require() inline if I am only
// going to use something a single time. I am certain some will find
// this a terrible practice
this.connection = require('mysql').createConnection(connectionInfo);
function getTables(){
// return a promise from invoking the node-style 'query' method
// of self.connection with parameter 'SHOW TABLES'.
return Q.ninvoke(self.connection,'query', 'SHOW TABLES');
};
function doTableEntries(theResults){
// note that because promises only pass a single parameter around,
// if the 'denodeify-ed' callback has more than two parameters (the
// first being the err param), the parameters will be stuffed into
// an array. In this case, the content of the 'fields' param of the
// mysql callback is in theResults[1]
var tables = theResults[0];
// create an array of promises resulting from another Q.ninvoke()
// query call, chained to .then(). Note that then() expects a function,
// so recordEntry() in fact builds and returns a new one-off function
// for actually recording the entry (see recordEntry() impl. below)
var tableDefinitionGetters = [];
for (var i = 0; i < tables.length ; i++){
// I noticed in your original code that your Tables_in_[] did not
// match your connection details ('mvc' vs 'test'), but the below
// should work and is a more generalized solution
var tableName = tables[i]['Tables_in_'+connectionInfo.database];
tableDefinitionGetters.push(Q.ninvoke(self.connection, 'query', 'SHOW CREATE TABLE ' + tableName)
.then(recordEntry(tableName)) );
}
// now that you have an array of promises, you can use Q.allSettled
// to return a promise which will be settled (resolved or rejected)
// when all of the promises in the array are settled. Q.all is similar,
// but its promise will be rejected (immediately) if any promise in the
// array is rejected. I tend to use allSettled() in most cases.
return Q.allSettled(tableDefinitionGetters);
};
function recordEntry (tableName){
return function(createTableQryResult){
self.backup += "DROP TABLE " + tableName + "\n\n";
self.backup += createTableQryResult[0][0]["Create Table"] + "\n\n";
};
};
function saveFile(){
// Q.denodeify return a promise-enabled version of a node-style function
// the below is probably excessively terse with its immediate invocation
return (Q.denodeify(require('fs').writeFile))(filename, self.backup);
}
// with the above all done, now you can actually make the magic happen,
// starting with the promise-return Q.ninvoke to connect to the DB
// note that the successive .then()s will be executed iff (if and only
// if) the preceding item resolves successfully, .catch() will get
// executed in the event of any upstream error, and finally() will
// get executed no matter what.
Q.ninvoke(this.connection, 'connect')
.then(getTables)
.then(doTableEntries)
.then(saveFile)
.then( function() {console.log('Success'); } )
.catch( function(err) {console.log('Something went awry', err); } )
.finally( function() {self.connection.destroy(); } );
};
var myConnection = {
host : '127.0.0.1',
user : 'root',
password : 'root',
database : 'test'
};
// I have left this as constructor-based calling approach, but the
// constructor just does it all so I just ignore the return value
new MysqlBackup(myConnection,'./backup_test.txt');