I'm setting up a web scraper using Node.js and want to grab some html from a url and save it as a variable. A stripped down version follows.
var request = require('request');
var get_html = function(){
var url = "http://www.google.com";
var html = '';
request.get(url,function(error, response, body){
html += body;
});
return html;
};
console.log(get_html());
It seems that the function returns before request can concatenate the html to the variable html. As far as I can see, request only allows me to manipulate the html within the callback function or pipe it to a file. Is there anyway to just return it as a variable?
request.get is asynchronous and it will return result in the callback function.
You need to adapt your code a little bit like this
var request = require('request');
// get_html receive callback to process result
var get_html = function(callback) {
var url = "http://www.google.com";
var html = '';
request.get(url,function(error, response, body){
return callback(body); // call callback and parse result to it
});
};
// call get_html function
// and log html result here
get_html(function (body) { console.log(body); });
Code with a lot of function callbacks looks not beautiful.
I prefer promise than callback.
If you wish to use promise, try 'request-promise' lib.
It appears that request.get is async, so you have to put return html; in the callback. Otherwise it's returning instantly, before request.get can finish running.
Related
I am using Promise bluebird to process a json array objects from file. The problem arises if I want to store data in a json array (called list) and return this in the final process.
The list is empty/undefined after the return of list or even in the final process. Running the code, I always have 1 value that is not false which trigger the adding/push of the json in the list.
Can you help me with this issue? Below you will find my code.
Thanks in advance !!!
var Promise = require('bluebird');
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
fs.readdirAsync(dir).map(function (filename) {
return fs.readFileAsync(dir + "/" + filename, "utf8");
}).then(function(result){
var list=[];
result.map(function(row, index){
Promise.coroutine(function*() {
update(row, index).then(function(value){
if (value!=false){
var trade_update = new updated_Item(row.ID, row.Quantity, row.Price, row.Remark);
list.push(trade_update);
console.log(JSON.stringify(list)); <-- This works. It gives me data
}
return list;
})
})();
});
console.log('list: ' + JSON.stringify(list)); <-- output: list:[]
return list;
}).finally(function(result){
console.log('Final outcome: '+ ' ' + JSON.stringify(result)); <-- output: Final outcome: undefined
})
With the help of Samuel my code is now:
var Promise = require('bluebird');
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
function updateOrder(done){
fs.readdirAsync(dir).map(function (filename) {
return fs.readFileAsync(dir + "/" + filename, "utf8");
}).then(function(result){
var list=[];
result.map(function(row, index){
Promise.coroutine(function*() {
update(row, index).then(function(value){
if (value!=false){
var trade_update = new updated_Item(row.ID, row.Quantity, row.Price, row.Remark);
list.push(trade_update);
done(list);
}
})
})();
});
//done(list); <--if I put the done callback here, it will give me an empty list. I though once the result.map finished processing all the values give me the end result.
}
}
updateOrder(function(resultList){
console.log('List' + JSON.stringify(resultList));
})
This code give me whole resultList everytime the list has been updated (pushed) now.
I would to receive the resultList at the end once the function updateOrder is finished.
As noted in the comment. Promise.coroutine is asynchronous so this means that a result is not going to get return straight after your code reaches it. And this pretty much explains the phenomenon you are seeing where the latter print statements you got in the code is suggesting that list is undefined.
What you could do is wrap the entire code you got there in a function, then add a callback function as a parameter for the async functions to invoke when it has finished its duty, together returning the populated list back for later processing.
I have written a pseudo code for your case, unfortunately I couldn't test it on my IDE but the concept is there and it should work.
Consider my pseudo code:
var Promise = require('bluebird');
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
// Wrap everything you got into a function with a `done` parameter (callback fn)
function doStuff(done) {
fs.readdirAsync(dir).map(function (filename) {
return fs.readFileAsync(dir + "/" + filename, "utf8");
}).then(function(result){
var list=[];
result.map(function(row, index){
Promise.coroutine(function*() {
update(row, index).then(function(value){
if (value!=false){
var trade_update = new updated_Item(row.ID, row.Quantity, row.Price, row.Remark);
list.push(trade_update);
}
done(list);
})
})();
});
}).finally(function(result){
console.log('File read finish, but this doesnt mean I have finished doing everything!');
})
}
// call your function and provide a callback function for the async method to call
doStuff(function(resultList) {
console.log('list: ' + JSON.stringify(resultList));
// Continue processing the list data.
});
I have trouble moving certain code outside a test into a function that needs to return a value.
Here is part of my code for the test file
function getCountOfTopics(browser){
var count;
browser.getText('#sumTopics',
function(result){
count = result.value;
console.log(result.value);
}
);
return count;
};
module.exports = {
'Create article' : function(browser){
var noOfThreadsByInlineCode, noOfThreadsByFunction;
browser.getText('#sumTopics',
function(result){
noOfThreadsByInlineCode = result.value;
}
);
noOfThreadsByFunction = getCountOfTopics(browser);
browser.end();
}
}
Now, the variable noOfThreadsByInlineCode indeed gets the value in the DOM, but the variable noOfThreadsByFunction is undefined. The console does indeed print the correct value, so the function does get the correct value out of the DOM.
I would appreciate help in updating the function so that I do get the value returned.
One word answer is Asynchronisity. The code doesn't wait for your callback to get complete, thats what the feature of Node JS is.
If you are in desperately in need for the content inside of the callback you can write this variable into a file and then access it anywhere you want inside your code. Here's a bit of a workaround:
Save something in a file:
var fs = require('fs');
iThrowACallBack(function(response){
fs.writeFile('youCanSaveData.txt', this.response, function(err) {
if (err) throw err;
console.log('Saved!');
browser.pause(5000);
});
});
Access it somewhere else:
iAccessThefile(){
response = fs.readFileSync('youCanSaveData.txt').toString('utf-8');
}
Hope it helps.
You return variable 'count' outside the callback,that is why.You can take a look this topic How to return value from an asynchronous callback function?
function getCountOfTopics(browser){
var count;
browser.getText('#sumTopics',
function(result){
count = result.value;
console.log(result.value);
/// result.value is available in this callback.
}
);
What do you want to do with the 'value'?
ps:do not remember custom_command.I think it is very helpful for this issue.
I have a for loop that calls the following function twice:
var getJSON = function (url, callback) {
var http = require('https');
http.get(url, function (res) {
var body = '';
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', function () {
//console.log("Got response: ", body);
var response = JSON.parse(body);
return callback(response);
});
}).on('error', function (e) {
console.log("Got error: ", e);
return callback(-1);
});
}
The url parameter is at first "https://api.bitok.com/open_api/btc_eur/ticker" and the second time "https://api.bitok.com/open_api/btc_usd/ticker". The callback parameter is just another function that the program should go. The problem is that if it only works the first time (no matter which of both endpoints), the second time is falling and not printing the error, not sure what to do.
EDIT
Here is the for loop where I call the function, I also can't understand what is wrong, "exch.pairs_list.length" is equal to 2.
for (var i = 0; i < exch.pairs_list.length; i++) {
getJSON(url, callback);
}
No error is been thrown, the problem is the callback function, it's only been called once, should be twice.
I'm sorry guys, the mistake was very silly, I was trying to print the result before I actually receive response from the API.
I'm trying to get JSONP working with a server running on an Arduino.
This is my JS code:
window.onload = init;
function init()
{
//alert("Test");
SendRequest();
}
function SendRequest()
{
alert("Sending request");
var url = "http://192.168.1.177";
var request = new XMLHttpRequest();
request.open("GET", url);
request.error = function(e) {
alert("ERROR");
};
request.send(null);
}
function ArduinoJSONP()
{
alert("Callback received!!!");
}
The callback function is never reached.
But if I just direct my browser directly to the Arduino IP I see the following displayed in the browser:
ArduinoJSONP({"data": 12345})
So it seems the server is sending the response with correct JSONP format but somehow the function is not invoked. Is there anything else I need to for JS to call the function? I even tried moving the function to the HTML body but it didn't help either.
Thank you.
You are not handling server response at all. If you doing it without any libraries you need to eval result that was returned by the server.
And actually JSONP implementation is not XHR, you have to inject it as a script tag into html with correct src attribute.
Just use a library that already have all this logic abstracted for you.
Simply inject script tag into HTML tree:
function SendRequest()
{
var element = document.createElement('script');
var s = document.getElementsByTagName('script')[0];
element.type = 'text/javascript';
element.async = true;
element.src = 'http://192.168.1.177';
s.parentNode.insertBefore(element, s);
}
You can mark it with unique id. Hookup to onload event and once executed remove that script.
I have a metro application(HTML5 & WinJS) in which am trying to display service data . Actually here am retrieving JSON data from my service but am unable to bind this data into listview . Anyone give me some working example.
Thank you.
You can use the WinJS.xhr() for this. You can read more about it on this link https://msdn.microsoft.com/pt-br/library/windows/apps/br229787.aspx and here is an example:
var path = "data/file.json";
function getData(path) {
WinJS.xhr({ url: path }).then(
function (response) {
var json = JSON.parse(response.responseText);
// Since this is an asynchronous function, you can't
// return the data, so you can:
// 1) retrieve the data to a namespace once the app loads.
var list = new WinJS.Binding.List(json);
Somenomespace.data = list;
// 2) or do all the binding inside the function.
var listView = document.getElementById("listViewID");
listView.winControl.itemDataSource = list.dataSource;
});
}
If you use the built in JSON.parse(jsonString) function you can loop through the content using a normal for loop as it then is a normal object and add it as usuall. Just remember to process or render the data.
Her is an example from code i had in a search page using listview:
var response = JSON.parse(data) ;
var originalResults = new WinJS.Binding.List();
for (x in response) {
originalResults.push(response[x]);
}
this.populateFilterBar(element, originalResults);
this.applyFilter(this.filters[0], originalResults);