JSON document mysteriously "empties" - json

Question

I have written a small script in Node.js to scrape a web page and get the some links. The scrapping part is done with Cheerio. My code is here (simplified for space):
var request = require('request');
var cheerio = require('cheerio');
var base_url = 'http://www.naftemporiki.gr/finance/';
var mutuals = {};
mutuals.date = new Date();
mutuals.companies = [];
var company = {};
request(base_url + 'mtfCompanies', function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('.blueRow.texttd.name a').each(function (i, element) {
var a = $(this);
company = {};
company.name = a.text();
company.link = a.attr('href');
mutuals.companies.push(company);
});
}
//console.log(mutuals); // 1st place
});
console.log(mutuals); // 2nd place
Here comes the fun part: When I try to output the JSON document from the "1st place", inside the 'request' block, it comes out nice and true. An example is here:
{ date: Wed Nov 26 2014 10:35:09 GMT+0200 (EET),
companies:
[ { name: ' J.P. MORGAN ASSET MANAGEMENT',
link: 'mtfCompany?id=J.P.+MORGAN+ASSET+MANAGEMENT' },
{ name: ' BNP PARIBAS INVESTMENT PARTNERS',
link: 'mtfCompany?id=BNP+PARIBAS+INVESTMENT+PARTNERS' },
{ name: ' PICTET', link: 'mtfCompany?id=PICTET' },
{ name: ' ALLIANZ ΑΕΔΑΚ',
link: 'mtfCompany?id=ALLIANZ+%ce%91%ce%95%ce%94%ce%91%ce%9a' },
{ name: ' ALLIANZ ΑΕΔΑΚ (ΑΝΤΙΠΡ.)',
link: 'mtfCompany?id=ALLIANZ+%ce%91%ce%95%ce%94%ce%91%ce%9a+(%ce%91%ce%9d%ce%a4%ce%99%ce%a0%ce%a1.)' },
{ name: ' ALLIANZ ΕΛΛΑΣ Α.Ε.',
link: 'mtfCompany?id=ALLIANZ+%ce%95%ce%9b%ce%9b%ce%91%ce%a3+%ce%91.%ce%95.' }]}
When I try to output the JSON document from the "2nd place", outside of ANY block and at the end of execution, this is what I get:
{ date: Wed Nov 26 2014 10:35:09 GMT+0200 (EET), companies: [] }
It looks like the 'companies' array inside the JSON document gets emptied. I have a suspicion that the 'mutuals.companies = [];' line gets executed again for some reason.
Can anyone help with this?
UPDATE 1:
Changed my code as suggested to use 'async.series...'. This is the updated version:
var request = require('request'),
async = require('async'),
cheerio = require('cheerio');
var base_url = 'http://www.naftemporiki.gr/finance/';
var mutuals = {};
mutuals.date = new Date();
mutuals.companies = [];
var company = {};
async.series([
function(callback) {
request(base_url + 'mtfCompanies', function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('.blueRow.texttd.name a').each(function (i, element) {
var a = $(this);
company = {};
company.name = a.text();
company.link = a.attr('href');
mutuals.companies.push(company);
});
}
});
callback(null, 'one');
},
function (callback) {
console.log(mutuals);
callback(null, 'two');
}
]);
Still does not work. Still the JSON outputted is this:
{ date: Wed Nov 26 2014 10:35:09 GMT+0200 (EET), companies: [] }

Answer 1

Your "2nd place" is printing the variable before the request finishes.
Your "1st place" works because it's located inside the callback of the request. The request is made, the data is pulled, the callback then gets called and is successfully printed.
This is the way asynchronous code works. Nothing blocks. So when you issue your request, node stores the callback function so that it can execute code with the results of the request.
Update 1:
The issue with your update is mostly the same. In your first function in the series, the callback gets called before request has finished. If you move callback into the function passed to request then it gets called after request finishes.
function(callback) {
request(base_url + 'mtfCompanies', function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('.blueRow.texttd.name a').each(function (i, element) {
var a = $(this);
company = {};
company.name = a.text();
company.link = a.attr('href');
mutuals.companies.push(company);
});
callback(null, 'one');
}
});
},
Suggestion 1
Developing in node.js with callbacks can leave you with a deep nested structure. Don't let your if statements make nesting worse. Use early returns instead of deeper nesting. Example:
function(callback) {
request(base_url + 'mtfCompanies', function (error, response, html) {
if(error) return callback(error);
if(response.statusCode !== 200) return callback('status code not 200');
var $ = cheerio.load(html);
$('.blueRow.texttd.name a').each(function (i, element) {
var a = $(this);
company = {};
company.name = a.text();
company.link = a.attr('href');
mutuals.companies.push(company);
});
callback(null, 'one');
});
},
Suggestion 2
When using async it can help to simplify things by using named functions. Example:
var request = require('request'),
async = require('async'),
cheerio = require('cheerio');
var base_url = 'http://www.naftemporiki.gr/finance/';
var mutuals = {};
mutuals.date = new Date();
mutuals.companies = [];
var company = {};
function getPage(callback) {
request(base_url + 'mtfCompanies', function (error, response, html) {
if(error) return callback(error);
if(response.statusCode !== 200) return callback('status code not 200');
var $ = cheerio.load(html);
$('.blueRow.texttd.name a').each(function (i, element) {
var a = $(this);
company = {};
company.name = a.text();
company.link = a.attr('href');
mutuals.companies.push(company);
});
callback(null, 'one');
});
}
function printMutuals(callback) {
console.log(mutuals);
callback(null, 'two');
}
async.series([
getPage,
printMutuals
]);