NodeJs Store Json from multiple requests to MySQL DB - mysql

Question

I am working on a crawler that stores the retrieved and parsed data into my MySQL DB.
Storing the results works pretty well, but I am stuck when it comes to end the connection.
I am using request, cheerio, async and mysql:
// DB params
var con = mysql.createConnection({
host: "localhost",
user: "user",
password: "password",
database: "mydatabase",
});
// Open connection to DB
con.connect(function(err) {
if (err) {
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
// Array of Cities to crawl
var cities = ["olten", "zurich"];
// Todays Date
var today = new Date();
...
today = dd + '.' + mm + '.' + yyyy;
// Search every city
async.each(cities, function(item, callback){
// The Page to crawl
var page = "https://www.thepageto.com/search_xhr?fn=" + /*cities[i]*/ item + "&tn=&db=" + today + "&sort=trip_date&order";
console.log("Visiting page " + page);
request(page, function(error, response, body) {
if (error) {console.log("Error: " + error);}
// Check status code (200 is HTTP OK)
console.log("Status code: " + response.statusCode);
if (response.statusCode === 200) {
// Parse JSON
var data = JSON.parse(body);
// Use only result > html code
var content = data.html.results;
// Parse html
var $ = cheerio.load(content);
// Iterate through offers & Extract information & Store in a Object
$('.trip.relative').each(function() {
// Empty Object, has the same structure as the Db table
var json = {id: "", driver: "", rating: "", date: "", time: "", start: "", stops: "", end: "", price: "", url: "", query: ""};
// Assigning values
json.driver = $(this).find('.username').text();
...
// Save filled object in Db
save(json, callback);
});
}
});
}, function(err){
if (err) {
console.log('could not save');
} else {
console.log('lets end connection');
con.end(function(err) {if (err) throw err;});
}
}
);
function save(item, callback) {
con.query('INSERT INTO offers SET ?', item, callback);
}
The problem is, that the DB connection is ended before anything could be stored into the DB. The con.end() is always fired before the actual query or before the query completed. I have the intension that the save function has to return something when its done. If i run the script without the con.end(), the data is stored into the DB perfectly.
I am new to Node so I am still learning and hopefully you can help me with this.
Edit:
I now get an:
ReferenceError: callback is not defined
at save (/home/juki/crawler/crawler.js:143:46)
As well as an:
Error: Callback was already called.
at Query._callback (/home/juki/crawler/node_modules/async/dist/async.js:839: 36)

Answer 1

It looks like you do not call callback from your crawler.
You also do not seem to wait for save to finish. Try to rework save function, so it takes callback too. Something like:
function save(item, callback) {
con.query('INSERT INTO offers SET ?', item, callback);
}
And then call it like:
save(json, callback);
where callback is the one you get from async.each.