Node.js asynchronous nested mysql queries - mysql

Question

To set the scenario for the code, the database stores Documents, and each document has the potential to have Images associated with them.
I have been trying to write a route that queries the database for each Document that has Images related to them, storing this data in JSON which is returned to the ajax request when completed, so the data can be viewed on the page. The closest I have got so far is the below attempt (see code).
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
async.waterfall([
// First function is to get each document which has an image related
function getDocuments(callback){
connection.query(`SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added
FROM Document WHERE doc_type = 'image'`, function(err, results) {
if (err) {
callback(err, null);
return;
}
// The Object containing the array where the data from the db needs to be stored
var documents = {
'docs': []
};
// foreach to iterate through each result found from the first db query (getDocuments)
results.forEach(function(result) {
// New object to store each document
var document = {};
document.entry = result;
// This is the array where each image assciated with a document will be stored
document.entry.images = [];
// Push each document to the array (above)
documents.docs.push(document);
var doc_id = result.document_id;
})
// Returning the results as 'documents' to the next function
callback(null, documents);
})
},
function getImages(documents, callback){
// Variable assignement to the array of documents
var doc_array = documents.docs;
// Foreach of the objects within document array
async.forEachOf(doc_array, function(doc, key, callback){
// Foreach object do the following series of functions
async.waterfall([
function handleImages(callback){
// The id of the document to get the images for
var doc_id = doc.entry.document_id;
connection.query(`SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`, function(err, rows) {
if (err) {
callback(err, null);
return;
}
callback(null, rows);
})
},
// Function below to push each image to the document.entry.images array
//
function pushImages(rows, callback){
// If multiple images are found for that document, the loop iterates through each pushing to the images array
for (var j = 0; j < rows.length; j++) {
// Creating new object for each image found so the data can be stored within this object, then pushed into the images array
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image);
}
callback(null, doc_array);
}
], function(err, doc_array){
if (err) {
console.log('Error in second waterfall callback:')
callback(err);
return;
}
console.log(doc.entry);
// callback(null, doc_array);
})
}, function(err, doc_array){
if (err) {
callback(err);
return;
}
callback(null, doc_array);
});
callback(null, doc_array);
}
], function(err, doc_array) {
if (err){
console.log('Error is: '+err);
return;
}
// The response that should return each document with each related image in the JSON
res.send(doc_array);
})
});
At the moment the results returned are:
1:
{entry: {document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}}
entry:
{document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}
doc_date_added:"2016-10-24"
doc_name:"DocumentNameHere"
doc_version_no:1
document_id:1
images:[]
As can be seen above, the images array remains empty even though with testing, the images are being found (console.log).
I hope someone is able to assist with this, as I am struggling to find the problem with this complex one.
Thanks

Answer 1

There are several async operations going on here and each operation needs a callback. See revised code:
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
var getDocuments = function(next) {
// Function for getting documents from DB
var query = `SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added FROM Document WHERE doc_type = 'image'`; // Set the query
connection.query(query, function(err, results) {
// Run the query async
if(err) {
// If err end execution
next(err, null);
return;
}
var documentList = []; // Array for holding docs
for(var i=0; i<results.length; i++) {
// Loop over results, construct the document and push to an array
var documentEntry = results[i];
var documentObject = {};
documentObject.entry = documentEntry;
documentObject.entry.images = [];
documentObject.id = documentEntry.document_id;
documentList.push(documentObject);
}
next(null, documents); // Pass to next async operation
});
};
var getImages = function(documents, next) {
// Function for getting images from documents
var finalDocs = []; // Blank arry for final documents with images
for (var i=0; i<documents.length; i++) {
// Loop over each document and construct the query
var id = documents[i].id;
var query = `SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`;
connection.query(query, function(err, images) {
// Execute the query async
if(err) {
// Throw error to callback
next(err, null);
return;
}
var processedDoc = processImages(documents[i], images); // Call a helper function to process all images into the document object
finalDocs.push(processedDoc); // Push the processed doc
if(i === documents.length) {
// If there are no more documents move onto next async
next(null, finalDocs);
}
});
}
};
var processImages = function(doc, images) {
for (var i=0; i< images.length; i++) {
// Loop over each document image - construct object
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image); // Push image into document object
}
return doc; // Return processed doc
};
getDocuments(function(err, docs) {
if(err) {
// Your error handler
}
if(docs) {
getImages(docs, function(err, finalDocs) {
if(err) {
// Your error handler
}
if(finalDocs) {
console.log(finalDocs);
res.status(200).json(finalDocs); // Send response
}
});
}
});
});
First we create a function to get documents - This function accepts a callback as its parameter. We run our query and construct our doc list. Then we return the document list by executing our callback
Next we run a function to get our images for each document. This function accepts our document list and a callback as our parameters. It retrieves images for each document and calls a helper function (sync)
Our helper function processes the images into each document and returns the processed document.
We then finish our operations by returning an array of processed documents via the second callback.
Other notes
We could tide this up by structuring this procedural style code into a contained JSON object
The nesting of the function executions at the end of the document could be cleaned up further
I've avoided using the async libray as it helps better understanding of the callback model
Event emitters could be used to flatten the callbacks - see https://nodejs.org/dist/latest-v7.x/docs/api/events.html
Hope this helps
Dylan