Streaming responses from MongoDB to client via Hapi - json

Question

What's the best approach to streaming MongoDB query responses to the client via Hapi? I've seen some examples with http or request, but not hapi.
The problem is that I'm getting concatenated and stringified JSON objects on the client side, but I can't can't call JSON.parse on the result because together it's not valid JSON.
Some solutions I've seen suggest concatenating on the server side before sending to the client, but that seems to defeat the value of streams.
For example:
const Hapi = require('hapi'),
MongoClient = require('mongodb').MongoClient,
Readable = require('stream').Readable;
// Connection url
const url = 'mongodb://localhost:27017/test';
// Create a server with a host and port
const server = new Hapi.Server();
server.connection({
host: 'localhost',
port: 8000
});
// Add the route
server.route({
method: 'GET',
path: '/stream',
handler: function (request, reply) {
let docs = [{ a: 1, b: 1 }, { a: 2, b: 2 }, { a: 3, b: 3 }, { a: 4, b: 4 }];
// Connect using MongoClient
MongoClient.connect(url, (err, db) => {
// Create a collection we want to drop later
const col = db.collection('stream_example');
// Insert documents into collection
col.insertMany(docs, { w: 1 }, function (err) {
if (err) return console.log(err);
// Peform a find to get a cursor
const stream = col.find()
.stream({
transform: function (doc) {
return JSON.stringify(doc);
}
});
reply(new Readable().wrap(stream));
});
});
}
});
// Start the server
server.start(err => {
if (err) {
throw err;
}
console.log('Server running at:', server.info.uri);
});
Returns a response.result of:
"{"_id":"57b0b99d681bb97a9321f03e","a":1,"b":1}{"_id":"57b0b99d681bb97a9321f03f","a":2,"b":2}{"_id":"57b0b99d681bb97a9321f040","a":3,"b":3}{"_id":"57b0b99d681bb97a9321f041","a":4,"b":4}"
Which is not valid JSON and cannot be parsed.
I've tried piping this stream into the event-stream module's .join('\n') stream to add newlines while also pushing string'd "[" and "]" before and after to build a stringified JSON Array, but have not yet been successful. This feels too hacky anyways.
Is there a better way?

Answer 1

A valid JSON has to be sent by using a stream transform.
Basically you have to:
start the stream with '['
then append stringified JSON object
add ',' after each of the objects
end stream with ']'
so the final result recevied in the stream would be valid JSON, like
[
{'key': 'value'},
{'key': 'value'},
]
Some of the solutions:
http://andyfiedler.com/2017/01/mongodb-stream-to-hapi
https://github.com/nlindley/hapi-mongo-stream
https://github.com/dominictarr/JSONStream

Answer 2

Here is an example of how I have been using Mongo with Hapi.
From BoardRepo.js:
module.exports = {
GetAllBoards: function (facility) {
return new Promise(function (resolve, reject) {
var db = mongo.ETestDatabase;
db.collection('boards').find({ "Location.Facility": facility }).toArray().then(r => {
resolve(r);
}).catch(err => {
logger.error('Error getting all boards by facility - ' + err.message);
reject(err.message);
});
});
}
};
In the Hapi handler (BoardHandler.js):
module.exports = {
GetAllBoards: {
method: 'GET',
path: '/v1/boards/facility/all/{facility}',
config: {
auth: 'jwt',
plugins: { 'hapiAuthorization': { roles: ['ADMINISTRATOR', 'MANAGER', 'TECHNICIAN', 'OPERATOR'] } },
description: 'Gets all boards per facility',
tags: ['api'],
handler: (request, reply) => {
logger.info('[' + request.auth.credentials.username + '] GetAllBoards requested');
var facility = request.params.facility;
repo.GetAllBoards(facility)
.then(boards => {
if (boards !== null) {
reply(boards);
} else {
reply().code(404);
}
})
.catch(err => {
geh.HandleError(request.auth.credentials.username, err, reply);
});
}
}
}
};