Quick question guys. I think i got the logic down but I am messed up because express doesnt have a pipe method?
anyways, I have this for my express endpoint
app.get('/:report/top', function(req, res) {
readDataTop(global[req.params.report], function(err, obj) {
res.header("Cache-Control", "max-age=3600");
res.json(obj);
});
});
and currently i am doing this to get data for it.
function readDataTop (x, callback) {
console.log("Read "+x[6]+" and Sent Cached Top Half");
jf.readFile( "loadedreports/top"+x[6], 'utf8', callback);
};
(jf just makes writing json.parse and with try/catch blocks easier) anyways I want to instead of just using fs.readfile i want to open a stream because these files are getting huge.
How would I pipe stuff to express but still use json?
Something like this?
function readDataTop (x, callback) {
console.log("Read "+x[6]+" and Sent Cached Top Half");
var rstream = fs.createReadStream("loadedreports/top"+req.params.report+".json");
rstream.pipe(callback);
};
Any ideas? Req is undefined but I thought im passing it through the pipe correctly?
Related
I am using Node JS and have a JS file, which opens a connection to an API, works with the receving API data and then saves the changed data into a JSON file. Next I have an HTML file, which takes the data from the JSON file and puts it into a table. At the end I open the HTML file in my browser to look at the visualized table and its data.
What I would like to happen is, that the table (or more specific a DIV with an ID inside the table) from the HTML file refreshes itself, when the JSON data gets updated from the JS file. Kinda like a "live table/website", that I can watch change over time without the need to presh F5.
Instead of just opening the HTML locally, I have tried it by using the JS file and creating a connection with the file like this:
const http = require('http');
const path = require('path');
const browser = http.createServer(function (request, response) {
var filePath = '.' + request.url;
if (filePath == './') {
filePath = './Table.html';
}
var extname = String(path.extname(filePath)).toLowerCase();
var mimeTypes = {
'.html': 'text/html',
'.css': 'text/css',
'.png': 'image/png',
'.js': 'text/javascript',
'.json': 'application/json'
};
var contentType = mimeTypes[extname] || 'application/octet-stream';
fs.readFile(filePath, function(error, content) {
response.writeHead(200, { 'Content-Type': contentType });
response.end(content, 'utf-8');
});
}).listen(3000);
This creates a working connection and I am able to see it in the browser, but sadly it doesn't update itself like I wish. I thought about some kind of function, which gets called right after the JSON file got saved and tells the div to reload itself.
I also read about something like window.onload, location.load() or getElementById(), but I am not able to figure out the right way.
What can I do?
Thank you.
Websockets!
Though they might sound scary, it's very easy to get started with websockets in NodeJS, especially if you use Socket.io.
You will need two dependencies in your node application:
"socket.io": "^4.1.3",
"socketio-wildcard": "^2.0.0"
your HTML File:
<script type="module" src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.0/socket.io.js"></script>
Your CLIENT SIDE JavaScript file:
var socket = io();
socket.on("update", function (data) { //update can be any sort of string, treat it like an event name
console.log(data);
// the rest of the code to update the html
})
your NODE JS file:
import { Server } from "socket.io";
// other code...
let io = new Server(server);
let activeConnections = {};
io.sockets.on("connection", function (socket) {
// 'connection' is a "magic" key
// track the active connections
activeConnections[socket.id] = socket;
socket.on("disconnect", function () {
/* Not required, but you can add special handling here to prevent errors */
delete activeConnections[socket.id];
})
socket.on("update", (data) => {
// Update is any sort of key
console.log(data)
})
})
// Example with Express
app.get('/some/api/call', function (req, res) {
var data = // your API Processing here
Object.keys(activeConnections).forEach((conn) => {
conn.emit('update', data)
}
res.send(data);
})
Finally, shameful self promotion, here's one of my "dead" side projects using websockets, because I'm sure I forgot some small detail, and this might help. https://github.com/Nhawdge/robert-quest
I would like to know how can I use JSON Vulnerability Protection with express.js.
http://haacked.com/archive/2008/11/20/anatomy-of-a-subtle-json-vulnerability.aspx
The problem is I used to write res.send(jsonObj) from controllers, which will sent data directly to the client.
But I want to intercept the response and modify it with something and send to the client. The client can then undo the modification and retrieve the original data.
I saw the res.format function, but it is not working as for my need.
I was using res.json instead of res.send to send JSON, so I modified the code from your answer thus:
app.use(function (req, res, next) {
res.json = function (data) {
var strData = typeof data == 'object' ? JSON.stringify(data) : data;
strData = expressOptions.jsonPrefix + strData;
res.set('Content-Type', 'text/json');
res.send.call(res, strData);
};
next();
});
Although I've implemented this "just to be sure", I don't think this is a serious vulnerability. If you read this, which is linked from this (which is where I think you got your inspiration to write your Express middleware), it seems that the JSON Vulnerability doesn't exist in "modern" browsers, as in, as far back as IE 6 and FireFox 3.
So I'm not sure why AngularJS is telling people to implement this protection these days. Would appreciate if someone enlightened me in the comments! :)
Finally I ended up doing this:
app.use(function (req, res, next){
var actualSend = res.send;
res.send = function (data) {
if (typeof data == "object") {
var strData = expressOptions.jsonPrefix + JSON.stringify(data);
res.set('Content-Type', 'text/json');
actualSend.call (res, strData);
} else {
actualSend.call (res, data);
}
};
next();
});
Where expressOptions.jsonPrefix is the prefix I wanted.
I added it before my route configurations.
I have a an exports function I'm calling that should return a json array of draft results. In the route below in app.js, when I console.log draft_results, I get undefined
app.get('/draft-results', function(req, res) {
var draft_results = fantasy.getDraftResults(req, res);
console.log(util.inspect(draft_results, false, null));
//looks in views folder by default
res.render('draft-results', {
draft_results: draft_results
});
});
In my other file, this is the function that should be returning the json array. If i console.log draft, the data is there.
exports.getDraftResults = function(req, res, cb) {
oauth.get(
"http://fantasysports.yahooapis.com/fantasy/v2/league/" + conf.LEAGUE_ID + "/draftresults?format=json",
req.user.accessToken,
req.user.tokenSecret,
function(e, data, resp) {
if (e) console.error(e);
data = JSON.parse(data);
var draft = data.fantasy_content.league[1].draft_results;
res.json(draft);
}
);
};
I feel like I am returning the data incorrectly, and I can't seem to find any other good examples out there. Could someone please assist?
getDraftResults() is asynchronous. That means the results it generates occur sometime later. Thus, it cannot return its results directly from the function like you are trying to use.
It is unclear what you want to be doing here. Inside of getDraftResults() you are creating a JSON response back to the web request that started all this. That, in itself would be fine and will work as you have it (except the error handling is missing).
But, in your app.get() handler, you have completely different code that seems to thing that getDraftResults() is going to return a value (it has no return value at all) and then you will later use that return value.
So, if you just want getDraftResults to make a JSON response to the original web request, it's already doing that and you can remove the rest of what you have in the app.get() handler. If that's not really what you want to do and you want to use the response from getDraftResults() inside of the app.get() handler, then you will have to change the design of both functions and likely pass a callback to getDraftResults() so the callback can supply the asynchronous response and you can then continue the rest of the app.get() functionality in that callback.
If you're trying to do the latter, then here's a scaffolding (I don't know exactly what you're trying to accomplish so I can't be too detailed here):
app.get('/draft-results', function(req, res) {
fantasy.getDraftResults(req, function(err, draft_results) {
if (err) {
// send some sort of error response here
console.error(err);
return;
}
console.log(util.inspect(draft_results, false, null));
//looks in views folder by default
res.render('draft-results', {
draft_results: draft_results
});
});
});
exports.getDraftResults = function(req, cb) {
oauth.get(
"http://fantasysports.yahooapis.com/fantasy/v2/league/" + conf.LEAGUE_ID + "/draftresults?format=json",
req.user.accessToken,
req.user.tokenSecret,
function(e, data, resp) {
if (e) {
console.error(e);
cb(e);
return;
}
data = JSON.parse(data);
var draft = data.fantasy_content.league[1].draft_results;
// send results back to caller
cb(null, draft);
}
);
};
I'm using Winston and Morgan for all the back-end logging in Sails.js and I need to be able to log the responses from HTTP get requests. I need to log them in a file. My logFile currently takes shows all the http requests but it does not show the responses. I have searched all the options for Morgan and Winston and can't find a way/option to do this. I was just wondering if any of you had any advice on how to accomplish this?
Thanks!
You can write a middleware function for ExpressJS that will log the body once a response is sent. Basing it off of Node's http module to see how Connect (and therefore Express) manages the response body (which is a stream): you can hook into the two methods that write to that stream to grab the chunks and then concat/decode them to log it. Simple solution and could be made more robust but it shows the concept works.
function bodyLog(req, res, next) {
var write = res.write;
var end = res.end;
var chunks = [];
res.write = function newWrite(chunk) {
chunks.push(chunk);
write.apply(res, arguments);
};
res.end = function newEnd(chunk) {
if (chunk) { chunks.push(chunk); }
end.apply(res, arguments);
};
res.once('finish', function logIt() {
var body = Buffer.concat(chunks).toString('utf8');
// LOG BODY
});
next();
}
And then set it before any routes are assigned in the main app router:
app.use(bodyLog);
// assign routes
I would assume you could also use this as an assignment for a variable in Morgan but I haven't looked into how async variable assignment would work.
So I'm selecting Activities from the mongodb and populating User for each.
var query = Activity.find(query).populate("user");
return query.sort({created:"desc"}).exec(function(err, activities) {
debugger;
if (!err) {
return res.json(activities);
} else {
res.status(400).json(err);
}
});
As you can see I have a debugger; breakpoint is there, When I'm pring activities it prints an array of activities with the user object populated.
Also when I'm calling something like activities[0].toJSON() I get everything good!
But the response comes back with the user property empty !
I looked into the source of express.response.json(OBJ) and saw this line:
var body = JSON.stringify(val, replacer, spaces);
val is my activities
When calling JSON.stringify(activities) it will create a json with an empty user field.. any suggestions ?
Try the lean option. That gives back plain JS objects with no mongoose weirdness. Also, your error handling seems a little awkward, can be simplified.
var query = Activity.find(query).populate("user");
query.sort({created:"desc"}).lean().exec(function(err, activities) {
if (err) return res.status(400).json(err);
res.json(activities);
});
I would go even further, not hard-coding error sending in routes but simply passing along via if (err) return next(err) to error-handling middleware defined elsewhere in your app. You can still set the status, then use detection in your middleware, something like this:
app.use(function(err, req, res, next){
err.status = err.status || 500;
res.status(err.status).json(err);
});