NodeJS http.request end processing before data processes - json

Can some explain why the the http.request end function is running before any data is actually retrieved? And how would I debug this any further? Should I be checking an http status?
This is going to work with Google Home app, but I took that code out and getting same error running locally. The http.request is from what a teacher provided in a class.
You can paste: people/?search=Luke%20Skywalker
into http://swapi.com (SW = StarWars API) to see the expected result.
'use strict';
/*eslint no-undef: "error"*/
/*eslint-env node*/
/*eslint-disable no-console */
let http = require('http');
let starWarsAPI = `www.swapi.co`;
//function to get details of the Star Wars Characters
//exports.handler = function(event, context, callback) {
//console.log("event=" + JSON.stringify(event));
//console.log("context=" + JSON.stringify(context));
//let characterName = event.result.parameters.StarWarsCharacter;
let characterName = "Luke Skywalker";
console.log("**** characterName=" + characterName);
let options = searchPeopleRequestOptions(characterName);
console.log("options=" + JSON.stringify(options));
makeRequest(options, function( data, error) {
console.log(" Processing data.results");
let person = data.results[0];
if (person) {
let height = person.height;
let mass = person.mass;
let response = person.name + " is " + height + " centimeters tall, weighs " + mass + " kilograms";
console.log("**** response=" + response);
//callback(null, {"speech": response});
}
else {
console.log ("No person found");
//callback(null, {"speech": "I'm not sure that character exists!"});
}
});
//};
console.log("The end");
//create a function to read first and last names from the API.
function searchPeopleRequestOptions(argCharacterName) {
var pathValue = `/api/people/?search=`+
encodeURIComponent(argCharacterName);
return {
host: starWarsAPI,
path: pathValue
};
}
function makeRequest(options, callback) {
var responseString = "";
var request = http.request(options,
function(response) {
response.on('data', function(data) {
responseString += data;
console.log("responseString=" + responseString);
});
response.on('end', function() {
console.log("end: responseString=" + responseString);
// dies on next line because responseString is empty
var responseJSON = JSON.parse(responseString);
callback(responseJSON, null);
});
response.on('error', function (error) {
console.log('\n Error received: ' + error);
});
});
request.end();
}
This is what I see when I run it:
E:\GitHub\NealWalters\GoogleHomeTest
λ node indexTest.js
**** characterName=Luke Skywalker
options={"host":"www.swapi.co","path":"/api/people/?search=Luke%20Skywalker"}
The end
end: responseString=
undefined:1
I'm not sure what's writing out the "undefined: 1" line.

If you look at the server's response status code, it will be 301: Moved Permanently.
And value of location field of response is:
https://swapi.co/api/people/?search=Luke%20Skywalker
instead
http://swapi.co/api/people/?search=Luke%20Skywalker
As we can see, the protocol changed from http to https.
The problem is that the http client supplied with the node.js does not support redirection for permanently changed URL.
So, you can use https module instead http (just change the require('https')).
Or use packages that support redirection. For example axios or request.

Related

CouchDb 2.1.1 Admin API Compaction PUT Request

I am working in NodeJS with CouchDB 2.1.1.
I'm using the http.request() method to set various config settings using the CouchDB API.
Here's their API reference, yes, I've read it:
Configuration API
Here's an example of a working request to set the logging level:
const http = require('http');
var configOptions = {
host: 'localhost',
path: '/_node/couchdb#localhost/_config/',
port:5984,
header: {
'Content-Type': 'application/json'
}
};
function setLogLevel(){
configOptions.path = configOptions.path+'log/level';
configOptions.method = 'PUT';
var responseString = '';
var req = http.request(configOptions, function(res){
res.on("data", function (data) {
responseString += data;
});
res.on("end", function () {
console.log("oldLogLevel: " + responseString);
});
});
var data = '\"critical\"';
req.write(data);
req.end();
}
setLogLevel();
I had to escape all the quotes and such, which was expected.
Now I'm trying to get CouchDb to accept a setting for compaction.
The problem is that I'm attempting to replicate this same request to a different setting but that setting doesn't have a simple structure, though it appears to be "just a String" as well.
The CouchDB API is yelling at me about invalid JSON formats and I've tried a boatload of escape sequences and attempts to parse the JSON in various ways to get it to behave the way I think it should.
I can use Chrome's Advanced Rest Client to send this payload, and it is successful:
Request Method: PUT
Request URL: http://localhost:5984/_node/couchdb#localhost/_config/compactions/_default
Request Body: "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"60%\"}, {from, \"23:00\"}, {to, \"04:00\"}]"
This returns a "200 OK"
When I execute the following function in my node app, I get a response of:
{"error":"bad_request","reason":"invalid UTF-8 JSON"}
function setCompaction(){
configOptions.path = configOptions.path+'compactions/_default';
configOptions.method = 'PUT';
var responseString = '';
var req = http.request(configOptions, function(res){
res.on("data", function (data) {
responseString += data;
});
res.on("end", function () {
console.log("oldCompaction: " + responseString);
});
});
var data = "\"[{db_fragmentation, \"70%\"}, {view_fragmentation, \"60%\"}, {from, \"23:00\"}, {to, \"04:00\"}]\"";
req.write(data);
req.end();
}
Can someone point at what I'm missing here?
Thanks in advance.
You need to use node's JSON module to prepare the data for transport:
var data = '[{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "23:00"}, {to, "04:00"}]';
// Show the formatted data for the requests' payload.
JSON.stringify(data);
> '"[{db_fragmentation, \\"70%\\"}, {view_fragmentation, \\"60%\\"}, {from, \\"23:
00\\"}, {to, \\"04:00\\"}]"'
// Format data for the payload.
req.write(JSON.stringify(data));

Get random wiki page from cloud functions

I tried to get a random Wikipedia page over their API via Google Cloud Functions. The Wikipedia API works fine. This is my request:
https://de.wikipedia.org/w/api.php?action=query&format=json&generator=random
For testing you can change the format to jsonfm in see the result in the browser. Click here 👍.
But it seems that my functions get destroyed even before the request was completely successfully. If I want to parse the data (or even if I want to log that data) I got a
SyntaxError: Unexpected end of json
The log look like (for example) that (no I haven't cut it by myself):
DATA: ue||"},"query":{"pages":{"2855038":{"pageid":2855038,"ns":0,"title":"Thomas Fischer
Of course, that is not a valid json and can't be parsed. Whatever this is my function:
exports.randomWikiPage = function getRandomWikiPage (req, res) {
const httpsOptions = {
host: "de.wikipedia.org",
path: "/w/api.php?action=query&format=json&generator=random"
};
const https = require('https');
https.request(httpsOptions, function(httpsRes) {
console.log('STATUS: ' + httpsRes.statusCode)
console.log('HEADERS: ' + JSON.stringify(httpsRes.headers))
httpsRes.setEncoding('utf8')
httpsRes.on('data', function (data) {
console.log("DATA: " + data)
const wikiResponse = JSON.parse(data);
const title = wikiResponse.query.title
res.status(200).json({"title": title})
});
}).end();
};
I've already tried to return something here. Like that video explained. But as I look into the node docs https.request don't return a Promise. So return that is wrong. I've also tried to extract the on('data', callback) into it's own function so that I can return the callback. But I haven't a success with that either.
How have to look my function that it return my expected:
{"title": "A random Wikipedia Page title"}
?
I believe your json comes through as a stream in chunks. You're attempting to parse the first data chunk that comes back. Try something like:
https.request(httpsOptions, function(httpsRes) {
console.log('STATUS: ' + httpsRes.statusCode)
console.log('HEADERS: ' + JSON.stringify(httpsRes.headers))
httpsRes.setEncoding('utf8')
let wikiResponseData = '';
httpsRes.on('data', function (data) {
wikiResponseData += data;
});
httpRes.on('end', function() {
const wikiResponse = JSON.parse(wikiResponseData)
const title = wikiResponse.query.title
res.status(200).json({"title": title})
})
}).end();
};

aws lambda s3 function isn't called inside alexa skills kit

I am trying to create a skill for Amazon Echo that will call a JSON file from AWS S3. When I call the code from s3 basic get function it works. And the Amazon Alexa code works on its own.
But when I call them together the function gets skipped. So for the following code the console gets called before and after s3.getObject(). But the middle one gets skipped. I do not understand why.
I also checked whether s3 was being called, and it is.
let aws = require('aws-sdk');
let s3 = new aws.S3({ apiVersion: '2006-03-01'});
function callS3() {
console.log('loading S3 function');
var myData = [];
const params = {
Bucket: 'cvo-echo',
Key: 'data.json'
};
console.log("trying to get s3");
s3.getObject(params, (err, data) => {
if (err) {
console.log('error in s3 get: \n' + err);
//const message = `Error getting object ${key} from bucket ${bucket}.
// Make sure they exist and your bucket is in same region as this function.
//console.log(message);
} else {
console.log('CONTENT TYPE: ', data.ContentType);
console.log('Data body: \n' + data.Body.toString());
myData = JSON.parse(data.Body.toString());
console.log('myData.length = ' + myData.length);
}
console.log('myData >> ' + myData);
});
console.log('finished callS3() func');
return myData;
}
This might be a control flow issue, I've worked with amazons sdk before and was running into similar issues. Try implementing async within your code to have a better control of what happens when. This way methods won't skip.
UPDATE: adding some code examples of what you could do.
function callS3(callback) {
console.log('loading S3 function');
var myData = [];
const params = {
Bucket: 'cvo-echo',
Key: 'data.json'
};
console.log("trying to get s3");
s3.getObject(params, (err, data) => {
if (err) {
console.log('error in s3 get: \n' + err);
//const message = `Error getting object ${key} from bucket ${bucket}.
// Make sure they exist and your bucket is in same region as this function.
//console.log(message);
callback(err,null);//callback the error.
} else {
console.log('CONTENT TYPE: ', data.ContentType);
console.log('Data body: \n' + data.Body.toString());
myData = JSON.parse(data.Body.toString());
console.log('myData.length = ' + myData.length);
console.log('myData >> ' + myData);
console.log('finished callS3() func');
//Include the callback inside of the S3 call to make sure this function returns until the S3 call completes.
callback(null,myData); // first element is an error and second is your data, first element is null if no error ocurred.
}
});
}
/*
This MIGHT work without async but just in case you can read more about
async.waterfall where functions pass down values to the next function.
*/
async.waterfall([
callS3()//you can include more functions here, the callback from the last function will be available for the next.
//myNextFunction()
],function(err,myData){
//you can use myData here.
})
It's a timing issue. Here is an example of loading a JSON file from an S3 share when a session is started.
function onLaunch(launchRequest, session, callback) {
var sBucket = "your-bucket-name";
var sFile = "data.json";
var params = {Bucket: sBucket, Key: sFile};
var s3 = new AWS.S3();
var s3file = s3.getObject(params)
new AWS.S3().getObject(params, function(err, data) {
if (!err) {
var json = JSON.parse(new Buffer(data.Body).toString("utf8"));
for(var i = 0; i < json.length; i++) {
console.log("name:" + json[i].name + ", age:" + json[i].age);
}
getWelcomeResponse(callback);
} else {
console.log(err.toString());
}
});
}

AWS Lambda http request: Unable to stringify body as json: Converting circular structure to JSON

I would like to return the result of an HTTP request in my AWS Lambda function:
var http = require('http');
exports.someFunction = function(event, context) {
var url = "http://router.project-osrm.org/trip?loc=47.95,12.95&loc=47.94,12.94";
http.get(url, function(res) {
context.succeed(res);
}).on('error', function(e) {
context.fail("Got error: " + e.message);
});
}
It should return exactly what I get when I open the url directly in my browser (try it to see the expected json).
AWS Lambda return the following error message when I call context.succeed(res):
{
"errorMessage": "Unable to stringify body as json: Converting circular structure to JSON",
"errorType": "TypeError"
}
I assume that I need to use some property of res instead of res itself, but I couldn't figure out which one contains the actual data I want.
If you are using the raw http module you need to listen for data and end events.
exports.someFunction = function(event, context) {
var url = "http://router.project-osrm.org/trip?loc=47.95,12.95&loc=47.94,12.94";
http.get(url, function(res) {
// Continuously update stream with data
var body = '';
res.on('data', function(d) {
body += d;
});
res.on('end', function() {
context.succeed(body);
});
res.on('error', function(e) {
context.fail("Got error: " + e.message);
});
});
}
Using another module such as request https://www.npmjs.com/package/request would make it so you don't have to manage those events and your code could go back to almost what you had before.

Parse JSON object with Node.js

I have a little node application that runs and gets stats from an httpprovider
Its returning the value every second in this format:
{ WOWZA_CONNECTIONS_CURRENT: 21,
WOWZA_CONNECTIONS_TOTAL: 4879,
WOWZA_CONNECTIONS_BYTES_IN: 303242,
WOWZA_CONNECTIONS_BYTES_OUT: 96372 }
but I need it to be returning in a different format, exactly like this:
WOWZA_CONNECTIONS_CURRENT 21 Rem-East-v4-Edge-1
WOWZA_CONNECTIONS_TOTAL 4879 Rem-East-v4-Edge-1
WOWZA_CONNECTIONS_BYTES_IN 303242 Rem-East-v4-Edge-1
WOWZA_CONNECTIONS_BYTES_OUT 96372 Rem-East-v4-Edge-1
How would I accomplish this?
Here is my Code:
#!/usr/bin/env node
// Requires.
// request, jsdom, optimist, all of which can be install with npm.
// NOTE: -----------------------------------------------------------------------
// Uses jsdom 0.2.13, 0.2.14 has a bug
// https://github.com/tmpvar/jsdom/issues/436
// Install w/ npm via `npm install jsdom#0.2.13`
// -----------------------------------------------------------------------------
// Require modules.
var request = require('request'),
jsdom = require('jsdom'),
argv = require('optimist').argv;
// Make sure that at least the --uri argument was passed.
if (argv.uri.length == 0) {
console.log('URI Required! Script should be called with one argument which is the URI of the connectioncounts HTTP provider to query.');
return;
}
/**
*
*/
var Collector = {
stats: {},
get_stats: function(uri, callback) {
request({ uri: uri }, function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log('Error when contacting ' + uri);
}
jsdom.env({
html: body,
scripts: [
'http://code.jquery.com/jquery.min.js'
]
}, function (err, window) {
// User jQuery to Gather some stats from the connectioncounts HTTP
// provider.
var $ = window.jQuery;
Collector.stats['WOWZA_CONNECTIONS_CURRENT'] = parseInt($('ConnectionsCurrent').html());
Collector.stats['WOWZA_CONNECTIONS_TOTAL'] = parseInt($('ConnectionsTotal').html());
Collector.stats['WOWZA_CONNECTIONS_BYTES_IN'] = parseFloat($('MessagesInBytesRate').html());
Collector.stats['WOWZA_CONNECTIONS_BYTES_OUT'] = parseFloat($('MessagesOutBytesRate').html());
callback(window);
});
});
},
}
/**
* Wrap Collector.get_stats call in a closure it works better with setInterval.
*/
var callDelay = function() {
Collector.get_stats(argv.uri, function(response) {
// Print out collected stats.
console.log(Collector.stats);
});
}
// Get stats once right away.
callDelay();
// Allo repeating at specified interval if --repeat is set.
if (argv.repeat != undefined) {
// Default to every 30 seconds if no delay is specified.
if (argv.delay == undefined) {
argv.delay = 30000;
}
setInterval(callDelay, argv.delay);
}
EDIT:
I achieved the output i needed by changing the output section of the code link so.
// Print out collected stats.
//console.log(Collector.stats);
console.log('WOWZA_CONNECTIONS_CURRENT ' + Collector.stats.WOWZA_CONNECTIONS_CURRENT + ' Rem-East-v4-Edge-1');
console.log('WOWZA_CONNECTIONS_TOTAL ' + Collector.stats.WOWZA_CONNECTIONS_TOTAL + ' Rem-East-v4-Edge-1')
console.log('WOWZA_CONNECTIONS_BYTES_IN ' + Collector.stats.WOWZA_CONNECTIONS_BYTES_IN + ' Rem-East-v4-Edge-1')
console.log('WOWZA_CONNECTIONS_BYTES_OUT ' + Collector.stats.WOWZA_CONNECTIONS_BYTES_OUT + ' Rem-East-v4-Edge-1')
You don't need to parse this as JSON, it is already there in an object. Just construct a string the way you want it by accessing it via Collector.stats.