The action did not produce a valid response and exited unexpectedly - ibm-cloud-functions

I want to call a Node-RED flow from IBM Cloud Functions.
const https = require('https');
function main(params) {
const path = "/" + params.route + "?" + params.query_params ;
const options = {
hostname: params.hostname,
path: path,
port: 443,
method: 'GET'
};
return new Promise((resolve, reject) => {
https.get(options, (resp) => {
resp.on('data', (d) => {
let s = d.toString();
obj = JSON.parse(s);
resolve({ "gw_result": obj })
});
});
})
}
In the Node-RED flow I'm using a HTTP request to get data from another server. For test purposes I used a GET request to google.com but have same results using another Node-RED endpoint.
As soon as I invoke the web action I get the error message "The action did not produce a valid response and exited unexpectedly". The output of the Node-RED flow appears some seconds later in the web action's log although the Node-RED flow works properly and promptly (I used debug Node-RED debug nodes to check this).
The https GET request to Node-RED works well when I replace the http request in Node-RED by something else, e.g. a Function node, even when I use a Delay node to delay the response for a second or so.

This code works, although google.com does not return an object, of course.
var rp = require('request-promise');
function main(params) {
var uri = params.hostname + params.route + params.query_params
return new Promise(function (resolve, reject) {
rp(uri)
.then(function (parsedBody) {
obj = JSON.parse(parsedBody);
resolve({ "gw_result": obj
});
})
.catch(function (err) {
resolve({ message: 'failed!!', error: err.toString() });
});
});
}

Related

Reading Arduino data using WebSocket with HTML & nodejs

I can't quite grasp the last step in this puzzle. Everything compiles, and it is "bug-free". This is my first foray into communications/full-stack and I have been stumped despite numerous excellent tutorials.
[WORKING] Arduino reads and interprets sensor data
[WORKING] index.js acquires data with serial communication over USB
[WORKING] index.js creates a WebSocket connection using nodejs
[WORKING] index.html performs the WebSocket handshake
[WORKING] index.html uses Plotly to create a real-time line graph
[WIP] index.html passes through the Arduino data in the Plotly function
Chopped down index.html:
<script src="server/plotly.min.js"></script>
<script>
//connection to the web socket server
const ws = new WebSocket("ws://localhost:5000");
let foo = 0.0;
//working
ws.addEventListener("open", () => {
console.log("We Are Connected");
ws.send("TestData");
});
//working
ws.addEventListener("message", e => {
console.log(e);
console.log("Data Recieved! Success.");
});
</script>
Rest of the file is just the graphing function which I would like to pass through Arduino data.
index.js
const WebSocket = require("ws");
const wss = new WebSocket.Server({ port: 5000 });
//create a serial port that allows serial connection from Arduino
let SerialPort = require("serialport");
let port = new SerialPort('COM4', { baudRate: 9600 });
let Readline = require("#serialport/parser-readline");
let parser = port.pipe(new Readline({ delimiter: '\n' }));
wss.on("connection", ws => {
//working
console.log("New Client Connection");
//this is what I need to passthrough my Plotly arg
parser.on("data", data => {
//event is firing but can't get client to grab this. Console logs data correctly.
console.log(RPM: ${data});
});
//working on both ends
ws.on("message", data => {
console.log("TEST")
ws.send(data);
});
//doesn't log?
port.on("open", () => {
console.log("Serial Port Open");
});
});
//working
console.log("The server is ON");
I'm looking for a strategy or method to grab the sensor data in my HTML file. Is it something simple conceptually I am missing? Thank you.
You established a websocket server, and it is works. If you want to send message to websocket, define a socket to the websocket server, get out the Serial part from websocket server, and run is standalone, and send data from this to the websocket.
Like this:
const http = require('http');
const WebSocketServer = require('websocket').server;
const server = http.createServer();
server.listen(5000);
const wsServer = new WebSocketServer({
httpServer: server
});
let SerialPort = require("serialport");
var serialPort = new SerialPort("COM5", {
baudRate: 9600,
parser: new SerialPort.parsers.Readline("\n")
});
var connection;
wsServer.on('request', function(request) {
connection = request.accept(null, request.origin);
connection.on('message', function(message) {
console.log('Received Message:', message.utf8Data);
connection.sendUTF('Hi this is WebSocket server!');
});
connection.on('close', function(reasonCode, description) {
console.log('Client has disconnected.');
});
});
serialPort.on('open',function(){
//connection.sendUTF('Hi this is WebSocket server!');
console.log('open');
serialPort.on('data', function(data){
readData = data.toString();
console.log("N<", readData);
if( typeof connection!="undefined")
connection.sendUTF( readData);
});
});

Stripe Error Message 405 - "append .json to uri to use rest api"

I'm using Stripe, and trying to send a test webhook to my URL and database hosted by Firebase. When I "send test webhook," I get the following error message in the Stripe Console:
Test Webhook Error: 405
"append .json to your request URI to use the rest API"
My code is a direct copy of the tutorial: https://github.com/GaryH21/Stripe-Webhooks-Tutorial/blob/master/functions/index.js
Here is the code of my index.js:
const functions = require('firebase-functions');
const stripe = require("stripe")(functions.config().keys.webhooks);
const admin = require('firebase-admin')
admin.initializeApp();
const endpointSecret = functions.config().keys.signing;
exports.events = functions.https.onRequest((request, response) => {
let sig = request.headers["stripe-signature"];
try {
let event = stripe.webhooks.constructEvent(request.rawBody, sig, endpointSecret)
return admin.database().ref('/events').push(event)
.then((snapshot) => {
return response.json({ received: true, ref: snapshot.ref.toString() })
})
.catch((err) => {
console.error(err)
return response.status(500).end() // error saving to database
})
} catch (err) {
return response.status(400).end() // signing signature failed
}
})
exports.exampleDataBaseTrigger = functions.database.ref('/events/{eventId}').onCreate((snapshot, context) => {
return console.log({
eventId: context.params.eventid,
data: snapshot.val()
})
})
The only time in the tutorial and in my code that .json is used is in the line: return response.json({ received: true, ref: snapshot.ref.toString() })
Should I be appending .json onto "request" somewhere, such as in request.RawBody?
It isn't a problem with the signing keys, as that would give the 400 Error message, which I already dealt with and fixed.
I would be happy to share the code of other files in my app, but as far as I can tell none of the rest is relevant to the problem. Thank you very much.

From ES2018 async/await to ES2015 Promises . ... timeout

I am trying to convert an ES2018 async function into an ES2015 (ES6) function, but I get a timeout, guess my ES2015 version is wrong...but where?
ES2018 version
async function connectGoogleAPI () {
// Create a new JWT client using the key file downloaded from the Google Developer Console
const client = await google.auth.getClient({
keyFile: path.join(__dirname, 'service-key.json'),
scopes: 'https://www.googleapis.com/auth/drive.readonly'
});
// Obtain a new drive client, making sure you pass along the auth client
const drive = google.drive({
version: 'v2',
auth: client
});
// Make an authorized request to list Drive files.
const res = await drive.files.list();
console.log(res.data);
return res.data;
}
ES2015 version w/Promise
function connectGoogleAPI () {
return new Promise((resolve, reject) => {
const authClient = google.auth.getClient({
keyFile: path.join(__dirname, 'service-key.json'),
scopes: 'https://www.googleapis.com/auth/drive.readonly'
});
google.drive({
version: 'v2',
auth: authClient
}), (err, response) => {
if(err) {
reject(err);
} else {
resolve(response);
}
}
});
}
You haven't translated the await of getClient. Remember, await = then (roughly). You're also falling prey to the promise creation anti-pattern: When you already have a promise (from getClient), you almost never need to use new Promise. Just use then.
Here's an example with each await converted into a then, using the chain for the subsequent operations:
function connectGoogleAPI () {
// Create a new JWT client using the key file downloaded from the Google Developer Console
return google.auth.getClient({
keyFile: path.join(__dirname, 'service-key.json'),
scopes: 'https://www.googleapis.com/auth/drive.readonly'
}).then(client => {
// Obtain a new drive client, making sure you pass along the auth client
const drive = google.drive({
version: 'v2',
auth: client
});
// Make an authorized request to list Drive files.
return drive.files.list();
}).then(res => {
console.log(res.data);
return res.data;
});
}
That last part can be just
}).then(res => res.data);
...if you remove the console.log. (Or we could abuse the comma operator.)
Notes:
Each await needs to become a then handler (there were two in the original, awaiting getClient and drive.files.list)
In a then handler, if you have to wait for another promise (such as the one from drive.files.list) you typically return it from the handler, and then use another handler to handle that result (which is why I have return drive.files.list() and then a separate handler for converting res to res.data)
Re that second point: Sometimes nesting is appropriate, such as when you need to combine the result with some intermediate value you only have with in your then handler. (For instance, if we wanted to combine res.data with client.) But generally, prefer not to nest.

Nodejs, Cloud Firestore Upload Tasks - Auth error:Error: socket hang up

I'm coding a function that runs API calls and requests JSON from a huge database in sequence via offsets. The JSON response is parsed and then the subsequent data within is uploaded to our Cloud Firestore server.
Nodejs (Node 6.11.3) & Latest Firebase Admin SDK
The information is parsed as expected, and prints to the console perfectly. When the data attempts to upload to our Firestore database however, the console is spammed with the error message:
Auth error:Error: socket hang up
(node:846) UnhandledPromiseRejectionWarning: Unhandled promise rejection
(rejection id: -Number-): Error: Getting metadata from plugin failed with
error: socket hang up
and occasionally:
Auth error:Error: read ECONNRESET
The forEach function collects the items from the downloaded JSON and processes the data before uploading to the Firestore database. Each JSON has up to 1000 items of data (1000 documents worth) to pass through the forEach function. I understand that this might be a problem if the function repeats before the upload set finishes?
I'm a coding newbie and understand that the control flow of this function isn't the best. However, I can't find any information on the error that the console prints. I can find plenty of information on socket hang ups, but none on the Auth error section.
I'm using a generated service account JSON as a credential to access our database, which uses the firebase-adminsdk account. Our read/write rules for the database are currently open to allow any access (as we're in development with no real users).
Here's my function:
Firebase initialisation & offset zero-ing
const admin = require('firebase-admin');
var serviceAccount = require("JSON");
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: "URL"
});
var db = admin.firestore();
var offset = 0;
var failed = false;
Running the function & setting HTTP Headers
var runFunction = function runFunction() {
var https = require('https');
var options = {
host: 'website.com',
path: (path including an offset and 1000 row specifier),
method: 'GET',
json: true,
headers: {
'content-type': 'application/json',
'Authorization': 'Basic ' + new Buffer('username' + ':' + 'password').toString('base64')
}
};
Running the HTTP Request & Re-running the function if we haven't reached the end of the response from the API
if (failed === false) {
var req = https.request(options, function (res) {
var body = '';
res.setEncoding('utf8');
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', () => {
console.log('Successfully processed HTTPS response');
body = JSON.parse(body);
if (body.hasOwnProperty('errors')) {
console.log('Body ->' + body)
console.log('API Call failed due to server error')
console.log('Function failed at ' + offset)
req.end();
return
} else {
if (body.hasOwnProperty('result')) {
let result = body.result;
if (Object.keys(result).length === 0) {
console.log('Function has completed');
failed = true;
return;
} else {
result.forEach(function (item) {
var docRef = db.collection('collection').doc(name);
console.log(name);
var upload = docRef.set({
thing: data,
thing2: data,
})
});
console.log('Finished offset ' + offset)
offset = offset + 1000;
failed = false;
}
if (failed === false) {
console.log('Function will repeat with new offset');
console.log('offset = ' + offset);
req.end();
runFunction();
} else {
console.log('Function will terminate');
}
}
}
});
});
req.on('error', (err) => {
console.log('Error -> ' + err)
console.log('Function failed at ' + offset)
console.log('Repeat from the given offset value or diagnose further')
req.end();
});
req.end();
} else {
req.end();
}
};
runFunction();
Any help would be greatly appreciated!
UPDATE
I've just tried changing the rows of JSON that I pull at a time and subsequently upload at a time using the function - from 1000 down to 100. The socket hang up errors are less frequent so it is definitely due to overloading the database.
Ideally it would be perfect if each forEach array iteration waited for the previous iteration to complete before commencing.
UPDATE #2
I've installed the async module and I'm currently using the async.eachSeries function to perform one document upload at a time. All errors mid-upload disappear - however the function will take an insane amount of time to finish (roughly 9 hours for 158,000 documents). My updated loop code is this, with a counter implemented:
async.eachSeries(result, function (item, callback) {
// result.forEach(function (item) {
var docRef = db.collection('collection').doc(name);
console.log(name);
var upload = docRef.set({
thing: data,
thing2: data,
}, { merge: true }).then(ref => {
counter = counter + 1
if (counter == result.length) {
console.log('Finished offset ' + offset)
offset = offset + 1000;
console.log('Function will repeat with new offset')
console.log('offset = ' + offset);
failed = false;
counter = 0
req.end();
runFunction();
}
callback()
});
});
Also, after a period of time the database returns this error:
(node:16168) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: -Number-): Error: The datastore operation timed out, or the data was temporarily unavailable.
It seems as if now my function is taking too long... instead of not long enough. Does anyone have any advice on how to make this run faster without stated errors?
The write requests as part of this loop were simply exceeding Firestore's quota - thus the server was rejecting the majority of them.
To solve this issue I converted my requests to upload in chunks of 50 or so items at a time, with Promises confirming when to move onto the next chunk upload.
The answer was posted here -> Iterate through an array in blocks of 50 items at a time in node.js, and the template for my working code is as below:
async function uploadData(dataArray) {
try {
const chunks = chunkArray(dataArray, 50);
for (const [index, chunk] of chunks.entries()) {
console.log(` --- Uploading ${index + 1} chunk started ---`);
await uploadDataChunk(chunk);
console.log(`---Uploading ${index + 1} chunk finished ---`);
}
} catch (error) {
console.log(error)
// Catch en error here
}
}
function uploadDataChunk(chunk) {
return Promise.all(
chunk.map((item) => new Promise((resolve, reject) => {
setTimeout(
() => {
console.log(`Chunk item ${item} uploaded`);
resolve();
},
Math.floor(Math.random() * 500)
);
}))
);
}
function chunkArray(array, chunkSize) {
return Array.from(
{ length: Math.ceil(array.length / chunkSize) },
(_, index) => array.slice(index * chunkSize, (index + 1) * chunkSize)
);
}
Pass the data array through to uploadData - using uploadData(data); and post your upload code for each item into uploadDataChunk inside the setTimeout block (before the resolve() line) within the chunk.map function.
I got around this by chaining the promises in the loop with a wait of 50 milliseconds in between each.
function Wait() {
return new Promise(r => setTimeout(r, 50))
}
function writeDataToFirestoreParentPhones(data) {
let chain = Promise.resolve();
for (let i = 0; i < data.length; ++i) {
var docRef = db.collection('parent_phones').doc(data[i].kp_ID_for_Realm);
chain = chain.then(()=> {
var setAda = docRef.set({
parent_id: data[i].kf_ParentID,
contact_number: data[i].contact_number,
contact_type: data[i].contact_type
}).then(ref => {
console.log(i + ' - Added parent_phones with ID: ', data[i].kp_ID_for_Realm);
}).catch(function(error) {
console.error("Error writing document: ", error);
});
})
.then(Wait)
}
}
For me this turned out to be a network issue.
Uploading 180,000 documents in batches of 10,000 was no trouble for me before and today having used a public, slower wifi connection, I received that error.
Switching back to my 4G mobile connection sorted the problem for me. Not sure whether it's a speed issue - could have been a security issue - but I'll go with that assumption.

Protractor: HTTP Response Testing

I'm using Protracotr for e2e testing.
I want to test response from HTTP in protractor.
Basically:
I have running some NodeJS server.
I want to send request to this server
Receive some JSON data
Parse those data
Check if they are correct
I'm using "http" NODEJS Lib to make http calls GET+POST.
var http = require('http');
describe("Some test", function() {
function httpGet(siteUrl) {
http.get(siteUrl, function(response) {
response.setEncoding('utf8');
response.on("data", function(chunk) {
bodyString += chunk;
});
response.on('end', function() {
defer.fulfill({
bodyString: bodyString
});
});
}).on('error', function(e) {
defer.reject("Got http.get error: " + e.message);
});
return defer.promise;
}
it('Test case', function(){
httpGet("http://localhost:3333/path/1/10").then(function(result) {
var json_data = JSON.parse(result.bodyString);
for (var i = 0; i < json_data.length; ++i) {
console.log("label: " + json_data[i].label);
expect(json_data[i].label).toEqual('abc');
}
});
});
});
I can see response message nice parsed in console.log, but I'm not able to test anything, my test results are:
Finished in 0.019 seconds
1 test, 0 assertions, 0 failures
label: Text1
label: Text2
[launcher] 0 instance(s) of WebDriver still running
[launcher] chrome #1 passed
Process finished with exit code 0
The console log is write down after finish of test and no assertion has been done.
Any help please, how to test those responses (in JSON format) from server in Protractor?
For async tests, you'll need to pass the done callback to your function. Then call done() on success or done.fail() on failure. See Jasmine's Asynchronous support documentation.
it('Test case', function(done){
httpGet("http://localhost:3333/path/1/10").then((result) => {
var json_data = JSON.parse(result.bodyString);
for (var i = 0; i < json_data.length; ++i) {
console.log("label: " + json_data[i].label);
}
done();
}).catch(err => {
done.fail();
});
});