Puppeteer Google Cloud Function Pub/Sub Trigger can't open browser - google-cloud-functions

I'm trying to create a Puppeteer function in GCP which can be triggered by Pub/Sub messages. The function is callable, but doesn't behave as expected and throws a Timeout Error once browser tries to initialize. Could the trigger possibly be using a NodeJS environment different from HTTP trigger?
I'm also very new to NodeJS, so I apologize ahead of time if the issue is blatantly obvious.
I've created an HTTP trigger for the function which behaves as expected. I copy/paste the Puppeteer Function below into the index.js when creating the Cloud Function, but separated in example for clarity that both triggers are running the identical function.
Puppeteer Function
const puppeteer = require('puppeteer');
scrapeUglyWebsite = () => {
return new Promise(async(resolve, reject) => {
await puppeteer.launch({
headless: true,
args: ['--no-sandbox']
})
.then(async (browser) => {
const page = await browser.newPage();
await page.goto('http://suzannecollinsbooks.com/', {waitUntil: 'load', timeout: 0})
.then(async () => {
//Wait for content to load
await page.waitForFunction('document.body !== null && document.body.innerText.includes(\'Jon Scieszka\')');
//Evaluate page contents
const dom_eval = await page.evaluate(() => document.body.innerText.includes("Here’s a picture of me with a rat"));
await browser.close();
resolve(dom_eval);
});
}).catch((err) => {
reject(err);
});
});
};
HTTP Trigger - index.js
exports.cloudFunctionTest = (req, res) => {
scrapeUglyWebsite()
.then((results) => {
if(results) {
res.send('Suzanne Collins takes pictures with rats.');
} else {
res.send("Suzzane Collins doesn't take pictures with rats.");
};
})
.catch((err) => {
res.send(err.toString());
});
Pub/Sub Trgger - index.js
exports.cloudFunctionTest = (data, context) => {
scrapeUglyWebsite()
.then((results) => {
if(results) {
console.log('Suzanne Collins takes pictures with rats.');
} else {
console.log("Suzzane Collins doesn't take pictures with rats.");
};
})
.catch((err) => {
console.log(err.toString());
});
};
package.json
{
"name": "test",
"version": "0.0.1",
"engines": {
"node": "8"
},
"dependencies": {
"puppeteer": "^1.6.0"
}
}
HTTP Trigger behaves correctly with the expected result
Suzanne Collins takes pictures with rats.
Pub/Sub Trigger throws the following error with no output
TimeoutError: Timed out after 30000 ms while trying to connect to Chrome! The only Chrome revision guaranteed to work is r662092

I know this is late but the reason that the TimeoutError occurs is because cloud functions do not automatically wait for async tasks to finish completing. So in exports.cloudFunctionTest, scrapeUglyWebsite() is called but the function does not wait for the promise to be fulfilled, so the program terminates. Hence the error
More info here on how background functions work in NodeJs
In order for the function to wait for scrapeUglyWebsite(), you need to return a promise that completes when scrapeUglyWebsite() and the resulting code is complete.
Personally, I got it to work by simply wrapping the code currently in the function I am exporting in another async function and then returning the promise of the wrapper function.
async function wrapper() {
try {
const result = await scrapeUglyWebsite();
if(results) {
console.log('Suzanne Collins takes pictures with rats.');
} else {
console.log("Suzzane Collins doesn't take pictures with rats.");
};
} catch (err) {
console.log(err.toString());
}
}
Then in the function you want to export:
exports.cloudFunctionTest = (data, context) => {
return wrapper();
};

Related

trying to deploy a todolist in blockchain, when adding a new task, I get invalid address error

I am making a simple block chain using the ETH blockchain technology.
I am making a simple todolist, following this tutorial.
My todolist works fine and I can see the tasks in there, how ever when I try to add a new task I get this following error:
Uncaught (in promise) Error: invalid address
v http://localhost:3000/js/web3.min.js:2
l http://localhost:3000/js/web3.min.js:2
formatInput http://localhost:3000/js/web3.min.js:2
formatInput http://localhost:3000/js/web3.min.js:2
toPayload http://localhost:3000/js/web3.min.js:2
e http://localhost:3000/js/web3.min.js:2
sendTransaction http://localhost:3000/js/web3.min.js:2
execute http://localhost:3000/js/web3.min.js:2
synchronizeFunction http://localhost:3000/vendor/truffle-contract/dist/truffle-contract.js:206
synchronizeFunction http://localhost:3000/vendor/truffle-contract/dist/truffle-contract.js:157
promise callback*synchronizeFunction/< http://localhost:3000/vendor/truffle-contract/dist/truffle-contract.js:156
createTask http://localhost:3000/js/app.js:124
onsubmit http://localhost:3000/:1
web3.min.js:2:4288
createTask http://localhost:3000/js/app.js:125
AsyncFunctionThrow self-hosted:696
(Async: async)
onsubmit http://localhost:3000/:1
​
this is my app.js
App = {
loading: false,
contracts: {},
load: async () => {
await App.loadWeb3();
await App.loadAccount();
await App.loadContract();
await App.render();
},
// https://medium.com/metamask/https-medium-com-metamask-breaking-change-injecting-web3-7722797916a8
loadWeb3: async () => {
if (typeof web3 !== "undefined") {
App.web3Provider = web3.currentProvider;
web3 = new Web3(web3.currentProvider);
} else {
window.alert("Please connect to Metamask.");
}
// Modern dapp browsers...
if (window.ethereum) {
window.web3 = new Web3(ethereum);
try {
// Request account access if needed
await ethereum.enable();
// Acccounts now exposed
web3.eth.sendTransaction({
/* ... */
});
} catch (error) {
// User denied account access...
}
}
// Legacy dapp browsers...
else if (window.web3) {
App.web3Provider = web3.currentProvider;
window.web3 = new Web3(web3.currentProvider);
// Acccounts always exposed
web3.eth.sendTransaction({
/* ... */
});
}
// Non-dapp browsers...
else {
console.log(
"Non-Ethereum browser detected. You should consider trying MetaMask!"
);
}
},
loadAccount: async () => {
// Set the current blockchain account
App.account = web3.eth.accounts[0];
console.log(App.account);
},
loadContract: async () => {
// Create a JavaScript version of the smart contract
const todoList = await $.getJSON("TodoList.json");
App.contracts.TodoList = TruffleContract(todoList);
App.contracts.TodoList.setProvider(App.web3Provider);
// Hydrate the smart contract with values from the blockchain
App.todoList = await App.contracts.TodoList.deployed();
// console.log(todoList);
},
render: async () => {
// Prevent double render
if (App.loading) {
return;
}
// Update app loading state
App.setLoading(true);
// Render Account
$("#account").html(App.account);
// Render Tasks
await App.renderTasks();
// Update loading state
App.setLoading(false);
},
renderTasks: async () => {
// Load the total task count from the blockchain
const taskCount = await App.todoList.taskCount();
const $taskTemplate = $(".taskTemplate");
// Render out each task with a new task template
for (var i = 1; i <= taskCount; i++) {
// Fetch the task data from the blockchain
const task = await App.todoList.tasks(i);
const taskId = task[0].toNumber();
const taskContent = task[1];
const taskCompleted = task[2];
// Create the html for the task
const $newTaskTemplate = $taskTemplate.clone();
$newTaskTemplate.find(".content").html(taskContent);
$newTaskTemplate
.find("input")
.prop("name", taskId)
.prop("checked", taskCompleted)
.on("click", App.toggleCompleted);
// Put the task in the correct list
if (taskCompleted) {
$("#completedTaskList").append($newTaskTemplate);
} else {
$("#taskList").append($newTaskTemplate);
}
// Show the task
$newTaskTemplate.show();
}
},
createTask: async () => {
App.setLoading(true);
const content = $("#newTask").val();
await App.todoList.createTask(content);
window.location.reload();
},
toggleCompleted: async (e) => {
App.setLoading(true);
const taskId = e.target.name;
await App.todoList.toggleCompleted(taskId);
window.location.reload();
},
setLoading: (boolean) => {
App.loading = boolean;
const loader = $("#loader");
const content = $("#content");
if (boolean) {
loader.show();
content.hide();
} else {
loader.hide();
content.show();
}
},
};
$(() => {
$(window).load(() => {
App.load();
});
});
in console it shows me the address to be: 0xc5cfa0a0345f74e26cecfd8ec3a5cfa3843955ac
I am using metamask and genache, and I tried to connect my smart contacrt to my memask wallet, so I know I am connected, but I am not sure why I am getting this error.
SHould I delete my metamask and do it again?
I tried to look for this solution here, but the solutions are mostly old and doesnt make scnese what I need to do.
Help would be really apperciated.
fixed it, in the app.js you have to replace this line:
App.account = web3.eth.accounts[0];
with the following line:
web3.eth.defaultAccount=web3.eth.accounts[0]

MySQL querying in NodeJS

I'm trying to write an app that will check whether or not an webpage has changed.
I'm using NodeJs mysql because I'm familiar with them.
So at the moment, I have the problem that my query is too 'slow', so my function will not return true, even though it should. Can I force my app to wait for the query somehow?
Edit:
checkEntry(webpage, callback) {
var oldPage;
this.mysql.query('SELECT OldWebpagecol FROM Web_Scraping.OldWebpage WHERE idOldWebpage = 15', (err, rows) => {
if (err) console.log(err);
oldPage = rows[0].OldWebpagecol;
if (webpage === oldPage) {
return true;
}
return false;
})
}
You can use async/await when you make a request.
For example,
(async () => {
let response = await fetch(‘/api/users’);
})();

get real-time json data from twilio runtime with axios

I am trying to achieve real-time data from twilio server-less function. I am using a boilerplate function edited a little bit.What I want is json data in server and voice response in call consecutively .but the following code is not sending json data to server.
const axios = require('axios');
exports.handler = function (context, event, callback) {
let twiml = new Twilio.twiml.VoiceResponse();
twiml.say('you are welcome ');
const instance = axios.create({
baseURL: 'http://fafc4eac4162.ngrok.io/',
timeout: 3000,
});
instance
.post('/test', {
id: 1,
title: 'Twilio'
})
.then((response) => {
console.log(JSON.stringify(response.data));
})
.catch((error) => {
console.log(error);
return callback(error);
});
return callback(null, twiml);
};
It shows below error,but it sends data successfully if I do not use the voice response callback return callback(null, twiml) and rather use simple return callback(null, response.data);
{"message":"timeout of 3000ms exceeded","name":"Error","stack":"Error: timeout of 3000ms
exceeded\n at createError (/var/task/node_modules/axios/lib/core/createError.js:16:15)\n
at RedirectableRequest.handleRequestTimeout
(/var/task/node_modules/axios/lib/adapters/http.js:280:16)\n at Object.onceWrapper
(events.js:286:20)\n at RedirectableRequest.emit (events.js:198:13)\n at
Timeout._onTimeout (/var/task/node_modules/follow-redirects/index.js:166:13)\n at
ontimeout (timers.j...
The return callback(null, twiml); should be in the .then block.
.then((response) => {
console.log(JSON.stringify(response.data));
return callback(null, twiml);
})
Also, the error indicates the 3000ms timeout is hit, is your application returning a 200-OK?

How do i subscribe live stream video in agora?

I am setting up agora SDK to my angular project and am getting the following error.
Code:
This is my sample code and is calling the startCall method in ngOnInit. I have a div element with id.
startCall() {
this.agoraService.client.join(null, '1000', null, (uid) => {
this.localStream = this.agoraService.createStream(uid, true, null, null, true, false);
this.localStream.setVideoProfile('720p_3');
this.subscribeToStreams();
});
}
private subscribeToStreams() {
this.localStream.on("accessAllowed", () => {
console.log("accessAllowed");
});
// The user has denied access to the camera and mic.
this.localStream.on("accessDenied", () => {
console.log("accessDenied");
});
this.localStream.init(() => {
console.log("getUserMedia successfully");
this.localStream.play('agora_local');
this.agoraService.client.publish(this.localStream, function (err) {
console.log("Publish local stream error: " + err);
});
this.agoraService.client.on('stream-published', function (evt) {
console.log("Publish local stream successfully");
});
}, function (err) {
console.log("getUserMedia failed", err);
});
// Add
this.agoraService.client.on('error', (err) => {
console.log("Got error msg:", err.reason);
if (err.reason === 'DYNAMIC_KEY_TIMEOUT') {
this.agoraService.client.renewChannelKey("", () => {
console.log("Renew channel key successfully");
}, (err) => {
console.log("Renew channel key failed: ", err);
});
}
});
// Add
this.agoraService.client.on('stream-added', (evt) => {
const stream = evt.stream;
this.agoraService.client.subscribe(stream, (err) => {
console.log("Subscribe stream failed", err);
});
});
// Add
this.agoraService.client.on('stream-subscribed', (evt) => {
const stream = evt.stream;
if (!this.remoteCalls.includes(`agora_remote${stream.getId()}`)) this.remoteCalls.push(`agora_remote${stream.getId()}`);
setTimeout(() => stream.play(`agora_remote${stream.getId()}`), 2000);
});
// Add
this.agoraService.client.on('stream-removed', (evt) => {
const stream = evt.stream;
stream.stop();
this.remoteCalls = this.remoteCalls.filter(call => call !== `#agora_remote${stream.getId()}`);
console.log(`Remote stream is removed ${stream.getId()}`);
});
// Add
this.agoraService.client.on('peer-leave', (evt) => {
const stream = evt.stream;
if (stream) {
stream.stop();
this.remoteCalls = this.remoteCalls.filter(call => call === `#agora_remote${stream.getId()}`);
console.log(`${evt.uid} left from this channel`);
}
});
}
I have a div element with id.
Uncaught (in promise) TypeError: Failed to execute 'getStats' on 'RTCPeerConnection': The callback provided as parameter 1 is not a function.
at Object.C.t.getStats (AgoraRTCSDK.min.js:2)
at AgoraRTCSDK.min.js:2
at ZoneDelegate.push../node_modules/zone.js/dist/zone.js.ZoneDelegate.invokeTask (zone.js:423)
at Zone.push../node_modules/zone.js/dist/zone.js.Zone.runTask (zone.js:195)
at push../node_modules/zone.js/dist/zone.js.ZoneTask.invokeTask (zone.js:498)
at ZoneTask.invoke (zone.js:487)
at timer (zone.js:2281)
Does anyone face the same issue? Can anyone help me with this?
Thanks.
I have followed this link
https://docs.agora.io/en/Interactive%20Broadcast/web_prepare?platform=Web
Steps I have done,enter code here
1. Import the Agora Web SDK to Your Project
2. Create and Initialize a Client
3. Join a Channel
4. And finally, Subscribe to the remote stream
When testing Agora's WebSDK (or any WebRTC application) make sure that you are using an https connection when trying to run your code as most browsers do not allow getMedia access without an https connection.
There are a number of solutions for using https connections on your local machine. I use the ngrok tool to easily run https connections on my local machine

What is the proper way to capture a HTTP response with Puppeteer?

I am trying to capture the http response status from a user sign-up.
My code looks like this:
it.only('returns a 400 response if email is taken', async () => {
await page.goto(`${process.env.DOMAIN}/sign-up`)
await page.waitFor('input[id="Full Name"]')
await page.type('input[id="Full Name"]', 'Luke Skywalker')
await page.type('input[id="Email"]', 'LukeSkywalker#voyage.com')
await page.type('input[id="Password"]', 'LukeSkywalker123', {delay: 100})
await page.click('input[type="submit"]', {delay: 1000})
const response = await page.on('response', response => response)
console.log('request status', response.status)
// expect(response).toEqual(400)
})
The docs give an example of intercepting the request and doing things with it:
await page.setRequestInterception(true);
page.on('request', request => {
request.respond({
status: 404,
contentType: 'text/plain',
body: 'Not Found!'
});
});
And I have tried a similar pattern to no avail, along with many other patterns. Everything I do returns the page, a huge object with no status on it that I can see. Any help is much appreciated.
WHAT WORKED:
thank you to #tomahaug for steering me in the correct direction. My first problem was placement, the listener needs to go be set up before the request is made, I had it just after the request. Makes sense. My biggest issue was assigning the listener to a variable, so that I could call the expect as my last line. Assigning it to a variable caused the page to be returned. What I needed to do was just run the test inside the listener. While using done() throws and error for me I closed off my test as follows below, the working version of my code:
it.only('returns a 400 response if email is taken', async () => {
await page.goto(`${process.env.DOMAIN}/sign-up`)
await page.waitFor('input[id="Full Name"]')
await page.type('input[id="Full Name"]', 'Luke Skywalker')
await page.type('input[id="Email"]', 'LukeSkywalker#voyage1.com')
await page.type('input[id="Password"]', 'LukeSkywalker123', {delay: 100})
await page.on('response', response => {
if (response.request().method === 'POST' && response.url === `${process.env.USERS_API_DOMAIN}/sessions`) {
expect(response.status).toEqual(400)
}
})
await page.click('input[type="submit"]', {delay: 1000})
})
after(async function () {
await browser.close()
})
Hope this helps someone else!
I believe you should do something along those lines. Note the callback function done.
What the code does, is that it attaches a listener for responses, then clicks the submit button. When a response is received it checks the status code, asserts it, and terminates the test by calling done.
You might want to have an if-statement that checks that it is the actual response from your form that you are checking in the callback, as the response handler might emit events for other concurrent requests.
it.only('returns a 400 response if email is taken', () => {
await page.goto(`${process.env.DOMAIN}/sign-up`)
await page.waitFor('input[id="Full Name"]')
await page.type('input[id="Full Name"]', 'Luke Skywalker')
await page.type('input[id="Email"]', 'LukeSkywalker#voyage.com')
await page.type('input[id="Password"]', 'LukeSkywalker123', {delay: 100})
page.on('response', (response) => {
if (
response.request().method === 'POST' &&
response.url === `${process.env.USERS_API_DOMAIN}/sessions`)
{
expect(response.status).toEqual(400)
}
})
await page.click('input[type="submit"]', {delay: 1000})
})
I have not tested the code, but it should give you the right idea.
Edit: Adjusted to reflect what worked out in the end.
If you need to manipulate the request/response, use page.setRequestInterception(true) and page.on/page.once (as documented).
However, if all you need is to assert something about the response, the simplest and most idiomatic way to do so is with page.waitForResponse:
const updateDashboardResponse = await page.waitForResponse(response =>
response.url().includes('updateDashboard')
);
expect(updateDashboardResponse.status()).toBe(200);
This allows test flow to remain linear and avoids ambiguity around closing a test before a page.on handler receives a response event.
The accepted answer (which was also edited into the question) is incorrect. It introduces a race condition due to a 1 second delay added to the click call. At best, this slows down the test suite unnecessarily, and at worst it generates false failures should the request take longer than a second to resolve (unlikely if it's mocked, but it doesn't change the fact that the code is unsafe).
Whenever there's a callback in a Jest test case, the correct way to ensure it's been executed and all assertions depending on it firing have been made without adding artificial delays is to call done() from the callback. If there is a throw in the callback that makes done unreachable, call done(error) in the error handler to report the test case failure to Jest.
To do this, you'll need to add done as the parameter to the callback passed to the it, test or only function so that it's available in the block. This allows Jest's test runner to treat the test as asynchronous and not to resolve it until done is called. Without done, the test suite ignores the callback's assertions. async/await doesn't help because it's a separate asynchronous chain than the callback.
You only need to specify done as a parameter or return a promise (async implicitly returns a promise), never both. However, you'd still likely want to use await for Puppeteer library calls rather than then. You can use an async IIFE that eventually fires the done() call when all assertions have fired to get the best of both worlds.
For example,
it.only('returns a 400 response if email is taken', done => {
(async () => {
page.on('response', response => {
if (response.request().method === 'POST' &&
response.url === `${process.env.USERS_API_DOMAIN}/sessions`) {
try { /* try-catch pattern shown for illustration */
expect(response.status).toEqual(400);
done();
}
catch (err) {
done(err);
}
}
});
await page.goto(`${process.env.DOMAIN}/sign-up`);
await page.waitFor('input[id="Full Name"]');
await page.type('input[id="Full Name"]', 'Luke Skywalker');
await page.type('input[id="Email"]', 'LukeSkywalker#voyage.com');
await page.type('input[id="Password"]', 'LukeSkywalker123', {delay: 100});
await page.click('input[type="submit"]');
})();
});
With this in mind, this answer shows a likely better approach using waitForResponse which lets you skip the callback and done entirely. The callback to waitForResponse is a string URL or function predicate that should return true for the target response that's being waited on:
it.only('returns a 400 response if email is taken', async () => {
await page.goto(`${process.env.DOMAIN}/sign-up`);
await page.waitFor('input[id="Full Name"]');
await page.type('input[id="Full Name"]', 'Luke Skywalker');
await page.type('input[id="Email"]', 'LukeSkywalker#voyage.com');
await page.type('input[id="Password"]', 'LukeSkywalker123', {delay: 100});
await page.click('input[type="submit"]');
const response = await page.waitForResponse(response =>
response.request().method === 'POST' &&
response.url === `${process.env.USERS_API_DOMAIN}/sessions`
);
expect(response.status).toEqual(400);
});
I should also mention waitFor is deprecated in favor of waitForSelector in the above snippets and that .url and .method are functions. I haven't verified the above code; it's there to relate to the original post and show the high-level patterns.
Minimal example
index.html
This is the web page we're testing.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
</head>
<body>
<button>Post</button>
<script>
document
.querySelector("button")
.addEventListener("click", e =>
fetch("https://jsonplaceholder.typicode.com/posts", {
method: "POST",
body: JSON.stringify({
title: "foo",
body: "bar",
userId: 1,
}),
headers: {
"Content-type": "application/json; charset=UTF-8",
},
})
.then(response => response.json())
.then(json => console.log(json))
)
;
</script>
</body>
</html>
index.test.js (async/await version):
describe("index page", () => {
it("should respond to POST", async () => {
const url = "https://jsonplaceholder.typicode.com/posts";
await page.goto("http://localhost:1234", {waitUntil: "load"});
await page.click("button");
const response = await page.waitForResponse(response =>
response.request().method() === "POST" &&
response.url() === url
);
const expectedBody = {
body: "bar",
id: 101,
title: "foo",
userId: 1,
};
expect(await response.json()).toEqual(expectedBody);
});
});
index.test.js (then version):
describe("index page", () => {
it("should respond to POST", () => {
const url = "https://jsonplaceholder.typicode.com/posts";
const expectedBody = {
body: "bar",
id: 101,
title: "foo",
userId: 1,
};
return page.goto("http://localhost:1234", {
waitUntil: "load"
})
.then(() => page.click("button"))
.then(() => page.waitForResponse(response =>
response.request().method() === "POST" &&
response.url() === url
))
.then(response => response.json())
.then(body => expect(body).toEqual(expectedBody))
;
});
});
index.test.js (done version):
describe("index page", () => {
it("should respond to POST", done => {
(async () => {
const url = "https://jsonplaceholder.typicode.com/posts";
const expectedBody = {
body: "bar",
id: 101,
title: "foo",
userId: 1,
};
await page.setRequestInterception(true);
page.on("response", async response => {
if (response.request().method() === "POST" &&
response.url() === url) {
try {
const body = await response.json();
expect(body).toEqual(expectedBody);
done();
}
catch (err) {
done(err);
}
}
});
await page.goto("http://localhost:1234", {
waitUntil: "load"
});
page.click("button");
})();
});
});
response.url is a function and you have to call it:
response.url()
The same is for response.request().method:
response.request().method()