how to get WSendpoint of a puppeteer-cluster browser - puppeteer

in a project that requires:
Starting each session with logging in credentials + notification/OTP
work with multiple accounts asynchronously
remote debugging or monitoring of the session
5+ different operations can be requested on a open session , in any order
I want it to be a puppeteer cluster browser with one remote debugging port to monitor it
but couldn't integrate WsEndpoints
const puppeteer = require('puppeteer-extra');
const { Cluster } = require('puppeteer-cluster');
class SingleBrowserCluster {
browserInstance
options
constructor() {
if (SingleBrowserCluster._instance) {
//throw new Error("Singleton classes can't be instantiated more than once.")
}
else{
SingleBrowserCluster._instance = this;
// ... Your rest of the constructor code goes after this
console.log("pre optArgs");
const optArgs = [
'--remote-debugging-port=3002',//works if dockerised
'--remote-debugging-address=0.0.0.0',// at localhost.3002
'--window-size=1920,1080',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-gpu', '--no-zygote', //'--single-process',
];
console.log("pre options");
this.options = {
headless: true,//for dockerization
args: optArgs,
defaultViewport: null,
waitUntil: 'networkidle2'
};
console.log("Do launch now");
return this;
}
}
async screenshotMethod({ page, data: url }) {
await page.goto(url);
console.log(`%c worker X is running on ${url} `, `color:green;`);
console.log("will wait 20 second");
await page.waitForTimeout(20000)
const path = url.replace(/[^a-zA-Z]/g, '_') + '.png';
await page.screenshot({ path });
};
async launchCluster (){
try {
this.browserInstance = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 3,
puppeteerOptions: this.options
})
console.log(this.browserInstance);
return this.browserInstance;
} catch (error) {
console.log(`%c ERRORR`,`color:red;`);
console.log(error);
}
}
}
const StartScraper = async (Url, useProxy) => new Promise((resolve, reject) => {
(async () => {
// get proxy url from environment files
const proxyUrl = process.env.PROXY_URL;
//--disable-dev-shm-usage
// By default, Docker runs a container with a /dev/shm shared memory space 64MB. This is typically too small for Chrome and will cause Chrome to crash when rendering large pages.
//his will write shared memory files into /tmp instead of /dev/shm. See crbug.com/736452 for more details.
var instanceOne1= new SingleBrowserCluster()//.launchCluster()
var browser= await instanceOne1.launchCluster();
browser.queue('https://www.google.com/', instanceOne1.screenshotMethod);
//THE PROBLEM LINE
const wsEndpoint = browser.wsEndpoint();
try {
const page = (await browser.pages())[0];
await page.goto(Url, { waitUntil: 'load' });
return resolve(wsEndpoint);
} catch (e) {
browser.close();
return reject(false);
}
})();
});
how can i have WSendpoint of any session in a puppeteer-cluster
( more info:
i will put those in a session file
to provide my next selected consequtive operation a connection point on its session
localhost/StartScraper creates WSendpoint
localhost/login==WSendpoint==>Connects to existing session do login stuff
localhost/listItems==WSendpoint==>Connects to existing session do listItems stuff
...
)

Related

Correct way to pass args in puppeteer-cluster via puppeteerOptions

I am trying to use args in my code to use a proxy service I have. If I remove the args altogether things run fine but if I have them in there I get an error stating: Error: Unable to restart chrome. I checked multiple examples and copied the same to my code but it seems to fail. Any ideas on how to implement this correctly?
Code:
const { Cluster } = require('puppeteer-cluster');
const vanillaPuppeteer = require('puppeteer');
const { addExtra } = require('puppeteer-extra');
const Stealth = require('puppeteer-extra-plugin-stealth')
async function main() {
// Create a custom puppeteer-extra instance using `addExtra`,
// so we could create additional ones with different plugin config.
const puppeteer = addExtra(vanillaPuppeteer)
puppeteer.use(Stealth())
let proxy_server = 'proxy.soax.com:9000';
let user = 'some_user_name';
let pass = 'some_password';
// Launch cluster with puppeteer-extra
const cluster = await Cluster.launch({
puppeteer,
puppeteerOptions: {
headless: false,
args: ['--proxy-server=' + proxy_server,
'--single-process',
'--no-zygote',
'--no-sandbox'],
sameDomainDelay: 1000,
retryDelay: 3000,
workerCreationDelay: 3000},
maxConcurrency: 2,
concurrency: Cluster.CONCURRENCY_CONTEXT,
monitor: false,
skipDuplicateUrls: true
})
// Define task handler
await cluster.task(async ({ page, data: url }) => {
await page.authenticate({
username: user,
password: pass,
});
await page.goto(url)
const { hostname } = new URL(url)
console.log(`checking on ${hostname}`)
await page.screenshot({ path: `${hostname}.png`, fullPage: true })
})
// Queue any number of tasks
cluster.queue('https://whatismyipaddress.com/')
await cluster.idle()
await cluster.close()
console.log(`All done`)
}
main().catch(console.warn)
I played around a bit and discovered by removing the arg --single-process then it works fine.

cannot connect client app with web3js to metamask

I am a dapp beginner. This is a demo app for a todolist
I am unable to get a connection to the blockchain in web3js. Websocket connection error
localhost is localhost:8545
using web3js CDN : https://cdn.jsdelivr.net/npm/web3#latest/dist/web3.min.js
this is my app.js
App = {
loading: false,
contracts: {},
load: async () => {
console.log('app loading ...')
console.log(web3);
await App.loadWeb3()
await App.loadAccount()
// await App.loadContract()
// await App.render()
},
// https://medium.com/metamask/https-medium-com-metamask-breaking-change-injecting-web3-7722797916a8
loadWeb3: async () => {
let web3 = new Web3('ws://localhost:8545');
if (typeof web3 !== 'undefined') {
App.web3Provider = web3.currentProvider
web3.setProvider('ws://localhost:8546');
web3.eth.getAccounts().then(console.log);
} else {
window.alert("Please connect to Metamask.")
}
// Modern dapp browsers...
if (window.ethereum) {
window.web3 = new Web3(ethereum)
try {
// Request account access if needed
await ethereum.enable()
// Acccounts now exposed
web3.eth.sendTransaction({/* ... */})
console.log('MetaMask is installed!');
} catch (error) {
// User denied account access...
}
}
// Legacy dapp browsers...
else if (window.web3) {
App.web3Provider = web3.currentProvider
window.web3 = new Web3(web3.currentProvider)
// Acccounts always exposed
web3.eth.sendTransaction({/* ... */})
}
// Non-dapp browsers...
else {
console.log('Non-Ethereum browser detected. You should consider trying MetaMask!')
}
},
loadAccount: async () => {
// Set the current blockchain account
App.account = web3.eth.accounts[0]
console.log(App.account)
// web3 set up by loadWeb3, includes all accounts, loading first one via MetaMask
},
loadContract: async () => {
// Create a JavaScript version of the smart contract
const todoList = await $.getJSON('TodoList.json')
App.contracts.TodoList = TruffleContract(todoList)
App.contracts.TodoList.setProvider(App.web3Provider)
// Hydrate the smart contract with values from the blockchain
App.todoList = await App.contracts.TodoList.deployed()
},
render: async () => {
// Prevent double render
if (App.loading) {
return
}
// Update app loading state
App.setLoading(true)
// Render Account
$('#account').html(App.account)
// Render Tasks
await App.renderTasks()
// Update loading state
App.setLoading(false)
},
renderTasks: async () => {
// Load the total task count from the blockchain
const taskCount = await App.todoList.taskCount()
const $taskTemplate = $('.taskTemplate')
// Render out each task with a new task template
for (var i = 1; i <= taskCount; i++) {
// Fetch the task data from the blockchain
const task = await App.todoList.tasks(i)
const taskId = task[0].toNumber()
const taskContent = task[1]
const taskCompleted = task[2]
// Create the html for the task
const $newTaskTemplate = $taskTemplate.clone()
$newTaskTemplate.find('.content').html(taskContent)
$newTaskTemplate.find('input')
.prop('name', taskId)
.prop('checked', taskCompleted)
.on('click', App.toggleCompleted)
// Put the task in the correct list
if (taskCompleted) {
$('#completedTaskList').append($newTaskTemplate)
} else {
$('#taskList').append($newTaskTemplate)
}
// Show the task
$newTaskTemplate.show()
}
},
createTask: async () => {
App.setLoading(true)
const content = $('#newTask').val()
await App.todoList.createTask(content)
window.location.reload()
},
toggleCompleted: async (e) => {
App.setLoading(true)
const taskId = e.target.name
await App.todoList.toggleCompleted(taskId)
window.location.reload()
},
setLoading: (boolean) => {
App.loading = boolean
const loader = $('#loader')
const content = $('#content')
if (boolean) {
loader.show()
content.hide()
} else {
loader.hide()
content.show()
}
}
}
$(() => {
$(window).load(() => {
App.load()
})
})
I get this error in the console :
Again I am a total newbie, any help is appreciated.
Any source of info did not help
Hey after metamask update , it no longer injects web3 .
You can check the blog below , It has shown how to connect metamask with our project .
https://dapp-world.com/blogs/01/how-to-connect-metamask-with-dapp--1616927367052
Only thing is its web application , you can relate it with your project .
Hope it works !

Want to measure the response time of web page using puppeteer script

I have a webpage and automated through puppeteer script as follow, I want to measure the response time and other metrics like what is the time taken to load the dom content, time for the first byte, etc.
const puppeteer = require('puppeteer');
const myURL = "http://localhost:8080/#/login";
const Username = "jmallick";
const BlockName = "TEST99";
const Password = "Test1234";
async function loginHandler (){
return new Promise (async (resolve, reject) =>{
let browserLaunch = await puppeteer.launch({headless : false,
args: ['--window-size=1920,1040']});
try{
const pageRender = await browserLaunch.newPage();
await pageRender.goto(myURL, {waitUntil: 'networkidle0'});
pageRender.setViewport() --> didn't worked
<want to measure the details here>
await pageRender.type('#txtUserName',Username);
await pageRender.type('#txtPassword',Password);
await pageRender.type('#txtBlockName',FirmName);
await Promise.all([
pageRender.click('#fw-login-btn'),
pageRender.waitForNavigation({ waitUntil: 'networkidle0' }),
pageRender.setViewport() --> didn't worked
<want to measure the details here>
]);
const homePage = await pageRender.$('[href="#/home"]');
await homePage.click({waitUntil: 'networkidle0'});
<want to measure the details here>
}
catch (e){
return reject(e);
}
finally{
browserLaunch.close();
}
});
}
loginHandler().then(console.log).catch(console.error);
Tried with pageRender.setViewport() but didn't work. Checked other pages but couldn't find the same match.

Firebase functions RangeError: Maximum call stack size exceeded

I have a Callable function that uploads an image and update Firestore and Storage accordingly. The function does what it should do. but I still get this error:
Unhandled error RangeError: Maximum call stack size exceeded
here is the function:
export const uploadImageToStripe = functions.https.onCall(async (data, context) => {
let businessDoc: DocumentSnapshot
try {
if (!fireStoreDB) {
fireStoreDB = admin.firestore();
fireStoreDB.settings(settings);
}
businessDoc = await fireStoreDB.collection('businesses').doc(data.business_id).get()
const bucketName = functions.config().storage.default_bucket;
const tempLocalFile = path.join(os.tmpdir(), 'img.jpg').trim();
const tempLocalDir = path.dirname(tempLocalFile);
const bucket = admin.storage().bucket(bucketName);
// Create the temp directory where the storage file will be downloaded.
await mkdirp(tempLocalDir);
console.log('Temporary directory has been created', tempLocalDir);
// Download file from bucket.
await bucket.file(data.photo_location).download({ destination: tempLocalFile });
console.log('The file has been downloaded to', tempLocalFile);
// Downloads the file
console.log(`gs://${bucketName}/${data.photo_location} downloaded to ${tempLocalDir}.`)
const uploadedFile: stripeM.fileUploads.IFileUpdate = await stripe.fileUploads.create({
file: {
data: fs.readFileSync(tempLocalFile),
name: 'img.jpg',
type: 'application.octet-stream',
}
});
if (!businessDoc.exists) {
throw new functions.https.HttpsError('not-found', `Couldn't find business document ` + data.business_id);
}
await stripe.accounts.update(businessDoc.data().stripeId,
{ document: uploadedFile.id });
await businessDoc.ref.update({ "photoNeeded": false })
return await bucket.file(data.photo_location).delete()
} catch (error) {
console.error(error);
await businessDoc.ref.update({ "photoNeeded": true })
throw new functions.https.HttpsError('unavailable', `failed to upload photo to stripe`);
}
})
Any ideas why I get this error?
This line throw the error:
return await bucket.file(data.photo_location).delete()
splitting it to:
await bucket.file(data.photo_location).delete()
return "Success"
solve it.

Puppeteer can't catch failing request & errors

I trying to collect data from failing requests and js error.
I'm using the following site: https://nitzani1.wixsite.com/marketing-automation/3rd-page
The site has a request to https://api.fixer.io/1latest, which returns a status code of 404,
also the page contains thw following js error:
"Uncaught (in promise) Fetch did not succeed"
I've tried to code bellow to catch the 404 and js error but couldn't.
Not sure what I'm doing wrong, any idea as to how to solve it?
const puppeteer = require('puppeteer');
function wait (ms) {
return new Promise(resolve => setTimeout(() => resolve(), ms));
}
var run = async () => {
const browser = await puppeteer.launch({
headless: false,
args: ['--start-fullscreen']
});
page = await browser.newPage();
page.on('error', err=> {
console.log('err: '+err);
});
page.on('pageerror', pageerr=> {
console.log('pageerr: '+pageerr);
});
page.on('requestfailed', err => console.log('requestfailed: '+err));
collectResponse = [];
await page.on('requestfailed', rf => {
console.log('rf: '+rf);
});
await page.on('response', response => {
const url = response.url();
response.buffer().then(
b => {
// console.log(url+' : '+response.status())
},
e => {
console.log('response err');
}
);
});
await wait(500);
await page.setViewport({ width: 1920, height: 1080 });
await page.goto('https://nitzani1.wixsite.com/marketing-automation/3rd-page', {
});
};
run();
The complete worked answer is:
const puppeteer = require('puppeteer');
const run = async () => {
const browser = await puppeteer.launch({
headless: true
});
const page = await browser.newPage();
// Catch all failed requests like 4xx..5xx status codes
page.on('requestfailed', request => {
console.log(`url: ${request.url()}, errText: ${request.failure().errorText}, method: ${request.method()}`)
});
// Catch console log errors
page.on("pageerror", err => {
console.log(`Page error: ${err.toString()}`);
});
// Catch all console messages
page.on('console', msg => {
console.log('Logger:', msg.type());
console.log('Logger:', msg.text());
console.log('Logger:', msg.location());
});
await page.setViewport({ width: 1920, height: 1080 });
await page.goto('https://nitzani1.wixsite.com/marketing-automation/3rd-page', { waitUntil: 'domcontentloaded' });
await page.waitFor(10000); // To be sure all exceptions logged and handled
await browser.close();
};
run();
Save in .js file and easily run it.
Current puppeteer 8.0.0^ have a very small amount of information in message.text(). So we need to get a description of the error from JSHandle.
Please check this comment with fully descriptive console errors from JSHandle object
Check the link here https://stackoverflow.com/a/66801550/9026103