const browser = await puppeteer.launch({args: ['--no-sandbox']});
const page = await browser.newPage();
await page.goto('$url', {waitUntil: 'load', timeout: 0});
const cookies = await page._client.send('Network.getAllCookies');
I want to check the if website is accessible with provided protocol.
If tried to access with http and failed, need to check whether its accessible with https
Is it possible to do with puppeteer.?
puppeteer will throw an error if it is not able to page.goto a page, so I'd suggest catching that error and then correcting the course:
try {
await page.goto('$url', {waitUntil: 'load', timeout: 0});
} catch(e) {
console.log(e.message);
}
Related
i use gologin service. gologin is a browser antidetect service where I can fake my browser identity / can manage browser fingerprint.
so I can freely do web-scraping without being detected.
in this case I want to be able to load my extension into that browser using the puppeteer.connect() method.
here's the code:
const puppeteer = require('puppeteer-core');
const GoLogin = require('gologin');
(async () => {
const GL = new GoLogin({
token: 'yU0token',
profile_id: 'yU0Pr0f1leiD',
});
const { status, wsUrl } = await GL.start();
const browser = await puppeteer.connect({
browserWSEndpoint: wsUrl.toString(),
ignoreHTTPSErrors: true,
});
const page = await browser.newPage();
await page.goto('https://myip.link/mini');
console.log(await page.content());
await browser.close();
await GL.stop();
})();
I don't know how. please help me, so i can load my extension using this puppeteer.connect()
Assume your wish is loading chrome-extension into your puppeteer browser.
Find chrome-extension Working Directory Where does Chrome store extensions?
Find your extension ID by go to chrome://extensions/
Sample code:
const puppeteer = require('puppeteer-core');
const MY_EXTENSION_PATH = '~/Library/Application Support/Google/Chrome/Default/Extensions/cdockenadnadldjbbgcallicgledbeoc/0.3.38_0'
async function loadExtension() {
return puppeteer.launch({
headless: 0,
args: [
`--disable-extensions-except=${MY_EXTENSION_PATH}`,
`--load-extension=${MY_EXTENSION_PATH}`,
],
});
}
Is there any way to avoid being detected by a website that I am using puppeteer? I just can't navigate around the https://www.footlocker.ca/ website using puppeteer. I have tried using stealth plugin and random user-agents to no avail.
Any advice on what else I can try?
This website use navigator.webdriver to check if you are real user or bot. so you can use the code below to delete navigator.webdriver value. docs.
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({
headless: false,
});
const page = await browser.newPage();
await page.evaluateOnNewDocument(() => {
delete navigator.__proto__.webdriver;
});
await page.goto("https://www.footlocker.ca", {
waitUntil: "domcontentloaded",
});
})();
Can someone explain why this code isn't working. I have a console log before I run page.evaluate() which logs what I expect, but the console log inside page.evaluate never runs.
const puppeteer = require('puppeteer');
(async () => {
try {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://www.example.com');
page.on('response', async response => {
const url = response.url();
if (url.includes('something')) {
console.log('this code runs');
await page.evaluate(() => {
console.log("this code doesn't run");
});
}
});
} catch (err) {
console.log(err);
}
})();
Console log doesn't work in page.evaluate()
https://github.com/GoogleChrome/puppeteer/issues/1944
Try to use this code for display console.log from evaluate
page.on('console', msg => {
for (let i = 0; i < msg.args().length; ++i)
console.log(`${i}: ${msg.args()[i]}`);
});
page.evaluate(() => console.log('hello', 5, {foo: 'bar'}));
https://pptr.dev/#?product=Puppeteer&version=v1.20.0&show=api-event-console
The code inside page.evaluate is run in the browser context, so the console.log works, but inside the Chrome console and not the Puppeteer one.
To display the logs of the Chrome context inside the Puppeteer console, you can set dumpio to true in the arguments when launching a browser using Puppeteer:
const browser = await puppeteer.launch({
dumpio: true
})
Console.log works but in the browser context. I'm guessing here that you are trying to see the log in the CLI. If you want to see the log set headless to false and then see the log in the browser console.
I'm trying to use headless Chrome (v66 on Win10, using C#) to take a series of screenshots from a 3D model in forge autodesk viewer.
The problem i'm facing is that once the model is loaded i set the camera into the first position, take a screenshot and then try to set the camera to the next position for the next screenshot. Once i try that (setting the camera position later then on the initial load), the webgl context is lost.
I have too little knowledge of WebGL / swiftshaders / etc, but what i find frustrating, is that when i position the camera directly after the load, it does work. (IE, the workaround is to spawn a seperate headless session per camera view, but since the loading of the geometry takes 20sec or more, thats not preferred)
So, this:
viewerApp.myCurrentViewer.addEventListener(Autodesk.Viewing.GEOMETRY_LOADED_EVENT,
function () {
_viewer = viewerApp.myCurrentViewer;
SetPerspective();
SetCamera(cams[0].position, cams[0].target);//no probs here
document.getElementById('MyViewerDiv').classList.add("geometry-loaded");
});
works (the camera is positioned), but when i execute a javascript function later (using driver.ExecuteScript($"SetCamera({JsonConvert.SerializeObject(target.Value.Position)},{JsonConvert.SerializeObject(target.Value.Target)});"); or on a timeout in the page itself, it outputs WebGL: CONTEXT_LOST_WEBGL: loseContext: context lost.
When i use a smaller model, everything works. Thus i think i understand the reason is too much memory/processing consumption, but why is it working at all then?
Looking at resource monitor, im not convinced that the consumption is actually problematic, my laptop should be capable (i7HQ7700, gtx1050, 16gbRam) I tried fiddling around with some GPU and GL flags of Chrome, to no avail. I suspect the GPU isn't used (which i found some posts that it actually can be used in headless...) Also, the forge viewer outputs GPU mem used, but that might be just the log message:
Starting ChromeDriver 2.38.552522 (437e6fbedfa8762dec75e2c5b3ddb86763dc9dcb) on port 62676
Only local connections are allowed.
[0517/203535.902:ERROR:gpu_process_transport_factory.cc(1007)] Lost UI shared context.
DevTools listening on ws://127.0.0.1:12556/devtools/browser/5b66c120-dc64-4211-a207-ac97152ace9a
---some ssl future warnings---
[0517/203540.524:INFO:CONSOLE(2)] "THREE.WebGLRenderer", source: https://developer.api.autodesk.com/modelderivative/v2/viewers/three.min.js (2)
[0517/203543.074:INFO:CONSOLE(0)] "[.Offscreen-For-WebGL-00000237DECBB270]RENDER WARNING: there is no texture bound to the unit 0", source: http://localhost:8881/Content/Screenshot.html
[0517/203543.074:INFO:CONSOLE(0)] "[.Offscreen-For-WebGL-00000237DECBB270]RENDER WARNING: there is no texture bound to the unit 0", source: http://localhost:8881/Content/Screenshot.html
[0517/203552.280:INFO:CONSOLE(2)] "Total geometry size: 8.434013366699219 MB", source: https://developer.api.autodesk.com/modelderivative/v2/viewers/three.min.js (2)
[0517/203552.281:INFO:CONSOLE(2)] "Number of meshes: 2909", source: https://developer.api.autodesk.com/modelderivative/v2/viewers/three.min.js (2)
[0517/203552.281:INFO:CONSOLE(2)] "Num Meshes on GPU: 2908", source: https://developer.api.autodesk.com/modelderivative/v2/viewers/three.min.js (2)
[0517/203552.281:INFO:CONSOLE(2)] "Net GPU geom memory used: 7494392", source: https://developer.api.autodesk.com/modelderivative/v2/viewers/three.min.js (2)
[0517/203558.143:INFO:CONSOLE(0)] "WebGL: CONTEXT_LOST_WEBGL: loseContext: context lost", source: http://localhost:8881/Content/Screenshot.html
To be complete, running the same program without the --headless flag, is working fine, so i guess the code itself is ok.
Is there any way to increase the allowed resources or anything?
(code for SetCamera)
function SetCamera(newPos, newTarget) {
nav = nav || viewerApp.myCurrentViewer.navigation;
nav.setPosition(newPos);
nav.setTarget(newTarget);
nav.orientCameraUp();
}
EDIT: Test case (currently on a test website, so this will we deleted at some point)
EDIT2: Result for running code below
NodeJS:
try {
const URN = '';
const Token = '';
(async () => {
const puppeteer = require('puppeteer');
const browser = await puppeteer.launch();
const page = await browser.newPage();
console.log('browsing');
await page.goto('https://rogerintelligentcloud.azurewebsites.net/test?urn=' + URN + '&token=' + Token);
//replace autodeskURN and token to point to your model
console.log("waiting");
await page.mainFrame().waitForSelector(
'.geometry-loaded', {
timeout: 60000
});
await takescreen(page, 'nodetest1');
await takescreen(page, 'nodetest2');
await takescreen(page, 'nodetest3');
await takescreen(page, 'nodetest4');
await takescreen(page, 'nodetest5');
await takescreen(page, 'nodetest6');
await page.evaluate("Test();");
await takescreen(page, 'nodetest11');
await takescreen(page, 'nodetest12');
await takescreen(page, 'nodetest13');
await takescreen(page, 'nodetest14');
await takescreen(page, 'nodetest15');
await takescreen(page, 'nodetest16');
await browser.close();
})();
} catch (e) {
console.log(e);
}
async function takescreen(page, name){
await page.screenshot({
path: 'c:\\temp\\'+name+'.png'
});
}
I didn't see any WebGL related error messages show up with your snippet and the rme_advanced_sample_project.rvt model, the only one I can see is page.delay is not defined. Here is my test code modified from the your code snippet and Philippe's forge-viewer-headless demo. If I missed something, please kindly point out. Thanks~
import puppeteer from 'puppeteer'
import 'babel-polyfill'
import path from 'path'
import os from 'os';
try {
const URN = 'YOUR_URN';
const Token = 'YOUR_TOKEN';
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(`https://rogerintelligentcloud.azurewebsites.net/test?urn=${ URN }&token=${ Token }`);
//replace autodeskURN and token to point to your model
await page.mainFrame().waitForSelector(
'.geometry-loaded', {
timeout: 60000
});
await delay(3000);
await page.screenshot({
path: path.join( os.tmpdir(), 'nodetest.png' )
})
await page.evaluate("Test();");
await page.evaluate("Test();");
await page.evaluate("Test();");
await page.evaluate("Test();");
await delay(3000);
const targetTxt = await page.evaluate(() => document.querySelector('body > :last-child').innerText );
console.log( targetTxt );
const targetLen = await page.evaluate(() => document.querySelectorAll('body > div:not(.box)').length );
console.log( targetLen );
await page.screenshot({
path: path.join( os.tmpdir(), 'nodetest2.png' )
})
await browser.close();
})();
function delay(timeout) {
return new Promise((resolve) => {
setTimeout(resolve, timeout);
});
}
}
catch (e) {
console.log(e);
}
Snapshot from my test result:
nodetest.png:
nodetest2.png:
Edit 2:
Tested with your code, it works fine on my machine.
But there is one change to run your code properly in my env. I modified your takescreen function definition:
function takescreen(page, name){
return page.screenshot({
path: 'c:\\temp\\'+name+'.png'
});
}
I want to close pages when puppeteer faces on any error , sometimes page the page that i try to load crashes and it doesnt call .close();
(async () => {
const page = await browser.newPage();
await page.setViewport({width: resWidth, height: resHeight});
await page.goto(d["entities"]["urls"][0]["expanded_url"], {timeout :90000});
await page.screenshot({path: './resimdata/'+d['id']+'.png' ,fullPage: true});
await page.close();
})();
There is an issue/PR on puppeteer repo regarding this which will be helpful in similar situation.
Related Issue link: https://github.com/GoogleChrome/puppeteer/issues/952
Meanwhile, you can try this little hack, if the PR is there on version 0.12+, we don't have to worry about the following code.
(async() => {
const browser = await puppeteer.launch({headless: false});
const page = await browser.newPage();
function handleClose(msg){
console.log(msg);
page.close();
browser.close();
process.exit(1);
}
process.on("uncaughtException", () => {
handleClose(`I crashed`);
});
process.on("unhandledRejection", () => {
handleClose(`I was rejected`);
});
await page.goto("chrome://crash");
})();
Which will output something like the following,
▶ node app/app.js
I was rejected