var wait = await page3.evaluate(()=> {
$('.modal').hide()
setTimeout(()=>{
return true
}, 3000);
});
Ex: wait 3 second in evaulate
You return a promise and resolve it when you want. Returning a promise will make it wait until it's resolved.
var wait = await page3.evaluate(()=> {
return new Promise((resolve,reject)=>{
$('.modal').hide()
setTimeout(()=>{
resolve(true)
}, 3000);
});
});
Related
I came a across a website that puppeteer can't handle. When making screenshot, Protocol error (Runtime.callFunctionOn): Target closed or Protocol error (Emulation.setDeviceMetricsOverride): Target closed is triggered. Before taking a screenshot, I scroll it so that all images are loaded. The page is large so I set '--disable-dev-shm-usage','--shm-size=3gb', params in hope to prevent any memory issues. This is sample code with url included. Any idea why the page is closed in the middle of the operation? In addition to puppeteer-cluster ("^0.23.0"), I am also using puppeteer-extra-plugin-stealth("^2.9.0") and puppeteer-extra("^3.2.3")
import puppeteer from 'puppeteer-extra';
import {Cluster} from 'puppeteer-cluster';
import StealthPlugin from "puppeteer-extra-plugin-stealth";
puppeteer.use(StealthPlugin());
const cluster = await Cluster.launch({
puppeteer,
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 1,
puppeteerOptions:{
headless : true,
args: [
'--disable-setuid-sandbox',
'--no-sandbox',
'--window-size=1920,1080',
'--disable-dev-shm-usage',
'--shm-size=3gb',
]
}
});
await cluster.task(async ({ page, data: url }) => {
let response = await page.goto(url, { waitUntil:"networkidle2" });
await Screenshot(page, screenshotPaths);
});
async function autoScroll(page){
await page.evaluate(async () => {
try {
await new Promise((resolve, reject) => {
let totalHeight = 0;
let distance = 389;
let counter = 0;
let timer = setInterval(() => {
counter++;
var scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if((totalHeight >= scrollHeight - window.innerHeight) || counter > 100){
clearInterval(timer);
resolve();
}
}, 50);
});
}catch (e) {
console.log("we got scrolling error:");
console.log(e);
}
});
}
async function Screenshot(page) {
let save = true;
try {
await page.waitForTimeout(6000);
await page.setViewport({ width:390, height:844});
await autoScroll(page);
await page.evaluate(() => window.scrollTo(0, 0));
await page.waitForTimeout(2000);
if(save) await page.screenshot({path: "./mobile.jpg", fullPage: true});
await page.setViewport({ width:1920, height:1080});
await autoScroll(page);
await page.evaluate(() => window.scrollTo(0, 0));
await page.waitForTimeout(2000);
if(save) await page.screenshot({path: "./desktop.jpg", fullPage: true});
}catch(error) {
console.log("we got screenshot error");
console.log(error);
}
}
cluster.queue("https://www.sinsay.com/si/sl/sale/woman/view-all-clothes");
await cluster.idle();
await cluster.close();
stack trace:
ProtocolError: Protocol error (Runtime.callFunctionOn): Target closed.
at /path/to/puppeteer/node_modules/puppeteer/lib/cjs/puppeteer/common/Connection.js:230:24
at new Promise (<anonymous>)
at CDPSession.send (/path/to/puppeteer/node_modules/puppeteer/lib/cjs/puppeteer/common/Connection.js:226:16)
at next (/path/to/puppeteer/node_modules/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.js:32:41)
at CDPSession.send (/path/to/puppeteer/node_modules/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.js:65:16)
at ExecutionContext._evaluateInternal (/path/to/puppeteer/node_modules/puppeteer/lib/cjs/puppeteer/common/ExecutionContext.js:204:50)
at ExecutionContext.evaluate (/path/to/puppeteer/node_modules/puppeteer/lib/cjs/puppeteer/common/ExecutionContext.js:110:27)
at DOMWorld.evaluate (/path/to/puppeteer/node_modules/puppeteer/lib/cjs/puppeteer/common/DOMWorld.js:123:24)
at processTicksAndRejections (node:internal/process/task_queues:96:5) {
originalMessage: ''
}
I am having a hard time understanding chaining promises in javaScript so I decided to make an example and practice some code this is what I want to do...
Do a "heavy task" for 5 seconds
Do a "medium task" for 3 seconds after heavy task finishes
successfully
If "medium task" is successful to a "small" task of 2 second
The "small task" must display the success message of the heavy task
If "medium task" fails then do an "error task" of 1 second stating
failure reason
while all these are going on do "Some other tasks..."
var p = new Promise(function(resolve, request) {
setTimeout(function() {
console.log("Inside heavy task...");
resolve("Heavy task was a success");
}, 5000);
})
.then(function(value) {
setTimeout(function(value) {
console.log("Inside medium task...");
resolve(value);
//reject("Medium task failed !");
}, 3000);
})
.then(function(value) {
console.log("Inside small task...");
console.log("From small task : " + value);
})
.catch( function(reson){
setTimeout(function(reason){
console.log("Inside error task...");
console.log("Failed due to "+reason);
},1000);
});
console.log("Some other tasks...");
I understand that my code is wrong can someone correct this and explain how this should be done.
Firstly, If you want to pass additional parameters to the callback function of setTimeout then it is done as follows
setTimeout(callback, time, param1, param2, ...)
Now in your code snippet, wrap the second setTimeout function in a promise and return that promise. Also if you want the small task to be asynchronous then wrap it inside a promise also and return that promise.
var p = new Promise(function(resolve, reject) {
setTimeout(function() {
console.log("Inside heavy task...");
resolve("Heavy task was a success");
}, 5000);
})
.then(function(value) {
var p2 = new Promise(function(resolve, reject) {
setTimeout(function() {
console.log("Inside medium task...");
resolve(value);
// reject("Medium task failed !");
}, 3000);
});
return p2;
})
.then(function(value) {
console.log("Inside small task...");
console.log("From small task : " + value);
})
.catch(function(reason) {
setTimeout(function(){
console.log("Inside error task...");
console.log("Failed due to "+ reason);
}, 1000);
});
console.log("Some other tasks...");
I hope this is helpful :)
I have a following function which uses streaming-query-rows of mysql node js module. How can i unit test the below function and also i want to mock the database behavior instead of connecting to database while unit test.
'processRow' and ''wirteCsvFile'' function both are synchronous task.
function executeTask(sql_connection,sql_query) {
let query = sql_connection.query(sql_query);
let showInfo = {};
let showids = [];
query
.on('error', (error) => {
console.error(`error executing query --> ${error}`);
})
.on('result', function (row) {
sql_connection.pause();
processRow(row, showInfo, showids, function () {
sql_connection.resume();
});
})
.on('end', function () {
showids.forEach(showid => {
if (showInfo[showid].faults.length === 0) {
delete showInfo[showid];
}
});
wirteCsvFile(showInfo, (error, done) => {
if (error) {
console.error(error);
} else {
console.log("done");
process.exit();
}
})
});
}
You can stub the query function to return whatever you want instead of making request to database:
sinon.stub(connection, "query").callsFake(() => /* whatever you want here */);
You should also break executeTask into smaller functions, for ex:
function errorHandler(error) {
console.error(`error executing query --> ${error}`);
}
function resultHandler(data, row) {
sql_connection.pause();
processRow(row, data.showInfo, data.showids, function() {
sql_connection.resume();
});
}
function endHandler(data) {
data.showids.forEach(showid => {
if (data.showInfo[showid].faults.length === 0) {
delete data.showInfo[showid];
}
});
wirteCsvFile(data.showInfo, (error, done) => {
if (error) {
console.error(error);
} else {
console.log("done");
process.exit();
}
})
}
function executeTask(sql_connection, sql_query) {
let query = sql_connection.query(sql_query);
let data = {
showInfo: {},
showids: [],
};
query.on('error', errorHandler)
.on('result', resultHandler.bind(null, data))
.on('end', endHandler.bind(null, data));
}
Now you can test errorHandler, resultHandler, endHandler separately
What I'm thinking is we can mock the sql_connection with a class of Event Emitter.
const sinon = require("sinon");
const assert = require('assert');
const EventEmitter = require('events');
const src = require('....'); // your source file that contain `executeTask`
// Create mock emitter
class QueryEmitter extends EventEmitter {}
describe('test execute task', function() {
const queryEmitter = new QueryEmitter();
// we build mock connection that contains all methods used as in `sql_connection`
const conn = {
query: sinon.stub().returns(queryEmitter),
pause: sinon.spy(),
resume: sinon.spy()
};
const query = 'SELECT *';
before(function() {
src.executeTask(conn, query);
});
it('calls query', function() {
assert(conn.query.calledWith(query));
});
it('on result', function() {
queryEmitter.emit('result');
assert(conn.pause.called);
// assert if processRow is called with correct arguments
// assert if conn.resume is called
});
it('on end', function() {
queryEmitter.emit('end');
// assert if writeCsvFile is called
});
// probably is not needed since you only call console.log here
it('on error', function() {
queryEmitter.emit('error');
});
});
Hope it helps
I am getting closer in my quest for a JSON response. In this example, the events variable gets populated AFTER it's returned, causing a blank output. I need it to wait. I have read that a promise is the way to go... but not sure how that would work... in my console.log you can see the array but Events.all(); returns null.
.factory('Events', function($http) {
var events="";
$http.get('http://appserver.falconinet.com/events.lasso').then(function(resp) {
events = resp.data;
console.log(events);
}, function(err) {
console.error('ERR', err);
// err.status will contain the status code
})
return {
all: function() {
return events;
},
get: function(eventId) {
for (var i = 0; i < events.length; i++) {
if (events[i].id === parseInt(eventId)) {
return events[i];
}
}
return null;
}
}
})
and here is my controller:
// events
.controller('EventsCtrl', function($scope, Events) {
$scope.events = Events.all();
})
.controller('EventDetailCtrl', function($scope, $stateParams, Events) {
$scope.event = Events.get($stateParams.eventId);
})
Following will return the promise created by $http as well as caches the loading of all events.
.factory('Events', function ($http, $q) {
function loadEvents(id) {
var promise = $http.get('http://appserver.falconinet.com/events.lasso', {cache: true});
// return the promise
return promise.then(function (resp) {
var events = resp.data;
if (id) {
// returns item or promise rejection
return getEventById(id, events);
} else {
return events;
}
}).catch (function (err) {
console.log('Events error ', err);
});
}
// helper function , returns event or promise rejection
function getEventById(id, events) {
for (var i = 0; i < events.length; i++) {
if (events[i].id === parseInt(eventId)) {
return events[i];
}
}
return $q.reject('None found');
}
return {
all: function () {
return loadEvents();
},
get: function (eventId) {
return loadEvents(eventId);
}
}
});
Then in controllers you need to resove your data in the promise then callback
.controller('EventsCtrl', function($scope, Events) {
Events.all().then(function(events){
$scope.events = events;
});
})
.controller('EventDetailCtrl', function($scope, $stateParams, Events) {
Events.get($stateParams.eventId).then(function(event){
$scope.event = event;
});
})
As you mentioned, wrapping your actual process in a new promise is the way to go. In order to do so, the usage of this factory needs some tweaks. Lemme try to write a sample from your script, but I don't promise to get it working on the first try :)
.factory('Events', function($http, $q) {
return {
all: function() {
var myPromise = $q.defer();
$http.get('http://appserver.falconinet.com/events.lasso')
.then(function(resp) {
myPromise.resolve(resp.data);
}, function(err) {
myPromise.reject(err);
});
return myPromise.promise;
},
get: function(eventId) {
var myPromise = $q.defer();
$http.get('http://appserver.falconinet.com/events.lasso')
.then(function(resp) {
var events = resp.data;
for (var i = 0; i < events.length; i++) {
if (events[i].id === parseInt(eventId)) {
myPromise.resolve(events[i]);
}
}
}, function(err) {
myPromise.reject(err);
});
return myPromise.promise;
}
}
});
Besides everything, this can be improved as you placer but I found pretty straightforward to do so after knowing how to handle promises.
Using them would be like this:
// Get all
Events.all().then(function(events){
$scope.events = events;
});
// Get one
Events.get(eventId).then(function(event){
$scope.events = event;
});
Has anyone seen this bug with AngularJS (v1.0.7) and Chrome (Version 30.0.1599.114) where canceling http GET request puts the sockets into a pending state thus maxing out the thread pool in chrome?
Code:
if ($scope.canceler !== undefined) {
$scope.canceler.resolve();
$scope.canceler = undefined;
}
$scope.canceler = $q.defer();
$http.get("/apicall", {
cache: myCache,
timeout: $scope.canceler.promise
}).success(function (results) {
}).
error(function (result) {
});
Might be the same bug 241844
You should update AngularJS to 1.1.5 to be able to cancel http calls.
ref: In AngularJS, how to stop ongoing $http calls on query change
Here is the working code and the JS fiddle. I have tested with AngularJS 1.2.0 and Chrome 32.0.1700.0 canary.
function Ctrl($rootScope, $scope, $http, $q, $timeout) {
var canceler = $q.defer();
console.log("Calling...");
$http.get("/echo/json", {
//Will return data after 5 seconds passed
data: {json: {id: "123"}, delay: 5},
timeout: canceler.promise
}).success(function (results) {
console.log("Success");
console.log(results);
}).
error(function (result) {
console.log("Error");
console.log(result);
});
$timeout(function () {
console.log("1 second passed");
// now, cancel it (before it may come back with data)
$rootScope.$apply(function () {
console.log("Canceling..");
canceler.resolve();
});
}, 1000);
}
http://jsfiddle.net/7Ma7E/4/
The request was become cancelled state.