I am using web3.js v1.0 with solidity ^0.4.17 with Ganache v1.1.0. I am trying to call a send transaction and it fails with the error message below.
Returned error: Error: Error: [ethjs-query] while formatting outputs
from RPC 'undefined' for method 'sendRawTransaction' Error:
[ethjs-format] hex string 'undefined' must be an alphanumeric 66 utf8
byte hex (chars: a-fA-F) string, is 0 bytes
MyContract.sol
function createStarCard(string name, uint price) public {
require(msg.sender == owner);
uint starCardId = starCards.push(StarCard(name, price));
starCardIdToOwner[starCardId] = owner;
}
App.js
createStarCard = ({ name, price }) => {
window.web3.eth.getAccounts().then((accounts) => {
this.state.ContractInstance.methods.createStarCard(name, price).send({
from: accounts[0],
gas: 300000,
}).then((receipt) => {
console.log(receipt)
}).catch((err) => {
console.log(err.message) <-- Caught error message
})
})
}
Google search results with the error message pointed me to the following issues, but they weren't helpful in my case:
https://github.com/MetaMask/metamask-extension/issues/1870
https://ethereum.stackexchange.com/questions/23679/callback-contain-no-result-error-error-ethjs-query-while-formatting-outputs
Update: sharing my constructor for App.js
constructor(props) {
super(props)
if (typeof window.web3 !== 'undefined') {
this.web3Provider = window.web3.currentProvider;
} else {
console.log("Use Ganache web3")
this.web3Provider = new Web3.providers.HttpProvider('http://localhost:7545');
}
window.web3 = new Web3(this.web3Provider);
const contractAddress = "0x1bdaf0cd259887258bc13a92c0a6da92698644c0"
const ContractInstance = new window.web3.eth.Contract(Abi, contractAddress);
this.state = {
ContractInstance,
}
}
It looks like the problem is with the Ganache mac app. I solved this by using ganache-cli instead.
I solved this by reinstalling Metamask.
Related
Here is my code, I have also added the options as well in the approve function which contains the from, to, gasLimit but that also doesn't work for me
const { POSClient, use } = require('#maticnetwork/maticjs');
const { Web3ClientPlugin } = require('#maticnetwork/maticjs-web3');
const HDWalletProvider = require('#truffle/hdwallet-provider');
require('dotenv').config();
use(Web3ClientPlugin);
async function getPOSClient() {
const posClient = new POSClient();
return await posClient.init({
network: 'testnet',
version: 'mumbai',
parent: {
provider: new HDWalletProvider(
process.env.PVT_KEY,
process.env.GOERLI_RPC
),
defaultConfig: {
from: process.env.FROM_ADDRESS,
},
},
child: {
provider: new HDWalletProvider(
process.env.PVT_KEY,
process.env.MUMBAI_RPC
),
defaultConfig: {
from: process.env.FROM_ADDRESS,
},
},
});
}
async function approveToken() {
const posClient = await getPOSClient();
// console.log('🚀 ~ file: index.js:36 ~ approveToken ~ posClient', posClient);
const erc20Token = posClient.erc20(process.env.ROOT_TOKEN, true);
const result = await erc20Token.approve('1000');
const txHash = await result.getTransactionHash();
console.log('txHash', txHash);
const receipt = await result.getReceipt();
console.log('receipt', receipt);
}
approveToken();
The error which I'm getting while approving the token,
Error: execution reverted: ERC20: approve to the zero address
0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002245524332303a20617070726f766520746f20746865207a65726f2061646472657373000000000000000000000000000000000000000000000000000000000000
I have also mapped the token from polygon token mapper portal also. Please give your feedback
This error occurs when you are trying to approve an ERC20 token to the zero address.
Assuming the approve interface is approve(spender, amount)
I believe that if you add the approved address to this call const result = await erc20Token.approve(ADDRESS,'1000'); this issue will be resolved.
Error message: Error: missing argument: in Contract constructor (count=0, expectedCount=1, code=MISSING_ARGUMENT, version=contracts/5.6.2)
contract:
contract KBMarket is ReentrancyGuard {
using Counters for Counters.Counter;
constructor() {
owner = payable(msg.sender);
}
and here is hardhat test.js:
describe("KBMarket", function () {
it("Should Mint And Trade NFTs", async function () {
const Market = await ethers.getContractFactory('KBMarket')
const market = await Market.deploy()
await market.deployed()
const marketAddress = market.adderss
}
}
Thanks in advance.
Your constructor should have an argument:
constructor(uint _feePercent)
In deploy.js,
const market = await Market.deploy(1)
you need to pass an argument here because the marketplace's constructor accepts the _feePercent as an argument.
I am developing a dapp where users can upload photographs and mint them to NFTs. My application uses web3( Alchemy web3), Alchemy, Ropsten and Metamask. I also use Hardhat in order to deploy my contract with the command:
npx hardhat run --network ropsten scripts/deploy.js
First of all I have installed Metamask in my browser(Firefox) and I have created my wallet. I have also used a Ropsten Faucet in order to get some ether. In order to deploy my contract I use my account's private key in hardhat.config.js. The problem is that in this way only my account is able to use the contract. My application is supposed to accept multiple users, each with their own Metamask wallet or configuration, performing their own transactions.
Therefore, can I deploy or change my contract so that it can be used by any user and not just the one who deployed it?
Here is my contract:
// SPDX-License-Identifier: MIT
pragma solidity ^0.8.2;
import "#openzeppelin/contracts/token/ERC721/ERC721.sol";
import "#openzeppelin/contracts/access/Ownable.sol";
import "#openzeppelin/contracts/utils/Counters.sol";
contract NFTminter is ERC721, Ownable {
using Counters for Counters.Counter;
mapping (uint256 => string) private _tokenURIs;
mapping(string => uint256) private hashes;
Counters.Counter private _tokenIdCounter;
constructor() ERC721("MyToken", "PcI") {
}
function _baseURI() internal pure override returns (string memory) {
return "ipfs://";
}
function safeMint(address to, string memory metadataURI) public onlyOwner returns (uint256) {
require(hashes[metadataURI] != 1 , "This metadataURI already exists.");
hashes[metadataURI] = 1;
uint256 tokenId = _tokenIdCounter.current();
_tokenIdCounter.increment();
_safeMint(to, tokenId);
_setTokenURI(tokenId, metadataURI);
return tokenId;
}
function _setTokenURI(uint256 tokenId, string memory _tokenURI) internal virtual {
require(_exists(tokenId), "ERC721URIStorage: URI set of nonexistent token");
_tokenURIs[tokenId] = _tokenURI;
}
}
Here is my hardhat.config.js:
require("#nomiclabs/hardhat-waffle");
const { MNEMONIC, ALCHEMY_HTTP } = require("./alchemy_secrets.json");
task("accounts", "Prints the list of accounts", async (taskArgs, hre) => {
const accounts = await hre.ethers.getSigners();
for (const account of accounts) {
console.log(account.address);
}
});
module.exports = {
networks: {
ropsten: {
url: ALCHEMY_HTTP,
accounts: ['PRIVATEKEYGOESHERE']
},
},
solidity: "0.8.4",
};
And here is the code which calls the contract and sends the transaction:
import Web3 from "web3";
import CONTRACT_ABI from "../ethContractABI";
import { AbiItem } from "web3-utils";
import {
ROPSTEN_CONTRACT_ADDRESS,
NFT_STORAGE_KEY,
ALCHEMY_API_KEY,
} from "../constants";
import { NFTStorage } from "nft.storage";
import { Contract } from "web3-eth-Contract";
import { AlchemyWeb3, createAlchemyWeb3 } from "#alch/alchemy-web3";
declare global {
interface Window {
ethereum: any;
web3: Web3;
}
}
interface postcardReturn {
ipfsLink?: string | undefined;
tokenID?: number | undefined;
errorMessage?: string | undefined;
}
async function convertToNft(
imageToUpload: File,
etherAddress: string,
privateKey: string
): Promise<postcardReturn> {
try {
await CreateWeb3Object();
const web3 = createAlchemyWeb3(ALCHEMY_API_KEY);
const metadata = await GetNFTmetadata(imageToUpload);
const NFTminter = createNftContract(web3, etherAddress);
//CheckIfTokenExists(NFTminter, metadata, etherAddress);
const receipt = await mintToken(
NFTminter,
metadata,
etherAddress,
web3,
privateKey
);
return {
ipfsLink: metadata.data.image.href,
tokenID: receipt!.events!.Transfer.returnValues.tokenId,
};
} catch (error: any) {
return returnError(error);
}
}
async function GetNFTmetadata(imageToUpload: File) {
const client = new NFTStorage({ token: NFT_STORAGE_KEY });
const metadata = await client.store({
name: "From: User",
description: "IMage to be converted to nft",
image: imageToUpload,
});
return metadata;
}
async function CreateWeb3Object() {
if (window.ethereum) {
try {
const enable = window.ethereum.enable();
return;
} catch (error) {
console.log(error);
}
}
}
async function CheckIfTokenExists(
NFTminter: Contract,
metadata: any,
etherAddress: string
) {
const check = await NFTminter.methods
.safeMint(etherAddress, metadata.url)
.estimateGas((error: any, gasAmount: any) => {
if (error) {
console.error(error);
return "An error has occured";
}
});
}
function createNftContract(web3: any, etherAddress: string) {
const NFTminter = new web3.eth.Contract(
CONTRACT_ABI,
ROPSTEN_CONTRACT_ADDRESS
);
return NFTminter;
}
async function mintToken(
NFTminter: Contract,
metadata: any,
etherAddress: string,
web3: AlchemyWeb3,
privateKey: string
) {
const nonce = await web3.eth.getTransactionCount(etherAddress, "latest");
const tx = {
from: etherAddress,
to: ROPSTEN_CONTRACT_ADDRESS,
nonce: nonce,
gas: 2000000,
maxPriorityFeePerGas: 1999999987,
data: NFTminter.methods
.safeMint(etherAddress, metadata.data.image.href)
.encodeABI(),
};
const signedTx = await web3.eth.accounts.signTransaction(tx, privateKey);
web3.eth.sendSignedTransaction(
signedTx.rawTransaction!
).then(console.log)
.catch(console.log);
const transactionReceipt = await web3.eth.sendSignedTransaction(
signedTx.rawTransaction!
);
console.log(transactionReceipt);
return transactionReceipt;
}
function returnError(error: any) {
if (error.message.includes("Internal JSON-RPC error."))
return {
errorMessage: "Internal JSON-RPC error.",
};
return {
errorMessage: error.message,
};
}
export default convertToNft;
The privateKey variable is given by the user before he clicks the upload button. If I change the account that I use, the exception is thrown in this line:
const signedTx = await web3.eth.accounts.signTransaction(tx, privateKey);
The exception says that the Caller is not the contract owner. I think that what I described as the problem is the issue, because I redeployed my contract with the Metamask account that I use now and I can normally send and sign transactions. However, If I change the account I receive an error.
Your safeMint function has onlyOwner modifier. That's why only contract deployer can use it.
Im trying to integrate web3js with Trezor in a truffle dev network or using ropsten test network.
The idea is to sign the transactions using the hardware wallet and then send a raw transaction using web3js
Im getting that we dont have balance to make the transaction, probably because web3js isnt taking one of the 10 truffle accounts and is using the trezor address that isnt in my local network..
On ropsten i have some ethers and i get "invalid address"
Is there a way to send a signed transactions (with trezor) using web3js into a truffle develop network? i mean, is there a way to include the trezor address into the truffle network?
The situation in truffle is explained more in details here, but the question could be generalized to "is there a way to include hardware wallets into truffle development network?" : https://github.com/trufflesuite/truffle/issues/973
Using ropsten I have managed to send a transaction and receive a transaction hash in the callback, but if we query for that transaction we get that the transaction doesnt exists.. so.. how is that possible?
I tried deploying a contract into Ropsten too and now im getting "Invalid address" when invoking a smart contract function. Maybe the signing function is wrong? anyone could integrate Trezor transaction signining with web3js?
Do you guys see anything wrong in the signing and sending process that we have followed? Maybe is there something wrong on the R, V and S parameters handling
..
Another important thing is that i am using https://github.com/ethereumjs/ethereumjs-tx for creating the raw transactions
Issues published in web3js, truffle and trezzor connect with more information:
https://github.com/trufflesuite/truffle/issues/973
https://github.com/ethereum/web3.js/issues/1669
https://github.com/trezor/connect/issues/130
kind regards
trezorLogin = async()=> {
let trezor= await this.getTrezor();
// site icon, optional. at least 48x48px
let hosticon = 'https://doc.satoshilabs.com/trezor-apps/_images/copay_logo.png';
// server-side generated and randomized challenges
let challenge_hidden = '';
let challenge_visual = '';
//use anonimous functions on callback otherwise returns cross origin errors
trezor.requestLogin(hosticon, challenge_hidden, challenge_visual, function (result){
if (result.success) {
console.log('Public key:', result.public_key); // pubkey in hex
console.log('Signature:', result.signature); // signature in hex
console.log('Version 2:', result.version === 2); // version field
console.log(result);
}else {
console.error('Error:', result.error);
}
});}
trezorSignTx= async(transaction)=> {
let trezor= await this.getTrezor();
// spend one change output
let address_n = "m/44'/60'/0'/0/0"
// let address_n = [44 | 0x80000000,
// 60 | 0x80000000,
// 0 | 0x80000000 ,
// 0 ]; // same, in raw form
let nonce = transaction.nonce.substring(2); // note - it is hex, not number!!!
let gas_price = transaction.gasPrice.substring(2);
let gas_limit = transaction.gasLimit.substring(2);
let to = transaction.to.substring(2);
// let value = '01'; // in hexadecimal, in wei - this is 1 wei
let value = transaction.value.substring(2); // in hexadecimal, in wei - this is about 18 ETC
let data = transaction.data.substring(2); // some contract data
// let data = null // for no data
let chain_id = 5777; // 1 for ETH, 61 for ETC
return new Promise (function (resolve,reject) {
trezor.ethereumSignTx(
address_n,
nonce,
gas_price,
gas_limit,
to,
value,
data,
chain_id,
function (response) {
if (response.success) {
console.log('Signature V (recovery parameter):', response.v); // number
console.log('Signature R component:', response.r); // bytes
console.log('Signature S component:', response.s); // bytes
resolve(response);
} else {
console.error('Error:', response.error); // error message
resolve(null);
}
});
})
}
getTrezorAddress = async() => {
let trezor= await this.getTrezor();
// spend one change output
let address_n = "m/44'/60'/0'/0/0";
trezor.ethereumGetAddress(address_n, function (result) {
if (result.success) { // success
console.log('Address: ', result.address);
} else {
console.error('Error:', result.error); // error message
}
});
}
getTrezor = async() => {
let trezorC;
await getTrezorConnect
.then(trezorConnect => {
trezorC= trezorConnect;
})
.catch((error) => {
console.log(error)
})
return trezorC;
}
sendTransaction= async(address, amount, id)=>{
let tokenInstance = this.props.smartContractInstance;
let getData = tokenInstance.mint.getData(address, amount);
let tx = {
nonce: '0x00',
gasPrice: '0x09184e72a000',
gasLimit: '0x2710',
to: CONTRACT_ADDRESS,
value: '0x00',
from:CONTRACT_OWNER_ADDRESS,
data: getData
};
let response = await this.trezorSignTx(tx);
let web3;
let _this = this;
if (response!=null){
getWeb3
.then(results => {
web3= results.web3;
let v = response.v.toString();
if (v.length % 2 != 0){
v="0"+v;
}
tx.r=Buffer.from(response.r,'hex');
tx.v=Buffer.from(v,'hex');
tx.s=Buffer.from(response.s,'hex');
let ethtx = new ethereumjs(tx);
console.dir(ethtx.getSenderAddress().toString('hex'), );
const serializedTx = ethtx.serialize();
const rawTx = '0x' + serializedTx.toString('hex');
console.log(rawTx);
//finally pass this data parameter to send Transaction
web3.eth.sendRawTransaction(rawTx, function (error, result) {
if(!error){
_this.props.addTokens(id)
.then(()=>{
_this.setState({modalOpen: true});
_this.props.getAllTransactions();
}
);
}else{
alert(error)
}
});
})
.catch((error) => {
console.log(error)
})
}else{
alert("There was an error signing with trezor hardware wallet")
}
}
The getTrezorConnect function is just get window.trezorConnect asynchronously because the object is injected as script
<script src="https://connect.trezor.io/4/connect.js"></script>
let getTrezorConnect = new Promise(function(resolve, reject) {
// Wait for loading completion
window.addEventListener('load', function() {
let trezorConnect = window.TrezorConnect
return resolve(trezorConnect)
})});
export default getTrezorConnect
Well, after a lot of trying we have managed to send a raw transaction signed with Trezor to Ropsten, Truffle (see the edit on the bottom of the answer) and also to a local private Geth network, so, the code is ok and there is no problem with Trezor integration on those environments
https://ropsten.etherscan.io/address/0x89e2c46b22881f747797cf67310aad1a831d50b7
This are the things that i had changed in order to make it possible to send signed transactions to the Ropsten testnet.
This assumes that you have your contract deployed into Ropsten and you have the contract address.
1) Get the address of your Trezor account
getTrezorAddress = async() => {
let trezor= await this.getTrezor();
// spend one change output
let address_n = "m/44'/1'/0'/0/0";
trezor.ethereumGetAddress(address_n, function (result) {
if (result.success) { // success
console.log('Address: ', result.address);
} else {
console.error('Error:', result.error); // error message
}
});
}
2) Put the trezor address into the from field of your raw transaction, get the nonce of the transaction by getting the transaction count for that address. Important: use the "pending" optional parameter on getTransactionCount to get all the transactions of the account, otherwise you will be overriting pending transactions.
getNonce = async(address) => {
let web3 = await this.getWeb3();
return new Promise (function (resolve,reject) {
web3.eth.getTransactionCount(address, "pending", function (error,result){
console.log("Nonce "+result);
resolve(result);
});
});
}
let count = null;
await this.getNonce("0xedff546ac229317df81ef9e6cb3b67c0e6425fa7").then(result => {
if(result.length % 2 !==0){
result = "0"+result;
}
count = "0x"+result;
});
let tx = {
nonce: count ,
gasPrice: web3.toHex(gasPriceGwei*1e9),
gasLimit: web3.toHex(gasLimit),
to: CONTRACT_ADDRESS,
value: '0x00',
data: getData,
chainId:chainId,
from:"yourTrezzorAddress"
};
3) The r, s, v parameters were incorrect, the right way to handle them is take that values for the trezor response and just convert it to hexa:
// response is the Trezor sign response
tx.v= response.v;
tx.r="0x"+response.r;
tx.s="0x"+response.s;
let ethtx = new ethereumjs(tx);.
const serializedTx = ethtx.serialize();
const rawTx = '0x' + serializedTx.toString('hex');
//finally pass this data parameter to send Transaction
web3.eth.sendRawTransaction(rawTx, someCallbackFunction);
Important: the mining time in ropsten will be between 15 and 30 secs so if in your someCallbackFunction you check for the transaction receipt, using the hash, you will get null as result, because the transaction is in a pending state.
4) To test it at ropsten we use Infura, so we change the web3 provider:
import Web3 from 'web3'
import HDWalletProvider from "truffle-hdwallet-provider";
let getWeb3 = new Promise(function(resolve, reject) {
// Wait for loading completion to avoid race conditions with web3 injection timing.
window.addEventListener('load', function() {
let results
let web3 = window.web3
// Checking if Web3 has been injected by the browser (Mist/MetaMask)
if (typeof web3 !== 'undefined') {
// Use Mist/MetaMask's provider.
web3 = new Web3(web3.currentProvider)
results = {
web3: web3
}
console.log('Injected web3 detected.');
return resolve(results)
} else {
// Fallback to localhost if no web3 injection. We've configured this to
// use the development console's port by default.
// let provider = new Web3.providers.HttpProvider("https://ropsten.infura.io/your_infura_api_key")
let mnemonic = "infura mnemonic"
let provider = new HDWalletProvider(mnemonic, "https://ropsten.infura.io/your_infura_api_key")
web3 = new Web3(provider)
results = {
web3: web3
}
console.log('No web3 instance injected, using Local web3.');
return resolve(results)
}
})
})
export default getWeb3
EDIT:
This also works on Truffle! check the last comments of this issue https://github.com/trufflesuite/truffle/issues/973
We developed a small library to use Truffle with Trezor hardware wallets: https://github.com/rarible/trezor-provider
It can be used like this:
const { createProvider } = require('#rarible/trezor-provider')
module.exports = {
networks: {
ropsten: {
provider: function() {
//websocket and http urls are supported
return createProvider({ url: "{infura or other}", path: "m/44'/60'/0'/0/0", chainId: 3 })
},
network_id: 3
}
}
};
Is there any way to run Headless Chrome/Chromium in a Google Cloud Function? I understand I can include and run statically compiled binaries in GCF. Can I get a statically compiled version of Chrome that would work for this?
The Node.js 8 runtime for Google Cloud Functions now includes all the necessary OS packages to run Headless Chrome.
Here is a code sample of an HTTP function that returns screenshots:
Main index.js file:
const puppeteer = require('puppeteer');
exports.screenshot = async (req, res) => {
const url = req.query.url;
if (!url) {
return res.send('Please provide URL as GET parameter, for example: ?url=https://example.com');
}
const browser = await puppeteer.launch({
args: ['--no-sandbox']
});
const page = await browser.newPage();
await page.goto(url);
const imageBuffer = await page.screenshot();
await browser.close();
res.set('Content-Type', 'image/png');
res.send(imageBuffer);
}
and package.json
{
"name": "screenshot",
"version": "0.0.1",
"dependencies": {
"puppeteer": "^1.6.2"
}
}
I've just deployed a GCF function running headless Chrome. A couple takeways:
you have to statically compile Chromium and NSS on Debian 8
you have to patch environment variables to point to NSS before launching Chromium
performance is much worse than what you'd get on AWS Lambda (3+ seconds)
For 1, you should be able to find plenty of instructions online.
For 2, the code that I'm using is the following:
static executablePath() {
let bin = path.join(__dirname, '..', 'bin', 'chromium');
let nss = path.join(__dirname, '..', 'bin', 'nss', 'Linux3.16_x86_64_cc_glibc_PTH_64_OPT.OBJ');
if (process.env.PATH === undefined) {
process.env.PATH = path.join(nss, 'bin');
} else if (process.env.PATH.indexOf(nss) === -1) {
process.env.PATH = [path.join(nss, 'bin'), process.env.PATH].join(':');
}
if (process.env.LD_LIBRARY_PATH === undefined) {
process.env.LD_LIBRARY_PATH = path.join(nss, 'lib');
} else if (process.env.LD_LIBRARY_PATH.indexOf(nss) === -1) {
process.env.LD_LIBRARY_PATH = [path.join(nss, 'lib'), process.env.LD_LIBRARY_PATH].join(':');
}
if (fs.existsSync('/tmp/chromium') === true) {
return '/tmp/chromium';
}
return new Promise(
(resolve, reject) => {
try {
fs.chmod(bin, '0755', () => {
fs.symlinkSync(bin, '/tmp/chromium'); return resolve('/tmp/chromium');
});
} catch (error) {
return reject(error);
}
}
);
}
You also need to use a few required arguments when starting Chrome, namely:
--disable-dev-shm-usage
--disable-setuid-sandbox
--no-first-run
--no-sandbox
--no-zygote
--single-process
I hope this helps.
As mentioned in the comment, work is being done on a possible solution to running a headless browser in a cloud function. A directly applicable discussion:"headless chrome & aws lambda" can be read on Google Groups.
The question at. had was can you run headless chrome or chromium in Firebase Cloud Functions... the answer is NO! since the node.js project will not have access any chrome/chromium executables and therefore will fail! (TRUST ME - I've Tried!).
A better solutions is to use the Phantom npm package, which uses PhantomJS under the hood:
https://www.npmjs.com/package/phantom
Docs and info can be found here:
http://amirraminfar.com/phantomjs-node/#/
or
https://github.com/amir20/phantomjs-node
The site i was trying to crawl had implemented screen scraping software, the trick is to wait for the page to load by searching for expected string, or regex match, i.e. i do a regex for a , if you need a regex of any complexity made for you - get in touch at https://AppLogics.uk/ - starting at £5 (GPB).
here is a typescript snippet to make the http or https call:
const phantom = require('phantom');
const instance: any = await phantom.create(['--ignore-ssl-errors=yes', '--load-images=no']);
const page: any = await instance.createPage();
const status = await page.open('https://somewebsite.co.uk/');
const content = await page.property('content');
same again in JavaScript:
const phantom = require('phantom');
const instance = yield phantom.create(['--ignore-ssl-errors=yes', '--load-images=no']);
const page = yield instance.createPage();
const status = yield page.open('https://somewebsite.co.uk/');
const content = yield page.property('content');
Thats the easy bit! if its a static page your pretty much done and you can parse the HTML into something like the cheerio npm package: https://github.com/cheeriojs/cheerio - an implementation of core JQuery designed for servers!
However if it is a dynamically loading page, i.e. lazy loading, or even anti-scraping methods, you will need to wait for the page to update by looping and calling the page.property('content') method and running a text search or regex to see if your page has finished loading.
I have created a generic asynchronous function returning the page content (as a string) on success and throws an exception on failure or timeout. It takes as parameters the variables for the page, text (string to search for that indicates success), error (string to indicate failure or null to not check for error), and timeout (number - self explanatory):
TypeScript:
async function waitForPageToLoadStr(page: any, text: string, error: string, timeout: number): Promise<string> {
const maxTime = timeout ? (new Date()).getTime() + timeout : null;
let html: string = '';
html = await page.property('content');
async function loop(): Promise<string>{
async function checkSuccess(): Promise <boolean> {
html = await page.property('content');
if (!isNullOrUndefined(error) && html.includes(error)) {
throw new Error(`Error string found: ${ error }`);
}
if (maxTime && (new Date()).getTime() >= maxTime) {
throw new Error(`Timed out waiting for string: ${ text }`);
}
return html.includes(text)
}
if (await checkSuccess()){
return html;
} else {
return loop();
}
}
return await loop();
}
JavaScript:
function waitForPageToLoadStr(page, text, error, timeout) {
return __awaiter(this, void 0, void 0, function* () {
const maxTime = timeout ? (new Date()).getTime() + timeout : null;
let html = '';
html = yield page.property('content');
function loop() {
return __awaiter(this, void 0, void 0, function* () {
function checkSuccess() {
return __awaiter(this, void 0, void 0, function* () {
html = yield page.property('content');
if (!isNullOrUndefined(error) && html.includes(error)) {
throw new Error(`Error string found: ${error}`);
}
if (maxTime && (new Date()).getTime() >= maxTime) {
throw new Error(`Timed out waiting for string: ${text}`);
}
return html.includes(text);
});
}
if (yield checkSuccess()) {
return html;
}
else {
return loop();
}
});
}
return yield loop();
});
}
I have personally used this function like this:
TypeScript:
try {
const phantom = require('phantom');
const instance: any = await phantom.create(['--ignore-ssl-errors=yes', '--load-images=no']);
const page: any = await instance.createPage();
const status = await page.open('https://somewebsite.co.uk/');
await waitForPageToLoadStr(page, '<div>Welcome to somewebsite</div>', '<h1>Website under maintenance, try again later</h1>', 1000);
} catch (error) {
console.error(error);
}
JavaScript:
try {
const phantom = require('phantom');
const instance = yield phantom.create(['--ignore-ssl-errors=yes', '--load-images=no']);
const page = yield instance.createPage();
yield page.open('https://vehicleenquiry.service.gov.uk/');
yield waitForPageToLoadStr(page, '<div>Welcome to somewebsite</div>', '<h1>Website under maintenance, try again later</h1>', 1000);
} catch (error) {
console.error(error);
}
Happy crawling!