puppeteer Works Local But Not On EC2 AWS - google-chrome

Question

local is fine but when i deployed on aws ec2 server
it stuck in page.goto, and got a **error ERR_CONNECTION_REFUSED**
so i tried install TightVNC for GUI
same as stuck in page.goto
i checked google-chrome is installed,
when i typing "google-chrome" on aws ec2 in tightVNC viewer, it can show chrome well
os: aws ec2 ubuntu18.04 (30Gib)
aws security group inbound:
5000 TCP 0.0.0.0/0
5901 TCP 0.0.0.0/0
80 TCP 0.0.0.0/0
3000 TCP 0.0.0.0/0
22 TCP 0.0.0.0/0
443 TCP 0.0.0.0/0
5901 TCP ::/0
node --version: v16.14.2
sudo node --version: v16.14.2
npm --version: 8.5.0
sudo npm --version: 8.5.0
google-chrome --version: Google Chrome 101.0.4951.64
which google-chrome: /usr/bin/google-chrome
i tried that 'executablePath: "/usr/bin/google-chrome"' but same as error
in the ~/file_name/node_modules/puppeteer/.local-chromium/linux-991974/chrome-linux
when i typing "ldd chrome | grep not" nothing show, i installed everything
i think most related with puppeteer and aws ec2 i searched and i tried
what's matter to me?
please let me know
const express = require("express");
const router = express.Router();
const puppeteer = require("puppeteer-extra");
const deepai = require("deepai");
const fs = require("fs");
const CryptoJS = require("crypto-js");
const axios = require("axios");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());
const puppeteerOptions = {
headless: true,
ignoreHTTPSErrors: false,
args: [
"--no-sandbox",
"--single-process",
"--no-zygote",
"--disable-setuid-sandbox",
"--disable-gpu",
"--use-gl=egl",
],
dumpio: true,
};
// executablePath: "/usr/bin/google-chrome",
//slowMo: 30, //사람이 타이핑하는 듯이 볼수 있음
// /usr/bin/google-chrome
// /usr/lib/node_modules/puppeteer/.local-chromium/linux-991974
let browser;
let pageNameH;
router.post("/rpa", (req, res) => {
(async function () {
browser = await puppeteer.launch(puppeteerOptions);
pageNameH = await browser.newPage();
await pageNameH.setDefaultNavigationTimeout(0);
await pageNameH.setViewport({ width: 1366, height: 768 });
console.log("Running tests..");
console.log("req.body.length", req.body.length); //5
console.log("req.body[0]", req.body[0]); //{ Link: 'www.coupang.com' }
console.log("req.body[0].Link", req.body[0].Link); //www.coupang.com
console.log("req.body", req.body);
var click = "";
var ScreenshotName = "";
for (var i = 0; i < req.body.length; i++) {
if (req.body[i].Link) {
await pageNameH.goto(req.body[i].Link);
// await pageNameH.waitFor(500);
console.log("Link");
} else if (req.body[i].Click) {
click = "#" + req.body[i].Click;
console.log("Click");
} else if (req.body[i].Input) {
await pageNameH.type(click, req.body[i].Input);
console.log("Input");
} else if (req.body[i] == "Click") {
await pageNameH.waitFor(500);
await pageNameH.keyboard.press("Enter");
await pageNameH.waitFor(500);
await pageNameH.waitForNavigation();
console.log("Button");
} else if (req.body[i].ScreenShot) {
ScreenshotName = req.body[i].ScreenShot + ".png";
await pageNameH.screenshot({
path: ScreenshotName,
fullPage: true,
});
console.log("ScreenShot");
} else if (req.body[i].sendSMS) {
var userphoneNo = req.body[i].sendSMS;
// browser.close();
browser.close();
console.log("browser.close");
deepai.setApiKey("241c47f5-d430-4e76-b078-10df6ce6be7f");
var resp = await deepai.callStandardApi("image-similarity", {
image1: fs.createReadStream("coupang.png"),
image2: fs.createReadStream(ScreenshotName),
});
if (resp["output"]["distance"] > 0) {
const user_phone_number = userphoneNo; // SMS를 수신할 전화번호
console.log("user_phone_number", user_phone_number);
const date = Date.now().toString(); // 날짜 string
// 환경 변수
const sens_service_id = "ncp:sms:kr:284757485087:projectd";
const sens_access_key = "***";
const sens_secret_key = "***";
const sens_call_number = "01012345678"; //회사 번호
// url 관련 변수 선언
const method = "POST";
const space = " ";
const newLine = "\n";
const url = `https://sens.apigw.ntruss.com/sms/v2/services/${sens_service_id}/messages`;
const url2 = `/sms/v2/services/${sens_service_id}/messages`;
// signature 작성 : crypto-js 모듈을 이용하여 암호화
console.log(1);
const hmac = CryptoJS.algo.HMAC.create(
CryptoJS.algo.SHA256,
sens_secret_key
);
console.log(2);
hmac.update(method);
hmac.update(space);
hmac.update(url2);
hmac.update(newLine);
hmac.update(date);
hmac.update(newLine);
console.log(sens_access_key);
hmac.update(sens_access_key);
const hash = hmac.finalize();
console.log(4);
const signature = hash.toString(CryptoJS.enc.Base64);
console.log(5);
// sens 서버로 요청 전송
const smsRes = await axios({
method: method,
url: url,
headers: {
"Contenc-type": "application/json; charset=utf-8",
"x-ncp-iam-access-key": sens_access_key,
"x-ncp-apigw-timestamp": date,
"x-ncp-apigw-signature-v2": signature,
},
data: {
type: "SMS",
countryCode: "82",
from: sens_call_number,
content: `쿠팡 홈쇼핑에 고객이 물건을 샀습니다.`,
messages: [{ to: `${user_phone_number}` }],
},
});
console.log("response", smsRes.data);
return res.status(200).json({
message: "SMS sent",
});
} else {
return res.status(404).json({ message: "고객이 물건을 사지 않았다" });
}
}
}
})();
});
module.exports = router;

Answer 1

ERR_CONNECTION_REFUSED means that the target site is actively denying connection from your server.
IP addresses of AWS and other large hosting providers are often banned by commercial sites to avoid scraping and botting because, let's face it, most of the times id someone visits from AWS it's probably a bot.
Try to use a more obscure provider and/or buy some proxies.