I want to scrape all element contents(every text between every tag) of a html webpage and want them to be stored in an array. I made some trials as below but it doesn't' seem to work. Can anyone help me to do this?
const request = require('request');
const cheerio = require('cheerio');
const url3 = 'https://www.amazon.com';
const array = [];
request({url: url3, json: true }, (err, res, body) => {
if (err)
{ return console.log(err); }
const $ = cheerio.load(body);
$('*').each(function(i, element){
array.push($(this).text());
});
console.log(array)
});
Related
I need help to convert a buffer to a base64 using Node and React.
I'm posting an image with an input to my databse and I'm not sure I'm doing it right. This is my first time doing this. Is there something missing here? Back-end POST request:
app.post("/api/img", (req, res) => {
const productimg = req.body.productimg;
const sqlInsert = "INSERT INTO users_files (file_src) VALUES (?)";
db.query(sqlInsert, [productimg], (err, result) => {
console.log(result);
})
})
In the database the image is a blob, and then with the GET request I get the image as a buffer. This is the GET request in the front-end:
useEffect(() => {
Axios.get('http://localhost:3001/api/get/img').then((base64String) => {
setBackenddata(base64String.data)
var blob = new Blob(productImg[0]?.file_src, {
type: "image/jpg",
});
var reader = new FileReader();
reader.readAsDataURL(blob);
reader.onloadend = function () {
let base64String = reader.result;
setProductImg(base64String);
};
console.log(base64String)
})
}, [])
And this is the get function in the back-end:
app.get('/api/get/img', (req, res) => {
const sqlGet = "SELECT * FROM users_files";
db.query(sqlGet, (err, result) => {
res.send(result);
})
})
This is what I get from the base64String:
The base64String does not seem to do anything, since it is still type: buffer.
In the back end I have tried with a Multer, but that did not work. I've been trying with different things in the back-end and the front-end, but since I'm new to this it is hard to know what's wrong.
I am new to programming...still self learning. I have below problem
I have below code in x.js file which I wish to reuse in same x.js file as well as in y.js file. both x.js and y.js are in controllers folder in a node.js project. The code is
''''''''
let indexmain = function(){
async.parallel({branch_count(callback) {
Branch.countDocuments({}, callback); documents of this collection},
customer_instance_count(callback) {
CustInstance.countDocuments({}, callback);
},
customer_instance_available_count(callback) {
CustInstance.countDocuments({ status:'Interested' }, callback);
},
}
);
return [branch_count, customer_instance_count, customer_instance_available_count];
''''''''
When used in below code inthe same file:
''''''''
const { body, validationResult } = require("express-validator");
const Branch = require('../models/brdetails');
const Customer = require('../models/custdetails');
const CustInstance = require('../models/custInstance');
const async = require('async');
exports.index = (req,res) => { (
branch_count = Indexmain[0],
customer_instance_count = Indexmain[1],
customer_instance_available_count = Indexmain[2]);
(err, results, next) => { if (err) { return next(err); }
res.render('index.jade', {error: err, data: results });
}
};
'''
There is no error, but page does not load at all. When checked with
'''
console.log("Indexmain is", Indexmain[0]);
'''
Output is "Indexmain is undefined".
What is to be done? Please help.
I am trying to generate a PDF file using the data stored in the Mysql database using Node js, Pdfkit and pdfkit-table. I need to print the records in database to a table in the PDF document.
The below code generates an empty PDF file. Please help me to solve the problem of why it does not generate a PDF file with data.
This is index.js file.
var express = require('express');
var router = express.Router();
var PDFDocument = require('pdfkit');
var orm = require('orm');
var PDFDoc = require("pdfkit-table");
router.use(orm.express("mysql://root:#localhost:/kirula_fashion", {
define: function (db, models, next) {
models.news = db.define("ledger", {
id : String,
date : String,
description : String,
debit : String,
credit : String,
});
next();
}
}));
router.get('/', function(req, res, next) {
var result = req.models.news.find({
}, function(error, news){
if(error) throw error;
res.render('index', { news:news, title: 'Generate PDF using NodeJS'
});
});
});
router.get('/pdf', function(req, res, next) {
var id = req.query.id;
const doc = new PDFDocument();
const docTable = new PDFDoc();
var result = req.models.news.find({id: id}, function(error, newspost){
if(error) throw error;
else{
if(newspost.length>0){
for(var i=0; i<newspost.length;i++){
var date = newspost[0]['date'];
var description = newspost[0]['description'];
var debit = newspost[0]['debit'];
var credit = newspost[0]['credit'];
var table = {
title: "Ledger records",
subtitle: "May - 2020",
headers: [
{ "label":"Date", "property":"date", "width":100 },
{ "label":"Description", "property":"description", "width":100 },
{ "label":"Debit", "property":"debit", "width":100 },
{ "label":"Credit", "property":"credit", "width":100 }
],
datas:
[
{ "date":date, "description":description, "debit":debit, "credit":credit},
{
"renderer": "function(value, i, irow){ return value + `(${(1+irow)})`; }"
}
],
};
docTable.table( table, {
width: 300,
});
}
}
}
var title = "Ledger for May 2020";
var filename = encodeURIComponent(title) + '.pdf';
res.setHeader('Content-disposition', 'attachment; filename="' + filename + '"');
res.setHeader('Content-type', 'application/pdf');
doc.pipe(res);
doc.end();
});
});
module.exports = router;
I encounter the same issue with datas options, however with rows options pdfkit-table work nicely, maybe mapping from [{..},{...}] to [[..],[...]] then use rows option
I'm working on a project right now, i need to create a qrcode that contains a specific information in NodeJS. I have started by creating the canvas in HTML, and take it in NodeJS
<canvas id="canvas" width="300" height="300"></canvas>
And then in my NodeJS file, i'm launching my function
const fs = require('fs');
const qrcode = require('qrcode');
module.exports = {
generateQr: function(link){
var canvas = new qrcode(document.getElementById('canvas'));
qrcode.toCanvas(canvas, link, function (error) {
if (error) console.error(error)
console.log('success!');
});
}
};
Unfortunately, i got the error :
ReferenceError: document is not defined
From the above code, does it look correct ? Does the Qrcode get the data i'm passing, and then what should i do so the QR-code appear in my HTML ?
Thank you for your help
document is a browser-specific global object, you can't access it in node
In node environment, you could generate an image with QR code and use it.
Example:
Async/Await style:
const fs = require('fs');
const qrcode = require('qrcode');
module.exports = {
generateQr: async link => {
const qrCodeDataUrl = await qrcode.toDataURL(link);
// ...
}
};
Promise style:
const fs = require('fs');
const qrcode = require('qrcode');
module.exports = {
generateQr: link => {
qrcode.toDataURL(link)
.then(data => {
const qrCodeDataUrl = data;
// ...
})
.catch(err => {
console.error(error);
// ...
});
}
};
Callback style:
const fs = require('fs');
const qrcode = require('qrcode');
module.exports = {
generateQr: link => {
qrcode.toDataURL(link, function(err, data) {
if (error) console.error(error);
const qrCodeDataUrl = data;
// ...
});
}
};
To render it in an HTML-file you could use template-engine of your choice:
Example with ejs:
const ejs = require('ejs');
// In some route
const qrCodeDataUrl = await generateQr('some link');
const html = ejs.render('<img src="<%= qrCodeDataUrl %>" />', {qrCodeDataUrl});
res.header('Content-Type', 'text/html');
res.send(html);
// ...
Note: It's a simplified example. Please check ejs docs for more details
I would like to scrape this page: calendar events
for specific data, like formattedDate and description. How do I go about that in a module in Node.JS. I am having a hard time understanding the process in Node.JS.
Any help would go a long way, thanks in advance.
it's pretty simple, you can import the request module and use it. For example, see code below.
const request = require("request");
request("MY_URL", (error, response, body) => {
console.log('body:', body);
});
Also, you can try this here, on Repl.it
First of all, you need to parse your JSON, this allows you to access fields from received json.
const data = JSON.parse(body);
Now, if you want to access some information about an event you need to loop events and access what you need, something like:
const events = data.bwEventList.events;
events.map((data, index) => console.log(data.calendar))
Final code also on Repl.it
from nodeJS docs here
const http = require('http');
http.get('http://umd.bwcs-hosting.com/feeder/main/eventsFeed.do?f=y&sort=dtstart.utc:asc&fexpr=(categories.href!=%22/public/.bedework/categories/sys/Ongoing%22%20and%20categories.href!=%22/public/.bedework/categories/Campus%20Bulletin%20Board%22)%20and%20(entity_type=%22event%22%7Centity_type=%22todo%22)&skinName=list-json&count=30', (res) => {
const { statusCode } = res;
const contentType = res.headers['content-type'];
let error;
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
return;
}
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
console.log(parsedData["bwEventList"]["resultSize"]);
} catch (e) {
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
see console.log(parsedData["bwEventList"]["resultSize"]);
slice parsedData as an array until you get what you want