I am trying to get weather forecast data from API of weatherapi.com but when I parsed the JSON data it show error that unexpected end of json input. I also tried setTimeout function as if it takes times to fetching data but not helpful.
const express = require('express');
const https = require('https');
const bodyParser = require('body-parser');
const app = express();
app.use(express.static("public"));
app.use(bodyParser.urlencoded({extended:true}));
app.post("/weather-data",function(req, res){
var city_name = req.body.city_name;
city_name = city_name.toUpperCase();
const key = "4b6f380fa80745beb2c174529222912";
const days = 1;
url = "https://api.weatherapi.com/v1/forecast.json?key="+key+"&q="+city_name;
https.get(url,(response)=>{
console.log(response.statusCode);
const status = response.statusCode;
if(status == 200){
response.on("data",function(data){
const WeatherData = JSON.parse(data);
const region = WeatherData.location.region;
const country = WeatherData.location.country;
console.log("region is "+region+" and country is "+country);
});
}
});
});
Note that response.on("data") event is triggered every time a chunk of data arrives and it can happen multiple times per request (not necessarily all data arrive simultaneously, especially for large payloads).
You should buffer the data and parse it only after all data arrived:
let dataBuffer = '';
response.on("data", function(data) {
dataBuffer += data;
});
response.on("end", function() {
const weatherData = JSON.parse(dataBuffer);
...
...
});
Related
I want to use message.attachments to get images from that and upload those to my website (not on localhost). How would i do that?
I already have a working upload form on my website, but how would i do it from a discord.js bot?
According to the discord.js docs, each attachment has a URL property. You can use this to get the image and upload it.
Example code:
const URLsToFetch = [];
const attachments = message.attachments.array();
for(let i = 0;i<attachments.length;++i){
URLsToFetch.push(attachments[i].url);
}
This would get all the attachment URLs, which you can use http, request, or some other similar module to download it and then write it to wherever you put your uploads:
const http = require("http");
const https = require("https");
const {URL} = require("url");
const fs = require("fs");
for(let url of URLsToFetch){
const uri = new URL(url);
const protocol = uri.protocol;
let proto = http;
if(protocol === "https:"){
proto = https;
}
proto.get(uri,response=>{
const chunks = [];
response.on("data",chunk={
chunks.push(chunk);
});
response.on("end",()=>{
const file = Buffer.concat(chunks);
fs.writeFile("path/to/filename",file,err=>{
if(err){throw err} // error
// successfully wrote file
});
});
});
}
I am trying to do web scraping and i would like to display the data in JSON format.
My task is to extract each post from the website and display its relevant data in JSON format. My issue is that i cannot seem to target the row () and then target each id. I can input the id in my code but i would like for the program to seacrh for the id and console log the data of each id in the row.
Example: I want to get the title for the first post by id.
I hope i am making sense.
The website i am trying to extract data from:
My code:
var express = require('express');
var path = require('path');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var app = express();
var port = 8080;
var url= "https://news.ycombinator.com/";
request(url, function(err,resp,body){
var $ = cheerio.load(body);
var title = $('tr');
var uri
var author
var points
var comments
var rank
var posts = {
postTitle : title,
postUri : uri,
postAuthor : author,
postPoints : points,
postComments : comments,
postRank : rank
}
console.log(posts)
})
app.listen(port);
console.log('server is listening on' + port);
The trick with hackernews is that three tr elements display one row. Thats why each element of rows inherits three subsequent elements of tr. Inside rows.map each item is one row and you can access the attributes "rowwise".
let cheerio = require('cheerio')
let request = require('request');
const url = "https://news.ycombinator.com/";
request(url, function(err,resp,body){
let $ = cheerio.load(body);
const tr = $('.itemlist > tr');
let rows = Array((tr.length - 2)/3); //the last two are the More button
for (var i = 0; i < (tr.length - 2)/3; ++i){
rows[i] = tr.slice(3*i, 3*(i+1));
}
res = rows.map(function(item, index) {
return {
postTitle: $(item).find('.storylink').text(),
postUri: $(item).find('.storylink').attr('href'),
postComments: $(item).find('a+ a').text(),
}
})
console.log(res);
})
Which gives you:
[ { postTitle: 'CockroachDB beta-20161013',
postUri: 'https://jepsen.io/analyses/cockroachdb-beta-20161013',
postComments: '10 comments' },
{ postTitle: 'Attacking the Windows Nvidia Driver',
postUri: 'https://googleprojectzero.blogspot.com/2017/02/attacking-windows-nvidia-driver.html',
postComments: '7 comments' },
{ postTitle: 'DuckDuckGo Donates $300K to Raise the Standard of Trust Online',
postUri: 'https://spreadprivacy.com/2017-donations-d6e4e4230b88#.kazx95v27',
postComments: '25 comments' },
... ]
I have the following code that finds any documents missing DateUTC and loop through and set the date using a datestring value contained in the document. The data is imported via JSON files so dates need to be cast to UTC date. However i am getting 'Process Out of Memory' when there are >1000 records. The documents are quite large JSON weather readings. I am new to MongoDb and Node but this is not a lot of data to update so I figure i am doing something wrong.
'use strict';
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var db = mongoose.connection;
var Grib = require('./models/grib.js');
var fs = require('fs');
var moment = require('moment');
//register call back events
db.on('error', console.error);
db.once('open', function () {
//do our DB work here
var today = moment().startOf('day');
var tomorrow = moment(today).add(1, 'day');
var yesterday = moment(today).add(-1, 'day');
Grib.find({ 'header.dateUTC': null }, {}, {}, function (err, array) {
for (var i = 0, len = array.length; i < len; i++) {
array[i].header.dateUTC = moment.utc(array[i].header.refTime).toDate();
array[i].save(function (err) {
if (err) console.log('save error:' + err);
});
};
console.log('Number of grib dates updated:' + array.length);
});
You can always paginate. For example fetch in batches by a 1000, just keep in mind you have to sort objects for it to work.
var index = 0;
Grib.find({ 'header.dateUTC': null }).sort('created_at').skip(index).limit(1000).exec(function (err, array) {
index += 1000;
.
.
.
be aware of nodejs async nature. I would suggest to take a look at async each function.
Ok so for fun I decided to scrape all the users who go to my college who are signed up on the website moodle.
This is the program I made with Node.js and cheerio that scrapes the site, but I can not seem to get the text that is inside the H2 tag.
This is the website I am scraping from, http://moodle.ramapo.edu/user/profile.php?id=2101
All I need to do is just change the ID number and it loops through every student.
var request = require('request'),
cheerio = require('cheerio');
urls = [];
//For just single page, eventually will loop through each page.
request('http://moodle.ramapo.edu/user/profile.php?id=2101', function(err, resp, body){
if (!err && resp.statusCode == 200) {
var $ = cheerio.load(body);
$('h2.main', '#yui_3_9_1_2_1410303448188_167').each(function(){
//Not sure how to retrieve just the text name of person
});
console.log(urls);
};
});
How do I just select the text inside the H2 tag so that I can log all of them to my console?
That's not the way I'd go about it. Below is a code snippet that should help you out, all you'll need to do is wrap it in a loop and iterate through the urls you want to scrape. I'd also suggest you check out this tutorial Scraping the Web With Node.js
var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function(req, res){
url = 'http://moodle.ramapo.edu/user/profile.php?id=2101';
request(url, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
var name;
$('.main').filter(function(){
var data = $(this);
name = data.text();
console.log("name = " + name);
})
}
res.send('Check your console!')
})
})
app.listen('8081')
exports = module.exports = app;
I am creating a script using node.js,fbgraph api and express framework. I POST access_token of user from a page index.html on nodejs server. I am able to retrieve the access_token and I used the fbgraph api to retrieve further user info. But when i try to send the response Json object i am getting this error Cannot GET /.
Here are my code , I am not able to understand where is problem coming , everything seems to work. I checked other questions also , they are not helpful in my case, I dont need to show any template. I only want to return response.
NOTE: In my project folder file structure s like this :-
node_modules
app.js
package.json
CODE: app.js
var bodyParser = require('body-parser');
var express = require('express');
var graph = require('fbgraph');
var app = express();
app.use(bodyParser());
//Retrieve POST data
app.post('/', function(req, res) {
// console.log(req.body.access_token);
var access_token = req.body.access_token;
//set access token
graph.setAccessToken(access_token);
//Graph Api request
graph.get("/me?access_token="+access_token, function(err, b_res) {
// console.log(b_res)
var name = b_res.name;
var id = b_res.id;
var profileUrl = b_res.link;
//Retrieve profile url
graph.get("/"+id+"/?fields=picture", function(err, g_res) {
//JSON object to be returned
var userObj = {
"name": name,
"id": id,
"profilerl": profileUrl,
"picurl": g_res.picture.data.url
};
console.log(userObj);
res.json(userObj);
//res.send(userObj);
});
});
});
app.use(express.static(__dirname + '/'));
app.set('port', process.env.PORT || 3000);
var server = app.listen(app.get('port'));
As discussed in above comments you can perform db operations here in the same page using userObj attributes in where clause or whatever other operation you want and then pass db returned object in the view like this
var bodyParser = require('body-parser');
var express = require('express');
var graph = require('fbgraph');
var app = express();
app.use(bodyParser());
//Retrieve POST data
app.post('/', function(req, res) {
// console.log(req.body.access_token);
var access_token = req.body.access_token;
//set access token
graph.setAccessToken(access_token);
//Graph Api request
graph.get("/me?access_token="+access_token, function(err, b_res) {
// console.log(b_res)
var name = b_res.name;
var id = b_res.id;
var profileUrl = b_res.link;
//Retrieve profile url
graph.get("/"+id+"/?fields=picture", function(err, g_res) {
//JSON object to be returned
var userObj = {
"name": name,
"id": id,
"profilerl": profileUrl,
"picurl": g_res.picture.data.url
};
console.log(userObj);
//res.json(userObj);
//res.send(userObj);
//perform db operation using userObj and when you get the returned object from db pass it to the view. Let say dataAfterDbOpeations is the returned object of ur query
res.render('views/index', {data: dataAfterDbOpeations})
});
});
});
app.use(express.static(__dirname + '/'));
app.set('port', process.env.PORT || 3000);
var server = app.listen(app.get('port'));