I am starting with Mongo db. I want to insert two variables into mongo. So, in the same file
1.- I define the two variables
2.- I create a function that returns a JSON format object with the two variables
3.- I create an app in Express to fill the values of the two variables and send them out as well
4.- I connected to Mongo and insert the JSON object creating a collection and passing in the first argument a call to the function that returns the JSON object with two arguments that are the two variables.
RESULT CHECKING IN THE CONSOLE:
1.- The connection is correct
2.- There is a JSON object inserted but empty
I think I have a problem of scopes.How would it be the right sequence?
// Express files
var express = require('express');
var app = express();
// Mongo files
var mongodb=require("mongodb")
var MongoClient = mongodb.MongoClient;
var MONGODB_URI="mongodb://user:psswd#00000.mlab.com:00000/"
// Variables
var one;
var two;
// JSON object to insert in mongo
var doc=function(one,two){
return{
"one":one,
"two": two
}
}
// App in Express
app.get("new/:which",function(req,res){
one=req.params.which
var randomNum=Math.round(Math.random()*10000)
two=req.headers["x-forwarded-host"]+("/")+randomNum.toString()
res.end(JSON.stringify(doc(one,two)))
})
// Mongo connection and insertion of JSON object
MongoClient.connect(MONGODB_URI,function(err,db){
if (err) {
console.log('Unable to connect to the mongoDB server.
Error:', err);
} else {
console.log('Connection established to', MONGODB_URI);
}
var collection=db.collection("url")
collection.insert(doc(one,two),function(){
if(err) throw err
console.log(JSON.stringify(doc(one,two)))
db.close()
})
})
// Express files
var express = require('express');
var app = express();
//mongoose files
var mongoose = require('mongoose');
mongoose.Promise = require('bluebird');
mongoose.connect('mongodb://user:psswd#00000.mlab.com:00000/'), {
useMongoClient: true,
});
//Define the document schema
var Schema = new mongoose.Schema({
one: {
type: String, //or maybe Number
required: true
},
two: {
type: String,
required: true
},
});
var Model = mongoose.model('model', Schema);
app.get("/new/:which",function(req,res){
one=req.params.which
var randomNum=Math.round(Math.random()*10000)
two=req.headers["x-forwarded-host"]+ ("/")+randomNum.toString();
var new_doc = new Model({
one: one,
two: two
});
new_doc.save(err=>{
err ? res.send(err) : res.send('added!');
});
});
I recommend to use mongoose npm package for working with mongo
You can split the code into modules for more comfortability
I found the answer to the question of the scopes in a freecourse for Node developers in MongoDB University showing how to put all together Mongo University
The sequence is the following:
//Dependencies
var express = require('express');
var app = express();
var mongodb=require("mongodb")
var MongoClient = mongodb.MongoClient;
var MONGODB_URI
="mongodb://<user>:<psswd>#000000.mlab.com:41358"
// Connection to mongo
MongoClient.connect(MONGODB_URI,function(err,db){
app.use("/new/:which",function(req,res){
var one=req.params.which
var two=req.headers["x-forwarded-host"]+"/"+randomNum
// Insertion of documents
db.collection("url").insertOne({"one":one,
"two":two})
res.send({"one":one,"two":two})
})
var listener = app.listen(8000)
}
So, the app need to be within the scope of the mongo connection and the collection methods within the scope of the app.
At the same level that the app will be the port.
The documents inserted in the database and the result send it out to the client as well.
Related
I am pretty new to web scraping techniques though I already have solid knowledge in terms of PHP / HTML / CSS.
After reading a few tutorials and a lot of tries, I finally managed to scrape my first results as a test.
I use Cheerio + Node.js, and here was the code of my test:
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
request('http://www.passion-de-vin.com/contact/', function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var parsedResults = [];
$('.form-headline').filter(function(i, element) {
var a = $(this).children();
var titre = a.first().text();
release2 = titre.replace(/(\r?\n)/g, '');
release = release2.replace(/\s\s/g, '');
titre = titre;
// Our parsed meta data object
var metadata = {
titre,
};
// Push meta-data into parsedResults array
parsedResults.push(metadata);
fs.writeFile('output.json', JSON.stringify(parsedResults, null, 4), function(err){
console.log('File successfully written! - Check your project directory for the output.json file');})
});
// Log our finished parse results in the terminal
console.log(parsedResults);
}
});
I have the result log in my JSON file.
Now I would like to know and understand how I can transmit information to that form, post a result and see or get the result of the post.
So far, all I have read has been unclear to me
So I am trying my hand at Node.js. I want to build a simple crawler which scans a page and then returns all links back in a json file. However, when I run the script it returns 0 links.
Here is my code in its entirety:
var request = require('request');
var cheerio = require('cheerio');
var fs = require("fs");
var url = 'https://stackoverflow.com/questions';
//Create the blank array to fill:
var obj = {
table: []
};
var i = 0;
request(url, function(err, resp, body){
$ = cheerio.load(body);
links = $('a'); //jquery get all hyperlinks
$(links).each(function(i, link){
var actualLink = $(link).attr('href');
obj.table.push({id: i, url:actualLink}); //add some data
i++;
});
});
var json = JSON.stringify(obj);
console.log(json);
The output in the terminal is so:
$ !!
node nodetest.js
{"table":[]}
Can anyone see why this is blank? Bonus points for writing the final json to a file :)
You must use obj inside the success callback of the request, that's where it gets populated:
request(url, function(err, resp, body) {
$ = cheerio.load(body);
links = $('a'); //jquery get all hyperlinks
$(links).each(function(i, link) {
var actualLink = $(link).attr('href');
obj.table.push({id: i, url:actualLink}); //add some data
});
// Only here you can be sure that the "obj" variable is properly
// populated because that's where the HTTP request completes
var json = JSON.stringify(obj);
console.log(json);
});
In your code you have placed the console.log outside the request success which is asynchronous and thus the obj variable is not yet populated.
Also notice that you don't need the i variable. It will be passed to the each callback automatically, you don't need to be explicitly declaring or incrementing it.
As far as writing the result to a file is concerned, you could use the fs.writeFile function:
fs.writeFile("/tmp/test", json, function(err) {
if(!err) {
console.log("File successfully saved");
}
});
Trying to get sessions working using Node.js and Express for my personal project. I'm using a MySQL database as my session store, and have installed the following modules:
express
body-parser, to get POST data
mysql, to connect to my DB
cors
express-session
express-mysql-session
bcrypt, to compare POSTed data with DB hashes
I'm sending a POST request containting login info through javascript from a page, which I compare with an hash from the database. If it matches, I create the session for the user. I want to prevent the user from logging in again if it has a session, but it looks like the session cookie isn't being stored. I verified this by looking at req.session, but the user object I created never appears there.
Records in the database are being created: if I login with correct data, a new record is created. I'm not sure if this is supposed to happen but if I login again with the same user it creates a new record.
Here's what I've got:
var express = require('express');
var app = express();
var bodyParser = require('body-parser');
var mysql = require('mysql');
var cors = require('cors');
var session = require('express-session');
var MySQLStore = require('express-mysql-session')(session);
var bcrypt = require('bcrypt');
var options = { ... };
var pool = mysql.createPool(options);
var sessionConnection = mysql.createConnection(options);
var sessionStore = new MySQLStore({
expiration: 10800000,
createDatabaseTable: true,
schema: {
tableName: 'USERS_SESSIONS',
columnNames: {
session_id: 'session_id',
expires: 'expires',
data: 'data'
}
}
}, sessionConnection);
// i'll change key & secret later
app.use(session({
key: '69Atu22GZTSyDGW4sf4mMJdJ42436gAs',
secret: '3dCE84rey8R8pHKrVRedgyEjhrqGT5Hz',
store: sessionStore,
resave: false,
saveUninitialized: true
}));
app.use(cors());
...
var router = express.Router();
router.post('/login', function(req, res){
if (req.session.user){
console.log('already logged in');
}else{
// get connection from pool, retrieve user record
// PLAIN is password from request, row.HASH is the record's hash
bcrypt.compare(PLAIN, row.HASH, function(err, match){
// do error handling
// when match = true do this
req.session.user = {
id: row.ID,
nickname: row.NICK,
isAuthed: true
};
res.sendStatus(200); // else send 401
return;
});
}
});
After successfully logging in, I check my session like this:
router.get('/session', function(req, res){
res.json(req.session);
});
And I get the following:
{"cookie":{"originalMaxAge":null,"expires":null,"httpOnly":true,"path":"/"}}
As you can see there's no user object. I'm not getting any errors and I can't figure out where's the problem.
var cookieParser = require('cookie-parser');
Please add app.use(cookieParser()); before
app.use(session({
key: '69Atu22GZTSyDGW4sf4mMJdJ42436gAs',
secret: '3dCE84rey8R8pHKrVRedgyEjhrqGT5Hz',
store: sessionStore,
resave: false,
saveUninitialized: true
}));
For detailed documentation
https://www.npmjs.com/package/cookie-parser
I have the following code which accepts data from the url and print the json formatted data.I want to publish the same data to mqtt using node.js.Is there any sample code for the same?
`var request = require('request')
var JSONStream = require('JSONStream')
`var es = require('event-stream')`
`request({url: 'http://isaacs.couchone.com/registry/_all_docs'})
`.pipe(JSONStream.parse('rows.*'))
.pipe(es.mapSync(function (data) {
console.log(data);
console.error(data)
return data
}))
You could use the mqtt node library MQTT.js
Your current code becomes something like that:
var request = require('request');
var JSONStream = require('JSONStream');
var es = require('event-stream');
var mqtt = require('mqtt');
request({url: 'http://isaacs.couchone.com/registry/_all_docs'})
.pipe(JSONStream.parse('rows.*'))
.pipe(es.mapSync(function (data) {
console.log(data);
console.error(data);
//MQTT publish starts here
var client = mqtt.createClient(1883, 'localhost');
client.publish('demoTopic', JSON.stringify(data));
client.end();
return data;
}))
The above code assumes the broker is running on the local machine on port 1883.
Just use a node.js library for mqtt such as MQTT.js https://github.com/adamvr/MQTT.js
Also you can run your own multi-protocol broker in node.js by installing mosca https://github.com/mcollina/mosca
I am creating a script using node.js,fbgraph api and express framework. I POST access_token of user from a page index.html on nodejs server. I am able to retrieve the access_token and I used the fbgraph api to retrieve further user info. But when i try to send the response Json object i am getting this error Cannot GET /.
Here are my code , I am not able to understand where is problem coming , everything seems to work. I checked other questions also , they are not helpful in my case, I dont need to show any template. I only want to return response.
NOTE: In my project folder file structure s like this :-
node_modules
app.js
package.json
CODE: app.js
var bodyParser = require('body-parser');
var express = require('express');
var graph = require('fbgraph');
var app = express();
app.use(bodyParser());
//Retrieve POST data
app.post('/', function(req, res) {
// console.log(req.body.access_token);
var access_token = req.body.access_token;
//set access token
graph.setAccessToken(access_token);
//Graph Api request
graph.get("/me?access_token="+access_token, function(err, b_res) {
// console.log(b_res)
var name = b_res.name;
var id = b_res.id;
var profileUrl = b_res.link;
//Retrieve profile url
graph.get("/"+id+"/?fields=picture", function(err, g_res) {
//JSON object to be returned
var userObj = {
"name": name,
"id": id,
"profilerl": profileUrl,
"picurl": g_res.picture.data.url
};
console.log(userObj);
res.json(userObj);
//res.send(userObj);
});
});
});
app.use(express.static(__dirname + '/'));
app.set('port', process.env.PORT || 3000);
var server = app.listen(app.get('port'));
As discussed in above comments you can perform db operations here in the same page using userObj attributes in where clause or whatever other operation you want and then pass db returned object in the view like this
var bodyParser = require('body-parser');
var express = require('express');
var graph = require('fbgraph');
var app = express();
app.use(bodyParser());
//Retrieve POST data
app.post('/', function(req, res) {
// console.log(req.body.access_token);
var access_token = req.body.access_token;
//set access token
graph.setAccessToken(access_token);
//Graph Api request
graph.get("/me?access_token="+access_token, function(err, b_res) {
// console.log(b_res)
var name = b_res.name;
var id = b_res.id;
var profileUrl = b_res.link;
//Retrieve profile url
graph.get("/"+id+"/?fields=picture", function(err, g_res) {
//JSON object to be returned
var userObj = {
"name": name,
"id": id,
"profilerl": profileUrl,
"picurl": g_res.picture.data.url
};
console.log(userObj);
//res.json(userObj);
//res.send(userObj);
//perform db operation using userObj and when you get the returned object from db pass it to the view. Let say dataAfterDbOpeations is the returned object of ur query
res.render('views/index', {data: dataAfterDbOpeations})
});
});
});
app.use(express.static(__dirname + '/'));
app.set('port', process.env.PORT || 3000);
var server = app.listen(app.get('port'));