I am migrating from SQL to NOSQL but still want to maintain a separate record and use ObjectIDs to link in some cases. I have this loop which works... in part.
router.get('/col', function(req, res, next) {
var stream = Record.find().stream();
stream.on('data', function (doc) {
if(doc.oldcolid){
Collection.findOne({da: doc.oldcolid}, function(err, col){
doc.collectionId = col._id
doc.save()
console.log(col.name)
})
}
})
stream.on('error', function (err) {
console.log(err)
})
stream.on('close', function () {
console.log("done")
})
});
However, it seems to finish and miss many records.
> db.records.count({collectionId: null})
12130
> db.records.count({collectionId: {$exists: true}})
5882
> db.records.count({oldcolid: {$exists: true}})
18012
I stored the old ID from the MySQL database in both collections to do the link. There are no obvious errors but it consistently seems to fall over. I do not seem to hit the on('close') function either.
OK, so thinking about this in reverse was simpler.
var stream = Collection.find().stream();
stream.on('data', function (doc) {
console.log(doc.da)
stream.pause()
Record.update({oldcolid: doc.da}, {collectionId: doc._id}, { multi: true }, function (err, raw) {
if(err) {console.log(err)}
console.log(raw)
stream.resume()
})
})
stream.on('error', function (err) {
console.log(err)
})
stream.on('close', function () {
console.log("done")
})
Fewer operations, less trouble.
Related
I have the following mongoose "update" path:
app.put('/update', async (req, res) => {
const newTaskName = req.body.todoName
const newDays = req.body.days
const id = req.body.id
try {
await TodoModel.findById(id, async (err, updatedTodo) => {
updatedTodo.todoName = newTaskName
updatedTodo.daysToDo = newDays
await updatedTodo.save()
res.send("updated")
})
} catch(err) {
console.log(err)
}
})
Separately I have a path that returns all data from the Mongo table:
app.get('/read', async (req, res) => {
TodoModel.find({}, (err, result) => {
if (err) {
res.send(err)
}
res.send(result)
})
})
How can I both update and send back the full updated list within the response?
Separate question, not necessary to answer, but would be nice - perhaps this approach is all wrong? some background:
In my MERN app I am calling to add an item to a list and then want to immediately render the updated list as currently read from the database, since I don't want to assume the insertion was successful
I tried using some asynchronous workarounds with no luck
Fixed!
Upon further inspection of Mongoose documentation, I found that by using the findOneAndUpdate method instead of findById, I am able to utilize a callback that will return the updated item:
app.put('/update', async (req, res) => {
const id = req.body.id
let updateSet = req.body
delete updateSet.id
try {
ShoppingModel.findOneAndUpdate({ _id: id }, { $set: updateSet }, { new: true }, (err, doc) => {
if (err) return console.log(err)
res.send(doc)
})
} catch (err) {
console.log(err)
}
})
I need to save to db inside an async waterfall series.
I've tried to integrate these two function after the clean function
function connectDb(next) {
pool.getConnection(function(err, connection) {
if (err) console.log(err);
conn = connection;
}, next);
},
function saveDb(next) {
let sql = "UPDATE media SET media_url = ? WHERE media_url = ?";
conn.query(sql, [dstKey, srcKey], function (error, results, fields) {
if (error) {
conn.release();
console.log(error);
}else{
console.log("media db updated");
}
}, next)
}
The problem is that these two functions block the code execution. How can I integrate it in the function below? I've tried to wrap the function in promise but it is also not working.
async.waterfall([
function download(next) {
s3.getObject({
//param
},
next);
},
function transform(response, next) {
resizeMedia(response.Body ).then( ( file ) => { next();} ).catch( (err) => { reject(err) } ); }
},
function upload(next) {
var fileData = fs.createReadStream('/tmp/'+dstKey);
if (isVideo ) { var ContentType = 'video/mp4' }
if (isAudio ) { var ContentType = 'audio/mp3' }
s3.putObject({
//param
},
next);
},
function clean(next) {
// Stream the transformed image to a different S3 bucket.
fs.unlinkSync('/tmp/'+dstKey);
s3.deleteObject({
//param
},
next);
}
], function (err) {
if (err) {
console.error('Error');
callback(null, "Error");
return;
} else {
console.log('Success');
callback(null, "Done");
return;
}
callback(null, "Done");
return;
}
);
The purpose of async waterflow is to block the waterfall until the callback is called.
P.S. Usually you should not create a new db connection each time. The connection should be done once when the application start and get used whenever you need.
I highly recommend you to use knex.js instead, it return promises by default and if you like to use it inside async waterfall (and wait for resolve) you can call .asCallback.
I've found the problem, if someone fall into the same issue here my solution:
If a waterfall function has a response, this response is automatically added as first argument in the next function. In my code the mistake was simple (after night's sleep), the s3.deleteObject and s3.putObject has response, this response need to be setted as first argument and the callback as last, as you say I've used only the callback as argument (next) and this broke my code.
[...]
function upload(next) {
s3.putObject({
//param
},
next);
},
function clean(response, next) { // response in arguments
s3.deleteObject({
//param
},
next);
}
[...]
I'm trying to use a SQL query with NodeJs but the async part really mess me up. When i print the return it gets "undefined". How can i sync this?
function SQL_test(blos) {
DB.query("Select profesor_id from materiador where materia_id = '"+blos+"';",function (err, results, fields) {
return results;
});
}
console.log(SQL_test(1));
Thanks!
So your answer is currently a promise. You'll have to read about Async and Await to do more synchronous JS.
Most of the JS for NodeJS is currently async. Below is a rewritten version of your example properly utilizing the callback method for your DB.
function callback (err, results, fields) {
if (err) {
console.log(err);
return err;
}
console.log(results, fields);
return results;
};
function SQL_test(blos) {
DB
.query("Select profesor_id from materiador where materia_id = '"+blos+"';", callback);
}
SQL_test(1);
To do the same thing synchronously you have to still have an outer level promise, otherwise Async and Await won't work how you want it to. There's no true synchronous way to handle this is javascript because it executes without waiting for the response.
function sqlTest(blos) {
return new Promise((resolve, reject) => {
DB.query(
`Select profesor_id from materiador where materia_id = '${blos}';`,
(err, results) => {
if (err) return reject(err)
resolve(results)
}
);
}
)
sqlTest(1)
.then(console.log)
.catch(console.error)
From my Model, I fetch some articles from a MySQL database for a user.
Model
var mysql = require('mysql');
var db = mysql.createPool({
host: 'localhost',
user: 'sampleUser',
password: '',
database: 'sampleDB'
});
fetchArticles: function (user, callback) {
var params = [user.userId];
var query = `SELECT * FROM articles WHERE userId = ? LOCK IN SHARE MODE`;
db.getConnection(function (err, connection) {
if (err) {
throw err;
}
connection.beginTransaction(function (err) {
if (err) {
throw err;
}
return connection.query(query, params, function (err, result) {
if (err) {
connection.rollback(function () {
throw err;
});
}
//console.log(result);
});
});
});
}
This is working and the function fetches the result needed. But it's not returning the result to the controller function (I am returning it but I'm not able to fetch it in the controller function. I guess, I did something wrong here).
When I did console.log(result) this is what I got.
[ RowDataPacket {
status: 'New',
article_code: 13362,
created_date: 2017-10-22T00:30:00.000Z,
type: 'ebook'} ]
My controller function looks like this:
var Articles = require('../models/Articles');
exports.getArticle = function (req, res) {
var articleId = req.body.articleId;
var article = {
userId: userId
};
Articles.fetchArticles(article, function (err, rows) {
if (err) {
res.json({ success: false, message: 'no data found' });
}
else {
res.json({ success: true, articles: rows });
}
});
};
Can anyone help me figure out what mistakes I made here?
I'm pretty new to nodejs. Thanks!
The simple answer is that you're not calling the callback function, anywhere.
Here's the adjusted code:
fetchArticles: function (user, callback) {
var params = [user.userId];
var query = `SELECT * FROM articles WHERE userId = ? LOCK IN SHARE MODE`;
db.getConnection(function (err, connection) {
if (err) {
// An error. Ensure `callback` gets called with the error argument.
return callback(err);
}
connection.beginTransaction(function (err) {
if (err) {
// An error. Ensure `callback` gets called with the error argument.
return callback(err);
}
return connection.query(query, params, function (err, result) {
if (err) {
// An error.
// Rollback
connection.rollback(function () {
// Once the rollback finished, ensure `callback` gets called
// with the error argument.
return callback(err);
});
} else {
// Query success. Call `callback` with results and `null` for error.
//console.log(result);
return callback(null, result);
}
});
});
});
}
There's no point in throwing errors inside the callbacks on the connection methods, since these functions are async.
Ensure you pass the error to the callback instead, and stop execution (using the return statement).
One more thing, without knowing the full requirements of this:
I'm not sure you need transactions for just fetching data from the database, without modifying it; so you can just do the query() and skip on using any beginTransaction(), rollback() and commit() calls.
Some time ago I decided to switch from PHP to node. In my first projects I didn't want to use any ORM since I thought that I didn't need to complicate my life so much learning another thing (at the moment I was learning node and angular) therefor I decided to use mysql package without anything else. It is important to say that I have some complex queries and I didn't want to learn from sctratch how to make them work using one of the 9000 ORM node have, This is what I've been doing so far:
thing.service.js
Thing.list = function (done) {
db.query("SELECT * FROM thing...",function (err,data) {
if (err) {
done(err)
} else {
done(null,data);
}
});
};
module.exports = Thing;
thing.controler.js
Thing = require('thing.service.js');
Thing.list(function (err,data) {
if (err) {
res.status(500).send('Error D:');
} else {
res.json(data);
}
});
how can I promisify this kind of functions using bluebird ? I've already tried but .... here I am asking for help. This is what I tried
var Thing = Promise.promisifyAll(require('./models/thing.service.js'));
Thing.list().then(function(){})
I have done this way and it is working fine.
const connection = mysql.createConnection({.....});
global.db = Bluebird.promisifyAll(connection);
db.queryAsync("SELECT * FROM users").then(function(rows){
console.log(rows);});
I have never had much luck with promisifyAll and IMO I prefer to handle my internal checks manually. Here is an example of how I would approach this:
//ThingModule
var Promises = require('bluebird');
Things.list = function(params) {
return new Promises(function(resolve, reject) {
db.query('SELECT * FROM thing...', function(err, data) {
return (err ? reject(err) : resolve(data));
});
});
}
//usage
var thinger = require('ThingModule');
thinger.list().then(function(data) {
//do something with data
})
.error(function(err) {
console.error(err);
})
You can also create a function that fires SQL like this :-
function sqlGun(query, obj, callback) {
mySQLconnection.query(query, obj, function(err, rows, fields) {
if (err) {
console.log('Error ==>', err);
// throw err;
return (err, null);
}
console.log(query)
if (rows.length) {
return callback(null, rows);
} else {
return callback(null, [])
}
});
}
Where mySQLconnection is the connection object you get after mysql.createConnection({}).
After that, you can promisify the function and use the promise like below :-
var promisified = Promise.promisify(sqlGun);
promisified(query, {}).then( function() {} );