I have an array of (few thousands of) data object that I need to insert or update depending on a condition. A simple upsert() method is implemented in my model class.
Implementation
csv.upsert = async function (values, condition) {
let obj = await csv.findOne({ where: condition });
if (obj) {
// update
console.log("existing record updated")
return await obj.update(values);
} else {
// insert
console.log("new record inserted")
return await csv.create(values);
}
}
This upsert method is then used, where I loop through the array of objects to insert or update them in db.
Usage
try {
await models.sequelize.authenticate();
await models.sequelize.sync();
let dataToBeInserted = getArrayOfDatatoInsert();
dataToBeInserted.map(async function (data) {
let condition = {
'categorygroup': data.categorygroup,
'category': data.category,
'country': data.country,
'city': data.city
};
await csvModel.upsert(data, condition);
})
// await restofthestuff();
} catch (error) {
console.log("error", error);
}
For test I took a dataset where all of my data needs to be updated.
When I run this method:
I can see in the (along with sequelize log turned on) log that "existing record updated" message is printed for each and every record that exists which is desired output. Only the last few (30) data gets updated in the db. Where as it works for csv.create(values)
~ How can I update all the records and obviously not just the last 30 data, any help's appreciated. ~
EDIT: Apparently I got this to work by using csv.update(values, {where: condition}) instead of using obj.update(values).
New question: I didn't look further into the sequelize's update method but is this a bug or am I doing something wrong here?
As detailed in the commented code below, your log is after your return and so will never be executed.
Also you were not using await in an async function, so either don't make it async or use await.
csv.upsert = async function (values, condition) {
const obj = await csv.findOne({ where: condition });
// you can await here, no need for then as you are in an async function
if (obj) { // update
// you need to log before your return condition
console.log("existing record updated")
return obj.update(values);
// since we return here, the rest of the code will not be executed
// you can skip the use of else
}
// we are in this part of the code only if object is falsy
// insert
console.log("new record inserted")
return csv.create(values);
}
You could also use Promise.all to ensure all the upsert are done:
await Promise.all(dataToBeInserted.map(async function (data) {
let condition = {
'categorygroup': data.categorygroup,
'category': data.category,
'country': data.country,
'city': data.city
};
await csvModel.upsert(data, condition);
}))
this will also ensure that if an error occur it gets catched by your try / catch
Maybe that will help you find what's causing the unexpected behaviours.
I'm using knex in loopback for DB operation with mysql.
My task is to update the 2 table by using the transaction.
When I enter new entry in one tabe, i want to use id of that entry for 2nd query operation.
But when transaction throw the error it not rolling back the data/ removing the first table entry if second table entry throws error. but in my case transaction always do commit not rollback i put my example code in below:
addTest : (data) => {
return new promise(function(resolve, reject) {
knex.transaction(function(t) {
return knex('foo')
.transacting(t)
.insert({
foo_id: data.foo_id ? data.foo_id : null,
foo_name: data.foo_name ? data.foo_name : null,
date_entered : new Date()
})
.then(function() {
return knex('bar')
.transacting(t)
.insert({
bar_id: data.bar_id ? data.bar_id : null,
bar_name : data.bar_name ? data.bar_name : null
})
})
.then(t.commit)
.catch(function(e) {
t.rollback();
throw e;
})
})
.then(function() {
// it worked
// resolve('sucess');
console.log('success');
})
.catch(function(e) {
// it failed
console.log('error'+e);
});
});
}
please, provide me suitable suggestion.
thank you
You can avoid having to call t.commit or t.rollback youself. See the docs.
Make your code inside the transaction function something like this
return t.insert({}).into('foo').returning('id')
.then( function(idArray) {
return t.insert({fooId: idArray[0]}).into('bar')
})
That lets knex handle the commiting and rolling back itself based on the result result of that promise. Also, note how I got the inserted fooId and applied it to the bar object for insertion. That was kind of mentioned in the question.
There is a field called counter in a model and whenever if I call a custom remote method like
Request URL
http://loopback:3000/api/models/increment_counter
Request body
EMPTY
Response
{
"counter" : [Value after Increment]
}
Currently to increment First i have to get the counter value from db and increment it one and update the couter value, This involves two queries, Is it possible to do it in a single NodeAPI call like the below mysql query.I am currently using Mysql as DB.
mysql_query("
UPDATE model
SET counter = counter + 1
WHERE model_id = '1'
");
Thank you
Given the MySQL syntax you want, you seem to need an atomic counter.
database transactions
With the MySQL connector, you can use database transactions. It is supported by the MySQL connector. It's a bit overkill for just atomic counters but it will get the job done.
Here is an example, inside a custom remote method
MyModel.someRemoteMethodForIncrementing = function(callback) {
// Start the transaction
MyModel.beginTransaction({
isolationLevel: MyModel.Transaction.READ_COMMITTED
}, function(err, tx) {
// Retrieve the current counter value as part of the transaction
MyModel.findById(id, {
transaction: tx
}, function(err, data) {
if (err) {
console.log(err);
return tx.rollback(function(err) {
callback(err);
});
}
// Increment the counter as part of the transaction
var inc = data.counter + 1;
MyModel.updateAttributes({
counter: inc
}, {
transaction: tx
}, function(err, newData) {
if (err) {
console.log(err);
return tx.rollback(function(err) {
callback(err);
});
}
// Commit the transaction to make it happen
tx.commit(function(err) {
if (err) return callback(err);
// Counter should have been incremented
callback();
});
});
});
});
};
I haven't tested it but it should work, let me know
extented operators
In the future, you will be able to use the $inc (increment) extended operator but so far it's only supported by MongoDB connector, not MySQL.
Just for reference, here is the syntax (works only with MongoDB)
PUT api/Model/:id?filter={"$inc":{"name":propertyToIncrement, "count":incrementAmount}}
There is an ongoing PR that I am trying to get landed to get MySQL support, but there are many things to be done before it can get merged.
Yes you can do it in a single loopback remote method call. Assume you are sending an Id in your request
yourModel.remoteMethod = function(data, cb){
yourModel.findById(data.id, function(err, object){
if(err){
cb(err, null);
}
object.counter += 1;
object.save(function(saveErr, afterSaveObj){
cb(saveErr, afterSaveObj.counter);
})
});
}
here cb is a callback which loopback passes to your remoteMethod.
I am using sequelize with mysql in my project. The current use case is about populating a table with 'roles_master' with some default roles.
The function createDefaultRoles implementation checks:
if there are rows in the table
if yes, and force is true, deletes existing rows and inserts new rows
if table empty, directly inserts new rows
My code is as below...
/**
* Creates a pre-defined set of roles
* If roles table exists and already contains data,
* this function simply returns, without creating any roles.
* The list of roles is defined in the file "ecp_db_defaults.js"
*
* #param force -- boolean flag to indicate whether to drop existng roles and recreate afresh
**/
function createDefaultRoles(force, callback) {
console.log("Executing createDefaultRoles...");
db.Role.count().then(function(result) {
if (result>0) {
if (force) {
console.log("Emptying the roles table...");
db.Role.destroy({force:true, where: {}}).then(function() {
db.Role.bulkCreate(dbConfig.roles).done(function() {
console.log("1. Successfully created user roles.");
if (callback) return callback(null); else return;
}).catch(function(e){
console.log("Error while creating roles...", e);
if (callback) return callback(e);
});
}).catch(function(e) {
console.error("Error while destroying roles table.", e);
return (callback)?callback(e):e;
});
}
//if (callback) return callback(null); else return;
} else {
db.Role.bulkCreate(dbConfig.roles).done(function(){
console.log("2. Successfully created user roles.");
if (callback) return callback(null);
})
}
}).catch(function(e){
console.error("Error while querying roles table.", e);
if (callback) return callback(e);
});
}
The problem is that, it works partially. While, it is able to delete the existing rows and add new ones, it also throws an exception which is caught and printed as below..
Executing createDefaultRoles...
Emptying the roles table... Error while destroying roles table. [TypeError: Cannot read property 'catch'of undefined]
Successfully created user roles.
Finally, the function fails, because of the above error. I have tried matching all .then() and .catch() promises correctly. They seem to be fine.
Any help in solving this is highly appreciated.
I guess the problem is in the block
db.Role.bulkCreate(dbConfig.roles).done(function() {
console.log("1. Successfully created user roles.");
if (callback) return callback(null); else return;
}).catch(function(e){
console.log("Error while creating roles...", e);
if (callback) return callback(e);
});
Since .done() returns undefined you cannot append a 'catch' on this.
If you replace .done with .then it should work as expected
I'm using Sequelize in my Nodejs project and I found a problem that I'm having a hard time to solve.
Basically I have a cron that gets an array of objects from a server than inserts it on my database as a object ( for this case, cartoons ). But if I already have one of the objects, I have to update it.
Basically I have a array of objects and a could use the BulkCreate() method. But as the Cron starts again, it doesn't solve it so I was needing some sort of update with an upsert true flag. And the main issue: I must have a callback that fires just once after all these creates or updates. Does anyone have an idea of how can I do that? Iterate over an array of object.. creating or updating it and then getting a single callback after?
Thanks for the attention
From the docs, you don't need to query where to perform the update once you have the object. Also, the use of promise should simplify callbacks:
Implementation
function upsert(values, condition) {
return Model
.findOne({ where: condition })
.then(function(obj) {
// update
if(obj)
return obj.update(values);
// insert
return Model.create(values);
})
}
Usage
upsert({ first_name: 'Taku' }, { id: 1234 }).then(function(result){
res.status(200).send({success: true});
});
Note
This operation is not atomic.
Creates 2 network calls.
which means it is advisable to re-think the approach and probably just update values in one network call and either:
Look at the value returned (i.e. rows_affected) and decide what to do.
Return success if update operation succeeds. This is because whether the resource exists is not within this service's responsibility.
You can use upsert
It's way easier.
Implementation details:
MySQL - Implemented as a single query INSERT values ON DUPLICATE KEY UPDATE values
PostgreSQL - Implemented as a temporary function with exception handling: INSERT EXCEPTION WHEN unique_constraint UPDATE
SQLite - Implemented as two queries INSERT; UPDATE. This means that the update is executed regardless of whether the row already
existed or not
MSSQL - Implemented as a single query using MERGE and WHEN (NOT) MATCHED THEN Note that SQLite returns undefined for created, no
matter if the row was created or updated. This is because SQLite
always runs INSERT OR IGNORE + UPDATE, in a single query, so there
is no way to know whether the row was inserted or not.
Update 07/2019 now with async/await
async function updateOrCreate (model, where, newItem) {
// First try to find the record
const foundItem = await model.findOne({where});
if (!foundItem) {
// Item not found, create a new one
const item = await model.create(newItem)
return {item, created: true};
}
// Found an item, update it
const item = await model.update(newItem, {where});
return {item, created: false};
}
I liked the idea of Ataik, but made it a little shorter:
function updateOrCreate (model, where, newItem) {
// First try to find the record
return model
.findOne({where: where})
.then(function (foundItem) {
if (!foundItem) {
// Item not found, create a new one
return model
.create(newItem)
.then(function (item) { return {item: item, created: true}; })
}
// Found an item, update it
return model
.update(newItem, {where: where})
.then(function (item) { return {item: item, created: false} }) ;
}
}
Usage:
updateOrCreate(models.NewsItem, {slug: 'sometitle1'}, {title: 'Hello World'})
.then(function(result) {
result.item; // the model
result.created; // bool, if a new item was created.
});
Optional: add error handling here, but I strongly recommend to chain all promises of one request and have one error handler at the end.
updateOrCreate(models.NewsItem, {slug: 'sometitle1'}, {title: 'Hello World'})
.then(..)
.catch(function(err){});
This might be an old question, but this is what I did:
var updateOrCreate = function (model, where, newItem, onCreate, onUpdate, onError) {
// First try to find the record
model.findOne({where: where}).then(function (foundItem) {
if (!foundItem) {
// Item not found, create a new one
model.create(newItem)
.then(onCreate)
.catch(onError);
} else {
// Found an item, update it
model.update(newItem, {where: where})
.then(onUpdate)
.catch(onError);
;
}
}).catch(onError);
}
updateOrCreate(
models.NewsItem, {title: 'sometitle1'}, {title: 'sometitle'},
function () {
console.log('created');
},
function () {
console.log('updated');
},
console.log);
User.upsert({ a: 'a', b: 'b', username: 'john' })
It will try to find record by hash in 1st param to update it, if it will not find it - then new record will be created
Here is example of usage in sequelize tests
it('works with upsert on id', function() {
return this.User.upsert({ id: 42, username: 'john' }).then(created => {
if (dialect === 'sqlite') {
expect(created).to.be.undefined;
} else {
expect(created).to.be.ok;
}
this.clock.tick(1000);
return this.User.upsert({ id: 42, username: 'doe' });
}).then(created => {
if (dialect === 'sqlite') {
expect(created).to.be.undefined;
} else {
expect(created).not.to.be.ok;
}
return this.User.findByPk(42);
}).then(user => {
expect(user.createdAt).to.be.ok;
expect(user.username).to.equal('doe');
expect(user.updatedAt).to.be.afterTime(user.createdAt);
});
});
Sound likes you want to wrap your Sequelize calls inside of an async.each.
This can be done with the custom event emitter.
Assuming your data is in a variable called data.
new Sequelize.Utils.CustomEventEmitter(function(emitter) {
if(data.id){
Model.update(data, {id: data.id })
.success(function(){
emitter.emit('success', data.id );
}).error(function(error){
emitter.emit('error', error );
});
} else {
Model.build(data).save().success(function(d){
emitter.emit('success', d.id );
}).error(function(error){
emitter.emit('error', error );
});
}
}).success(function(data_id){
// Your callback stuff here
}).error(function(error){
// error stuff here
}).run(); // kick off the queries
you can use findOrCreate and then update methods in sequelize. here is a sample with async.js
async.auto({
getInstance : function(cb) {
Model.findOrCreate({
attribute : value,
...
}).complete(function(err, result) {
if (err) {
cb(null, false);
} else {
cb(null, result);
}
});
},
updateInstance : ['getInstance', function(cb, result) {
if (!result || !result.getInstance) {
cb(null, false);
} else {
result.getInstance.updateAttributes({
attribute : value,
...
}, ['attribute', ...]).complete(function(err, result) {
if (err) {
cb(null, false);
} else {
cb(null, result);
}
});
}
}]
}, function(err, allResults) {
if (err || !allResults || !allResults.updateInstance) {
// job not done
} else {
// job done
});
});
Here is a simple example that either updates deviceID -> pushToken mapping or creates it:
var Promise = require('promise');
var PushToken = require("../models").PushToken;
var createOrUpdatePushToken = function (deviceID, pushToken) {
return new Promise(function (fulfill, reject) {
PushToken
.findOrCreate({
where: {
deviceID: deviceID
}, defaults: {
pushToken: pushToken
}
})
.spread(function (foundOrCreatedPushToken, created) {
if (created) {
fulfill(foundOrCreatedPushToken);
} else {
foundOrCreatedPushToken
.update({
pushToken: pushToken
})
.then(function (updatedPushToken) {
fulfill(updatedPushToken);
})
.catch(function (err) {
reject(err);
});
}
});
});
};
2022 update:
You can use the upsert function:
https://sequelize.org/api/v6/class/src/model.js~model#static-method-upsert
Insert or update a single row. An update will be executed if a row which matches the supplied values on either the primary key or a unique key is found. Note that the unique index must be defined in your sequelize model and not just in the table. Otherwise you may experience a unique constraint violation, because sequelize fails to identify the row that should be updated.
Implementation details:
MySQL - Implemented with ON DUPLICATE KEY UPDATE`
PostgreSQL - Implemented with ON CONFLICT DO UPDATE. If update data contains PK field, then PK is selected as the default conflict key.
Otherwise first unique constraint/index will be selected, which can satisfy conflict key requirements.
SQLite - Implemented with ON CONFLICT DO UPDATE
MSSQL - Implemented as a single query using MERGE and WHEN (NOT) MATCHED THEN
Note that Postgres/SQLite returns null for created, no matter if the row was created or updated