I'm using knex in loopback for DB operation with mysql.
My task is to update the 2 table by using the transaction.
When I enter new entry in one tabe, i want to use id of that entry for 2nd query operation.
But when transaction throw the error it not rolling back the data/ removing the first table entry if second table entry throws error. but in my case transaction always do commit not rollback i put my example code in below:
addTest : (data) => {
return new promise(function(resolve, reject) {
knex.transaction(function(t) {
return knex('foo')
.transacting(t)
.insert({
foo_id: data.foo_id ? data.foo_id : null,
foo_name: data.foo_name ? data.foo_name : null,
date_entered : new Date()
})
.then(function() {
return knex('bar')
.transacting(t)
.insert({
bar_id: data.bar_id ? data.bar_id : null,
bar_name : data.bar_name ? data.bar_name : null
})
})
.then(t.commit)
.catch(function(e) {
t.rollback();
throw e;
})
})
.then(function() {
// it worked
// resolve('sucess');
console.log('success');
})
.catch(function(e) {
// it failed
console.log('error'+e);
});
});
}
please, provide me suitable suggestion.
thank you
You can avoid having to call t.commit or t.rollback youself. See the docs.
Make your code inside the transaction function something like this
return t.insert({}).into('foo').returning('id')
.then( function(idArray) {
return t.insert({fooId: idArray[0]}).into('bar')
})
That lets knex handle the commiting and rolling back itself based on the result result of that promise. Also, note how I got the inserted fooId and applied it to the bar object for insertion. That was kind of mentioned in the question.
Related
I am using knex 0.19.4 in node js 10.x. I have 2 SQL statements - Insert and Update which has to happen as a transaction.
// Using Var, so that below code has access to this variable
var sqlStageInsert = kx('stage').insert({
officeid: 'OFF000',
taskid: 'T002',
});
var sqlTaskPcUpdate = kx('task')
.update({ pc: 100})
.where('task.taskno', taskno)
.limit(1);
1st Try - 2nd SQL sqlTaskPcUpdate Not getting Executed
const sqlUpdateInsert = kx.transaction(function (trx) {
sqlStageInsert.transacting(trx)
.then(function () {
console.log(sqlTaskPcUpdate.toString()); // This is outputing correct SQL
return sqlTaskPcUpdate.transacting(trx);
})
.then(trx.commit)
.catch(trx.rollback);
});
await sqlUpdateInsert;
2nd Try - Getting error Transaction query already complete. This is based on Commit/rollback a knex transaction using async/await
await kx.transaction(async (trx) => {
try {
await sqlStageInsert.transacting(trx);
await sqlTaskPcUpdate.transacting(trx);
trx.commit();
} catch (error) {
trx.rollback();
throw error;
}
});
I would suggest you to try inserting the data in stage table first and then retrieve a common value which belongs to both the table for applying in where class of updating task table(Assuming both the table contain any one common column having same data).
Please note that as per knexjs.org website, knex.transaction() uses returning statement with respect to PostgreSQL to insert/ update more than one table to maintain consistency, however MySQL won't support transaction due to which I'm using return values in below code.
FYR, http://knexjs.org/#Transactions
Please refer below code snippet for your reference :
db.transaction(trx => {
trx.insert({
officeid: 'OFF000',
taskid: 'T002'
})
.into('stage')
.then(() => {
return trx('stage').where({taskid}).then(resp => resp[0].taskno)
})
.then(stageTaskno => {
return trx('task')
.update({pc: 100})
.where({stageTaskno})
.limit(1)
.then(resp => {
resp.json(resp)
})
})
.then(trx.commit)
.catch(trx.rollback)
});
hope this is helpful, cheers!
I have an array of (few thousands of) data object that I need to insert or update depending on a condition. A simple upsert() method is implemented in my model class.
Implementation
csv.upsert = async function (values, condition) {
let obj = await csv.findOne({ where: condition });
if (obj) {
// update
console.log("existing record updated")
return await obj.update(values);
} else {
// insert
console.log("new record inserted")
return await csv.create(values);
}
}
This upsert method is then used, where I loop through the array of objects to insert or update them in db.
Usage
try {
await models.sequelize.authenticate();
await models.sequelize.sync();
let dataToBeInserted = getArrayOfDatatoInsert();
dataToBeInserted.map(async function (data) {
let condition = {
'categorygroup': data.categorygroup,
'category': data.category,
'country': data.country,
'city': data.city
};
await csvModel.upsert(data, condition);
})
// await restofthestuff();
} catch (error) {
console.log("error", error);
}
For test I took a dataset where all of my data needs to be updated.
When I run this method:
I can see in the (along with sequelize log turned on) log that "existing record updated" message is printed for each and every record that exists which is desired output. Only the last few (30) data gets updated in the db. Where as it works for csv.create(values)
~ How can I update all the records and obviously not just the last 30 data, any help's appreciated. ~
EDIT: Apparently I got this to work by using csv.update(values, {where: condition}) instead of using obj.update(values).
New question: I didn't look further into the sequelize's update method but is this a bug or am I doing something wrong here?
As detailed in the commented code below, your log is after your return and so will never be executed.
Also you were not using await in an async function, so either don't make it async or use await.
csv.upsert = async function (values, condition) {
const obj = await csv.findOne({ where: condition });
// you can await here, no need for then as you are in an async function
if (obj) { // update
// you need to log before your return condition
console.log("existing record updated")
return obj.update(values);
// since we return here, the rest of the code will not be executed
// you can skip the use of else
}
// we are in this part of the code only if object is falsy
// insert
console.log("new record inserted")
return csv.create(values);
}
You could also use Promise.all to ensure all the upsert are done:
await Promise.all(dataToBeInserted.map(async function (data) {
let condition = {
'categorygroup': data.categorygroup,
'category': data.category,
'country': data.country,
'city': data.city
};
await csvModel.upsert(data, condition);
}))
this will also ensure that if an error occur it gets catched by your try / catch
Maybe that will help you find what's causing the unexpected behaviours.
There is a field called counter in a model and whenever if I call a custom remote method like
Request URL
http://loopback:3000/api/models/increment_counter
Request body
EMPTY
Response
{
"counter" : [Value after Increment]
}
Currently to increment First i have to get the counter value from db and increment it one and update the couter value, This involves two queries, Is it possible to do it in a single NodeAPI call like the below mysql query.I am currently using Mysql as DB.
mysql_query("
UPDATE model
SET counter = counter + 1
WHERE model_id = '1'
");
Thank you
Given the MySQL syntax you want, you seem to need an atomic counter.
database transactions
With the MySQL connector, you can use database transactions. It is supported by the MySQL connector. It's a bit overkill for just atomic counters but it will get the job done.
Here is an example, inside a custom remote method
MyModel.someRemoteMethodForIncrementing = function(callback) {
// Start the transaction
MyModel.beginTransaction({
isolationLevel: MyModel.Transaction.READ_COMMITTED
}, function(err, tx) {
// Retrieve the current counter value as part of the transaction
MyModel.findById(id, {
transaction: tx
}, function(err, data) {
if (err) {
console.log(err);
return tx.rollback(function(err) {
callback(err);
});
}
// Increment the counter as part of the transaction
var inc = data.counter + 1;
MyModel.updateAttributes({
counter: inc
}, {
transaction: tx
}, function(err, newData) {
if (err) {
console.log(err);
return tx.rollback(function(err) {
callback(err);
});
}
// Commit the transaction to make it happen
tx.commit(function(err) {
if (err) return callback(err);
// Counter should have been incremented
callback();
});
});
});
});
};
I haven't tested it but it should work, let me know
extented operators
In the future, you will be able to use the $inc (increment) extended operator but so far it's only supported by MongoDB connector, not MySQL.
Just for reference, here is the syntax (works only with MongoDB)
PUT api/Model/:id?filter={"$inc":{"name":propertyToIncrement, "count":incrementAmount}}
There is an ongoing PR that I am trying to get landed to get MySQL support, but there are many things to be done before it can get merged.
Yes you can do it in a single loopback remote method call. Assume you are sending an Id in your request
yourModel.remoteMethod = function(data, cb){
yourModel.findById(data.id, function(err, object){
if(err){
cb(err, null);
}
object.counter += 1;
object.save(function(saveErr, afterSaveObj){
cb(saveErr, afterSaveObj.counter);
})
});
}
here cb is a callback which loopback passes to your remoteMethod.
I have 2 sequelize models (Event and Inventory associated with 2 tables). I created a Event._create method so that I can use it to create an event in event db with multiple products recorded in the inventory db at the same time. Each inventory is associated with the event_id of the newly created event.
Because all these stuff should success or fail altogether, I use sequelize's transaction to achieve this.
Initially I was thinking about doing something like this.
sequelize.transactionPromise = Promise.promisify(sequelize.transaction, sequelize);
return sequelize.transactionPromise({autocommit: 0})
.then(function(t) {
return Event.create(ev, {transaction: t})
.then(function(event){
var event_id = event.id; // ------ (*)
return Promise.resolve([1, ..., event_number])
.then(function(){
Inventory.create({product_id: some_product_id, event_id: event_id},
{transaction: t});
});
.then(function(){
return Promise.cast(t.commit())
.then(function() { // successfully committed
return res.json(d);
}).catch(function(err){ // cannot commit somehow
return res.json(500, err.toString());
});
}).catch(function(err){ // error rollback
return Promise.cast(t.rollback())
.then(function() {
return Promise.reject('rollback: ' + err.toString());
});
});
});
But this doesn't work because before the transaction is commiteed the (*) has no value and gives me event_id of NULL.
Instead I do something like below:
var Event = sequelize.model('Event');
var Inventory = sequelize.model('Inventory');
var _create = function(t, ev){
var ev_id_secret = {secret: 'some random secret'};
return Promise.cast(Event.create(ev_id_secret))
.then(function(d){
ev_id_secret.id = d.id;
return true;
}).then(function(){
return Promise.resolve(_.range(ev.number_of_products))
.map(function(){
var inventory = {
event_id: ev_id_secret.id,
product_id: ev.product_id
};
return Promise.cast(Inventory._create(t, inventory));
});
}).then(function(){ // thennable a transaction
return Promise.cast(Event.update(ev, ev_id_secret, {transaction: t}));
});
};
So I can do something like this.
sequelize.transactionPromise=Promise.promisify(sequelize.transaction, sequelize);
return sequelize.transactionPromise({autocommit: 0})
.then(function(t) {
return Event._create(t, ev)
.then(function(){
return Promise.cast(t.commit())
.then(function() {
return res.json(d);
}).catch(function(err){
return res.json(500, err.toString());
});
}).catch(function(err){
return Promise.cast(t.rollback())
.then(function() {
return Promise.reject('rollback: ' + err.toString());
});
});
}).catch(function(err){
console.log(err.stack);
res.json(500, {error: err.toString()});
});
What I do with _create is that I just insert an empty event (with randomly generated secret) in Event db and some empty products in Inventory db, later get event_id using this secret to query, update the event and the inventory accordingly.
The thing is when the promise is rejected, the transaction rollback() is called and leave empty event and product records in the db. So I have to deal with the empty records later, which is really disturbing.
So my question is how do I do transaction between 2 tables correctly? Am I on the right track?
p.s: as a side question you can see my code is full of return Promise.xxx statements, this ensures the control flow but the promise gets really messy. Is there something I can do to improve my code? Thanks.
Your original implementation was correct. However, after you created the event, the object should come back with the id already populated without having to commit (I'm assuming you're using the standard auto-incremented sequelize id's). All SQL databases function this way. I recommend debugging your application as to determine the root of the problem.
Here's a sample implementation of sequelize transaction from one of my applications. I've just tested and confirmed that calling db.Deposit.create() resolves to an object with valid id. I am using Bluebird, Sequelize 2.0.0-rc2, and Postgresql 9.3
module.exports.createDeposit = function(deposit) {
return db.sequelize.transaction({isolationLevel: 'READ COMMITTED' })
.then(function(t){
var strQuery = 'UPDATE "Accounts" '
+ 'SET "balance"="balance" + :amount, "updatedAt"=NOW() '
+ 'WHERE "id"= :AccountId RETURNING *';
return sql.query(strQuery,
db.Account.build(),
{ raw: true, transaction : t },
{ AccountId: deposit.AccountId, amount: deposit.amount })
.then(function(account){
if (!account) throw new Error('Account does not exist')
return db.Deposit.create(deposit, { transaction: t });
})
.then(function(dbDeposit){
// If successful, dbDeposit object contains a valid id
if (!dbDeposit) throw new Error('Failed to create deposit');
return t.commit()
.then(function(){
return dbDeposit;
});
})
.catch(function(e){
return t.rollback()
.then(function(){
throw e;
});
});
});
};
Your original implementation was correct as far as i can tell but can be improved by levering managed transactions and full promise support in the latest versions of Sequelize:
return sequelize.transaction({autocommit: false}, function(t) {
return Event.create(ev, {transaction: t}).then(function(event) {
return Promise.map(_.range(event_number), function (number) {
return Inventory.create({product_id: some_product_id, event_id: event.get('id')}, {transaction: t});
});
});
}).then(function(){
// Automatically comitted at this point if the promise chain returned to the transaction successfully resolved
return res.status(200).json(d);
}).catch(function(err){
// Automatically rolledback at this point if the promise chain returned to the transaction was rejected
return res.status(403).json();
});
You can read more about managed transactions on the docs: http://sequelize.readthedocs.org/en/latest/docs/transactions/
I'm using Sequelize in my Nodejs project and I found a problem that I'm having a hard time to solve.
Basically I have a cron that gets an array of objects from a server than inserts it on my database as a object ( for this case, cartoons ). But if I already have one of the objects, I have to update it.
Basically I have a array of objects and a could use the BulkCreate() method. But as the Cron starts again, it doesn't solve it so I was needing some sort of update with an upsert true flag. And the main issue: I must have a callback that fires just once after all these creates or updates. Does anyone have an idea of how can I do that? Iterate over an array of object.. creating or updating it and then getting a single callback after?
Thanks for the attention
From the docs, you don't need to query where to perform the update once you have the object. Also, the use of promise should simplify callbacks:
Implementation
function upsert(values, condition) {
return Model
.findOne({ where: condition })
.then(function(obj) {
// update
if(obj)
return obj.update(values);
// insert
return Model.create(values);
})
}
Usage
upsert({ first_name: 'Taku' }, { id: 1234 }).then(function(result){
res.status(200).send({success: true});
});
Note
This operation is not atomic.
Creates 2 network calls.
which means it is advisable to re-think the approach and probably just update values in one network call and either:
Look at the value returned (i.e. rows_affected) and decide what to do.
Return success if update operation succeeds. This is because whether the resource exists is not within this service's responsibility.
You can use upsert
It's way easier.
Implementation details:
MySQL - Implemented as a single query INSERT values ON DUPLICATE KEY UPDATE values
PostgreSQL - Implemented as a temporary function with exception handling: INSERT EXCEPTION WHEN unique_constraint UPDATE
SQLite - Implemented as two queries INSERT; UPDATE. This means that the update is executed regardless of whether the row already
existed or not
MSSQL - Implemented as a single query using MERGE and WHEN (NOT) MATCHED THEN Note that SQLite returns undefined for created, no
matter if the row was created or updated. This is because SQLite
always runs INSERT OR IGNORE + UPDATE, in a single query, so there
is no way to know whether the row was inserted or not.
Update 07/2019 now with async/await
async function updateOrCreate (model, where, newItem) {
// First try to find the record
const foundItem = await model.findOne({where});
if (!foundItem) {
// Item not found, create a new one
const item = await model.create(newItem)
return {item, created: true};
}
// Found an item, update it
const item = await model.update(newItem, {where});
return {item, created: false};
}
I liked the idea of Ataik, but made it a little shorter:
function updateOrCreate (model, where, newItem) {
// First try to find the record
return model
.findOne({where: where})
.then(function (foundItem) {
if (!foundItem) {
// Item not found, create a new one
return model
.create(newItem)
.then(function (item) { return {item: item, created: true}; })
}
// Found an item, update it
return model
.update(newItem, {where: where})
.then(function (item) { return {item: item, created: false} }) ;
}
}
Usage:
updateOrCreate(models.NewsItem, {slug: 'sometitle1'}, {title: 'Hello World'})
.then(function(result) {
result.item; // the model
result.created; // bool, if a new item was created.
});
Optional: add error handling here, but I strongly recommend to chain all promises of one request and have one error handler at the end.
updateOrCreate(models.NewsItem, {slug: 'sometitle1'}, {title: 'Hello World'})
.then(..)
.catch(function(err){});
This might be an old question, but this is what I did:
var updateOrCreate = function (model, where, newItem, onCreate, onUpdate, onError) {
// First try to find the record
model.findOne({where: where}).then(function (foundItem) {
if (!foundItem) {
// Item not found, create a new one
model.create(newItem)
.then(onCreate)
.catch(onError);
} else {
// Found an item, update it
model.update(newItem, {where: where})
.then(onUpdate)
.catch(onError);
;
}
}).catch(onError);
}
updateOrCreate(
models.NewsItem, {title: 'sometitle1'}, {title: 'sometitle'},
function () {
console.log('created');
},
function () {
console.log('updated');
},
console.log);
User.upsert({ a: 'a', b: 'b', username: 'john' })
It will try to find record by hash in 1st param to update it, if it will not find it - then new record will be created
Here is example of usage in sequelize tests
it('works with upsert on id', function() {
return this.User.upsert({ id: 42, username: 'john' }).then(created => {
if (dialect === 'sqlite') {
expect(created).to.be.undefined;
} else {
expect(created).to.be.ok;
}
this.clock.tick(1000);
return this.User.upsert({ id: 42, username: 'doe' });
}).then(created => {
if (dialect === 'sqlite') {
expect(created).to.be.undefined;
} else {
expect(created).not.to.be.ok;
}
return this.User.findByPk(42);
}).then(user => {
expect(user.createdAt).to.be.ok;
expect(user.username).to.equal('doe');
expect(user.updatedAt).to.be.afterTime(user.createdAt);
});
});
Sound likes you want to wrap your Sequelize calls inside of an async.each.
This can be done with the custom event emitter.
Assuming your data is in a variable called data.
new Sequelize.Utils.CustomEventEmitter(function(emitter) {
if(data.id){
Model.update(data, {id: data.id })
.success(function(){
emitter.emit('success', data.id );
}).error(function(error){
emitter.emit('error', error );
});
} else {
Model.build(data).save().success(function(d){
emitter.emit('success', d.id );
}).error(function(error){
emitter.emit('error', error );
});
}
}).success(function(data_id){
// Your callback stuff here
}).error(function(error){
// error stuff here
}).run(); // kick off the queries
you can use findOrCreate and then update methods in sequelize. here is a sample with async.js
async.auto({
getInstance : function(cb) {
Model.findOrCreate({
attribute : value,
...
}).complete(function(err, result) {
if (err) {
cb(null, false);
} else {
cb(null, result);
}
});
},
updateInstance : ['getInstance', function(cb, result) {
if (!result || !result.getInstance) {
cb(null, false);
} else {
result.getInstance.updateAttributes({
attribute : value,
...
}, ['attribute', ...]).complete(function(err, result) {
if (err) {
cb(null, false);
} else {
cb(null, result);
}
});
}
}]
}, function(err, allResults) {
if (err || !allResults || !allResults.updateInstance) {
// job not done
} else {
// job done
});
});
Here is a simple example that either updates deviceID -> pushToken mapping or creates it:
var Promise = require('promise');
var PushToken = require("../models").PushToken;
var createOrUpdatePushToken = function (deviceID, pushToken) {
return new Promise(function (fulfill, reject) {
PushToken
.findOrCreate({
where: {
deviceID: deviceID
}, defaults: {
pushToken: pushToken
}
})
.spread(function (foundOrCreatedPushToken, created) {
if (created) {
fulfill(foundOrCreatedPushToken);
} else {
foundOrCreatedPushToken
.update({
pushToken: pushToken
})
.then(function (updatedPushToken) {
fulfill(updatedPushToken);
})
.catch(function (err) {
reject(err);
});
}
});
});
};
2022 update:
You can use the upsert function:
https://sequelize.org/api/v6/class/src/model.js~model#static-method-upsert
Insert or update a single row. An update will be executed if a row which matches the supplied values on either the primary key or a unique key is found. Note that the unique index must be defined in your sequelize model and not just in the table. Otherwise you may experience a unique constraint violation, because sequelize fails to identify the row that should be updated.
Implementation details:
MySQL - Implemented with ON DUPLICATE KEY UPDATE`
PostgreSQL - Implemented with ON CONFLICT DO UPDATE. If update data contains PK field, then PK is selected as the default conflict key.
Otherwise first unique constraint/index will be selected, which can satisfy conflict key requirements.
SQLite - Implemented with ON CONFLICT DO UPDATE
MSSQL - Implemented as a single query using MERGE and WHEN (NOT) MATCHED THEN
Note that Postgres/SQLite returns null for created, no matter if the row was created or updated