Django unique constraint not working properly [duplicate] - mysql

This question already has answers here:
Unique constraint that allows empty values in MySQL
(3 answers)
Closed 2 years ago.
I have a model with a custom _id that has to be unique, and soft delete, deleted objects don't have to have a unique _id, so I did it as follows:
class MyModel(models.Model):
_id = models.CharField(max_length=255, db_index=True)
event_code = models.CharField(max_length=1, blank=True, default='I')
deleted = models.BooleanField(default=False)
deleted_id = models.IntegerField(blank=True, null=True)
objects = MyModelManager() # manager that filters out deleted objects
all_objects = MyModelBaseManager() # manager that returns every object, including deleted ones
class Meta:
constraints = [
UniqueConstraint(fields=['_id', 'event_code', 'deleted', 'deleted_id'], name='unique_id')
]
def delete(self, *args, **kwargs):
self.deleted = True
self.deleted_id = self.max_related_deleted_id() + 1
self.save()
def undelete(self, *args, **kwargs):
self.deleted = False
self.deleted_id = None
self.save()
def max_related_deleted_id(self):
# Get max deleted_id of deleted objects with the same _id
max_deleted_id = MyModel.all_objects.filter(Q(_id=self._id) & ~Q(pk=self.pk) & Q(deleted=True)).aggregate(Max('deleted_id'))['deleted_id__max']
return max_deleted_id if max_deleted_id is not None else 0
The whole logic of the deleted_id is working, I tested it out, the problem is, the UniqueConstraint is not working, for example:
$ MyModel.objects.create(_id='A', event_code='A')
$ MyModel.objects.create(_id='A', event_code='A')
$ MyModel.objects.create(_id='A', event_code='A')
$ MyModel.objects.filter(_id='A').values('pk', '_id', 'event_code', 'deleted', 'deleted_id')
[{'_id': 'A',
'deleted': False,
'deleted_id': None,
'event_code': 'A',
'pk': 1},
{'_id': 'A',
'deleted': False,
'deleted_id': None,
'event_code': 'A',
'pk': 2},
{'_id': 'A',
'deleted': False,
'deleted_id': None,
'event_code': 'A',
'pk': 3}]
Here is the migration that created the unique constraint:
$ python manage.py sqlmigrate myapp 0003
BEGIN;
--
-- Create constraint unique_id on model MyModel
--
ALTER TABLE `myapp_mymodel` ADD CONSTRAINT `unique_id` UNIQUE (`_id`, `event_code`, `deleted`, `deleted_id`);
COMMIT;
Any help is appreciated!
Django version = 2.2
Python version = 3.7
Database = MySQL 5.7

Ok, I figured out my problem, I'm gonna post this here in case someone runs into it:
The problem is with MySQL, as stated in this post, mysql allows multiple null values in a unique constraint, so I had to change the default of deleted_id to 0 and now it works.

Related

Handling Redshift identity columns in SQLAlchemy

This question is related to a Question posted a few years ago.
I'm using the redshift-sqlalchemy package to connect SQLAlchemy to Redshift.
In Redshift I have a simple "companies" table:
CREATE TABLE apis
(
id INTEGER IDENTITY(1,1) NOT NULL,
name VARCHAR(255) NOT NULL,
PRIMARY KEY(id)
);
On the SQLAlchemy side I have mapped it like so:
Base = declarative_base()
class Company(Base):
__tablename__ = 'companies'
id = Column(BigInteger, primary_key=True, redshift_identity=(1, 1))
name = Column(String, nullable=False)
def __init__(self, name: str):
self.name = name
If I try to create a company:
company = Company(name = 'Hoge')
session.add(company)
session.commit()
then I get this error:
(sqlalchemy.exc.ProgrammingError) (redshift_connector.error.ProgrammingError) {'S': 'ERROR', 'C': '42P01', 'M': 'relation "companies_id_seq" does not exist', 'F': '../src/pg/src/backend/catalog/namespace.c', 'L': '267', 'R': 'LocalRangeVarGetRelid'}
[SQL: INSERT INTO companies (id, name) VALUES (%s, %s)]
[parameters: [{'name': 'Hoge'}]]
(Background on this error at: https://sqlalche.me/e/14/f405)
I think the problem is that you are trying to insert data also in the ID column with the IDENTITY option set.
SQL: INSERT INTO companies (id, name) VALUES (%s, %s)
If I execute SQL directly on a Redshift table I get the following error.
ERROR: cannot set an identity column to a value.
Please tell me how to define the auto-populated ID column in sqlalchemy-redshift model?

Modifying entry in django admin creates duplicate

I am using the django admin to modify records in a table. The problem is that whenever I modify an entry, when I click save, instead of modifying that entry, the old one is not modified and a new entry containing the modified details is being added.
For example, if I have the following:
Aardvark | Orycteropus | Some description | aardvark | animals/images/aardvark.jpg
when I change the first field to Aardvarkon, I get the following:
Aardvark | Orycteropus | Some description | aardvark | animals/images/aardvark.jpg
Aardvarkon | Orycteropus | Some description | aardvark | animals/images/aardvark.jpg
I have the following django model:
def article_file_name(instance, filename):
return ANIMAL_IMAGES_BASE_DIR[1:] + instance.ai_species_species_sanitized + '.jpg'
class ai_species(models.Model):
ai_species_species = models.CharField('Species', max_length=100, primary_key=True, db_column='species')
ai_species_genus = models.ForeignKey(ai_genera, max_length=50, blank=False, null=False, db_column='genus')
ai_species_description = models.CharField('Description', max_length=65000, db_column='description')
ai_species_species_sanitized = models.CharField(max_length=100, blank=False, null=False, db_column='species_sanitized')
image_url = models.ImageField(max_length=100, storage=OverwriteStorage(), validators=[validate_jpg_extension], upload_to=article_file_name)
class Meta:
db_table = 'Species'
verbose_name = 'Animal species'
verbose_name_plural = 'Animal species'
def __unicode__(self): # Required, don't remove.
return self.ai_species_species
And the following helpers:
def validate_jpg_extension(value):
if not value.name.lower().endswith('.jpg') and not value.name.lower().endswith('.jpeg'):
raise ValidationError(u'Invalid file format! Only jpg or jpeg files allowed!')
class OverwriteStorage(FileSystemStorage):
def get_available_name(self, name):
# If the filename already exists, remove it.
if self.exists(name):
os.remove(os.path.join(settings.MEDIA_ROOT, name))
return name
This is the MySQL table schema for this table:
This is a very counter-intuitive behavior and I haven't found any other occurrences of this online. Any help on this would be greatly appreciated.
Here's the culprit:
ai_species_species = models.CharField('Species', max_length=100, primary_key=True, db_column='species')
Since you've defined the species as the primary key, any time you change this field in the admin it will create a new record (because there isn't already a record with that primary key).
FYI, primary keys aren't supposed to be things that change for a given record, since changing the primary key will invalidate every foreign key (ForeignKey, OneToOneField, and ManyToManyField) that refers to the record.
BTW, you don't need to be prefixing the field names with ai_species_; it's cluttering. Removing those prefixes would remove the need for the db_column parameters as well.

SQLAlchemy - Auto Lookup Foreign Key Relationship for Insert

I am trying to get an SQLAlchemy ORM class to automatically:
either lookup the foreign key id for a field
OR
for entries where the field isn't yet in foreign key table, add the row to the foreign key table - and use the auto generated id in the original table.
To illustrate:
Class Definition
class EquityDB_Base(object):
#declared_attr
def __tablename__(cls):
return cls.__name__.lower()
__table_args__ = {'mysql_engine': 'InnoDB'}
__mapper_args__= {'always_refresh': True}
id = Column(Integer, primary_key=True)
def fk(tablename, nullable=False):
return Column("%s_id" % tablename, Integer,
ForeignKey("%s.id" % tablename),
nullable=nullable)
class Sector(EquityDB_Base, Base):
name = Column(String(40))
class Industry(EquityDB_Base, Base):
name = Column(String(40))
sector_id = fk('sector')
sector = relationship('Sector', backref='industries')
class Equity(EquityDB_Base, Base):
symbol = Column(String(10), primary_key=True)
name = Column(String(40))
industry_id = fk('industry')
industry = relationship('Industry', backref='industries')
Using the Class to Set Industry and Sector
for i in industry_record[]:
industry = Industry(id=i.id,
name=i.name,
sector=Sector(name=i.sector_name))
session.merge(industry)
Result
Unfortunately, when I run this - the database adds individual rows to the sector table for each duplicate use of 'sector_name' - for instance, if 10 industries use 'Technology' as their sector name, I get 10 unique sector_id for each one of the 10 industries.
What I WANT - is for each time a sector name is presented that is already in the database, for it to auto-resolve to the appropriate sector_id
I am clearly just learning SQLAlchemy, but can't seem to figure out how to enable this behavior.
Any help would be appreciated!
See answer to a similar question create_or_get entry in a table.
Applying the same logic, you would have something like this:
def create_or_get_sector(sector_name):
obj = session.query(Sector).filter(Sector.name == sector_name).first()
if not obj:
obj = Sector(name = sector_name)
session.add(obj)
return obj
and use it like below:
for i in industry_record[:]:
industry = Industry(id=i.id,
name=i.name,
sector=create_or_get_sector(sector_name=i.sector_name))
session.merge(industry)
One thing you should be careful about is which session instance is used there in the create_or_get_sector.

Joining 2 Tables on Multiple Non Foreign Key Columns in Flask with SQLAlchemy and Retrieving All Columns

I have a few tables shown below that I would like to join on columns that are not foreign keys to each other's tables and then have access to the columns of both. Here are the classes:
class Yi(db.Model):
year = db.Column(db.Integer(4), primary_key=True)
industry_id = db.Column(db.String(5), primary_key=True)
wage = db.Column(db.Float())
complexity = db.Column(db.Float())
class Ygi(db.Model, AutoSerialize):
year = db.Column(db.Integer(4), primary_key=True)
geo_id = db.Column(db.String(8), primary_key=True)
industry_id = db.Column(db.String(5), primary_key=True)
wage = db.Column(db.Float())
So, what I would like to get are the columns of both tables joined by the IDs I specify, in this case Year and industry_id. Is this possible? Here is the SQL I've written to achieve this...
SELECT
yi.complexity, ygi.*
FROM
yi, ygi
WHERE
yi.year = ygi.year and
yi.industry_id = ygi.industry_id
One dirty way is :
q=session.query(Ygi,Yi.complexity).\
filter(Yi.year==Ygi.year).\
filter(Yi.industry_id==Ygi.industry_id)
Which gives you :
SELECT ygi.year AS ygi_year, ygi.geo_id AS ygi_geo_id,
ygi.industry_id AS ygi_industry_id, ygi.wage AS ygi_wage,
yi.complexity AS yi_complexity
FROM ygi, yi
WHERE yi.year = ygi.year
AND yi.industry_id = ygi.industry_id
I find this dirty because it does not use the join() method.
You can figure out how to use the join() with the SQLAlchemy documentation
Then, you can choose to use a virtual model : see answer of TokenMacGuy in this question Mapping a 'fake' object in SQLAlchemy.
It will be a good solution.
Or you will just have a YiYgi class that will not be a sqlalchemy.Base derived class but just an object. It more a "hand-fashion" way to do it.
The class will have a classmethod get() method that will:
call the query you build just before,
call the init with the returned request lines and build up one instance per line
This is an example :
class YiYgi(object):
def __init__(self,year, geo_id, industry_id, wage, complexity):
# Initialize all your fields
self.year = year
self.geo_id = geo_id
self.industry_id = industry_id
self.wage = wage + 100 # You can even make some modifications to the values here
self.complexity = complexity
#classmethod
def get_by_year_and_industry(cls, year, industry_id):
""" Return a list of YiYgi instances, void list if nothing available """
q = session.query(Ygi,Yi.complexity).\
filter(Yi.year==Ygi.year).\
filter(Yi.industry_id==Ygi.industry_id)
results = q.all()
yiygi_list = []
for result in results:
# result is a tuple with (YGi instance, Yi.complexity value)
ygi_result = result[0]
yiygi = YiYgi(ygi_result.ygi_year,
ygi_result.geo_id,
ygi_result.industry_id,
ygi_result.wage,
result[1])
yiygi_list.append(yiygi)
return yiygi_list

sqlalchemy FetchedValue and primary_key

I'm trying to create a table that uses a UUID_SHORT() as a primary key. I have a trigger that inserts a value when you do an insert. I'm having trouble making sqlalchemy recognize a column as a primary_key without complaining about not providing a default. If I do include a default value, it will use that default value even after flush despite declaring server_default=FetchedValue(). The only way I can seem to get things to work properly is if the column is not a primary key.
I'm using Pyramid, SQLAlchemy ORM, and MySQL.
Here's the model object:
Base = declarative_base()
class Patient(Base):
__tablename__ = 'patient'
patient_id = Column(BigInteger(unsigned=True), server_default=FetchedValue(), primary_key=True, autoincrement=False)
details = Column(Binary(10000))
in initializedb.py I have:
with transaction.manager:
patient1 = Patient(details = None)
DBSession.add(patient1)
DBSession.flush()
print(patient1.patient_id)
running ../bin/initialize_mainserver_db development.ini gives me the following error:
2012-11-01 20:17:22,168 INFO [sqlalchemy.engine.base.Engine][MainThread] BEGIN (implicit)
2012-11-01 20:17:22,169 INFO [sqlalchemy.engine.base.Engine][MainThread] INSERT INTO patient (details) VALUES (%(details)s)
2012-11-01 20:17:22,169 INFO [sqlalchemy.engine.base.Engine][MainThread] {'details': None}
2012-11-01 20:17:22,170 INFO [sqlalchemy.engine.base.Engine][MainThread] ROLLBACK
Traceback (most recent call last):
File "/sites/metrics_dev/lib/python3.3/site-packages/sqlalchemy/engine/base.py", line 1691, in _execute_context
context)
File "/sites/metrics_dev/lib/python3.3/site-packages/sqlalchemy/engine/default.py", line 333, in do_execute
cursor.execute(statement, parameters)
File "/sites/metrics_dev/lib/python3.3/site-packages/mysql/connector/cursor.py", line 418, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/sites/metrics_dev/lib/python3.3/site-packages/mysql/connector/cursor.py", line 345, in _handle_result
self._handle_noresultset(result)
File "/sites/metrics_dev/lib/python3.3/site-packages/mysql/connector/cursor.py", line 321, in _handle_noresultset
self._warnings = self._fetch_warnings()
File "/sites/metrics_dev/lib/python3.3/site-packages/mysql/connector/cursor.py", line 608, in _fetch_warnings
raise errors.get_mysql_exception(res[0][1],res[0][2])
mysql.connector.errors.DatabaseError: 1364: Field 'patient_id' doesn't have a default value
Running a manual insert using the mysql client results in the everything working fine, so the problem seems to be with SQLAlchemy.
mysql> insert into patient(details) values (null);
Query OK, 1 row affected, 1 warning (0.00 sec)
mysql> select * from patient;
+-------------------+---------+
| patient_id | details |
+-------------------+---------+
| 94732327996882980 | NULL |
+-------------------+---------+
1 row in set (0.00 sec)
mysql> show triggers;
+-----------------------+--------+---------+-------------------------------------+--------+---------+----------+----------------+----------------------+----------------------+--------------------+
| Trigger | Event | Table | Statement | Timing | Created | sql_mode | Definer | character_set_client | collation_connection | Database Collation |
+-----------------------+--------+---------+-------------------------------------+--------+---------+----------+----------------+----------------------+----------------------+--------------------+
| before_insert_patient | INSERT | patient | SET new.`patient_id` = UUID_SHORT() | BEFORE | NULL | | root#localhost | utf8 | utf8_general_ci | latin1_swedish_ci |
+-----------------------+--------+---------+-------------------------------------+--------+---------+----------+----------------+----------------------+----------------------+--------------------+
1 row in set (0.00 sec)
Here's what I did as a work-around...
DBSession.execute(
"""CREATE TRIGGER before_insert_patient BEFORE INSERT ON `patient`
FOR EACH ROW BEGIN
IF (NEW.patient_id IS NULL OR NEW.patient_id = 0) THEN
SET NEW.patient_id = UUID_SHORT();
END IF;
END""")
and in the patient class:
patient_id = Column(BigInteger(unsigned=True), default=text("uuid_short()"), primary_key=True, autoincrement=False, server_default="0")
So, the trigger only does something if someone accesses the database directly and not through the python code. And hopefully no one does patient1 = Patient(patient_id=0, details = None) as SQLAlchemy will use the '0' value instead of what the trigger produces
For completeness, here are two additional possible solutions for your question (also available here), based on your answer. They are slightly simpler than your solution (omitting passing parameters with correct default values) and using SQLAlchemy constructs for defining the triggers.
#!/usr/bin/env python3
from sqlalchemy import BigInteger, Column, create_engine, DDL, event
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.schema import FetchedValue
from sqlalchemy.sql.expression import func
Base = declarative_base()
class PatientOutputMixin(object):
'''
Mixin to output human readable representations of models.
'''
def __str__(self):
return '{}'.format(self.patient_id)
def __repr__(self):
return str(self)
class Patient1(Base, PatientOutputMixin):
'''
First version of ``Patient`` model.
'''
__tablename__ = 'patient_1'
patient_id = Column(BigInteger, primary_key=True,
default=func.uuid_short())
# the following trigger is only required if columns are inserted in the table
# not using the above model/table definition, otherwise it is redundant
create_before_insert_trigger = DDL('''
CREATE TRIGGER before_insert_%(table)s BEFORE INSERT ON %(table)s
FOR EACH ROW BEGIN
IF NEW.patient_id IS NULL THEN
SET NEW.patient_id = UUID_SHORT();
END IF;
END
''')
event.listen(Patient1.__table__, 'after_create',
create_before_insert_trigger.execute_if(dialect='mysql'))
# end of optional trigger definition
class Patient2(Base, PatientOutputMixin):
'''
Second version of ``Patient`` model.
'''
__tablename__ = 'patient_2'
patient_id = Column(BigInteger, primary_key=True,
default=0, server_default=FetchedValue())
create_before_insert_trigger = DDL('''
CREATE TRIGGER before_insert_%(table)s BEFORE INSERT ON %(table)s
FOR EACH ROW BEGIN
SET NEW.patient_id = UUID_SHORT();
END
''')
event.listen(Patient2.__table__, 'after_create',
create_before_insert_trigger.execute_if(dialect='mysql'))
# test models
engine = create_engine('mysql+oursql://test:test#localhost/test?charset=utf8')
Base.metadata.bind = engine
Base.metadata.drop_all()
Base.metadata.create_all()
Session = sessionmaker(bind=engine)
session = Session()
for patient_model in [Patient1, Patient2]:
session.add(patient_model())
session.add(patient_model())
session.commit()
print('{} instances: {}'.format(patient_model.__name__,
session.query(patient_model).all()))
Running the above script produces the following (sample) output:
Patient1 instances: [22681783426351145, 22681783426351146]
Patient2 instances: [22681783426351147, 22681783426351148]