I'm trying to create a package that manages DB connection, ORM models and migration. It's separated from web service project such as a Flask application, so flask-sqlalchemy is not considered.
This is how I organize my project (only list out parts related to this question):
alembic.ini
src/
* my_project/
* db/
- connections.py
* models/
* abstract/
- abstract_base.py
* realized/
- realized_model.py
migrations/
* versions/
- env.py
- script.py.mako
src/my_project/db/connections.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
ENGINE = create_engine("db://url")
ModelBase = declarative_base(bind=ENGINE)
Session = sessionmaker(ENGINE)
src/my_project/models/abstract/abstract_base.py:
from sqlalchemy import Column, Integer, DateTime
from my_project.db.connections import ModelBase
class BaseWithTimestamp(ModelBase):
__abstract__ = True
id = Column(Integer, primary_key=True, nullable=False)
created_at = Column(DateTime, nullable=False)
src/my_project/models/realized/realized_model.py:
from sqlalchemy import Column, String
from my_project.models.abstract.abstract_base import BaseWithTimestamp
class Note(BaseWithTimestamp):
__tablename__ = "notes"
content = Column(String(300), nullable=True)
env.py (the same as alembic default except metadata setup):
# ...
from my_project.db.connections import ModelBase
# ...
target_metadata = ModelBase.metadata
# ...
Supposedly, when linked to an empty database, alembic should generate migration script that creates table notes with three columns specified in model Note when running revision command with auto-generation switched on. However, what I got is an empty migration script.
Hence I tried doing this in interactive shell to see what's stored in Base's metadata:
from my_project.db.connections import ModelBase
ModelBase.metadata.tables # => FacadeDict({})
Note/notes is expected to appear in metadata's table list, but above result shows that no table was memorized in Base's metatdata, which I think is the root cause for generating empty migration script. Is there anything I'm missing or doing wrong?
Seems that one needs to declare all related classes explicitly right after declarative base model is declared/imported, so that these extended models can get added to metadata:
migrations/env.py
# ...
from my_project.db.connections import ModelBase
from my_project.models.abstract.abstract_base import BaseWithTimestamp
from my_project.models.realized.realized_model import Note
# ...
target_metadata = ModelBase.metadata
# ...
Related
I am using SQLALCHEMY and I can't update the table, i don't know why it doesn't work.
models.py
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, Date
Base = declarative_base()
class Book(Base):
__tablename__='books'
id= Column(Integer, primary_key=True)
title=Column(String)
author = Column(String)
pages = Column(String)
published=Column(Date)
def __repr__(self):
return "<Book(title='{}', author='{}', pages={}, published={})>"\
.format(self.title, self.author, self.pages, self.published)
crud.py
from sqlalchemy import create_engine, update
from sqlalchemy.orm import sessionmaker
engine = create_engine(config.DATABASE_URI)
Session = sessionmaker(bind=engine)
s= Session()
def recreate_database():
models.Base.metadata.drop_all(engine)
models.Base.metadata.create_all(engine)
I insert the first row on the table 'books' :
book = models.Book(
title='Deep Learning',
author='Ian Goodfellow',
pages=775,
published=datetime(2016,11,18)
)
using:
recreate_database()
s.add(book)
Then I verify that row is added:
<Book(title='Deep Learning', author='Ian Goodfellow', pages=775, published=2016-11-18 00:00:00)>
And I try to update that valor changing the value of 'pages' to 900 using:
libro=models.Base.metadata.tables['books']
u=update(libro)
u=u.values({"pages": 900})
u=u.where(libro.c.title=="Deep Learning")
engine.execute(u)
s.commit()
s.close()
I don't get error but it does not update the value of pages.
I appreciate if you could tell me where is the error on my code to update the value.
Thanks !!
its s.execute(u) not engine.execute(u)
You also declared pages to be a string column when it actually is an integer (maybe you want to change that).
edit: if you already have the book object, you can also use
book.pages = 1200
s.commit()
to change the value in your database.
SQLAlchemy, Is there a way to create table definition class directly, beacause the table has already existed in DB, most documents tell me to make the definition mannually.
Base = declarative_base()
# ORM defininition
class Dep(Base):
__tablename__='dep'
id=Column(Integer,primary_key=True,autoincrement=True)
dname=Column(String(64),nullable=False,index=True)
# Add new obs
session = sessionMaker()
row_obj=Dep(dname='saleman')
session.add(row_obj)
There are several of ways to reflect tables in SQLAlchemy.
I'll put here examples inspired by the documentation.
1. Giving the class an autoloaded __table__
from sqlalchemy import Table, create_engine
from sqlalchemy.orm import declarative_base
Base = declarative_base() # same declarative_base() as usual
engine = create_engine("sqlite:///mydatabase.db") # get your engine
class User(Base):
__table__ = Table("user", Base.metadata, autoload_with=engine)
2. Retrieving classes with the Automap extension
from sqlalchemy import create_engine
from sqlalchemy.ext.automap import automap_base
Base = automap_base() # rather than declarative_base()
engine = create_engine("sqlite:///mydatabase.db") # get your engine
Base.prepare(autoload_with=engine) # reflect the tables and classes
User = Base.classes.user # retrieve classes from table name
3. Inheriting from the DeferredReflection extension
from sqlalchemy.ext.declarative import DeferredReflection
from sqlalchemy.orm import declarative_base
Base = declarative_base() # same declarative_base() as usual
class User(DeferredReflection, Base): # define your class
__tablename__ = "user"
engine = create_engine("sqlite:///mydatabase.db") # get your engine
User.prepare(engine) # reflect the table and attributes
The two first examples need an engine to create the class, which can be inconvenient for testing and such, so I prefer the third one which allows me to have class and engine defined in different places, but all three will work.
I was trying to make my ORM code a bit more elegant by using classmethods in sqlalchemy and I wanted to make a method called get which would just retrieve a single existing instance of the ORM object with a few parameters. But since it seems I need a session to do it, the only way I could figure out how to do it was to pas the session as a parameter to the get method. It is working but I can't shake the feeling that I am building an antipattern.
Here is my ORM class (simplified):
class GeocellZone(Base):
__tablename__ = "geocell_zone"
__table_args__ = (UniqueConstraint("study_area_id", "version_id", "name"),)
id = Column(Integer, primary_key=True)
study_area_id = Column(Integer, ForeignKey("study_area.id"))
version_id = Column(Integer, ForeignKey("version.id"))
name = Column(Text)
geometry = Column(Geometry(srid=4326))
#classmethod
def get(cls, session, version, study_area, name):
return session.query(cls) \
.filter(cls.study_area_id == study_area.id) \
.filter(cls.version_id == version.id) \
.filter(cls.name == name) \
.first()
And here is what it looks like when I call it in my application:
import os
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from myapp.orm import *
engine = create_engine(
f"postgresql://{os.getenv('DB_USER')}:{os.getenv('DB_PASS')}#{os.getenv('DB_HOST')}:{os.getenv('DB_PORT')}/{os.getenv('DB_NAME')}",
echo=False
)
session = sessionmaker(bind=engine)()
GeocellZone.get(session, version, study_area, "Antwerpen")
This works, it returns the exact GeocellZone instance that I want. But I feel dirty passing the session to the ORM class like this. Am I overthinking this? Or is there a better way to do this?
IIUC, the format is set in log.py (lines 33..38 in 1.4.7):
def _add_default_handler(logger):
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
)
logger.addHandler(handler)
But all my attempts to set the format to a bare '%(message)s' have failed. For example,
logging.StreamHandler(sys.stdout).setFormatter('%(message)s')
has no effect.
When developing a program in SQLAlchemy, I want to see only the query executed by engine, and these extra fields asctime, levelname and name are distracting.
There is an entire subchapter about engine logging in SQLAlchemy documentation, but it talks only about levels and says nothing about formatting. On the other hand, I guess changing formatting should be possible, because in SQLAlchemy tutorial (see, e.g., here), the logging messages are presented just the way I would like them.
First some remarks:
When you write
: logging.StreamHandler(sys.stdout)
you're calling a constructor, i.e. you're getting a new instance of
logging.StreamHandler class, not the same handler which could have been used in
sqla's log module: _add_default_handler().
Modifying it won't have any effect as long as you don't add this handler to some active logger.
It you read carefully the docs page you mentionned (https://docs.sqlalchemy.org/en/14/core/engines.html#configuring-logging), you'll find some hints :
It’s important to note that these two flags work independently of
any existing logging configuration, and will make use of
logging.basicConfig() unconditionally. This has the effect of being
configured in addition to any existing logger configurations.
Therefore, when configuring logging explicitly, ensure all echo flags
are set to False at all times, to avoid getting duplicate log lines.
logging.basicConfig() accepts a good set of parameters, but deals only with
the root logger, from which other loggers will inherit settings.
As a first step I suggest you keep the level and logger name in output, to let
you know who is speaking in your messages.
>>> import logging
>>> # 1. Get rid of timestamp for all modules, and set other defaults
... logging.basicConfig(format="%(levelname)s %(name)s %(message)s", level="INFO")
... logging.info("Root logger talking")
...
INFO root Root logger talking
>>> # 2. Preset top level logger for SQLAlchemy
... logging.getLogger('sqlalchemy').setLevel("INFO")
...
>>> # 4. Run your code
... import sqlalchemy as sqla
... engine = sqla.create_engine('sqlite:///:memory:')
...
>>> from sqlalchemy.ext.declarative import declarative_base
...
... Base = declarative_base()
... from sqlalchemy import Column, Integer, String
>>> class User(Base):
... __tablename__ = 'users'
...
... id = Column(Integer, primary_key=True)
... name = Column(String)
...
INFO sqlalchemy.orm.mapper.Mapper (User|users) _configure_property(id, Column)
INFO sqlalchemy.orm.mapper.Mapper (User|users) _configure_property(name, Column)
INFO sqlalchemy.orm.mapper.Mapper (User|users) Identified primary key columns: ColumnSet([Column('id', Integer(), table=<users>, primary_key=True, nullable=False)])
INFO sqlalchemy.orm.mapper.Mapper (User|users) constructed
>>> Base.metadata.create_all(engine)
INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1
INFO sqlalchemy.engine.base.Engine ()
INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1
INFO sqlalchemy.engine.base.Engine ()
INFO sqlalchemy.engine.base.Engine PRAGMA main.table_info("users")
INFO sqlalchemy.engine.base.Engine ()
INFO sqlalchemy.engine.base.Engine PRAGMA temp.table_info("users")
INFO sqlalchemy.engine.base.Engine ()
INFO sqlalchemy.engine.base.Engine
CREATE TABLE users (
id INTEGER NOT NULL,
name VARCHAR,
PRIMARY KEY (id)
)
INFO sqlalchemy.engine.base.Engine ()
INFO sqlalchemy.engine.base.Engine COMMIT
>>>
To answer your question more specifically, here's a solution:
import logging, sys
sql_logger = logging.getLogger("sqlalchemy.engine.base.Engine")
hdlr = logging.StreamHandler(sys.stdout)
hdlr.setFormatter(logging.Formatter("[SQL] %(message)s"))
sql_logger.addHandler(hdlr)
sql_logger.setLevel(logging.INFO)
import sqlalchemy as sqla
engine = sqla.create_engine('sqlite:///:memory:')
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
from sqlalchemy import Column, Integer, String
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String)
Base.metadata.create_all(engine)
You'll need some more work to handle other modules log messages,
while avoiding duplicates
Below is a form that populates an empty parent object and creates its children. It was necessary to manually invoke a ModelFormField, which was a minor annoyance. It works great. However, when I use the form to do an update, only the object is updated -- the children are created fresh.
What is the correct way to propogate an update to the children in this framework? Effectively, I'd like to the two print statements below to print the same thing. I'd be especially grateful if the form would create (delete) children if the formdata had extra (was missing) data for the children.
from multidict import MultiDict
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from wtforms_alchemy import ModelFieldList, ModelForm, ModelFormField
Base = declarative_base()
class Child(Base):
__tablename__ = "child"
id = Column(Integer, primary_key=True)
name = Column(String)
parent_id = Column(Integer, ForeignKey("parent.id"))
class Parent(Base):
__tablename__ = "parent"
id = Column(Integer, primary_key=True)
name = Column(String)
children = relationship(Child)
class ChildForm(ModelForm):
class Meta:
model = Child
class ParentForm(ModelForm):
class Meta:
model = Parent
children = ModelFieldList(ModelFormField(ChildForm)) # annoyed!
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
# Form data
formdata = MultiDict({"name": "Foo", "children-0-name": "hotdog"})
# Create initial object
question = Parent()
form = ParentForm(formdata, obj=question)
form.populate_obj(question)
if not form.validate():
raise RuntimeError(form.errors)
session.add(question)
session.commit()
# prints: "(1, 1)"
print((question.id, question.children[0].id))
# Retrieve object and update with same data
question_get = session.query(Parent).get(question.id)
form = ParentForm(formdata, obj=question_get)
form.populate_obj(question_get)
if not form.validate():
raise RuntimeError(form.errors)
session.add(question_get)
session.commit()
# prints: "(1, 2)", want it to print the same as above
print((question_get.id, question_get.children[0].id))
In digging through the code, it looks like object ids are expected for updates. Therefore, one must use a form on update which includes the id and pass around the ids.
--- orig.py 2018-05-01 20:51:09.000000000 -0700
+++ fixed.py 2018-05-01 20:56:16.000000000 -0700
## -4,6 +4,7 ##
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
+from wtforms import IntegerField
from wtforms_alchemy import ModelFieldList, ModelForm, ModelFormField
## -24,6 +25,7 ##
class ChildForm(ModelForm):
class Meta:
model = Child
+ id = IntegerField() # Optional field used in update operations.
class ParentForm(ModelForm):
class Meta:
## -53,7 +55,11 ##
# Retrieve object and update with same data
question_get = session.query(Parent).get(question.id)
-form = ParentForm(formdata, obj=question_get)
+child_id = question_get.children[0].id
+formdata_update = MultiDict({"name": "Foo",
+ "children-0-id": child_id,
+ "children-0-name": "pizza"})
+form = ParentForm(formdata_update, obj=question_get)
form.populate_obj(question_get)
if not form.validate():
raise RuntimeError(form.errors)
Nice sleuthing. Still this seems rather cumbersome. Have you subsequently found a easier way to do an update on a form with a one to many relationship? WTForms are supposed to save developer time. But this seems like more overhead than doing it without WTForms.