ProgrammingError - sqlalchemy - on_conflict_do_update - sqlalchemy

Following this question:
As Ilja Everilä mentioned in his answer, I created a table object:
from sqlalchemy import *
metadata = MetaData()
idTagTable = Table('id_tag', metadata,
Column('id', String(255), primary_key = True),
Column('category', String(20), nullable = False),
Column('createddate', Date, nullable = False),
Column('updatedon', Date, nullable = False)
)
After creating a table object, I changed insert and update statements:
insert_statement = sqlalchemy.dialects.postgresql.insert(idTagTable)
upsert_statement = insert_statement.on_conflict_do_update(
constraint=PrimaryKeyConstraint('id'),
set_={"updatedon": insert_statement.excluded.updateon,
"category":insert_statement.excluded.category}
)
insert_values = df.to_dict(orient='records')
conn.execute(upsert_statement, insert_values)
Now I am getting Programming Error:
Traceback (most recent call last):
File "<ipython-input-66-0fc6a1bf9c6b>", line 7, in <module>
conn.execute(upsert_statement, insert_values)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
return meth(self, multiparams, params)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
compiled_sql, distilled_params
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
context)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
exc_info
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1159, in _execute_context
context)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 467, in do_executemany
cursor.executemany(statement, parameters)
ProgrammingError: (psycopg2.ProgrammingError) syntax error at or near
")"
LINE 1: ...category) VALUES ('sports') ON CONFLICT () DO UPDAT...
^
Not Able to understand why I am getting this error.

The PrimaryKeyConstraint object you're using as constraint= argument is not bound to any table and would seem to produce nothing when rendered, as seen in ON CONFLICT (). Instead pass the primary key(s) of your table as the conflict_target and Postgresql will perform unique index inference:
upsert_statement = insert_statement.on_conflict_do_update(
constraint=idTagTable.primary_key,
set_={"updatedon": insert_statement.excluded.updateon,
"category":insert_statement.excluded.category}
)

Related

sqlalchemy Object with OrderingList - Session.commit() method fails

Model that has attribute in type of sqlalchemy OrderingList fails on Database Session Commit step (as Database I am using PostgreSQL 13):
from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import relationship
from database.base_class import Base
from sqlalchemy.ext.orderinglist import ordering_list
class ContentType(Enum):
SCENE_HEADING = 'scene_heading'
ACTION = 'action'
CHARACTER = 'character'
PARENTHETICAL = 'parenthetical'
DIALOGUE = 'dialogue'
SHOT = 'shot'
TRANSITION = 'transition'
TEXT = 'text'
class EditorElement(Base):
"""
Class that represents the Table of Editor Elements.
"""
id = Column(Integer, primary_key=True, autoincrement=True, index=True)
content = Column(String, nullable=True)
content_type = Column(ContentType, nullable=True)
screenplay_id = Column(Integer, ForeignKey('screenplays.id', ondelete="CASCADE"), nullable=False)
screenplay = relationship("Screenplay", back_populates="elements")
class Screenplay(Base):
"""
Class that represents the screenplay table in the database.
"""
id = Column(Integer, primary_key=True, autoincrement=True, index=True)
name = Column(String, index=True, nullable=False)
elements = relationship("EditorElement", back_populates="screenplay", order_by="EditorElement.content_type",
collection_class=ordering_list('content_type'))
I am using FastAPI framework and in the CRUD file where the Screenplay is created, I have the following class:
class CRUDScreenplay(CRUDBase[models.Screenplay, schemas.ScreenplayBase, schemas.ScreenplayBase]):
#staticmethod
def create(db: Session, *, obj_in: schemas.ScreenplayCreate) -> models.Screenplay:
db_obj = models.Screenplay()
db_obj.name = obj_in['name']
for element in obj_in.get('elements'):
e = models.EditorElement(
content=element.content,
content_type=element.content_type,
screenplay_id=db_obj.id
)
db_obj.elements.append(e)
db.add(db_obj)
db.commit()
db.refresh(db_obj)
return db_obj
Using the PyCharm debugger I was able to check when the error occurs, it seems to appear after Session.commit(). Here is the object elements before the commit() method
And here is the elements object after the commit() method.
And the error in the console:
INFO: 127.0.0.1:52649 - "POST /api/v1/screenplays/ HTTP/1.1" 500 Internal Server Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\sql\sqltypes.py", line 1649, in _object_value_for_elem
return self._object_lookup[elem]
KeyError: 'scene_heading'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 372, in run_asgi
result = await app(self.scope, self.receive, self.send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\uvicorn\middleware\proxy_headers.py", line 75, in __call__
return await self.app(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\fastapi\applications.py", line 261, in __call__
await super().__call__(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\middleware\errors.py", line 181, in __call__
raise exc
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\middleware\errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\middleware\cors.py", line 84, in __call__
await self.app(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\exceptions.py", line 82, in __call__
raise exc
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\fastapi\middleware\asyncexitstack.py", line 21, in __call__
raise e
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\routing.py", line 656, in __call__
await route.handle(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\routing.py", line 259, in handle
await self.app(scope, receive, send)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\routing.py", line 61, in app
response = await func(request)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\fastapi\routing.py", line 235, in app
response_data = await serialize_response(
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\fastapi\routing.py", line 130, in serialize_response
value, errors_ = await run_in_threadpool(
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\starlette\concurrency.py", line 39, in run_in_threadpool
return await anyio.to_thread.run_sync(func, *args)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\anyio\to_thread.py", line 28, in run_sync
return await get_asynclib().run_sync_in_worker_thread(func, *args, cancellable=cancellable,
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\anyio\_backends\_asyncio.py", line 818, in run_sync_in_worker_thread
return await future
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\anyio\_backends\_asyncio.py", line 754, in run
result = context.run(func, *args)
File "pydantic\fields.py", line 854, in pydantic.fields.ModelField.validate
File "pydantic\fields.py", line 1071, in pydantic.fields.ModelField._validate_singleton
File "pydantic\fields.py", line 1118, in pydantic.fields.ModelField._apply_validators
File "pydantic\class_validators.py", line 313, in pydantic.class_validators._generic_validator_basic.lambda12
File "pydantic\main.py", line 678, in pydantic.main.BaseModel.validate
File "pydantic\main.py", line 562, in pydantic.main.BaseModel.from_orm
File "pydantic\main.py", line 1001, in pydantic.main.validate_model
File "pydantic\utils.py", line 409, in pydantic.utils.GetterDict.get
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\orm\attributes.py", line 481, in __get__
return self.impl.get(state, dict_)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\orm\attributes.py", line 941, in get
value = self._fire_loader_callables(state, key, passive)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\orm\attributes.py", line 977, in _fire_loader_callables
return self.callable_(state, passive)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\orm\strategies.py", line 911, in _load_for_state
return self._emit_lazyload(
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\orm\strategies.py", line 1051, in _emit_lazyload
result = result.unique().scalars().all()
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\engine\result.py", line 1371, in all
return self._allrows()
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\engine\result.py", line 401, in _allrows
rows = self._fetchall_impl()
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\engine\result.py", line 1284, in _fetchall_impl
return self._real_result._fetchall_impl()
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\engine\result.py", line 1696, in _fetchall_impl
return list(self.iterator)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\orm\loading.py", line 147, in chunks
fetch = cursor._raw_all_rows()
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\engine\result.py", line 393, in _raw_all_rows
return [make_row(row) for row in rows]
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\engine\result.py", line 393, in <listcomp>
return [make_row(row) for row in rows]
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\sql\sqltypes.py", line 1768, in process
value = self._object_value_for_elem(value)
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\sql\sqltypes.py", line 1651, in _object_value_for_elem
util.raise_(
File "C:\Users\userh\Playground\screenplay-writer\backend\.venv\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
LookupError: 'scene_heading' is not among the defined enum values. Enum name: None. Possible values: None
However, the data was successfully created in the database, but FastAPI does not return anything due to this error.
There are some related issues in GitHub but I am was not able to figure out what is going on there:
Issue 1
Issue 2
Issue 3
Issue 4
UPDATE
Alembic (which is often used with FastAPI) or SLQAlchemy created the PostgreSQL ENUM Type in Uppercase.
I changed the definition of ContentType(enum.Enum) as it was in the example of the official documentation LINK. Thanks to #fchancel for pointing on that:
class ContentType(str, enum.Enum):
HEADING = 'HEADING'
ACTION = 'ACTION'
CHARACTER = 'CHARACTER'
PARENTHETICAL = 'PARENTHETICAL'
DIALOGUE = 'DIALOGUE'
SHOT = 'SHOT'
TRANSITION = 'TRANSITION'
TEXT = 'TEXT'
So, now my code works
It seems that when creating your elements, the content_type section is a problem, since it can't find heading_scene. To understand this, we need to see how you call your function and what obj_in contains.
However, it's possible that the source of the error is just your class ContentType(Enum) which to work properly should be class ContentType(str, Enum)

SQLAlchemy many-to-many UNIQUE constraint failed association table

I connected the tables 'projects' and 'project_freelancer' through a many-to-many relationship, because one project can have multiple freelancer and one freelancer can do multiple projects.
In 'projects' I have only one primary key 'url'.
In 'projects_freelancer' I have a composite primary key out of 'url' and 'freel_url'
Therefore I definded my association table like this:
association_table = Table('association', Base.metadata,
Column('projects_url', ForeignKey('projects.url', ondelete="CASCADE"), primary_key=True),
Column('project_freelancer_url'),
Column('project_freelancer_freel_url'),
ForeignKeyConstraint(
('project_freelancer_url', 'project_freelancer_freel_url'),
('project_freelancer.url', 'project_freelancer.freel_url')),
)
And here are my class-definitions (short version):
class Project(Base):
__tablename__ = "projects"
url = Column(String, primary_key=True)
datetime = Column(DateTime)
children2 = relationship("ProjectFree", secondary=association_table, back_populates='parents2', cascade="all, delete") # Many-to-many relationship
class ProjectFree(Base):
__tablename__ = "project_freelancer"
url = Column(String, primary_key=True)
freel_url = Column(String, primary_key=True)
update = Column(DateTime)
parents2 = relationship("Project", secondary=association_table, back_populates="children2", cascade="all, delete")
When filling my db I get the following error message:
ERROR [21-11-18 09:58:47] scrapy.core.scraper _itemproc_finished :249 Error processing {'added': datetime.datetime(2021, 11, 18, 9, 58, 47, 654089),
'freel_bidtext': 'Hi,\n'
'\n'
'I hope all is well with you.\n'
'\n'
'I have read the job description and finds it a perfect '
'match for me as I have extensive hands-on experience in '
'React JS/Vue JS development.\n'
'\n'
'I have developed a lot of websites and sto',
'freel_country': 'Pakistan',
'freel_deadline': 'in 7 days',
'freel_earnings': '0.0',
'freel_name': 'abdulhaseebbasit',
'freel_price': '$140 USD',
'freel_ranking': 3,
'freel_rating': '0.0',
'freel_reviews': '0',
'freel_url': 'https://www.freelancer.com/u/abdulhaseebbasit',
'update': datetime.datetime(1900, 1, 1, 0, 0),
'url': 'https://www.freelancer.com/projects/javascript/looking-for-react-developer-32135012/?ngsw-bypass=&w=f'}
Traceback (most recent call last):
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1802, in _execute_context
self.dialect.do_execute(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\default.py", line 719, in do_execute
cursor.execute(statement, parameters)
sqlite3.IntegrityError: UNIQUE constraint failed: association.projects_url
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\twisted\internet\defer.py", line 858, in _runCallbacks
current.result = callback( # type: ignore[misc]
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\scrapy\utils\defer.py", line 150, in f
return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
File "C:\Users\myuser\PycharmProjects\crawlerDiss\diss\diss\pipelines.py", line 42, in process_item
return self.handle_ProjectFreelancer(item, spider)
File "C:\Users\myuser\PycharmProjects\crawlerDiss\diss\diss\pipelines.py", line 160, in handle_ProjectFreelancer
session.commit()
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\session.py", line 1428, in commit
self._transaction.commit(_to_root=self.future)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\session.py", line 829, in commit
self._prepare_impl()
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\session.py", line 808, in _prepare_impl
self.session.flush()
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\session.py", line 3345, in flush
self._flush(objects)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\session.py", line 3485, in _flush
transaction.rollback(_capture_exception=True)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\util\langhelpers.py", line 70, in __exit__
compat.raise_(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\session.py", line 3445, in _flush
flush_context.execute()
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\unitofwork.py", line 456, in execute
rec.execute(self)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\unitofwork.py", line 579, in execute
self.dependency_processor.process_saves(uow, states)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\dependency.py", line 1182, in process_saves
self._run_crud(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\orm\dependency.py", line 1245, in _run_crud
connection.execute(statement, secondary_insert)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1289, in execute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\sql\elements.py", line 325, in _execute_on_connection
return connection._execute_clauseelement(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1481, in _execute_clauseelement
ret = self._execute_context(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1845, in _execute_context
self._handle_dbapi_exception(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\base.py", line 2026, in _handle_dbapi_exception
util.raise_(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1802, in _execute_context
self.dialect.do_execute(
File "c:\users\myuser\pycharmprojects\crawlerdiss\venv\lib\site-packages\sqlalchemy\engine\default.py", line 719, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) UNIQUE constraint failed: association.projects_url
[SQL: INSERT INTO association (projects_url, project_freelancer_url, project_freelancer_freel_url) VALUES (?, ?, ?)]
[parameters: ('https://www.freelancer.com/projects/javascript/looking-for-react-developer-32135012/?ngsw-bypass=&w=f', 'https://www.freelancer.com/projects/javascript/looking-for-react-developer-32135012/?ngsw-bypass=&w=f', 'https://www.freelancer.com/u/abdulhaseebbasit')]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
In the db-browser I can see, that only unique combinations are filled into the association table. So for a project already in the association table, with a different freelancer the error occurs. Do you have an idea what I have to change? Should I change the unique behavior anywhere?
Finally I get the solution and it was quite easy (as is so often the case). In the association table, I had to add , primary_key = True for each Foreign Key. Here is the final association_table:
association_table = Table('association', Base.metadata,
Column('projects_url', ForeignKey('projects.url', ondelete="CASCADE"), primary_key=True),
Column('project_freelancer_url', primary_key = True),
Column('project_freelancer_freel_url', primary_key = True),
ForeignKeyConstraint(
('project_freelancer_url', 'project_freelancer_freel_url'),
('project_freelancer.url', 'project_freelancer.freel_url')),
)
The idea come to my mind by reading the documentations tip here.

How to use marshmallow-sqlalchemy with async code?

I'm trying to use marshmallow-sqlalchemy with aiohttp and I have followed their docs with the basic example and I'm getting an error.
I have this schema:
from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
from db.customer import Customer
class CustomerSchema(SQLAlchemyAutoSchema):
class Meta:
model = Customer
include_relationships = True
load_instance = True
And then the following code for the query:
from sqlalchemy import select
from db import db_conn
from db.customer import Customer
from queries.schema import CustomerSchema
customer_schema = CustomerSchema()
async def get_all_users():
async with db_conn.get_async_sa_session() as session:
statement = select(Customer)
results = await session.execute(statement)
_ = (results.scalars().all())
print(_)
response = customer_schema.dump(_, many=True)
print(response)
For the first print statement I'm getting
[<db.customer.Customer object at 0x10a183340>, <db.customer.Customer object at 0x10a183940>, <db.customer.Customer object at 0x10b0cd9d0>]
But then it fails with
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 60, in await_only
raise exc.MissingGreenlet(
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: http://sqlalche.me/e/14/xd2s)
So how can I use marshmallow-sqlalchemy to serialize the SqlAlchemy reponse?
Another options (packages, etc) or a generic custom solutions are OK too.
For the time being I'm using this:
statement = select(Customer)
results = await session.execute(statement)
_ = (results.scalars().all())
response = {}
for result in _:
value = {k: (v if not isinstance(v, sqlalchemy.orm.state.InstanceState) else '_') for k, v in result.__dict__.items()}
response[f'customer {value["id"]}'] = value
return response
Full traceback:
Traceback (most recent call last):
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/aiohttp/web_protocol.py", line 422, in _handle_request
resp = await self._request_handler(request)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/aiohttp/web_app.py", line 499, in _handle
resp = await handler(request)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/aiohttp/web_urldispatcher.py", line 948, in _iter
resp = await method()
File "/Users/ruslan/OneDrive/Home/Dev/projects/code/education/other/cft/views/user.py", line 24, in get
await get_all_users()
File "/Users/ruslan/OneDrive/Home/Dev/projects/code/education/other/cft/queries/user.py", line 18, in get_all_users
response = customer_schema.dump(_, many=True)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 547, in dump
result = self._serialize(processed_obj, many=many)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 509, in _serialize
return [
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 510, in <listcomp>
self._serialize(d, many=False)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 515, in _serialize
value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/fields.py", line 310, in serialize
value = self.get_value(obj, attr, accessor=accessor)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow_sqlalchemy/fields.py", line 27, in get_value
return super(fields.List, self).get_value(obj, attr, accessor=accessor)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/fields.py", line 239, in get_value
return accessor_func(obj, check_key, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/schema.py", line 472, in get_attribute
return get_value(obj, attr, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/utils.py", line 239, in get_value
return _get_value_for_key(obj, key, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/marshmallow/utils.py", line 253, in _get_value_for_key
return getattr(obj, key, default)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 480, in __get__
return self.impl.get(state, dict_)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 931, in get
value = self.callable_(state, passive)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 879, in _load_for_state
return self._emit_lazyload(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 1036, in _emit_lazyload
result = session.execute(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1689, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1582, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/sql/lambdas.py", line 481, in _execute_on_connection
return connection._execute_clauseelement(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1451, in _execute_clauseelement
ret = self._execute_context(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1813, in _execute_context
self._handle_dbapi_exception(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1998, in _handle_dbapi_exception
util.raise_(exc_info[1], with_traceback=exc_info[2])
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1770, in _execute_context
self.dialect.do_execute(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 449, in execute
self._adapt_connection.await_(
File "/Users/ruslan/.local/share/virtualenvs/cft-RKlbQ9iX/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 60, in await_only
raise exc.MissingGreenlet(
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: http://sqlalche.me/e/14/xd2s)
The problem in this case is that the Marshmallow schema is configured to load related models (include_relationships=True). Since the initial query doesn't load them automatically, the schema triggers a query to fetch them, and this causes the error.
The simplest solution, demonstrated in the docs, is to eagerly load the related objects with their "parent":
async def get_all_users():
async with db_conn.get_async_sa_session() as session:
# Let's assume a Customer has a 1 to many relationship with an Order model
statement = select(Customer).options(orm.selectinload(Customer.orders))
results = await session.execute(statement)
_ = (results.scalars().all())
print(_)
response = customer_schema.dump(_, many=True)
print(response)
There is more discussion in the Preventing Implicit IO when Using AsyncSession section of the docs.

I'm getting django.db.utils.IntegrityError: FOREIGN KEY constraint failed in cmd

I copied a json file from online. I'm using it for django project.
I entered the following lines in cmd -
>>> import json
>>> from blog.models import Post
>>> with open('posts.json') as f:
... posts_json=json.load(f)
...
>>> for post in posts_json:
... post = Post(title=post['title'], content=post['content'], author_id=post['
user_id'])
... post.save()
...
And the error I got was -
Traceback (most recent call last):
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\sqlite3\base.py", line 396, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.IntegrityError: FOREIGN KEY constraint failed
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<console>", line 3, in <module>
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\base.py", line 745, in save
self.save_base(using=using, force_insert=force_insert,
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\base.py", line 782, in save_base
updated = self._save_table(
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\base.py", line 887, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields
, raw)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\base.py", line 924, in _do_insert
return manager._insert(
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\query.py", line 1204, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\models\sql\compiler.py", line 1391, in execute_sql
cursor.execute(sql, params)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\utils.py", line 100, in execute
return super().execute(sql, params)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._e
xecute)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\Admin\.virtualenvs\my_projects-I2onQxk3\lib\site-packages\djang
o\db\backends\sqlite3\base.py", line 396, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.IntegrityError: FOREIGN KEY constraint failed
This is my models.py file -
from django.db import models
from django.utils import timezone
from django.contrib.auth.models import User
from django.urls import reverse
class Post(models.Model):
title=models.CharField(max_length=100)
content=models.TextField()
date_posted=models.DateTimeField(default=timezone.now)
author=models.ForeignKey(User,on_delete=models.CASCADE)
def __str__(self):
return self.title
def get_absolute_url(self):
return reverse('post-detail',kwargs={'pk':self.pk})
This is the json file I copied from online https://raw.githubusercontent.com/CoreyMSchafer/code_snippets/master/Django_Blog/snippets/posts.json
This is a later reply but maybe it will help someone else in the future.
I had the same issue now and the same code and posts.json as well, it seems that in posts.json the user_id it either 1 or 2 so the error means that there is no user with id 1 or 2 in your database.
I would say that there is no user with id = 2 I supposed you have created a superuser which is id = 1 and if you create a second user it may not take id = 2, so you need to find out what is that id and change it in your posts.json file.
That's how I just fixed my problem.
Sorry for my English.
The only foreign key here is author, so a foreign key constraint failing must mean that there's no user with an id corresponding to the user_id field in a record in that data.
Since by a quick glance it looks like there's only user ids 1 and 2, make sure you have those two users in your database, then try again.

Sqlalchemy class _MatchType(sqltypes.Float, sqltypes.MatchType): AttributeError

sometimes execute query and got this error:
ct = db.session.query(CIType).filter(
CIType.type_name == key).first() or \
db.session.query(CIType).filter(CIType.type_id == key).first()
full error info
2016-08-11 14:27:26,177 ERROR /usr/lib/python2.6/site-packages/flask/app.py 1306 - Exception on /api/v0.1/projects/search-indexer-rafael/product [GET]
Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/flask/app.py", line 1687, in wsgi_app
response = self.full_dispatch_request()
File "/usr/lib/python2.6/site-packages/flask/app.py", line 1360, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/lib/python2.6/site-packages/flask/app.py", line 1358, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/lib/python2.6/site-packages/flask/app.py", line 1344, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/data/webapps/cmdb-api/core/special.py", line 175, in get_project_product
product = ProjectManager().get_for_product(project_name)
File "/data/webapps/cmdb-api/lib/special/project.py", line 18, in __init__
self.ci_type = CITypeCache.get("project")
File "/data/webapps/cmdb-api/models/cmdb.py", line 458, in get
ct = db.session.query(CIType).filter(
File "/usr/lib64/python2.6/site-packages/sqlalchemy/orm/scoping.py", line 149, in do
def do(self, *args, **kwargs):
File "/usr/lib64/python2.6/site-packages/sqlalchemy/util/_collections.py", line 903, in __call__
item = dict.get(self, key)
File "/usr/lib/python2.6/site-packages/flask_sqlalchemy.py", line 201, in __init__
bind=db.engine,
File "/usr/lib/python2.6/site-packages/flask_sqlalchemy.py", line 754, in engine
return self.get_engine(self.get_app())
File "/usr/lib/python2.6/site-packages/flask_sqlalchemy.py", line 771, in get_engine
return connector.get_engine()
File "/usr/lib/python2.6/site-packages/flask_sqlalchemy.py", line 451, in get_engine
self._engine = rv = sqlalchemy.create_engine(info, **options)
File "/usr/lib64/python2.6/site-packages/sqlalchemy/engine/__init__.py", line 344, in create_engine
of 0 indicates no limit; to disable pooling, set ``poolclass`` to
File "/usr/lib64/python2.6/site-packages/sqlalchemy/engine/strategies.py", line 50, in create
File "/usr/lib64/python2.6/site-packages/sqlalchemy/engine/url.py", line 116, in get_dialect
return self.get_dialect().driver
File "/usr/lib64/python2.6/site-packages/sqlalchemy/util/langhelpers.py", line 170, in load
fn.__func__.__doc__ = doc
File "/usr/lib64/python2.6/site-packages/sqlalchemy/dialects/__init__.py", line 33, in _auto_fn
try:
File "/usr/lib64/python2.6/site-packages/sqlalchemy/dialects/mysql/__init__.py", line 8, in <module>
from . import base, mysqldb, oursql, \
File "/usr/lib64/python2.6/site-packages/sqlalchemy/dialects/mysql/base.py", line 681, in <module>
class _MatchType(sqltypes.Float, sqltypes.MatchType):
AttributeError: 'module' object has no attribute 'MatchType'
code
#special.route("/api/v0.1/projects/<string:project_name>/product",
methods=["GET"])
def get_project_product(project_name):
product = ProjectManager().get_for_product(project_name)
return jsonify(product=product)
...
goto
class ProjectManager(object):
def __init__(self):
self.ci_type = CITypeCache.get("project")
...
then
class CITypeCache(object):
#classmethod
def get(cls, key):
if key is None:
return
ct = cache.get("CIType::ID::%s" % key) or \
cache.get("CIType::Name::%s" % key)
if ct is None:
ct = db.session.query(CIType).filter(
CIType.type_name == key).first() or \
db.session.query(CIType).filter(CIType.type_id == key).first()
if ct is not None:
CITypeCache.set(ct)
return ct
The sqlalchemy's version is SQLAlchemy-1.0.8-py2.6.egg-info
and after many same error, I can't catch this error any more. What's the reason of this error?
I assume CIType.type_name and CIType.type_id have different data types (perhaps string and numeric types). It may lead to situation when:
db.session.query(CIType).filter(CIType.type_name == key).first()
is valid expression but:
db.session.query(CIType).filter(CIType.type_id == key).first()
produces error because of type mismatch. You need to convert key to the type_id column type in this expression.
The second expression is calculated when first expression returns no results. As written in Python documentation:
The expression x or y first evaluates x; if x is true, its value is returned; otherwise, y is evaluated and the resulting value is returned.
For example:
>>> a = 1 or 2 + '2'
>>> print a
1
>>> a = 0 or 2 + '2'
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: unsupported operand type(s) for +: 'int' and 'str'