I have an array of the objects and then try to serialize it using the following statement:
serializer = MovieWithDescriptionSerializer(movies, many=True)
data = serializer.data
The class and the serializer are as below:
class MovieWithDescription(object):
id = 0
name = ''
description = ''
rating = ''
year = 0
def __init__(self, uid, name, description):
self.id = uid
self.name = name
self.description = description
class MovieWithDescriptionSerializer(serializers.Serializer):
class Meta:
model = MovieWithDescription
fields = ('id', 'name', 'description')
id = serializers.IntegerField()
name = serializers.StringRelatedField()
description = serializers.StringRelatedField()
The data is saved to session:
request.session['movies'] = data
And read on the other page:
movies = request.session['movies']
However when I tried to deserialize it I learned that the movies variable contains list. So it looks like I don't need to deserialize and just need to iterate through the list to process the data. What I'm doing wrong with this serialization? Is there any more simple way to serialize data than to use Django Rest Framework?
Answering your question from the comments section; also to clarify for whoever is migrating from a Java environment:
In Django Rest Framework; Serialization happens as follows:
Python Object --> serializer.data ==> Python Native Types --> Renderer Class ==> JSON
serializer.data is constructed by calling to_representation on each of the fields declared on the Serializer.
#property
def data(self):
...
if not hasattr(self, '_data'):
if self.instance is not None and not getattr(self, '_errors', None):
self._data = self.to_representation(self.instance)
elif hasattr(self, '_validated_data') and not getattr(self, '_errors', None):
self._data = self.to_representation(self.validated_data)
else:
self._data = self.get_initial()
return self._data
To representation is supposed to take an object instance and return a dict of primitive types:
def to_representation(self, instance):
"""
Object instance -> Dict of primitive datatypes.
"""
...
So if serializer.data does not render to JSON, where does this magic happen?
It happens inside the Response object. When you construct a Response object with the data attribute (data here is a dict of primitive or a list of dict of primitives), the rendered_content method then defines how the data is rendered, sets the appropriate Content-Type headers, and so on.
#property
def rendered_content(self):
renderer = getattr(self, 'accepted_renderer', None)
accepted_media_type = getattr(self, 'accepted_media_type', None)
context = getattr(self, 'renderer_context', None)
assert renderer, ".accepted_renderer not set on Response"
assert accepted_media_type, ".accepted_media_type not set on Response"
assert context is not None, ".renderer_context not set on Response"
context['response'] = self
media_type = renderer.media_type
charset = renderer.charset
content_type = self.content_type
if content_type is None and charset is not None:
content_type = "{0}; charset={1}".format(media_type, charset)
elif content_type is None:
content_type = media_type
self['Content-Type'] = content_type
ret = renderer.render(self.data, accepted_media_type, context)
if isinstance(ret, six.text_type):
assert charset, (
'renderer returned unicode, and did not specify '
'a charset value.'
)
return bytes(ret.encode(charset))
if not ret:
del self['Content-Type']
return ret
This allows you to do something neat; you can define a single Django Rest Framework View, that can render XML, JSON, and many more -> you can find them under rest_framework.renderers.
For example, you can define an APIView that supports multiple render formats based on the query parameter (?format=json, ?format=xml, ?format=csv) as follows:
class ExampleAPIView(APIView):
renderer_classes = (JSONRenderer, XMLRenderer, CSVRenderer)
...
XML and CSV require additional package installations. Read more here: http://www.django-rest-framework.org/api-guide/renderers/#xml
Related
I've written Graphene models for polymorphic entities represented in my database w/SQLalchemy.
The problem is simple:
I want to create an interface that reflects my SQLAlchemy models for Graphene but also either a) implements Node or b) does not conflict with Node and allows me to retrieve the model's ID without needing to add ... on Node {id} to the query string.
have to exclude the ID field from my ORM-based interface or field conflicts with the Node interface, by doing so in order to get the ID then you need to add ...on Node { id }, which is ugly.
I created an SQLAlchemyInterface object that extends graphene.Interface. Many (but not all) of my models used this as well as Node as interfaces. The first problem was that this contains an ID field and it conflicted with the Node interface.
I excluded the id field to not interfere with Node, but then found I could not directly query ID on my models anymore, and had to add ... on Node {id} to the query string.
I then decided to have this SQLAlchemyInterface extend Node. I don't love this approach because I need to use another (named) Node interface for all of my models that don't necessarily need to implement SQLAlchemyInterface
class SQLAlchemyInterface(Node):
#classmethod
def __init_subclass_with_meta__(
cls,
model=None,
registry=None,
only_fields=(),
exclude_fields=(),
connection_field_factory=default_connection_field_factory,
_meta=None,
**options
):
_meta = SQLAlchemyInterfaceOptions(cls)
_meta.name = f'{cls.__name__}Node'
autoexclude_columns = exclude_autogenerated_sqla_columns(model=model)
exclude_fields += autoexclude_columns
assert is_mapped_class(model), (
"You need to pass a valid SQLAlchemy Model in " '{}.Meta, received "{}".'
).format(cls.__name__, model)
if not registry:
registry = get_global_registry()
assert isinstance(registry, Registry), (
"The attribute registry in {} needs to be an instance of "
'Registry, received "{}".'
).format(cls.__name__, registry)
sqla_fields = yank_fields_from_attrs(
construct_fields(
model=model,
registry=registry,
only_fields=only_fields,
exclude_fields=exclude_fields,
connection_field_factory=connection_field_factory
),
_as=Field
)
if not _meta:
_meta = SQLAlchemyInterfaceOptions(cls)
_meta.model = model
_meta.registry = registry
connection = Connection.create_type(
"{}Connection".format(cls.__name__), node=cls)
assert issubclass(connection, Connection), (
"The connection must be a Connection. Received {}"
).format(connection.__name__)
_meta.connection = connection
if _meta.fields:
_meta.fields.update(sqla_fields)
else:
_meta.fields = sqla_fields
super(SQLAlchemyInterface, cls).__init_subclass_with_meta__(_meta=_meta, **options)
#classmethod
def Field(cls, *args, **kwargs): # noqa: N802
return NodeField(cls, *args, **kwargs)
#classmethod
def node_resolver(cls, only_type, root, info, id):
return cls.get_node_from_global_id(info, id, only_type=only_type)
#classmethod
def get_node_from_global_id(cls, info, global_id, only_type=None):
try:
node: DeclarativeMeta = one_or_none(session=info.context.get('session'), model=cls._meta.model, id=global_id)
return node
except Exception:
return None
#staticmethod
def from_global_id(global_id):
return global_id
#staticmethod
def to_global_id(type, id):
return id
Interface impls, Models + Query code examples:
class CustomNode(Node):
class Meta:
name = 'UuidNode'
#staticmethod
def to_global_id(type, id):
return '{}:{}'.format(type, id)
#staticmethod
def get_node_from_global_id(info, global_id, only_type=None):
type, id = global_id.split(':')
if only_type:
# We assure that the node type that we want to retrieve
# is the same that was indicated in the field type
assert type == only_type._meta.name, 'Received not compatible node.'
if type == 'User':
return one_or_none(session=info.context.get('session'), model=User, id=global_id)
elif type == 'Well':
return one_or_none(session=info.context.get('session'), model=Well, id=global_id)
class ControlledVocabulary(SQLAlchemyInterface):
class Meta:
name = 'ControlledVocabularyNode'
model = BaseControlledVocabulary
class TrackedEntity(SQLAlchemyInterface):
class Meta:
name = 'TrackedEntityNode'
model = TrackedEntityModel
class Request(SQLAlchemyObjectType):
"""Request node."""
class Meta:
model = RequestModel
interfaces = (TrackedEntity,)
class User(SQLAlchemyObjectType):
"""User Node"""
class Meta:
model = UserModel
interfaces = (CustomNode,)
class CvFormFieldValueType(SQLAlchemyObjectType):
class Meta:
model = CvFormFieldValueTypeModel
interfaces = (ControlledVocabulary,)
common_field_kwargs = {'id': graphene.UUID(required=False), 'label': graphene.String(required=False)}
class Query(graphene.ObjectType):
"""Query objects for GraphQL API."""
node = CustomNode.Field()
te_node = TrackedEntity.Field()
cv_node = ControlledVocabulary.Field()
# Non-Tracked Entities:
users: List[User] = SQLAlchemyConnectionField(User)
# Generic Query for any Tracked Entity:
tracked_entities: List[TrackedEntity] = FilteredConnectionField(TrackedEntity, sort=None, filter=graphene.Argument(TrackedEntityInput))
# Generic Query for any Controlled Vocabulary:
cv: ControlledVocabulary = graphene.Field(ControlledVocabulary, controlled_vocabulary_type_id=graphene.UUID(required=False),
base_entry_key=graphene.String(required=False),
**common_field_kwargs)
cvs: List[ControlledVocabulary] = FilteredConnectionField(ControlledVocabulary, sort=None, filter=graphene.Argument(CvInput))
#staticmethod
def resolve_with_filters(info: ResolveInfo, model: Type[SQLAlchemyObjectType], **kwargs):
query = model.get_query(info)
log.debug(kwargs)
for filter_name, filter_value in kwargs.items():
model_filter_column = getattr(model._meta.model, filter_name, None)
log.debug(type(filter_value))
if not model_filter_column:
continue
if isinstance(filter_value, SQLAlchemyInputObjectType):
log.debug(True)
filter_model = filter_value.sqla_model
q = FilteredConnectionField.get_query(filter_model, info, sort=None, **kwargs)
# noinspection PyArgumentList
query = query.filter(model_filter_column == q.filter_by(**filter_value))
log.info(query)
else:
query = query.filter(model_filter_column == filter_value)
return query
def resolve_tracked_entity(self, info: ResolveInfo, **kwargs):
entity: TrackedEntity = Query.resolve_with_filters(info=info, model=BaseTrackedEntity, **kwargs).one()
return entity
def resolve_tracked_entities(self, info, **kwargs):
query = Query.resolve_with_filters(info=info, model=BaseTrackedEntity, **kwargs)
tes: List[BaseTrackedEntity] = query.all()
return tes
def resolve_cv(self, info, **kwargs):
cv: List[BaseControlledVocabulary] = Query.resolve_with_filters(info=info, model=BaseControlledVocabulary, **kwargs).one()
log.info(cv)
return cv
def resolve_cvs(self, info, **kwargs):
cv: List[BaseControlledVocabulary] = Query.resolve_with_filters(info=info, model=BaseControlledVocabulary, **kwargs).all()
return cv
schema:
schema = Schema(query=Query, types=[*tracked_members, *cv_members])
I would like to be able to not extend Node with SQLAlchemyInterface and rather add Node back to the list of interfaces for TrackedEntity and ControlledVocabulary but be able to perform a query like this:
query queryTracked {
trackedEntities{
id
(other fields)
... on Request {
(request specific fields)
}
}
I am using Marshmallow to send instance of my Decision class to JSON. However, this will also dump the attributes which are None, e.g. my attribute score will translate to null in JSON. After that I am unable to read the JSON again using the same approach.
https://repl.it/repls/VoluminousMulticoloredFacts
The last line is where it currently fails. I need to either NOT dump None to JSON or skip null during loading:
import json
from marshmallow import Schema, fields, post_load
json_data = """{
"appid": "2309wfjwef",
"strategy": "First Strategy"
}"""
# Output class definition
class Decision(object):
def __init__(self, appid = None, strategy = None, score = None):
self.appid = appid
self.strategy = strategy
self.score = score
class DecisionSchema(Schema):
appid = fields.Str()
strategy = fields.Str()
score = fields.Int()
#post_load
def make_decision(self, data):
return Decision(**data)
# Deserialization into object
dec_json = json.loads(json_data)
schema = DecisionSchema()
dec = schema.load(dec_json).data
print(dec.strategy)
# Dump results back to JSON
schema = DecisionSchema()
out = schema.dumps(dec)
print(out.data)
# Load back from dump
schema = DecisionSchema()
dec = schema.load(out).data
#print(dec.strategy) # returns error currently
An "official" answer from marshmallow development team can be found in this comment in the bugtracker:
Use a post_dump method.
from marshmallow import Schema, fields, post_dump
class BaseSchema(Schema):
SKIP_VALUES = set([None])
#post_dump
def remove_skip_values(self, data, **kwargs):
return {
key: value for key, value in data.items()
if value not in self.SKIP_VALUES
}
class MySchema(BaseSchema):
foo = fields.Field()
bar = fields.Field()
sch = MySchema()
sch.dump({'foo': 42, 'bar': None}).data # {'foo': 42}
As I point out in a further comment, there's a shortcoming: it will also remove None when the field's allow_none is True.
As I pointed out in my comment above this messes with the order if you use the
class Meta:
fields = (
'field1', 'field2'
)
ordered = True
To fix this I used this:
# Remove None fields
#pre_dump
def remove_skip_values(self, data):
return {
key: value for key, value in data.items()
if value is not None
}
This works for my dictonary of objects
I have a django model with a JSONField (django.contrib.postgres.fields.JSONField)
Is there any way that I can validate model data against a json schema file?
(pre-save)
Something like my_field = JSONField(schema_file=my_schema_file)
I wrote a custom validator using jsonschema in order to do this.
project/validators.py
import django
from django.core.validators import BaseValidator
import jsonschema
class JSONSchemaValidator(BaseValidator):
def compare(self, value, schema):
try:
jsonschema.validate(value, schema)
except jsonschema.exceptions.ValidationError:
raise django.core.exceptions.ValidationError(
'%(value)s failed JSON schema check', params={'value': value}
)
project/app/models.py
from django.db import models
from project.validators import JSONSchemaValidator
MY_JSON_FIELD_SCHEMA = {
'schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
'properties': {
'my_key': {
'type': 'string'
}
},
'required': ['my_key']
}
class MyModel(models.Model):
my_json_field = models.JSONField(
default=dict,
validators=[JSONSchemaValidator(limit_value=MY_JSON_FIELD_SCHEMA)]
)
That's what the Model.clean() method is for (see docs). Example:
class MyData(models.Model):
some_json = JSONField()
...
def clean(self):
if not is_my_schema(self.some_json):
raise ValidationError('Invalid schema.')
you could use cerberus to validate your data against a schema
from cerberus import Validator
schema = {'name': {'type': 'string'}}
v = Validator(schema)
data = {'name': 'john doe'}
v.validate(data) # returns "True" (if passed)
v.errors # this would return the error dict (or on empty dict in case of no errors)
it's pretty straightforward to use (also due to it's good documentation -> validation rules: http://docs.python-cerberus.org/en/stable/validation-rules.html)
I wrote a custom JSONField that extends models.JSONField and validates attribute's value by using jsonschema (Django 3.1, Python 3.7).
I didn't use the validators parameter for one reason: I want to let users define the schema dynamically.So I use a schema parameter, that should be:
None (by default): the field will behave like its parent class (no JSON schema validation support).
A dict object. This option is suitable for a small schema definition (for example: {"type": "string"});
A str object that describes a path to the file where the schema code is contained. This option is suitable for a big schema definition (to preserve the beauty of the model class definition code). For searching I use all enabled finders: django.contrib.staticfiles.finders.find().
A function that takes a model instance as an argument and returns a schema as dict object. So you can build a schema based on the state of the given model instance. The function will be called every time when the validate() is called.
myapp/models/fields.py
import json
from jsonschema import validators as json_validators
from jsonschema import exceptions as json_exceptions
from django.contrib.staticfiles import finders
from django.core import checks, exceptions
from django.db import models
from django.utils.functional import cached_property
class SchemaMode:
STATIC = 'static'
DYNAMIC = 'dynamic'
class JSONField(models.JSONField):
"""
A models.JSONField subclass that supports the JSON schema validation.
"""
def __init__(self, *args, schema=None, **kwargs):
if schema is not None:
if not(isinstance(schema, (bool, dict, str)) or callable(schema)):
raise ValueError('The "schema" parameter must be bool, dict, str, or callable object.')
self.validate = self._validate
else:
self.__dict__['schema_mode'] = False
self.schema = schema
super().__init__(*args, **kwargs)
def check(self, **kwargs):
errors = super().check(**kwargs)
if self.schema_mode == SchemaMode.STATIC:
errors.extend(self._check_static_schema(**kwargs))
return errors
def _check_static_schema(self, **kwargs):
try:
schema = self.get_schema()
except (TypeError, OSError):
return [
checks.Error(
f"The file '{self.schema}' cannot be found.",
hint="Make sure that 'STATICFILES_DIRS' and 'STATICFILES_FINDERS' settings "
"are configured correctly.",
obj=self,
id='myapp.E001',
)
]
except json.JSONDecodeError:
return [
checks.Error(
f"The file '{self.schema}' contains an invalid JSON data.",
obj=self,
id='myapp.E002'
)
]
validator_cls = json_validators.validator_for(schema)
try:
validator_cls.check_schema(schema)
except json_exceptions.SchemaError:
return [
checks.Error(
f"{schema} must be a valid JSON Schema.",
obj=self,
id='myapp.E003'
)
]
else:
return []
def deconstruct(self):
name, path, args, kwargs = super().deconstruct()
if self.schema is not None:
kwargs['schema'] = self.schema
return name, path, args, kwargs
#cached_property
def schema_mode(self):
if callable(self.schema):
return SchemaMode.DYNAMIC
return SchemaMode.STATIC
#cached_property
def _get_schema(self):
if callable(self.schema):
return self.schema
elif isinstance(self.schema, str):
with open(finders.find(self.schema)) as fp:
schema = json.load(fp)
else:
schema = self.schema
return lambda obj: schema
def get_schema(self, obj=None):
"""
Return schema data for this field.
"""
return self._get_schema(obj)
def _validate(self, value, model_instance):
super(models.JSONField, self).validate(value, model_instance)
schema = self.get_schema(model_instance)
try:
json_validators.validate(value, schema)
except json_exceptions.ValidationError as e:
raise exceptions.ValidationError(e.message, code='invalid')
Usage:
myapp/models/__init__.py
def schema(instance):
schema = {}
# Here is your code that uses the other
# instance's fields to create a schema.
return schema
class JSONSchemaModel(models.Model):
dynamic = JSONField(schema=schema, default=dict)
from_dict = JSONField(schema={'type': 'object'}, default=dict)
# A static file: myapp/static/myapp/schema.json
from_file = JSONField(schema='myapp/schema.json', default=dict)
Another solution using jsonschema for simple cases.
class JSONValidatedField(models.JSONField):
def __init__(self, *args, **kwargs):
self.props = kwargs.pop('props')
self.required_props = kwargs.pop('required_props', [])
super().__init__(*args, **kwargs)
def validate(self, value, model_instance):
try:
jsonschema.validate(
value, {
'schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
'properties': self.props,
'required': self.required_props
}
)
except jsonschema.exceptions.ValidationError:
raise ValidationError(
f'Value "{value}" failed schema validation.')
class SomeModel(models.Model):
my_json_field = JSONValidatedField(
props={
'foo': {'type': 'string'},
'bar': {'type': 'integer'}
},
required_props=['foo'])
I was following this tutorial as a starting point to using models/controllers with python. Bear with me as I'm learning python.
I'm basically trying to get a json representation of my database table with key:value pairs as one would typically expect.
I can't seem to figure out the proper way of doing this.
My Model looks like this:
from app import db
import json
# define a base model for other database tables to inherit
class Base(db.Model):
__abstract__ = True
id = db.Column(db.Integer,primary_key=True)
date_created = db.Column(db.DateTime, default=db.func.current_timestamp())
date_modified = db.Column(db.DateTime, default=db.func.current_timestamp(), onupdate=db.func.current_timestamp())
# define a members model
class Member(Base):
__tablename__ = "members"
# member name
fname = db.Column(db.String(48), nullable=False)
sname = db.Column(db.String(48), nullable=False)
title = db.Column(db.String(90), nullable=True)
# new instance instantiation procedure
def __init__(self, fname, sname, title):
self.fname = fname
self.sname = sname
self.title = title
def __repr__(self):
# return '<Member %r>' % (self.fname)
# return '[{"id":self.id,"fname":self.fname,"sname":self.sname,"title":self.title}]'
return json.dumps(self.__dict__)
that will return an error: TypeError: datetime.datetime(2015, 2, 18, 11, 50, 1) is not JSON serializable
My Controller looks like the following:
# import flask dependencies
from flask import Blueprint, request, jsonify, Response
# import the database object from the main app module
from app import db
# import module models (i.e. Members)
from app.team_members.models import Member
# define the blueprint: 'member' set its url prefix: app.url/members
team_members = Blueprint('member',__name__,url_prefix='/members')
# set the route and accepted methods
#team_members.route('/list/',methods=['GET'])
def list():
members = Member.query.all()
return jsonify(dict(('member%d' % i, member) for i, member in enumerate(Member.query.all(), start=1)))
# resp = Response(response=members, status=200, mimetype="application/json")
# return resp
I understand that jsonify will not return a list, but a dictionary for security reasons. And If I understand correctly, I need __repr__ to return a string.
Any help would be much appreciated.
I am overwriting the get_obj_list function. Following some parameters and a random function I would like to return an object related to the actual object that follows the parameters. This works fine. How can I return only this obj instead of a one-entry-list? Is there another function that better fits my purpose?
class SentenceRandomResource(ModelResource):
class Meta:
queryset = Sentence.objects.filter()
resource_name = 'sentence/random'
always_return_data = True
authorization = ReadOnlyAuthorization()
filtering = {'internal': ALL}
def obj_get_list(self, bundle, **kwargs):
if 'case' in bundle.request.GET.keys() and 'lemma' in bundle.request.GET.keys() :
if 'number' in bundle.request.GET.keys() :
words = Word.objects.filter(case = bundle.request.GET['case'], number = bundle.request.GET['number'], lemma = bundle.request.GET['lemma'])
else :
words = Word.objects.filter(case = bundle.request.GET['case'], lemma = bundle.request.GET['lemma'])
number_of_words = len(words)
if number_of_words > 0 :
random_index = int(random.random()*number_of_words)+0
random_word = words[random_index]
sentence = random_word.sentence
return [sentence]
else: ...
else: ...
Thanks to method prepend_url you may add some special functionality not included in RESTful principles.
import random
from tastypie.http import HttpBadRequest
class SentenceRandomResource(ModelResource):
class Meta:
queryset = Sentence.objects.filter()
resource_name = 'sentence/random'
always_return_data = True
authorization = ReadOnlyAuthorization()
filtering = {'internal': ALL}
def prepend_urls(self, *args, **kwargs):
name = 'get_one_random'
return [url(r"^(?P<resource_name>%s)/%s%s$" %
(self._meta.resource_name, name, trailing_slash()),
self.wrap_view(name), name="api_%s" % name)]
def get_one_random(self, request, **kwargs):
"""
Gets one random sentence of sentences with provided `case` and `lemma`
params.
"""
case = request.GET.get('case')
lemma = request.GET.get('lemma')
number = request.GET.get('number')
if case and lemma:
query_params = {'case': case, 'lemma': lemma}
if number is not None:
query_params['number'] = number
words = Word.objects.filter(**query_params)
word = random.choice(words)
return self.create_response(request, {'sentence': word.sentence.__dict__})
else:
return self.error_response(request, {'error': 'lemma and case are required.'},
response_class=HttpBadRequest)
Example use:
GET ..../sentence/random/get_one_random/?case=1&lemma=2
{'sentence': 'asdfasdf'}