Currently working on a quick little project in python and am attempting to encode an object into a JSON string. I've done this several times before without any problem except for now. Usually I just do the following.
def ClassToEncode :
def __init__(self, arg1, arg2, ..., argn) :
self.attr1 = arg1
self.attr2 = arg2
...
self.attrn = argn
...
def toJSON(self) :
return json.dumps(self, default=lambda o: o.__dict__)
But the problem is that one of my class attributes is a datetime.datetime object and I am being thrown the following error
AttributeError: 'datetime.datetime' object has no attribute '__dict__'
Any thoughts or wraparounds that could enable the functionality of including the datetime attribute into the JSON output??
Thanks in advance!
You can use the isoformat() method on a datetime object to convert it to an ISO-8601-formatted time string, then serialize it as JSON just fine. On the other end, call datetime.datetime.strptime() on the formatted string to convert it back into a datetime object:
>>> from datetime import datetime as dt
>>> now = dt.now()
>>> now
datetime.datetime(2014, 9, 4, 3, 19, 44, 214096)
>>> isonow = now.isoformat()
>>> isonow
'2014-09-04T03:19:44.214096'
>>> fmt = "%Y-%m-%dT%H:%M:%S.%f"
>>> newtime = dt.strptime(isonow, fmt)
>>> newtime
datetime.datetime(2014, 9, 4, 3, 19, 44, 214096)
Another way is to modify your toJSON() method to use a customized dictionary in which you customize the data:
import datetime
def ClassToEncode :
def __init__(self, arg1, arg2, ..., argn) :
self.attr1 = arg1
self.attr2 = arg2
...
self.attrn = datetime.datetime.utcnow()
...
def customDict(self):
dup = self.__dict__.copy()
# configure dup to contain fields that you want to send
dup['attrn'] = self.createdAt.isoformat() # datetime object
del dup['attr2'] # Some private field you may want to hide
return dup
def toJSON(self):
return json.dumps(self, default=lambda o: o.customDict())
While the accepted answer works great. If you don't need any date formatting you could quickly get away with direct string conversion.
So something like below should work
str(datetime.datetime.now())
json.dumps should be able to serialize this string now.
Related
I know I can use ruamel.yaml to load a file with tags in it. But when I want to dump without them i get an error. Simplified example :-
from ruamel.yaml import YAML
from json import dumps
import sys
yaml = YAML()
data = yaml.load(
"""
!mytag
a: 1
b: 2
c: 2022-05-01
"""
)
try:
yaml2 = YAML(typ='safe', pure=True)
yaml.default_flow_style = True
yaml2.dump(data, sys.stdout)
except Exception as e:
print('exception dumping using yaml', e)
try:
print(dumps(data))
except Exception as e:
print('exception dumping using json', e)
exception dumping using cannot represent an object: ordereddict([('a', 1), ('b', 2), ('c', datetime.date(2022, 5, 1))])
exception dumping using json Object of type date is not JSON serializable
I cannot change the load() without getting an error on the tag. How to get output with tags stripped (YAML or JSON)?
You get the error because the neither the safe dumper (pure or not), nor JSON, do know about the ruamel.yaml internal
types that preserve comments, tagging, block/flow-style, etc.
Dumping as YAML, you could register these types with alternate dump methods. As JSON this is more complex
as AFAIK you can only convert the leaf-nodes (i.e. the YAML scalars, you would e.g. be
able to use that to dump the datetime.datetime instance that is loaded as the value of key c).
I have used YAML as a readable, editable and programmatically updatable config file with
an much faster loading JSON version of the data used if its file is not older than the corresponding YAML (if
it is older JSON gets created from the YAML). The thing to do in order to dump(s) is
recursively generate Python primitives that JSON understands.
The following does so, but there are other constructs besides datetime
instances that JSON doesn't allow. E.g. when using sequences or dicts
as keys (which is allowed in YAML, but not in JSON). For keys that are
sequences I concatenate the string representation of the elements
:
from ruamel.yaml import YAML
import sys
import datetime
import json
from collections.abc import Mapping
yaml = YAML()
data = yaml.load("""\
!mytag
a: 1
b: 2
c: 2022-05-01
[d, e]: !myseq [42, 196]
{f: g, 18: y}: !myscalar x
""")
def json_dump(data, out, indent=None):
def scalar(obj):
if obj is None:
return None
if isinstance(obj, (datetime.date, datetime.datetime)):
return str(obj)
if isinstance(obj, ruamel.yaml.scalarbool.ScalarBoolean):
return obj == 1
if isinstance(obj, bool):
return bool(obj)
if isinstance(obj, int):
return int(obj)
if isinstance(obj, float):
return float(obj)
if isinstance(obj, tuple):
return '_'.join([str(x) for x in obj])
if isinstance(obj, Mapping):
return '_'.join([f'{k}-{v}' for k, v in obj.items()])
if not isinstance(obj, str): print('type', type(obj))
return obj
def prep(obj):
if isinstance(obj, dict):
return {scalar(k): prep(v) for k, v in obj.items()}
if isinstance(obj, list):
return [prep(elem) for elem in obj]
if isinstance(obj, ruamel.yaml.comments.TaggedScalar):
return prep(obj.value)
return scalar(obj)
res = prep(data)
json.dump(res, out, indent=indent)
json_dump(data, sys.stdout, indent=2)
which gives:
{
"a": 1,
"b": 2,
"c": "2022-05-01",
"d_e": [
42,
196
],
"f-g_18-y": "x"
}
I have this sqlite3.register_converter function :
def str_to_dict(s: ByteString) -> Dict:
if s and isinstance(s, ByteString):
s = s.decode('UTF-8').replace("'", '"')
return json.loads(s)
raise TypeError(f'value : "{s}" should be a byte string')
which returns this exception text :
File "/usr/lib64/python3.7/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 30 (char 29)
when encounter with this string :
s = b"{'foo': {'bar': [('for', 'grid')]}}"
It seems that the issue comes from the nested list/tuple/dictionary but what I don't understand is that in the sqlite shell, the value is correctly returned with a select command :
select * from table;
whereas the same command issued from a python script returned the exception above :
class SqliteDb:
def __init__(self, file_path: str = '/tmp/database.db'):
self.file_path = file_path
self._db = sqlite3.connect(self.file_path, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
if self._db:
self._cursor = self._db.cursor()
else:
raise ValueError
# register data types converters and adapters
sqlite3.register_adapter(Dict, dict_to_str)
sqlite3.register_converter('Dict', str_to_dict)
sqlite3.register_adapter(List, list_to_str)
sqlite3.register_converter('List', str_to_list)
def __del__(self):
self._cursor.close()
self._db.close()
def select_from(self, table_name: str):
with self._db:
query = f'SELECT * FROM {table_name}'
self._cursor.execute(query)
if __name__ == '__main__':
try:
sq = SqliteDb()
selection_item = sq.select_from("table")[0]
print(f'selection_item : {selection_item}')
except KeyboardInterrupt:
print('\n')
sys.exit(0)
s, the value is already saved in database with no issue. Only the selection causes this issue.
So, anybody has a clue why ?
Your input is really a Python dict literal, and contains structures such as the tuple ('for', 'grid') that cannot be directly parsed as JSON even after you replace single quotes with double quotes.
You can use ast.literal_eval instead to parse the input:
from ast import literal_eval
def str_to_dict(s: ByteString) -> Dict:
return literal_eval(s.decode())
I am using Marshmallow to send instance of my Decision class to JSON. However, this will also dump the attributes which are None, e.g. my attribute score will translate to null in JSON. After that I am unable to read the JSON again using the same approach.
https://repl.it/repls/VoluminousMulticoloredFacts
The last line is where it currently fails. I need to either NOT dump None to JSON or skip null during loading:
import json
from marshmallow import Schema, fields, post_load
json_data = """{
"appid": "2309wfjwef",
"strategy": "First Strategy"
}"""
# Output class definition
class Decision(object):
def __init__(self, appid = None, strategy = None, score = None):
self.appid = appid
self.strategy = strategy
self.score = score
class DecisionSchema(Schema):
appid = fields.Str()
strategy = fields.Str()
score = fields.Int()
#post_load
def make_decision(self, data):
return Decision(**data)
# Deserialization into object
dec_json = json.loads(json_data)
schema = DecisionSchema()
dec = schema.load(dec_json).data
print(dec.strategy)
# Dump results back to JSON
schema = DecisionSchema()
out = schema.dumps(dec)
print(out.data)
# Load back from dump
schema = DecisionSchema()
dec = schema.load(out).data
#print(dec.strategy) # returns error currently
An "official" answer from marshmallow development team can be found in this comment in the bugtracker:
Use a post_dump method.
from marshmallow import Schema, fields, post_dump
class BaseSchema(Schema):
SKIP_VALUES = set([None])
#post_dump
def remove_skip_values(self, data, **kwargs):
return {
key: value for key, value in data.items()
if value not in self.SKIP_VALUES
}
class MySchema(BaseSchema):
foo = fields.Field()
bar = fields.Field()
sch = MySchema()
sch.dump({'foo': 42, 'bar': None}).data # {'foo': 42}
As I point out in a further comment, there's a shortcoming: it will also remove None when the field's allow_none is True.
As I pointed out in my comment above this messes with the order if you use the
class Meta:
fields = (
'field1', 'field2'
)
ordered = True
To fix this I used this:
# Remove None fields
#pre_dump
def remove_skip_values(self, data):
return {
key: value for key, value in data.items()
if value is not None
}
This works for my dictonary of objects
I am working with eXistDB in python and leveraging the eulxml library to handle mapping from the xml in the database into custom objects. I want to then serialize these objects to json (for another application to consume) but I'm running into issues. jsonpickle doesn't work (it ends up returning all sorts of excess garbage and the value are the fields aren't actually encoded but rather their eulxml type) and the standard json.dumps() is simply giving me empty json (this was after trying to implement the solution detailed here). The problem seems to stem from the fact that the __dict__ values are not initialised __oninit__ (as they are mapped as class properties) so the __dict__ appears empty upon serialization. Here is some sample code:
Serializable Class Object
class Serializable(dict):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# hack to fix _json.so make_encoder serialize properly
self.__setitem__('dummy', 1)
def _myattrs(self):
return [
(x, self._repr(getattr(self, x)))
for x in self.__dir__()
if x not in Serializable().__dir__()
]
def _repr(self, value):
if isinstance(value, (str, int, float, list, tuple, dict)):
return value
else:
return repr(value)
def __repr__(self):
return '<%s.%s object at %s>' % (
self.__class__.__module__,
self.__class__.__name__,
hex(id(self))
)
def keys(self):
return iter([x[0] for x in self._myattrs()])
def values(self):
return iter([x[1] for x in self._myattrs()])
def items(self):
return iter(self._myattrs())
Base Class
from eulxml import xmlmap
import inspect
import lxml
import json as JSON
from models.serializable import Serializable
class AlcalaBase(xmlmap.XmlObject,Serializable):
def toJSON(self):
return JSON.dumps(self, indent=4)
def to_json(self, skipBegin=False):
json = list()
if not skipBegin:
json.append('{')
json.append(str.format('"{0}": {{', self.ROOT_NAME))
for attr, value in inspect.getmembers(self):
if (attr.find("_") == -1
and attr.find("serialize") == -1
and attr.find("context") == -1
and attr.find("node") == -1
and attr.find("schema") == -1):
if type(value) is xmlmap.fields.NodeList:
if len(value) > 0:
json.append(str.format('"{0}": [', attr))
for v in value:
json.append(v.to_json())
json.append(",")
json = json[:-1]
json.append("]")
else:
json.append(str.format('"{0}": null', attr))
elif (type(value) is xmlmap.fields.StringField
or type(value) is str
or type(value) is lxml.etree._ElementUnicodeResult):
value = JSON.dumps(value)
json.append(str.format('"{0}": {1}', attr, value))
elif (type(value) is xmlmap.fields.IntegerField
or type(value) is int
or type(value) is float):
json.append(str.format('"{0}": {1}', attr, value))
elif value is None:
json.append(str.format('"{0}": null', attr))
elif type(value) is list:
if len(value) > 0:
json.append(str.format('"{0}": [', attr))
for x in value:
json.append(x)
json.append(",")
json = json[:-1]
json.append("]")
else:
json.append(str.format('"{0}": null', attr))
else:
json.append(value.to_json(skipBegin=True))
json.append(",")
json = json[:-1]
if not skipBegin:
json.append('}')
json.append('}')
return ''.join(json)
Sample Class that implements Base
from eulxml import xmlmap
from models.alcalaMonth import AlcalaMonth
from models.alcalaBase import AlcalaBase
class AlcalaPage(AlcalaBase):
ROOT_NAME = "page"
id = xmlmap.StringField('pageID')
year = xmlmap.IntegerField('content/#yearID')
months = xmlmap.NodeListField('content/month', AlcalaMonth)
The toJSON() method on the base is the method that is using the Serializable class and is returning empty json, e.g. "{}". The to_json() is my attempt to for a json-like implementation but that has it's own problems (for some reason it skips certain properties / child objects for no reason I can see but thats a thread for another day).
If I attempt to access myobj.keys or myobj.values (both of which are exposed via Serializable) I can see property names and values as I would expect but I have no idea why json.dumps() produces an empty json string.
Does anyone have any idea why I cannot get these objects to serialize to json?! I've been pulling my hair out for weeks with this. Any help would be greatly appreciated.
So after a lot of playing around, I was finally able to fix this with jsonpickle and it took only 3 lines of code:
def toJson(self):
jsonpickle.set_preferred_backend('simplejson')
return jsonpickle.encode(self, unpicklable=False)
I used simplejson to eliminate some of the additional object notation that was being added and the unpicklable property removed the rest (I'm not sure if this would work with the default json backend as I didn't test it).
Now when I call toJson() on any object that inherits from this base class, I get very nice json and it works brilliantly.
according to http://dev.mysql.com/doc/refman/5.0/en/datetime.html. i got to find a way to convert the string value 'YYYY-MM-DD HH:MM:SS' to a timestamp int.
i looked up in python's doc.
i tried:
print(time.strptime('2013-01-12 15:27:43', '%Y-%m-%d %H:%M:%S'))
python give me a result like this.
time.struct_time(tm_year=2013, tm_mon=1, tm_mday=12, tm_hour=15, tm_min=27, tm_sec=43, tm_wday=5, tm_yday=12, tm_isdst=-1)
i tried this to convert timestamp to YYYY-MM-DD HH:MM:SS format
print(time.strftime('%Y-%m-%d %H:%M:%S',time.time()))
python give me a type error.
i only use timestamp to calculate time and date, i hope there's already a way in python, simple and efficient , and don't have to create temp data.
according to the answer i write two methods. hope it would be helpful
import time
def convertTimestampToSQLDateTime(value):
return time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(value))
def convertSQLDateTimeToTimestamp(value):
return time.mktime(time.strptime(value, '%Y-%m-%d %H:%M:%S'))
Happy to update this if I'm not properly understanding, but here are a few examples which may help. Note that this uses the datetime module instead of time.
>>> import datetime
Here we set up an example timestamp ts and a format f:
>>> ts = '2013-01-12 15:27:43'
>>> f = '%Y-%m-%d %H:%M:%S'
Similar to what you did above, we use the strptime function (from datetime.datetime) to convert our string into a datetime object based on the formatting parameter:
>>> datetime.datetime.strptime(ts, f)
datetime.datetime(2013, 1, 12, 15, 27, 43)
Now in reverse - here we use datetime.datetime.now() to get the current time as a datetime object:
>>> now = datetime.datetime.now()
>>> now
datetime.datetime(2013, 1, 12, 0, 46, 54, 490219)
In the datetime case, the strftime method is actually called on the datetime object itself, with the formatting parameter as an argument:
>>> now.strftime(f)
'2013-01-12 00:46:54'
In your situation, the reason you were getting an error is because time.time() returns a float:
>>> time.time()
1357980846.290231
But time.strftime needs a time tuple, similar to what you had above. Without getting into the maddening spiral that is time, a function such as time.localtime() will return the aforementioned time tuple and will return as you expect:
>>> now = time.localtime()
>>> now
time.struct_time(tm_year=2013, tm_mon=1, tm_mday=12, tm_hour=0, tm_min=55, tm_sec=55, tm_wday=5, tm_yday=12, tm_isdst=0)
>>> f = '%Y-%m-%d %H:%M:%S'
>>> time.strftime(f, now)
'2013-01-12 00:55:55'
I'm only adding this class to potentially save the next guy a little time. If anyone finds this useful, upvote RocketDonkey's answer.
## dev on v3.7.6
from datetime import datetime
from time import mktime, time
class Time:
'''\
*Convenience class for easy format conversion*\n
Accepts time() float, datetime object, or SQL datetime str.\n
If no time arg is provided, object is initialized with time().\n
id kwarg can be used to keep track of objects.\n
Access formats as instance.t, instance.dt, or instance.sql.\
'''
f = '%Y-%m-%d %H:%M:%S'
def __init__(self, *arg, id=None) -> None:
self.id = id
if len(arg) == 0:
self.t = time()
self.dt = self._dt
self.sql = self._sql
else:
arg = arg[0]
if isinstance(arg, float) or arg == None:
if isinstance(arg, float):
self.t = arg
else:
self.t = time()
self.dt = self._dt
self.sql = self._sql
elif isinstance(arg, datetime):
self.t = arg.timestamp()
self.dt = arg
self.sql = self._sql
elif isinstance(arg, str):
self.sql = arg
if '.' not in arg:
self.dt = datetime.strptime(self.sql, Time.f)
else:
normal, fract = arg.split('.')
py_t = datetime.strptime(normal, Time.f)
self.dt = py_t.replace(
microsecond=int(fract.ljust(6, '0')[:6]))
self.t = self.dt.timestamp()
#property
def _dt(self) -> datetime:
return datetime.fromtimestamp(self.t)
#property
def _sql(self) -> str:
t = self.dt
std = t.strftime(Time.f)
fract = f'.{str(round(t.microsecond, -3))[:3]}'
return std + fract
def __str__(self) -> str:
if self.id == None:
return self.sql
else:
return f'Time obj "{self.id}": {self.sql}'
def test():
def test_one(*arg):
t = Time(*arg, id=type(*arg))
print(t)
print(t.t)
print(t.dt)
sql = '2020-01-22 15:30:33.433'
time_float = 1579927395.3708763
dt_obj = datetime.now()
for datum in [sql, time_float, dt_obj, None]:
test_one(datum)
if __name__ == '__main__':
test()