Django mysql when saving a object got warning - mysql

I was just writing a little app to store words in my mysql database using Django.I read data from a text file which is extremely well organised,like this:
The text file is like this:
DELUGE
DELUSION
DELVE
DEMAGOGUE
DEMANDING
DEMOLITION
DEMONSTRATE
DEMORALIZE
DEMOTIC
DEMUR
DENIGRATE
DENOUEMENT
DENOUNCE
DENT
DENUDE
DEPLETE
DEPLORE
DEPLOY
And then I read date from it using open('thefile').readlines like this:
for line in open('/home/jacos/sorted-gre.txt').readlines():
... if line:
... p = Word(word_spelling = line)
... p.save()
The word_spelling field is the primary key.
Then came this warning:
Traceback (most recent call last):
File "<console>", line 4, in <module>
File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 460, in save
self.save_base(using=using, force_insert=force_insert, force_update=force_update)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 553, in save_base
result = manager._insert(values, return_id=update_pk, using=using)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/manager.py", line 195, in _insert
return insert_query(self.model, values, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 1436, in insert_query
return query.get_compiler(using=using).execute_sql(return_id)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 791, in execute_sql
cursor = super(SQLInsertCompiler, self).execute_sql(None)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 735, in execute_sql
cursor.execute(sql, params)
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/util.py", line 34, in execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/mysql/base.py", line 86, in execute
return self.cursor.execute(query, args)
File "/usr/lib/pymodules/python2.7/MySQLdb/cursors.py", line 176, in execute
if not self._defer_warnings: self._warning_check()
File "/usr/lib/pymodules/python2.7/MySQLdb/cursors.py", line 92, in _warning_check
warn(w[-1], self.Warning, 3)
Data truncated for column 'word_spelling' at row 1
As a result, only part of these words were stored in mysql. I'd like to know why.

CharFields have a max_length attribute. What did you set when you generate database
for Word.object.get(pk=1).word_spelling?
nothing related with your warning but
it's recommended to close file or open it with a with statement.
with open('/home/jacos/sorted-gre.txt') as f:
for line in f.readlines():
if line:
p = Word(word_spelling = line)
p.save()

Related

Having "make_aware expects a naive datetime" while migrate

I have developed an application with Django.
This is working fine in my PC with sqlite backend.
But when I am trying to go live with linux server and mysql backend then I am getting bellow error while first time migration.
(env-bulkmailer) [root#localhost bulkmailer]# python3 manage.py migrate
Traceback (most recent call last):
File "/var/www/bulkmailer-folder/bulkmailer/manage.py", line 22, in <module>
main()
File "/var/www/bulkmailer-folder/bulkmailer/manage.py", line 18, in main
execute_from_command_line(sys.argv)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/core/management/__init__.py", line 446, in execute_from_command_line
utility.execute()
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/core/management/__init__.py", line 440, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/core/management/base.py", line 402, in run_from_argv
self.execute(*args, **cmd_options)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/core/management/base.py", line 448, in execute
output = self.handle(*args, **options)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/core/management/base.py", line 96, in wrapped
res = handle_func(*args, **kwargs)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/core/management/commands/migrate.py", line 114, in handle
executor = MigrationExecutor(connection, self.migration_progress_callback)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/migrations/executor.py", line 18, in __init__
self.loader = MigrationLoader(self.connection)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/migrations/loader.py", line 58, in __init__
self.build_graph()
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/migrations/loader.py", line 235, in build_graph
self.applied_migrations = recorder.applied_migrations()
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/migrations/recorder.py", line 82, in applied_migrations
return {
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/models/query.py", line 394, in __iter__
self._fetch_all()
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/models/query.py", line 1866, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/models/query.py", line 117, in __iter__
for row in compiler.results_iter(results):
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/models/sql/compiler.py", line 1336, in apply_converters
value = converter(value, expression, connection)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/db/backends/mysql/operations.py", line 331, in convert_datetimefield_value
value = timezone.make_aware(value, self.connection.timezone)
File "/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/utils/timezone.py", line 291, in make_aware
raise ValueError("make_aware expects a naive datetime, got %s" % value)
ValueError: make_aware expects a naive datetime, got 2022-11-20 12:39:18.866299+00:00
In settings-
USE_TZ = True
I have run mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql also as django doc.
I am using django 4.1.3 and mysql community 8.0.30
Thanks in advance.
Ran into the same issue. At some point, django assumes that the the data is timezone-naive without checking. Here's the work-around.
Update the make_aware function that is listed in your stack trace here:
/var/www/bulkmailer-folder/env-bulkmailer/lib64/python3.9/site-packages/django/utils/timezone.py", line 291, in make_aware
Instead of raising an error if the value is already aware, just return the aware value. See the last else statement below.
def make_aware(value, timezone=None, is_dst=NOT_PASSED):
"""Make a naive datetime.datetime in a given time zone aware."""
if is_dst is NOT_PASSED:
is_dst = None
else:
warnings.warn(
"The is_dst argument to make_aware(), used by the Trunc() "
"database functions and QuerySet.datetimes(), is deprecated as it "
"has no effect with zoneinfo time zones.",
RemovedInDjango50Warning,
)
if timezone is None:
timezone = get_current_timezone()
if _is_pytz_zone(timezone):
# This method is available for pytz time zones.
return timezone.localize(value, is_dst=is_dst)
else:
# Check that we won't overwrite the timezone of an aware datetime.
if is_aware(value):
# ADD THIS
return value
# REMOVE THE FOLLOWING LINE
# raise ValueError("make_aware expects a naive datetime, got %s" % value)
# This may be wrong around DST changes!
return value.replace(tzinfo=timezone)

How to solve a duplicate column name error in web2py

I'm running web2py on pythonanywhere. I've started getting a duplicate column name error when the db.py file runs. I've tried restoring the database from a backup and also dropping the table and adding it back, without success. At this point I'm completely locked out of my app.
I've tried removing all but the last field in the table, but then the problem appears in the next table.
I'm wondering if web2py uses a cache that needs to be cleared.
Here is the relevant portion of my db.py file:
db.define_table('library',
Field('title', 'string'),
Field('created','datetime'),
Field('duration','float'),
Field('error_message','string'),
Field('external_id','string'),
Field('hosting_type','string'),
Field('source_id','string', unique = True),
Field('last_modified','datetime'),
Field('media_type','string'),
Field('mime_type','string'),
Field('relationships','string'),
Field('source_schema','string'),
Field('source_url','string'),
Field('status','string'),
Field('trim_in_point','string'),
Field('trim_out_point','string'),
Field('source_type','string'),
Field('poster','string'),
Field('background_poster_filename','string'),
Field('background_poster','upload', required=False, requires = IS_EMPTY_OR(IS_IMAGE(extensions=('png', 'jpg', 'jpeg'), maxsize=(1920, 1080)))),
Field('sources','list:string',length = 4096),
Field('tracks','string'),
singular="Library",
plural="Library"
)
Here is the error ticket:
Ticket ID
67.0.14.100.2022-06-23.16-49-09.ecf75fd3-1bb4-4f76-b557-95e072c6681f
<class 'gluon.contrib.pymysql.err.InternalError'> (1060, "Duplicate column name 'title'")
Version
web2py™ Version 2.21.1-stable+timestamp.2020.11.28.04.10.44
Python Python 3.7.10: /usr/local/bin/uwsgi (prefix: /home/ghdev/.virtualenvs/ghdevvirtualenv)
Traceback
Traceback (most recent call last):
File "/home/ghdev/web2py/gluon/restricted.py", line 219, in restricted
exec(ccode, environment)
File "/home/ghdev/web2py/applications/ghrokucms/models/db.py", line 283, in <module>
plural="Library"
File "/home/ghdev/web2py/gluon/packages/dal/pydal/base.py", line 660, in define_table
table = self.lazy_define_table(tablename, *fields, **kwargs)
File "/home/ghdev/web2py/gluon/packages/dal/pydal/base.py", line 701, in lazy_define_table
polymodel=polymodel,
File "/home/ghdev/web2py/gluon/packages/dal/pydal/adapters/base.py", line 920, in create_table
return self.migrator.create_table(*args, **kwargs)
File "/home/ghdev/web2py/gluon/packages/dal/pydal/migrator.py", line 376, in create_table
fake_migrate=fake_migrate,
File "/home/ghdev/web2py/gluon/packages/dal/pydal/migrator.py", line 544, in migrate_table
self.adapter.execute(sub_query)
File "/home/ghdev/web2py/gluon/packages/dal/pydal/adapters/__init__.py", line 69, in wrap
return f(*args, **kwargs)
File "/home/ghdev/web2py/gluon/packages/dal/pydal/adapters/base.py", line 468, in execute
rv = self.cursor.execute(command, *args[1:], **kwargs)
File "/home/ghdev/web2py/gluon/contrib/pymysql/cursors.py", line 166, in execute
result = self._query(query)
File "/home/ghdev/web2py/gluon/contrib/pymysql/cursors.py", line 322, in _query
conn.query(q)
File "/home/ghdev/web2py/gluon/contrib/pymysql/connections.py", line 835, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/home/ghdev/web2py/gluon/contrib/pymysql/connections.py", line 1019, in _read_query_result
result.read()
File "/home/ghdev/web2py/gluon/contrib/pymysql/connections.py", line 1302, in read
first_packet = self.connection._read_packet()
File "/home/ghdev/web2py/gluon/contrib/pymysql/connections.py", line 981, in _read_packet
packet.check_error()
File "/home/ghdev/web2py/gluon/contrib/pymysql/connections.py", line 393, in check_error
err.raise_mysql_exception(self._data)
File "/home/ghdev/web2py/gluon/contrib/pymysql/err.py", line 107, in raise_mysql_exception
raise errorclass(errno, errval)
gluon.contrib.pymysql.err.InternalError: (1060, "Duplicate column name 'title'")
I'm at a loss to figure how to resolve this. Any help would be greatly appreciated.
Thanks.
I solved this problem. In addition to dropping the table, I needed to delete the associated file in the web2py database directory. I then had to manually add back the table to the mySql database.
SOLVED.

Run python errors.use kereas to implement CNN

I am learning Deep Learning and want to use python-kereas to implement CNN, but when I run in command, it looks like some errors.
This is my source code. https://github.com/lijhong/CNN-kereas.git
And my fault is like this:
Traceback (most recent call last):
File "/home/ah0818lijhong/CNN-kereas/cnn-kereas.py", line 167, in <module>
model.fit(x_train, y_train,epochs=3)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/models.py", line 845, in fit
initial_epoch=initial_epoch)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/engine/training.py", line 1485, in fit
initial_epoch=initial_epoch)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/engine/training.py", line 1140, in _fit_loop
outs = f(ins_batch)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 2073, in __call__
feed_dict=feed_dict)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 778, in run
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 778, in run
run_metadata_ptr)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 982, in _run
feed_dict_string, options, run_metadata)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1032, in _do_run
target_list, options, run_metadata)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[0,868] = 115873 is not in [0, 20001)
[[Node: embedding_1/Gather = Gather[Tindices=DT_INT32, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](embedding_1/embeddi
ngs/read, _recv_embedding_1_input_0)]]
Caused by op u'embedding_1/Gather', defined at:
File "/home/ah0818lijhong/CNN-kereas/cnn-kereas.py", line 122, in <module>
model_left.add(embedding_layer)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/models.py", line 422, in add
layer(x)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 554, in __call__
output = self.call(inputs, **kwargs)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/layers/embeddings.py", line 119, in call
out = K.gather(self.embeddings, inputs)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 966, in gather
return tf.gather(reference, indices)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 1207, in gather
validate_indices=validate_indices, name=name)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 768, in apply_op
op_def=op_def)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): indices[0,868] = 115873 is not in [0, 20001)
[[Node: embedding_1/Gather = Gather[Tindices=DT_INT32, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](embedding_1/embeddi
ngs/read, _recv_embedding_1_input_0)]]
I hope someone can help me fix it.

have no idea what the exception is here

I am trying to read a csv file. using below command:
sample = pd.read_csv("C:/kushal/DataMining/hillary/out.txt" ,header = 0, delimiter = "\t")
Unfortunately it gives me some exception about which I have no idea what is causing it. Does anyone knows anything about this exception
sample["ExtractedBodyText"][1]
Traceback (most recent call last):
File "<pyshell#75>", line 1, in <module>
hillary["ExtractedBodyText"][1]
File "C:\Python34\lib\site-packages\pandas\core\frame.py", line 1914, in __getitem__
return self._getitem_column(key)
File "C:\Python34\lib\site-packages\pandas\core\frame.py", line 1921, in _getitem_column
return self._get_item_cache(key)
File "C:\Python34\lib\site-packages\pandas\core\generic.py", line 1090, in _get_item_cache
values = self._data.get(item)
File "C:\Python34\lib\site-packages\pandas\core\internals.py", line 3102, in get
loc = self.items.get_loc(item)
File "C:\Python34\lib\site-packages\pandas\core\index.py", line 1692, in get_loc
return self._engine.get_loc(_values_from_object(key))
File "pandas\index.pyx", line 137, in pandas.index.IndexEngine.get_loc (pandas\index.c:3979)
File "pandas\index.pyx", line 157, in pandas.index.IndexEngine.get_loc (pandas\index.c:3843)
File "pandas\hashtable.pyx", line 668, in pandas.hashtable.PyObjectHashTable.get_item (pandas\hashtable.c:12265)
File "pandas\hashtable.pyx", line 676, in pandas.hashtable.PyObjectHashTable.get_item (pandas\hashtable.c:12216)
KeyError: 'ExtractedBodyText'

IMDbPY Importing to MySQL Problem

I am having a problem while trying to import imdb data from text files to MySQL database using imdbpy2sql.py script.
It throws the following error. It seems like the exception handler code tries to insert a duplicated record into cast_info table with existing primary key.
Can anyone help me fix this problem or suggest any work around solution?
SCANNING actor: Hall, Stephan
SCANNING actor: Halsey, William F.
* FLUSHING CharactersCache...
* TOO MANY DATA (100000 items in CharactersCache), recursion: 1
* SPLITTING (run 1 of 2), recursion: 1
* FLUSHING CharactersCache...
Traceback (most recent call last):
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 2951, in <module>
run()
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 2812, in run
castLists(_charIDsList=characters_imdbIDs)
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 1576, in castLists
doCast(f, roleid, rolename)
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 1535, in doCast
cid = CACHE_CID.addUnique(role)
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 957, in addUnique
else: return self.add(key, miscData)
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 950, in add
self[key] = c
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 860, in __setitem__
self.flush()
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 912, in flush
self.flush(quiet=quiet, _recursionLevel=_recursionLevel)
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 883, in flush
self._toDB(quiet)
File "D:\project\IMDB\IMDbPY-4.7\bin\imdbpy2sql.py", line 1186, in _toDB
CURS.executemany(self.sqlstr, self.converter(l))
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 206, in executemany
r = r + self.execute(query, a)
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 174, in execute
self.errorhandler(self, exc, value)
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
_mysql_exceptions.IntegrityError: (1062, "Duplicate entry '745684' for key 'PRIMARY'")
A lot of similar problems were fixed in the development version.
Please try again with the version in the Mercurial repository:
https://bitbucket.org/alberanid/imdbpy/