Op:__inference_predict_function_82024 error in neural network - deep-learning

i am getting this error when i run a simple neural network model using MNIST dataset.
Heres my model
customNN = Sequential()
# input layer
customNN.add(Dense(4, activation = "relu",input_shape = (28,28)))
# Hidden layer
customNN.add(Dense(16,activation = "relu"))
customNN.add(Dense(32,activation = "relu"))
customNN.add(Dense(64,activation = "relu"))
customNN.add(Dense(100,activation = "relu"))
customNN.add(Dense(128,activation = "relu"))
# flatten() function is used to get a copy of an given array collapsed into one dimension.
customNN.add(Flatten())
# output layer
customNN.add(Dense(10,activation = "softmax"))
when i compile it successfully done with MNIST dataset
customNN.compile(optimizer="adam", loss= "categorical_crossentropy", metrics=["accuracy"])
customNN.fit(xtrain,ytrain, epochs=10)
import cv2
def input_prepare(img):
img = np.asarray(img) # convert to array
img = cv2.resize(img, (28, 28 )) # resize to target shape
img = cv2.bitwise_not(img) # [optional] turned bg to black - {bitwise_not} turns 1's into 0's and 0's into 1's
img = img / 255 # normalize
img = img.reshape(1, 784) # reshape it to input placeholder shape
return img
img = cv2.imread('4.jpg')
orig = img.copy() # save for plotting later on
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # gray scaling
img = input_prepare(img)
pred = customNN.predict(img)
plt.imshow(cv2.cvtColor(orig, cv2.COLOR_BGR2RGB))
plt.title(np.argmax(pred, axis=1))
plt.show()
But when i run this code i am getting the following error
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-29-509e1856bd1e> in <module>
16
17
---> 18 pred = customNN.predict(img)
19 plt.imshow(cv2.cvtColor(orig, cv2.COLOR_BGR2RGB))
20 plt.title(np.argmax(pred, axis=1))
D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py in error_handler(*args, **kwargs)
68 # To get the full stack trace, call:
69 # `tf.debugging.disable_traceback_filtering()`
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
D:\anaconda3\lib\site-packages\tensorflow\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
51 ctx.ensure_initialized()
52 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 53 inputs, attrs, num_outputs)
54 except core._NotOkStatusException as e:
55 if name is not None:
InvalidArgumentError: Graph execution error:
Detected at node 'sequential/dense/Tensordot/GatherV2_1' defined at (most recent call last):
File "D:\anaconda3\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "D:\anaconda3\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "D:\anaconda3\lib\site-packages\ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\traitlets\config\application.py", line 1041, in launch_instance
app.start()
File "D:\anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 583, in start
self.io_loop.start()
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\platform\asyncio.py", line 215, in start
self.asyncio_loop.run_forever()
File "D:\anaconda3\lib\asyncio\base_events.py", line 541, in run_forever
self._run_once()
File "D:\anaconda3\lib\asyncio\base_events.py", line 1786, in _run_once
handle._run()
File "D:\anaconda3\lib\asyncio\events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\ioloop.py", line 687, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\ioloop.py", line 740, in _run_callback
ret = callback()
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\gen.py", line 821, in inner
self.ctx_run(self.run)
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\gen.py", line 782, in run
yielded = self.gen.send(value)
File "D:\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 361, in process_one
yield gen.maybe_future(dispatch(*args))
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "D:\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 268, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "D:\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 541, in execute_request
user_expressions, allow_stdin,
File "C:\Users\Deadpool\AppData\Roaming\Python\Python37\site-packages\tornado\gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "D:\anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 300, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "D:\anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2976, in run_cell
raw_cell, store_history, silent, shell_futures, cell_id
File "D:\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "D:\anaconda3\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "D:\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3258, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "D:\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "D:\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-29-509e1856bd1e>", line 18, in <module>
pred = customNN.predict(img)
File "D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 65, in error_handler
return fn(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\engine\training.py", line 2350, in predict
tmp_batch_outputs = self.predict_function(iterator)
File "D:\anaconda3\lib\site-packages\keras\engine\training.py", line 2137, in predict_function
return step_function(self, iterator)
File "D:\anaconda3\lib\site-packages\keras\engine\training.py", line 2123, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "D:\anaconda3\lib\site-packages\keras\engine\training.py", line 2111, in run_step
outputs = model.predict_step(data)
File "D:\anaconda3\lib\site-packages\keras\engine\training.py", line 2079, in predict_step
return self(x, training=False)
File "D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 65, in error_handler
return fn(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\engine\training.py", line 561, in __call__
return super().__call__(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 65, in error_handler
return fn(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\engine\base_layer.py", line 1132, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 96, in error_handler
return fn(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\engine\sequential.py", line 413, in call
return super().call(inputs, training=training, mask=mask)
File "D:\anaconda3\lib\site-packages\keras\engine\functional.py", line 511, in call
return self._run_internal_graph(inputs, training=training, mask=mask)
File "D:\anaconda3\lib\site-packages\keras\engine\functional.py", line 668, in _run_internal_graph
outputs = node.layer(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 65, in error_handler
return fn(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\engine\base_layer.py", line 1132, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 96, in error_handler
return fn(*args, **kwargs)
File "D:\anaconda3\lib\site-packages\keras\layers\core\dense.py", line 244, in call
outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])
Node: 'sequential/dense/Tensordot/GatherV2_1'
indices[0] = 2 is not in [0, 2)
[[{{node sequential/dense/Tensordot/GatherV2_1}}]] [Op:__inference_predict_function_82024]
Cannot figure out whats the problem? i am newbie in DL and trying to improve. any help to find out the problem is highly appreciated
thanks
i have tried a new image to detect the handwriting, but shows error

Related

Pytorch torchvision.transforms execute randomly?

I am doing this transformation:
self.transform = transforms.Compose( {
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
} )
and then
image = Image.open(img_name)
if self.transform:
image = self.transform(image)
this works for the first epoch then how the hell it crashes for the second epoch?
why the f normalize getting PIL-image and not torch.tensor? is the execution of each transforms Compose items random?
Traceback (most recent call last): File
"/home/ubuntu/projects/ssl/src/train_supervised.py", line 63, in
main() File "/home/ubuntu/projects/ssl/src/train_supervised.py", line 60, in main
train() File "/home/ubuntu/projects/ssl/src/train_supervised.py", line 45, in train
for i, data in enumerate(tqdm_): File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/tqdm/std.py",
line 1195, in iter
for obj in iterable: File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/dataloader.py",
line 530, in next
data = self._next_data() File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/dataloader.py",
line 1224, in _next_data
return self._process_data(data) File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/dataloader.py",
line 1250, in _process_data
data.reraise() File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/_utils.py",
line 457, in reraise
raise exception TypeError: Caught TypeError in DataLoader worker process 0. Original Traceback (most recent call last): File
"/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
data = fetcher.fetch(index) File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index] File
"/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 49, in
data = [self.dataset[idx] for idx in possibly_batched_index] File "/home/ubuntu/projects/ssl/src/data_loader.py", line 44, in
getitem
image = self.transform(image) File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torchvision/transforms/transforms.py", line 95, in call
img = t(img) File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/nn/modules/module.py",
line 1110, in _call_impl
return forward_call(*input, **kwargs) File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torchvision/transforms/transforms.py", line 270, in forward
return F.normalize(tensor, self.mean, self.std, self.inplace) File
"/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torchvision/transforms/functional.py", line 341, in normalize
raise TypeError(f"Input tensor should be a torch tensor. Got {type(tensor)}.") TypeError: Input tensor should be a torch tensor.
Got <class 'PIL.Image.Image'>.
Python set iteration order is not deterministic. Kindly use list instead ([] rather than {}).

'ner_ontonotes_bert_mult' model Custom train

When I want to train this 'ner_ontonotes_bert_mult' model with my custom dataset it is showing the error below. (I have saved my datset in the ~\.deeppavlov\downloads\ontonotes folder that was mentioned in [deeppavlov documentation][1]. )
PS C:\Users\sghanta\Desktop\NER> & c:/Users/sghanta/Desktop/NER/env/Scripts/Activate.ps1
(env) PS C:\Users\sghanta\Desktop\NER> & c:/Users/sghanta/Desktop/NER/env/Scripts/python.exe c:/Users/sghanta/Desktop/NER/train_model.py
C:\Users\sghanta\Desktop\NER\env\lib\site-packages\numpy\_distributor_init.py:32: UserWarning: loaded more than 1 DLL from .libs:
C:\Users\sghanta\Desktop\NER\env\lib\site-packages\numpy\.libs\libopenblas.PYQHXLVVQ7VESDPUVUADXEVJOBGHJPAY.gfortran-win_amd64.dll
C:\Users\sghanta\Desktop\NER\env\lib\site-packages\numpy\.libs\libopenblas.WCDJNK7YVMPZQ2ME2ZZHJJRJ3JIKNDB7.gfortran-win_amd64.dll
stacklevel=1)
Traceback (most recent call last):
File "c:/Users/sghanta/Desktop/NER/train_model.py", line 12, in <module>
ner_model = train_model(configs.ner.ner_ontonotes_bert_mult)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\__init__.py", line 29, in train_model
train_evaluate_model_from_config(config, download=download, recursive=recursive)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\commands\train.py", line 92, in train_evaluate_model_from_config
data = read_data_by_config(config)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\commands\train.py", line 58, in read_data_by_config
return reader.read(data_path, **reader_config)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\dataset_readers\conll2003_reader.py", line 56, in read
dataset[name] = self.parse_ner_file(file_name)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\dataset_readers\conll2003_reader.py", line 106, in parse_ner_file
raise Exception(f"Input is not valid {line}")
Exception: Input is not valid
O
(env) PS C:\Users\sghanta\Desktop\NER>
After cleaning the dataset the above error has gone but this is the new error.
New Error
2021-08-12 02:43:35.335 ERROR in 'deeppavlov.core.common.params'['params'] at line 112: Exception in <class 'deeppavlov.models.bert.bert_sequence_tagger.BertSequenceTagger'>
Traceback (most recent call last):
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 1365, in _do_call
return fn(*args)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 1350, in _run_fn
target_list, run_metadata)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 1443, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Assign requires shapes of both tensors to match. lhs shape= [13,13] rhs shape= [37,37]
[[{{node save/Assign_76}}]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saver.py", line 1290, in restore
{self.saver_def.filename_tensor_name: save_path})
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 956, in run
run_metadata_ptr)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 1359, in _do_run
run_metadata)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\client\session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Assign requires shapes of both tensors to match. lhs shape= [13,13] rhs shape= [37,37]
[[node save/Assign_76 (defined at C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\framework\ops.py:1748) ]]
Original stack trace for 'save/Assign_76':
File "c:/Users/sghanta/Desktop/NER/train_model.py", line 12, in <module>
ner_model = train_model(configs.ner.ner_ontonotes_bert_mult)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\__init__.py", line 29, in train_model
train_evaluate_model_from_config(config, download=download, recursive=recursive)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\commands\train.py", line 121, in train_evaluate_model_from_config
trainer.train(iterator)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\trainers\nn_trainer.py", line 334, in train
self.fit_chainer(iterator)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\trainers\fit_trainer.py", line 104, in fit_chainer
component = from_params(component_config, mode='train')
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\common\params.py", line 106, in from_params
component = obj(**dict(config_params, **kwargs))
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\models\tf_backend.py", line 76, in __call__
obj.__init__(*args, **kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\models\tf_backend.py", line 28, in _wrapped
return func(*args, **kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\models\bert\bert_sequence_tagger.py", line 529, in __init__
**kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\models\bert\bert_sequence_tagger.py", line 259, in __init__
self.load()
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\models\tf_backend.py", line 28, in _wrapped
return func(*args, **kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\models\bert\bert_sequence_tagger.py", line 457, in load
return super().load(exclude_scopes=exclude_scopes, **kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\models\tf_model.py", line 251, in load
return super().load(exclude_scopes=exclude_scopes, **kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\deeppavlov\core\models\tf_model.py", line 54, in load
saver = tf.train.Saver(var_list)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saver.py", line 828, in __init__
self.build()
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saver.py", line 840, in build
self._build(self._filename, build_save=True, build_restore=True)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saver.py", line 878, in _build
build_restore=build_restore)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saver.py", line 508, in _build_internal
restore_sequentially, reshape)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saver.py", line 350, in _AddRestoreOps
assign_ops.append(saveable.restore(saveable_tensors, shapes))
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\training\saving\saveable_object_util.py", line 73, in restore
self.op.get_shape().is_fully_defined())
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\ops\state_ops.py", line 227, in assign
validate_shape=validate_shape)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\ops\gen_state_ops.py", line 66, in assign
use_locking=use_locking, name=name)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\framework\op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\framework\ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\framework\ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "C:\Users\sghanta\Desktop\NER\env\lib\site-packages\tensorflow_core\python\framework\ops.py", line 1748, in __init__
self._traceback = tf_stack.extract_stack()
Can anyone explain how to solve it.
[1]: http://docs.deeppavlov.ai/en/master/features/models/ner.html
conll2003_reader dataset reader failed to parse the following line:
O
conll2003_reader dataset reader expects that line is either empty or contains a token and a label. In your case only label is present.
So, I would suggest to clean your data from empty lines with labels.
Sample text from DeepPavlov docs:
EU B-ORG
rejects O
the O
call O
of O
Germany B-LOC
to O
boycott O
lamb O
from O
Great B-LOC
Britain I-LOC
. O
China B-LOC

Airflow DAG throws RecursionError triggered via Web console

I get following error, when I am trying to trigger DAG which is using custom BaseOperator.
here is the error,
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.7/site-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask_login/utils.py", line 258, in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/www/utils.py", line 290, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/www/utils.py", line 337, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/www/views.py", line 1213, in trigger
external_trigger=True
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/models/dag.py", line 1659, in create_dagrun
session=session)
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 70, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/models/dag.py", line 1346, in create_dagrun
run.refresh_from_db()
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/models/dagrun.py", line 109, in refresh_from_db
DR.run_id == self.run_id
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3347, in one
ret = self.one_or_none()
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3316, in one_or_none
ret = list(self)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3389, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3414, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1101, in _execute_clauseelement
distilled_params,
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1478, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.7/site-packages/mysql/connector/cursor_cext.py", line 272, in execute
self._handle_result(result)
File "/usr/local/lib/python3.7/site-packages/mysql/connector/cursor_cext.py", line 163, in _handle_result
self._handle_resultset()
File "/usr/local/lib/python3.7/site-packages/mysql/connector/cursor_cext.py", line 651, in _handle_resultset
self._rows = self._cnx.get_rows()[0]
File "/usr/local/lib/python3.7/site-packages/mysql/connector/connection_cext.py", line 318, in get_rows
else self._cmysql.fetch_row()
SystemError: <method 'fetch_row' of '_mysql_connector.MySQL' objects> returned a result with an error set
My Code is as follows:
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
class TestOperator(BaseOperator):
template_fields = ('param1')
ui_color = '#A7E6A7'
#apply_defaults
def __init__(self,param1,*args, **kwargs):
self.param1 = param1
super(TestOperator, self).__init__(*args, **kwargs)
def execute(self):
print ('welcome to airflow')
class TestOperatorPlugin(AirflowPlugin):
name = "TestOperator_plugin"
operators = [TestOperator]
--here is Dag,
from TestOperator import TestOperator
from airflow import DAG
from datetime import datetime
prog_args = {
'depends_on_past': False, 'param1' : 'testOne'
}
testMYDAG = DAG('TestMYDAG', start_date = datetime(2020, 2, 18) , description='TestMYDAG', default_args = prog_args, schedule_interval=None)
testOp = TestOperator(task_id='test_dag', dag=testMYDAG )
testOp
Team, I had resolved issue by adding encoding scheme to utf-8 in airflow.cfg file, Also, it needs to be appended to sql_alchemy_connection string.

Intermittent IndexError with MySQL and Django running on Ubuntu

Here is what I'm getting:
Traceback (most recent call last):
File "/.../.env/lib/python3.6/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/.../.env/lib/python3.6/site-packages/django/core/handlers/base.py", line 126, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/.../.env/lib/python3.6/site-packages/django/core/handlers/base.py", line 124, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/.../.env/lib/python3.6/site-packages/django/contrib/auth/decorators.py", line 20, in _wrapped_view
if test_func(request.user):
File "/.../.env/lib/python3.6/site-packages/django/contrib/auth/decorators.py", line 44, in <lambda>
lambda u: u.is_authenticated,
File "/.../.env/lib/python3.6/site-packages/django/utils/functional.py", line 213, in inner
self._setup()
File "/.../.env/lib/python3.6/site-packages/django/utils/functional.py", line 347, in _setup
self._wrapped = self._setupfunc()
File "/.../.env/lib/python3.6/site-packages/django/contrib/auth/middleware.py", line 24, in <lambda>
request.user = SimpleLazyObject(lambda: get_user(request))
File "/.../.env/lib/python3.6/site-packages/django/contrib/auth/middleware.py", line 12, in get_user
request._cached_user = auth.get_user(request)
File "/.../.env/lib/python3.6/site-packages/django/contrib/auth/__init__.py", line 189, in get_user
user = backend.get_user(user_id)
File "/.../.env/lib/python3.6/site-packages/django/contrib/auth/backends.py", line 98, in get_user
user = UserModel._default_manager.get(pk=user_id)
File "/.../.env/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/.../.env/lib/python3.6/site-packages/django/db/models/query.py", line 393, in get
num = len(clone)
File "/.../.env/lib/python3.6/site-packages/django/db/models/query.py", line 250, in __len__
self._fetch_all()
File "/.../.env/lib/python3.6/site-packages/django/db/models/query.py", line 1186, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/.../.env/lib/python3.6/site-packages/django/db/models/query.py", line 63, in __iter__
for row in compiler.results_iter(results):
File "/.../.env/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1007, in apply_converters
value = row[pos]
IndexError: list index out of range
As you can see, there is none of my code in the stacktrace, all I know is that this code is happening somewhere in a very simple view that gets a queryset and renders it to JSON.
Most of the times it works, sometimes this traceback is thrown and the page 500s.
What the hell is going on?
Sometimes I get other weird errors:
unsupported operand type(s) for +=: 'int' and 'str'
triggered by
count += Model.objects.filter(...).count()
I've tried to replicate from a shell, but that's not working.

Run python errors.use kereas to implement CNN

I am learning Deep Learning and want to use python-kereas to implement CNN, but when I run in command, it looks like some errors.
This is my source code. https://github.com/lijhong/CNN-kereas.git
And my fault is like this:
Traceback (most recent call last):
File "/home/ah0818lijhong/CNN-kereas/cnn-kereas.py", line 167, in <module>
model.fit(x_train, y_train,epochs=3)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/models.py", line 845, in fit
initial_epoch=initial_epoch)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/engine/training.py", line 1485, in fit
initial_epoch=initial_epoch)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/engine/training.py", line 1140, in _fit_loop
outs = f(ins_batch)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 2073, in __call__
feed_dict=feed_dict)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 778, in run
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 778, in run
run_metadata_ptr)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 982, in _run
feed_dict_string, options, run_metadata)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1032, in _do_run
target_list, options, run_metadata)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[0,868] = 115873 is not in [0, 20001)
[[Node: embedding_1/Gather = Gather[Tindices=DT_INT32, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](embedding_1/embeddi
ngs/read, _recv_embedding_1_input_0)]]
Caused by op u'embedding_1/Gather', defined at:
File "/home/ah0818lijhong/CNN-kereas/cnn-kereas.py", line 122, in <module>
model_left.add(embedding_layer)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/models.py", line 422, in add
layer(x)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 554, in __call__
output = self.call(inputs, **kwargs)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/layers/embeddings.py", line 119, in call
out = K.gather(self.embeddings, inputs)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 966, in gather
return tf.gather(reference, indices)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 1207, in gather
validate_indices=validate_indices, name=name)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 768, in apply_op
op_def=op_def)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/ah0818lijhong/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): indices[0,868] = 115873 is not in [0, 20001)
[[Node: embedding_1/Gather = Gather[Tindices=DT_INT32, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](embedding_1/embeddi
ngs/read, _recv_embedding_1_input_0)]]
I hope someone can help me fix it.