I created a function build_model to tune hyperparameters. However, the function fails to create objects within it, the rlr object (ReduceLROnPlateau). I know the function has run because I tested it by inserting some print statements. Why are the objects in the function not being created?
NameError: name 'rlr' is not defined
#error:
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
<ipython-input-34-00e7981884ae> in <module>()
56 validation_freq=1,
57 epochs=1, #run 1 EPOCH TRIAL FIRST! originally 50
---> 58 callbacks=[rlr,ckpt,es])
59
60 # save weights
NameError: name 'rlr' is not defined
#My Code:
from tensorflow.keras.callbacks import ReduceLROnPlateau,ModelCheckpoint,EarlyStopping
from keras.models import Sequential
from tensorflow import keras
from tensorflow.keras.applications import EfficientNetB0
import tensorflow as tf
from tensorflow.keras.optimizers import Adam
from kerastuner.tuners import RandomSearch
from tensorflow.keras.applications.resnet50 import ResNet50
model_fn = EfficientNetB0(include_top=False, input_shape= (224,224,3), pooling='avg') # , we
def build_model(hp):
model = keras.Sequential()
model.add(model_fn)
#for i in range(hp.Int('num_layers', 2, 20)):
model.add(layers.Dense(units=hp.Int('units_' + str(i),
min_value=32,
max_value=512,
step=32),
activation='relu'))
model.add(keras.layers.Dropout(0.4))
model.add(layers.Dense(2, activation='linear'))
model.summary()
patience = hp.Int('patience', 1, 3, default=1)
callbacks = tf.keras.callbacks.ReduceLROnPlateau(patience=patience)
rlr=ReduceLROnPlateau(monitor='val_loss', factor=0.1,
patience=5, min_lr=0.00001, min_delta=0.001)
ckpt=ModelCheckpoint('models/checkpoint_female', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
es=EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20, min_delta=0.0001)
model.compile(
optimizer=keras.optimizers.Adam(
hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])),
loss='mean_squared_error',
metrics=['mean_absolute_error'])
return model
tuner = RandomSearch(
build_model,
objective='val_mean_absolute_error',
max_trials=2,#5
executions_per_trial=2,#3
directory='tuner',
project_name='Tuner Output')
tuner.search_space_summary()
tuner.search(train_generator_F, steps_per_epoch=200, epochs=2, validation_data=valid_generator_F)
TModel=tuner.get_best_models(num_models=1)[0]
#summary of best model
TModel.summary()
history=TModel.fit_generator(generator= train_generator_F,
steps_per_epoch=STEP_SIZE_TRAIN_F,
validation_data=valid_generator_F,
validation_steps=STEP_SIZE_VALID_F,
validation_freq=1,
epochs=1,
callbacks=[rlr,ckpt,es])
TModel.save_weights('models/TunedEnet100v1.h5')
Related
An error occur when try to execute a custom activation function all the commands work until reaching the last one hits an error!
Tensorflow version is: 2.9.1
keras version is: 2.9.0
Thanks in advance.
The code
import tensorflow
from tensorflow.keras.datasets import mnist
from tensorflow.keras import backend as K
from keras.utils.generic_utils import get_custom_objects
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten, Activation
from tensorflow.keras.layers import Conv2D, MaxPooling2D
import numpy as np
import matplotlib.pyplot as plt
# Custom activation function
def custom_activation(x):
return K.cast(K.x**2) # I also tried the Square(x)
# Before creating the model, I update Keras' custom objects:
get_custom_objects().update({'custom_activation': Activation(custom_activation)})
# Model configuration
img_width, img_height = 28, 28
batch_size = 32
no_epochs = 5
no_classes = 10
verbosity = 1
# Load MNIST dataset
(input_train, target_train), (input_test, target_test) = mnist.load_data()
# Reshape data
input_train = input_train.reshape(input_train.shape[0], img_width, img_height, 1)
input_test = input_test.reshape(input_test.shape[0], img_width, img_height, 1)
input_shape = (img_width, img_height, 1)
# Parse numbers as floats
input_train = input_train.astype('float32')
input_test = input_test.astype('float32')
# Normalize data: [0, 1].
input_train = input_train / 255
input_test = input_test / 255
# Convert target vectors to categorical targets
target_train = tensorflow.keras.utils.to_categorical(target_train, no_classes)
target_test = tensorflow.keras.utils.to_categorical(target_test, no_classes)
# Create the model
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation=custom_activation, input_shape=((input_shape))))
The Error
When trying to execute the following line:
model.add(Conv2D(32, kernel_size=(3, 3), activation=custom_activation, input_shape=((input_shape))))
This error appears:
AttributeError: Exception encountered when calling layer "conv2d_4" (type Conv2D).
module 'keras.api._v2.keras.backend' has no attribute 'x'
Call arguments received by layer "conv2d_4" (type Conv2D):
• inputs=tf.Tensor(shape=(None, 28, 28, 1), dtype=float32)
I actually want to estimate a normalizing flow with a mixture of gaussians as the base distribution, so I'm sort of stuck with torch. However you can reproduce my error in my code by just estimating a mixture of Gaussian model in torch. My code is below:
import numpy as np
import matplotlib.pyplot as plt
import sklearn.datasets as datasets
import torch
from torch import nn
from torch import optim
import torch.distributions as D
num_layers = 8
weights = torch.ones(8,requires_grad=True).to(device)
means = torch.tensor(np.random.randn(8,2),requires_grad=True).to(device)#torch.randn(8,2,requires_grad=True).to(device)
stdevs = torch.tensor(np.abs(np.random.randn(8,2)),requires_grad=True).to(device)
mix = D.Categorical(weights)
comp = D.Independent(D.Normal(means,stdevs), 1)
gmm = D.MixtureSameFamily(mix, comp)
num_iter = 10001#30001
num_iter2 = 200001
loss_max1 = 100
for i in range(num_iter):
x = torch.randn(5000,2)#this can be an arbitrary x samples
loss2 = -gmm.log_prob(x).mean()#-densityflow.log_prob(inputs=x).mean()
optimizer1.zero_grad()
loss2.backward()
optimizer1.step()
The error I get is:
0
8.089411823514835
Traceback (most recent call last):
File "/home/cameron/AnacondaProjects/gmm.py", line 183, in <module>
loss2.backward()
File "/home/cameron/anaconda3/envs/torch/lib/python3.7/site-packages/torch/tensor.py", line 221, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/home/cameron/anaconda3/envs/torch/lib/python3.7/site-packages/torch/autograd/__init__.py", line 132, in backward
allow_unreachable=True) # allow_unreachable flag
RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.
After as you see the model runs for 1 iteration.
There is ordering problem in your code, since you create Gaussian mixture model outside of training loop, then when calculate the loss the Gaussian mixture model will try to use the initial value of the parameters that you set when you define the model, but the optimizer1.step() already modify that value so even you set loss2.backward(retain_graph=True) there will still be the error: RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation
Solution to this problem is simply create new Gaussian mixture model whenever you update the parameters, example code running as expected:
import numpy as np
import matplotlib.pyplot as plt
import sklearn.datasets as datasets
import torch
from torch import nn
from torch import optim
import torch.distributions as D
num_layers = 8
weights = torch.ones(8,requires_grad=True)
means = torch.tensor(np.random.randn(8,2),requires_grad=True)
stdevs = torch.tensor(np.abs(np.random.randn(8,2)),requires_grad=True)
parameters = [weights, means, stdevs]
optimizer1 = optim.SGD(parameters, lr=0.001, momentum=0.9)
num_iter = 10001
for i in range(num_iter):
mix = D.Categorical(weights)
comp = D.Independent(D.Normal(means,stdevs), 1)
gmm = D.MixtureSameFamily(mix, comp)
optimizer1.zero_grad()
x = torch.randn(5000,2)#this can be an arbitrary x samples
loss2 = -gmm.log_prob(x).mean()#-densityflow.log_prob(inputs=x).mean()
loss2.backward()
optimizer1.step()
print(i, loss2)
Using Keras DL library with Tensorflow backend, I'm attempting to extract features from the intermediate layer of VGG-16 to perform binary classification on my task of interest.
The data set contains 1000 (500 for each class) training samples and 200 (100 for each class) test samples. I'm training a small fully connected model on top of the extracted features. On running the code, I could see that the size of the train_data is (10000,6,6,512), validation_data is (2000,6,6,512) (float32), train_labels is (1000,2) and validation_labels is (200,2) (int32).
Here is the code:
########################load libraries#########################################
import numpy as np
import time
from keras.layers import Dense, Dropout, Flatten
from keras.models import Model
from keras import applications
from keras.optimizers import SGD
from keras.preprocessing.image import ImageDataGenerator
#########################image characteristics#################################
img_rows=100 #dimensions of image, to be varied suiting the input requirements of the pre-trained model
img_cols=100
channel = 3 #RGB
num_classes = 2
batch_size = 10
nb_epoch = 10
###############################################################################
''' This code uses VGG-16 as a feature extractor'''
feature_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(img_rows, img_cols, 3))
#get the model summary
feature_model.summary()
#extract feature from the intermediate layer
feature_model = Model(input=feature_model.input, output=feature_model.get_layer('block5_conv2').output)
#get the model summary
feature_model.summary()
#declaring image data generators
train_datagen = ImageDataGenerator()
val_datagen = ImageDataGenerator()
generator = train_datagen.flow_from_directory(
'f1_100/train',
target_size=(img_rows, img_cols),
batch_size=batch_size,
class_mode=None,
shuffle=False)
train_data = feature_model.predict_generator(generator, 1000)
train_labels = np.array([[1, 0]] * 500 + [[0, 1]] * 500)
generator = val_datagen.flow_from_directory(
'f1_100/test',
target_size=(img_rows, img_cols),
batch_size=batch_size,
class_mode=None,
shuffle=False)
validation_data = feature_model.predict_generator(generator, 200)
validation_labels = np.array([[1,0]] * 100 + [[0,1]] * 100)
###############################################################################
#addding the top layers and training them on the extracted features
from keras.models import Sequential
model = Sequential()
model.add(Flatten(input_shape=train_data.shape[1:]))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
sgd = SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
print('-'*30)
print('Start Training the top layers on the extracted features...')
print('-'*30)
#measure the time and train the model
t=time.time()
hist = model.fit(train_data, train_labels, nb_epoch=nb_epoch, batch_size=batch_size,
validation_data=(validation_data, validation_labels),
verbose=2)
#print the history of the trained model
print(hist.history)
print('Training time: %s' % (time.time()-t))
###############################################################################
However, on running the code, I'm getting the following error:
Traceback (most recent call last):
File "<ipython-input-14-cc5b1b34cc67>", line 46, in <module>
verbose=2)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\models.py", line 960, in fit
validation_steps=validation_steps)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1581, in fit
batch_size=batch_size)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1426, in _standardize_user_data
_check_array_lengths(x, y, sample_weights)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 250, in _check_array_lengths
'and ' + str(list(set_y)[0]) + ' target samples.')
ValueError: Input arrays should have the same number of samples as target arrays. Found 10000 input samples and 1000 target samples.
You see, you have a batch_size of 10.
The feature_model.predict_generator() uses the steps param (in your case 1000) a total of 10 (batch_size) times. So a total of 10000 training samples are generated in that.
But in the next line, you are declaring the labels to be only 1000. (500 1s and 500 0s).
So you have two options:
1) Either change the steps in predict_generator() like this (which I believe is what you want, to generate 1000 samples in train and 200 samples in validation):
train_data = feature_model.predict_generator(generator, 100)
validation_data = feature_model.predict_generator(generator, 20)
2) Or you can change the numbers in labels:
train_labels = np.array([[1, 0]] * 5000 + [[0, 1]] * 5000)
validation_labels = np.array([[1,0]] * 1000 + [[0,1]] * 1000)
I am writing a tensorflow program to find dogs vs cats prediction i got a error in this part of the code when i tried to execute this cell in juypter notebook.
2.Below is the code:
import tflearn
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.core import input_data, dropoutfully_connected
from tflearn.layers.estimator import regression
import tensorflow as tf
tf.reset_default_graph()
convnet = input_data(shape=[None, IMG_SIZE, IMG_SIZE, 1],name='input')
convnet = conv_2d(convnet, 32, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = conv_2d(convnet, 64, 5, activation='relu')
convnet = max_pool_2d(convnet, 5)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 2, activation='softmax')
convnet = regression(convnet, optimizer='adam', learning_rate=LR, loss='categorical_crossentropy', name='targets')
model = tflearn.DNN(convnet, tensorboard_dir='log')
----- - ---------------------------------------------------------------------
Error for the above code below (can anyone help to resolve this error):
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
<ipython-input-6-2ef14dea0c38> in <module>()
----> 1 import tflearn
2 from tflearn.layers.conv import conv_2d, max_pool_2d
3 from tflearn.layers.core import input_data, dropout, fully_connected
4 from tflearn.layers.estimator import regression
5
/home/aravind/anaconda3/lib/python3.6/site-packages/tflearn/__init__.py in <module>()
19
20 # Predefined ops
---> 21 from .layers import normalization
22 from . import metrics
23 from . import activations
/home/aravind/anaconda3/lib/python3.6/site-packages/tflearn/layers/__init__.py in <module>()
8 from .normalization import batch_normalization, local_response_normalization
9 from .estimator import regression
---> 10 from .recurrent import lstm, gru, simple_rnn, bidirectional_rnn, \
11 BasicRNNCell, BasicLSTMCell, GRUCell
12 from .embedding_ops import embedding
/home/aravind/anaconda3/lib/python3.6/site-packages/tflearn/layers/recurrent.py in <module>()
6 import tensorflow as tf
7 from tensorflow.python.ops import array_ops
----> 8 from tensorflow.contrib.rnn.python.ops.core_rnn import static_rnn as _rnn, \
9 static_bidirectional_rnn as _brnn
10 from tensorflow.python.ops.rnn import rnn_cell_impl as _rnn_cell, \
ModuleNotFoundError: No module named 'tensorflow.contrib.rnn.python.ops.core_rnn'
I have trained LeNet for MNIST using Caffe and now I would like to export this model to be used within Keras.
To this end I tried to extract weights from caffe.Net and use them to initialize Keras's network. However, I received different predictions from the two models. So I have tried to debug them layer by layer, starting with the first one. The code I have tested is as follows:
# import caffe and load facce.Net from prototxt and caffemodel files
import sys
sys.path.append('/opt/caffe/python')
import caffe
net = caffe.Net('lenet_train_test.prototxt', 'save/mnist_iter_500000.caffemodel', caffe.TEST)
# this should be the kernel weights and bias for the first convolution layer 'conv1'
c1_w = net.params['conv1'][0].data
c1_b = net.params['conv1'][1].data
# import Keras and build a convolution layer using the same parameters as in lenet_train_test.prototxt and instantiate it with the weights above
import keras
from keras.layers import Convolution2D
conv1 = Convolution2D(20, 5, 5, border_mode='valid', input_shape=(1, 28, 28), weights=[c1_w,c1_b], activation='linear')
from keras.models import Sequential
model=Sequential()
model.add(conv1)
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
# load MNIST data and do scaling like I did when training Caffe model
from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_test = X_test.reshape(X_test.shape[0], 1, 28, 28)
X_test = X_test.astype('float32')
X_test *= 0.00392157
# set the first test example and get the prediction
net.blobs['data'].data[0] = X_test[0][0]
net.forward()
out0 = net.blobs['conv1'].data[0] # this is the prediction for 20 kernels
m0 = out0[0] # just consider the first one
m0.shape # >>> (24,24)
# pass the same example through the conv1 layer in Keras model
import numpy as np
a = np.zeros( (1,1,28,28) )
a[0] = X_test[0]
out1 = model.predict_on_batch(a) # this is the prediction for 20 kernels
m1 = out1[0][0] # just consider the first one
m1.shape # >>> (24,24) the same size
# I get a lots of 'False'
m0 == m1
Have I done anything wrong in the layer construction? or maybe Caffe and Keras implement the convolution2D differently?