I want to read data from csv file in tensorflow .So I've been trying out different ways of reading a CSV file with 2000 lines and each line with 93 features,and I hope to get one-hot value.
my dataset is like this:
the first column is data of 93 features,and the second column is labels of 16 one-hot .
this is my code
import tensorflow as tf
# data_input = pd.read_csv('ans_string.csv')
# data_train = pd.read_csv('ans_result.csv')
x = tf.placeholder(tf.float32,[None,93])
W = tf.Variable(tf.zeros([93,16]))
b = tf.Variable(tf.zeros([16]))
sess = tf.InteractiveSession()
filename_queue = tf.train.string_input_producer(["dataset.csv"])
reader = tf.TextLineReader()
key,value = reader.read(filename_queue)
# _,csv_row = reader.read(filename_queue)
# data = tf.decode_csv(csv_row,record_fefaults = rDeraults)
record_defaults_key = [[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1]]
record_defaults_value = [[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1]]
list_result_key = tf.decode_csv(key,record_defaults = record_defaults_key)
list_result_value = tf.decode_csv(value,record_defaults = record_defaults_value)
features = tf.stack(list_result_key)
labels = tf.stack(list_result_value)
y = tf.nn.softmax(tf.matmul(x,W)+b)
y_ = tf.placeholder(tf.float32,[None,16])
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y),reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
with tf.Session() as sess:
# something happened
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord)
tf.global_variables_initializer().run()
for _ in range (1000):
example,label = sess.run([features,labels])
print(sess.run(example,label))
sess.run(train_step,feed_dict={x:example,y_:label})
correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
print(sess.run(accuracy.eval({x:example,y_:label})))
coord.request_stop()
coord.join(threads)
I want to train my model,but I got Error like this.
How can I fix it?
Related
I'm coding a nn that has a img input and numeric, so img will be handled by cnn and numeric by dnn , then they concatenate and make calculations in a fully connected , here is code :
img_input = Input(shape = (99,4,4))
rr_input = Input(shape = (1))
price_input = Input(shape = (20))
legal_action_input = Input(shape = (9))
numeric_input = concatenate([rr_input,price_input, legal_action_input])
cnn = Conv2D(99, (4,4), activation='relu',padding='same', input_shape = (99,4,4))(img_input),
cnn = Dropout(0.2)(cnn)
cnn = Conv2D(99, (4,4), strides = (2,2), activation = 'relu', padding = 'same')(cnn)
cnn = Dropout(0.2)(cnn)
cnn = Conv2D(99, (4,4), strides = (2,2), activation = 'relu', padding = 'same')(cnn)
cnn = Dropout(0.2)(cnn)
cnn = Conv2D(99, (4,4), strides = (2,2), activation = 'relu', padding = 'same')(cnn)
cnn = MaxPooling3D(pool_size = (1,1,1),strides = 3)(cnn)
cnn = Dropout(0.2)(cnn)
cnn = Dense(64, activation = 'relu')(cnn)
cnn = Flatten()(cnn)
cnn = Model(inputs=img_input, outputs=cnn)
dnn = Dense(128, activation='relu', input_shape = (30,))(numeric_input)
dnn = Dropout(0.2)(dnn)
dnn = Dense(128, activation='relu')(dnn)
dnn = Dropout(0.2)(dnn)
dnn = Dense(64, activation='relu')(dnn)
#dnn = Flatten()(dnn)
dnn = Model(inputs=numeric_input, outputs=dnn)
merged = Concatenate()([cnn,dnn])
model = Dense(128, activation='relu')(merged)
model = Dropout(0.2)(model)
model = Dense(64, activation='relu')(model)
model = Dropout(0.2)(model)
model = Dense(9, activation='linear')(model)
model.compile(loss="mse", optimizer=Adam(args.lr))
but when I run them console prints :
--------------------------------------------------------------------------- TypeError Traceback (most recent call
last) Cell In[7], line 28
25 #dnn = Flatten()(dnn)
26 dnn = Model(inputs=numeric_input, outputs=dnn)
---> 28 merged = Concatenate()([cnn,dnn])
30 model = Dense(128, activation='relu')(merged)
31 model = Dropout(0.2)(model)
File
~/miniconda3/lib/python3.9/site-packages/keras/utils/traceback_utils.py:70,
in filter_traceback..error_handler(*args, **kwargs)
67 filtered_tb = _process_traceback_frames(e.traceback)
68 # To get the full stack trace, call:
69 # tf.debugging.disable_traceback_filtering()
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
File
~/miniconda3/lib/python3.9/site-packages/keras/layers/merging/concatenate.py:97,
in Concatenate.build(self, input_shape)
94 #tf_utils.shape_type_conversion
95 def build(self, input_shape):
96 # Used purely for shape validation.
---> 97 if len(input_shape) < 1 or not isinstance(input_shape[0], tuple):
98 raise ValueError(
99 "A Concatenate layer should be called on a list of "
100 f"at least 1 input. Received: input_shape={input_shape}"
101 )
102 if all(shape is None for shape in input_shape):
TypeError: object of type 'NoneType' has no len()
where is the error ?
Why am I getting this error when I try to use writerow
db.writerow(loanList[i])
TypeError: writerows() argument must be iterable
'''
import csv
header = ['Name','Results']
file = open("loanResults.csv", "w", newline = "")
a = 10000
b = [0.032,0.043,0.037,0.043,0.044,0.029,0.028,0.030]
c = [6,7,4,3,4,6,7,9,]
loanList = []
def monthlyRepayment(principalAmount,interest,year):
for i in range(len(b)):
finalAmount = principalAmount * ((1 + interest[i]) **year[i])
monthlyAmount = (finalAmount / (year[i]*12))
loanList.append(round(monthlyAmount))
print(round(monthlyAmount))
testCase1 = monthlyRepayment(a,b,c)
db = csv.writer(file)
db.writerow(header)
for i in range(len(loanList)):
db.writerows(loanList[i])
file.close()
'''
I am here using ResNet50 to create a regression model. I ran into a problem when I wanted to test a model using other data. The length of the dataset is 2050. Then I separate it into training and testing data. I divide it by 1500 as training data and 500 as test data. At the time of the training process, I had good results and was able to predict quite accurately. but when I want to test it using testing data, the prediction results are bad.
below is the model loss result
the code :
Insole = pd.read_csv('1119_Rwalk40s1_list.txt', header=None, low_memory=False)
SIData = np.asarray(Insole)
df = pd.read_csv('1119_Rwalk40s1.csv', low_memory=False)
columns = ['Fx','Fy','Fz','Mx','My','Mz']
selected_df = df[columns]
FCDatas = selected_df[:2050]
SmartInsole = np.array(SIData)
FCData = np.array(FCDatas)
xX = SmartInsole
yY = FCData
scaler_x = MinMaxScaler(feature_range=(0, 1))
scaler_x.fit(xX)
xscale = scaler_x.transform(xX)
scaler_y = MinMaxScaler(feature_range=(0, 1))
scaler_y.fit(yY)
yscale = scaler_y.transform(yY)
SIDataPCA = xscale
pca = PCA(n_components=12)
pca.fit(SIDataPCA)
SIdata_pca = pca.transform(SIDataPCA)
#For Training
trainX = SIdata_pca[:1500]
trainY = yscale[:1500]
#For Testing
testX = SIdata_pca[1500]
testY = yscale[1500:]
X_train, X_test, y_train, y_test = train_test_split(trainX, trainY, test_size=0.20, random_state=2)
Below is the my resnet model structure:
Below is identity blok:
def identity_block(input_tensor,units):
x = layers.Dense(units)(input_tensor)
x = layers.Activation('relu')(x)
x = layers.Dense(units)(x)
x = layers.Activation('relu')(x)
x = layers.Dense(units)(x)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
Below is dens_block:
def dens_block(input_tensor,units):
x = layers.Dense(units)(input_tensor)
x = layers.Activation('relu')(x)
x = layers.Dense(units)(x)
x = layers.Activation('relu')(x)
x = layers.Dense(units)(x)
shortcut = layers.Dense(units)(input_tensor)
x = layers.add([x, shortcut])
x = layers.Activation('relu')(x)
return x
Resnet50 model:
def ResNet50Regression():
Res_input = layers.Input(shape=(12,))
width = 32
x = dens_block(Res_input,width)
x = identity_block(x,width)
x = identity_block(x,width)
x = dens_block(x,width)
x = identity_block(x,width)
x = identity_block(x,width)
x = dens_block(x,width)
x = identity_block(x,width)
x = identity_block(x,width)
x = layers.Dense(6,activation="sigmoid")(x)
model = models.Model(inputs=Res_input, outputs=x)
return model
model = ResNet50Regression()
model.compile(loss='mse',
optimizer=Adam(),
metrics=['mse'])
history = model.fit(X_train, y_train,
batch_size=32,
epochs=50,
validation_data=(X_test, y_test),
verbose=2)
model.save('Resnet50-1203.h5')
ypred = model.predict(trainX)
x=[]
colors=['red','green','brown','teal','gray','black','maroon','orange','purple']
colors2=['green','red','orange','black','maroon','teal','blue','gray','brown']
x = np.arange(0,1500)*40/1500
for i in range(0,6):
plt.figure(figsize=(15,6))
plt.plot(x,trainY[0:1500,i],color=colors[i])
plt.plot(x,ypred[0:1500,i], markerfacecolor='none',color=colors2[i])
plt.title('Result for ResNet Regression (Training Data)')
plt.ylabel(columns[i])
plt.xlabel('Time(s)')
plt.legend(['FP Data', 'SI Prediction'], loc='best')
# plt.savefig('Regression Result.png'[i])
plt.show()
Testing Model using other data code:
new_model = load_model('Resnet50-1203.h5')
model.evaluate(testX, testY)
Test_xX_model = new_model.predict(testX)
x=[]
colors=['red','green','brown','teal','gray','black','maroon','orange','purple']
colors2=['green','red','orange','black','maroon','teal','blue','gray','brown']
x = np.arange(0,550)*40/550
for i in range(0,6):
plt.figure(figsize=(15,6))
plt.plot(x,testY[0:550,i],color=colors[i])
plt.plot(x,Test_xX_model[0:550,i], markerfacecolor='none',color=colors2[i])
plt.title('Result for ResNet Regression (Testing Data)')
plt.ylabel(columns[i])
plt.xlabel('Time(s)')
plt.legend(['FP Data', 'SI Prediction'], loc='best')
# plt.savefig('Regression Result.png'[i])
plt.show()
1 of traning data predictions results:
1 of testing data predictions results:
what should i do for this case?
I have created a spacy transformer model for named entity recognition. Last time I trained till it reached 90% accuracy and I also have a model-best directory from where I can load my trained model for predictions. But now I have some more data samples and I wish to resume training this spacy transformer. I saw that we can do it by changing the config.cfg but clueless about 'what to change?'
This is my config.cfg after running python -m spacy init fill-config ./base_config.cfg ./config.cfg:
[paths]
train = null
dev = null
vectors = null
init_tok2vec = null
[system]
gpu_allocator = "pytorch"
seed = 0
[nlp]
lang = "en"
pipeline = ["transformer","ner"]
batch_size = 128
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
tokenizer = {"#tokenizers":"spacy.Tokenizer.v1"}
[components]
[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"#scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100
[components.ner.model]
#architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = false
nO = null
[components.ner.model.tok2vec]
#architectures = "spacy-transformers.TransformerListener.v1"
grad_factor = 1.0
pooling = {"#layers":"reduce_mean.v1"}
upstream = "*"
[components.transformer]
factory = "transformer"
max_batch_items = 4096
set_extra_annotations = {"#annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
[components.transformer.model]
#architectures = "spacy-transformers.TransformerModel.v3"
name = "roberta-base"
mixed_precision = false
[components.transformer.model.get_spans]
#span_getters = "spacy-transformers.strided_spans.v1"
window = 128
stride = 96
[components.transformer.model.grad_scaler_config]
[components.transformer.model.tokenizer_config]
use_fast = true
[components.transformer.model.transformer_config]
[corpora]
[corpora.dev]
#readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[corpora.train]
#readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[training]
accumulate_gradient = 3
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
patience = 1600
max_epochs = 0
max_steps = 20000
eval_frequency = 200
frozen_components = []
annotating_components = []
before_to_disk = null
[training.batcher]
#batchers = "spacy.batch_by_padded.v1"
discard_oversize = true
size = 2000
buffer = 256
get_length = null
[training.logger]
#loggers = "spacy.ConsoleLogger.v1"
progress_bar = false
[training.optimizer]
#optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001
[training.optimizer.learn_rate]
#schedules = "warmup_linear.v1"
warmup_steps = 250
total_steps = 20000
initial_rate = 0.00005
[training.score_weights]
ents_f = 1.0
ents_p = 0.0
ents_r = 0.0
ents_per_type = null
[pretraining]
[initialize]
vectors = ${paths.vectors}
init_tok2vec = ${paths.init_tok2vec}
vocab_data = null
lookups = null
before_init = null
after_init = null
[initialize.components]
[initialize.tokenizer]
As you can see there is a 'vectors' parameter under [initialize] so I tried giving vectors from 'model-best' like this:
But it gave me this error
OSError: [E884] The pipeline could not be initialized because the vectors could not be found at './model-best/ner'. If your pipeline was already initialized/trained before, call 'resume_training' instead of 'initialize', or initialize only the components that are new.
For those who are wondering that I have been given the wrong path. No, that directory exists. You can see directory structure,
So, please guide me on how I can successfully resume the training from previous weights.
Thank you!
The vectors setting is not related to the transformer or what you're trying to do.
In the new config, you want to use the source option to load the components from the existing pipeline. You would modify the [component] blocks to contain only the source setting and no other settings:
[components.ner]
source = "/path/to/model-best"
[components.transformer]
source = "/path/to/model-best"
See: https://spacy.io/usage/training#config-components
Vector sizes refer to word vectors here. To use the vocabulary from the previously trained Spacy pipeline, you can use the following the code:
[components.ner]
source = "/path/to/model-best"
[initialize]
vectors = ${paths.vectors}
[initialize.before_init]
#callbacks: "spacy.copy_from_base_model.v1"
tokenizer: "/path/to/model-best"
vocab: "/path/to/model-best"
The acc gyro in model.fit is (200 * 3)，in the Input layer shape is (200 * 3). Why is there such a problem? Error when checking input: expected acc_input to have 3 dimensions, but got array with shape (200, 3).This is a visualization of my model.
Here's my code：
WIDE = 20
FEATURE_DIM = 30
CHANNEL = 1
CONV_NUM = 64
CONV_LEN = 3
CONV_LEN_INTE = 3#4
CONV_LEN_LAST = 3#5
CONV_NUM2 = 64
CONV_MERGE_LEN = 8
CONV_MERGE_LEN2 = 6
CONV_MERGE_LEN3 = 4
rnn_size=128
acc_input_tensor = Input(shape=(200,3),name = 'acc_input')
gyro_input_tensor = Input(shape=(200,3),name= 'gyro_input')
Acc_input_tensor = Reshape(target_shape=(20,30,1))(acc_input_tensor)
Gyro_input_tensor = Reshape(target_shape=(20,30,1))(gyro_input_tensor)
acc_conv1 = Conv2D(CONV_NUM,(1, 1*3*CONV_LEN),strides= (1,1*3),padding='valid',activation=None)(Acc_input_tensor)
acc_conv1 = BatchNormalization(axis=1)(acc_conv1)
acc_conv1 = Activation('relu')(acc_conv1)
acc_conv1 = Dropout(0.2)(acc_conv1)
acc_conv2 = Conv2D(CONV_NUM,(1,CONV_LEN_INTE),strides= (1,1),padding='valid',activation=None)(acc_conv1)
acc_conv2 = BatchNormalization(axis=1)(acc_conv2)
acc_conv2 = Activation('relu')(acc_conv2)
acc_conv2 = Dropout(0.2)(acc_conv2)
acc_conv3 = Conv2D(CONV_NUM,(1,CONV_LEN_LAST),strides=(1,1),padding='valid',activation=None)(acc_conv2)
acc_conv3 = BatchNormalization(axis=1)(acc_conv3)
acc_conv3 = Activation('relu')(acc_conv3)
acc_conv3 = Dropout(0.2)(acc_conv3)
gyro_conv1 = Conv2D(CONV_NUM,(1, 1*3*CONV_LEN),strides=(1,1*3),padding='valid',activation=None)(Gyro_input_tensor)
gyro_conv1 = BatchNormalization(axis=1)(gyro_conv1)
gyro_conv1 = Activation('relu')(gyro_conv1)
gyro_conv1 = Dropout(0.2)(gyro_conv1)
gyro_conv2 = Conv2D(CONV_NUM,(1, CONV_LEN_INTE),strides=(1,1),padding='valid',activation=None)(gyro_conv1)
gyro_conv2 = BatchNormalization(axis=1)(gyro_conv2)
gyro_conv2 = Activation('relu')(gyro_conv2)
gyro_conv2 = Dropout(0.2)(gyro_conv2)
gyro_conv3 = Conv2D(CONV_NUM,(1, CONV_LEN_LAST),strides=(1,1),padding='valid',activation=None)(gyro_conv2)
gyro_conv3 = BatchNormalization(axis=1)(gyro_conv3)
gyro_conv3 = Activation('relu')(gyro_conv3)
gyro_conv3 = Dropout(0.2)(gyro_conv3)
sensor_conv_in = concatenate([acc_conv3, gyro_conv3], 2)
sensor_conv_in = Dropout(0.2)(sensor_conv_in)
sensor_conv1 = Conv2D(CONV_NUM2,kernel_size=(2, CONV_MERGE_LEN),padding='SAME')(sensor_conv_in)
sensor_conv1 = BatchNormalization(axis=1)(sensor_conv1)
sensor_conv1 = Activation('relu')(sensor_conv1)
sensor_conv1 = Dropout(0.2)(sensor_conv1)
sensor_conv2 = Conv2D(CONV_NUM2,kernel_size=(2, CONV_MERGE_LEN2),padding='SAME')(sensor_conv1)
sensor_conv2 = BatchNormalization(axis=1)(sensor_conv2)
sensor_conv2 = Activation('relu')(sensor_conv2)
sensor_conv2 = Dropout(0.2)(sensor_conv2)
sensor_conv3 = Conv2D(CONV_NUM2,kernel_size=(2, CONV_MERGE_LEN3),padding='SAME')(sensor_conv2)
sensor_conv3 = BatchNormalization(axis=1)(sensor_conv3)
sensor_conv3 = Activation('relu')(sensor_conv3)
conv_shape = sensor_conv3.get_shape()
print conv_shape
x1 = Reshape(target_shape=(int(conv_shape[1]), int(conv_shape[2]*conv_shape[3])))(sensor_conv3)
x1 = Dense(64, activation='relu')(x1)
gru_1 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru1')(x1)
gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru1_b')(x1)
gru1_merged = merge([gru_1, gru_1b], mode='sum')
gru_2 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru2_b')(gru1_merged)
x = merge([gru_2, gru_2b], mode='concat')
x = Dropout(0.25)(x)
n_class=2
x = Dense(n_class)(x)
model = Model(input=[acc_input_tensor,gyro_input_tensor], output=x)
model.compile(loss='mean_squared_error',optimizer='adam')
model.fit(inputs=[acc,gyro],outputs=labels,batch_size=1, validation_split=0.2, epochs=2,verbose=1 ,
shuffle=False)
The acc gyro in model.fit is (200 * 3)，in the Input layer shape is (200 * 3). Why is there such a problem? Error when checking input: expected acc_input to have 3 dimensions, but got array with shape (200, 3)
Shape (None, 200, 3) is used in Keras for batches, None means batch_size, because in the time of creating or reshaping input arrays, the batch size might be unknown, so if you will be using batch_size = 128 your batch input matrix will have shape (128, 200, 3)