Related
I am using Transfer Learning (EfficientNet b0), to train a model, using Adam Optimizer and CrossEntropyLoss for a image classification task.
The model yields 96% val acc and 99% train accuracy. However Inference fails. All images tested always yield the same prediction values.
I tried changing the Learning Rate to a very small value, making the batch size smaller.
What am I doing wrong ?
Following is my train and infer code.
from efficientnet_pytorch import EfficientNet
from torch import nn
from torchvision import models
#using efficientnet model based transfer learning
class EffNet(nn.Module):
def __init__(self, numClasses):
self.numClasses = numClasses
self.effNet = {0: models.efficientnet_b0(pretrained = False, num_classes= self.numClasses),
1: models.efficientnet_b1(pretrained = False, num_classes= self.numClasses),
2: models.efficientnet_b2(pretrained = False, num_classes= self.numClasses),
3: models.efficientnet_b3(pretrained = False, num_classes= self.numClasses),
4: models.efficientnet_b4(pretrained = False, num_classes= self.numClasses),
5: models.efficientnet_b5(pretrained = False, num_classes= self.numClasses),
6: models.efficientnet_b6(pretrained = False, num_classes= self.numClasses),
7: models.efficientnet_b7(pretrained = False, num_classes= self.numClasses)
}
# self.effNet = {0: EfficientNet.from_name(model_name='efficientnet-b0', num_classes=self.numClasses),
# 1: EfficientNet.from_name(model_name='efficientnet-b1', num_classes=self.numClasses),
# 2: EfficientNet.from_name(model_name='efficientnet-b2', num_classes=self.numClasses),
# 3: EfficientNet.from_name(model_name='efficientnet-b3', num_classes=self.numClasses),
# 4: EfficientNet.from_name(model_name='efficientnet-b4', num_classes=self.numClasses),
# 5: EfficientNet.from_name(model_name='efficientnet-b5', num_classes=self.numClasses),
# 6: EfficientNet.from_name(model_name='efficientnet-b6', num_classes=self.numClasses),
# 7: EfficientNet.from_name(model_name='efficientnet-b7', num_classes=self.numClasses)
# }
def getEffnetClassification(self, num_layers, fine_tune):
effNet = self.effNet[num_layers]
if fine_tune:
print('[INFO]: Fine-tuning all layers...')
for params in effNet.parameters():
params.requires_grad = True
elif not fine_tune:
print('[INFO]: Freezing hidden layers...')
for params in effNet.parameters():
params.requires_grad = False
# # Change the final classification head.
n_features = effNet.classifier[1].in_features
effNet.classifier = nn.Linear(in_features=n_features, out_features=self.numClasses)
return effNet
from torchvision import models, transforms
import sys
from efficientNet import EffNet
class Network():
def __init__(self, num_classes):
self.numClasses = num_classes
def getNetwork(self, network_name, num_layers=2, fine_tune=False):
if "efficient" in network_name:
#network = EfficientNet.from_name(network_name)
nw = EffNet(self.numClasses)
print("In EffNet")
print(num_layers)
print(self.numClasses)
print(fine_tune)
network = nw.getEffnetClassification(num_layers=num_layers, fine_tune=fine_tune)
elif "resnet" in network_name:
nw = resNet()
network = nw.getResnetClassification(num_layers)
else:
try:
method = getattr(models, network_name)
except AttributeError:
raise NotImplementedError("Pytorch does not implement `{}`".format(method_name))
network = method(pretrained=False)
return network
Training Code::
print("Printing Phase")
print(phase)
torch.cuda.empty_cache()
gc.collect()
bestValLoss = float('inf')
bestAcc = 0
bestLoss = float('inf')
total_step = len(self.dataLoader[phase])
losses = list()
acc = list()
valLosses = list()
valAcc = list()
datadict = {}
if(patience!=None):
earlystop = EarlyStopping(patience = patience,verbose = True)
for epoch in range(self.epochs):
startTime = datetime.now()
print('Epoch {}/{}'.format(epoch+1, self.epochs))
print('-' * 10)
for phase in ['train', 'val']:
if phase == 'train':
self.model.train()
else:
self.model.eval()
running_loss = 0.0
running_corrects = 0
total=0
for batch_id, (imgName, inputs, labels) in enumerate(self.dataLoader[phase]): # Change Here
print("BatchID")
print(batch_id)
inputs = inputs.to(device)
labels = labels.to(device)
self.optimizer.zero_grad()
outputs = self.model(inputs)
#print(outputs)
#print(labels)
loss = self.criterion(outputs, labels.long())
##loss = self.criterion(outputs.float(), labels.float()) # This was changed only for BCEWithLogitsLoss. if you change Loss Type change to line above
if phase == 'train':
loss.backward()
self.optimizer.step()
_, preds = torch.max(outputs, 1)
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
total += labels.size(0)
epoch_loss = running_loss / len(self.dataLoader[phase])
epoch_acc = running_corrects.double() / total
print('{} loss: {:.4f}, acc: {:.4f}'.format(phase,
epoch_loss,
epoch_acc))
endTime = datetime.now()
print('{} loss: {:.4f}, acc: {:.4f}'.format(phase,
epoch_loss,
epoch_acc))
print('Time For Epoch {} :: {} seconds '.format(epoch, (endTime-startTime).total_seconds()))
losses.append(epoch_loss)
acc.append(epoch_acc.cpu().numpy().item())
print("Printing in Epoch")
print(losses)
print(acc)
if epoch_acc > bestAcc:
bestAcc = epoch_acc
bestModelWeights = copy.deepcopy(self.model.state_dict())
torch.save(bestModelWeights, self.path_model)
if epoch_loss < bestLoss:
bestLoss = epoch_loss
torch.save(self.model.state_dict(), self.path_model)
runningValAcc, runningValLoss = self.evalAndSave(bestValLoss, epoch)
valLosses.append(runningValLoss)
valAcc.append(runningValAcc.cpu().numpy().item())
print("Printing Val Data")
print(valLosses)
print(valAcc)
if runningValLoss < bestValLoss and patience:
earlystop(runningValLoss, self.model)
bestValLoss = runningValLoss
if earlystop.early_stop:
earlystop.save_checkpoint(runningValLoss, self.model)
print("Early Stopping")
break
print("Final Print in Trainer Before Return")
print("Training Losses")
print(losses)
print("Training Accuracy")
print(acc)
print("Validation Losses")
print(valLosses)
print("Validation Accuracy")
print(valAcc)
datadict["trainLoss"] = losses
datadict["trainAcc"] = acc
datadict["valLoss"] = valLosses
datadict["valAcc"] = valAcc
datadict["optimizer"] = self.optimizer
datadict["criterion"] = self.criterion
return self.model, datadict
Inference Script:::
with torch.no_grad():
self.model.eval()
#sm = nn.Softmax(dim = 1)
for batch_id, (imgPaths, inputs, labels) in enumerate(self.dataLoader):
for imgPath, image, label in zip(imgPaths, inputs, labels):
output= None
predictedClassName = None
predictions = None
topclass = None
topk = None
new_record = pd.DataFrame()
image = image.to(device)
label = label.to(device)
print("Printing Label")
print(label)
image_tensor = image.unsqueeze_(0)
output = self.model(image_tensor.cuda())
#print(output)
predictions = torch.exp(output.data)
#_, predictions = torch.max(output.data, 1)
#predictions = sm(output) #.softmax(output.data, dim=1)
topk, topclass = torch.max(predictions, 1)
print("Predictions")
print(predictions)
print("Checking")
print(topk)
print(topclass)
correct += torch.sum(topk == label.long()).cpu().numpy()
totalPredictions.extend(topk.cpu().numpy())
#print("Predictions")
#print(totalPredictions)
totalGT.extend([label.cpu().numpy().item()])
#print("Label")
#print([label.cpu().numpy()])
#print(label.cpu().numpy().item())
#k = output_.item()==label.item()
print("FileName")
print(imgPath)
className = self.dictClasses.get(label.item(), None)
predictedClassName = self.dictClasses.get(topclass.item(), None)
print("Predicted Class Names")
print(predictedClassName)
#print(predictedClassNameList)
predictedClassNameList.append(predictedClassName)
new_record = pd.DataFrame([[imgPath, topk.cpu().numpy().item(), predictedClassName]],columns=[ "FileName" , "Confidence" , "Classification"])
dfInfoTable = pd.concat([dfInfoTable,new_record])
cnt = cnt + 1
if cnt%100 == 0:
path = validationFigureLoc + "\\" + "ValidationOuput.csv"
dfInfoTable.to_csv(path, index=False, header=True)
print(cnt)
I created a Neural Network that takes two greyscale images 14x14 pixels portraying a digit (from MNIST database) and returns 1 if the first digit is less or equal to the second digit, returns 0 otherwise. The code runs, but every time the initial weights are the same. They should be random
Forcing the initial weights to be random, by using the following line of code in the Net class, does not help.
torch.nn.init.normal_(self.layer1.weight, mean=0.0, std=0.01)
Here is the code of the "main.py" file:
import os; os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
import torch
import torch.nn as nn
from dlc_practical_prologue import *
class Net(nn.Module):
def __init__(self):
super().__init__()
self.layer1 = nn.Linear(2*14*14, 32)
#torch.nn.init.normal_(self.layer1.weight, mean=0.0, std=0.01)
#self.layer2 = nn.Linear(100, 100)
#self.layer3 = nn.Linear(100, 100)
self.layer2 = nn.Linear(32, 1)
def forward(self, x):
x = torch.relu(self.layer1(x))
#x = torch.relu(self.layer2(x))
#x = torch.relu(self.layer3(x))
x = torch.sigmoid(self.layer2(x))
return x
if __name__ == '__main__':
# Data initialization
N = 1000
train_input, train_target, train_classes, _, _, _, = generate_pair_sets(N)
_, _, _, test_input, test_target, test_classes = generate_pair_sets(N)
train_input = train_input.view(-1, 2*14*14)
test_input = test_input.view(-1, 2*14*14)
train_target = train_target.view(-1, 1)
test_target = test_target.view(-1, 1)
# I convert the type to torch.float32
train_input, train_target, train_classes, test_input, test_target, test_classes = \
train_input.type(torch.float32), train_target.type(torch.float32), train_classes.type(torch.long), \
test_input.type(torch.float32), test_target.type(torch.float32), test_classes.type(torch.long)
# Create the neural network
net = Net()
# Training
learning_rate = 0.01
# Use MSELoss
loss = nn.MSELoss()
# Use Adam optimizer
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
EPOCHS = 50
for param in net.parameters():
print(param)
for epoch in range(EPOCHS):
target_predicted = net(train_input)
l = loss(train_target, target_predicted) #loss = nn.MSELoss()
#l = loss(target_predicted, train_target)
l.backward()
optimizer.step()
optimizer.zero_grad()
#print(l)
# Testing
total = 1000
correct = 0
with torch.no_grad():
correct = ( test_target == net(test_input).round() ).sum()
print("Accuracy %.2f%%" % (correct / total * 100))
Here is the code for "dlc_practical_monologue.py":
import os; os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
import torch
from torchvision import datasets
import argparse
import os
import urllib
######################################################################
parser = argparse.ArgumentParser(description='DLC prologue file for practical sessions.')
parser.add_argument('--full',
action='store_true', default=False,
help = 'Use the full set, can take ages (default False)')
parser.add_argument('--tiny',
action='store_true', default=False,
help = 'Use a very small set for quick checks (default False)')
parser.add_argument('--seed',
type = int, default = 0,
help = 'Random seed (default 0, < 0 is no seeding)')
parser.add_argument('--cifar',
action='store_true', default=False,
help = 'Use the CIFAR data-set and not MNIST (default False)')
parser.add_argument('--data_dir',
type = str, default = None,
help = 'Where are the PyTorch data located (default $PYTORCH_DATA_DIR or \'./data\')')
# Timur's fix
parser.add_argument('-f', '--file',
help = 'quick hack for jupyter')
args = parser.parse_args()
if args.seed >= 0:
torch.manual_seed(args.seed)
######################################################################
# The data
def convert_to_one_hot_labels(input, target):
tmp = input.new_zeros(target.size(0), target.max() + 1)
tmp.scatter_(1, target.view(-1, 1), 1.0)
return tmp
def load_data(cifar = None, one_hot_labels = False, normalize = False, flatten = True):
if args.data_dir is not None:
data_dir = args.data_dir
else:
data_dir = os.environ.get('PYTORCH_DATA_DIR')
if data_dir is None:
data_dir = './data'
if args.cifar or (cifar is not None and cifar):
print('* Using CIFAR')
cifar_train_set = datasets.CIFAR10(data_dir + '/cifar10/', train = True, download = True)
cifar_test_set = datasets.CIFAR10(data_dir + '/cifar10/', train = False, download = True)
train_input = torch.from_numpy(cifar_train_set.data)
train_input = train_input.transpose(3, 1).transpose(2, 3).float()
train_target = torch.tensor(cifar_train_set.targets, dtype = torch.int64)
test_input = torch.from_numpy(cifar_test_set.data).float()
test_input = test_input.transpose(3, 1).transpose(2, 3).float()
test_target = torch.tensor(cifar_test_set.targets, dtype = torch.int64)
else:
print('* Using MNIST')
######################################################################
# import torchvision
# raw_folder = data_dir + '/mnist/raw/'
# resources = [
# ("https://fleuret.org/dlc/data/train-images-idx3-ubyte.gz", "f68b3c2dcbeaaa9fbdd348bbdeb94873"),
# ("https://fleuret.org/dlc/data/train-labels-idx1-ubyte.gz", "d53e105ee54ea40749a09fcbcd1e9432"),
# ("https://fleuret.org/dlc/data/t10k-images-idx3-ubyte.gz", "9fb629c4189551a2d022fa330f9573f3"),
# ("https://fleuret.org/dlc/data/t10k-labels-idx1-ubyte.gz", "ec29112dd5afa0611ce80d1b7f02629c")
# ]
# os.makedirs(raw_folder, exist_ok=True)
# # download files
# for url, md5 in resources:
# filename = url.rpartition('/')[2]
# torchvision.datasets.utils.download_and_extract_archive(url, download_root=raw_folder, filename=filename, md5=md5)
######################################################################
mnist_train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True)
mnist_test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True)
train_input = mnist_train_set.data.view(-1, 1, 28, 28).float()
train_target = mnist_train_set.targets
test_input = mnist_test_set.data.view(-1, 1, 28, 28).float()
test_target = mnist_test_set.targets
if flatten:
train_input = train_input.clone().reshape(train_input.size(0), -1)
test_input = test_input.clone().reshape(test_input.size(0), -1)
if args.full:
if args.tiny:
raise ValueError('Cannot have both --full and --tiny')
else:
if args.tiny:
print('** Reduce the data-set to the tiny setup')
train_input = train_input.narrow(0, 0, 500)
train_target = train_target.narrow(0, 0, 500)
test_input = test_input.narrow(0, 0, 100)
test_target = test_target.narrow(0, 0, 100)
else:
print('** Reduce the data-set (use --full for the full thing)')
train_input = train_input.narrow(0, 0, 1000)
train_target = train_target.narrow(0, 0, 1000)
test_input = test_input.narrow(0, 0, 1000)
test_target = test_target.narrow(0, 0, 1000)
print('** Use {:d} train and {:d} test samples'.format(train_input.size(0), test_input.size(0)))
if one_hot_labels:
train_target = convert_to_one_hot_labels(train_input, train_target)
test_target = convert_to_one_hot_labels(test_input, test_target)
if normalize:
mu, std = train_input.mean(), train_input.std()
train_input.sub_(mu).div_(std)
test_input.sub_(mu).div_(std)
return train_input, train_target, test_input, test_target
######################################################################
def mnist_to_pairs(nb, input, target):
input = torch.functional.F.avg_pool2d(input, kernel_size = 2)
a = torch.randperm(input.size(0))
a = a[:2 * nb].view(nb, 2)
input = torch.cat((input[a[:, 0]], input[a[:, 1]]), 1)
classes = target[a]
target = (classes[:, 0] <= classes[:, 1]).long()
return input, target, classes
######################################################################
def generate_pair_sets(nb):
if args.data_dir is not None:
data_dir = args.data_dir
else:
data_dir = os.environ.get('PYTORCH_DATA_DIR')
if data_dir is None:
data_dir = './data'
train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True)
train_input = train_set.data.view(-1, 1, 28, 28).float()
train_target = train_set.targets
test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True)
test_input = test_set.data.view(-1, 1, 28, 28).float()
test_target = test_set.targets
return mnist_to_pairs(nb, train_input, train_target) + \
mnist_to_pairs(nb, test_input, test_target)
######################################################################
Note that I have to add the following line of code to run the code on Windows 10, while it is not necessary to run it on Linux.
import os; os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
Also on Linux I always get the same initial weights.
Please, can you help me?
Correct me if I'm wrong here but only the weights of the first layer should be the same each time you run this. The thing is when you import the dlc_practical_monologue.py there's this thing in it:
if args.seed >= 0:
torch.manual_seed(args.seed)
which fires up if the seed is >=0 (default is 0).
This should only initialize the first layer with the same weights for each run. Check if this is the case.
The solution was to delete the following lines from "dlv_practical_prologue.py":
if args.seed >= 0:
torch.manual_seed(args.seed)
I'm working on facial expression recognition, and I'm using Keras. I've collected many datasets, and then I have applied data augmentation on the images, I've got about 500 000 images saved (as pixels) on a .csv file (same format as fer2013.csv).
This is the code I'm using :
def Zerocenter_ZCA_whitening_Global_Contrast_Normalize(list):
Intonumpyarray = numpy.asarray(list)
data = Intonumpyarray.reshape(img_width,img_height)
data2 = ZeroCenter(data)
data3 = zca_whitening(flatten_matrix(data2)).reshape(img_width,img_height)
data4 = global_contrast_normalize(data3)
data5 = numpy.rot90(data4,3)
return data5
def load_data():
train_x = []
train_y = []
val_x = []
val_y = []
test_x = []
test_y = []
f = open('ALL.csv')
csv_f = csv.reader(f)
for row in csv_f:
if str(row[2]) == "Training":
temp_list_train = []
for pixel in row[1].split():
temp_list_train.append(int(pixel))
data = Zerocenter_ZCA_whitening_Global_Contrast_Normalize(temp_list_train)
train_y.append(int(row[0]))
train_x.append(data.reshape(data_resh).tolist())
elif str(row[2]) == "PublicTest":
temp_list_validation = []
for pixel in row[1].split():
temp_list_validation.append(int(pixel))
data = Zerocenter_ZCA_whitening_Global_Contrast_Normalize(temp_list_validation)
val_y.append(int(row[0]))
val_x.append(data.reshape(data_resh).tolist())
elif str(row[2]) == "PrivateTest":
temp_list_test = []
for pixel in row[1].split():
temp_list_test.append(int(pixel))
data = Zerocenter_ZCA_whitening_Global_Contrast_Normalize(temp_list_test)
test_y.append(int(row[0]))
test_x.append(data.reshape(data_resh).tolist())
return train_x, train_y, val_x, val_y, test_x, test_y
And then I load data and feed them to the generator :
Train_x, Train_y, Val_x, Val_y, Test_x, Test_y = load_data()
Train_x = numpy.asarray(Train_x)
Train_x = Train_x.reshape(Train_x.shape[0],img_rows,img_cols)
Test_x = numpy.asarray(Test_x)
Test_x = Test_x.reshape(Test_x.shape[0],img_rows,img_cols)
Val_x = numpy.asarray(Val_x)
Val_x = Val_x.reshape(Val_x.shape[0],img_rows,img_cols)
Train_x = Train_x.reshape(Train_x.shape[0], img_rows, img_cols, 1)
Test_x = Test_x.reshape(Test_x.shape[0], img_rows, img_cols, 1)
Val_x = Val_x.reshape(Val_x.shape[0], img_rows, img_cols, 1)
Train_x = Train_x.astype('float32')
Test_x = Test_x.astype('float32')
Val_x = Val_x.astype('float32')
Train_y = np_utils.to_categorical(Train_y, nb_classes)
Test_y = np_utils.to_categorical(Test_y, nb_classes)
Val_y = np_utils.to_categorical(Val_y, nb_classes)
datagen = ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
shear_range=0.03,
zoom_range=0.03,
vertical_flip=False)
datagen.fit(Train_x)
model.fit_generator(datagen.flow(Train_x, Train_y,
batch_size=batch_size),
samples_per_epoch=Train_x.shape[0],
nb_epoch=nb_epoch,
validation_data=(Val_x, Val_y))
When I run the code, RAM usage gets bigger and bigger until the pc freezes (I've have 16 Gb). It get stuck when loading_data() is called. Any solution for this problem that can fits my code ?
Seems to be a duplicate of this question. Basically, you'll have to use fit_generator() instead of fit() and pass in a function that loads the data into your model one batch at a time instead of all at once.
So I am having trouble obtaining the href link for the next pages of the url. I got up to obtaining all the text and what not that the tag contains but I can't seem to wrap my head around to removing the text that I don't need and just obtaining the href and navigating through the pages.
Here is my code:
import requests
from bs4 import BeautifulSoup
import webbrowser
import time
jobsearch = input("What type of job?: ")
location = input("What is your location: ")
url = ("https://ca.indeed.com/jobs?q=" + jobsearch + "&l=" + location)
base_url = 'https://ca.indeed.com/'
r = requests.get(url)
rcontent = r.content
prettify = BeautifulSoup(rcontent, "html.parser")
filter_words = ['engineering', 'instrumentation', 'QA']
all_job_url = []
nextpages = []
filtered_job_links = []
http_flinks = []
flinks = []
def all_next_pages():
pages = prettify.find_all('div', {'class':'pagination'})
for next_page in pages:
next_page.find_all('a')
nextpages.append(next_page)
print(next_page)
all_next_pages()
Here is a way to get the links of the search result items. Find row result class and then find a tag, it contains all the information you need.
import requests
from bs4 import BeautifulSoup
import webbrowser
import time
jobsearch = input("What type of job?: ")
location = input("What is your location: ")
url = ("https://ca.indeed.com/jobs?q=" + jobsearch + "&l=" + location)
base_url = 'https://ca.indeed.com/'
r = requests.get(url)
rcontent = r.text
prettify = BeautifulSoup(rcontent, "lxml")
filter_words = ['engineering', 'instrumentation', 'QA']
all_job_url = []
nextpages = []
filtered_job_links = []
http_flinks = []
flinks = []
def all_next_pages():
pages = prettify.find_all('div', {'class':' row result'})
for next_page in pages:
info = next_page.find('a')
url = info.get('href')
title = info.get('title')
print(title,url)
all_next_pages()
I have been writing a simple plugin for hackerrank to compile and run code, of a problem on hackerrank from my system.
For ex: I need to test code for this problem https://www.hackerrank.com/challenges/solve-me-first
So, I ran my script like:
python hackerrank.py https://www.hackerrank.com/challenges/solve-me-first solve-me-first.cpp
I get following output:
Traceback (most recent call last):
File "hackerrank.py", line 126, in <module>
h.run()
File "hackerrank.py", line 113, in run
if self.compile_and_test() == "NOT_FOUND":
File "hackerrank.py", line 51, in compile_and_test
j = self.r.json()
File "/usr/lib/python2.7/dist-packages/requests/models.py", line 741, in json
return json.loads(self.text, **kwargs)
File "/usr/lib/python2.7/dist-packages/simplejson/__init__.py", line 488, in loads
return _default_decoder.decode(s)
File "/usr/lib/python2.7/dist-packages/simplejson/decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "/usr/lib/python2.7/dist-packages/simplejson/decoder.py", line 389, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.scanner.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Here is my hackerrank.py file:
import requests
import time
import sys
import os.path
class HackerRank:
def __init__(self,url,code="",ext=".py"):
self.code = code
self.ext = ext
self.set_language()
self.problem_url = url
self.s = requests.session()
self.set_post_url()
def set_post_url(self):
#rootUrl = "https://www.hackerrank.com/"
#l = self.problem_url.split("/")
#print l
#if l[3] == "challenges":
# contestUrl = "/contests/master/challenges/" + l[4]
#else:
# # it's a contest
# contestUrl = "/".join(l[3:])
#self.post_url = rootUrl + "/rest/" + contestUrl + "/compile_tests/"
self.post_url = self.problem_url
def set_language(self):
if self.ext == ".py":
self.language = "python"
if self.ext == ".c":
self.language = "c"
if self.ext == ".cpp":
self.language = "cpp"
if self.ext == ".java":
self.language = "java"
# TODO : find out the language value for other exts
def set_code(self,code):
self.code = code
def generate_payload(self):
self.payload = {'code' : self.code, 'language' : self.language}
def compile_and_test(self):
self.generate_payload()
self.r = self.s.post(self.post_url, params=self.payload)
if self.r.status_code == 404:
print "not found 1"
return "NOT_FOUND"
print "yes"
j = self.r.json()
print j
self.submission_id = j['model']['id']
self.get_url = self.post_url + "/submissions/code/" + str(self.submission_id)
self.rr = self.s.get(self.get_url, cookies = self.s.cookies)
return self.rr
def fetch(self,last_status):
if self.r.status_code == 404:
return
self.rr = self.s.get(self.get_url, cookies = self.s.cookies)
self.res = self.rr.json()
if self.res['model']['status'] == 0:
new_status = self.res['model']['status_string']
if new_status != last_status:
print(new_status)
time.sleep(1)
self.fetch(new_status)
else:
return
def compiler_message(self):
return self.res['model']['compilemessage']
def testcase_message(self):
return self.res['model']['testcase_message']
def expected_output(self):
return self.res['model']['expected_output']
def stdin(self):
return self.res['model']['stdin']
def stdout(self):
return self.res['model']['stdout']
def dump(self):
cm = self.compiler_message()
tm = self.testcase_message()
eo = self.expected_output()
stdin = self.stdin()
stdout = self.stdout()
s = ""
for i in range(len(tm)):
s += (cm + "\n\n")
s += ("Testcase# " + str(i) + "\n")
s += ("Sample Input:\n\n")
s += (stdin[i])
s += ("\n\n")
s += ("Your Output:\n\n")
s += (stdout[i])
s += ("\n\n")
s += ("Expected Output:\n\n")
s += (eo[i])
s += ("\n\n")
s += ("Compiler Message:\n\n")
s += (tm[i])
s += ("\n\n")
print(s)
return s
def run(self):
if self.compile_and_test() == "NOT_FOUND":
print "not found"
return "404 : NOT_FOUND"
self.fetch("")
return self.dump()
if __name__=="__main__":
url = sys.argv[1]
codefile = sys.argv[2]
print url,codefile
ext = os.path.splitext(codefile)[1]
code = open(codefile).read()
h = HackerRank(url,code,ext)
h.run()
I am newbie to json and plugins. Can you help me out.