Kaggle competition: Digit Recognizer
shutil.rmtree(train_path)
 gc.collect()<load_pretrained>
%matplotlib inline
if torch.cuda.is_available() :
torch.backends.cudnn.deterministic = True
if not os.path.exists(root_path + '/working/test/'):
os.makedirs(root_path + '/working/test/')
 Archive(root_path + '/input/tensorflow-speech-recognition-challenge/test.7z' ).extractall(root_path + '/working')
 test_path = root_path + '/working/test'<predict_on_test>
train_df = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_df = pd.read_csv(".. /input/digit-recognizer/test.csv" )
test_data,test_labels,_ = get_image_list(test_path)
 test_data = test_data[0]
test_labels = test_labels[0]
test_datagen = data_generator(test_data, test_labels, batch_size, None)
 test_size = len(test_data)
 test_steps = np.ceil(test_size /(batch_size))
y_pred = model.predict_generator(test_datagen, steps = test_steps, verbose=1 )<prepare_output>
train_df.iloc[:,1:]
y_labs = np.argmax(y_pred, axis=1 )<define_variables>
class MNISTDataset(Dataset):

 def __init__(self, dataframe,
transform = transforms.Compose([transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,)) ])
):
df = dataframe
self.n_pixels = 784
if len(df.columns)== self.n_pixels:
self.X = df.values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = None
else:
self.X = df.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = torch.from_numpy(df.iloc[:,0].values)
 self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
if self.y is not None:
return self.transform(self.X[idx]), self.y[idx]
else:
return self.transform(self.X[idx] )
inv_map = inv_map = {v: k for k, v in classes_map.items() }<normalization>
class MNISTResNet(ResNet):
def __init__(self):
super().__init__(BasicBlock, [2, 2, 2, 2], num_classes=10)
 self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=1, padding=3,bias=False)
 model = MNISTResNet()
print(model )
train_audio_sample = test_data[16]
x,sr = librosa.load(train_audio_sample, sr = 16000)
 ipd.Audio(x, rate=sr )<normalization>
def train(train_loader, model, criterion, optimizer, epoch):
model.train()
loss_train = 0
for batch_idx,(data, target)in enumerate(train_loader):
if torch.cuda.is_available() :
data = data.cuda()
target = target.cuda()
output = model(data)
 loss = criterion(output, target)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(batch_idx + 1)% 100 == 0:
print('Train Epoch: {} [{}/{}({:.0f}%)]\tLoss: {:.6f}'.format(
 epoch,(batch_idx + 1)* len(data), len(train_loader.dataset),
100.*(batch_idx + 1)/ len(train_loader), loss.data.item()))
train_audio_sample = test_data[100]
x,sr = librosa.load(train_audio_sample, sr = 16000)
 ipd.Audio(x, rate=sr )<normalization>
def validate(val_loader, model, criterion):
model.eval()
loss = 0
correct = 0
for _,(data, target)in enumerate(val_loader):
if torch.cuda.is_available() :
data = data.cuda()
target = target.cuda()
output = model(data)
 loss += criterion(output, target ).data.item()
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.data.view_as(pred)).cpu().sum()
loss /= len(val_loader.dataset)
 print('
On Val set Average loss: {:.4f}, Accuracy: {}/{}({:.3f}%)
 '.format(
 loss, correct, len(val_loader.dataset),
100.0 * float(correct)/ len(val_loader.dataset)))
train_audio_sample = test_data[1001]
x,sr = librosa.load(train_audio_sample, sr = 16000)
 ipd.Audio(x, rate=sr )<filter>
train_transforms = transforms.Compose(
 [transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,)) ])
 val_test_transforms = transforms.Compose(
 [transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,)) ] )
inv_map[y_labs[1001]]<set_options>
total_epoches = 20
step_size = 5
base_lr = 0.01
batch_size = 64
optimizer = optim.Adam(model.parameters() , lr=base_lr)
 criterion = nn.CrossEntropyLoss()
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=0.1)
 if torch.cuda.is_available() :
model = model.cuda()
criterion = criterion.cuda()
shutil.rmtree(test_path )<save_to_csv>
def split_dataframe(dataframe=None, fraction=0.9, rand_seed=1):
df_1 = dataframe.sample(frac=fraction, random_state=rand_seed)
 df_2 = dataframe.drop(df_1.index)
 return df_1, df_2
my_submission = pd.DataFrame({'fname': test_data, 'label': [inv_map[x] for x in y_labs]})
 my_submission.to_csv('submission.csv', index=False )<set_options>
for epoch in range(total_epoches):
print("
Train Epoch {}: lr = {}".format(epoch, exp_lr_scheduler.get_lr() [0]))
train_df_new, val_df = split_dataframe(dataframe=train_df, fraction=0.9, rand_seed=epoch)
 train_dataset = MNISTDataset(train_df_new, transform=train_transforms)
 val_dataset = MNISTDataset(val_df, transform=val_test_transforms)
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size, shuffle=True)
 val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=batch_size, shuffle=False)
 train(train_loader=train_loader, model=model, criterion=criterion, optimizer=optimizer, epoch=epoch)
 validate(val_loader=val_loader, model=model, criterion=criterion)
 exp_lr_scheduler.step()
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
def prediciton(test_loader, model):
model.eval()
test_pred = torch.LongTensor()
for i, data in enumerate(test_loader):
if torch.cuda.is_available() :
data = data.cuda()
output = model(data)
 pred = output.cpu().data.max(1, keepdim=True)[1]
test_pred = torch.cat(( test_pred, pred), dim=0)
 return test_pred
train = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')
 test = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')
 train_Id = train.Id
test_Id = test.Id
train.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True )<set_options>
test_batch_size = 64
test_dataset = MNISTDataset(test_df, transform=val_test_transforms)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=test_batch_size, shuffle=False)
 test_pred = prediciton(test_loader, model)
 test_pred_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1), test_pred.numpy() ],
columns=['ImageId', 'Label'])
 print(test_pred_df.head() )
pd.options.display.max_columns = None
pd.options.display.max_rows = 80
pd.options.display.float_format = '{:.2f}'.format
%matplotlib inline
warnings.filterwarnings('ignore')
 %matplotlib inline<data_type_conversions>
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index=False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(test_pred_df, filename="submission.csv" )
train['pd_date'] = pd.to_datetime(train['Open Date'], format='%m/%d/%Y')
 train['Open_Year'] = train['pd_date'].dt.strftime('%Y')
 train['Open_Month'] = train['pd_date'].dt.strftime('%m')
 train["Open Date"] = pd.to_datetime(train["Open Date"])
 train["Day"] = train["Open Date"].apply(lambda x:x.day)
 train["kijun"] = "2015-04-27"
train["kijun"] = pd.to_datetime(train["kijun"])
 train["BusinessPeriod"] =(train["kijun"] - train["Open Date"] ).apply(lambda x: x.days)
 train = train.drop('kijun', axis=1)
 train = train.drop('pd_date',axis=1)
 train = train.drop('Open Date',axis=1 )<data_type_conversions>
test_pred_df.to_csv('submission.csv', index=False )