Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import torch.nn as nn
import torch.optim as optim
import torchvision
from torch.autograd import Variable
import torch
from torchvision import transforms
labelencoder = LabelEncoder()
cols_to_drop = []
columns = full_data.columns.values
for column in columns:
if full_data[column].dtype not in(np.int64, np.float64)and full_data[column].nunique() > 2:
dummies = pd.get_dummies(full_data[column], prefix = str(column))
cols_to_drop.append(column)
 full_data = pd.concat([full_data, dummies], axis = 1)
 elif full_data[column].dtype not in(np.int64, np.float64)and full_data[column].nunique() < 3:
full_data[column] = labelencoder.fit_transform(full_data[column])
 cols_to_drop.append(column)
 full_data.drop(cols_to_drop, axis = 1, inplace = True)
 print(f'The new dataframe comprises of {test_data.shape[0]} rows and {test_data.shape[1]} columns.
 ' )<split>
def to_np(v):

 if isinstance(v,(np.ndarray, np.generic)) : return v
if isinstance(v,(list,tuple)) : return [to_np(o)for o in v]
if isinstance(v, Variable): v=v.data
if torch.cuda.is_available() :
if isinstance(v, torch.cuda.HalfTensor): v=v.float()
if isinstance(v, torch.FloatTensor): v=v.float()
return v.cpu().numpy()
def score(model, x, y=None, ret=0):

 y_pred = model(x)
 if ret == 1:
return to_np(y_pred)
 elif ret == 2:
return to_np(y_pred ).argmax(1)
 else:
ypred_argmax = to_np(y_pred ).argmax(axis=1)
 return np.sum(ypred_argmax == to_np(y)) , y_pred
def accuracy(preds, targs):

 preds = torch.max(preds, dim=1)[1]
return(preds==targs ).float().mean()
def show(img, title=None):

 plt.imshow(img, cmap="gray")
 if title is not None: plt.title(title)
 def plots(ims, figsize=(12, 6), rows=2, titles=None):

 f = plt.figure(figsize=figsize)
 cols = len(ims)//rows
for i in range(len(ims)) :
sp = f.add_subplot(rows, cols, i+1)
 sp.axis('Off')
 if titles is not None: sp.set_title(titles[i], fontsize=16)
 plt.imshow(ims[i], cmap='gray' )
train_df = full_data[:train_data.shape[0]]
test_df = full_data[train_data.shape[0]:]<choose_model_class>
PATH = ".. /input/"
tf.random.set_seed(42)
 model = tf.keras.Sequential([
tf.keras.layers.Dense(1000),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(100),
tf.keras.layers.Dense(100),
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(1)
 ])
 model.compile(loss=tf.keras.losses.mae,
optimizer=tf.keras.optimizers.Adam(learning_rate = 0.0003),
metrics=['mae'])
 history = model.fit(train_df,sale_price, epochs=400, verbose=0)
 pd.DataFrame(history.history ).plot()
plt.ylabel("loss")
 plt.xlabel("epochs")
 plt.show()
DNN_predictions = model.predict(test_df)
 DNN_predictions = tf.squeeze(DNN_predictions, axis = 1)
 DNN_predictions = np.array(DNN_predictions)
 DNN_predictions<choose_model_class>
train = pd.read_csv(f'{PATH}train.csv')
 test = pd.read_csv(f'{PATH}test.csv' )
xgboost = XGBRegressor(learning_rate=0.008,
n_estimators=6000,
max_depth=8,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:squarederror',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 xgb = xgboost.fit(train_df,sale_price)
 training_accuracy = xgb.score(train_df,sale_price)
 print("Training accuracy: %.2f%%" %(training_accuracy * 100.0))
xgb_predictions = xgb.predict(test_df )<save_to_csv>
idx = np.random.permutation(train.shape[0])
 valid_idx, train_idx = idx[:10000], idx[10000:]
xtrain, xvalid = train.iloc[train_idx,:].values, train.iloc[valid_idx,:].values
ytrain, yvalid = xtrain[:, 0], xvalid[:,0]
xtrain, xvalid = xtrain[:, 1:], xvalid[:, 1:]
predictions = [(xgb_pred + DNN_pred)/ 2 for xgb_pred, DNN_pred in zip(xgb_predictions, DNN_predictions)]
submission = pd.DataFrame({'ID':test_data['Id'],'SalePrice':predictions})
 submission.to_csv('submission.csv',index = False)
 submission<import_modules>
xtrain =(xtrain-mean)/std
xvalid =(xvalid-mean)/std
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.preprocessing import MinMaxScaler, RobustScaler
from sklearn.metrics import mean_squared_error, explained_variance_score
from sklearn.linear_model import RidgeCV, Ridge
from sklearn.ensemble import VotingRegressor, GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor, StackingRegressor
from xgboost import XGBRegressor<set_options>
def rotate_cv(im_arr):

 output = []
degs = np.random.randint(-20, 20, len(im_arr))
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
r,c,*_ = img.shape
M = cv2.getRotationMatrix2D(( c//2,r//2),degs[i],1)
 out = cv2.warpAffine(img,M,(c,r))
output.append(out)
 return np.array(output)
 def translate_cv(im_arr):

 output = []
pos = np.random.randint(-7, 7, len(im_arr))
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
r, c, *_ = img.shape
M = np.float32([[1, 0, pos[i]],[0, 1, 0]])
 dst = cv2.warpAffine(img, M,(c,r))
output.append(dst)
 return np.array(output)
 def gaussBlur_cv(im_arr):

 output = []
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
y = int(np.random.randint(3)*0.5)
 if y: dst = cv2.GaussianBlur(img,(0, 0), 2)
 else: dst = img
output.append(dst)
 return np.array(output)
 def sobel_cv(im_arr):

 output = []
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
y = int(np.random.randint(3)*0.5)
 if y: dst = cv2.Sobel(img, cv2.CV_8U, 1, 0)
 else: dst = img
output.append(dst)
 return np.array(output)
 def dilate_cv(im_arr):

 output = []
for i in range(len(im_arr)) :
img = np.reshape(im_arr[i],(28, 28))
y = int(np.random.randint(3)*0.5)
 kernel = np.ones(( 3,3), np.uint8)
 if y: dst = cv2.dilate(img, kernel, iterations=1)
 else: dst = img
output.append(dst)
 return np.array(output )
%matplotlib inline
plt.rcParams['figure.figsize'] =(25, 15 )<load_from_csv>
class DataLoader() :
def __init__(self, xt, yt, batch_size=32):

 self.xt, self.yt = xt, yt
self.bs = batch_size
def __iter__(self):
xt_trans = translate_cv(dilate_cv(self.xt))
xt_rot = rotate_cv(xt_trans)
 xout, yout = [], []
for i in range(len(self.xt)) :
xout.append(xt_rot[i])
 yout.append(self.yt[i])
 if len(xout)== self.bs:
yield(Variable(torch.cuda.DoubleTensor(np.array(xout, dtype="float32"))),
Variable(torch.cuda.LongTensor(np.array(yout, dtype="float32"))))
xout, yout = [], []
if len(xout)> 0: yield(Variable(torch.cuda.DoubleTensor(np.array(xout, dtype="float32"))),
Variable(torch.cuda.LongTensor(np.array(yout, dtype="float32"))))
def __len__(self):
if len(self.xt)%self.bs==0 : return len(self.xt)//self.bs
else : return len(self.xt)//self.bs+1
data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 data.drop(columns=['Id'], inplace=True )<data_type_conversions>
%time dl = DataLoader(xtrain_imgs, ytrain )
data.columns[data.isna().any() ].tolist()<count_values>
class valGenerator() :
def __init__(self, xv, yv, batch_size=500):

 self.data = np.zeros(( len(xv), 28*28+1))
self.bs = batch_size
self.data[:, 0] = yv
self.data[:, 1:785] = xv
def __iter__(self):
while True:
idxs = np.random.permutation(len(self.data)) [0:self.bs]
yield(Variable(torch.cuda.DoubleTensor(self.data[idxs, 1:785].reshape(( -1, 1, 28, 28)))) ,
Variable(torch.cuda.LongTensor(self.data[idxs, 0])) )
def cleanNumericalData(df, feature):
mode = df[feature].dropna().value_counts().index[0]
df[feature] = df[feature].replace(np.nan, mode)
 return df[feature]<drop_column>
vgen = valGenerator(xvalid, yvalid )
nonObjCols = data.select_dtypes(exclude=['object'] ).columns
for feature in nonObjCols:
data[feature] = cleanNumericalData(data, feature )<feature_engineering>
class ConvNet(nn.Module):
def __init__(self):
super(ConvNet, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(1, 64, 3, padding=1),
nn.ReLU() ,
nn.Conv2d(64, 64, 3, padding=1),
nn.ReLU() ,
nn.MaxPool2d(2, padding=1),
nn.BatchNorm2d(64),
nn.Dropout(0.25),
nn.Conv2d(64, 32, 3, padding=1),
nn.ReLU() ,
nn.Conv2d(32, 32, 3, padding=1),
nn.ReLU() ,
nn.MaxPool2d(2, padding=1),
nn.Dropout(0.20), 
)
 self.fc = nn.Sequential(
 nn.Linear(32*8*8, 500),
nn.ReLU() ,
nn.BatchNorm1d(500),
nn.Dropout(p=0.5),
nn.Linear(500, 100),
nn.ReLU() ,
nn.Linear(100, 10),
nn.ReLU() ,
nn.Softmax(dim=1)
)
 def forward(self, x):
x = self.conv(x)
 x = x.view(-1, x.size(1)*x.size(2)*x.size(3))
return self.fc(x )
data['MasVnrType'] = data['MasVnrType'].replace(np.nan, data.MasVnrType.value_counts().index[0])
 data['Electrical'] = data['Electrical'].replace(np.nan, data.Electrical.value_counts().index[0] )<prepare_x_and_y>
def set_lrs(opt, lr):

 for pg in opt.param_groups: pg['lr'] = lr
data = pd.get_dummies(data)
 columns = data.columns
data = pd.DataFrame(data, columns = columns)
 y = data.SalePrice.values
X = data.loc[:, data.columns != 'SalePrice'].values<split>
class lrAnnealing() :
def __init__(self, ini_lr, epochs, itr_per_epoch, mult_dec=False):

 self.epochs = epochs
self.ipe = itr_per_epoch
self.m_dec = mult_dec
self.ppw =(self.ipe * self.epochs)// 4
self.count = 0
self.lr = ini_lr
self.values = np.cos(np.linspace(np.arccos(self.lr), np.arccos(1e-5), self.ppw))
self.mult = 1
def __call__(self, opt):
self.count += 1
set_lrs(opt, self.values[self.count-1]*self.mult)
 if self.count == len(self.values):
self.count = 0
if self.m_dec: self.mult /= 2
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=18 )<normalization>
y = np.concatenate([np.cos(np.arange(0,1,.01)) ,
np.cos(np.arange (.3, 1,.01)) ,
np.cos(np.arange (.5, 1,.01)) ] )
inScale = RobustScaler()
outScale = RobustScaler()
X_train = inScale.fit_transform(X_train)
 y_train = outScale.fit_transform(y_train.reshape(-1, 1))
X_test = inScale.transform(X_test)
 y_test = outScale.transform(y_test.reshape(-1, 1)) 
<train_model>
x = np.array(np.arange(0, 220, 1))
estimators = [('ridge', ridge),('ridgecv', rcv),('randomforest', rf),('gradientboost', gbr),('xgboost', xb)]
sr = StackingRegressor(estimators=estimators, final_estimator=RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1), cv = 10 ).fit(X_train, y_train.ravel())
 data_predicted = sr.predict(X_test)
 MAX = 200
x = range(len(data_predicted)) [0:MAX]
plt.scatter(x,y_test[0:MAX],color='g', linewidths=5)
 plt.scatter(x,data_predicted[0:MAX],color='r', linewidths=3)
 plt.plot(data_predicted[0:MAX], color = 'r', linewidth=2)
 mse = mean_squared_error(y_test, data_predicted)
 print("MSE: %.4f" % mse)
 evs = explained_variance_score(y_test, data_predicted)
 print("EVS: %.4f" % evs)
 score = sr.score(X_test, y_test)
 print("Score: %.4f" % score )<load_from_csv>
def fit(model, lr, train_dl, valGen, n_epochs, crit, opt, metrics, annln=True, mult_dec=False):

 if annln: annl = lrAnnealing(lr, 40, 500, mult_dec=mult_dec)
 for epoch in range(n_epochs):
losses = []
vl = iter(valGen)
 dl = iter(train_dl)
 length = len(train_dl)
 for t in range(length):
if annln: annl(opt)
 xt, yt = next(dl)
 xt = xt.view(-1, 1, 28, 28)
 xt, yt = xt, yt
y_pred = model(xt)
 l = crit(y_pred, yt)
 losses.append(l)
 opt.zero_grad()
l.backward()
opt.step()
xv, yv = next(vl)
 yright =[]
valLosses = []
mult = len(xv)//100
for i in range(0, len(xv), 100):
if i == mult*100:
a, b = score(model, xv[i:len(xv)], yv[i:len(xv)])
 yright.append(a)
 valLosses.append(crit(b, yv[i:len(xv)] ).data.cpu().numpy())
 else:
a, b = score(model, xv[i:i+100], yv[i:i+100])
 yright.append(a)
 valLosses.append(crit(b, yv[i:i+100] ).data.cpu().numpy())
 val_scores = np.sum(yright)/len(xv)
 meanValLoss = 0
for vliter in range(len(valLosses)) :
meanValLoss += valLosses[vliter]
meanValLoss/=len(valLosses)
 mean_loss = 0
for liter in range(len(losses)) :
mean_loss += losses[liter].data.cpu().numpy()
mean_loss/=len(losses)
 if epoch == 69:
print("Epoch " + str(epoch)+ "::"
+ " loss: " + str(mean_loss)
 + ", valLoss: " + str(meanValLoss)
 +", valAcc: " + str(val_scores*100))
data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 data.head()
id_values = data.Id.values<drop_column>
nets = [0]*10
for i in range(10):
nets[i] = ConvNet().cuda().double()
loss = nn.CrossEntropyLoss()
opt = optim.SGD(nets[i].parameters() , 1e-3, momentum=0.999, weight_decay=1e-3, nesterov=True)
 print(f'{i}th ConvNet:')
 %time fit(nets[i], 1e-2, dl, vgen, 70, loss, opt, metrics, mult_dec=True )
nonObjCols = data.select_dtypes(exclude=['object'] ).columns
for feature in nonObjCols:
data[feature] = cleanNumericalData(data, feature )<feature_engineering>
test =(test-mean)/std
test = np.resize(test,(28000, 1, 28, 28))
test = Variable(torch.DoubleTensor(test)).cuda()
yp = np.zeros(( 28000 ,2), dtype='int'); yp.shape
for i in range(0, 28000, 100):
prob_sum = np.zeros(( 100, 10), dtype='float64')
 for j in range(10):
prob_sum += to_np(nets[j](test[i:i+100]))
yp[i:i+100] = np.array(np.vstack([np.arange(i+1, i+101), prob_sum.argmax(1)] ).T)
 res = pd.DataFrame(yp, index = np.arange(len(yp)) , columns=["ImageId", "Label"])
 res.to_csv("submission.csv", index=False)
 res.head()