Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_pretrained>
Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten
Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D
%matplotlib inline
print(os.listdir(".. /input/"))
def init_weights(m):
if type(m)==nn.Linear or type(m)==nn.Conv2d:
nn.init.xavier_uniform_(m.weight )<compute_test_metric>
def date_time(x):
if x==1:
return 'Timestamp: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
 if x==2:
return 'Timestamp: {:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now())
 if x==3:
return 'Date now: %s' % datetime.datetime.now()
if x==4:
return 'Date today: %s' % datetime.date.today()
def accuracy(y_hat, y):
return(torch.argmax(y_hat, dim=1)==y ).sum().float().mean()<set_options>
input_directory = r".. /input/digit-recognizer/"
output_directory = r".. /output/"
training_dir = input_directory + "train_images"
testing_dir = input_directory + r"test_images"
if not os.path.exists(output_directory):
os.mkdir(output_directory)
 figure_directory = ".. /output/figures"
if not os.path.exists(figure_directory):
os.mkdir(figure_directory)
 model_output_directory = ".. /output/models/"
if not os.path.exists(model_output_directory):
os.mkdir(model_output_directory)
 file_name_pred_batch = figure_directory+r"/result"
file_name_pred_sample = figure_directory+r"/sample"
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<find_best_params>
train_df = pd.read_csv(input_directory + "train.csv")
 train_df.rename(index=str, columns={"label": "target"}, inplace=True)
 train_df.head()
net = get_net()
net<init_hyperparams>
test_df = pd.read_csv(input_directory + "test.csv")
 test_df.rename(index=str, columns={"label": "target"}, inplace=True)
 test_df.head()
test_labels = None
batch_size= 64
learning_rate = 0.03
lr = learning_rate
weight_decay = 0
num_epochs = 100<train_model>
def get_data(train_X=None, train_Y=None, test_X=None, batch_size=32):
print("Preprocessing and Generating Data Batches.......
 ")
 rescale = 1.0/255
train_batch_size = batch_size
validation_batch_size = batch_size*5
test_batch_size = batch_size*5
train_shuffle = True
val_shuffle = True
test_shuffle = False
train_datagen = ImageDataGenerator(
 horizontal_flip=False,
vertical_flip=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
rescale=rescale,
validation_split=0.25)
 train_generator = train_datagen.flow(
 x=train_X,
y=train_Y,
batch_size=batch_size,
shuffle=True,
sample_weight=None,
seed=42,
save_to_dir=None,
save_prefix='',
save_format='png',
subset='training')
 validation_generator = train_datagen.flow(
 x=train_X,
y=train_Y,
batch_size=validation_batch_size,
shuffle=False,
sample_weight=None,
seed=42,
save_to_dir=None,
save_prefix='',
save_format='png',
subset='validation')
 test_datagen = ImageDataGenerator(rescale=rescale)
 test_generator = test_datagen.flow(
 x=test_X,
y=None,
batch_size=test_batch_size,
shuffle=False,
sample_weight=None,
seed=42,
save_to_dir=None,
save_prefix='',
save_format='png')
 class_weights = get_weight(np.argmax(train_Y, axis=1))
steps_per_epoch = len(train_generator)
 validation_steps = len(validation_generator)
 print("
Preprocessing and Data Batch Generation Completed.
 ")
 return train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps
def get_weight(y):
class_weight_current = cw.compute_class_weight('balanced', np.unique(y), y)
 return class_weight_current
train_dataloader = load_array(( train_features, train_labels), batch_size)
 with torch.no_grad() :
for X, y in train_dataloader:
print(X.shape)
 print(X, "
")
 print(len(y))
print(y, "
")
 y_hat = net(X)
 print(y_hat)
 print(accuracy(y_hat, y))
break<train_model>
def get_model(model_name, input_shape=(96, 96, 3), num_class=2, weights='imagenet', dense_units=1024, internet=False):
inputs = Input(input_shape)
 if model_name == "Xception":
base_model = Xception(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNet50":
base_model = ResNet50(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNet101":
base_model = keras.applications.resnet.ResNet101(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNet152":
base_model = keras.applications.resnet.ResNet152(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNet50V2":
base_model = resnet_v2.ResNet50V2(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNet101V2":
base_model = resnet_v2.ResNet101V2(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNet152V2":
base_model = resnet_v2.ResNet152V2(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNeXt50":
base_model = resnext.ResNeXt50(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "ResNeXt101":
base_model = resnext.ResNeXt101(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "InceptionV3":
base_model = InceptionV3(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "InceptionResNetV2":
base_model = InceptionResNetV2(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "DenseNet201":
base_model = DenseNet201(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "NASNetMobile":
base_model = NASNetMobile(include_top=False, weights=weights, input_shape=input_shape)
 elif model_name == "NASNetLarge":
base_model = NASNetLarge(include_top=False, weights=weights, input_shape=input_shape)
 x = base_model.output
x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = GlobalAveragePooling2D()(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(dense_units )(x)
 x = BatchNormalization()(x)
 x = Activation(activation='relu' )(x)
 x = Dropout(0.5 )(x)
 if num_class>1:
outputs = Dense(num_class, activation="softmax" )(x)
 else:
outputs = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=base_model.input, outputs=outputs)
 model.summary()
return model
def get_conv_model(num_class=2, input_shape=None, dense_units=256):
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(dense_units, activation = "relu"))
model.add(Dropout(0.5))
if num_class>1:
model.add(Dense(num_class, activation='softmax'))
else:
model.add(Dense(num_class, activation='sigmoid'))
print(model.summary())
 return model
def train(net, train_features, train_labels, test_features, test_labels, num_epochs, learning_rate, weight_decay,batch_size):
train_ls, test_ls = [], []
net = net.to(device)
 net.apply(init_weights)
 train_acc = []
optimizer = torch.optim.Adam(net.parameters() , lr = learning_rate, weight_decay=weight_decay)
 train_dataloader = load_array(( train_features, train_labels),batch_size)
 for epoch in range(num_epochs):
curr_acc = 0
numer = 0
for X, y in train_dataloader:
X = X.to(device)
 y = y.to(device)
 y_hat = net(X)
 l = loss(y_hat, y.unsqueeze(1))
curr_acc += accuracy(y_hat, y)
 optimizer.zero_grad()
l.backward()
optimizer.step()
numer += len(y)
 train_ls.append(log_rmse(net, train_features, train_labels))
train_acc.append(curr_acc/ numer)
 if test_labels is not None:
test_ls.append(log_rmse(net, test_features, test_labels))
print(f'for epoch {epoch} rmse: {train_ls[-1]}, accuracy : {train_acc[-1]}')
 return train_ls, test_ls 
<install_modules>
print("Settting Callbacks")
 def step_decay(epoch, lr):
lrate = lr
if epoch==2:
lrate = 0.0001
return lrate
checkpoint = ModelCheckpoint(
 model_file,
monitor='val_acc',
save_best_only=True)
 early_stopping = EarlyStopping(
 monitor='val_loss',
patience=10,
verbose=1,
restore_best_weights=True)
 reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.6,
patience=2,
min_lr=0.0000001,
verbose=1)
 learning_rate_scheduler = LearningRateScheduler(step_decay, verbose=1)
 callbacks = [reduce_lr, early_stopping]
print("Set Callbacks at ", date_time(1))
!pip install -U d2l 
<train_model>
print("Getting Base Model", date_time(1))
dim = 28
input_shape =(dim, dim, 1)
 num_class = len(set(train_df["target"].values))
weights = 'imagenet'
dense_units = 256
model = get_conv_model(num_class=num_class, input_shape=input_shape, dense_units=dense_units)
 print("Loaded Base Model", date_time(1))
train_and_pred(train_features, test_features, train_labels, test_data,
num_epochs, lr, weight_decay, batch_size)
<choose_model_class>
loss = 'categorical_crossentropy'
metrics = ['accuracy']
def get_net() :
net = nn.Sequential(nn.Linear(in_features, 256), nn.ReLU() , nn.Linear(256,out_features))
return net
net = get_net()
net<train_model>
train_X = train_df.drop(columns=["target"] ).values
train_X = train_X.reshape(train_X.shape[0], dim, dim,1)
 train_Y = train_df["target"].values
train_Y = keras.utils.to_categorical(train_Y, 10)
 test_X = test_df.values
test_X = test_X.reshape(test_X.shape[0], dim, dim,1 )
train_and_pred(train_features, test_features, train_labels, test_data,
num_epochs, lr, weight_decay, batch_size )<choose_model_class>
batch_size = 128
train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(train_X=train_X, train_Y=train_Y, test_X=test_X, batch_size=batch_size)
def get_net() :
net = nn.Sequential(nn.Linear(in_features, 256), nn.ReLU() ,nn.Linear(256,64), nn.ReLU() , nn.Linear(64,out_features))
return net
net = get_net()
net<train_model>
print("Starting Trainning...
 ")
 start_time = time.time()
print(date_time(1))
print("
Compliling Model...
 ")
 learning_rate = 0.001
optimizer = Adam(learning_rate)
 model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
 steps_per_epoch = len(train_generator)
 validation_steps = len(validation_generator)
 verbose = 1
epochs = 100
print("Trainning Model...
 ")
 history = model.fit_generator(
 train_generator,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=verbose,
callbacks=callbacks,
validation_data=validation_generator,
validation_steps=validation_steps,
class_weight=class_weights)
 elapsed_time = time.time() - start_time
elapsed_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print("
Elapsed Time: " + elapsed_time)
 print("Completed Model Trainning", date_time(1))
train_and_pred(train_features, test_features, train_labels, test_data,
num_epochs, lr, weight_decay, batch_size )<set_options>
ypreds = model.predict_generator(generator=test_generator, steps = len(test_generator), verbose=1)
%matplotlib inline 
<load_from_csv>
ypred = np.argmax(ypreds, axis=1 )
train_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )<drop_column>
sample_df = pd.read_csv(input_directory + "sample_submission.csv")
 sample_df.head()
train_data = train_data.drop(train_data[(train_data['OverallQual'] > 9)&(train_data['SalePrice'] < 220000)].index)
 train_data = train_data.drop(train_data[(train_data['GrLivArea'] > 4000)&(train_data['SalePrice'] < 300000)].index)
 train_data = train_data.drop(train_data[(train_data['GarageArea'] > 1200)&(train_data['SalePrice'] < 300000)].index )<drop_column>
test_gen_id = test_generator.index_array
sample_submission_id = sample_df["ImageId"]
len(test_gen_id), len(sample_submission_id )
Target = 'SalePrice'
train_data.dropna(axis=0, subset=[Target], inplace=True)
 all_data = pd.concat([train_data, test_data],keys=['train','test'])
 print('У тренировочного датасета {} рядов и {} признаков'.format(train_data.shape[0], train_data.shape[1]))
print('У тестового датасета {} рядов и {} признаков'.format(test_data.shape[0], test_data.shape[1]))
print('Объединённый датасет содержит в себе {} рядов и {} признаков'.format(all_data.shape[0], all_data.shape[1]))<drop_column>
sample_list = list(sample_df.ImageId)
 ids = [x+1 for x in test_generator.index_array]
pred_dict = dict(( key, value)for(key, value)in zip(ids, ypred))
pred_list_new = [pred_dict[f] for f in sample_list]
test_df = pd.DataFrame({'ImageId': sample_list,'Label': pred_list_new})
 test_df.to_csv('submission.csv', header=True, index=False )