Kaggle competition: Machine Learning Lab - CAS Data Science FS 20
<SOS> metric: mape Kaggle data source: machine-learning-lab-cas-data-science-fs-20<choose_model_class>
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer, make_column_transformer
from sklearn.pipeline import Pipeline
model0 = torchvision.models.resnet152(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet152_0.pt')
 num_ftrs = model0.fc.in_features
model0.fc = nn.Linear(num_ftrs,5)
 model0.load_state_dict(state)
 model0 = model0.to(device)
 model1 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet02.pt')
 num_ftrs = model1.fc.in_features
model1.fc = nn.Linear(num_ftrs,5)
 model1.load_state_dict(state)
 model1 = model1.to(device)
 model2 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet01.pt')
 num_ftrs = model2.fc.in_features
model2.fc = nn.Linear(num_ftrs,5)
 model2.load_state_dict(state)
 model2 = model2.to(device)
 model3 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet00.pt')
 num_ftrs = model3.fc.in_features
model3.fc = nn.Linear(num_ftrs,5)
 model3.load_state_dict(state)
 model3 = model3.to(device)
 model4 = torchvision.models.resnet101(pretrained=False)
 state = torch.load('.. /input/resnet/FinalResnet0.pt')
 num_ftrs = model4.fc.in_features
model4.fc = nn.Linear(num_ftrs,5)
 model4.load_state_dict(state)
 model4 = model4.to(device)
<predict_on_test>
town_data = pd.read_csv('/kaggle/input/town-data/town_data.csv', header=[1])
 town_data = town_data.replace('*', np.nan)
 town_data = town_data.replace('X', np.nan )
def compute_predictions(model, model_type, data_loader, device):
if model_type == 'train':
predictions = []
correct_pred, num_examples = 0, 0
tqdm_notebook()
for i,(inputs, labels)in enumerate(tqdm_notebook(data_loader)) :
inputs = inputs.to(device)
 labels = labels.to(device)
 outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 predictions.append(preds)
 num_examples += labels.size(0)
 correct_pred +=(preds==labels ).sum()
return predictions, correct_pred.item() /num_examples*100
else:
predictions = []
img_ids = []
out = []
tqdm_notebook()
for i,(inputs, img_id)in enumerate(tqdm_notebook(data_loader)) :
inputs = inputs.to(device)
 outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 predictions.extend(preds)
 img_ids.extend(img_id)
 out.extend(outputs)
 predictions = [pred.item() for pred in predictions]
final_predictions = pd.DataFrame(np.array([img_ids, predictions])).transpose()
final_predictions.columns = ['id_code', 'diagnosis']
final_predictions_df = final_predictions.copy()
return final_predictions_df, out, img_ids<predict_on_test>
train_data = pd.read_csv('/kaggle/input/machine-learning-lab-cas-data-science-fs-20/houses_train.csv', index_col=0 )
with torch.set_grad_enabled(False):
model0.eval()
model1.eval()
model2.eval()
model3.eval()
model4.eval()
print('Computing Test Predictions')
 test_predictions0,out0,ids0 = compute_predictions(model0, 'test', test_loader, device)
 test_predictions1,out1,ids1 = compute_predictions(model1, 'test', test_loader, device)
 test_predictions2,out2,ids2 = compute_predictions(model2, 'test', test_loader, device)
 test_predictions3,out3,ids3 = compute_predictions(model3, 'test', test_loader, device)
 test_predictions4,out4,ids4 = compute_predictions(model4, 'test', test_loader, device)
<save_to_csv>
train_data = train_data.join(town_data.set_index('Gemeindename'), on='municipality_name' )
out =(torch.stack(out0)+torch.stack(out1)+torch.stack(out2)+torch.stack(out3)+torch.stack(out4)) /5
img_ids = np.array(ids0)
 _, predictions = torch.max(out, 1)
 predictions = predictions.cpu().numpy()
final_predictions = pd.DataFrame(np.array([img_ids, predictions])).transpose()
final_predictions.columns = ['id_code', 'diagnosis']
final_predictions_df = final_predictions.copy()
final_predictions.to_csv("submission.csv",index=False )<import_modules>
train_data = train_data.fillna(-1 )
%matplotlib inline
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
<prepare_x_and_y>
X_data = train_data.drop(columns='price')
 y_data = train_data['price']
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png'
)<load_pretrained>
X_train, X_dev, y_train, y_dev = train_test_split(X_data, y_data, stratify=X_data['object_type_name'], test_size=0.1 )
model=load_model('.. /input/model211/val_model2_11.h5' )<choose_model_class>
pipeline = Pipeline([ 
('pre', make_column_transformer(( OneHotEncoder(handle_unknown='ignore'), ['zipcode', 'municipality_name', 'object_type_name']), remainder='passthrough')) , 
('clf', LGBMRegressor(
 num_leaves=50,
learning_rate=0.05,
n_estimators=200 
))
] )
for layer in model.layers[:]:
print(layer )<predict_on_test>
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test 
<save_to_csv>
parameters = {
'clf__num_leaves': [10,20,30,40,50,60,70,80,90,100],
'clf__max_depth': [3,4,5,6,7,8,9,10],
'clf__n_estimators': [100,150,200,500]
}
clf = RandomizedSearchCV(pipeline, parameters, cv=5, n_iter=40, verbose=1, n_jobs=-1 )
with open("submission.csv", "w")as f:
f.write(test_df.to_csv(index=False))<count_values>
clf.fit(X_train, y_train )
test_df['diagnosis'].value_counts()<data_type_conversions>
est = clf.best_estimator_
data = np.asarray(x_test[1], dtype="int32")
 inputImg = np.expand_dims(data,0 ).astype(np.float32 )<split>
y_dev_pred = est.predict(X_dev )
input_details = mobilenetV2F_interpreter.get_input_details()
mobilenetV2F_interpreter.set_tensor(input_details[0]['index'], inputImg)
 mobilenetV2F_interpreter.invoke()
output_details = mobilenetV2F_interpreter.get_output_details()
output_data = mobilenetV2F_interpreter.get_tensor(output_details[0]['index'] )<set_options>
def mean_absolute_percentage_error(y_true, y_pred):
return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 100
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<set_options>
mean_absolute_percentage_error(y_dev, y_dev_pred )
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = 1.0 /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)
 else:
p_data_fp32.add_(-step_size * group['lr'], exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class PlainRAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super(PlainRAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(PlainRAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class AdamW(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, warmup = 0):
defaults = dict(lr=lr, betas=betas, eps=eps,
weight_decay=weight_decay, warmup = warmup)
 super(AdamW, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(AdamW, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
if group['warmup'] > state['step']:
scheduled_lr = 1e-8 + state['step'] * group['lr'] / group['warmup']
else:
scheduled_lr = group['lr']
step_size = group['lr'] * math.sqrt(bias_correction2)/ bias_correction1
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32)
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 p.data.copy_(p_data_fp32)
 return loss<set_options>
est.fit(X_data, y_data )
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )<set_options>
X_test = pd.read_csv('/kaggle/input/machine-learning-lab-cas-data-science-fs-20/houses_test.csv', index_col=0 )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )<feature_engineering>
X_test = X_test.join(town_data.set_index('Gemeindename'), on='municipality_name')
 X_test = X_test.fillna(-1 )
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(5 )<count_values>
y_test_pred = est.predict(X_test )
df['diagnosis'].value_counts()<train_model>
X_test_submission = pd.DataFrame(index=X_test.index )
im = Image.open(df['path'][1])
 width, height = im.size
print(width,height)
 im.show()
plt.imshow(np.asarray(im))<define_variables>
X_test_submission['price'] = y_test_pred
<feature_engineering><EOS> 
X_test_submission.to_csv('lightgbm_submission_all.csv', header=True, index_label='id' )