Kaggle competition: IESB Sul - IGM - Maio 2019
import os
import time
import random
import numpy as np
import pandas as pd
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from pathlib import Path
from PIL import Image, ImageEnhance, ImageOps, ImageDraw
from torchvision import transforms, models
from efficientnet_pytorch import EfficientNet
import torch
from torch import optim
from torch.optim.lr_scheduler import ReduceLROnPlateau
from torch import nn
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import WeightedRandomSampler
from torch import cuda<set_options>
df = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 class AverageMeter() :
def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<categorify>
df = df.append(test )
class MelanomaDataset(Dataset):
def __init__(self, base_dir, info, transform=None, phase='train'):
self.base_dir = base_dir
self.info = info
self.transform = transform
self.phase = phase
def __len__(self):
return len(self.info)
 def __getitem__(self, index):
p = Path(self.base_dir, self.info.loc[index, 'image_name'] + '.jpg')
 img = Image.open(p)
 img_transformed = self.transform(img, self.phase)
 return {
'inputs': img_transformed,
'labels': torch.tensor(self.info.loc[index, 'target'], dtype=torch.int64)
 }<choose_model_class>
df['area'] = df['area'].str.replace(',','')
 df['densidade_dem'] = df['densidade_dem'].str.replace(',','')
 df['area'] = df['area'].astype(float)
 df['densidade_dem'] = df['densidade_dem'].astype(float )
def load_model(model, out_features):
net = None
if model.startswith('efficientnet'):
net = EfficientNet.from_pretrained(model, num_classes=out_features)
 for name, param in net.named_parameters() :
if '_fc' in name:
param.requires_grad = True
else:
param.requires_grad = False
elif model == 'vgg19':
net = models.vgg19(pretrained=True)
 net.classifier[6] = nn.Linear(in_features=net.classifier[6].weight.size(1), out_features=out_features)
 for name, param in net.named_parameters() :
if name in ['classifier.6.weight', 'classifier.6.bias']:
param.requires_grad = True
else:
param.requires_grad = False
elif model == 'resnext':
net = models.resnext101_32x8d(pretrained=True)
 net.fc = nn.Linear(in_features=net.fc.weight.size(1), out_features=out_features)
 for name, param in net.named_parameters() :
if 'fc' in name:
param.requires_grad = True
else:
param.requires_grad = False
elif model == 'densenet':
net = models.densenet161(pretrained=True)
 net.classifier = nn.Linear(in_features=net.classifier.weight.size(1), out_features=out_features)
 for name, param in net.named_parameters() :
if 'classifier' in name:
param.requires_grad = True
else:
param.requires_grad = False
return net<load_pretrained>
df['populacao'] = df['populacao'].str.replace(',','')
 df['populacao'] = df['populacao'].str.replace('(2)','')
 df['populacao'] = df['populacao'].str.replace('(','')
 df['populacao'] = df['populacao'].str.replace(')','')
 df['populacao'] = df['populacao'].str.replace('.','')
 df['populacao'] = df['populacao'].astype(int )
def eval_model(model, dataset, batch_size, criterion, num_workers, device=None):
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
 losses = AverageMeter()
preds = []
targets = []
with torch.no_grad() :
for data in dataloader:
inputs = data['inputs'].to(device)
 labels = data['labels'].to(device)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 losses.update(loss.item() , inputs.size(0))
preds.append(outputs[:, 1])
 targets.append(labels)
 preds = torch.cat(preds, dim=-1 ).cpu().numpy()
targets = torch.cat(targets, dim=-1 ).cpu().numpy()
auc = roc_auc_score(targets, preds)
 return losses.avg, auc
def train_model(
 model_id,
dataset_train,
dataset_valid,
batch_size,
model,
criterion,
optimizer,
scheduler,
num_epochs,
freezed_epochs,
base_dir,
num_workers,
sampler,
device,
early_stopping 
):
model.to(device)
 if sampler != None:
dataloader_train = DataLoader(dataset_train, batch_size=batch_size, sampler=sampler, num_workers=num_workers)
 else:
dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
 for epoch in range(1, num_epochs + 1):
losses = AverageMeter()
s_time = time.time()
if epoch == freezed_epochs + 1:
model.load_state_dict(torch.load(f'./state_dict_{model_id}.pt', map_location=device))
for param in model.parameters() :
param.requires_grad = True
for g in optimizer.param_groups:
g['lr'] = 1e-4
model.train()
for data in dataloader_train:
optimizer.zero_grad()
inputs = data['inputs'].to(device)
 labels = data['labels'].to(device)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
losses.update(loss.item() , inputs.size(0))
model.eval()
loss_valid, auc_valid = eval_model(model, dataset_valid, batch_size, criterion, num_workers, device)
 if epoch == 1 or auc_valid > max_auc:
saved = True
max_auc = auc_valid
torch.save(model.state_dict() , f'./state_dict_{model_id}.pt')
 counter = 0
else:
saved = False
counter += 1
e_time = time.time()
print(f'epoch: {epoch}, loss_train: {losses.avg:.4f}, loss_valid: {loss_valid:.4f}, auc_valid: {auc_valid:.4f}, saved: {saved}, {(e_time - s_time):.4f}sec')
 if epoch < freezed_epochs + 1:
counter = 0
else:
scheduler.step(auc_valid)
 if early_stopping != None:
if counter == early_stopping:
break
def predict(dataset, batch_size, model, device=None):
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
 model.to(device)
 model.eval()
preds = []
with torch.no_grad() :
for data in dataloader:
inputs = data['inputs'].to(device)
 outputs = model(inputs)
 preds.append(outputs)
 preds = torch.cat(preds)
 return preds<set_options>
df['comissionados_por_servidor'] = df['comissionados_por_servidor'].str.replace('
df['comissionados_por_servidor'] = df['comissionados_por_servidor'].str.replace('!','')
 df['comissionados_por_servidor'] = df['comissionados_por_servidor'].str.replace('%','')
 df['comissionados_por_servidor'] = df['comissionados_por_servidor'].astype(float)
 df['comissionados_por_servidor'] = df['comissionados_por_servidor']/100
seed_everything(123)
 torch.backends.cudnn.benchmark = True
config = {
'INPUT_DIR' : '.. /input/jpeg-melanoma-256x256',
'MODEL' : 'efficientnet-b0',
'SIZE' : 256,
'BATCH_SIZE' : 128,
'NUM_FOLDS' : 5,
'NUM_EPOCHS' : 20,
'FREEZED_EPOCHS' : 5,
'LEARNING_RATE' : 1e-3,
'EARLY_STOPPING' : 3,
'UNIFORM_AUGMENT': True,
'TTA' : 5,
'NUM_WORKERS' : 4,
'DEVICE' : 'cuda'
}<load_from_csv>
df_porte = pd.get_dummies(df['porte'])
 df_regiao = pd.get_dummies(df['regiao'] )
train = pd.read_csv(Path(config['INPUT_DIR'], 'train.csv'))
test = pd.read_csv(Path(config['INPUT_DIR'], 'test.csv'))
sub = pd.read_csv(Path(config['INPUT_DIR'], 'sample_submission.csv'))
transform = ImageTransform(config['SIZE'], config['UNIFORM_AUGMENT'] )<create_dataframe>
df = pd.concat([df, df_porte], axis=1)
 df = pd.concat([df, df_regiao], axis=1 )
prediction = pd.DataFrame()
skf = StratifiedKFold(n_splits=config['NUM_FOLDS'], shuffle=True)
 for i,(tr_idx, va_idx)in enumerate(skf.split(train, train['target'])) :
tr = train.loc[tr_idx, :]
va = train.loc[va_idx, :]
tr.reset_index(drop=True, inplace=True)
 va.reset_index(drop=True, inplace=True)
 dataset_train = MelanomaDataset(Path(config['INPUT_DIR'], 'train'), tr, transform=transform, phase='train')
 dataset_valid = MelanomaDataset(Path(config['INPUT_DIR'], 'train'), va, transform=transform, phase='valid')
 net = load_model(config['MODEL'], 2)
 criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters() , lr=config['LEARNING_RATE'])
 scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=2, factor=0.2)
 class_sample_count = np.array([len(np.where(tr['target'] == t)[0])for t in np.unique(tr['target'])])
 weight = 1./ class_sample_count
samples_weight = np.array([weight[t] for t in tr['target']])
 sampler = WeightedRandomSampler(samples_weight, len(samples_weight))
print(f"---- fold: {i + 1} ------------")
 train_model(
 f"{config['MODEL']}_{i + 1}",
dataset_train,
dataset_valid,
config['BATCH_SIZE'],
net,
criterion,
optimizer,
scheduler,
config['NUM_EPOCHS'],
config['FREEZED_EPOCHS'],
config['INPUT_DIR'],
config['NUM_WORKERS'],
sampler,
config['DEVICE'],
config['EARLY_STOPPING'] 
)
 test['target'] = 0
dataset_test = MelanomaDataset(Path(config['INPUT_DIR'], 'test'), test, transform=transform, phase='test')
 for _ in range(config['TTA']):
pred_test = predict(dataset_test, config['BATCH_SIZE'], net, device=config['DEVICE'])
 pred_test = pd.DataFrame(torch.softmax(pred_test, 1)[:, 1].cpu().numpy())
 prediction = pd.concat([prediction, pred_test], axis=1)
 sub['target'] = prediction.mean(axis=1)
 sub.to_csv('./submission.csv', index=False )<set_options>
import matplotlib.pyplot as plt
import seaborn as sns
pd.set_option('display.max_columns',500)
 pd.set_option('display.max_rows',100 )<load_from_csv>
test = df[df['nota_mat'].isnull() ]
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' )<filter>
df = df[~df['nota_mat'].isnull() ]
sales = sales[sales['item_price'] < 250000]
sales = sales[sales['item_cnt_day'] < 1000]<feature_engineering>
train, valid = train_test_split(df, random_state=42 )
sales.loc[sales.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
sales.loc[sales.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
sales.loc[sales.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
sales.loc[sales.shop_id == 39, 'shop_id'] = 40
test.loc[test.shop_id == 39, 'shop_id'] = 40<feature_engineering>
removed_cols = ['municipio', 'estado', 'codigo_mun', 'porte', 'regiao', 'nota_mat','capital']
feats = [c for c in df.columns if c not in removed_cols]
sales['date'] = pd.to_datetime(sales['date'], format='%d.%m.%Y')
 sales['month'] = sales['date'].dt.month
sales['year'] = sales['date'].dt.year
sales = sales.drop('date', axis = 1 )<feature_engineering>
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
to_append = test[['shop_id', 'item_id']].copy()
to_append['date_block_num'] = sales['date_block_num'].max() + 1
to_append['year'] = 2015
to_append['month'] = 11
to_append['item_cnt_day'] = 0
to_append['item_price'] = 0
sales = pd.concat([sales, to_append], ignore_index=True, sort=False )<categorify>
models = {'RandomForest': RandomForestClassifier(random_state=42, min_samples_leaf=8,
min_samples_split=3, n_jobs=-1, n_estimators=75),
'ExtraTrees': ExtraTreesClassifier(random_state=42, n_jobs=-1),
'GradientBoosting': GradientBoostingClassifier(random_state=42),
'DecisionTree': DecisionTreeClassifier(random_state=42),
'AdaBoost': AdaBoostClassifier(random_state=42),
'KNM 11': KNeighborsClassifier(n_neighbors=11, n_jobs=-1),}
holiday_dict = {
1: 7,
2: 4,
3: 3,
4: 9,
5: 4,
6: 4,
7: 3,
8: 9,
9: 5,
10: 9,
11: 6,
12: 5,
}
sales['holidays_in_month'] = sales['month'].map(holiday_dict )<feature_engineering>
from sklearn.metrics import accuracy_score
period = sales[['date_block_num','year','month','holidays_in_month']].drop_duplicates().reset_index(drop=True)
 period['days'] = period.apply(lambda r: monthrange(r.year, r.month)[1], axis=1 )<categorify>
def run_model(models, train, valid, feats, y_name):
model.fit(train[feats], train[y_name])
 preds = model.predict(valid[feats])
 return accuracy_score(valid[y_name], preds )
shops['shop_city'] = cities
shops.loc[shops['shop_city'] == '!Якутск', 'shop_city' ] = 'Якутск'
ls = LabelEncoder()
shops['shop_city'] = ls.fit_transform(shops['shop_city'])
 shops['shop_type'] = shops['shop_name'].apply(lambda x: 'МТРЦ' if 'МТРЦ' in x
else 'ТРЦ' if 'ТРЦ' in x
else 'ТРК' if 'ТРК' in x
else 'ТЦ' if 'ТЦ' in x
else 'ТК' if 'ТК' in x
else 'NO_DATA')
 ls1 = LabelEncoder()
shops['shop_type'] = ls1.fit_transform(shops['shop_type'] )<categorify>
scores = []
for name, model in models.items() :
score = run_model(model, train.fillna(-1), valid.fillna(-1), feats, 'nota_mat')
 scores.append(score)
 print(name, ':', score )
ls = LabelEncoder()
item_cats['main_cat'] = ls.fit_transform(main_cat)
 sub_cat = item_cats['item_category_name'].str.split('-' ).map(lambda row : row[1].strip()
if len(row)> 1
else row[0].strip())
 item_cats['sub_cat'] = ls.fit_transform(sub_cat )<categorify>
train = train.append(valid )
items['name_1'], items['name_2'] = items['item_name'].str.split('[', 1 ).str
items['name_1'], items['name_3'] = items['item_name'].str.split('(', 1 ).str
items['name_2'] = items['name_2'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items['name_3'] = items['name_3'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items.drop('name_1', axis = 1 , inplace=True)
 items = items.fillna('0')
 ls = LabelEncoder()
items['name_2'] = ls.fit_transform(items['name_2'])
 items['name_3'] = ls.fit_transform(items['name_3'] )<feature_engineering>
rf = RandomForestClassifier(random_state=42, min_samples_leaf=8, min_samples_split=3, n_jobs=-1, n_estimators=200 )
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
items['item_name'] = items['item_name'].apply(lambda x: name_correction(x))
items.head()
print('Unique item names after correction:', len(items['item_name'].unique()))<merge>
rf.fit(train[feats].fillna(-1), train['nota_mat'] )
items = items.merge(item_cats.drop('item_category_name', axis=1),
on = ['item_category_id'], how = 'left')
 items = items[['item_name','item_id','item_category_id','main_cat','sub_cat','name_2','name_3']]<data_type_conversions>
test['nota_mat'] = rf.predict(test[feats].fillna(-1))
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype in ["float64"]]
int_cols = [c for c in df if df[c].dtype in ["int64"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 return df
def position_colum(df,column_name):

 assert column_name in df, "Column {} not in the dataset".format(column_name)
 colums = df.columns
for i in range(len(colums)) :
if(column_name == colums[i]):
position = i
return(position)
 def mean_encoding_reg(df, col_to_enc,target_col, nb_split):

 col = [col_to_enc,target_col]
df_small = df[col]
target_mean_fold_enc = []
kf = KFold(n_splits=nb_split, shuffle=False)
 for ind_tr, ind_val in kf.split(df_small):
mat_tr, mat_val = df_small.iloc[ind_tr], df_small.iloc[ind_val]
target_tr_mean_estimate = mat_tr.groupby(col_to_enc)[target_col].mean()
target_val_mean = df_small.loc[ind_val,col_to_enc].map(target_tr_mean_estimate)
 for mean in target_val_mean :
target_mean_fold_enc.append(mean)
 pos_categorical_feat = position_colum(df,col_to_enc)
 new_df = df.copy()
new_df.insert(pos_categorical_feat,col_to_enc +'_mean_enc',pd.Series(target_mean_fold_enc ).fillna(0))
return(( downcast_dtypes(new_df)) )<merge>
test[['codigo_mun','nota_mat']].to_csv('randomforest.csv', index=False )