Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<train_model>
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )
learn.fit_one_cycle(10, lr_max=0.33 )<train_model>
n_comp_GENES = 450
n_comp_CELLS = 2
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
QT_n_quantile_min=50,
QT_n_quantile_max=1000,
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )
learn.fit_one_cycle(50, lr_max=0.091 )<train_model>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drugs = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
learn.fit_one_cycle(80, lr_max=5e-2 )<train_model>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
learn.fit_one_cycle(300, lr_max=1e-2 )<categorify>
to_tst = to.new(test_df)
 to_tst.process()
to_tst.all_cols.head()<train_model>
def calc_QT_par_kurt(QT_n_quantile_min=10, QT_n_quantile_max=200):
a = np.array([[1.53655,1], [30.4929,1]])
 b = np.array([QT_n_quantile_min, QT_n_quantile_max])
 return np.linalg.solve(a, b )
tst_dl = dls.valid.new(to_tst)
 tst_dl.show_batch()<predict_on_test>
def n_quantile_for_kurt(kurt, calc_QT_par_kurt_transform):
return int(calc_QT_par_kurt_transform[0]*kurt + calc_QT_par_kurt_transform[1] )
learn.metrics = []
tst_preds,_ = learn.get_preds(dl=tst_dl )<create_dataframe>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
res1 = np.expm1(tst_preds)
 res2 = list(map(lambda x: x[0], res1.numpy()))
res3 = list(map(lambda x: x[1], res1.numpy()))
submit = pd.DataFrame({'ConfirmedCases': res2, 'Fatalities': res3})
 submit.index = test_df.ForecastId<concatenate>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
submit = pd.concat([submit, outlier_df], sort=True ).sort_index()<merge>
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
test = submit.merge(test_ori[['ForecastId', 'Date', 'Country_Region', 'Province_State']], on='ForecastId', how='left')
 test.index = test_ori.ForecastId
submit.ConfirmedCases = test.groupby(['Country_Region', 'Province_State'] ).ConfirmedCases.cummax()
submit.Fatalities = test.groupby(['Country_Region', 'Province_State'] ).Fatalities.cummax()<save_to_csv>
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
submit.to_csv('submission.csv')
 submit<define_variables>
train_features = train_features.drop(CELLS+GENES , axis=1 )
min_date = test_df.Date.min()
max_date = train_df.Date.max()
f, axes = plt.subplots(10, 1, figsize=(16, 60))
def plot_preds(country, ax):
subset = test_hlp[(test_hlp['Country_Region'] == country)]
idx = subset.index
dates = subset.Date
predicted = submit.iloc[idx].ConfirmedCases
predicted.index = dates
combined = pd.DataFrame({'pred': predicted})
 sns.lineplot(data=combined, ax=axes[ax] ).set_title(country)
 plot_preds('Italy', 0)
 plot_preds('Spain', 1)
 plot_preds('Germany', 2)
 plot_preds('Poland', 3)
 plot_preds('Czechia', 4)
 plot_preds('Russia', 5)
 plot_preds('Iran', 6)
 plot_preds('Sweden', 7)
 plot_preds('Japan', 8)
 plot_preds('Belgium', 9 )<filter>
test_features = test_features.drop(CELLS+GENES , axis=1 )
submit.iloc[test_hlp[test_hlp['Country_Region'] == 'Poland'].index]<import_modules>
data = train_features.append(test_features)
 data
import numpy as np
import pandas as pd
import datetime as datetime
from sklearn import preprocessing<set_options>
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
register_matplotlib_converters() 
<import_modules>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor<load_from_csv>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
training_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 training_set.shape<load_from_csv>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
test_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test_set.shape<filter>
target_non_scored_cols = train_targets_nonscored.drop('sig_id', axis=1 ).columns.values.tolist()
N_train=len(training_set[training_set['Country_Region']==my_country])
 N_train<filter>
train_non_scored = train.drop(target_cols , axis = 1 ).copy()
N_test=len(test_set[test_set['Country_Region']==my_country])
 N_test<prepare_output>
train_non_scored = train_non_scored.merge(train_targets_nonscored , how = 'inner' )
df_test = test_set.copy()
df_test['DsType']='test'
df_test['ConfirmedCases']=0
df_test['Fatalities']=0
df_test.info()<concatenate>
target_non_scored = train_non_scored[['sig_id']+target_non_scored_cols]
df_union=pd.concat([df_train,df_test],sort=False ).copy()
df_union.fillna('ND', inplace = True )<feature_engineering>
folds_non_scored = train_non_scored.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds_non_scored.loc[v_idx, 'kfold'] = int(f)
 folds_non_scored['kfold'] = folds_non_scored['kfold'].astype(int)
 folds_non_scored
df_union['Month']=df_union['Date'].apply(lambda s : int(s.replace('-','')[4:6]))
df_union['Day']=df_union['Date'].apply(lambda s : int(s.replace('-','')[6:9]))
df_union['Date']=df_union['Date'].apply(lambda s : datetime.datetime.strptime(s, '%Y-%m-%d'))
df_union['DateOrd']=df_union['Date'].apply(lambda s : s.toordinal())
 df_union['Province_Norm']=df_union['Country_Region']+'-'+df_union['Province_State']
df_union.tail()<categorify>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
le1 = preprocessing.LabelEncoder()
le1.fit(df_union['Country_Region'])
 df_union['Country'] = le1.transform(df_union['Country_Region'] )<categorify>
le2 = preprocessing.LabelEncoder()
le2.fit(df_union['Province_Norm'])
 df_union['Province'] = le2.transform(df_union['Province_Norm'])
 df_union.drop('Province_Norm',axis=1,inplace=True )<define_variables>
t_max_tr = max(df_union['Date'][df_union['DsType']=='train'])
 t_min_te = min(df_union['Date'][df_union['DsType']=='test'])
 Noverlap=(t_max_tr-t_min_te ).days+1
Noverlap<concatenate>
def cut_neg(y):
m = 0
for i in range(0,len(y)) :
if y[i]<m : y[i]=m
return y<find_best_model_class>
def polynomial_trend(deg,X_train,y_true,X_test):
pf = PolynomialFeatures(degree=deg)
 pr = pf.fit_transform(X_train)
 lrm = LinearRegression()
lrm.fit(pr, y_true)
 y_valid = lrm.predict(pf.fit_transform(X_train))
y_pred = lrm.predict(pf.fit_transform(X_test))
y_valid = cut_neg(y_valid)
 y_pred = cut_neg(y_pred)
 return y_valid,y_pred<compute_test_metric>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
Nlim = 5
delta_max = 0.1
def calc_trends(X_train,y_true,X_test,deg):
pct_hist = 100*sum(1 for x in y_true if x > 0)/N_train
y=np.array(y_true ).flatten()
y_last = y[len(y)-1]
y_check = y[len(y)-1-Nlim]
if(y_check==0):
den = 1
else :
den = y_check
diff = np.abs(y_last-y_check)/den
lim = np.mean(y_true[len(y_true)-Nlim:len(y_true)])
 if(diff<=delta_max):
y_valid = np.full([len(y_true)],lim)
 y_pred = np.full([len(X_test)],lim)
 msg1 = 'cost'
msg2 = 'cost'
if(diff>delta_max):
y_valid,y_pred = polynomial_trend(deg,X_train,y_true,X_test)
 msg1 = 'poly'
msg2 = 'poly'
return y_valid, y_pred, msg1, msg2<feature_engineering>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
df_train = df_union[df_union['DsType']=='train'].drop('DsType',axis=1)
 df_test = df_union[df_union['DsType']=='test'].drop('DsType',axis=1)
 df_train['ConfirmedCasesValid'] = 0
df_train['FatalitiesValid'] = 0
df_train['ConfirmedCasesTrend'] = 0
df_train['FatalitiesTrend'] = 0
df_train['ConfirmedCasesResid'] = 0
df_train['FatalitiesResid'] = 0
df_test['ConfirmedCases'] = 0
df_test['Fatalities'] = 0
df_test['ConfirmedCasesTrend'] = 0
df_test['FatalitiesTrend'] = 0
df_test['ConfirmedCasesResid'] = 0
df_test['FatalitiesResid'] = 0<data_type_conversions>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
y1_train = df_train['ConfirmedCases'].astype(float)
 y2_train = df_train['Fatalities'].astype(float )<filter>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def isPos(x):
if(x>0):
return 1
else :
return 0<feature_engineering>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
df_train['ConfirmedCasesPos']=df_train['ConfirmedCases'].apply(isPos)
 df_train['FatalitiesPos']=df_train['Fatalities'].apply(isPos)
 df_test['ConfirmedCasesPos']=df_test['ConfirmedCases'].apply(isPos)
 df_test['FatalitiesPos']=df_test['Fatalities'].apply(isPos )<groupby>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 num_non_scored_targets=len(target_non_scored_cols)
 hidden_size=1500
sel_cols = ['Country_Region','Province_State','ConfirmedCasesPos','FatalitiesPos']
df_tr_cnt = df_train[sel_cols].groupby(by=['Country_Region','Province_State'] ).sum()
df_te_cnt = df_test[sel_cols].groupby(by=['Country_Region','Province_State'] ).sum()<filter>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
sel_cols = ['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']
df_tr_last = df_train[df_train['Date']==max(df_train['Date'])][sel_cols]
df_te_last = df_test[df_test['Date']==max(df_test['Date'])][sel_cols]<merge>
def run_training(fold, seed , pretrain):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 if pretrain:
model = Model(
 num_features=num_features,
num_targets=num_non_scored_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_non_scored.pth"))
model.batch_norm3 = nn.BatchNorm1d(hidden_size)
 model.dropout3 = nn.Dropout(0.5)
 model.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
else:
model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 if pretrain:
scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-3, epochs=EPOCHS, steps_per_epoch=len(trainloader))
epochs = 25
else:
scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
epochs = copy.copy(EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
df_tr_last = df_tr_last.merge(df_tr_cnt,on=['Country_Region','Province_State'])
 df_te_last = df_te_last.merge(df_te_cnt,on=['Country_Region','Province_State'] )<merge>
train_x = process_data(folds_non_scored )
df_last = df_tr_last.merge(df_te_last,on=['Country_Region','Province_State'])
 df_last['DeltaConfirmedCases']=100*(df_last['ConfirmedCases_y']-df_last['ConfirmedCases_x'])/df_last['ConfirmedCases_x']
df_last['DeltaFatalities']=100*(df_last['Fatalities_y']-df_last['Fatalities_x'])/df_last['Fatalities_x']
df_last['DeltaConfirmedCases']=df_last['DeltaConfirmedCases'].apply(np.abs)
 df_last['DeltaFatalities']=df_last['DeltaFatalities'].apply(np.abs)
 df_last.fillna(0,inplace=True)
 df_last = df_last.replace([np.inf, -np.inf], 0 )<sort_values>
val_idx = train_x[train_x['kfold'] == 0].index
n_head = 5
d_lim1 = 120
d_lim2 = 200
sel_var ='ConfirmedCases'
sel_cols = ['Country_Region','Province_State',sel_var+'_x','Delta'+sel_var,sel_var+'Pos_x']
condition =(df_last['Delta'+sel_var]>d_lim1)&(df_last['Delta'+sel_var]<d_lim2)
 df_last[condition][sel_cols].sort_values(by=['Delta'+sel_var], ascending=True ).head(n_head )<sort_values>
def run_training_non_scored(fold, seed):
seed_everything(seed)
 train = process_data(folds_non_scored)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_non_scored_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_non_scored_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_non_scored_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-4, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target_non_scored.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(10):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_non_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_non_scored_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_non_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target_non_scored.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
n_head = 5
d_lim1 = 0
d_lim2 = 10
sel_var = 'Fatalities'
sel_cols = ['Country_Region','Province_State',sel_var+'_x','Delta'+sel_var,sel_var+'Pos_x']
condition =(df_last['Delta'+sel_var]>d_lim1)&(df_last['Delta'+sel_var]<d_lim2)
 df_last[condition][sel_cols].sort_values(by=['Delta'+sel_var], ascending=True ).head(n_head )<prepare_x_and_y>
def run_k_fold_none_scored(NFOLDS, seed):
oof = np.zeros(( len(train_non_scored), len(target_non_scored_cols)))
 predictions = np.zeros(( len(test), len(target_non_scored_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training_non_scored(fold, seed)
 print(oof_.shape)
 print(oof.shape)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def plotExample() :
train_cond =(( df_train['Country_Region']==my_country)&(df_train['Province_State']==my_province))
test_cond =(( df_test['Country_Region']==my_country)&(df_test['Province_State']==my_province))
x_train_plt = df_train[train_cond]['Date']
y_train_plt = df_train[train_cond][my_variable]
y_valid_plt = df_train[train_cond][my_variable+'Valid']
y_trend_plt = df_train[train_cond][my_variable+'Trend']
y_resid_plt = df_train[train_cond][my_variable+'Resid']
x_test_plt = df_test[test_cond]['Date']
y_test_plt = df_test[test_cond][my_variable]
plt.rcParams["figure.figsize"] =(12,6)
 fig, ax = plt.subplots()
ax.plot(x_train_plt,y_train_plt,'o',color='orange', label='y_true')
 ax.plot(x_train_plt,y_valid_plt,'x',color='gray' , label ='y_valid')
 ax.plot(x_train_plt,y_trend_plt,'.',color='lightblue', label='y_trend')
 ax.plot(x_train_plt,y_resid_plt,'-',color='green', label='y_resid')
 ax.plot(x_test_plt,y_test_plt,'*',color='red', label='y_test')
 ax.set_xticks([])
 ax.legend()<filter>
def run_k_fold(NFOLDS, seed , pretrain=False):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed, pretrain)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
sel_cols = ['Country_Region','Province_State','ConfirmedCases_x','ConfirmedCases_y','DeltaConfirmedCases','ConfirmedCasesPos_x']
df_last[sel_cols][(( df_last['Country_Region']==my_country)&(df_last['Province_State']==my_province)) ]<filter>
for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed, pretrain=True)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
sel_cols = ['Country_Region','Province_State','ConfirmedCases_x','ConfirmedCases_y','DeltaConfirmedCases','ConfirmedCasesPos_x']
df_last[sel_cols][(( df_last['Country_Region']==my_country)&(df_last['Province_State']==my_province)) ]<filter>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<filter><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )