Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
print("pandas version: {}".format(pd.__version__))
print("NumPy version: {}".format(np.__version__))
print("matplotlib version: {}".format(matplotlib.__version__))
print("seaborn version: {}".format(sns.__version__))
print("scikit-learn version: {}".format(sklearn.__version__))
print("statsmodels version: {}".format(statsmodels.__version__))
print("SciPy version: {}".format(sp.__version__))
import numpy as np
import pandas as pd
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import Linear, LayerNorm, ReLU, Dropout
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm
import os
import copy
from sklearn.cluster import KMeans
from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold
from torch.utils.data import Dataset,TensorDataset, DataLoader,RandomSampler
import time,datetime 
<feature_engineering>
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import seaborn as sns
def Get_nowtime(fmat='%Y-%m-%d %H:%M:%S'):
return datetime.datetime.strftime(datetime.datetime.now() ,fmat)
 def Metric(target,pred):
metric = 0
for i in range(target.shape[-1]):
metric +=(np.sqrt(np.mean(( target[:,:,i]-pred[:,:,i])**2)) /target.shape[-1])
 return metric
def Write_log(logFile,text,isPrint=True):
if isPrint:
print(text)
 logFile.write(text)
 logFile.write('
')
 return None
def Seed_everything(seed=1017):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
Seed_everything()
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv')!= -1 or classname.find('Linear')!= -1:
print(classname)
 try:
nn.init.xavier_uniform_(m.weight)
 except:
pass
try:
nn.init.zeros_(m.bias)
 except:
pass
if classname.find('GRU')!= -1 or classname.find('LSTM')!= -1:
print(classname)
 for name, param in m.named_parameters() :
print(name)
 if 'bias_ih' in name:
torch.nn.init.zeros_(param)
 elif 'bias_hh' in name:
torch.nn.init.zeros_(param)
 elif 'weight_ih' in name:
nn.init.xavier_uniform_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param )<compute_test_metric>
data = pd.read_csv('.. /input/titanic/train.csv')
 data_val = pd.read_csv('.. /input/titanic/test.csv')
 data1 = data.copy(deep = True)
 data_cleaner = [data1, data_val]
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
def mcrmse(y_actual, y_pred, weight=None, num_scored=5):
score = 0
for i in range(5):
if weight is not None:
score += torch.sqrt(torch.mean(( y_actual[:,:,i]-y_pred[:,:,i])**2*weight)) / num_scored
else:
score += torch.sqrt(torch.mean(( y_actual[:,:,i]-y_pred[:,:,i])**2)) / num_scored
return score
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
<load_from_disk>
print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True)
 def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test)
 class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
num_target=5
self.cate_emb = nn.Embedding(14,100)
 self.gru = nn.GRU(100*3+3, 256, num_layers=1, batch_first=True, dropout=0.5, bidirectional=True)
 self.gru1 = nn.GRU(512, 256, num_layers=1, batch_first=True, dropout=0.5, bidirectional=True)
 self.predict = nn.Linear(512,num_target)
 def forward(self, cateX,contX):
cate_x = self.cate_emb(cateX ).view(cateX.shape[0],cateX.shape[1],-1)
 sequence = torch.cat([cate_x,contX],-1)
 x, h = self.gru(sequence)
 x, h = self.gru1(x)
 x = F.dropout(x,0.5,training=self.training)
 predict = self.predict(x)
 return predict
def train_and_predict(type = 0, FOLD_N = 5):
gkf = GroupKFold(n_splits=FOLD_N)
 device = torch.device('cuda:%s'%0 if torch.cuda.is_available() else 'cpu')
 log = open('./train.log','w',1)
 all_best_model = []
oof = []
for fold,(train_index, valid_index)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
Write_log(log,'fold %s train start:%s'%(fold,Get_nowtime()))
t_train = train.iloc[train_index]
train_x = preprocess_inputs(t_train)
 train_cate_x = torch.LongTensor(train_x[:,:,:3])
 train_cont_x = torch.Tensor(train_x[:,:,3:])
 train_y = torch.Tensor(np.array(t_train[pred_cols].values.tolist() ).transpose(( 0, 2, 1)))
 w_train = torch.Tensor(np.log(t_train['signal_to_noise'].values.reshape(-1,1)+1.1)/2)
 t_valid = train.iloc[valid_index]
t_valid = t_valid[t_valid['SN_filter'] == 1]
valid_x = preprocess_inputs(t_valid)
 valid_count = valid_x.shape[0]
valid_cate_x = torch.LongTensor(valid_x[:,:,:3])
 valid_cont_x = torch.Tensor(valid_x[:,:,3:])
 valid_y = torch.Tensor(np.array(t_valid[pred_cols].values.tolist() ).transpose(( 0, 2, 1)))
 train_data = TensorDataset(train_cate_x,train_cont_x,train_y,w_train)
 train_data_loader = DataLoader(dataset=train_data,shuffle=True,batch_size=64,num_workers=1)
 valid_data = TensorDataset(valid_cate_x,valid_cont_x,valid_y)
 valid_data_loader = DataLoader(dataset=valid_data,shuffle=False,batch_size=32,num_workers=1)
 valid_y = valid_y.numpy()
model = Net()
model.apply(weights_init)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 all_valid_metric = []
all_epoch_valid_metric = []
not_improve_epochs = 0
best_valid_metric = 1e9
for epoch in range(60):
running_loss = 0.0
t0 = datetime.datetime.now()
model.train()
for n,data in enumerate(train_data_loader):
cate_x,cont_x,y,weight = [x.to(device)for x in data]
outputs = model(cate_x,cont_x)
 optimizer.zero_grad()
loss = mcrmse(y,outputs[:,:68,:],weight)
 loss.backward()
optimizer.step()
running_loss += loss.item()
running_loss = running_loss / n
valid_loss = 0.0
all_pred = []
model.eval()
for data in valid_data_loader:
cate_x,cont_x,y = [x.to(device)for x in data]
outputs = model(cate_x,cont_x)
 all_pred.append(outputs.detach().cpu().numpy())
 loss = mcrmse(y,outputs[:,:68,:])
 valid_loss +=(loss.item() * cate_x.shape[0])
 valid_loss = valid_loss / valid_count
all_pred = np.concatenate(all_pred,0)
 valid_metric = Metric(valid_y[:,:,[0,1,3]],all_pred[:,:68,[0,1,3]])
 all_epoch_valid_metric.append(valid_metric)
 t1 = datetime.datetime.now()
Write_log(log,'epoch %s | train mean loss:%.6f | valid loss:%.6f | valid metric:%.6f | ⏰:%ss'%(str(epoch ).rjust(3),running_loss,valid_loss,valid_metric,(t1-t0 ).seconds))
if valid_metric < best_valid_metric:
Write_log(log,'[epoch %s] save better model'%(epoch))
torch.save(model.state_dict() ,'./gru-cate-emb-100-fold-%s.cpkt'%(fold))
best_valid_metric = valid_metric
best_model = copy.deepcopy(model.state_dict())
 not_improve_epochs = 0
else:
not_improve_epochs += 1
Write_log(log,'Not improve epoch +1 ---> %s'%not_improve_epochs)
 all_best_model.append(best_model)
 model.load_state_dict(best_model)
 model.eval()
all_id = []
all_y_id = []
for i,row in t_valid.iterrows() :
for j in range(row['seq_length']):
all_id.append(row['id']+'_%s'%j)
 for k in range(len(row['reactivity'])) :
all_y_id.append(row['id']+'_%s'%k)
 all_id = np.array(all_id ).reshape(-1,1)
 all_y_id = np.array(all_y_id ).reshape(-1,1)
 all_pred = []
for data in valid_data_loader:
cate_x,cont_x,y = [x.to(device)for x in data]
outputs = model(cate_x,cont_x)
 all_pred.append(outputs.detach().cpu().numpy())
 all_pred = np.concatenate(all_pred,0)
 t_valid_metric = Metric(valid_y[:,:,[0,1,3]],all_pred[:,:68,[0,1,3]])
 t_oof = pd.DataFrame(all_pred.reshape(-1,5),columns=['reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C'])
 t_oof['id_seqpos'] = all_id
t_target_df = pd.DataFrame(valid_y.reshape(-1,5),columns=['label_reactivity','label_deg_Mg_pH10','label_deg_pH10','label_deg_Mg_50C','label_deg_50C'])
 t_target_df['id_seqpos'] = all_y_id
t_oof = t_oof.merge(t_target_df,how='left',on='id_seqpos')
 all_valid_metric.append(t_valid_metric)
 Write_log(log,'fold %s valid metric:%.6f'%(fold,t_valid_metric))
oof.append(t_oof)
 oof = pd.concat(oof)
 oof_metirc = 0
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
oof_metirc +=(np.sqrt(np.mean(( oof.loc[~oof['label_%s'%col].isna() ,'label_%s'%col].values-oof.loc[~oof['label_%s'%col].isna() ,col].values)**2)) / 3.0)
 log.close()
os.rename('./train.log','gru-cate-emb-100-0.6%f-%.6f.log'%(np.mean(all_valid_metric),oof_metirc))
def Pred(df):
test_x = preprocess_inputs(df)
 test_cate_x = torch.LongTensor(test_x[:,:,:3])
 test_cont_x = torch.Tensor(test_x[:,:,3:])
 test_data = TensorDataset(test_cate_x,test_cont_x)
 test_data_loader = DataLoader(dataset=test_data,shuffle=False,batch_size=64,num_workers=1)
 all_id = []
for i,row in df.iterrows() :
for j in range(row['seq_length']):
all_id.append(row['id']+'_%s'%j)
 all_id = np.array(all_id ).reshape(-1,1)
 all_pred = np.zeros(len(all_id)*5 ).reshape(len(all_id),5)
 for fold in range(FOLD_N):
model.load_state_dict(all_best_model[fold])
 model.eval()
t_all_pred = []
for data in test_data_loader:
cate_x,cont_x = [x.to(device)for x in data]
outputs = model(cate_x,cont_x)
 t_all_pred.append(outputs.detach().cpu().numpy())
 t_all_pred = np.concatenate(t_all_pred,0)
 all_pred += t_all_pred.reshape(-1,5)
 all_pred /= FOLD_N
sub = pd.DataFrame(all_pred,columns=['reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C'])
 sub['id_seqpos'] = all_id
return sub
public_sub = Pred(test.loc[test['seq_length']==107])
 private_sub = Pred(test.loc[test['seq_length']==130])
 sub = pd.concat([public_sub,private_sub] ).reset_index(drop=True)
 return oof[['id_seqpos']+['reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C']+['label_reactivity','label_deg_Mg_pH10','label_deg_pH10','label_deg_Mg_50C','label_deg_50C']],\
sub[['id_seqpos']+['reactivity','deg_Mg_pH10','deg_pH10','deg_Mg_50C','deg_50C']] 
<train_model>
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 dataset.drop('Cabin', axis=1, inplace=True )
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_ 
<save_to_csv>
print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
oof,sub = train_and_predict()
oof.to_csv('./oof.csv',index=False)
 sub.to_csv('./submission.csv',index=False )<set_options>
varlist = ['Sex']
def binary_map(x):
return x.map({'male': 1, "female": 0})
 for dataset in data_cleaner:
dataset[varlist] = dataset[varlist].apply(binary_map )
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
def allocate_gpu_memory(gpu_number=0):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
 if physical_devices:
try:
print("Found {} GPU(s)".format(len(physical_devices)))
 tf.config.set_visible_devices(physical_devices[gpu_number], 'GPU')
 tf.config.experimental.set_memory_growth(physical_devices[gpu_number], True)
 print("
except RuntimeError as e:
print(e)
 else:
print("Not enough GPU hardware devices available")
 allocate_gpu_memory()
Ver='GRU_LSTM1'
aug_data = '.. /input/openvaccine-augmentation-data/aug_data1.csv'
debug = False<choose_model_class>
dummy1 = pd.get_dummies(data1['Embarked'], prefix='Embarked', drop_first=True)
 data1 = pd.concat([data1, dummy1], axis=1 )
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def build_model(seq_len=107, pred_len=68, dropout=0.5, embed_dim=100, hidden_dim=256, type=0):
inputs = L.Input(shape=(seq_len, 6))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:]
embed = L.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 if type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 return model<categorify>
dummy1 = pd.get_dummies(data_val['Embarked'], prefix='Embarked', drop_first=True)
 data_val = pd.concat([data_val, dummy1], axis=1 )
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(pred_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score<load_from_disk>
dummy1 = pd.get_dummies(data1['Pclass'], prefix='Pclass', drop_first=True)
 data1 = pd.concat([data1, dummy1], axis=1 )
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True )<load_pretrained>
dummy1 = pd.get_dummies(data_val['Pclass'], prefix='Pclass', drop_first=True)
 data_val = pd.concat([data_val, dummy1], axis=1 )
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test )<train_model>
dummy1 = pd.get_dummies(data1['Sex'], prefix='Male', drop_first=True)
 data1 = pd.concat([data1, dummy1], axis=1 )
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_<load_from_csv>
dummy1 = pd.get_dummies(data_val['Sex'], prefix='Male', drop_first=True)
 data_val = pd.concat([data_val, dummy1], axis=1 )
aug_df = pd.read_csv(aug_data)
 display(aug_df.head() )<merge>
data1['FamilySize'] = data1['SibSp'] + data1['Parch'] + 1
data1.head(2 )
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )<split>
data_val['FamilySize'] = data_val['SibSp'] + data_val['Parch'] + 1
data_val.head(2 )
if debug:
train = train[:200]
test = test[:200]<split>
data1= data1.rename(columns={ 'Male_1' : 'Male'})
 data_val= data_val.rename(columns={ 'Male_1' : 'Male'} )
def train_and_predict(type = 0, FOLD_N = 5):
gkf = GroupKFold(n_splits=FOLD_N)
 public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 holdouts = []
holdout_preds = []
for cv,(tr_idx, vl_idx)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
trn = train.iloc[tr_idx]
x_trn = preprocess_inputs(trn)
 y_trn = np.array(trn[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(trn.signal_to_noise+1.1)/2
val = train.iloc[vl_idx]
x_val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
x_val = preprocess_inputs(val)
 y_val = np.array(val[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
model = build_model(type=type)
 model_short = build_model(seq_len=107, pred_len=107,type=type)
 model_long = build_model(seq_len=130, pred_len=130,type=type)
 history = model.fit(
 x_trn, y_trn,
validation_data =(x_val, y_val),
batch_size=64,
epochs=60,
sample_weight=w_trn,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model{Ver}_cv{cv}.h5')
 ] 
)
 fig = px.line(
 history.history, y=['loss', 'val_loss'],
labels={'index': 'epoch', 'value': 'Mean Squared Error'},
title='Training History')
 fig.show()
model.load_weights(f'model{Ver}_cv{cv}.h5')
 model_short.load_weights(f'model{Ver}_cv{cv}.h5')
 model_long.load_weights(f'model{Ver}_cv{cv}.h5')
 holdouts.append(train.iloc[vl_idx])
 holdout_preds.append(model.predict(x_val_all))
if cv == 0:
public_preds = model_short.predict(public_inputs)/FOLD_N
private_preds = model_long.predict(private_inputs)/FOLD_N
else:
public_preds += model_short.predict(public_inputs)/FOLD_N
private_preds += model_long.predict(private_inputs)/FOLD_N
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds<predict_on_test>
drop_column = ['PassengerId', 'Pclass', 'Name', 'Sex', 'Ticket', 'Fare', 'Embarked']
data1.drop(drop_column, axis=1, inplace = True )
val_df, val_preds, test_df, test_preds = [], [], [], []
if debug:
nmodel = 1
else:
nmodel = 4
for i in range(nmodel):
holdouts, holdout_preds, public_df, public_preds, private_df, private_preds = train_and_predict(i)
 val_df += holdouts
val_preds += holdout_preds
test_df.append(public_df)
 test_df.append(private_df)
 test_preds.append(public_preds)
 test_preds.append(private_preds )<prepare_output>
from sklearn.model_selection import train_test_split
preds_ls = []
for df, preds in zip(test_df, test_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
preds_ls = []
for df, preds in zip(val_df, val_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds_ls.append(single_df)
 holdouts_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()<save_to_csv>
from sklearn.model_selection import train_test_split
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )<load_from_disk>
X = data1.drop(['Survived'], axis=1)
 X.head()
def print_mse(prd):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(prd, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))<compute_test_metric>
y = data1['Survived']
y.head()
print_mse(holdouts_df )<compute_test_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7, test_size=0.3, random_state=100 )
print_mse(holdouts_df[holdouts_df.SN_filter == 1] )<import_modules>
survived =(sum(data1['Survived'])/len(data1['Survived'].index)) *100
survived
import pandas as pd
import numpy as np<import_modules>
logreg = LogisticRegression()
import pandas as pd
import numpy as np<load_from_csv>
rfe = RFE(logreg, 15)
 rfe = rfe.fit(X_train, y_train )
df1 = pd.read_csv('/kaggle/input/gru-lstm-mix-with-custom-loss-tunning/ensemble_final.csv')
 df2 = pd.read_csv('/kaggle/input/mvan-covid-mrna-vaccine-analysis-notebook-268/submission.csv')
 df3 = pd.read_csv('/kaggle/input/gru-lstm-mix-with-custom-loss/ensemble_final.csv' )<prepare_output>
rfe.support_
sub = df1
sub['reactivity'] =(df1.reactivity.values + df2.reactivity.values + df3.reactivity.values)/3
sub['deg_Mg_pH10'] =(df1.deg_Mg_pH10.values + df2.deg_Mg_pH10.values + df3.deg_Mg_pH10.values)/3
sub['deg_pH10'] =(df1.deg_pH10.values + df2.deg_pH10.values + df3.deg_pH10.values)/3
sub['deg_Mg_50C'] =(df1.deg_Mg_50C.values + df2.deg_Mg_50C.values +df3.deg_Mg_50C.values)/3
sub['deg_50C'] =(df1.deg_50C.values + df2.deg_50C.values + df3.deg_50C.values)/3<save_to_csv>
list(zip(X_train.columns, rfe.support_, rfe.ranking_))
sub.to_csv('submission.csv', index = False )<set_options>
import statsmodels.api as sm
warnings.filterwarnings('ignore')
<load_from_csv>
X_train_sm = sm.add_constant(X_train[col])
 logm1 = sm.GLM(y_train,X_train_sm, family = sm.families.Binomial())
 res = logm1.fit()
res.summary()
FOLDS = 5
EPOCHS = 130
BATCH_SIZE = 64
LR = 0.001
VERBOSE = 2
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
 train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines = True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines = True)
 sample_sub = pd.read_csv('.. /input/stanford-covid-vaccine/sample_submission.csv')
 target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
token2int = {x: i for i, x in enumerate('().ACGUBEHIMSX')}
def preprocess_inputs(df, cols = ['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols]\ 
.applymap(lambda seq: [token2int[x] for x in seq])\ 
.values\ 
.tolist() 
), 
(0, 2, 1)
)
 train_inputs = preprocess_inputs(train)
 train_labels = np.array(train[target_cols].values.tolist() ).transpose(0, 2, 1)
 public_test_df = test[test['seq_length'] == 107]
private_test_df = test[test['seq_length'] == 130]
public_test = preprocess_inputs(public_test_df)
 private_test = preprocess_inputs(private_test_df )<compute_train_metric>
y_train_pred = res.predict(X_train_sm)
 y_train_pred[:10]
def CMCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1)
 def build_model(seq_len = 107, pred_len = 68, embed_dim = 85, dropout = 0.10):
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2 ** i for i in range(n)]
x = tf.keras.layers.Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = tf.keras.layers.Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = tf.keras.layers.Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = tf.keras.layers.Multiply()([tanh_out, sigm_out])
 x = tf.keras.layers.Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = tf.keras.layers.Add()([res_x, x])
 return res_x
inputs = tf.keras.layers.Input(shape =(seq_len, 3))
embed = tf.keras.layers.Embedding(input_dim = len(token2int), output_dim = embed_dim )(inputs)
 reshaped = tf.reshape(embed, shape =(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.SpatialDropout1D(dropout )(reshaped)
 x = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(128,
dropout = dropout,
return_sequences = True,
kernel_initializer = 'orthogonal'))(reshaped)
 x = wave_block(x, 16, 3, 12)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(dropout )(x)
 x = wave_block(x, 32, 3, 8)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(dropout )(x)
 x = wave_block(x, 64, 3, 4)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(dropout )(x)
 x = wave_block(x, 128, 3, 1)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(dropout )(x)
 truncated = x[:, :pred_len]
out = tf.keras.layers.Dense(5, activation = 'linear' )(truncated)
 model = tf.keras.models.Model(inputs = inputs, outputs = out)
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer = opt,
loss = tf.keras.losses.MeanSquaredLogarithmicError() ,
metrics = [tf.keras.metrics.RootMeanSquaredError() ])
 return model
def mcrmse(y_true, y_pred):
y_true_ = y_true.reshape(163200, 5)
 y_pred_ = y_pred.reshape(163200, 5)
 y_true0 = y_true_[:, 0]
y_true1 = y_true_[:, 1]
y_true2 = y_true_[:, 2]
y_true3 = y_true_[:, 3]
y_true4 = y_true_[:, 4]
y_pred0 = y_pred_[:, 0]
y_pred1 = y_pred_[:, 1]
y_pred2 = y_pred_[:, 2]
y_pred3 = y_pred_[:, 3]
y_pred4 = y_pred_[:, 4]
rmse0 = math.sqrt(metrics.mean_squared_error(y_true0, y_pred0))
rmse1 = math.sqrt(metrics.mean_squared_error(y_true1, y_pred1))
rmse2 = math.sqrt(metrics.mean_squared_error(y_true2, y_pred2))
rmse3 = math.sqrt(metrics.mean_squared_error(y_true3, y_pred3))
rmse4 = math.sqrt(metrics.mean_squared_error(y_true4, y_pred4))
return np.mean([rmse0, rmse1, rmse2, rmse3, rmse4])
 def train_and_evaluate(train_inputs, train_labels, public_test, private_test):
oof_preds = np.zeros(( train_inputs.shape[0], 68, 5))
public_preds = np.zeros(( public_test.shape[0], 107, 5))
private_preds = np.zeros(( private_test.shape[0], 130, 5))
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for fold,(train_index, val_index)in enumerate(kfold.split(train_inputs)) :
print(f'Training fold {fold + 1}')
 checkpoint = tf.keras.callbacks.ModelCheckpoint(f'fold_{fold + 1}.h5',
monitor = 'val_loss',
save_best_only = True,
save_weights_only = True 
)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
mode = 'min',
factor = 0.5,
patience = 5,
verbose = 1,
min_delta = 0.00001 
)
 x_train, x_val = train_inputs[train_index], train_inputs[val_index]
y_train, y_val = train_labels[train_index], train_labels[val_index]
K.clear_session()
model = build_model()
history = model.fit(x_train, y_train,
validation_data =(x_val, y_val),
batch_size = BATCH_SIZE,
epochs = EPOCHS,
callbacks = [checkpoint, cb_lr_schedule],
verbose = VERBOSE)
 model.load_weights(f'fold_{fold + 1}.h5')
 oof_preds[val_index] = model.predict(x_val)
 short = build_model(seq_len = 107, pred_len = 107)
 short.load_weights(f'fold_{fold + 1}.h5')
 public_preds += short.predict(public_test)/ FOLDS
long = build_model(seq_len = 130, pred_len = 130)
 long.load_weights(f'fold_{fold + 1}.h5')
 private_preds += long.predict(private_test)/ FOLDS
print('-'*50)
 print('
')
 mean_col_rmse = mcrmse(train_labels, oof_preds)
 print(f'Our out of folds mean columnwise root mean squared error is {mean_col_rmse}')
 return public_preds, private_preds<compute_test_metric>
y_train_pred = y_train_pred.values.reshape(-1)
 y_train_pred[:10]
public_preds, private_preds = train_and_evaluate(train_inputs, train_labels, public_test, private_test )<prepare_output>
y_train_pred_final = pd.DataFrame({'Survived':y_train.values, 'Survived_Prob':y_train_pred})
 y_train_pred_final['PassengerId'] = y_train.index
y_train_pred_final.head()
def inference_format(public_test_df, public_preds, private_test_df, private_preds, target_cols):
predictions = []
for test, preds in [(public_test_df, public_preds),(private_test_df, private_preds)]:
for index, uid in enumerate(test['id']):
single_pred = preds[index]
single_df = pd.DataFrame(single_pred, columns = target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
predictions.append(single_df)
 predictions = pd.concat(predictions)
 return predictions
predictions = inference_format(public_test_df, public_preds, private_test_df, private_preds, target_cols)
 submission = sample_sub[['id_seqpos']].merge(predictions, on = ['id_seqpos'])
 submission.to_csv('submission.csv', index = False)
 print('Submission saved')
 submission.head()<import_modules>
y_train_pred_final['predicted'] = y_train_pred_final.Survived_Prob.map(lambda x: 1 if x > 0.5 else 0)
 y_train_pred_final.head()
!pip install /kaggle/input/timm-package/timm-0.1.26-py3-none-any.whl
PKGPATH='.. /input/kprostate111/prostatev111'
sys.path.append(PKGPATH)
 os.environ['KMP_DUPLICATE_LIB_OK']='True'
warnings.filterwarnings("ignore" )<import_modules>
from sklearn import metrics
from utils.imageutils import rotatecrop, pairrot, cropcoords, noisecrop, Mish
from utils.imageutils import padsplit, padimg, pixctr, cropper, condenseImgls
from utils.imageutils import padsplitv2, chunk_squaresv2, condenseImglsv2, cropperv2
from utils.imageutils import chunk_squares, frame_combine, balancedSampler
from utils.utilities import get_logger, dumpobj, loadobj, Identity
from utils.utilities import SpatialDropout, qwk5<set_options>
confusion = metrics.confusion_matrix(y_train_pred_final.Survived, y_train_pred_final.predicted)
 print(confusion )
logger = get_logger('Sequence model :', 'INFO')
 device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 try:
logger.info('Device : {}'.format(torch.cuda.get_device_name(0)))
 except:
logger.info('Device : Not available')
 logger.info('Cuda available : {}'.format(torch.cuda.is_available()))
n_gpu = torch.cuda.device_count()
logger.info('Cuda n_gpus : {}'.format(n_gpu))<load_from_csv>
print(metrics.accuracy_score(y_train_pred_final.Survived, y_train_pred_final.predicted))
NPIXELS=20*10**6
NSAMP=300
DATATYPE = 'test'
INPATH = '.. /input/prostate-cancer-grade-assessment'
SAMPLE = f'{INPATH}/sample_submission.csv'
DATA = f'{INPATH}/{DATATYPE}_images'
subdf = pd.read_csv(f'{INPATH}/sample_submission.csv')
 tstdf = pd.read_csv(f'{INPATH}/{DATATYPE}.csv')
 if DATATYPE=='train':
folds= pd.read_csv(f'{INPATH}/.. /train-clust/train_clust.csv.gzip',
usecols = ['image_id', 'fold'],
compression = 'gzip')
 removedf = pd.read_csv('.. /input/remove/removedf.csv')
 alldf = pd.read_csv(f'{INPATH}/{DATATYPE}.csv')
 alldf = alldf[~alldf.image_id.isin(removedf.imgname)]
alldf = pd.merge(alldf, folds, on='image_id')
 tstdf = alldf.query('fold == 0' ).reset_index(drop=True)[:NSAMP]
print(tstdf.isup_grade.value_counts())
 print(tstdf.shape )<define_variables>
from statsmodels.stats.outliers_influence import variance_inflation_factor
class options:
max_len = 36
tilesize = 224
data_path = f'{DATATYPE}_images'
bsize = 2<categorify>
vif = pd.DataFrame()
vif['Features'] = X_train[col].columns
vif['VIF'] = [variance_inflation_factor(X_train[col].values, i)for i in range(X_train[col].shape[1])]
vif['VIF'] = round(vif['VIF'], 2)
 vif = vif.sort_values(by = "VIF", ascending = False)
 vif
def cropload(imname, thresh = 255):
img = MultiImage(imname)[1]
krows = np.where(np.min(img, 0)< 255)[0]
kcols = np.where(np.min(img, 1)< 255)[0]
img = img[kcols[0]: kcols[-1] + 1, krows[0]: krows[-1] + 1]
return img
val_transforms = A.Compose([
A.RandomCrop(224*6, 224*6, always_apply=True, p=1),
A.NoOp() ,
])
 def get_tiles(img, mode, n_tiles , tile_size, cutoff = 230, tile_cutoff = 0.0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 slice_size = 2
img3 = img2.reshape(\
img2.shape[0] // tile_size, \
tile_size , \
img2.shape[1]* slice_size // tile_size, \
tile_size // slice_size, \
3 \ 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size//slice_size,3)
 bb, hh, ww, cc = img3.shape
idx =(img3[:,:,:,1].reshape(bb,-1)< cutoff ).sum(1)>(hh*ww)*tile_cutoff
img3 = img3[idx]
img3 = np.pad(img3,[[0, slice_size -(img3.shape[0] % slice_size)],[0,0],[0,0],[0,0]], constant_values=255)
 img3 = np.concatenate(img3, 1)\ 
.reshape(tile_size, img3.shape[0]//slice_size, tile_size, 3)\ 
.transpose(1,0,2,3 ).reshape(-1, tile_size, tile_size,3)
 if len(img3)< n_tiles:
padder = n_tiles-len(img3)
 img3 = np.pad(img3,[[padder//2,padder-(padder//2)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = sorted(np.argsort(( img3.reshape(img3.shape[0],-1)< cutoff ).sum(-1)) [-n_tiles:])
 img3 = img3[idxs]
return img3, 0
def chunks(lst, n):

 for i in range(0, len(lst), n):
yield lst[i:i + n]
tst_transforms = A.Compose([
A.Transpose(p=0.5),
A.VerticalFlip(p=0.5),
A.HorizontalFlip(p=0.5),
])
 class ProstateDataset(Dataset):
def __init__(self, df, path, transform,
tile_mode=0, modtype='train'):
self.path = path
self.data_path = options.data_path
self.df = df
logger.info(f'Dataset {modtype} {self.df.shape}')
 if modtype!='test':
self.df.gleason_score = self.df.gleason_score.str.replace('negative', '0+0')
 self.gleasonmap = {'0': 0, '3': 1, '4': 2, '5': 3}
self.tile_mode = tile_mode
self.basesz = 224*6
self.transform = transform
self.type = modtype
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
imname = self.df.iloc[idx].image_id
imgpath = f'{INPATH}/{self.data_path}/{imname}.tiff'
mskpath = f'{INPATH}/{self.data_path}/{imname}_mask.png'
imgorig = cropload(imgpath, thresh = 255)
 BASESZ = 224*6
sample = {}
for iii in range(4):
img = imgorig.copy()
for _ in range(iii):
img = np.rot90(img)
 hhh, www, _ = img.shape
thresh = 230
BASE =(img[:,:, 1]<thresh ).sum()
FILLED_RATIO= 0
n_tiles = 1
image_size = 2000
while(( FILLED_RATIO)<0.85)and(image_size>228):
n_tiles += 1
image_size = BASESZ//n_tiles
image_size = int(image_size +(4-image_size%4))
tiles, OK = get_tiles(img, 2*random.randint(0,1), n_tiles**2, image_size, cutoff=thresh)
 FILLED =(tiles[:,:,:, 1]<thresh ).sum()
FILLED_RATIO = FILLED/BASE
if FILLED_RATIO < 0.8:
dsize =(1+FILLED_RATIO)/2
img = cv2.resize(img,(int(www * dsize), int(hhh * dsize)) , \
interpolation=cv2.INTER_CUBIC)
 BASE =(img[:,:, 1]<thresh ).sum()
FILLED_RATIO= 0
n_tiles = 1
image_size = 2000
while(( FILLED_RATIO)<0.85)and(image_size>228):
n_tiles += 1
image_size = BASESZ//n_tiles
image_size = int(image_size +(4-image_size%4))
tiles, OK = get_tiles(img, 2*random.randint(0,1), n_tiles**2, image_size, cutoff=thresh)
 FILLED =(tiles[:,:,:, 1]<thresh ).sum()
FILLED_RATIO = FILLED/BASE
fillls = []
if FILLED_RATIO < 0.9:
tilels = []
for tile_cutoff in [0.0, 0.1, 0.2]:
tilels.append(get_tiles(img, 2*random.randint(0,1), n_tiles**2, image_size, cutoff=thresh, tile_cutoff = tile_cutoff)[0])
 fillls.append(( tilels[-1][:,:,:, 1]<thresh ).sum() /BASE)
 fillls = np.array(fillls ).round(3)
 tiles = tilels[np.argmax(fillls)]
else:
fillls = np.array([FILLED_RATIO])
 logtxt = f'flip {iii} img size {image_size} init filled {FILLED_RATIO:.4f}'
image_size = tiles.shape[1]
n_row_tiles = int(np.sqrt(tiles.shape[0]))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
this_img = tiles[i]
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.uint8)
 images = 255 - images
images = self.transform(image=images)['image']
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 sample[f'image{iii}'] = images
if self.type=='train':
sample['isup_grade'] = torch.zeros(5)
 sample['isup_grade'][: self.df.iloc[idx].isup_grade ] = 1.
 sample['gleason1'], sample['gleason2'] = map(self.gleasonmap.get, \
self.df.gleason_score.iloc[idx].split('+'))
return sample
def collatefn(batch):
batch = [b for b in batch if b is not None]
b = dict(( f'image{i}', torch.stack([torch.tensor(l[f'image{i}'])for l in batch])) for i in range(4))
if 'isup_grade' in batch[0]:
b['isup_grade'] = torch.stack([i['isup_grade'] for i in batch])
 for ii in ['gleason1', 'gleason2']:
b[ii] = torch.tensor([l[ii] for l in batch])
 return b<create_dataframe>
col = col.drop(['Parch','SibSp','FamilySize', 'Embarked_S', 'Embarked_Q'], 1)
 col
trndataargs = {'path': INPATH}
tstdataset = ProstateDataset(tstdf, modtype=DATATYPE, transform = val_transforms, **trndataargs )<data_type_conversions>
X_train_sm = sm.add_constant(X_train[col])
 logm2 = sm.GLM(y_train,X_train_sm, family = sm.families.Binomial())
 res = logm2.fit()
res.summary()
<load_pretrained>
y_train_pred = res.predict(X_train_sm)
 y_train_pred[:10]
loaderls = []
trnloaderargs = {'num_workers' : 4, 'collate_fn' : collatefn}
tstloader = DataLoader(tstdataset, shuffle=False, batch_size=options.bsize, **trnloaderargs )<choose_model_class>
y_train_pred = y_train_pred.values.reshape(-1)
 y_train_pred[:10]
class SeqNet(nn.Module):
def __init__(self, architecture = 'mixnet_m', pretrained=True, \
dense_units = 256, dropout = 0.2, isuplabels = 5, \
nblocks=4, concatfinal = 1,
glabels = 4):
super(SeqNet, self ).__init__()
logger.info('Architecture {} dense {} dropout {}'.format(\
architecture, dense_units, dropout))
self.arch=architecture
if 'eff' in self.arch:
self.bbmod = timm.create_model(architecture, pretrained)
 self.dense_units = dense_units
self.bbmod.classifier = Identity()
self.myctfc = nn.Linear(self.bbmod.num_features, isuplabels)
 def forward(self, x):
emb = self.bbmod(x)
 out_isup = self.myctfc(emb)
 return out_isup 
<load_pretrained>
y_train_pred_final = pd.DataFrame({'Survived':y_train.values, 'Survived_Prob':y_train_pred})
 y_train_pred_final['PassengerId'] = y_train.index
y_train_pred_final.head()
models = []
for cpt_path in sorted(glob.glob('.. /input/mnetlv33/*')) :
logger.info(f'Load {cpt_path}')
 models.append(SeqNet(architecture = 'tf_efficientnet_b0_ns', pretrained=False, nblocks=4, concatfinal = 1,\
dense_units = 256, dropout = 0.0, isuplabels = 5, glabels = 4))
cpt = torch.load(cpt_path, map_location=torch.device(device))
models[-1].load_state_dict(cpt)
 models[-1].to(device);
models[-1].eval() ;
del cpt
gc.collect()<define_variables>
y_train_pred_final['predicted'] = y_train_pred_final.Survived_Prob.map(lambda x: 1 if x > 0.5 else 0)
 y_train_pred_final.head()
ttafns = [
lambda x: x,
lambda x: x.flip(-1),
lambda x: x.flip(-2),
lambda x: x.flip(-1, -2),
lambda x: x.transpose(-1, -2),
lambda x: x.transpose(-1, -2 ).flip(-1),
lambda x: x.transpose(-1, -2 ).flip(-2),
lambda x: x.transpose(-1, -2 ).flip(-1, -2),
]
iters = 4
modlens = len(models)
 if os.path.exists(f'{INPATH}/{DATATYPE}_images/'):
predsls = []
for loader in [tstloader]:
with torch.no_grad() :
ypredval = []
for step, batch in enumerate(loader):
batchls = []
for i in range(iters):
x = batch[f'image{i}'].to(device, dtype=torch.float)
 for midx in range(modlens):
for f in ttafns:
outisup = models[midx](f(x))
outisup = torch.sigmoid(outisup)
 batchls.append(outisup)
 out = sum(batchls)/len(batchls)
 out = out.cpu().sum(1 ).detach().numpy()
ypredval.append(out)
 del x, batch
if step%10==0:
logger.info('Val step {} of {}'.format(step, len(loader)))
 torch.cuda.empty_cache()
gc.collect()
predsls.append(np.concatenate(ypredval, 0))<data_type_conversions>
confusion = metrics.confusion_matrix(y_train_pred_final.Survived, y_train_pred_final.predicted)
 print(confusion )
subdf = tstloader.df[['image_id']]
subdf['isup_grade'] =(sum(predsls)/len(predsls)).round().astype(np.int32 )<save_to_csv>
print(metrics.accuracy_score(y_train_pred_final.Survived, y_train_pred_final.predicted))
subdf.to_csv("submission.csv", index=False)
 subdf.head(20 )<count_values>
from statsmodels.stats.outliers_influence import variance_inflation_factor
if DATATYPE=='train':
logger.info(( subdf.isup_grade == tstloader.df.isup_grade ).value_counts() )<set_options>
vif = pd.DataFrame()
vif['Features'] = X_train[col].columns
vif['VIF'] = [variance_inflation_factor(X_train[col].values, i)for i in range(X_train[col].shape[1])]
vif['VIF'] = round(vif['VIF'], 2)
 vif = vif.sort_values(by = "VIF", ascending = False)
 vif
if DATATYPE=='train':
logger.info(qwk5(subdf.isup_grade, tstloader.df.isup_grade))<import_modules>
fpr, tpr, thresholds = metrics.roc_curve(y_train_pred_final.Survived, y_train_pred_final.Survived_Prob, drop_intermediate = False )
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import tensorflow as tf
from sklearn.model_selection import train_test_split<load_pretrained>
numbers = [float(x)/10 for x in range(10)]
for i in numbers:
y_train_pred_final[i]= y_train_pred_final.Survived_Prob.map(lambda x: 1 if x > i else 0)
 y_train_pred_final.head()
zip_ref = zipfile.ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip', 'r')
 print(zip_ref.namelist())
 embeddings = zip_ref.open('glove.840B.300d/glove.840B.300d.txt', 'r' )<compute_test_metric>
cutoff_df = pd.DataFrame(columns = ['prob','accuracy','sensi','speci'])
 num = [0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
for i in num:
cm1 = metrics.confusion_matrix(y_train_pred_final.Survived, y_train_pred_final[i])
 total1=sum(sum(cm1))
accuracy =(cm1[0,0]+cm1[1,1])/total1
speci = cm1[0,0]/(cm1[0,0]+cm1[0,1])
 sensi = cm1[1,1]/(cm1[1,0]+cm1[1,1])
 cutoff_df.loc[i] =[ i ,accuracy,sensi,speci]
print(cutoff_df )
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.decode().split(" ")) for o in embeddings )<load_from_csv>
y_train_pred_final['final_predicted'] = y_train_pred_final.Survived_Prob.map(lambda x: 1 if x > 0.56 else 0)
 y_train_pred_final.head()
train_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')
 test_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')
 print('训练集维度：
',train_data.shape)
 print('测试集维度：
',test_data.shape)
 train_data.sample(5 )<define_variables>
metrics.accuracy_score(y_train_pred_final.Survived, y_train_pred_final.final_predicted )
train_input = list(train_data['question_text'])
 train_label = list(train_data['target'])
 test_input = list(test_data['question_text'] )<string_transform>
confusion2 = metrics.confusion_matrix(y_train_pred_final.Survived, y_train_pred_final.final_predicted)
 confusion2
stop=stopwords.words('english')
 def remove_stop_words(x):
for word in stop:
token = " " + word + " "
if(x.find(token)!= -1):
x = x.replace(token, " ")
 return x
train_input_rsw = list(map(remove_stop_words, train_input))
test_input_rsw = list(map(remove_stop_words, test_input))<define_variables>
X_test_sm = sm.add_constant(X_test )
max_features=100000
embed_size = 300
max_length = 60<string_transform>
y_test_pred = res.predict(X_test_sm)
 y_test_pred[:10]
tokenizer=Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(train_input_rsw)
 word_index = tokenizer.word_index
n_words=min(max_features,len(word_index))
embedding_matrix = np.zeros(( n_words+1, 300))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<string_transform>
y_pred_1 = pd.DataFrame(y_test_pred)
 y_pred_1.head()
sequences = tokenizer.texts_to_sequences(train_input_rsw)
 train_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')
 print(train_input_padded.shape)
 sequences = tokenizer.texts_to_sequences(test_input_rsw)
 test_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')
 print(test_input_padded.shape )<split>
y_test_df = pd.DataFrame(y_test)
 y_test_df['PassengerId'] = y_test_df.index
y_pred_1.reset_index(drop=True, inplace=True)
 y_test_df.reset_index(drop=True, inplace=True)
 y_pred_final = pd.concat([y_test_df, y_pred_1],axis=1)
 y_pred_final.head()
train_text, cv_text, train_target, cv_target = train_test_split(train_input_padded, train_label, test_size = 0.1, random_state=2 )<import_modules>
y_pred_final= y_pred_final.rename(columns={ 0 : 'Survived_Prob'})
 y_pred_final.head()
from keras.models import Sequential
from keras.layers import Embedding,Bidirectional,LSTM,Dropout,Conv1D,MaxPooling1D,Dense<choose_model_class>
y_pred_final['final_predicted'] = y_pred_final.Survived_Prob.map(lambda x: 1 if x > 0.47 else 0)
 y_pred_final.head()
lstm=Sequential()
lstm.add(Embedding(n_words+1,300,input_length=max_length,weights=[embedding_matrix], trainable=False))
lstm.add(Bidirectional(LSTM(256,return_sequences=True)))
 lstm.add(Dropout(0.2))
lstm.add(Conv1D(100,5,activation='relu'))
lstm.add(MaxPooling1D(pool_size=4))
lstm.add(LSTM(128))
lstm.add(Dropout(0.4))
lstm.add(Dense(1,activation='sigmoid'))
lstm.summary()
lstm.compile(loss='binary_crossentropy',optimizer='adam', metrics=['acc'])
<set_options>
metrics.accuracy_score(y_pred_final.Survived, y_pred_final.final_predicted )
del embeddings_index
gc.collect()<train_model>
data_val_Id = data_val['PassengerId']
data_val = data_val[col]
data_val.head()
history=lstm.fit(np.array(train_text), np.array(train_target), epochs = 5, validation_data=(np.array(cv_text),np.array(cv_target)) , batch_size=1024,verbose=1 )<find_best_params>
data_val_sm = sm.add_constant(data_val)
 y_val_pred = res.predict(data_val_sm)
 y_val_pred[:10]
cv_predictions = lstm.predict(cv_text, batch_size=512)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 result = f1_score(cv_target,(cv_predictions>thresh ).astype(int))
thresholds.append([thresh, result])
 print("F1 score at threshold {} is {}".format(thresh, result))
thresholds.sort(key=lambda x: x[1], reverse=True)
 print("Best value {}".format(thresholds[0]))
best_thresh = thresholds[0]<predict_on_test>
y_val_1 = pd.DataFrame(y_val_pred)
 y_val_1.head()
predictions = lstm.predict(cv_text)
 predictions = np.around(predictions ).astype(int)
 df = pd.DataFrame({'pred': predictions.flatten() , 'actual': cv_target})
 df.head()
pd.crosstab(df['pred'], df['actual'], margins=True )<predict_on_test>
y_val_1= y_val_1.rename(columns={ 0 : 'Survived_Prob'})
 y_val_1['PassengerId'] = data_val_Id
y_val_1['final_predicted'] = y_val_1.Survived_Prob.map(lambda x: 1 if x > 0.57 else 0)
 y_val_1.head()
predictions = lstm.predict(test_input_padded )<data_type_conversions>
output = pd.DataFrame({'PassengerId': y_val_1.PassengerId, 'Survived': y_val_1.final_predicted})
 output.to_csv('my_submission_GLM.csv', index=False)
 print("Your submission was successfully saved!" )