Kaggle competition: Titanic - Machine Learning from Disaster
tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])
 questions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])
 questions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])
 questions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')
 questions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16' )<drop_column>
plt.style.use('seaborn-darkgrid')
 warnings.filterwarnings('ignore' )
questions_df.drop(columns=['content_correctness'], inplace=True )<merge>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True )<drop_column>
target = train_data['Survived']
train_data.drop(['Survived'], axis = 1, inplace = True )
del bundle_agg
del part_agg
del tags1_agg
gc.collect()<train_model>
full_data = pd.concat([train_data, test_data], axis = 0 )
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )<categorify>
full_data.drop(['PassengerId', 'Ticket', 'Cabin', 'Fare'], axis = 1, inplace = True )
MAX_SEQ = 160
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for user_id in group.index:
q, qa = group[user_id]
if len(q)>MAX_SEQ:
group[user_id] =(q[-MAX_SEQ:],qa[-MAX_SEQ:])
 pickle.dump(group, open("group.pkl", "wb"))
del group
gc.collect() 
<define_variables>
%pip install nameparser
full_data['Name_Title'] = full_data['Name'].apply(lambda x: HumanName(x ).title)
 print(full_data.Name_Title.value_counts())
 unique_title = list(full_data.Name_Title.value_counts().index )
features = [
'lagtime',
'lagtime_mean',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_explation_false_mean',
'content_explation_true_mean',
'content_count',
'content_sum',
'task_container_correctness',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'tags1',
'tags1_correctness_mean',
'bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'part',
'tags1',
'bundle_id',
'prior_question_had_explanation',
] 
<init_hyperparams>
full_data.drop(['Name'], axis = 1, inplace = True )
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=20000*1000)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.09)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df)) 
<drop_column>
top_title = ['Mr.','Miss.','Mrs.','Master.']
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
full_data.replace({'Sex': {'male': 0, 'female': 1}}, inplace = True )
for i in range(0,num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target])
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()
del tr_data
del va_data
gc.collect() 
<data_type_conversions>
full_data.isnull().sum() [full_data.isnull().sum() > 0].sort_values(ascending = False )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
guess_ages = np.zeros(( 2,3))
for i in range(0, 2):
for j in range(0, 3):
guess_df = full_data[(full_data['Sex'] == i)&(full_data['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
full_data.loc[(full_data.Age.isnull())&(full_data.Sex == i)&(full_data.Pclass == j+1),'Age'] = guess_ages[i,j]
full_data['Age'] = full_data['Age'].astype(int )
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()<categorify>
full_data['Embarked'].fillna(full_data['Embarked'].mode() [0], inplace = True )
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<categorify>
full_data.isnull().sum() [full_data.isnull().sum() > 0].sort_values(ascending = False )
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()<feature_engineering>
full_data['Family'] = full_data['SibSp'] + full_data['Parch'] + 1
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<choose_model_class>
full_data.loc[full_data['Family'] == 1, 'Family'] = 'Alone'
full_data.loc[full_data['Family'] == 2, 'Family'] = 'Partner'
full_data.loc[(full_data['Family'] != 'Alone')&(full_data['Family'] != 'Partner'), 'Family'] = 'More'
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = len(skills)
 self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 return x, questions
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SAKT_model = SAKTModel(n_skill, embed_dim=128)
 try:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt"))
except:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt", map_location='cpu'))
SAKT_model.to(device)
 SAKT_model.eval()
group = pickle.load(open("group.pkl", "rb"))
print(psutil.virtual_memory().percent)
<split>
iter_test = env.iter_test()
prior_test_df = None<data_type_conversions>
full_data.drop(['SibSp', 'Parch'], axis = 1, inplace = True)
 print('Types of Passengers Travelling:',full_data.Family.value_counts())
 full_data.head(10 )
%%time
for(test_df, sample_prediction_df)in iter_test:
if(prior_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
lgbm_final = sub_preds / len(clfs)
 test_df[target] = np.array(SAKT_outs)* 0.5 + lgbm_final * 0.5
env.predict(test_df[['row_id', target]] )<install_modules>
full_data.loc[ full_data['Age'] <= 10, 'Age'] = 0
full_data.loc[(full_data['Age'] > 10)&(full_data['Age'] <= 20), 'Age'] = 1
full_data.loc[(full_data['Age'] > 20)&(full_data['Age'] <= 30), 'Age'] = 2
full_data.loc[(full_data['Age'] > 30)&(full_data['Age'] <= 40), 'Age'] = 3
full_data.loc[(full_data['Age'] > 40)&(full_data['Age'] <= 50), 'Age'] = 4
full_data.loc[(full_data['Age'] > 50)&(full_data['Age'] <= 60), 'Age'] = 5
full_data.loc[(full_data['Age'] > 60)&(full_data['Age'] <= 70), 'Age'] = 6
full_data.loc[ full_data['Age'] > 70, 'Age'] = 7
!pip install efficientnet_pytorch >> /dev/null<set_options>
dummy_df = pd.get_dummies(full_data, columns = ['Pclass', 'Embarked', 'Name_Title', 'Family'], drop_first = True )
random.seed(a=42)
 os.environ["CUDA_VISIBLE_DEVICES"] = "0"
gpus = tf.config.experimental.list_physical_devices('GPU')
 try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 except RuntimeError as e:
print(e)
 CFG = dict(
 batch_size = 32,
read_size = 256,
crop_size = 235,
net_size = 224,
LR = 1e-4,
epochs = 30,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
tta_steps = 15,
es_patience = 4, 
)
 GCS_PATH = f'.. /input//melanoma-{CFG["read_size"]}x{CFG["read_size"]}'
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))
 malig_files = sorted(glob(f'.. /input/malignant-v2-{CFG["read_size"]}x{CFG["read_size"]}/*.tfrec'))
malig_files = malig_files[15:]
files_train = np.concatenate([files_train, malig_files])
<normalization>
train = dummy_df[:len(train_data)]
test = dummy_df[len(train_data):]
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3])
 def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target'], example['image_name']
def read_unlabeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], 0, example['image_name']
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def get_dataset(files, cfg, augment = False, shuffle = False,
labeled=True):
AUTO = tf.data.experimental.AUTOTUNE
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, label, fn:(prepare_image(img, augment=augment, cfg=cfg),
label, fn),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'])
 ds = ds.prefetch(AUTO)
 return ds<train_model>
rf_clf = RandomForestClassifier(n_estimators = 100)
 rf_clf.fit(train, target)
 rf_pred = rf_clf.predict(test)
 print('Accuracy Score:',round(rf_clf.score(train, target)*100,2))
def get_model() :
model = EfficientNet.from_pretrained('efficientnet-b0', num_classes=1)
 model.to(device)
 return model
def get_opt_loss_fn(model):
optimizer = torch.optim.Adam(model.parameters() , lr=CFG["LR"])
 loss_object = torch.nn.BCEWithLogitsLoss().to(device)
 return optimizer, loss_object
def get_train_fn() :
def train_step(images, labels, model, optimizer, loss_object):
images = torch.from_numpy(images.numpy() ).permute(0,3,1,2 ).to(device)
 labels = torch.from_numpy(labels.numpy() ).to(device ).float()
if not model.training:
model.train()
predictions = model.forward(images)
 loss = loss_object(predictions[:,0], labels)
 loss.backward()
optimizer.step()
optimizer.zero_grad()
return loss.cpu().detach()
return train_step
def test_step(images, labels):
images = torch.from_numpy(images.numpy() ).permute(0,3,1,2 ).to(device)
 labels = torch.from_numpy(labels.numpy() ).to(device ).float()
if model.training:
model.eval()
with torch.no_grad() :
predictions = model.forward(images)
 loss = loss_object(predictions[:,0], labels)
 return loss.cpu().detach()
def pred_step(images):
images = torch.from_numpy(images.numpy() ).permute(0,3,1,2 ).to(device)
 if model.training:
model.eval()
with torch.no_grad() :
return torch.sigmoid(model(images)).cpu().numpy()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<split>
svc_clf = SVC(C = 2, kernel = 'rbf')
 svc_clf.fit(train, target)
 svc_pred = svc_clf.predict(test)
 print('Accuracy Score:',round(svc_clf.score(train, target)*100,2))
fold_cv_scores = []
submission_scores = []
folds = KFold(n_splits=5, shuffle = True, random_state = 42)
 fold_num = 0
for tr_idx, va_idx in folds.split(files_train):
print(f"Starting fold: {fold_num}")
 no_imp = 0
CFG['batch_size'] = 32
checkpoint_filepath = f"checkpoint_{fold_num}.h5"
files_train_tr = files_train[tr_idx]
files_train_va = files_train[va_idx]
ds_train = get_dataset(files_train_tr, CFG, augment=True, shuffle=True)
 ds_val = get_dataset(files_train_va, CFG, augment=False, shuffle=False)
 model = get_model()
optimizer, loss_object = get_opt_loss_fn(model)
 train_fn = get_train_fn()
torch.cuda.empty_cache()
bestLoss = float("inf")
 for e in range(CFG["epochs"]):
trainLoss = 0
tk0 = tqdm(ds_train)
 for idx,(x,y,_)in enumerate(tk0):
loss = train_fn(x, y, model, optimizer, loss_object)
 trainLoss += loss.numpy()
tk0.set_postfix(loss=trainLoss/(idx+1))
testLoss = 0
tk0 = tqdm(ds_val)
 for idx,(x,y,_)in enumerate(tk0):
loss = test_step(x,y)
 testLoss += loss.numpy()
tk0.set_postfix(loss=testLoss/(idx+1))
testLoss /= idx
if testLoss < bestLoss:
no_imp = 0
bestLoss = testLoss
torch.save(model.state_dict() , checkpoint_filepath)
 else:
no_imp += 1
if no_imp > CFG["es_patience"]:
print("Early stopping.. ")
 break
model.load_state_dict(torch.load(checkpoint_filepath))
CFG['batch_size'] = 256
ds_valAug = get_dataset(files_train_va, CFG, augment=True)
 ds_testAug = get_dataset(files_test, CFG, augment=True, labeled=False)
 for t in range(CFG['tta_steps']):
for idx,(x,y,fn)in enumerate(ds_valAug):
predictions = pred_step(x)
 for j in range(predictions.shape[0]):
fold_cv_scores.append([fold_num,
fn[j].numpy().decode("utf-8"),
predictions[j,0].item() ,
y[j].numpy() ])
 for idx,(x,y,fn)in enumerate(ds_testAug):
predictions = pred_step(x)
 for j in range(predictions.shape[0]):
submission_scores.append([fold_num,
fn[j].numpy().decode("utf-8"),
predictions[j,0].item() ])
 fold_num += 1 
<create_dataframe>
ada_clf = AdaBoostClassifier(n_estimators = 100)
 ada_clf.fit(train, target)
 ada_pred = ada_clf.predict(test)
 print('Accuracy Score:',round(ada_clf.score(train, target)*100,2))
df_fold = pd.DataFrame(fold_cv_scores, columns=["Fold", "Filename", "Pred", "Label"])
 df_sub = pd.DataFrame(submission_scores, columns=["Fold", "Filename", "Pred"] )<compute_test_metric>
lgbm_clf = LGBMClassifier(n_estimators = 400)
 lgbm_clf.fit(train, target)
 lgbm_pred = lgbm_clf.predict(test)
 print('Accuracy Score:',round(lgbm_clf.score(train, target)*100,2))
df_fold = df_fold.groupby(["Filename"] ).mean().reset_index()
print("CV ROCAUC: ")
 print(roc_auc_score(df_fold["Label"], df_fold["Pred"]))<save_to_csv>
xgb_clf = XGBClassifier(n_estimators = 200)
 xgb_clf.fit(train, target)
 xgb_pred = xgb_clf.predict(test)
 print('Accuracy Score:',round(xgb_clf.score(train, target)*100,2))
df_sub = df_sub.groupby(["Filename"] ).mean().reset_index()
df_sub = df_sub[["Filename", "Pred"]]
df_sub.columns=["image_name", "target"]
df_sub.to_csv("submission.csv", index=False )<load_from_csv>
base_estimators = [('lgbm',lgbm_clf), 
('rf',rf_clf)]
stc_clf = StackingClassifier(estimators = base_estimators, final_estimator = svc_clf)
 stc_clf.fit(train, target)
 stc_pred = stc_clf.predict(test)
 print('Accuracy Score:',round(stc_clf.score(train, target)*100,2))
def MinMaxBestBaseStacking(input_folder, best_base, output_path):
sub_base = pd.read_csv(best_base)
 all_files = os.listdir(input_folder)
 outs = [pd.read_csv(os.path.join(input_folder, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "target" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub['is_iceberg_max'] = concat_sub.iloc[:, 1:6].max(axis=1)
 concat_sub['is_iceberg_min'] = concat_sub.iloc[:, 1:6].min(axis=1)
 concat_sub['is_iceberg_mean'] = concat_sub.iloc[:, 1:6].mean(axis=1)
 concat_sub['is_iceberg_median'] = concat_sub.iloc[:, 1:6].median(axis=1)
 cutoff_lo = 0.60
cutoff_hi = 0.40
concat_sub['is_iceberg_base'] = sub_base['target']
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:, 1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:, 1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_base']))
concat_sub[['image_name', 'target']].to_csv(output_path,
index=False, float_format='%.12f' )<load_from_csv>
estimators = [('rf',rf_clf), 
('xgb',xgb_clf), 
('lgbm',lgbm_clf)]
vot_clf = VotingClassifier(estimators = estimators, voting = 'soft')
 vot_clf.fit(train, target)
 vot_pred = vot_clf.predict(test)
 print('Accuracy Score:',round(vot_clf.score(train, target)*100,2))
MinMaxBestBaseStacking('.. /input/cs0099/', '.. /input/cs0099/submission_mean.csv', 'submission.csv')
<import_modules>
submit_data = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 submit_data.head()
import pandas as pd, numpy as np, os
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<load_from_csv>
submit_data['Survived'] = rf_pred
submit_data.head()
<prepare_x_and_y><EOS> 
submit_data.to_csv('output.csv', index = False )