Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<filter>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
col_list[np.where(LGBM_ranking<=5)]<set_options>
from fastai.vision import *
from fastai.metrics import error_rate
import torchvision.models as mdls
pd.set_option('display.max_rows', 500 )<train_model>
path = Path('')
 path_train = '.. /input/aptos2019-blindness-detection/train_images'
path_test = '.. /input/aptos2019-blindness-detection/test_images'
def rmse_score(y_test, y_pred):
return np.sqrt(mean_squared_error(y_test, y_pred))
def clear_save_file(file_name, variable=None, save=False, file_type='pickle'):
file_path = file_name + '.' + file_type
try:
os.remove(file_path)
 except:
pass
if save:
if file_type == 'pickle':
with open(file_path, 'wb')as handle:
pickle.dump(variable, handle)
 gc.collect()
print(len(gc.get_objects()))
def create_data(val_month, feature_list, model_name, validation=False, meta=False):
print('Fetching data for ', model_name)
 with open('df_data.pickle', 'rb')as handle:
df_data = pickle.load(handle)
 gc.collect()
month_train =(df_data['date_block_num'] < val_month)
 X_train = df_data.loc[month_train, feature_list].values
y_train = df_data[month_train].item_cnt_month.values
month_test =(df_data['date_block_num'] == val_month)
 X_test = df_data.loc[month_test, feature_list].values
if validation:
y_test = df_data[month_test].item_cnt_month.values
del df_data
gc.collect()
if model_name == 'Linear':
clear_save_file('X_train', X_train, save=True, file_type='pickle')
 del X_train
clear_save_file('y_train', y_train, save=True, file_type='pickle')
 del y_train
clear_save_file('X_test', X_test, save=True, file_type='pickle')
 del X_test
if validation:
clear_save_file('y_test', y_test,save=True, file_type='pickle')
 del y_test
gc.collect()
print('Data on disk now!')
 if model_name == 'LGBM':
X_train_LGBM = lgb.Dataset(data=X_train, label=y_train)
 del X_train
del y_train
gc.collect()
print(len(gc.get_objects()))
clear_save_file('X_train_LGBM',file_type='bin')
 X_train_LGBM.save_binary('X_train_LGBM.bin')
 clear_save_file('X_test', X_test, save=True, file_type='pickle')
 if validation:
X_test_LGBM = lgb.Dataset(data=X_test, label=y_test, reference=X_train_LGBM)
 del X_train_LGBM
gc.collect()
print(len(gc.get_objects()))
clear_save_file('y_test', y_test, save=True, file_type='pickle')
 del y_test
gc.collect()
print(len(gc.get_objects()))
clear_save_file('X_test_LGBM',file_type='bin')
 X_test_LGBM.save_binary('X_test_LGBM.bin')
 del X_test_LGBM
gc.collect()
print(len(gc.get_objects()))
else:
del X_train_LGBM
gc.collect()
print(len(gc.get_objects()))
print('data on disk now!')
 def gridsearch_LGBM(num_feature_list, params, feature_ranking, col_list):
final_param_list = []
final_param_score = []
for num_feature in num_feature_list:
print('Now testing for ', num_feature, ' features...')
 feature_list = col_list[np.where(LGBM_ranking <= num_feature)]
create_data(33, feature_list, 'LGBM', validation=True, meta=False)
 key_list = np.array(list(params.keys()))
param_dict = {key: np.arange(len(params[key])) for key in key_list}
combinations = list(product(*[param_dict[key] for key in key_list]))
print('Totalling {} combinations to be tested!'.format(len(combinations)))
 count = 0
for combination in combinations:
ts = time.time()
param_combination = {key_list[k_pos]: params[key_list[k_pos]][v_pos] for k_pos, v_pos in enumerate(combination)}
param_combination['metric'] = 'rmse'
model = lgb.train(param_combination,train_set=lgb.Dataset('X_train_LGBM.bin'), num_boost_round=1000, verbose_eval=False,
early_stopping_rounds=10, valid_sets=lgb.Dataset('X_test_LGBM.bin'))
cur_iter = model.current_iteration()
with open('X_test.pickle', 'rb')as handle:
X_test = pickle.load(handle)
 pred = model.predict(X_test)
 with open('y_test.pickle', 'rb')as handle:
y_test = pickle.load(handle)
 score = -mean_squared_error(pred, y_test)
 param_combination['iteration'] = cur_iter
param_combination['num_feature'] = num_feature
final_param_list.append(param_combination)
 final_param_score.append(score)
 del model
del X_test
del y_test
gc.collect()
print(len(gc.get_objects()))
count = count + 1
print(count, '/', len(combinations), ' is done! used time: ', time.time() - ts)
 clear_save_file('X_train_LGBM', file_type='bin')
 clear_save_file('X_test_LGBM', file_type='bin')
 clear_save_file('X_test', file_type='pickle')
 clear_save_file('y_test', file_type='pickle')
 best_idx = np.argmax(final_param_score)
 print('best score is: ', final_param_score[best_idx])
 print('best param combination is: ')
 print(final_param_list[best_idx])
 class feature_selector(BaseEstimator, TransformerMixin):
def __init__(self, feature_rank=0, num_feature=2):
self.feature_rank = feature_rank
self.num_feature = num_feature
self.feature_selected = None
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.feature_selected = np.where(self.feature_rank <= self.num_feature)[0]
print('now number of feature is: ', self.num_feature)
 return X[:, self.feature_selected]
class feature_selector(BaseEstimator, TransformerMixin):
def __init__(self, feature_rank=0, num_feature=2):
self.feature_rank = feature_rank
self.num_feature = num_feature
self.feature_selected = None
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.feature_selected = np.where(self.feature_rank <= self.num_feature)[0]
print('now number of feature is: ', self.num_feature)
 return X[:, self.feature_selected]
class get_feature_column(BaseEstimator, TransformerMixin):
def __init__(self, num_component=3):
self.num_component = num_component
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
print('returning number of pca vector: ', self.num_component)
 return X[:, 0: self.num_component]
class my_scaler(BaseEstimator, TransformerMixin):
def __init__(self):
self.std = np.array([])
 self.mean = np.array([])
 def fit(self, X, y=None):
for i in range(X.shape[1]):
std = np.std(X[:, i].astype(np.float32))
mean = np.mean(X[:, i].astype(np.float32))
self.mean = np.append(self.mean, mean)
 self.std = np.append(self.std, std)
 return self
def transform(self, X, y=None):
for i in range(X.shape[1]):
X[:, i] =(X[:, i] - self.mean[i])/ self.std[i]
return X.astype(np.float16)
 def my_grid_search(pipeline, params, X_path, y_path, cv=3, cv_by_index=False):
key_list = np.array(list(params.keys()))
param_dict = {key: np.arange(len(params[key])) for key in key_list}
combinations = list(product(*[param_dict[key] for key in key_list]))
print('Totalling {} combinations to be tested!'.format(len(combinations)))
 param_list = []
param_score = []
for combination in combinations:
param_combination = {key_list[k_pos]: params[key_list[k_pos]][v_pos] for k_pos, v_pos in enumerate(combination)}
my_score = []
if cv_by_index:
train_idx, val_idx = cv
with open(X_path, 'rb')as handle:
X = pickle.load(handle)
 with open(y_path, 'rb')as handle:
y = pickle.load(handle)
 my_pipeline = clone(pipeline)
 my_pipeline.set_params(**param_combination)
 my_pipeline.fit(X[train_idx], y[train_idx])
 val_pred = my_pipeline.predict(X[val_idx])
 val_score = -mean_squared_error(val_pred, y[val_idx])
 my_score.append(val_score)
 del my_pipeline
del X
del y
gc.collect()
len(gc.get_objects())
 else:
for train_idx, val_idx in KFold(n_splits=cv ).split(X):
my_pipeline = clone(pipeline)
 my_pipeline.set_params(**param_combination)
 my_pipeline.fit(X[train_idx], y[train_idx])
 val_pred = my_pipeline.predict(X[val_idx])
 val_score = -mean_squared_error(val_pred, y[val_idx])
 my_score.append(val_score)
 del my_pipeline
gc.collect()
len(gc.get_objects())
 param_list.append(param_combination)
 param_score.append(np.mean(my_score))
best_idx = np.argmax(param_score)
 print('best score is: ', param_score[best_idx])
 print('best param combination is: ')
 print(param_list[best_idx])
 def run_meta_LR(col_list, feature_ranking, meta_train, meta_test, meta_train_month, meta_col=0):
meta_months = [28, 29, 30, 31, 32, 33, 34]
meta_test_month = 34
feature_list = col_list[np.where(feature_ranking <= 80)]
for month in meta_months:
ts = time.time()
create_data(month, feature_list, 'Linear', validation=False, meta=False)
 with open('X_train.pickle', 'rb')as handle:
X_train = pickle.load(handle)
 with open('X_test.pickle', 'rb')as handle:
X_test = pickle.load(handle)
 with open('y_train.pickle', 'rb')as handle:
y_train = pickle.load(handle)
 scaler = my_scaler()
scaler.fit(X_train)
 X_train = scaler.transform(X_train)
 X_test = scaler.transform(X_test)
 model = LinearRegression(normalize=False)
 model.fit(X_train, y_train)
 meta_pred = model.predict(X_test)
 loc =(meta_train_month == month)
 if month != meta_test_month:
meta_train[loc, meta_col] = meta_pred
else:
meta_test[:, meta_col] = meta_pred
del X_train
del X_test
del y_train
clear_save_file('X_train')
 clear_save_file('y_train')
 clear_save_file('X_test')
 gc.collect()
print(len(gc.get_objects()))
print('LR for month ', month, ' is done! used time: ', time.time() - ts)
 def run_meta_LR_PCA(col_list, meta_train, meta_test, meta_train_month, meta_col=1):
meta_months = [28, 29, 30, 31, 32, 33, 34]
meta_test_month = 34
feature_num = 56
for month in meta_months:
ts = time.time()
create_data(month, col_list, 'Linear', validation=False, meta=False)
 with open('X_train.pickle', 'rb')as handle:
X_train = pickle.load(handle)
 with open('X_test.pickle', 'rb')as handle:
X_test = pickle.load(handle)
 with open('y_train.pickle', 'rb')as handle:
y_train = pickle.load(handle)
 print('starting PCA...')
 ipca = IncrementalPCA(batch_size=40)
 ipca.fit(X_train)
 print('PCA trained, now converting X...')
 X_train = ipca.transform(X_train)[:, 0: feature_num]
X_test = ipca.transform(X_test)[:, 0: feature_num]
del ipca
gc.collect()
print(len(gc.get_objects()))
print('PCA finished...')
 model = LinearRegression(normalize=False)
 model.fit(X_train, y_train)
 meta_pred = model.predict(X_test)
 loc =(meta_train_month == month)
 if month != meta_test_month:
meta_train[loc, meta_col] = meta_pred
else:
meta_test[:, meta_col] = meta_pred
del X_train
del X_test
del y_train
clear_save_file('X_train')
 clear_save_file('y_train')
 clear_save_file('X_test')
 gc.collect()
print(len(gc.get_objects()))
print('LR for month ', month, ' is done! used time: ', time.time() - ts)
 def run_meta_LGBM(col_list, feature_ranking, meta_train, meta_test, meta_train_month, meta_col=2):
feature_list = col_list[np.where(feature_ranking < 60)]
meta_months = [28, 29, 30, 31, 32, 33, 34]
meta_test_month = 34
for month in meta_months:
ts = time.time()
create_data(month, feature_list, 'LGBM', validation=False, meta=False)
 print('now running the model...')
 params = {'max_depth': 9, 'num_leaves': 64, 'learning_rate': 0.1, 'metric': 'rmse'}
model = lgb.train(params,train_set=lgb.Dataset('X_train_LGBM.bin'), num_boost_round=80, verbose_eval=False)
 with open('X_test.pickle', 'rb')as handle:
X_test = pickle.load(handle)
 meta_pred = model.predict(X_test)
 loc =(meta_train_month == month)
 if month != meta_test_month:
meta_train[loc, meta_col] = meta_pred
else:
meta_test[:, meta_col] = meta_pred
del X_test
del model
clear_save_file('X_train_LGBM', file_type='bin')
 clear_save_file('X_test')
 gc.collect()
print(len(gc.get_objects()))
print('LGBM for month ', month, ' is done! used time: ', time.time() - ts )<prepare_x_and_y>
train = pd.read_csv(path/'.. /input/aptos2019-blindness-detection/train.csv')[:100]
test = pd.read_csv(path/'.. /input/aptos2019-blindness-detection/sample_submission.csv')
with open('df_data.pickle', 'rb')as handle:
df_data = pickle.load(handle)
 meta_train_months = [28, 29, 30, 31, 32, 33]
meta_test_month = [34]
months = df_data.date_block_num
meta_train_months_series = months[months.isin(meta_train_months)]
meta_test_month_series = months[months.isin(meta_test_month)]
y_train_meta = df_data[months.isin(meta_train_months)].item_cnt_month.values
X_train_meta = np.zeros([y_train_meta.shape[0], 3])
 X_test_meta = np.zeros([meta_test_month_series.shape[0], 3])
 del df_data
gc.collect()
print(len(gc.get_objects()))<statistical_test>
test2 = test.copy()
test2['id_code'] = np.arange(0,test.shape[0],dtype=np.int ).astype('str' )
run_meta_LR(col_list, LR_ranking, X_train_meta, X_test_meta, meta_train_months_series, meta_col=0)
 gc.collect()
print(len(gc.get_objects()))
run_meta_LR_PCA(col_list, X_train_meta, X_test_meta, meta_train_months_series, meta_col=1)
 gc.collect()
print(len(gc.get_objects()))
run_meta_LGBM(col_list, LGBM_ranking, X_train_meta, X_test_meta, meta_train_months_series, meta_col=2)
 gc.collect()
print(len(gc.get_objects()))<train_model>
tfms = get_transforms(flip_vert=True,max_rotate = 10,max_warp = 0,max_zoom =1.05,max_lighting = 0 )
model = LinearRegression()
model.fit(X_train_meta, y_train_meta)
<predict_on_test>
tfms[1].append(tfms[0][1])
 tfms[1].append(tfms[0][2])
meta_pred = model.predict(X_test_meta)
 gc.collect()
print(len(gc.get_objects()))<drop_column>
gc.collect()
df_submission = df_test.copy().drop(['shop_id', 'item_id', 'date_block_num'], axis=1)
 df_submission['item_cnt_month'] = meta_pred
df_submission.head() 
<save_to_csv>
image_dic = {}
df_submission.to_csv('my_submission.csv', index=False )<set_options>
def load_image_on_ram(p,names,k):
for i in tqdm(names):
path= p+i+'.png'
image = read_image2(path,k)
 image_dic[i] = image
env = riiideducation.make_env()<load_from_csv>
path1 = '.. /input/aptos2019-blindness-detection/train_images/'
path2 = '.. /input/aptos2019-blindness-detection/test_images/'
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<load_from_csv>
load_image_on_ram(path1,train.id_code.unique() ,1 )
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)<sort_values>
image_type = lambda x : load_image_on_ram(path2,test.id_code.unique() ,x )
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<count_unique_values>
def im_b(path=""):
img = path.split('/')[-1]
image = image_dic[img]/255
return image
train[(train.content_type_id == False)].task_container_id.nunique()<groupby>
class MyImageItemList(ImageList):
def open(self,path:PathOrStr)->Image:
image = im_load(path)
 xx = vision.Image(px=pil2tensor(image,np.float32))
return xx
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2<groupby>
im_load = im_b
group3['avg_questions_seen'] = group3.avg_questions.cumsum()<set_options>
def get_data(sz=256):
data =(MyImageItemList.from_df(df = train,path = '',folder = path1)
.split_by_rand_pct (.15)
.label_from_df(cols = 'diagnosis' 
)
.add_test(MyImageItemList.from_df(df = test,path='',folder =path2)) 
.transform(tfms,size=sz,resize_method = ResizeMethod.SQUISH, padding_mode = 'zeros')
.databunch(bs = 8,num_workers = 4))
data.normalize(imagenet_stats)
 return data
data = get_data(400 )
group3.iloc[0].avg_questions_seen<groupby>
data.show_batch(rows=3,figsize=(8,8),ds_type = DatasetType.Test )
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']<merge>
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=data.c )
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )<groupby>
!cp '.. /input/ef5-ahe/ef5_ah_400_1.pth' models/
!cp '.. /input/ef5-400/ef5_400_1.pth' models/
!cp '.. /input/ef5-b15/ef5_b15_400_1.pth' models/
!cp '.. /input/ef5-b15/ef5_b15_320_1.pth' models/
!cp '.. /input/ef5-b-w-400/ef5_b_400_2.pth' models/
!cp '.. /input/ef4-b-ls-400/ef5_b_400_1.pth' models/
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']<groupby>
!ls models
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']<merge>
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<merge>
!nvidia-smi
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def run_subm(learn=learn, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
preds,y = learn.TTA(scale=1,ds_type=DatasetType.Test)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 return test_df
question2.quest_pct = round(question2.quest_pct,5 )<drop_column>
learn.data = get_data(400 )
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )<filter>
learn.load('ef5_b_400_1');
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )<groupby>
%%time
preds1 = learn.TTA(beta =.2, scale=1,ds_type=DatasetType.Test )
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']<groupby>
v1 = torch.tensor([1.0,1.0,.7,.7,1.0] )
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )<groupby>
learn.load('ef5_b_400_2');
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<drop_column>
gc.collect()
del(train )<merge>
%%time
preds2 = learn.TTA(beta =.2, scale=1,ds_type=DatasetType.Test )
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left" )<merge>
v2 = torch.tensor([1.0,.7,1.0,.7,1.0] )
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left" )<categorify>
learn.data = get_data(400 )
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<load_from_csv>
gc.collect()
<categorify>
learn.load('ef5_b15_400_1');
gc.collect()
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )<merge>
%%time
preds3 = learn.TTA(beta =.2, scale=1,ds_type=DatasetType.Test )
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1<prepare_x_and_y>
v3 = torch.tensor([1.0,1.0,.5,1.0,1.0] )
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<drop_column>
learn.data = get_data(320)
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]<data_type_conversions>
learn.load('ef5_b15_320_1');
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True)
<data_type_conversions>
gc.collect()
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )<create_dataframe>
%%time
preds4 = learn.TTA(beta =.2, scale=1,ds_type=DatasetType.Test )
y_true = np.array(y_val)
 lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'],free_raw_data=False)
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train, free_raw_data=False )<find_best_params>
v4 = torch.tensor([1.0,1.0,.5,1.0,1.0] )
def objective(trial):
params = {
'num_leaves': trial.suggest_int('num_leaves', 32, 512),
'boosting_type': 'gbdt',
'max_bin': trial.suggest_int('max_bin', 700, 900),
'objective': 'binary',
'metric': 'auc',
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 16),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 8),
'min_child_samples': trial.suggest_int('min_child_samples', 4, 80),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 1.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 1.0),
'early_stopping_rounds': 10
}
model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_eval], verbose_eval=1000, num_boost_round=1300)
 val_pred = model.predict(X_val)
 score = roc_auc_score(y_true, val_pred)
 print(f"AUC = {score}")
 return score<import_modules>
image_type(3)
 learn.data = get_data(400 )
import optuna<find_best_params>
learn.load('ef5_ah_400_1')
 gc.collect()
study = optuna.create_study(direction='maximize')
 study.optimize(objective, n_trials=5 )<find_best_params>
%%time
preds5 = learn.TTA(beta =.2, scale=1,ds_type=DatasetType.Test )
print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))<train_model>
v5 = torch.tensor([1.0,.7,.7,1.0,.7] )
model = lgb.train(trial.params, lgb_train, valid_sets=[lgb_train, lgb_eval], verbose_eval=1000,num_boost_round=1300 )<compute_train_metric>
gc.collect()
y_pred = model.predict(X_val)
 roc_auc_score(y_true, y_pred )<import_modules>
learn.load('ef5_400_1')
 gc.collect()
import matplotlib.pyplot as plt
import seaborn as sns<split>
%%time
preds6 = learn.TTA(beta =.2, scale=1,ds_type=DatasetType.Test )
iter_test = env.iter_test()<merge>
test_preds = torch.argmax(preds,dim=1 )
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
test['diagnosis'] = np.array(test_preds,dtype = np.int)
 test.to_csv('submission.csv',index = False )
<load_from_csv><EOS> 
test['diagnosis'].value_counts()