Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
warnings.filterwarnings('ignore')
 %matplotlib inline
sns.set_style('white' )
df['Fence']=df['Fence'].fillna('None' )<data_type_conversions>
labelled = pd.read_csv('.. /input/train.csv' )
df['FireplaceQu']=df['FireplaceQu'].fillna('None' )<feature_engineering>
unlabelled = pd.read_csv('.. /input/test.csv' )
df['LotFrontage']=df['LotFrontage'].fillna(df['LotFrontage'].median() )<feature_engineering>
passengerID = unlabelled[['PassengerId']]
df['GarageArea']=df['GarageArea'].fillna(0)
 df['GarageArea'].isnull().sum()<count_values>
data = pd.concat([labelled, unlabelled], axis= 0, sort= False )
df['GarageQual'].value_counts()<categorify>
class_mean_age = data.pivot_table(values='Age', index='Pclass', aggfunc='median' )
def impute_GarageQual(cols):
GarageArea=cols[1]
GarageQual=cols[0]
if pd.isnull(GarageQual):
if 0<GarageArea<400:
return 'Fa'
elif 400<GarageArea<550:
return 'Ta'
elif 550<GarageArea<700:
return 'Gd'
else:
return 'Ex'
else:
return GarageQual<feature_engineering>
null_age = data['Age'].isnull()
df['GarageQual']=df['GarageQual'].fillna('None')
 df['GarageQual'].isnull().sum()<categorify>
data.loc[null_age,'Age'] = data.loc[null_age,'Pclass'].apply(lambda x: class_mean_age.loc[x] )
def impute_GarageYrBlt(cols):
GarageYrBlt=cols[0]
GarageQual=cols[1]
if pd.isnull(GarageYrBlt):
if GarageQual=='Po':
return 1920
elif GarageQual=='Fa':
return 1930
elif GarageQual=='Ta':
return 1975
elif GarageQual=='Gd':
return 1980
elif GarageQual=='Ex':
return 2000
else:
return 2010
else:
return GarageYrBlt 
<count_missing_values>
data.Age.isnull().sum()
df['GarageYrBlt'].isnull().sum()<feature_engineering>
class_mean_fare = data.pivot_table(values= 'Fare', index= 'Pclass', aggfunc='median' )
df['GarageYrBlt']=df['GarageYrBlt'].fillna(0)
 df['GarageYrBlt'].isnull().sum()<categorify>
null_fare = data['Fare'].isnull()
def impute_GarageCond(cols):
GarageArea=cols[1]
GarageCond=cols[0]
if pd.isnull(GarageCond):
if 0<GarageArea<400:
return 'Fa'
elif 400<GarageArea<550:
return 'Ta'
elif 550<GarageArea<700:
return 'Gd'
else:
return 'Ex'
else:
return GarageCond<feature_engineering>
data.loc[null_fare, 'Fare'] = data.loc[null_fare, 'Pclass'].apply(lambda x: class_mean_fare.loc[x] )
df['GarageCond']=df['GarageCond'].fillna('None')
 df['GarageCond'].isnull().sum()<count_values>
data.Fare.isnull().sum()
df['GarageType'].value_counts()<categorify>
data.Embarked.value_counts()
def impute_GarageType(cols):
GarageArea=cols[1]
GarageType=cols[0]
if pd.isnull(GarageType):
if 0<GarageArea<400:
return 'Detchd'
elif 400<GarageArea<600:
return 'Attchd'
elif 600<GarageArea<700:
return 'BuiltIn'
else:
return '2Types'
else:
return GarageType<feature_engineering>
data['Embarked'] = data.Embarked.fillna('S' )
df['GarageType']=df['GarageType'].fillna('None')
 df['GarageType'].isnull().sum()<feature_engineering>
data.Embarked.isnull().sum()
df['TotalBsmtSF']=df['TotalBsmtSF'].fillna(df['TotalBsmtSF'].mean())
 df['TotalBsmtSF'].isnull().sum()<categorify>
data['Title'] = data.Name.apply(lambda x : x[x.find(',')+2:x.find('.')] )
def impute_BsmtExposure(cols):
TotalBsmtSF=cols[1]
BsmtExposure=cols[0]
if pd.isnull(BsmtExposure):
if 0<TotalBsmtSF<900:
return 'No'
elif 900<TotalBsmtSF<1000:
return 'Mn'
elif 1000<TotalBsmtSF<1400:
return 'Av'
else:
return 'Gd'
else:
return BsmtExposure<data_type_conversions>
data.Title.value_counts()
df['BsmtExposure']=df['BsmtExposure'].fillna('None')
 df['BsmtExposure'].isnull().sum()<categorify>
rare_titles =(data['Title'].value_counts() < 10 )
def impute_BsmtCond(cols):
TotalBsmtSF=cols[1]
BsmtCond=cols[0]
if pd.isnull(BsmtCond):
if 0<TotalBsmtSF<700:
return 'Fa'
elif 700<TotalBsmtSF<1000:
return 'TA'
else:
return 'Gd'
else:
return BsmtCond<feature_engineering>
data['Title'] = data['Title'].apply(lambda x : 'Other' if rare_titles.loc[x] == True else x )
df['BsmtCond']=df['BsmtCond'].fillna('None')
 df['BsmtCond'].isnull().sum()<categorify>
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
def impute_BsmtQual(cols):
TotalBsmtSF=cols[1]
BsmtQual=cols[0]
if pd.isnull(BsmtQual):
if 0<TotalBsmtSF<800:
return 'TA'
elif 800<TotalBsmtSF<1400:
return 'Gd'
else:
return 'Ex'
else:
return BsmtQual 
<feature_engineering>
data['IsAlone'] = 0
df['BsmtQual']=df['BsmtQual'].fillna('None')
 df['BsmtQual'].isnull().sum()<feature_engineering>
data['IsAlone'].loc[ data['FamilySize'] == 1] = 1
df['BsmtFinType2']=df['BsmtFinType2'].fillna('NA')
 df['BsmtFinType2'].isnull().sum()<categorify>
data['AgeBins'] = 0
def impute_BsmtFinType1(cols):
TotalBsmtSF=cols[1]
BsmtFinType1=cols[0]
if pd.isnull(BsmtFinType1):
if 0<TotalBsmtSF<800:
return 'Unf'
else:
return 'GLQ'
else:
return BsmtFinType1<data_type_conversions>
data['AgeBins'].loc[(data['Age'] >= 11)&(data['Age'] < 20)] = 1
data['AgeBins'].loc[(data['Age'] >= 20)&(data['Age'] < 60)] = 2
data['AgeBins'].loc[data['Age'] >= 60] = 3
df['BsmtFinType1']=df['BsmtFinType1'].fillna('None')
 df['BsmtFinType1'].isnull().sum()<feature_engineering>
data['FareBins'] = pd.qcut(data['Fare'], 4 )
df['MasVnrArea']=df['MasVnrArea'].fillna(df['MasVnrArea'].mean() )<feature_engineering>
data.drop(columns=['PassengerId','Name','Ticket', 'Cabin', 'Age', 'Fare', 'SibSp', 'Parch'], inplace= True )
df['MSZoning']=df['MSZoning'].fillna('RL' )<data_type_conversions>
data = pd.get_dummies(
 data, columns=['Embarked', 'Sex', 'Title'], drop_first=True )
df['Utilities']=df['Utilities'].fillna('AllPub' )<feature_engineering>
label = LabelEncoder()
data['FareBins'] = label.fit_transform(data['FareBins'] )
df['Functional']=df['Functional'].fillna('Typ' )<feature_engineering>
labelled = data[data.Survived.isnull() == False].reset_index(drop=True)
 unlabelled = data[data.Survived.isnull() ].drop(columns = ['Survived'] ).reset_index(drop=True )
df['BsmtFullBath']=df['BsmtFullBath'].fillna(df['BsmtFullBath'].mean() )<feature_engineering>
labelled['Survived'] = labelled.Survived.astype('int64' )
df['BsmtHalfBath']=df['BsmtHalfBath'].fillna(df['BsmtHalfBath'].mean() )<feature_engineering>
scalers = [MinMaxScaler() , MaxAbsScaler() , StandardScaler() , RobustScaler() ,
Normalizer() , QuantileTransformer() , PowerTransformer() ]
df['GarageArea']=df['GarageArea'].fillna(df['GarageArea'].mean() )<feature_engineering>
scaler_score = {}
labelled_copy = labelled.copy(deep= True)
 for scaler in scalers:
scaler.fit(labelled_copy[['FamilySize']])
 labelled_copy['FamilySize'] = scaler.transform(labelled_copy[['FamilySize']])
 lr = LogisticRegressionCV(cv = 10, scoring= 'accuracy')
 lr.fit(labelled_copy.drop(columns=['Survived']), labelled_copy.Survived)
 score = lr.score(labelled_copy.drop(columns=['Survived']), labelled_copy.Survived)
 scaler_score.update({scaler:score} )
df['BsmtFinSF2']=df['BsmtFinSF2'].fillna(df['BsmtFinSF2'].mean() )<feature_engineering>
scaler = StandardScaler()
scaler.fit(labelled[['FamilySize']])
 labelled['FamilySize'] = scaler.transform(labelled[['FamilySize']])
 unlabelled['FamilySize'] = scaler.transform(unlabelled[['FamilySize']] )
df['Exterior1st']=df['Exterior1st'].fillna(method='ffill' )<data_type_conversions>
x_train, x_other, y_train, y_other = train_test_split(
 labelled.drop(columns=['Survived']), labelled.Survived, train_size=0.7 )
df['TotalBsmtSF']=df['TotalBsmtSF'].fillna(method='ffill' )<feature_engineering>
x_valid, x_test, y_valid, y_test = train_test_split(
 x_other, y_other, train_size=0.5 )
df['GarageCars']=df['GarageCars'].fillna(0 )<data_type_conversions>
features = labelled.drop(columns=['Survived'])
 target = labelled.Survived
df['BsmtUnfSF']=df['BsmtUnfSF'].fillna(method='ffill' )<data_type_conversions>
logistic_reg = LogisticRegressionCV(cv= 7 )
df['Electrical']=df['Electrical'].fillna(method='ffill' )<feature_engineering>
threshold = np.arange(1, 10, 0.5)*1e-1
df['BsmtFinSF1']=df['BsmtFinSF1'].fillna(df['BsmtFinSF1'].mean() )<feature_engineering>
print('The highest accuracy score is:', np.max(np.array(scores)) )
df['KitchenQual']=df['KitchenQual'].fillna(method='ffill' )<feature_engineering>
number_of_features = list(range(1,13))
df['SaleType']=df['SaleType'].fillna(method='ffill' )<feature_engineering>
print("Maximum accuracy score is :", max(scores_k))
df['Exterior2nd']=df['Exterior2nd'].fillna(method='ffill' )<drop_column>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
df=df.drop(['Id'],axis=1 )<categorify>
print("Optimal number of features : %d" % selector.n_features_ )
df=pd.get_dummies(df )<count_missing_values>
print("Maximum accuracy score is :", np.max(selector.grid_scores_))
df.isnull().sum()<prepare_x_and_y>
threshold = np.arange(1, 5, 0.1)*1e-1
train=df.iloc[:1460,:]<split>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
test=df.iloc[1460:,:] 
<prepare_x_and_y>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
X_train=train.drop(["SalePrice"],axis=1)
 y_train=train['SalePrice']<drop_column>
selector = sklearn.feature_selection.SelectFromModel(logistic_reg, threshold= 0.25)
 selector.fit(features, target)
 lr_selected_features = selector.get_support()
test=test.drop(['SalePrice'],axis=1 )<choose_model_class>
logistic_reg = LogisticRegressionCV(
 Cs=1, cv= 7, scoring='accuracy', max_iter=1000, refit=True )
XGB = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 XGB.fit(X_train,y_train )<choose_model_class>
lr_parameters_1 = {'solver': ['liblinear', 'saga'], 'penalty': ['l1']}
lr_parameters_2 = {'solver': ['newton-cg', 'lbfgs', 'sag'], 'penalty': ['l2']}
LGBM = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 LGBM.fit(X_train,y_train )<train_model>
rs_lr = RandomizedSearchCV(logistic_reg, param_distributions= lr_parameters_2, n_iter= 100 )
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 GBoost.fit(X_train,y_train )<predict_on_test>
rs_lr.fit(x_train.loc[:, lr_selected_features], y_train )
y_pred_XGB=XGB.predict(test)
 y_pred_LGBM=LGBM.predict(test)
 y_pred_GB=GBoost.predict(test )<create_dataframe>
print('Best Parameters are:
', rs_lr.best_params_,
'
Training accuracy score is:
', rs_lr.best_score_ )
y_pred_XGB=pd.DataFrame(y_pred_XGB)
 y_pred_LGBM=pd.DataFrame(y_pred_LGBM)
 y_pred_GB=pd.DataFrame(y_pred_GB )<prepare_output>
print('Validation accuracy score is:
', rs_lr.score(
 x_valid.loc[:, lr_selected_features], y_valid))
y_pred= 0.20 * y_pred_XGB[0] + 0.60 * y_pred_LGBM[0] + 0.20 * y_pred_GB[0]
y_pred<save_to_csv>
param_name = 'Cs'
param_range = [1, 10, 100, 1000]
train_score, valid_score = [], []
for cs in param_range:
lr = LogisticRegressionCV(Cs=cs, cv=7, scoring='accuracy', solver= 'newton-cg',
penalty= 'l2', refit=True, max_iter=1000)
 lr.fit(x_train.loc[:, lr_selected_features], y_train)
 train_score.append(
 lr.score(x_train.loc[:, lr_selected_features], y_train))
valid_score.append(
 lr.score(x_valid.loc[:, lr_selected_features], y_valid))
sub=pd.concat([df_test['Id'],pd.DataFrame(y_pred)],axis=1)
 sub.columns=['Id','SalePrice']
sub.to_csv('submission.csv',index=False)
 sub.head()<set_options>
lr = LogisticRegressionCV(Cs= 10, cv= 7, solver= 'newton-cg', penalty= 'l2')
 lr.fit(x_train.loc[:, lr_selected_features], y_train )
warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('seaborn')
<load_from_csv>
y_scores_lr = lr.predict_proba(x_test.loc[:, lr_selected_features])[:, 1]
lr_fpr, lr_tpr, lr_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_lr )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 features = [c for c in train.columns if c not in ['ID_code', 'target']]<sort_values>
lr_auc = sklearn.metrics.auc(x=lr_fpr, y=lr_tpr )
obs = train.isnull().sum().sort_values(ascending = False)
 percent = round(train.isnull().sum().sort_values(ascending = False)/len(train)*100, 2)
 pd.concat([obs, percent], axis = 1,keys= ['Number of Observations', 'Percent'] )<set_options>
lr_acc = lr.score(x_test.loc[:, lr_selected_features], y_test )
lin_pca = KernelPCA(n_components = 2, kernel="linear", fit_inverse_transform=True)
 rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.0433, fit_inverse_transform=True)
 sig_pca = KernelPCA(n_components = 2, kernel="sigmoid", gamma=0.001, coef0=1, fit_inverse_transform=True)
 plt.figure(figsize=(11, 4))
for subplot, pca, title in(( 131, lin_pca, "Linear kernel"),(132, rbf_pca, "RBF kernel, $\gamma=0.04$"), 
(133, sig_pca, "Sigmoid kernel, $\gamma=10^{-3}, r=1$")) :
PCA_train_x = PCA(2 ).fit_transform(train_scaled)
 plt.subplot(subplot)
 plt.title(title, fontsize=14)
 plt.scatter(PCA_train_x[:, 0], PCA_train_x[:, 1], c=target, cmap="nipy_spectral_r")
 plt.xlabel("$z_1$", fontsize=18)
 if subplot == 131:
plt.ylabel("$z_2$", fontsize=18, rotation=0)
 plt.grid(True)
 plt.show()<categorify>
print('For logistic Regression:
Area Under Curve: {},
Test Accuracy score: {}'.format(
 lr_auc, lr_acc))
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
nb = GaussianNB()
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}<prepare_x_and_y>
threshold = np.arange(1, 10, 0.5)*1e-1
num_folds = 11
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = KFold(n_splits=num_folds, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 nb.fit(selected_features, target)
 y_pred = nb.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(np.arange(1, 10, 0.5)*1e-1, np.array(scores))
num_sub = 26
print('Saving the Submission File')
 sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions
sub.to_csv('submission{}.csv'.format(num_sub), index=False)
 getValue = pd.DataFrame(getVal)
 getValue.to_csv("Validation_kfold.csv" )<load_from_csv>
print('The highest accuracy score is:', np.max(np.array(scores)) )
oof = pd.read_csv(".. /input/santander-outputs/Validation_Skfold.csv")['0']
oof_2 = pd.read_csv(".. /input/santander-outputs/Validation_kfold.csv")['0']
predictions = pd.read_csv(".. /input/santander-outputs/submission26_skfold.csv")["target"]
predictions_2 = pd.read_csv(".. /input/santander-outputs/submission26_kfold.csv")["target"]<load_from_csv>
number_of_features = list(range(1,13))
train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 features = [c for c in train.columns if c not in ['ID_code', 'target']]<drop_column>
print("Maximum accuracy score is :", max(scores_k))
target = train['target']
train = train.drop(["ID_code", "target"], axis=1 )<prepare_x_and_y>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
train_stack = np.vstack([oof,oof_2] ).transpose()
test_stack = np.vstack([predictions, predictions_2] ).transpose()
folds_stack = RepeatedKFold(n_splits=5, n_repeats=2, random_state=15)
 oof_stack = np.zeros(train_stack.shape[0])
 predictions_3 = np.zeros(test_stack.shape[0])
 for fold_,(trn_idx, val_idx)in enumerate(folds_stack.split(train_stack,target)) :
print("fold {}".format(fold_))
trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values
clf_3 = BayesianRidge()
clf_3.fit(trn_data, trn_y)
 oof_stack[val_idx] = clf_3.predict(val_data)
 predictions_3 += clf_3.predict(test_stack)/ 10<save_to_csv>
selector = sklearn.feature_selection.VarianceThreshold(threshold= 0.1)
 selector.fit(features, target)
 nb_selected_features = selector.get_support()
sample_submission = pd.read_csv('.. /input/santander-customer-transaction-prediction/sample_submission.csv')
 sample_submission['target'] = predictions_3
sample_submission.to_csv('submission_ashish.csv', index=False )<set_options>
nb_params = {'priors': [[0.7, 0.3], [0.6, 0.4],
[0.5, 0.5], [0.4, 0.6], [0.3, 0.7]]}
py.init_notebook_mode(connected=True)
 plt.style.use('ggplot')
 sns.set(font_scale=1)
 pd.set_option('display.max_columns', 500 )<load_from_csv>
rs_nb = RandomizedSearchCV(nb, param_distributions= nb_params,cv= 7 ,n_iter= 200 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
rs_nb.fit(x_train.loc[:, nb_selected_features], y_train )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
print('Best Parameters are:
', rs_nb.best_params_,
'
Training accuracy score is:
', rs_nb.best_score_ )
features = [c for c in train.columns if c not in ['ID_code', 'target']]
features_t = [c for c in test.columns if c not in ['ID_code']]
y = train.target<find_best_params>
print('Validation accuracy score is:
', rs_nb.score(
 x_valid.loc[:, nb_selected_features], y_valid))
%%time
def clusters(train,y):
distance = []
for cluster in range(2,11,1):
plt.figure(figsize=(10,7))
print('Started checking {}'.format(cluster))
kmeans = KMeans(cluster, random_state=2702)
 labels = kmeans.fit_predict(train)
 plt.title('Clusters {}'.format(cluster))
sns.countplot(labels, hue=y)
 plt.show()
distance.append(kmeans.inertia_)
 gc.collect()
return distance
train_dist = clusters(train[features],train.target )<normalization>
nb = GaussianNB(priors= [0.4, 0.6])
 nb.fit(x_train.loc[:, nb_selected_features], y_train )
sc = StandardScaler(copy=False)
 train_sc = pd.DataFrame(sc.fit_transform(train[features]), columns=features)
 test_sc = pd.DataFrame(sc.fit_transform(test[features_t]), columns=features_t)
 gc.collect()<prepare_x_and_y>
y_scores_nb = nb.predict_proba(x_test.loc[:, nb_selected_features])[:, 1]
nb_fpr, nb_tpr, nb_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_nb )
mn = MinMaxScaler(copy=False)
 train_mn = pd.DataFrame(mn.fit_transform(train[features]), columns=features)
 test_mn = pd.DataFrame(mn.fit_transform(test[features_t]), columns=features_t)
 gc.collect()<feature_engineering>
nb_auc = sklearn.metrics.auc(x=nb_fpr, y=nb_tpr )
qt = QuantileTransformer(n_quantiles = 200)
 train_qt = pd.DataFrame(mn.fit_transform(train[features]), columns=features)
 test_qt = pd.DataFrame(mn.fit_transform(test[features_t]), columns=features_t)
 gc.collect()<prepare_x_and_y>
nb_acc = nb.score(x_test.loc[:, nb_selected_features], y_test )
rs = RobustScaler(copy=False)
 train_rs = pd.DataFrame(rs.fit_transform(train[features]), columns=features)
 test_rs = pd.DataFrame(rs.fit_transform(test[features_t]), columns=features_t)
 gc.collect()<train_model>
print('For Gaussian Naive Bayes:
Area Under Curve: {},
Test Accuracy score: {}'.format(
 nb_auc, nb_acc))
X_train, X_test, y_train, y_test = train_test_split(
 train[features], y, test_size=0.3,stratify = y, random_state=2701)
 model = lgb.LGBMClassifier(
 n_estimators = 5000,
learning_rate= 0.1,
metric='auc', 
)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )<predict_on_test>
knn = KNeighborsClassifier(n_neighbors= 5 )
pred = model.predict(X_test)
 print(f'AUC: {roc_auc_score(pred, y_test)}' )<split>
threshold = [0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
X_train, X_test, y_train, y_test = train_test_split(
 train_sc[features], y, test_size=0.3,stratify = y, random_state=2701)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )<predict_on_test>
number_of_features = list(range(1,13))
pred = model.predict(X_test)
 print(f'AUC: {roc_auc_score(pred, y_test)}' )<split>
print("Maximum accuracy score is :", max(scores_k))
X_train, X_test, y_train, y_test = train_test_split(
 train_qt[features], y, test_size=0.3,stratify = y, random_state=2701)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )<split>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
X_train, X_test, y_train, y_test = train_test_split(
 train_rs[features], y, test_size=0.3,stratify = y, random_state=2701)
 model.fit(X_train, y_train)
 eli5.explain_weights(model )<predict_on_test>
selector = sklearn.feature_selection.VarianceThreshold(threshold= 0.1)
 selector.fit(features, target)
 knn_selected_features = selector.get_support()
pred = model.predict(X_test)
 print(f'AUC: {roc_auc_score(pred, y_test)}' )<init_hyperparams>
knn_params = {'n_neighbors': [5, 7, 9] , 'weights': [
'uniform', 'distance'], 'leaf_size': [5, 10, 20], 'p': [1, 2, 3]}
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}<train_model>
rs_knn = RandomizedSearchCV(knn, param_distributions= knn_params,
scoring='accuracy', cv= 7, n_iter= 200, refit=True )
num_folds = 11
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=2702)
 oof = np.zeros(len(train))
predictions = np.zeros(len(y))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, y.values)) :
print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=y.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=2000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(y, oof)) )<save_to_csv>
rs_knn.fit(x_train.loc[:, knn_selected_features], y_train )
sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions
sub.to_csv('submission.csv', index=False )<load_from_csv>
print('Best Parameters are:
', rs_knn.best_params_,
'
Training accuracy score is:
', rs_knn.best_score_ )
df1 = pd.read_csv('.. /input/sctp-blend-data/submission_1.csv')
 df2 = pd.read_csv('.. /input/sctp-blend-data/submission_2.csv')
 df3 = pd.read_csv('.. /input/sctp-blend-data/submission_3.csv')
 df4 = pd.read_csv('.. /input/sctp-blend-data/submission_4.csv' )<load_from_csv>
print('Validation accuracy score is:
', rs_knn.score(
 x_valid.loc[:, knn_selected_features], y_valid))
blend = df4['target'] *0.25 + df3['target'] * 0.25 + df2['target'] * 0.25 + df1['target'] * 0.25
sample = pd.read_csv('.. /input/santander-customer-transaction-prediction/sample_submission.csv' )<save_to_csv>
param_name = 'n_neighbors'
param_range = np.arange(3,21)
 train_score, valid_score = [], []
for k in param_range:
knn = KNeighborsClassifier(n_neighbors= k, weights= 'uniform', p= 2,leaf_size= 5)
 knn.fit(x_train.loc[:, knn_selected_features], y_train)
 train_score.append(
 knn.score(x_train.loc[:, knn_selected_features], y_train))
valid_score.append(
 knn.score(x_valid.loc[:, knn_selected_features], y_valid))
sample['target'] = blend
sample.to_csv('blend_ver10.csv',index=False )<set_options>
knn = KNeighborsClassifier(n_neighbors= 4, weights= 'uniform', p= 2, leaf_size= 5 )
warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('seaborn')
 random_state = 42
np.random.seed(random_state )<load_from_csv>
knn.fit(x_train.loc[:, knn_selected_features], y_train )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 features = [c for c in train.columns if c not in ['ID_code', 'target']]<sort_values>
y_scores_knn = knn.predict_proba(x_test.loc[:, knn_selected_features])[:, 1]
knn_fpr, knn_tpr, knn_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_knn )
obs = train.isnull().sum().sort_values(ascending = False)
 percent = round(train.isnull().sum().sort_values(ascending = False)/len(train)*100, 2)
 pd.concat([obs, percent], axis = 1,keys= ['Number of Observations', 'Percent'] )<set_options>
knn_auc = sklearn.metrics.auc(x=knn_fpr, y=knn_tpr )
lin_pca = KernelPCA(n_components = 2, kernel="linear", fit_inverse_transform=True)
 rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.0433, fit_inverse_transform=True)
 sig_pca = KernelPCA(n_components = 2, kernel="sigmoid", gamma=0.001, coef0=1, fit_inverse_transform=True)
 plt.figure(figsize=(11, 4))
for subplot, pca, title in(( 131, lin_pca, "Linear kernel"),(132, rbf_pca, "RBF kernel, $\gamma=0.04$"), 
(133, sig_pca, "Sigmoid kernel, $\gamma=10^{-3}, r=1$")) :
PCA_train_x = PCA(2 ).fit_transform(train_scaled)
 plt.subplot(subplot)
 plt.title(title, fontsize=14)
 plt.scatter(PCA_train_x[:, 0], PCA_train_x[:, 1], c=target, cmap="nipy_spectral_r")
 plt.xlabel("$z_1$", fontsize=18)
 if subplot == 131:
plt.ylabel("$z_2$", fontsize=18, rotation=0)
 plt.grid(True)
 plt.show()<categorify>
knn_acc = knn.score(x_test.loc[:, knn_selected_features], y_test )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
print('Area Under Curve: {}, Accuracy: {}'.format(knn_auc, knn_acc))
param = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 31,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 150,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state}<prepare_x_and_y>
svm = SVC(probability=True )
num_folds = 11
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
threshold = np.arange(1, 10, 0.5)*1e-1
num_sub = 34
print('Saving the Submission File')
 sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions
sub.to_csv('submission{}.csv'.format(num_sub), index=False)
 getValue = pd.DataFrame(getVal)
 getValue.to_csv("Validation.csv" )<load_from_csv>
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 svm.fit(selected_features, target)
 y_pred = svm.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(np.arange(1, 10, 0.5)*1e-1, np.array(scores))
submission = pd.read_csv(".. /input/multiple-data-nn/submission__nn__0.9005821511281362.csv" )<feature_engineering>
print('The highest accuracy score is:', np.max(np.array(scores)) )
submission['target'] = submission['target'] * 12 /8<save_to_csv>
number_of_features = list(range(1,13))
submission.to_csv('output.csv',index=False )<set_options>
print("Maximum accuracy score is :", max(scores_k))
warnings.filterwarnings('ignore' )<load_from_csv>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )<load_from_csv>
selector = sklearn.feature_selection.VarianceThreshold(threshold= 0.1)
 selector.fit(features, target)
 svm_selected_features = selector.get_support()
<create_dataframe>
svm = SVC(probability=True )
train = train_df.copy()
test = test_df.copy()<drop_column>
svm_parameters = {'kernel': ['linear', 'rbf', 'sigmoid'], 'gamma': [
'auto', 'scale'], 'shrinking': [True, False]}
del train_df
del test_df 
<categorify>
rs_svm = RandomizedSearchCV(svm, cv= 7, param_distributions= svm_parameters, n_iter= 200 )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y 
<prepare_x_and_y>
rs_svm.fit(x_train.loc[:, svm_selected_features], y_train )
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']
X_test = test[features].values<init_hyperparams>
print('Best Parameters are:
', rs_svm.best_params_,
'
Training accuracy score is:
', rs_svm.best_score_ )
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1}<prepare_x_and_y>
print('Validation accuracy score is:
', rs_svm.score(
 x_valid.loc[:, svm_selected_features], y_valid))
num_folds = 9
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = KFold(n_splits=num_folds, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
param_name = 'C'
param_range = np.arange(1,31)
 train_score, valid_score = [], []
for c in param_range:
svm = SVC(C= c,probability= True)
 svm.fit(x_train.loc[:, svm_selected_features], y_train)
 train_score.append(
 svm.score(x_train.loc[:, svm_selected_features], y_train))
valid_score.append(
 svm.score(x_valid.loc[:, svm_selected_features], y_valid))
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions['target']
sub.to_csv('submission_oof.csv', index=False )<load_from_csv>
svm = SVC(C=3, probability= True)
 svm.fit(features.loc[:, svm_selected_features], target )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']
print("Data is ready!" )<count_missing_values>
y_scores_svm = svm.predict_proba(x_test.loc[:, svm_selected_features])[:, 1]
svm_fpr, svm_tpr, svm_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_svm )
print("Missing data at training")
 train.isnull().values.any()<count_missing_values>
svm_auc = sklearn.metrics.auc(x=svm_fpr, y=svm_tpr )
print("Missing data at test")
 test.isnull().values.any()<drop_column>
svm_acc = svm.score(x_test.loc[:, svm_selected_features], y_test )
train = train.drop(["ID_code", "target"], axis=1 )<set_options>
print('For logistic Regression:
Area Under Curve: {},
Test Accuracy score: {}'.format(
 svm_auc, svm_acc))
sns.set_style('whitegrid')
 sns.countplot(target)
 sns.set_style('whitegrid' )<count_duplicates>
dt = DecisionTreeClassifier()
@jit
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
threshold = np.arange(1, 10, 0.5)*1e-1
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}<create_dataframe>
print('The highest accuracy score is:', np.max(np.array(scores)) )
num_folds = 11
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = KFold(n_splits=num_folds, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()<split>
number_of_features = list(range(1,13))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits<compute_test_metric>
print("Maximum accuracy score is :", max(scores_k))
print("
>> CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
submission = pd.DataFrame({"ID_code": test.ID_code.values})
 submission["target"] = predictions
submission.to_csv("submission.csv", index=False )<load_from_csv>
print("Optimal number of features : %d" % selector.n_features_ )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<filter>
print("Maximum accuracy score is :", np.max(selector.grid_scores_))
pos=predictors[train_df[predictors].mean() >0]
neg=predictors[train_df[predictors].mean() <=0]<feature_engineering>
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1]
idx = features = pos
for df in [train_df, test_df]:
df['sum_pos'] = df[idx].sum(axis=1)
 df['min_pos'] = df[idx].min(axis=1)
 df['max_pos'] = df[idx].max(axis=1)
 df['mean_pos'] = df[idx].mean(axis=1)
 df['std_pos'] = df[idx].std(axis=1)
 df['skew_pos'] = df[idx].skew(axis=1)
 df['kurt_pos'] = df[idx].kurtosis(axis=1)
 df['med_pos'] = df[idx].median(axis=1 )<feature_engineering>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
idx = features = neg
for df in [train_df, test_df]:
df['sum_neg'] = df[idx].sum(axis=1)
 df['min_neg'] = df[idx].min(axis=1)
 df['max_neg'] = df[idx].max(axis=1)
 df['mean_neg'] = df[idx].mean(axis=1)
 df['std_neg'] = df[idx].std(axis=1)
 df['skew_neg'] = df[idx].skew(axis=1)
 df['kurt_neg'] = df[idx].kurtosis(axis=1)
 df['med_neg'] = df[idx].median(axis=1 )<init_hyperparams>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
param = {
'num_leaves': 25,
'max_bin': 60,
'min_data_in_leaf': 5,
'learning_rate': 0.010614430970330217,
'min_sum_hessian_in_leaf': 0.0093586657313989123,
'feature_fraction': 0.056701788569420042,
'lambda_l1': 0.060222413158420585,
'lambda_l2': 4.6580550589317573,
'min_gain_to_split': 0.29588543202055562,
'max_depth': 50,
'save_binary': True,
'seed': 1234,
'feature_fraction_seed': 1234,
'bagging_seed': 1234,
'drop_seed': 1234,
'data_random_seed': 1234,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False
}<define_variables>
dt_params = {'criterion': ['gini'], 'min_samples_split': [
21, 22, 23], 'max_features': ['auto', 'log2', None]}
nfold = 10<define_variables>
rs_dt = RandomizedSearchCV(dt, param_distributions= dt_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 500 )
target = 'target'
predictors = train_df.columns.values.tolist() [2:]<split>
rs_dt.fit(x_train, y_train )
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 nround = 8000
clf = lgb.train(param, xg_train, nround, valid_sets = [xg_valid], verbose_eval=250)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=nround)
 predictions += clf.predict(test_df[predictors], num_iteration=nround)/ nfold
i = i + 1
print("
CV AUC: {:<0.4f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )<create_dataframe>
print('Best Parameters are:
', rs_dt.best_params_,
'
Training accuracy score is:
', rs_dt.best_score_ )
submission = pd.DataFrame({"ID_code": test_df.ID_code.values})
 submission["target"] = predictions
submission[:10]<save_to_csv>
print('Validation accuracy score is:
', rs_dt.score(x_valid, y_valid))
submission.to_csv("LGMB_210_featutes.csv", index=False )<import_modules>
param_name = 'max_depth'
param_range = np.arange(1, 21)
 train_score, valid_score = [], []
for depth in param_range:
dt = DecisionTreeClassifier(
 criterion='gini', max_features=None, min_samples_split=22, max_depth= depth)
 dt.fit(x_train, y_train)
 train_score.append(dt.score(x_train, y_train))
valid_score.append(dt.score(x_valid, y_valid))
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path<load_from_csv>
dt = DecisionTreeClassifier(criterion='gini', max_features=None, min_samples_split=22, max_depth= 3)
 dt.fit(features,target )
DATA_PATH = ".. /input/santander-customer-transaction-prediction/"
train = pd.read_csv(str(Path(DATA_PATH)/ "train.csv"))
test = pd.read_csv(str(Path(DATA_PATH)/ "test.csv"))
print("Train and test shapes", train.shape, test.shape )<count_values>
y_scores_dt = dt.predict_proba(x_test)[:, 1]
dt_fpr, dt_tpr, dt_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_dt)
 dt_auc = sklearn.metrics.auc(x=dt_fpr, y=dt_tpr )
train.target.value_counts()<feature_engineering>
dt_acc = dt.score(x_test, y_test )
def woe(X, y):
tmp = pd.DataFrame()
tmp["variable"] = X
tmp["target"] = y
var_counts = tmp.groupby("variable")["target"].count()
var_events = tmp.groupby("variable")["target"].sum()
var_nonevents = var_counts - var_events
tmp["var_counts"] = tmp.variable.map(var_counts)
 tmp["var_events"] = tmp.variable.map(var_events)
 tmp["var_nonevents"] = tmp.variable.map(var_nonevents)
 events = sum(tmp["target"] == 1)
 nonevents = sum(tmp["target"] == 0)
 tmp["woe"] = np.log(((tmp["var_nonevents"])/nonevents)/(( tmp["var_events"])/events))
tmp["woe"] = tmp["woe"].replace(np.inf, 0 ).replace(-np.inf, 0)
 tmp["iv"] =(tmp["var_nonevents"]/nonevents - tmp["var_events"]/events)* tmp["woe"]
iv = tmp.groupby("variable")["iv"].last().sum()
return tmp["woe"], tmp["iv"], iv<prepare_x_and_y>
print('Area Under Curve: {}, Accuracy: {}'.format(dt_auc, dt_acc))
iv_values = []
feats = ["var_{}".format(i)for i in range(200)]
y = train["target"]
for f in feats:
X = pd.qcut(train[f], 10, duplicates='drop')
 _, _, iv = woe(X, y)
 iv_values.append(iv)
 iv_inds = np.argsort(iv_values)[::-1][:50]
iv_values = np.array(iv_values)[iv_inds]
feats = np.array(feats)[iv_inds] 
<import_modules>
rf = ens.RandomForestClassifier()
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_predict
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import StandardScaler<find_best_model_class>
threshold = np.arange(1, 10, 0.5)*1e-1
feats = ["var_{}".format(i)for i in range(200)]
X = train[feats]
X_test = test[feats]
y = train["target"]
cvlist = list(StratifiedKFold(5, random_state=12345786 ).split(X, y))
scaler = StandardScaler()
X_sc = scaler.fit_transform(X)
 X_test_sc = scaler.fit_transform(X_test)
 lr = LogisticRegression()
y_preds_lr = cross_val_predict(lr, X_sc, y, cv=cvlist, method="predict_proba")[:, 1]
lr.fit(X_sc, y)
 y_test_preds_lr = lr.predict_proba(X_test_sc)[:, 1]
roc_auc_score(y, y_preds_lr )<predict_on_test>
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 rf.fit(selected_features, target)
 y_pred = rf.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(threshold, np.array(scores))
import lightgbm as lgb 
<import_modules>
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))
from scipy.stats import gmean<define_search_space>
print('The highest accuracy score is:', np.max(np.array(scores)) )
np.mean([0.9, 0.9, 0.9, 0.98, 0.9] )<define_search_space>
number_of_features = list(range(1,13))
gmean([0.9, 0.9, 0.9, 0.98, 0.9] )<install_modules>
print("Maximum accuracy score is :", max(scores_k))
!pip install -U lightgbm<train_model>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
model = lgb.LGBMClassifier(boosting_type='gbdt', n_estimators=200000, learning_rate=0.02, num_leaves=2, subsample=0.4, colsample_bytree=0.4, seed=1)
 y_preds_lgb = np.zeros(( len(y)))
 test_preds_allfolds = []
for i,(tr_idx, val_idx)in enumerate(cvlist):
X_dev, y_dev = X.iloc[tr_idx], y.iloc[tr_idx]
X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]
model.fit(X_dev, y_dev, eval_set=[(X_val, y_val)], eval_metric="auc", verbose=5000, early_stopping_rounds=1000)
 val_preds = model.predict_proba(X_val)[:, 1]
test_preds = model.predict_proba(X_test)[:, 1]
test_preds_allfolds.append(test_preds)
 y_preds_lgb[val_idx] = val_preds
print("Score for fold {} is {}".format(i, roc_auc_score(y_val, val_preds)))
 print("Overall Score for oof predictions ", roc_auc_score(y, y_preds_lgb))<save_to_csv>
print("Optimal number of features : %d" % selector.n_features_ )
sub = test[["ID_code"]]
sub["target"] = y_test_preds_lgb
sub.to_csv("submission_lgbm2_v1.csv", index=False )<compute_test_metric>
print("Maximum accuracy score is :", np.max(selector.grid_scores_))
weighted_preds = y_preds_lr* 0.05 + y_preds_lgb * 0.95
weighted_test_preds = y_test_preds_lr* 0.05 + y_test_preds_lgb * 0.95
roc_auc_score(y, weighted_preds )<load_from_csv>
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]
public_sub = pd.read_csv(".. /input/santander-lgb-new-features-rank-mean-10-folds/submission_LGBM.csv")
 public_sub.head()<prepare_output>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
sub["target"] = weighted_test_preds<save_to_csv>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
sub["target"] = 0.2*sub["target"].rank() + 0.8*public_sub["target"]
sub.to_csv("submission_blend.csv", index=False )<train_model>
rf_params = {'n_estimators': [200, 300, 400], 'criterion': ['gini'], 'min_samples_split': [
22, 20, 25], 'max_features': ['auto', 'log2', None], 'class_weight': [{0: 0.6, 1: 0.4}, {0: 0.6, 1: 0.4}, {0: 0.5, 1: 0.5}]}
concatenate, GaussianNoise, Reshape, TimeDistributed, LeakyReLU, PReLU, Embedding)
 class ROC_AUC(Callback):
def __init__(self, validation_data):
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
print("ROC AUC for this fold, is ", roc_auc_score(self.y_val, self.model.predict(X_val)))
 class NNv1(BaseEstimator, ClassifierMixin):
def __init__(self,
inp_shape=200,
gaussian_noise=0.01,
dense1_dim=32,
dense2_dim=32,
dense1_kwargs=None,
dense2_kwargs=None,
classifier_kwargs=None,
optimizer=SGD,
opt_kwargs=None, 
):
self.inp_shape = inp_shape
self.gaussian_noise = gaussian_noise
self.dense1_dim = dense1_dim
self.dense2_dim = dense2_dim
self.dense1_kwargs = dense1_kwargs
self.dense2_kwargs = dense2_kwargs
self.classifier_kwargs = classifier_kwargs
self.optimizer = optimizer
self.opt_kwargs = opt_kwargs
self._default_initiaization()
def _default_initiaization(self):
if self.dense1_kwargs is None:
self.dense1_kwargs = {"kernel_initializer": "glorot_uniform"}
if self.dense2_kwargs is None:
self.dense2_kwargs = {"kernel_initializer": "he_uniform"}
if self.classifier_kwargs is None:
self.classifier_kwargs = {"kernel_initializer": "he_uniform"}
if self.opt_kwargs is None:
self.opt_kwargs = {}
def _build_model(self):
inp = Input(shape=(self.inp_shape,))
x = Reshape(( self.inp_shape, 1))(inp)
 d1 = Dense(self.dense1_dim, activation='tanh', )(x)
 d2 = Dense(self.dense1_dim, activation='relu', )(x)
 x = concatenate([d1, d2])
 x = Flatten()(x)
 out = Dense(1, activation='sigmoid', **self.classifier_kwargs )(x)
 model = Model(inputs=inp, outputs=out)
 opt = self.optimizer(**self.opt_kwargs)
 model.compile(loss='binary_crossentropy', optimizer=opt)
 return model
def fit(self, X, y, *args, **kwargs):
self.model = self._build_model()
print(self.model.summary())
 self.model.fit(X, y, *args, **kwargs)
 return self
def predict(self, X, y=None, weight_path=None, **kwargs):
if self.model:
if weight_path is not None:
self.model.load_weights(weight_path)
 y_hat = self.model.predict(X, **kwargs)
 else:
raise ValueError("Model not fit yet")
 return y_hat 
<predict_on_test>
rs_rf = RandomizedSearchCV(rf, param_distributions= rf_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 200 )
model = NNv1(opt_kwargs = {"lr": 0.01, "momentum": 0.9, "nesterov": True, "clipnorm": 1})
 y_preds_nn = np.zeros(( len(y)))
 for tr_idx, val_idx in cvlist:
X_dev, y_dev = X_sc[tr_idx], y.iloc[tr_idx]
X_val, y_val = X_sc[val_idx], y.iloc[val_idx]
roc_auc = ROC_AUC(( X_val, y_val))
model.fit(X_dev, y_dev, validation_data=(X_val, y_val), epochs=40, batch_size=256, verbose=0, callbacks=[roc_auc])
 val_preds = model.predict(X_val, batch_size=5000)
 y_preds_nn[val_idx] = val_preds.flatten() 
<compute_test_metric>
rs_rf.fit(x_train, y_train )
roc_auc_score(y, y_preds_nn )<load_from_csv>
print('Best Parameters are:
', rs_rf.best_params_,
'
Training accuracy score is:
', rs_rf.best_score_ )
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
print('Validation accuracy score is:
', rs_rf.score(x_valid, y_valid))
!pip install -q efficientnet >> /dev/null<import_modules>
param_name = 'max_depth'
param_range = np.arange(1, 31)
 train_score, valid_score = [], []
for depth in param_range:
rf = ens.RandomForestClassifier(n_estimators= 300,
criterion='gini', max_features= 'auto', min_samples_split=22,
class_weight= {0: 0.5, 1: 0.5},max_depth= depth)
 rf.fit(x_train, y_train)
 train_score.append(rf.score(x_train, y_train))
valid_score.append(rf.score(x_valid, y_valid))
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<define_variables>
rf = ens.RandomForestClassifier(n_estimators= 300,
criterion='gini', max_features= 'auto', min_samples_split=22,
class_weight= {0: 0.5, 1: 0.5}, max_depth= 6)
 rf.fit(features,target )
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0,0,0,0,0]
INC2018 = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
y_scores_rf = rf.predict_proba(x_test)[:, 1]
rf_fpr, rf_tpr, rf_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_rf)
 rf_auc = sklearn.metrics.auc(x=rf_fpr, y=rf_tpr )
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
rf_acc = rf.score(x_test, y_test )
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
print('Area Under Curve: {}, Accuracy: {}'.format(rf_auc, rf_acc))
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
bg = ens.BaggingClassifier()
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
threshold = np.arange(1, 10, 0.5)*1e-1
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 bg.fit(selected_features, target)
 y_pred = bg.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(threshold, np.array(scores))
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
print('The highest accuracy score is:', np.max(np.array(scores)) )
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
number_of_features = list(range(1,13))
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
print("Maximum accuracy score is :", max(scores_k))
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
bg_params = {'n_estimators': [20, 25, 100], 'base_estimator': [
None, svm], 'max_features': [0.6, 0.7, 0.8], 'oob_score' : [True, False],
'max_samples': [0.6,0.7,0.8]}
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
rs_bg = RandomizedSearchCV(bg, param_distributions= bg_params,
scoring='accuracy', cv=StratifiedKFold(7), n_iter= 2000,refit=True )
!pip install tensorflow~=2.2.0 tensorflow_gcs_config~=2.2.0<install_modules>
rs_bg.fit(x_train, y_train )
!pip install -q efficientnet
!pip install pandas_summary
!pip install tensorflow-addons<import_modules>
print('Best Parameters are:
', rs_bg.best_params_,
'
Training accuracy score is:
', rs_bg.best_score_ )
import os
import re
import numpy as np
import pandas as pd
import random
import math
import matplotlib.pyplot as plt
from sklearn import metrics
from sklearn.model_selection import KFold, StratifiedKFold
import tensorflow as tf
from kaggle_datasets import KaggleDatasets
import efficientnet.tfkeras as efn
from tensorflow.keras import backend as K
import tensorflow_addons as tfa<define_variables>
print('Validation accuracy score is:
', rs_bg.score(x_valid, y_valid))
def seed_all(seed):

 random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = str(seed)
 os.environ['TF_KERAS'] = str(seed)
 tf.random.set_seed(seed)
 seed_all(42 )<train_on_grid>
bg = ens.BaggingClassifier(n_estimators= 25,
max_features= 0.8, base_estimator= svm, oob_score= True, max_samples= 0.8)
 bg.fit(features,target )
DEVICE = 'TPU'
MIXED_PRECISION = True
XLA_ACCELERATE = True
if DEVICE == 'TPU':
print('Connecting to TPU...')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print('Could not connect to TPU')
 tpu = None
if tpu:
try:
print('initializing TPU...')
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('TPU initialized')
 if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
 except _:
print('failed to initialize TPU')
 else:
DEVICE = 'GPU'
if DEVICE != 'TPU':
print('Using default strategy for CPU and single GPU')
 strategy = tf.distribute.get_strategy()
if DEVICE == 'GPU':
print('Num GPUs Available: ', len(tf.config.experimental.list_physical_devices('GPU')))
 if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<init_hyperparams>
y_scores_bg = bg.predict_proba(x_test)[:, 1]
bg_fpr, bg_tpr, bg_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_bg)
 bg_auc = sklearn.metrics.auc(x=bg_fpr, y=bg_tpr )
CFG = dict(
 epochs = 30,
batch_size = 128,
lr = 0.00032,
inp_size = 256,
eff_B = 0,
sprinkles_mode = 'normal',
sprinkles_prob = 1,
num_holes = 5,
side_length = 256//10 
)
<load_from_csv>
bg_acc = bg.score(x_test, y_test )
BASEPATH = '.. /input/siim-isic-melanoma-classification'
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(CFG['inp_size'],CFG['inp_size']))
GCS_PATHE = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(CFG['inp_size'],CFG['inp_size']))
training_files = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 training_files_2018 = tf.io.gfile.glob(GCS_PATHE + '/train%.2i*.tfrec'%(2*x)for x in range(15))
training_files += training_files_2018
test_files = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec' )<categorify>
print('Area Under Curve: {}, Accuracy: {}'.format(bg_auc, bg_acc))
def make_mask(num_holes,side_length,rows, cols, num_channels):

 row_range = tf.tile(tf.range(rows)[..., tf.newaxis], [1, num_holes])
 col_range = tf.tile(tf.range(cols)[..., tf.newaxis], [1, num_holes])
 r_idx = tf.random.uniform([num_holes], minval=0, maxval=rows-1,
dtype=tf.int32)
 c_idx = tf.random.uniform([num_holes], minval=0, maxval=cols-1,
dtype=tf.int32)
 r1 = tf.clip_by_value(r_idx - side_length // 2, 0, rows)
 r2 = tf.clip_by_value(r_idx + side_length // 2, 0, rows)
 c1 = tf.clip_by_value(c_idx - side_length // 2, 0, cols)
 c2 = tf.clip_by_value(c_idx + side_length // 2, 0, cols)
 row_mask =(row_range > r1)&(row_range < r2)
 col_mask =(col_range > c1)&(col_range < c2)
 mask = row_mask[:, tf.newaxis] & col_mask
mask = tf.reduce_any(mask, axis=-1)
 mask = mask[..., tf.newaxis]
mask = tf.tile(mask, [1, 1, num_channels])
 return mask
def sprinkles(image, cfg = CFG):

 num_holes = cfg['num_holes']
side_length = cfg['side_length']
mode = cfg['sprinkles_mode']
PROBABILITY = cfg['sprinkles_prob']
RandProb = tf.cast(tf.random.uniform([],0,1)< PROBABILITY, tf.int32)
 if(RandProb == 0)|(num_holes == 0): return image
img_shape = tf.shape(image)
 if mode is 'normal':
rejected = tf.zeros_like(image)
 elif mode is 'salt_pepper':
num_holes = num_holes // 2
rejected_high = tf.ones_like(image)
 rejected_low = tf.zeros_like(image)
 elif mode is 'gaussian':
rejected = tf.random.normal(img_shape, dtype=tf.float32)
 else:
raise ValueError(f'Unknown mode "{mode}" given.')
 rows = img_shape[0]
cols = img_shape[1]
num_channels = img_shape[-1]
if mode is 'salt_pepper':
mask1 = make_mask(num_holes,side_length,rows, cols, num_channels)
 mask2 = make_mask(num_holes,side_length,rows, cols, num_channels)
 filtered_image = tf.where(mask1, rejected_high, image)
 filtered_image = tf.where(mask2, rejected_low, filtered_image)
 else:
mask = make_mask(num_holes,side_length,rows, cols, num_channels)
 filtered_image = tf.where(mask, rejected, image)
 return filtered_image<normalization>
ada = ens.AdaBoostClassifier()
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))<categorify>
threshold = np.arange(1, 10, 0.5)*1e-1
def transform(image, label):
DIM = CFG['inp_size']
XDIM = DIM%2
if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
rot = 15.* tf.random.normal([1],dtype='float32')
 else:
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['img_inp'],tf.transpose(idx3))
return {'img_inp': tf.reshape(d,[DIM,DIM,3]), 'meta_inp': image['meta_inp']}, label<categorify>
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 ada.fit(selected_features, target)
 y_pred = ada.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(threshold, np.array(scores))
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [CFG['inp_size'], CFG['inp_size'], 3])
 return image<normalization>
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))
def data_augment(data, label):
data['img_inp'] = tf.image.random_flip_left_right(data['img_inp'])
 data['img_inp'] = tf.image.random_flip_up_down(data['img_inp'])
 data['img_inp'] = tf.image.random_hue(data['img_inp'], 0.01)
 data['img_inp'] = tf.image.random_saturation(data['img_inp'], 0.7, 1.3)
 data['img_inp'] = tf.image.random_contrast(data['img_inp'], 0.8, 1.2)
 data['img_inp'] = tf.image.random_brightness(data['img_inp'], 0.1)
 data['img_inp'] = sprinkles(data['img_inp'])
 return data, label<categorify>
print('The highest accuracy score is:', np.max(np.array(scores)) )
def read_labeled_tfrecord(example):

 LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data<feature_engineering>
number_of_features = list(range(1,13))
def read_unlabeled_tfrecord(example):

 UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data<categorify>
print("Maximum accuracy score is :", max(scores_k))
def read_complete_tfrecord(example):

 LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data<create_dataframe>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def load_complete_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_complete_tfrecord, num_parallel_calls = AUTO)
 return dataset<categorify>
print("Optimal number of features : %d" % selector.n_features_ )
def training_input(image, label, data):

 anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'img_inp': image, 'meta_inp': tabular}, label<categorify>
print("Maximum accuracy score is :", np.max(selector.grid_scores_))
def test_input(image, image_name, data):

 anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'img_inp': image, 'meta_inp': tabular}, image_name 
<categorify>
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]
def validation_input(image, image_name, target, data):

 anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'img_inp': image, 'meta_inp': tabular}, image_name, target<categorify>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
def get_training_dataset(filenames, labeled = True, ordered = False):

 dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(training_input, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):

 dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(training_input, num_parallel_calls = AUTO)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):

 dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(test_input, num_parallel_calls = AUTO)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_complete_dataset(filenames):
dataset = load_complete_dataset(filenames)
 dataset = dataset.map(validation_input, num_parallel_calls = AUTO)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
NUM_TRAINING_IMAGES = int(count_data_items(training_files)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(training_files)* 0.2)
 NUM_TEST_IMAGES = count_data_items(test_files)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // CFG['batch_size']
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
ada_params = {'n_estimators': [90, 100, 110], 'base_estimator': [None, svm],
'learning_rate': [0.09 ,0.1, 0.11]}
class IncreaseSprinklesHoles(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):

 if epoch <= 10:
CFG['num_holes'] = epoch + 5
CFG['side_length'] =(CFG['inp_size'] // 10)+(epoch // 2)
 if epoch >= 10:
CFG['num_holes'] = epoch + 5
CFG['side_length'] =(CFG['inp_size'] // 5)+(epoch // 2)
 if epoch >= 15:
CFG['num_holes'] = epoch + 5
CFG['side_length'] =(CFG['inp_size'] // 2)+(epoch // 2)
 sprinkles_cb = IncreaseSprinklesHoles()<choose_model_class>
rs_ada = RandomizedSearchCV(ada, param_distributions= ada_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 500 )
def get_model() :

 with strategy.scope() :
img_inp = tf.keras.layers.Input(shape =(CFG['inp_size'], CFG['inp_size'], 3), name = 'img_inp')
 meta_inp = tf.keras.layers.Input(shape =(9), name = 'meta_inp')
 effs = [0,1,2,3,4,5,6,7]
eff = effs[CFG['eff_B']]
constructor = getattr(efn, f'EfficientNetB{eff}')
 efnetb = constructor(weights = 'noisy-student', include_top = False)
 pt_depth = efnetb.get_output_shape_at(0)[-1]
pt_features = efnetb(img_inp)
 bn_features = tf.keras.layers.BatchNormalization()(pt_features)
 attn_layer = tf.keras.layers.Conv2D(64, kernel_size =(1, 1), padding = 'same', activation = 'swish' )(tf.keras.layers.Dropout(0.5 )(bn_features))
attn_layer = tf.keras.layers.Conv2D(16, kernel_size =(1, 1), padding = 'same', activation = 'swish' )(attn_layer)
 attn_layer = tf.keras.layers.Conv2D(8, kernel_size =(1,1), padding = 'same', activation = 'swish' )(attn_layer)
 attn_layer = tf.keras.layers.Conv2D(1, kernel_size =(1, 1), padding = 'valid', activation = 'sigmoid' )(attn_layer)
 up_c2_w = np.ones(( 1, 1, 1, pt_depth))
up_c2 = tf.keras.layers.Conv2D(pt_depth, kernel_size =(1, 1), padding = 'same', activation = 'linear', use_bias = False, weights = [up_c2_w])
 up_c2.trainable = False
attn_layer = up_c2(attn_layer)
 mask_features = tf.keras.layers.multiply([attn_layer, bn_features])
 gap_features = tf.keras.layers.GlobalAveragePooling2D()(mask_features)
 gap_mask = tf.keras.layers.GlobalAveragePooling2D()(attn_layer)
 gap = tf.keras.layers.Lambda(lambda x: x[0] / x[1], name = 'RescaleGAP' )([gap_features, gap_mask])
 gap_dr = tf.keras.layers.Dropout(0.5 )(gap)
 dr_steps = tf.keras.layers.Dropout(0.25 )(tf.keras.layers.Dense(128, activation = 'swish' )(gap_dr))
meta_layer = tf.keras.layers.Dense(16 )(meta_inp)
 meta_layer = tf.keras.layers.BatchNormalization()(meta_layer)
 meta_layer = tf.keras.layers.Activation('swish' )(meta_layer)
 meta_layer = tf.keras.layers.Dropout(0.2 )(meta_layer)
 meta_layer = tf.keras.layers.Dense(8 )(meta_inp)
 meta_layer = tf.keras.layers.BatchNormalization()(meta_layer)
 meta_layer = tf.keras.layers.Activation('swish' )(meta_layer)
 meta_layer = tf.keras.layers.Dropout(0.2 )(meta_layer)
 concat = tf.keras.layers.concatenate([dr_steps, meta_layer])
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dense(512, activation = 'swish' )(concat)
 concat = tf.keras.layers.Dropout(0.15 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid',dtype='float32' )(concat)
 model = tf.keras.models.Model(inputs = [img_inp, meta_inp], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = CFG['lr'])
 model.compile(
 optimizer = opt,
loss = [tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO,gamma = 2.0, alpha = 0.90)],
metrics = [tf.keras.metrics.AUC() ] 
)
 return model
def train_validate_predict(df_sub, folds = 3):

 models = []
oof_image_name = []
oof_target = []
oof_prediction = []
kfold = KFold(folds, shuffle = True, random_state = 42)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(training_files)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 CFG['num_holes'] = 5
CFG['side_length'] = CFG['inp_size']//10
train_dataset = get_training_dataset([training_files[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([training_files[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 2, min_delta = 0.00025, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.6, patience = 2 , verbose = 2, min_delta = 0.0005, min_lr=0.000001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = CFG['epochs'],
callbacks = [sprinkles_cb, cb_lr_schedule, early_stopping],
validation_data = val_dataset,
verbose = 1)
 models.append(model)
 number_of_files = count_data_items([training_files[x] for x in val_ind])
 dataset = get_complete_dataset([training_files[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('OOF_EfficientNetB%i_%i.csv'%(CFG['eff_B'],CFG['inp_size']), index = False)
 test_ds = get_test_dataset(test_files, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 df_sub.drop('target', inplace = True, axis = 1)
 df_sub = df_sub.merge(pred_df, on = 'image_name')
 df_sub.to_csv('sub_EfficientNetB%i_%i.csv'%(CFG['eff_B'],CFG['inp_size']), index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_validate_predict(df_sub )<compute_test_metric>
rs_ada.fit(x_train, y_train )
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Final OOF Roc Auc Score: ', roc_auc )<load_from_csv>
print('Best Parameters are:
', rs_ada.best_params_,
'
Training accuracy score is:
', rs_ada.best_score_ )
treino = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<load_from_csv>
print('Validation accuracy score is:
', rs_ada.score(x_valid, y_valid))
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)<count_values>
ada = ens.AdaBoostClassifier(n_estimators= 110, learning_rate= 0.09)
 ada.fit(features, target )
print(f'Number of Rows: {train_df.shape[0]}')
 print(f'Number of Cols: {train_df.shape[1]}' )<data_type_conversions>
y_scores_ada = ada.predict_proba(x_test)[:, 1]
ada_fpr, ada_tpr, ada_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_ada)
 ada_auc = sklearn.metrics.auc(x=ada_fpr, y=ada_tpr )
train_df = train_df.astype(data_types_dict )<count_missing_values>
ada_acc = ada.score(x_test, y_test )
train_df.isna().sum()<feature_engineering>
print('Area Under Curve: {}, Accuracy: {}'.format(ada_auc, ada_acc))
treino = treino.loc[treino.content_type_id == False]
treino['tempo_exercicio'] = pd.DataFrame(treino.prior_question_elapsed_time.shift(-1))
treino.loc[treino.timestamp == 0, ['prior_question_had_explanation']] = treino.loc[treino.timestamp == 0, ['prior_question_had_explanation']].fillna(False)
 treino = treino.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
<merge>
gb = ens.GradientBoostingClassifier()
treino = pd.merge(treino, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 treino.part = treino.part - 1<drop_column>
threshold = np.arange(1, 10, 0.5)*1e-1
treino.drop(['timestamp', 'content_type_id','question_id'], axis=1, inplace=True )<groupby>
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 gb.fit(selected_features, target)
 y_pred = gb.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(threshold, np.array(scores))
tempo_medio_estudantef = treino[['user_id','tempo_exercicio']].groupby(['user_id'] ).agg(['mean'])
 tempo_medio_estudantef.columns = ['tempo_medio_estudante']
tempo_medio_exerciciof = treino[['content_id','tempo_exercicio']].groupby(['content_id'] ).agg(['mean'])
 tempo_medio_exerciciof.columns = ['tempo_medio_exercicio']
tempo_medio_tipo_exerciciof = treino[['tempo_exercicio','part']].groupby(['part'] ).agg(['mean'])
 tempo_medio_tipo_exerciciof.columns = ['tempo_medio_tipo_exercicio']
acerto_medio_estudantef = treino[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'count'])
 acerto_medio_estudantef.columns = ['acerto_medio_estudante', 'count']
media_numero_exerciciof = acerto_medio_estudantef[['acerto_medio_estudante','count']].groupby(['count'] ).agg(['mean'])
 media_numero_exerciciof.columns = ["media_numero_exercicio"]
acerto_medio_exerciciof = treino[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 acerto_medio_exerciciof.columns = ["acerto_medio_exercicio"]
acerto_medio_tipo_exerciciof = treino[['part','answered_correctly']].groupby(['part'] ).agg(['mean'])
 acerto_medio_tipo_exerciciof.columns = ["acerto_medio_tipo_exercicio"]
treino = treino[~treino['prior_question_had_explanation'].isna() ]
viu_medio_estudantef = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['user_id'] ).agg(['mean','count'])
 viu_medio_estudantef.columns = ['viu_medio_estudante','count_had']
media_numero_viuf = viu_medio_estudantef[['viu_medio_estudante','count_had']].groupby(['count_had'] ).agg(['mean'])
 media_numero_viuf.columns = ["media_numero_viu"]
viu_medio_exerciciof = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['content_id'] ).agg(['mean'])
 viu_medio_exerciciof.columns = ["viu_medio_exercicio"]<create_dataframe>
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))
validation = pd.DataFrame()<remove_duplicates>
print('The highest accuracy score is:', np.max(np.array(scores)) )
for i in range(4):
last_records = treino.drop_duplicates('user_id', keep = 'last')
 treino = treino[~treino.index.isin(last_records.index)]
validation = validation.append(last_records)
 del(last_records )<drop_column>
number_of_features = list(range(1,13))
validation.drop(['tempo_exercicio'], axis=1, inplace=True )<create_dataframe>
print("Maximum accuracy score is :", max(scores_k))
X = pd.DataFrame()
for i in range(15):
last_records = treino.drop_duplicates('user_id', keep = 'last')
 treino = treino[~treino.index.isin(last_records.index)]
X = X.append(last_records)
 del(last_records )<groupby>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
tempo_medio_estudante = treino[['user_id','tempo_exercicio']].groupby(['user_id'] ).agg(['mean'])
 tempo_medio_estudante.columns = ['tempo_medio_estudante']
tempo_medio_exercicio = treino[['content_id','tempo_exercicio']].groupby(['content_id'] ).agg(['mean'])
 tempo_medio_exercicio.columns = ['tempo_medio_exercicio']
tempo_medio_tipo_exercicio = treino[['tempo_exercicio','part']].groupby(['part'] ).agg(['mean'])
 tempo_medio_tipo_exercicio.columns = ['tempo_medio_tipo_exercicio']
acerto_medio_estudante = treino[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'count'])
 acerto_medio_estudante.columns = ['acerto_medio_estudante', 'count']
media_numero_exercicio = acerto_medio_estudante[['acerto_medio_estudante','count']].groupby(['count'] ).agg(['mean'])
 media_numero_exercicio.columns = ["media_numero_exercicio"]
acerto_medio_exercicio = treino[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 acerto_medio_exercicio.columns = ["acerto_medio_exercicio"]
acerto_medio_tipo_exercicio = treino[['part','answered_correctly']].groupby(['part'] ).agg(['mean'])
 acerto_medio_tipo_exercicio.columns = ["acerto_medio_tipo_exercicio"]
treino = treino[~treino['prior_question_had_explanation'].isna() ]
viu_medio_estudante = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['user_id'] ).agg(['mean','count'])
 viu_medio_estudante.columns = ['viu_medio_estudante','count_had']
media_numero_viu = viu_medio_estudante[['viu_medio_estudante','count_had']].groupby(['count_had'] ).agg(['mean'])
 media_numero_viu.columns = ["media_numero_viu"]
viu_medio_exercicio = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['content_id'] ).agg(['mean'])
 viu_medio_exercicio.columns = ["viu_medio_exercicio"]
del(treino )<merge>
print("Optimal number of features : %d" % selector.n_features_ )
X = pd.merge(X, acerto_medio_estudante, on=['user_id'], how="left")
 X = pd.merge(X, tempo_medio_estudante, on=['user_id'], how="left")
 X = pd.merge(X, acerto_medio_exercicio, on=['content_id'], how="left")
 X = pd.merge(X, tempo_medio_exercicio, on=['content_id'], how="left")
 X = pd.merge(X, acerto_medio_tipo_exercicio, on=['part'], how="left")
 X = pd.merge(X, tempo_medio_tipo_exercicio, on=['part'], how="left")
 X = pd.merge(X, viu_medio_estudante, on=['user_id'], how="left")
 X = pd.merge(X, viu_medio_exercicio, on=['content_id'], how="left")
 X = pd.merge(X, media_numero_exercicio, on=['count'], how="left")
 X = pd.merge(X, media_numero_viu, on=['count_had'], how="left")
 X<merge>
print("Maximum accuracy score is :", np.max(selector.grid_scores_))
validation = pd.merge(validation, acerto_medio_estudante, on=['user_id'], how="left")
 validation = pd.merge(validation, tempo_medio_estudante, on=['user_id'], how="left")
 validation = pd.merge(validation, acerto_medio_exercicio, on=['content_id'], how="left")
 validation = pd.merge(validation, tempo_medio_exercicio, on=['content_id'], how="left")
 validation = pd.merge(validation, acerto_medio_tipo_exercicio, on=['part'], how="left")
 validation = pd.merge(validation, tempo_medio_tipo_exercicio, on=['part'], how="left")
 validation = pd.merge(validation, viu_medio_estudante, on=['user_id'], how="left")
 validation = pd.merge(validation, viu_medio_exercicio, on=['content_id'], how="left")
 validation = pd.merge(validation, media_numero_exercicio, on=['count'], how="left")
 validation = pd.merge(validation, media_numero_viu, on=['count_had'], how="left")
 validation<categorify>
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<count_missing_values>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
X.isna().sum()<prepare_x_and_y>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<data_type_conversions>
selector = sklearn.feature_selection.SelectKBest(k= 11)
 selector.fit(features, target)
 gb_selected_features = selector.get_support()
X['acerto_medio_estudante'].fillna(acerto_medio_estudante.acerto_medio_estudante.mean() ,inplace=True)
 X['acerto_medio_exercicio'].fillna(acerto_medio_exercicio.acerto_medio_exercicio.mean() ,inplace=True)
 X['tempo_medio_estudante'].fillna(tempo_medio_estudante.tempo_medio_estudante.mean() ,inplace = True)
 X['tempo_medio_exercicio'].fillna(tempo_medio_exercicio.tempo_medio_exercicio.mean() ,inplace = True)
 X['viu_medio_exercicio'].fillna(viu_medio_exercicio.viu_medio_exercicio.mean() ,inplace=True)
 X_val['acerto_medio_estudante'].fillna(acerto_medio_estudante.acerto_medio_estudante.mean() ,inplace=True)
 X_val['acerto_medio_exercicio'].fillna(acerto_medio_exercicio.acerto_medio_exercicio.mean() ,inplace=True)
 X_val['tempo_medio_estudante'].fillna(tempo_medio_estudante.tempo_medio_estudante.mean() ,inplace = True)
 X_val['tempo_medio_exercicio'].fillna(tempo_medio_exercicio.tempo_medio_exercicio.mean() ,inplace = True)
 X_val['viu_medio_exercicio'].fillna(viu_medio_exercicio.viu_medio_exercicio.mean() ,inplace=True )<train_model>
gb_params = {'n_estimators': [150, 160, 170], 'loss': ['deviance', 'exponential'],
'subsample': [0.7, 0.8, 0.9], 'max_features': ['auto', 'log2', None]}
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
lgb_train = lgb.Dataset(P, yp)
 lgb_eval = lgb.Dataset(P_val, yp_val, reference=lgb_train )<train_model>
rs_gb = RandomizedSearchCV(gb, param_distributions= gb_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 2000 )
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=10000,
early_stopping_rounds=12
)<predict_on_test>
rs_gb.fit(x_train.loc[:,gb_selected_features], y_train )
y_pred = model.predict(P_val)
 y_true = np.array(yp_val)
 y_predc = model.predict(P)
 y_truec = np.array(yp )<compute_test_metric>
print('Best Parameters are:
', rs_gb.best_params_,
'
Training accuracy score is:
', rs_gb.best_score_ )
confusion_matrix(y_true, y_pred.round() )<compute_test_metric>
print('Validation accuracy score is:
',
rs_gb.score(x_valid.loc[:,gb_selected_features], y_valid))
print(roc_auc_score(y_true, y_pred))
print(classification_report(y_true, y_pred.round()))
print(roc_auc_score(y_truec, y_predc))
print(classification_report(y_truec, y_predc.round()))<import_modules>
param_name = 'max_depth'
param_range = np.arange(1, 31)
 train_score, valid_score = [], []
for depth in param_range:
gb = ens.GradientBoostingClassifier(n_estimators= 170,
subsample= 0.9, max_features= 'auto', loss= 'exponential',max_depth= depth)
 gb.fit(x_train.loc[:,gb_selected_features], y_train)
 train_score.append(gb.score(x_train.loc[:,gb_selected_features], y_train))
valid_score.append(gb.score(x_valid.loc[:,gb_selected_features], y_valid))
import matplotlib.pyplot as plt
import seaborn as sns<set_options>
gb = ens.GradientBoostingClassifier(n_estimators= 170, subsample= 0.9, max_features= 'auto',
loss= 'exponential',max_depth= 4)
 gb.fit(features.loc[:, gb_selected_features],target )
warnings.filterwarnings('ignore' )<import_modules>
y_scores_gb = gb.predict_proba(x_test.loc[:, gb_selected_features])[:, 1]
gb_fpr, gb_tpr, gb_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_gb)
 gb_auc = sklearn.metrics.auc(x=gb_fpr, y=gb_tpr )
import pickle
import riiideducation
from tqdm import tqdm
import glob
import pandas as pd
import numpy as np
import plotly.graph_objects as go<define_variables>
gb_acc = gb.score(x_test.loc[:, gb_selected_features], y_test )
metrics = [
'questions_part',
'questions_difficulty',
'questions_count',
'questions_good_answer_reaction_time',
'questions_bad_answer_reaction_time',
'questions_batch_size',
'questions_variance',
'questions_difficulty_given_prev_explanation',
'questions_difficulty_given_prev_no_explanation',
'questions_conditionnal_proba_cluster',
'lectures_count',
'student_general_elo_theta',
'student_general_elo_proba',
'student_part_elo_proba',
'student_tags_elo_theta',
'student_tags_elo_proba',
'student_part_score',
'timestamp_student_general_last_batch_average',
'timestamp_student_general_reactivity_rolling_19',
'timestamp_student_general_part_reactivity_average',
'timestamp_student_general_reactivity_good_answer_average',
'timestamp_last_delta_between_questions',
'timestamp_delta_between_questions_rolling_10',
'timestamp_delta_between_questions_rolling_5',
'timestamp_last_time_between_answers',
'student_general_good_answers_average',
'student_general_good_answers_rolling_2',
'student_general_good_answers_rolling_6',
'student_part_good_answers_rolling_6',
'questions_seen_with_explanation',
'questions_conditionnal_count',
'questions_conditionnal_proba',
'lectures_conditionnal_proba',
]<load_pretrained>
print('Area Under Curve: {}, Accuracy: {}'.format(gb_auc, gb_acc))
with open('.. /input/lgb-training/model.pkl', 'rb')as f:
lgb = pickle.load(f )<load_from_csv>
xgboost = xgb.XGBClassifier()
train = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', nrows = 1000000, index_col = 0)
 questions = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('.. /input/riiid-test-answer-prediction/lectures.csv' )<groupby>
threshold = np.arange(1, 10, 0.5)*1e-2
questions_to_difficulty =(1-train[train.content_type_id == False].groupby('content_id')['answered_correctly'].mean() )<groupby>
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))
subtrain = train[train.content_type_id ==0]
subtrain['shift_elapse_time'] = subtrain['prior_question_elapsed_time'].shift(-1)
 good_answer_reaction_time = subtrain[subtrain.answered_correctly==True].dropna().groupby('content_id')['shift_elapse_time'].mean()
bad_answer_reaction_time = subtrain[subtrain.answered_correctly==False].dropna().groupby('content_id')['shift_elapse_time'].mean()<count_unique_values>
print('The highest accuracy score is:', np.max(np.array(scores)) )
( unique, counts)= np.unique(train.values[:,1], return_counts=True)
 qids = pd.Series(counts, index = unique ).sort_values(ascending=False)[:500]
qids.head()<filter>
number_of_features = list(range(1,13))
qids = qids.index<define_variables>
print("Maximum accuracy score is :", max(scores_k))
memory = 100
train_arr = train[['user_id', 'content_id', 'content_type_id', 'answered_correctly']].values
user_meta = {}
user_meta_count = {}
meta_proba_pos = {}
meta_count_pos = {}
meta_proba_neg = {}
meta_count_neg = {}
for i in tqdm(range(len(train))):
user_id, content_id, content_type_id, answered_correctly = train_arr[i]
if user_id not in user_meta.keys() :
user_meta = {}
user_meta_count = {}
user_meta[user_id] = {qid:0 for qid in qids}
user_meta_count[user_id] = {}
if content_type_id ==0:
A = content_id
X = np.array(list(user_meta[user_id].values()))
if A not in meta_proba_pos.keys() :
meta_proba_pos[A] = {i:0 for i in range(len(qids)) }
meta_count_pos[A] = {i:0 for i in range(len(qids)) }
meta_proba_neg[A] = {i:0 for i in range(len(qids)) }
meta_count_neg[A] = {i:0 for i in range(len(qids)) }
for B in np.where(X!=0)[0]:
if(X[B]==1):
if answered_correctly == 1:
meta_proba_pos[A][B] += 1
meta_count_pos[A][B] += 1
if(X[B]==-1):
if answered_correctly == 1:
meta_proba_neg[A][B] += 1
meta_count_neg[A][B] += 1
if A in qids:
for k,v in user_meta_count[user_id].copy().items() :
user_meta_count[user_id][k] = v-1
if v == 0:
del user_meta_count[user_id][k]
user_meta[user_id][k] = 0
user_meta[user_id][content_id] = 2*answered_correctly-1
if not A in user_meta_count[user_id].keys() :
user_meta_count[user_id][A] = memory 
<load_pretrained>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
with open('.. /input/riid-raw-samples/bad_answer_reaction_time', 'rb')as f:
bad_answer_reaction_time = pickle.load(f)
 with open('.. /input/riid-raw-samples/good_answer_reaction_time', 'rb')as f:
good_answer_reaction_time = pickle.load(f)
 with open('.. /input/riid-raw-samples/question_user_count', 'rb')as f:
question_user_count = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty', 'rb')as f:
questions_to_difficulty = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty', 'rb')as f:
questions_to_difficulty = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty_variance', 'rb')as f:
questions_to_difficulty_variance = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty_with_explanation', 'rb')as f:
questions_to_difficulty_with_explanation = pickle.load(f)
 with open('.. /input/riid-raw-samples/questions_to_difficulty_without_explanation', 'rb')as f:
questions_to_difficulty_without_explanation = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_count.pkl', 'rb')as f:
meta_count = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_count_neg.pkl', 'rb')as f:
meta_count_neg = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_proba.pkl', 'rb')as f:
meta_proba = pickle.load(f)
 with open('.. /input/riid-probabilities-2/meta_proba_neg.pkl', 'rb')as f:
meta_proba_neg = pickle.load(f)
 with open('.. /input/riid-probabilities/question_position.pkl', 'rb')as f:
question_position = pickle.load(f)
 with open('.. /input/riid-lectures/lecture_position.pkl', 'rb')as f:
lecture_position = pickle.load(f)
 with open('.. /input/riid-lectures/meta_count_lect.pkl', 'rb')as f:
meta_count_lect = pickle.load(f)
 with open('.. /input/riid-lectures/meta_proba_lect.pkl', 'rb')as f:
meta_proba_lect = pickle.load(f)
 clust = pd.read_csv('.. /input/riid-cluster/test_cluster_4',index_col=0)['0']
with open('.. /input/riid-cache-questions/cache_questions.pck', 'rb')as f:
question_cache = pickle.load(f )<define_variables>
print("Optimal number of features : %d" % selector.n_features_ )
proba_pos = {}
proba_neg = {}
THRESH_COUNT = 50
THRESH_PROBA = 0.01
for A,v in tqdm(meta_proba.items()):
proba_pos[A] = {}
proba_neg[A] = {}
for B in v.keys() :
if meta_count[A][B]>THRESH_COUNT:
proba = meta_proba[A][B]/meta_count[A][B]
improve = proba -(1-questions_to_difficulty[A])
 if improve > THRESH_PROBA:
proba_pos[A][question_position[B]] = improve
if meta_count_neg[A][B]>THRESH_COUNT:
proba = meta_proba_neg[A][B]/meta_count_neg[A][B]
decrease = proba -(1-questions_to_difficulty[A])
 if decrease < -THRESH_PROBA:
proba_neg[A][question_position[B]] = decrease
proba_lec = {}
thresh = 20
for k,v in tqdm(meta_proba_lect.items()):
proba_lec[k] = {}
for kk,vv in v.items() :
if meta_count_lect[k][kk]>thresh:
proba = meta_proba_lect[k][kk]/meta_count_lect[k][kk]
improve = proba -(1-questions_to_difficulty[k])
 if improve > 0.10:
proba_lec[k][lecture_position[kk]] = improve<categorify>
print("Maximum accuracy score is :", np.max(selector.grid_scores_))
all_prob = {}
for k,v in proba_pos.items() :
all_prob[k] = {}
for kk,vv in v.items() :
all_prob[k][(kk,1)] = vv
for k,v in proba_neg.items() :
if k not in all_prob.keys() :
all_prob[k] = {}
for kk,vv in v.items() :
all_prob[k][(kk,0)] = vv
all_q = set()
for k,v in all_prob.items() :
for kk in v.keys() :
all_q.add(kk[0])
 mprob1 = {}
for k,v in tqdm(all_prob.items()):
mprob1[k] = {}
kk = [e[0] for e in v.keys() ]
for elmt in all_q:
mprob1[k][elmt]=0
if elmt in kk:
if(elmt,1)in v.keys() :
mprob1[k][elmt]=v[(elmt,1)]
else:
mprob1[k][elmt]=v[(elmt,0)]<prepare_output>
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]
mprob = pd.DataFrame(mprob1 ).T
mprob.head()<predict_on_test>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
kmean = KMeans(n_clusters=20)
 c = kmean.fit_predict(mprob )<define_variables>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
ids = glob.glob('.. /input/riid-cache-6/content/drive/My Drive/riid/*')
 ids = [int(elmt.split('/')[-1])for elmt in ids]<categorify>
xgboost = xgb.XGBClassifier()
xgboost.fit(features, target )
questions_to_tag = questions[['question_id','tags']].set_index('question_id')[['tags']]['tags'].fillna('-1' ).apply(lambda x: [np.int64(elmt)for elmt in x.split(' ')] ).to_dict()
questions_to_parts = questions[['question_id','part']].set_index('question_id')['part'].to_dict()
lecture_to_type = lectures[['lecture_id','type_of']].set_index('lecture_id')['type_of'].to_dict()
lecture_to_part = lectures[['lecture_id','part']].set_index('lecture_id')['part'].to_dict()
lecture_to_tag = lectures[['lecture_id','tag']].set_index('lecture_id')['tag'].to_dict()
tags = []
for k,v in questions_to_tag.items() :
tags+= v
tags = {elmt:np.int32(0)for elmt in set(tags)}
parts = np.unique(list(questions_to_parts.values()))
question_list = list(questions_to_difficulty.keys() )<compute_test_metric>
y_scores_xgb = xgboost.predict_proba(x_test)[:, 1]
xgb_fpr, xgb_tpr, xgb_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_xgb)
 xgb_auc = sklearn.metrics.auc(x=xgb_fpr, y=xgb_tpr )
def qscore(answer, difficulty):

 if answer>0:
return(2*answer - 1)*difficulty
else:
return(2*answer - 1)*(1-difficulty)
 def list_feature_average(n, vals):

 if len(vals)>n:
return np.mean(vals[-n:])
 else:
return -1
def make_feature_average(count, vals):

 if count>0:
return vals/count
else:
return -1
def calculate_time_deltas(cache, timestamp, prior_question_elapsed_time,content_type_id, previous_part, user_id, batch_size):
if timestamp != cache[user_id]['ts-1']:
if prior_question_elapsed_time != -1:
cache[user_id]['average_reactivities'].append(prior_question_elapsed_time)
 cache[user_id]['last_ts_list'].append(timestamp)
 if len(cache[user_id]['average_reactivities'])>20:
cache[user_id]['average_reactivities'].pop(0)
 if len(cache[user_id]['last_ts_list'])>20:
cache[user_id]['last_ts_list'].pop(0)
 if previous_part in cache[user_id]['part_average_reactivity_tot'].keys() :
cache[user_id]['part_average_reactivity_tot'][previous_part]+=prior_question_elapsed_time
else:
if previous_part != -1:
cache[user_id]['part_average_reactivity_tot'][previous_part] = prior_question_elapsed_time
if len(cache[user_id]['last_ts_list'])>3:
cache[user_id]['time_between_questions'].append(cache[user_id]['last_ts_list'][-2]-cache[user_id]['last_ts_list'][-3]-cache[user_id]['average_reactivities'][-1]*cache[user_id]['batch_size_list'])
 if len(cache[user_id]['time_between_questions'])>20:
cache[user_id]['time_between_questions'].pop(0)
 if(cache[user_id]['last_delta_between_questions'] != -1)&(prior_question_elapsed_time != -1):
if cache[user_id]['average_delta_between_questions'] > 3600:
cache[user_id]['hour_count'] +=1
if cache[user_id]['average_delta_between_questions'] > 10000:
cache[user_id]['day_count'] +=1
if cache[user_id]['average_delta_between_questions'] < 3600:
cache[user_id]['average_delta_between_questions'] += cache[user_id]['last_delta_between_questions'] - prior_question_elapsed_time
if cache[user_id]['ts-1'] != -1:
if content_type_id == 0:
cache[user_id]['last_delta_between_questions'] =(timestamp - cache[user_id]['ts-1'])/batch_size
else:
cache[user_id]['average_time_on_lecture'] +=(timestamp - cache[user_id]['ts-1'])/batch_size
return cache<compute_test_metric>
xgb_acc = xgboost.score(x_test, y_test )
def get_new_theta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return theta + learning_rate_theta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote)
)
 def get_new_beta(is_good_answer, beta, left_asymptote, theta, nb_previous_answers):
return beta - learning_rate_beta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta, left_asymptote)
)
 def learning_rate_theta(nb_answers):
return max(0.3 /(1 + 0.01 * nb_answers), 0.04)
 def learning_rate_beta(nb_answers):
return 1 /(1 + 0.05 * nb_answers)
 def probability_of_good_answer(theta, beta, left_asymptote):
return left_asymptote +(1 - left_asymptote)* sigmoid(theta - beta)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))<categorify>
print('Area Under Curve: {}, Accuracy: {}'.format(xgb_auc, xgb_acc))
def create_cache(cache, user_id, ts, ids):
if user_id in ids:
with open(f'.. /input/riid-cache-6/content/drive/My Drive/riid/{user_id}', 'rb')as f:
id_cache = pickle.load(f)[user_id]
cache[user_id] = id_cache
else:
cache[user_id] = {}
cache[user_id]['previous_content_type_id'] = -1
cache[user_id]['previous_part'] = -1
cache[user_id]['previous_question'] = -1
cache[user_id]['questions_count'] = 0
cache[user_id]['part_lectures_count'] = {i:0 for i in range(1,8)}
cache[user_id]['lectures_part'] = {i:0 for i in range(1,8)}
cache[user_id]['part_count'] = {}
cache[user_id]['question_with_explanation'] = set()
cache[user_id]['question_seen'] = set()
cache[user_id]['first_bundle'] = -1
cache[user_id]['batch_size_list'] = -1
cache[user_id]['ts-1'] = -1
cache[user_id]['last_ts_part'] = {i:0 for i in range(1,8)}
cache[user_id]['last_delta_between_questions'] = -1
cache[user_id]['day_count'] = 0
cache[user_id]['hour_count'] = 0
cache[user_id]['last_ts_list'] = []
cache[user_id]['time_between_questions'] = []
cache[user_id]['average_reactivity_tot'] = 0
cache[user_id]['average_reaction_good_answer_tot'] = 0
cache[user_id]['average_reaction_bad_answer_tot'] = 0
cache[user_id]['average_delta_between_good_answer'] = 0
cache[user_id]['part_average_delta_between_good_answer'] = {}
cache[user_id]['average_reactivities'] = []
cache[user_id]['average_delta_between_questions'] = 0
cache[user_id]['average_time_on_lecture'] = -1
cache[user_id]['part_average_reactivity_tot'] = {}
cache[user_id]['last_questions'] = []
cache[user_id]['last_answers_correctly'] = []
cache[user_id]['last_answers_correctly_part'] = {i:[] for i in range(1,8)}
cache[user_id]['last_lectures'] = []
cache[user_id]['lectures_count'] = 0
cache[user_id]['general_good_answer'] = 0
cache[user_id]['part_good_answer'] = {}
cache[user_id]['part_score'] = {i:0 for i in range(1,8)}
cache[user_id]['student_parameters'] = {"theta": 0, "nb_answers": 0}
cache[user_id]['part_student_parameters'] = {i:{"theta": 0, "nb_answers": 0} for i in range(1,8)}
cache[user_id]['tag_student_parameters'] = {i:{"theta": 0, "nb_answers": 0} for i in tags.keys() }
cache[user_id]['tag_count'] = {}
cache[user_id]['tag_good_answers'] = {}
return cache<categorify>
lgboost = lgb.LGBMClassifier()
def update_cache(cache, arr, batch_size, question_cache):
left_asymptote = 1/4
row_id, timestamp, user_id, content_id, content_type_id, task_container_id, prior_question_elapsed_time, prior_question_had_explanation, _, _, answered_correctly,user_answer = arr
if not prior_question_elapsed_time:
prior_question_elapsed_time=-1
if not prior_question_had_explanation:
prior_question_had_explanation =-1
timestamp = int(float(timestamp)/1000)
 user_id = np.uint32(user_id)
 content_id = np.uint16(content_id)
 content_type_id = np.int8(content_type_id)
 task_container_id = np.uint32(task_container_id)
 prior_question_elapsed_time = int(float(prior_question_elapsed_time)/1000)
 prior_question_had_explanation = np.bool(prior_question_had_explanation)
 user_answer = np.int8(user_answer)
 answered_correctly = np.int8(answered_correctly)
 previous_content_type_id = cache[user_id]['previous_content_type_id']
part = -1
if content_type_id == 0:
if content_id not in question_cache.keys() :
question_cache[content_id] = {"beta": 0, "nb_answers": 0}
part = questions_to_parts[content_id]
tags = questions_to_tag[content_id]
difficulty = questions_to_difficulty[content_id]
count = question_user_count[content_id]
bad_answer_reaction = bad_answer_reaction_time[content_id]
good_answer_reaction = good_answer_reaction_time[content_id]
beta = question_cache[content_id]["beta"]
score = qscore(answered_correctly, difficulty)
 theta = cache[user_id]['student_parameters']["theta"]
theta_part = cache[user_id]['part_student_parameters'][part]["theta"]
for tag in tags:
theta_tag = cache[user_id]['tag_student_parameters'][tag]["theta"]
cache[user_id]['tag_student_parameters'][tag]["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta_tag, cache[user_id]['tag_student_parameters'][tag]["nb_answers"], 
)
 cache[user_id]['tag_student_parameters'][tag]["nb_answers"] += 1
cache[user_id]['student_parameters']["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta, cache[user_id]['student_parameters']["nb_answers"], 
)
 cache[user_id]['part_student_parameters'][part]["theta"] = get_new_theta(
 answered_correctly, beta, left_asymptote, theta_part, cache[user_id]['part_student_parameters'][part]["nb_answers"], 
)
 cache[user_id]['student_parameters']["nb_answers"] += 1
cache[user_id]['part_student_parameters'][part]["nb_answers"] += 1
cache[user_id]['questions_count'] +=1
if part in cache[user_id]['part_count'].keys() :
cache[user_id]['part_count'][part] += 1
else:
cache[user_id]['part_count'][part] = 1
cache[user_id]['last_answers_correctly'].append(answered_correctly)
 cache[user_id]['last_answers_correctly_part'][part].append(answered_correctly)
 if len(cache[user_id]['last_answers_correctly'])>20:
cache[user_id]['last_answers_correctly'].pop(0)
 if len(cache[user_id]['last_answers_correctly_part'][part])>20:
cache[user_id]['last_answers_correctly_part'][part].pop(0)
 if answered_correctly == 1:
cache[user_id]['average_reaction_good_answer_tot'] += prior_question_elapsed_time
if part in cache[user_id]['part_good_answer'].keys() :
cache[user_id]['part_good_answer'][part]+=1
else:
cache[user_id]['part_good_answer'][part]=1
cache[user_id]['general_good_answer'] += 1
score = qscore(answered_correctly, difficulty)
 if answered_correctly == 0:
cache[user_id]['average_reaction_bad_answer_tot'] += prior_question_elapsed_time
cache[user_id]['part_score'][part]+=score
for tag in tags:
if tag in cache[user_id]['tag_count'].keys() :
cache[user_id]['tag_count'][tag] +=1
cache[user_id]['tag_good_answers'][tag] += answered_correctly
else:
cache[user_id]['tag_count'][tag] = 1
cache[user_id]['tag_good_answers'][tag] = answered_correctly
if(cache[user_id]['ts-1'] != -1)&(timestamp != cache[user_id]['ts-1']):
cache[user_id]['last_delta_between_questions'] =(timestamp - cache[user_id]['ts-1'])
 cache[user_id]['average_delta_between_questions'] +=(timestamp - cache[user_id]['ts-1'])
 cache = calculate_time_deltas(cache, timestamp, prior_question_elapsed_time,content_type_id, cache[user_id]['previous_part'], user_id, batch_size)
 cache[user_id]['ts-1'] = timestamp
cache[user_id]['last_ts_part'][part] = timestamp
cache[user_id]['previous_content_type_id'] = content_type_id
if part != -1:
cache[user_id]['previous_part'] = part
if content_type_id == 0:
cache[user_id]['previous_question'] = content_id
if content_type_id == 0:
cache[user_id]['last_questions'].append(( content_id,answered_correctly))
if len(cache[user_id]['last_questions'])>100:
cache[user_id]['last_questions'].pop(0)
 if content_type_id == 1:
part = lecture_to_part[content_id]
cache[user_id]['lectures_count']+=1
cache[user_id]['lectures_part'][part]+=1
cache[user_id]['last_lectures'].append(content_id)
 if len(cache[user_id]['last_lectures'])>100:
cache[user_id]['last_lectures'].pop(0)
 if(prior_question_had_explanation != -1)&(cache[user_id]['previous_question'] != -1):
cache[user_id]['question_with_explanation'].add(content_id)
 cache[user_id]['question_seen'].add(content_id)
 if cache[user_id]['first_bundle'] == -1:
cache[user_id]['first_bundle'] = content_id
cache[user_id]['batch_size_list'] = batch_size
return cache<define_variables>
threshold = [0.001, 0.01,0.1,0.5]
def create_feature(cache, arr, batch_size, question_cache):
features = []
row_id, timestamp, user_id, content_id, content_type_id, task_container_id, prior_question_elapsed_time, prior_question_had_explanation, _, _ = arr
if not prior_question_elapsed_time:
prior_question_elapsed_time=-1
if not prior_question_had_explanation:
prior_question_had_explanation =-1
timestamp = float(timestamp)/1000
prior_question_elapsed_time = float(prior_question_elapsed_time)/1000
if content_type_id == 0:
part = questions_to_parts[content_id]
tags = questions_to_tag[content_id]
difficulty = questions_to_difficulty[content_id]
count = question_user_count[content_id]
bad_answer_reaction = bad_answer_reaction_time[content_id]
good_answer_reaction = good_answer_reaction_time[content_id]
var = questions_to_difficulty_variance[content_id]
if content_id in questions_to_difficulty_with_explanation.keys() :
difficulty1bis = questions_to_difficulty_with_explanation[content_id]
else:
difficulty1bis = -1
if content_id in questions_to_difficulty_without_explanation.keys() :
difficulty2bis = questions_to_difficulty_without_explanation[content_id]
else:
difficulty2bis = -1
kk = 0
features.append(part)
 features.append(difficulty)
 features.append(count)
 features.append(good_answer_reaction)
 features.append(bad_answer_reaction)
 features.append(batch_size)
 features.append(var)
 features.append(difficulty1bis)
 features.append(difficulty2bis)
 features.append(clust[content_id])
 features.append(cache[user_id]['lectures_count'])
 a=0
if user_id in cache.keys() :
features.append(cache[user_id]['student_parameters']['theta'])
 a+=1
else:
features.append(-1)
 if content_id in question_cache.keys() :
a+=1
else:
pass
if a ==2:
features.append(probability_of_good_answer(cache[user_id]['student_parameters']['theta'], question_cache[content_id]['beta'], 1/4))
else:
features.append(-1)
 if part in cache[user_id]['part_student_parameters'].keys() :
features.append(probability_of_good_answer(cache[user_id]['part_student_parameters'][part]['theta'], question_cache[content_id]['beta'], 1/4))
else:
features.append(-1)
 c = 0
p = 0
s = len(tags)
 for tag in tags:
c+=cache[user_id]['tag_student_parameters'][tag]['theta']
p+=probability_of_good_answer(cache[user_id]['tag_student_parameters'][tag]['theta'], question_cache[content_id]['beta'], 1/4)
 features.append(c/s)
 features.append(p/s)
 features.append(cache[user_id]['part_score'][part])
 features.append(( timestamp - cache[user_id]['ts-1'])/batch_size)
 features.append(list_feature_average(19, cache[user_id]['average_reactivities']))
if part in cache[user_id]['part_average_reactivity_tot'].keys() :
features.append(make_feature_average(cache[user_id]['part_count'][part], cache[user_id]['part_average_reactivity_tot'][part]))
else:
features.append(-1)
 features.append(make_feature_average(cache[user_id]['general_good_answer'], cache[user_id]['average_reaction_good_answer_tot']))
features.append(cache[user_id]['last_delta_between_questions'])
 features.append(list_feature_average(10,(cache[user_id]['time_between_questions'])))
 features.append(list_feature_average(5,(cache[user_id]['time_between_questions'])))
 if len(cache[user_id]['time_between_questions'])>1:
features.append(cache[user_id]['time_between_questions'][-1])
 else:
features.append(-1)
 features.append(make_feature_average(cache[user_id]['questions_count'], cache[user_id]['general_good_answer']))
features.append(list_feature_average(2, cache[user_id]['last_answers_correctly']))
features.append(list_feature_average(6, cache[user_id]['last_answers_correctly']))
features.append(list_feature_average(6, cache[user_id]['last_answers_correctly_part'][part]))
if content_id in cache[user_id]['question_with_explanation']:
features.append(1)
 else:
features.append(0)
 c = 0
c1 = 0
c0 = 0
p=0
if content_id in all_prob.keys() :
to_check = set(cache[user_id]['last_questions'] ).intersection(set(all_prob[content_id].keys()))
for content, answer in to_check:
p+=all_prob[content_id][(content,answer)]
if answer == 1:
c1+=1
else:
c0+=1
features.append(c1)
 if c1+c0:
features.append(p/(c1+c0))
else:
features.append(0)
 c = 0
p=0
if content_id in proba_lec.keys() :
to_check = set(cache[user_id]['last_lectures'] ).intersection(set(proba_lec[content_id].keys()))
for content in to_check:
p+=proba_lec[content_id][content]
c+=1
if c:
features.append(p/c)
 else:
features.append(0)
 return features<categorify>
number_of_features = list(range(1,13))
def update_cache_batch(cache, df_arr, question_cache):
batch_array = []
task_init = -1
user_id0 = -1
for arr in df_arr:
user_id = arr[2]
timestamp = arr[1]
task_container_id = arr[5]
if user_id not in cache.keys() :
cache = create_cache(cache, user_id, timestamp, ids)
 if(task_container_id == task_init)&(user_id == user_id0):
batch_array.append(arr)
 else:
batch_size = len(batch_array)
 for arr_ in batch_array:
cache = update_cache(cache, arr_, batch_size, question_cache)
 task_init = task_container_id
if user_id != user_id0:
user_id0 = user_id
batch_array = []
batch_array.append(arr)
 batch_size = len(batch_array)
 for arr_ in batch_array:
cache = update_cache(cache, arr_, batch_size, question_cache)
 return cache<define_variables>
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )
def calculate_features(cache, df_arr, question_cache):
X = []
batch_array = []
task_init = -1
user_id0 = -1
for arr in df_arr:
user_id = arr[2]
timestamp = arr[1]
task_container_id = arr[5]
if user_id not in cache.keys() :
cache = create_cache(cache, user_id, timestamp, ids)
 if(task_container_id == task_init)&(user_id == user_id0):
batch_array.append(arr)
 else:
batch_size = len(batch_array)
 for arr_ in batch_array:
feats = create_feature(cache, arr_, batch_size, question_cache)
 if len(feats)>0:
X.append(feats)
 task_init = task_container_id
if user_id != user_id0:
user_id0 = user_id
batch_array = []
batch_array.append(arr)
 batch_size = len(batch_array)
 for arr_ in batch_array:
feats = create_feature(cache, arr_, batch_size, question_cache)
 if len(feats)>0:
X.append(feats)
 X = np.array(X)
 return X<split>
print("Optimal number of features : %d" % selector.n_features_ )
env = riiideducation.make_env()
iter_test = env.iter_test()<data_type_conversions>
threshold = [0.001, 0.01, 0.05, 0.1 , 0.5]
p_test_df = pd.DataFrame()
cache = {}
for idx,(test_df, _)in enumerate(iter_test):
test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(float)
 test_df = test_df.fillna(-1)
 submit_df = test_df.loc[test_df['content_type_id'] == False, ['row_id']].copy()
if not p_test_df.empty:
answered_correctly = [int(elmt)for elmt in test_df.iloc[0]['prior_group_answers_correct'][1:-1].replace(' ','' ).split(',')]
prior_group_responses = [int(elmt)for elmt in test_df.iloc[0]['prior_group_responses'][1:-1].replace(' ','' ).split(',')]
p_test_df['answered_correctly'] = answered_correctly
p_test_df['user_response'] = answered_correctly
test_arr = p_test_df.values
update_cache_batch(cache, test_arr, question_cache)
 X = calculate_features(cache, test_df.values, question_cache)
 if len(X):
preds = 1-lgb.predict_proba(X)[:,0]
else:
preds = None
submit_df['answered_correctly'] = preds
env.predict(submit_df)
 p_test_df = test_df<install_modules>
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )
!pip --quiet install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
!pip install --quiet -r.. /input/treelite-treelite-runtime-version-093/treelite/requirements.txt --no-index --find-links.. /input/treelite-treelite-runtime-version-093/treelite
tqdm.tqdm.pandas()
%matplotlib inline
env = riiideducation.make_env()<load_from_csv>
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )
dtypes = {
"row_id": "int64",
"timestamp": "int64",
"user_id": "int32",
"content_id": "int16",
"content_type_id": "boolean",
"task_container_id": "int16",
"user_answer": "int8",
"answered_correctly": "int8",
"prior_question_elapsed_time": "float32",
"prior_question_had_explanation": "boolean"
}
data = pd.read_csv(".. /input/riiid-test-answer-prediction/train.csv", dtype=dtypes, nrows=int(1e6))
ques = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv(".. /input/riiid-test-answer-prediction/lectures.csv")
 ex_sub = pd.read_csv('.. /input/riiid-test-answer-prediction/example_sample_submission.csv')
 ex_test = pd.read_csv('.. /input/riiid-test-answer-prediction/example_test.csv')
 data.shape, ques.shape, lectures.shape, ex_sub.shape, ex_test.shape<feature_engineering>
selector = sklearn.feature_selection.SelectKBest(k= 11)
 selector.fit(features, target)
 lgb_selected_features = selector.get_support()
ql = pd.concat([ques, lectures.rename({"lecture_id": "question_id"}, axis=1)], axis=0 ).reset_index(drop=True)
 ql.tags = ql.tags.fillna(ql.tag)
 ql.type_of = ql.type_of.fillna("question")
 ql["content_type_id"] = ql["type_of"] != 'question'
ql = ql.fillna(-1)
 ql = ql.drop("tag", 1)
 ql = ql.rename({"question_id": "content_id"}, axis=1)
 ql.tags = ql.tags.apply(lambda x: [int(x)] if type(x)!= str else list(map(int, x.split())))
 ql['listening_lvl'] = ql.part.map({1: 2, 2: 2, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0})
 ql['multiple_q'] = ql.part.isin([3, 4, 6, 7])
 ql['tag_c'] = ql.tags.apply(len )<count_values>
lgboost = lgb.LGBMClassifier()
lgboost.fit(features.loc[:,lgb_selected_features], target )
lec_count = ql.loc[ql.content_type_id, 'tags'].transform(lambda x: x[0] ).value_counts()
ql['lec_available'] =(
 ql.loc[~ql.content_type_id, 'tags'].transform(
 lambda x: sum([lec_count.at[i] if i in lec_count.index else 0 for i in x]))
)<feature_engineering>
y_scores_lgb = lgboost.predict_proba(x_test.loc[:,lgb_selected_features])[:, 1]
lgb_fpr, lgb_tpr, lgb_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_lgb)
 lgb_auc = sklearn.metrics.auc(x=lgb_fpr, y=lgb_tpr )
ql['bundle_q_count'] = ql.groupby("bundle_id")['content_id'].transform('count')
 ql.loc[ql.content_type_id, 'bundle_q_count'] = -1<categorify>
lgb_acc = lgboost.score(x_test.loc[:,lgb_selected_features], y_test )
te = TransactionEncoder()
temp = ql[~ql.content_type_id]
temp = temp.merge(
 data[~data.content_type_id].groupby("content_id")['answered_correctly'].agg(['count', 'mean']),
on='content_id', how='left')
 temp['mean'] = temp['mean'].fillna(0.5)
 temp['count'] = temp['count'].fillna(0)
 temp = np.hstack([
te.fit_transform(temp.tags.to_list() ).astype(int),
pd.get_dummies(temp[['tag_c', 'part', 'count', 'mean']], columns=['part', 'tag_c'])
 ])
 temp = PCA(n_components=150 ).fit_transform(StandardScaler().fit_transform(temp))
ql.loc[~ql.content_type_id, 'cluster'] = KMeans(n_clusters=30 ).fit_predict(temp)
 ql.loc[ql.content_type_id, 'cluster'] = -1
ql.cluster = ql.cluster.astype(int)
 ql.sample(5 )<feature_engineering>
print('Area Under Curve: {}, Accuracy: {}'.format(lgb_auc, lgb_acc))
temp = ql.tags.progress_apply(pd.Series)
 ql['tagF'] = temp[0]
ql['tagS'] = temp[1]
ql['tagT'] = temp[2]
ql['tagL'] = ql.tags.apply(lambda x: x[-1])
 ql[['tagF', 'tagS', 'tagL', 'tagT']] =(ql[['tagF', 'tagS', 'tagL', 'tagT']] + 1 ).fillna(0)
 ql.sample(5 )<merge>
v = ens.VotingClassifier(estimators=[ 
('lr', lr),('NB', nb),('KNN', knn),('SVM', svm),('DT', dt), 
('RF', rf),('BG', bg),('AdaBoost', ada),('GBM', gb), 
('XGBM', xgboost),('LightGBM', lgboost)],
voting='soft',
weights= [1,1,1, 1.25, 1.25, 1.25, 1.25, 1.25, 1.75, 1.5, 1.5] )
data = data.drop(['part', 'bundle_id'], 1 ).merge(ql, on=['content_id', 'content_type_id'], how='left')
 data.shape<sort_values>
selector = sklearn.feature_selection.SelectKBest(k= 11)
 selector.fit(features, target)
 voting_selected_features = selector.get_support()
data = data.sort_values(by=['user_id', 'timestamp'])
 data['response_time'] =(
 data.groupby("user_id")['timestamp'] 
.transform(lambda x: x.diff().replace(0, np.nan)
.fillna(method='ffill' ).fillna(0))
)<data_type_conversions>
v.fit(features.loc[:, voting_selected_features], target )
data['res_time_avg'] =(
 data.timestamp - 
(data.timestamp *(data.task_container_id - 1)/ data.task_container_id)
)
 data['res_time_avg'] = data['res_time_avg'].replace(np.inf, np.nan)
 print("Corelation to response_time:
",
data.corr() ['response_time'].loc[['res_time_avg']], sep='')
 print("
Correlation to ans_correctness:
",
data.corr() ['answered_correctly'].loc[['response_time', 'res_time_avg']], sep='')
(data[['timestamp', 'response_time', 'res_time_avg']] 
.sample(10 ).fillna(0 ).astype(int))<merge>
y_scores_v = v.predict_proba(features.loc[:, voting_selected_features])[:, 1]
v_fpr, v_tpr, v_thresholds = sklearn.metrics.roc_curve(target, y_scores_v)
 v_auc = sklearn.metrics.auc(x=v_fpr, y=v_tpr )
data = data.sort_values(['user_id', 'timestamp'])
 data = data.merge(
(data[~data.content_type_id].groupby(['user_id', 'task_container_id'])
 [['prior_question_elapsed_time', 'prior_question_had_explanation']] 
.mean().groupby("user_id" ).shift(-1 ).reset_index() 
.rename({"prior_question_elapsed_time": 'pqet_shifted',
'prior_question_had_explanation': "pqhe_shifted"}, axis=1)) ,
on=['user_id', 'task_container_id'], how='left' )<sort_values>
v_acc = v.score(x_test.loc[:,voting_selected_features], y_test )
data = data.sort_values(by=['user_id', 'timestamp'])
 cut_off =(1000 * 60 * 60)
 cut_off = cut_off * 1
data['sessions'] =(
 data.groupby("user_id")['timestamp'].diff() > cut_off
).groupby(data['user_id'] ).cumsum()<categorify>
print('Area Under Curve: {}, Accuracy: {}'.format(v_auc, v_acc))
def post_process(fn0, fn1):

 fn_processed = fn0.drop(['prior_group_answers_correct', 'prior_group_responses'], 1)
 fn_processed['answered_correctly'] = eval(fn1['prior_group_answers_correct'].iloc[0])
 fn_processed['user_answer'] = eval(fn1['prior_group_responses'].iloc[0])
 return fn_processed<load_pretrained>
pd.DataFrame([(lr_auc, lr_acc),(nb_auc, nb_acc),(knn_auc, knn_acc),(dt_auc, dt_acc), 
(rf_auc, rf_acc),(svm_auc, svm_acc),(bg_auc, bg_acc),(ada_auc, ada_acc), 
(v_auc, v_acc),(gb_auc, gb_acc),(xgb_auc, xgb_acc),(lgb_auc, lgb_acc)],
columns=['AUC', 'Accuracy'],
index=['Logistic Regression', 'Naive Bayes', 'KNN', 'Decision Tree',
'Random Forest', 'SVM', 'Bagging', 'AdaBoost', 'Voting',
'Gradient Boost', 'XGBoost', 'Light Boost'] )
with open(".. /input/riiid-final-model-inputs/sample-batches.pkl", 'rb')as f:
batches = pickle.load(f)
 print("Batch sizes for each test sample:", list(map(lambda x: x[0].shape[0], batches)) )<compute_train_metric>
y_pred_v = pd.DataFrame(v.predict(unlabelled.loc[:, voting_selected_features]), columns=[
'Survived'], dtype='int64' )
temp = data.loc[~data.content_type_id, "answered_correctly"]
for value in [0, 1, 0.5, temp.mean() ]:
print("At {:.2f} the score is: {:.3f}".format(value, roc_auc_score(temp, np.full_like(temp, 0))))<compute_test_metric>
v_model = pd.concat([passengerID, y_pred_v], axis=1 )
<categorify><EOS> 
v_model.to_csv('voting.csv', index= False )