Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
RUN_ON_KAGGLE = True
RANDOM_SEED = 3
random.seed(RANDOM_SEED)
 np.random.seed(seed=RANDOM_SEED )
class skew_dummies(BaseEstimator, TransformerMixin):
def __init__(self,skew=0.75):
self.skew = skew
def fit(self,X,y=None):
return self
def transform(self,X):
X_numeric=X.select_dtypes(exclude=["object"])
 skewness = X_numeric.apply(lambda x: skew(x))
skewness_features = skewness[abs(skewness)>= self.skew].index
X[skewness_features] = np.log1p(X[skewness_features])
 X = pd.get_dummies(X)
 return X<categorify>
processed_data_dir = "/kaggle/input/riid-acp-data-joined-user-feat"
class labelenc(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self,X,y=None):
return self
def transform(self,X):
lab = LabelEncoder()
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
X[c] = lab.fit_transform(X[c])
 return X<train_on_grid>
logger.info("reading training data...")
 train = pd.read_pickle(
 os.path.join(processed_data_dir, "train-with-user-fea.pkl")
 )
class grid() :
def __init__(self,model):
self.model = model
def grid_get(self,X,y,param_grid):
grid_search = GridSearchCV(self.model,param_grid,cv=5, scoring="neg_mean_squared_error")
 grid_search.fit(X,y)
 print(grid_search.best_params_, np.sqrt(-grid_search.best_score_))
grid_search.cv_results_['mean_test_score'] = np.sqrt(-grid_search.cv_results_['mean_test_score'])
 print(pd.DataFrame(grid_search.cv_results_)[['params','mean_test_score','std_test_score']] )<prepare_x_and_y>
logger.info("reading validation data...")
 valid = pd.read_pickle(
 os.path.join(processed_data_dir, "eval-with-user-fea.pkl")
 )
class drop_cols(BaseEstimator, TransformerMixin):
def __init__(self, remove_cols = True):
self.remove_cols = remove_cols
pass
def fit(self,X,y=None):
return self
def transform(self,X):
if self.remove_cols:
del X['PoolQC']
del X['BsmtFinType1']
del X['LowQualFinSF']
del X['MoSold']
del X['Electrical']
del X['BldgType']
del X['SaleType']
del X['BsmtFinSF2']
del X['Exterior2nd']
del X['ExterCond']
del X['2ndFlrSF']
del X['3SsnPorch']
del X['Exterior1st']
del X['MasVnrType']
del X['GarageFinish']
del X['Alley']
del X['PoolArea']
del X['RoofStyle']
del X['MiscFeature']
del X['Fence']
del X['BsmtHalfBath']
del X['Street']
del X['LotConfig']
del X['TotalBsmtSF']
del X['GarageYrBlt']
return X<predict_on_test>
train.drop("row_id", inplace=True, axis=1)
 valid.drop("row_id", inplace=True, axis=1 )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<load_from_csv>
def pickle_load(file_path):
with open(file_path, "rb")as f:
return pickle.load(f )
pipe = Pipeline([ 
('feat_eng', feat_eng()), 
('add_feature', add_feature(additional=2)) , 
('lab_enc', labelenc()), 
('drop_cols', drop_cols()), 
('skew_dummies', skew_dummies(skew=1)) ,
])
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 trainWprice = pd.DataFrame(train)
 trainNoPrice = trainWprice.drop("SalePrice", axis=1)
 full=pd.concat([trainNoPrice,test], ignore_index=True)
 full.drop(['Id'],axis=1, inplace=True)
 print("full without Id and no price ", full.shape)
 FullDataPipe = pipe.fit_transform(full)
 print("FullDataPipe ", FullDataPipe.shape)
 n_train=train.shape[0]
trainFinal = pd.DataFrame(FullDataPipe[:n_train])
 testFinal = pd.DataFrame(FullDataPipe[n_train:])
 y= train.SalePrice
yFinal = np.log(train.SalePrice)
 scaler = RobustScaler()
trainFinal = scaler.fit_transform(trainFinal)
 testFinal = scaler.fit_transform(testFinal)
 print("trainFinal", trainFinal.shape)
 print("testFinal", testFinal.shape)
 print("yFinal", yFinal.shape )<train_model>
answer_cnt = defaultdict(int)
 answer_cnt.update(
 pickle_load(os.path.join(processed_data_dir, "answer_cnt.pkl")) 
)
 correct_sum = defaultdict(int)
 correct_sum.update(
 pickle_load(os.path.join(processed_data_dir, "correct_sum.pkl"))
)
trainFinalFI = pd.DataFrame(trainFinal)
 yFinalFI = yFinal
lasso=Lasso(alpha=0.001, copy_X=True, fit_intercept=True, max_iter=1000,
normalize=False, positive=False, precompute=False, random_state=None,
selection='cyclic', tol=0.0001, warm_start=False)
 lasso.fit(trainFinalFI,yFinalFI)
 FI_lasso = pd.DataFrame({"Feature Importance":lasso.coef_}, index=trainFinalFI.columns)
 FI_lasso[FI_lasso["Feature Importance"]!=0].sort_values("Feature Importance" ).plot(kind="barh",figsize=(15,25))
plt.xticks(rotation=90)
 plt.show()<create_dataframe>
logger.info("reading questions...")
 questions = pd.read_csv(
 os.path.join(data_folder, "questions.csv"),
dtype={
"question_id": "int16",
"bundle_id": "int32",
"correct_answer": "int8",
"part": "int8"
}
)
trainFinal = pd.DataFrame(trainFinal)
 ColsMissingValues = trainFinal.isnull().sum()
print("There are ", len(ColsMissingValues[ColsMissingValues>0]), " features with missing values")
 all_data_na =(trainFinal.isnull().sum() / len(trainFinal)) * 100
all_data_na = all_data_na.sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 print(missing_data.head(len(ColsMissingValues[ColsMissingValues>0])))
 print("_"*80)
 print("trainFinal ", trainFinal.shape)
 print("yFinal ", yFinal.shape )<compute_test_metric>
logger.info("reading lectures...")
 lectures = pd.read_csv(
 os.path.join(data_folder, "lectures.csv"),
dtype={
"lecture_id": "int16",
"tag": "int16",
"part": "int8"
}
)
def rmse_cv(model,X,y):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=5))
return rmse<define_search_model>
def add_user_feats(
 df,
user_correct_sum: dict,
user_answer_cnt: dict,
update=True 
):
correct_sum = np.zeros(len(df), dtype=np.int32)
 answer_cnt = np.zeros(len(df), dtype=np.int32)
 cols = ["user_id", "answered_correctly"] if update else ["user_id"]
for i, row in df[cols].iterrows() :
user_id = row["user_id"]
correct_sum[i] = user_correct_sum[user_id]
answer_cnt[i] = user_answer_cnt[user_id]
if update:
answer_res = row["answered_correctly"]
user_correct_sum[user_id] += answer_res
user_answer_cnt[user_id] += 1
user_feats = pd.DataFrame({
"correct_sum": correct_sum,
"answer_cnt": answer_cnt,
"acc":(correct_sum / answer_cnt ).astype(np.float32)
 })
 return pd.concat([df, user_feats], axis=1 )
<choose_model_class>
def update_user_feats(df, user_correct_sum: dict, user_answer_cnt: dict):
df = df[df["content_type_id"] == 0]
for _, row in df[["user_id", "answered_correctly"]].iterrows() :
user_id = row["user_id"]
answer_res = row["answered_correctly"]
user_correct_sum[user_id] += answer_res
user_answer_cnt[user_id] += 1
models = [
Lasso(alpha =0.0005, random_state=1),
ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3),
KernelRidge(alpha=2.2, coef0=2.5, degree=3, gamma=None, kernel='polynomial',kernel_params=None),
SVR(C=2, cache_size=200, coef0=0.1, degree=3, epsilon=0.005, gamma=0.005,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.01, verbose=False),
GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
learning_rate=0.05, loss='huber', max_depth=5,
max_features='sqrt', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=10, min_samples_split=10,
min_weight_fraction_leaf=0.0, n_estimators=2000,
n_iter_no_change=None, presort='auto', random_state=None,
subsample=1.0, tol=0.0001, validation_fraction=0.1, verbose=0,
warm_start=False),
]
names = ["LASSO", "ELA","KER", "SVR ", "GBR" ]<compute_train_metric>
def fast_left_join(left, right, on=None, left_on=None, right_on=None):
left_on = left_on or on
right_on = right_on or on
assert left_on in left.columns, f"{on} not found in left DF"
assert right_on == right.index.name, f"{right_on} not equals to right's index_name"
return pd.concat([
left.reset_index(drop=True),
right.reindex(left[left_on].values ).reset_index(drop=True)
 ], axis=1 )
ModScores = {}
for name, model in zip(names, models):
score = rmse_cv(model, trainFinal, yFinal)
 ModScores[name] = score.mean()
print("{}: {:.6f}".format(name,score.mean()))
print("trainFinal ", trainFinal.shape)
 print("_"*80)
 for key, value in sorted(ModScores.items() , key = itemgetter(1), reverse = False):
print(key, value )<define_search_space>
logger.info("generating content features...")
 content_df = train[
["content_id", "answered_correctly"]
].groupby("content_id" ).agg(
 content_acc=("answered_correctly", "mean")
 ).astype({"content_acc": "float32"} )
param_grid = [{'kernel': ["rbf"], 'degree': [3]},
{'gamma': [0.005],'coef0': [0.1,0.05],'tol': [0.01],
'C': [2.5,2.2],'epsilon': [0.005],},]<train_on_grid>
logger.info("joining content features to train...")
 train = fast_left_join(
 train,
content_df,
on="content_id"
)
model4cv = SVR()
grid_search = GridSearchCV(model4cv, param_grid)
 grid_search.fit(trainFinal, yFinal)
 print(grid_search.best_params_)
 print(grid_search.best_estimator_)
 print("_"*80)
 print(grid(grid_search.best_estimator_ ).grid_get(trainFinal, yFinal,{}))<choose_model_class>
_ = gc.collect()
lasso = Lasso(alpha =0.0005, random_state=1)
 ker = KernelRidge(alpha=2.2, coef0=2.5, degree=3, gamma=None, kernel='polynomial',
kernel_params=None)
 svr = SVR(C=2, cache_size=200, coef0=0.1, degree=3, epsilon=0.005, gamma=0.005,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.01, verbose=False)
 gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 ela = ElasticNet(alpha=0.01, copy_X=True, fit_intercept=True, l1_ratio=0.05,
max_iter=5000, normalize=False, positive=False, precompute=False,
random_state=None, selection='cyclic', tol=0.0001, warm_start=False)
<train_on_grid>
logger.info("join content features to validation...")
 valid = fast_left_join(
 valid,
content_df,
on="content_id"
)
model = SVR(C=2, cache_size=200, coef0=0.1, degree=3, epsilon=0.005, gamma=0.005,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.01, verbose=False)
 model.fit(trainFinal, yFinal)
 score = rmse_cv(model, trainFinal, yFinal)
 print(" model score: {:.5f}({:.4f})
 ".format(score.mean() , score.std()))
pred = np.exp(model.predict(testFinal))
pred = np.around(pred, decimals=4, out=None)
 print(pred )<save_to_csv>
train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean )
result=pd.DataFrame({'Id':test.Id, 'SalePrice':pred})
 result.to_csv("submission.csv",index=False)
 print(result )<import_modules>
logger.info("generating questions features...")
 question_df = questions[[
"question_id", "part"
]].set_index("question_id" )
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
logger.info("joining questions features to validation...")
 valid = fast_left_join(
 valid,
question_df,
left_on = "content_id",
right_on = "question_id"
)
training_set = pd.read_csv('.. /input/train.csv')
 testing_set = pd.read_csv('.. /input/test.csv')
 pID = testing_set['PassengerId']<count_missing_values>
logger.info("joining questions features to train...")
 train = fast_left_join(
 train,
question_df,
left_on = "content_id",
right_on = "question_id"
)
print(training_set.isnull().sum() ,"
")
 print(testing_set.isnull().sum() )<data_type_conversions>
logger.info("filling missing values...")
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )
for dataset in [training_set,testing_set]:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
training_set.drop(drop_column, axis=1, inplace = True)
 testing_set.drop(drop_column, axis=1, inplace = True)
 print(training_set.isnull().sum())
 print("-"*10)
 print(testing_set.isnull().sum() )<feature_engineering>
import lightgbm as lgb
for dataset in [training_set,testing_set]:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 training_set.info()
testing_set.info()
training_set.sample(10 )<filter>
TARGET = "answered_correctly"
FEATS = [
"correct_sum",
"answer_cnt",
"acc",
"part",
"content_acc",
"prior_question_elapsed_time",
"prior_question_had_explanation"
]
training_set[training_set["Name"].str.contains("Master")]<categorify>
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _ = gc.collect()
label = LabelEncoder()
for dataset in [training_set,testing_set]:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
training_set_x = ['Sex','Pclass', 'Embarked','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
training_set_x_calc = ['Sex_Code','Pclass', 'Embarked_Code','SibSp', 'Parch', 'Age', 'Fare']
training_set_xy = Target + training_set_x
print('Original X Y: ', training_set_xy, '
')
 training_set_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
training_set_xy_bin = Target + training_set_x_bin
print('Bin X Y: ', training_set_xy_bin, '
')
 training_set_dummy = pd.get_dummies(training_set[training_set_x],drop_first=True)
 training_set_x_dummy = training_set_dummy.columns.tolist()
training_set_xy_dummy = Target + training_set_x_dummy
print('Dummy X Y: ', training_set_xy_dummy, '
')
 training_set_dummy.head()<prepare_x_and_y>
lgb_train = lgb.Dataset(train[FEATS], y_tr, categorical_feature=["part"])
 lgb_valid = lgb.Dataset(valid[FEATS], y_va, reference=lgb_train, categorical_feature=["part"])
 del train, y_tr
_=gc.collect()
y = training_set['Survived']
X = training_set_dummy<categorify>
from sklearn.metrics import roc_auc_score
testing_set_dummy = pd.get_dummies(testing_set[training_set_x],drop_first=True )<normalization>
USE_OFFLINE_MODEL = True
model_path = os.path.join(processed_data_dir, "model_7fea.pkl")
 if USE_OFFLINE_MODEL:
logger.info("loading offline trained model...")
 model = pickle_load(model_path)
 else:
logger.info("start training model...")
 model = lgb.train(
 {
"objective": "binary",
"metric": "auc",
"boosting_type": "gbdt",
"learning_rate": 0.3,
"seed": 3
},
lgb_train,
valid_sets=[lgb_train, lgb_valid],
verbose_eval=20,
num_boost_round=1000,
early_stopping_rounds=10 
)
 auc = roc_auc_score(y_va, model.predict(valid[FEATS]))
logger.info(f"auc on validation: {auc}" )
ss = MinMaxScaler()
training_set_dummy_ss= ss.fit_transform(training_set_dummy)
 testing_set_dummy_ss= ss.fit_transform(testing_set_dummy )<choose_model_class>
del lgb_train, lgb_valid
_ = gc.collect()
classifiers = {'Gradient Boosting Classifier':GradientBoostingClassifier() ,'Adaptive Boosting Classifier':AdaBoostClassifier() ,'RadiusNN':RadiusNeighborsClassifier(radius=40.0),
'Linear Discriminant Analysis':LinearDiscriminantAnalysis() , 'GaussianNB': GaussianNB() , 'BerNB': BernoulliNB() , 'KNN': KNeighborsClassifier() ,
'Random Forest Classifier': RandomForestClassifier(min_samples_leaf=10,min_samples_split=20,max_depth=4),'Decision Tree Classifier': DecisionTreeClassifier() ,'Logistic Regression':LogisticRegression() , "XGBoost": xgb.XGBClassifier() }<split>
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
X_training, X_validating, y_training, y_validating = train_test_split(training_set_dummy, y, test_size=0.20, random_state=11 )<compute_train_metric>
import time
base_accuracy = 0
for Name,classify in classifiers.items() :
classify.fit(X_training,y_training)
 y_predictng = classify.predict(X_validating)
 print('Accuracy Score of '+str(Name)+ " : " +str(met.accuracy_score(y_validating,y_predictng)))
 if met.accuracy_score(y_validating,y_predictng)> base_accuracy:
predictions_test = classify.predict(testing_set_dummy)
 base_accuracy = met.accuracy_score(y_validating,y_predictng)
 else:
continue
predicted_test_value = pd.DataFrame({ 'PassengerId': pID,
'Survived': predictions_test })
 predicted_test_value.to_csv("PredictedTestScore.csv", index=False )<compute_train_metric>
pre_test_df = None
infer_start, iter_cnt = time.time() , 0
actuals = []
preds = []
for(test_df, sample_pred_df)in iter_test:
if pre_test_df is not None:
targets = eval(test_df["prior_group_answers_correct"].iloc[0])
 actuals.extend(targets)
 pre_test_df[TARGET] = targets
update_user_feats(pre_test_df, correct_sum, answer_cnt)
 pre_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats(
 test_df, correct_sum, answer_cnt, update=False 
)
 test_df = fast_left_join(
 test_df, content_df, on="content_id" 
)
 test_df = fast_left_join(
 test_df, question_df, left_on="content_id", right_on="question_id" 
)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 preds_ = model.predict(test_df[FEATS])
 test_df[TARGET] = preds_
preds.extend(preds_.tolist())
 set_predict(test_df[['row_id', TARGET]])
 iter_cnt += 1
infer_duration = time.time() - infer_start
logger.info(f"total iterations: {iter_cnt}, cost {infer_duration:.4f} secs, {infer_duration/iter_cnt:.4f} secs/iteration" )
<choose_model_class><EOS> 
roc_auc_score(actuals, preds[:len(actuals)] )