Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
%matplotlib inline
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
print("train score: ", GBoost.score(X,y))<load_from_csv>
data_tittrain=pd.read_csv(".. /input/titanic/train.csv")
 data_tittest=pd.read_csv(".. /input/titanic/test.csv")
 data_sample=pd.read_csv(".. /input/titanic/gender_submission.csv" )
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )<predict_on_test>
data_sample.head()
y_pred = GBoost.predict(test )<prepare_output>
merged = pd.concat([data_tittrain,data_tittest] )
submission['SalePrice'] = y_pred
submission.head(10 )<save_to_csv>
data_tittrain["Sex"].value_counts()
submission.to_csv('submission.csv', index=False )<save_to_csv>
data_tittest["Sex"].value_counts()
submission.to_csv('submission.csv', index=False )<import_modules>
data_tittrain["Pclass"].value_counts()
from tqdm import tqdm<load_from_csv>
data_tittest['Pclass'].value_counts()
train = pd.read_csv('.. /input/ames-housing-dataset/AmesHousing.csv')
 train.drop(['PID'], axis=1, inplace=True)
 origin = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.columns = origin.columns
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 print('Train:{} Test:{}'.format(train.shape,test.shape))<drop_column>
data_tittrain = data_tittrain.drop(columns=['Name','Cabin'])
 data_tittrain['family_member'] = data_tittrain['SibSp'] + data_tittrain['Parch']
data_tittrain = data_tittrain.drop(columns=['SibSp', 'Parch'] )
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True )<feature_engineering>
data_tittest = data_tittest.drop(columns=['Name','Cabin'])
 data_tittest['family_member'] = data_tittest['SibSp'] + data_tittest['Parch']
data_tittest = data_tittest.drop(columns=['SibSp', 'Parch'] )
l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()<save_to_csv>
X = data_tittrain.drop(columns=['Survived'])
 Y = data_tittrain['Survived']
cate_features_index = np.where(X.dtypes != float)[0]
submission.to_csv('result-with-best.csv', index=False )<install_modules>
from catboost import CatBoostClassifier, cv, Pool
from sklearn.model_selection import train_test_split
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
xtrain,xtest,ytrain,ytest = train_test_split(X,Y,train_size=0.82,random_state=42 )
np.seterr(divide = 'ignore', invalid = 'ignore' )<define_variables>
clf =CatBoostClassifier(eval_metric='Accuracy',use_best_model=True,random_seed=42 )
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
clf.fit(xtrain,ytrain,cat_features=cate_features_index,eval_set=(xtest,ytest), early_stopping_rounds=50 )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns = set(data_types_dict.keys())).to_pandas()<train_model>
test_id = data_tittest.PassengerId
data_tittest.isnull().sum()
print('Training dataset detailed information')
 print('*' * 50)
 print('Columns:', train_df.columns)
 print('*' * 50)
 print('Shape:', train_df.shape)
 print('*' * 50)
 print('NA values in each column:', sum(train_df.isna().sum()))
print('*' * 50 )<data_type_conversions>
prediction = clf.predict(data_tittest )
train_df = train_df[train_df[target] != -1].reset_index(drop = True, inplace = False)
 train_df['prior_question_had_explanation'].fillna(False, inplace = True)
 train_df = train_df.astype(data_types_dict )<feature_engineering>
df_sub = pd.DataFrame()
df_sub['PassengerId'] = test_id
df_sub['Survived'] = prediction.astype(np.int)
 df_sub.to_csv('gender_submission.csv', index=False )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns = ['lag'], inplace = True )<groupby>
%matplotlib inline
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )<groupby>
data_tittrain=pd.read_csv(".. /input/titanic/train.csv")
 data_tittest=pd.read_csv(".. /input/titanic/test.csv")
 data_sample=pd.read_csv(".. /input/titanic/gender_submission.csv" )
train_df = train_df.groupby('user_id' ).tail(24 ).reset_index(drop = True )<load_from_csv>
data_sample.head()
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols = [0, 3],
dtype = {'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 train_df.drop(columns = ['question_id'], inplace = True )<data_type_conversions>
merged = pd.concat([data_tittrain,data_tittest] )
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )<drop_column>
data_tittrain["Sex"].value_counts()
valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace = True )<init_hyperparams>
data_tittest["Sex"].value_counts()
features = ['content_id', 'prior_question_elapsed_time',
'prior_question_had_explanation', 'user_correctness',
'part', 'content_count']
params = {
'loss_function': 'Logloss',
'eval_metric': 'AUC',
'task_type': 'GPU' if torch.cuda.is_available() else 'CPU',
'grow_policy': 'Lossguide',
'iterations': 2500,
'learning_rate': 4e-2,
'random_seed': 0,
'l2_leaf_reg': 1e-1,
'depth': 15,
'max_leaves': 10,
'border_count': 128,
'verbose': 50,
}<define_variables>
data_tittrain["Pclass"].value_counts()
train_set = Pool(train_df[features], label = train_df[target])
 val_set = Pool(valid_df[features], label = valid_df[target] )<train_model>
data_tittest['Pclass'].value_counts()
model = CatBoostClassifier(**params)
 model.fit(train_set, eval_set = val_set, use_best_model = True )<data_type_conversions>
data_tittrain = data_tittrain.drop(columns=['Name','Cabin'])
 data_tittrain['family_member'] = data_tittrain['SibSp'] + data_tittrain['Parch']
data_tittrain = data_tittrain.drop(columns=['SibSp', 'Parch'] )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<split>
data_tittest = data_tittest.drop(columns=['Name','Cabin'])
 data_tittest['family_member'] = data_tittest['SibSp'] + data_tittest['Parch']
data_tittest = data_tittest.drop(columns=['SibSp', 'Parch'] )
try:
env = riiideducation.make_env()
except:
pass
iter_test = env.iter_test()
prior_test_df = None<prepare_output>
X = data_tittrain.drop(columns=['Survived'])
 Y = data_tittrain['Survived']
cate_features_index = np.where(X.dtypes != float)[0]
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop = True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 user_sum = np.zeros(len(test_df), dtype = np.int16)
 user_count = np.zeros(len(test_df), dtype = np.int16)
 content_sum = np.zeros(len(test_df), dtype = np.int32)
 content_count = np.zeros(len(test_df), dtype = np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_correctness'] = user_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df[target] = model.predict_proba(test_df[features])[:,1]
env.predict(test_df[['row_id', target]] )<import_modules>
from catboost import CatBoostClassifier, cv, Pool
from sklearn.model_selection import train_test_split
import optuna
from sklearn.feature_selection import RFECV
from lightgbm import LGBMClassifier
import pandas as pd
import joblib
import numpy as np
import riiideducation
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import optuna.integration.lightgbm as lgb
from optuna.samplers import TPESampler
import matplotlib.pyplot as plt<load_from_csv>
xtrain,xtest,ytrain,ytest = train_test_split(X,Y,train_size=0.82,random_state=42 )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16', 'part': 'int8'} 
)
 model = joblib.load('.. /input/lgb-model-2/Final_lgb_2.joblib')
 selector = joblib.load('.. /input/selector-2/Selector_2.joblib')
 user_answers_df = pd.read_csv('.. /input/preprocessingcontentuser/user.csv')
 content_answers_df = pd.read_csv('.. /input/preprocessingcontentuser/content.csv' )<define_variables>
clf =CatBoostClassifier(eval_metric='Accuracy',use_best_model=True,random_seed=42 )
features = [
'timestamp',
'user_id',
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'part',
'mean_user_accuracy',
'questions_skew',
'questions_std',
'questions_var',
'questions_sem',
'content_mean',
'content_skew',
'content_std',
'content_var',
'content_sem'
]
features = [features[i] for i in range(len(selector.support_)) if selector.support_[i] == True]<split>
clf.fit(xtrain,ytrain,cat_features=cate_features_index,eval_set=(xtest,ytest), early_stopping_rounds=50 )
env = riiideducation.make_env()
iter_test = env.iter_test()<merge>
test_id = data_tittest.PassengerId
data_tittest.isnull().sum()
for(test_df, sample_prediction_df)in iter_test:
test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df.drop(columns=['question_id'], inplace=True)
 test_df['prior_question_had_explanation'].fillna(bool(True), inplace=True)
 test_df = test_df.replace([-np.inf, np.inf], np.nan)
 test_df = test_df.fillna(test_df.mean())
 test_df = test_df[test_df['content_type_id'] != 1]
test_df = test_df.merge(user_answers_df, how='left', on='user_id')
 test_df = test_df.merge(content_answers_df, how='left', on='content_id')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(bool)
 test_df = test_df.replace([-np.inf, np.inf], np.nan)
 test_df = test_df.fillna(test_df.mean())
 test_df['answered_correctly'] = model.predict_proba(test_df[features])[:,1]
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
<import_modules>
prediction = clf.predict(data_tittest )
import numpy as np
import pandas as pd 
<set_options>
df_sub = pd.DataFrame()
df_sub['PassengerId'] = test_id
df_sub['Survived'] = prediction.astype(np.int)
 df_sub.to_csv('gender_submission.csv', index=False )