Kaggle competition: Titanic - Machine Learning from Disaster
<merge><EOS> 
import pandas as pd
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
import pandas as pd
CATEGORY = 20 
(
 items[items['item_id'].isin(X_test.item_id.unique())]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={'lgb_pred':'test_pred'}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('test_pred',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'item_category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )<load_pretrained>
pd.set_option('display.max_rows', 1000)
 %pip install ppscore
seed =2055
plt.style.use('fivethirtyeight' )
M = pd.read_pickle('/kaggle/input/sales-data-prep/matrix.pkl')
 M.columns<load_pretrained>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
M = pd.read_pickle('/kaggle/input/sales-data-prep/matrix.pkl')
 M.drop(["new_item_cat_enc_lag_1", "new_item_cat_enc_lag_2", "new_item_cat_enc_lag_3"], axis=1, inplace=True)
 M = M[M["date_block_num"] > 2]
M.fillna(0)
 for col in M.columns:
print(col,M[col].nunique())
 def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
X_train = M[M.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = M[M.date_block_num < 33]['item_cnt_month']
X_test = M[M.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_val = M[M.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_val = M[M.date_block_num == 33]['item_cnt_month']
del M
gc.collect()<train_model>
def basic_analysis(df1, df2):

 b = pd.DataFrame()
b['First df_mean'] = round(df1.mean() ,2)
 b['Second df_mean'] = round(df2.mean() ,2)
 c =(b['First df_mean']/b['Second df_mean'])
 if [c<=1]:
b['Variation, %'] = round(( 1-(( b['First df_mean']/b['Second df_mean'])))*100)
 else:
b['Variation, %'] = round(((b['First df_mean']/b['Second df_mean'])-1)*100)
 b['Influence'] = np.where(abs(b['Variation, %'])<= 9, "feature's effect on the target is not defined",
"feature value affects the target")
 return b
model = XGBRegressor(
 max_depth=10,
booster='gbtree',
n_estimators=1000,
min_child_weight=0.5,
subsample=0.8,
sampling_method="uniform",
colsample_bynode=1,
colsample_bytree=0.8,
eta=0.1,
tree_method='gpu_hist',
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_val, Y_val)],
verbose=True,
early_stopping_rounds = 20 )<load_pretrained>
train.loc[train['PassengerId'] == 631, 'Age'] = 48
train.loc[train['PassengerId'] == 69, ['SibSp', 'Parch']] = [0,0]
test.loc[test['PassengerId'] == 1106, ['SibSp', 'Parch']] = [0,0]
pickle.dump(model, open("model.pkl", "wb"))<load_pretrained>
train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
loaded_model = pickle.load(open("model.pkl", "rb"))<load_from_csv>
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.7 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2,["Age","SibSp","Parch","Fare"])
 train.loc[Outliers_to_drop]
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 Y_test = loaded_model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
} )<load_from_csv>
df = pd.concat(( train.loc[:,'Pclass':'Embarked'], test.loc[:,'Pclass':'Embarked'])).reset_index(drop=True )
items=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/items.csv")
 item_categories=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv" )<merge>
survived = train.drop(train[train['Survived'] != 1].index)
 not_survived = train.drop(train[train['Survived'] != 0].index)
 basic_analysis(survived,not_survived )
df = pd.merge(items, item_categories)
 df<save_to_csv>
def basic_details(df):
b = pd.DataFrame()
b['Missing value, %'] = round(df.isnull().sum() /df.shape[0]*100)
 b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )
submission.to_csv('my_submission.csv', index=False )<import_modules>
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df['Title'] = df['Title'].replace('Mlle', 'Miss')
 df['Title'] = df['Title'].replace('Ms', 'Miss')
 df['Title'] = df['Title'].replace('Mme', 'Mrs')
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
df['Title'] = df['Title'].map(title_mapping)
 df['Title'] = df['Title'].fillna(0)
 df = df.drop(['Name'], axis=1)
 df["Sex"][df["Sex"] == "male"] = 0
df["Sex"][df["Sex"] == "female"] = 1
df["Sex"] = df["Sex"].astype(int)
 df['Age'] = df.groupby(['Pclass'])['Age'].transform(lambda x: x.fillna(x.median()))
df["Age"] = df["Age"].astype(int)
 df['Age_cat'] = pd.qcut(df['Age'],q=[0,.16,.33,.49,.66,.83, 1], labels=False, precision=1)
 tickets = []
for i in list(df.Ticket):
if not i.isdigit() :
tickets.append(i.replace(".","" ).replace("/","" ).strip().split(" ")[0])
 else:
tickets.append("x")
 df["Ticket"] = tickets
df['Ticket_Frequency'] = df.groupby('Ticket')['Ticket'].transform('count')
 df = pd.get_dummies(df, columns= ["Ticket"], prefix = "T")
 df['Fare'] = df.groupby("Pclass")['Fare'].transform(lambda x: x.fillna(x.median()))
df['Zero_Fare'] = df['Fare'].map(lambda x: 1 if x == 0 else(0))
def fare_category(fr):
if fr <= 7.91:
return 1
elif fr <= 14.454 and fr > 7.91:
return 2
elif fr <= 31 and fr > 14.454:
return 3
return 4
df['Fare_cat'] = df['Fare'].apply(fare_category)
 df['Cabin'] = df['Cabin'].fillna('U')
 df['Cabin'] = df['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 cabin_category = {'A':9, 'B':8, 'C':7, 'D':6, 'E':5, 'F':4, 'G':3, 'T':2, 'U':1}
df['Cabin'] = df['Cabin'].map(cabin_category)
 df["Embarked"] = df["Embarked"].fillna("C")
 df["Embarked"][df["Embarked"] == "S"] = 1
df["Embarked"][df["Embarked"] == "C"] = 2
df["Embarked"][df["Embarked"] == "Q"] = 3
df["Embarked"] = df["Embarked"].astype(int)
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['FamilySize_cat'] = df['FamilySize'].map(lambda x: 1 if x == 1
else(2 if 5 > x >= 2
else(3 if 8 > x >= 5
else 4)
))
df['Alone'] = [1 if i == 1 else 0 for i in df['FamilySize']]
dummy_col=['Title', 'Sex', 'Age_cat', 'SibSp', 'Parch', 'Fare_cat', 'Embarked', 'Pclass', 'FamilySize_cat']
dummy = pd.get_dummies(df[dummy_col], columns=dummy_col, drop_first=False)
 df = pd.concat([dummy, df], axis = 1)
 dummy_fare = ['Fare']
dummy_f = pd.get_dummies(df[dummy_fare], columns=dummy_fare, drop_first=True)
 df = pd.concat([dummy_f, df], axis = 1)
 df['FareCat_Sex'] = df['Fare_cat']*df['Sex']
df['Pcl_Sex'] = df['Pclass']*df['Sex']
df['Pcl_Title'] = df['Pclass']*df['Title']
df['Age_cat_Sex'] = df['Age_cat']*df['Sex']
df['Age_cat_Pclass'] = df['Age_cat']*df['Pclass']
df['Title_Sex'] = df['Title']*df['Sex']
df['SmallF'] = df['FamilySize'].map(lambda s: 1 if s == 2 else 0)
 df['MedF'] = df['FamilySize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df['LargeF'] = df['FamilySize'].map(lambda s: 1 if s >= 5 else 0)
 df['Senior'] = df['Age'].map(lambda s:1 if s>70 else 0)
 df["Fare"] = df["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
import psutil
import joblib
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader<split>
def des_stat_feat(df):
df = pd.DataFrame(df)
 dcol= [c for c in df.columns if df[c].nunique() >=10]
d_median = df[dcol].median(axis=0)
 d_mean = df[dcol].mean(axis=0)
 q1 = df[dcol].apply(np.float32 ).quantile(0.25)
 q3 = df[dcol].apply(np.float32 ).quantile(0.75)
 for c in dcol:
df[c+str('_median_range')] =(df[c].astype(np.float32 ).values > d_median[c] ).astype(np.int8)
 df[c+str('_mean_range')] =(df[c].astype(np.float32 ).values > d_mean[c] ).astype(np.int8)
 df[c+str('_q1')] =(df[c].astype(np.float32 ).values < q1[c] ).astype(np.int8)
 df[c+str('_q3')] =(df[c].astype(np.float32 ).values > q3[c] ).astype(np.int8)
 return df
env = riiideducation.make_env()
iter_test = env.iter_test()<define_variables>
def basic_details(df):
b = pd.DataFrame()
b['Missing value'] = df.isnull().sum()
b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )
MAX_SEQ = 100<define_search_model>
df = df.loc[:,~df.columns.duplicated() ]
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<load_pretrained>
df.apply(lambda x: sum(x.isnull()),axis=0 )
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip" )<load_pretrained>
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.model_selection import train_test_split
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 try:
model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
model.to(device)
 model.eval()<feature_engineering>
le = LabelEncoder()
for col in df.select_dtypes('object' ).columns:
df[col] = le.fit_transform(df[col] )
prev_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(prev_test_df is not None)&(psutil.virtual_memory().percent < 90):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
X_train = df[:train.shape[0]]
X_test_fin = df[train.shape[0]:]
y = train.Survived
X_train['Y'] = y
df = X_train
df.head(20)
 X = df.drop('Y', axis=1)
 y = df.Y
import gc
import random
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader<define_variables>
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=2020 )
MAX_SEQ = 160 
<load_from_csv>
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(X_test_fin)
 params = {
'objective':'binary:logistic',
'max_depth':10,
'learning_rate':0.1,
'eval_metric':'auc',
'min_child_weight':1,
'subsample':0.64,
'colsample_bytree':0.4,
'seed':45,
'reg_lambda':2.79,
'reg_alpha':0.1,
'gamma':0,
'scale_pos_weight':1.68,
'nthread':-1
}
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
nrounds=12000
model = xgb.train(params, d_train, nrounds, watchlist, early_stopping_rounds=100,
maximize=True, verbose_eval=10 )
%%time
dtype = {'timestamp':'int64',
'user_id':'int32' ,
'content_id':'int16',
'content_type_id':'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)
 train_df.head()<sort_values>
accuracy = pd.DataFrame()
accuracy['predict'] = model.predict(d_valid)
 accuracy['predict'] = accuracy['predict'].apply(lambda x: 1 if x>0.6 else 0)
 accuracy_score(y_valid, accuracy['predict'] )
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )<count_unique_values>
sub = pd.DataFrame()
sub['PassengerId'] = test['PassengerId']
sub['Survived'] = model.predict(d_test)
 sub['Survived'] = sub['Survived'].apply(lambda x: 1 if x>0.6 else 0 )
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))<groupby>
leaks = {
897:1,
899:1,
930:1,
932:1,
949:1,
987:1,
995:1,
998:1,
999:1,
1016:1,
1047:1,
1083:1,
1097:1,
1099:1,
1103:1,
1115:1,
1118:1,
1135:1,
1143:1,
1152:1,
1153:1,
1171:1,
1182:1,
1192:1,
1203:1,
1233:1,
1250:1,
1264:1,
1286:1,
935:0,
957:0,
972:0,
988:0,
1004:0,
1006:0,
1011:0,
1105:0,
1130:0,
1138:0,
1173:0,
1284:0,
}
<define_variables><EOS> 
sub['Survived'] = sub.apply(lambda r: leaks[int(r['PassengerId'])] if int(r['PassengerId'])in leaks else r['Survived'], axis=1)
 sub.to_csv('submission.csv', index=False)
 sub.head()