Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size=MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 FFN()<categorify>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_len=len(train_df)
 train_len
def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 future_mask(5 )<choose_model_class>
dataset=pd.concat(objs=[train_df, test_df], axis=0 ).reset_index(drop=True )
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<set_options>
train_df.isnull().sum()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<choose_model_class>
dataset.isnull().sum()
def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)
 model = create_model()
model<train_model>
model(sample_batch[0], sample_batch[1])[0]<define_variables>
dataset_title=[i.split(",")[1].split('.')[0].strip() for i in dataset['Name']]
dataset['Title']=pd.Series(dataset_title)
 dataset.head()
LR = 2e-3
EPOCHS = 10
MODEL_PATH = '/kaggle/working/sakt.pth'<choose_model_class>
dataset['Title'] = dataset['Title'].replace(['Lady', 'the Countess','Capt', 'Col',\
'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 dataset[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
def do_train() :
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR,
steps_per_epoch=len(train_dataloader), epochs=EPOCHS)
 model.to(device)
 criterion.to(device)
 best_auc = 0.0
for epoch in range(EPOCHS):
train_epoch(model, train_dataloader, optimizer, criterion, scheduler, device)
 val_loss, avl_acc, val_auc = val_epoch(model, val_dataloader, criterion, device)
 print(f"epoch - {epoch + 1} val_loss - {val_loss:.3f} acc - {avl_acc:.3f} auc - {val_auc:.3f}")
 if best_auc < val_auc:
print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')
 best_auc = val_auc
torch.save(model.state_dict() , MODEL_PATH )<train_model>
train_df[['Sex','Survived']].groupby(['Sex'],as_index=False ).mean().sort_values(by='Survived',ascending=False)
do_train()<train_model>
Ticket = []
for i in list(dataset.Ticket):
if not i.isdigit() :
Ticket.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket.append("X")
 dataset["Ticket"] = Ticket
LR = LR/10.
 EPOCHS = 3
do_train()<load_pretrained>
dataset["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in dataset['Cabin'] ])
model = create_model()
model.load_state_dict(torch.load(MODEL_PATH))
model.to(device )<split>
freq_port=train_df.Embarked.dropna().mode() [0]
freq_port
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
dataset['Embarked']=dataset['Embarked'].fillna(freq_port )
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_answered_correctly = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_answered_correctly))
else:
group[prev_user_id] =(prev_group_content, prev_group_answered_correctly)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_answered_correctly = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_answered_correctly)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 item = next(iter(test_dataloader))
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
test_df['answered_correctly'] = output.cpu().numpy()
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<create_dataframe>
train_df[['Pclass','Survived']].groupby(['Pclass'],as_index=False ).mean().sort_values(by='Survived',ascending=False )
test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ )<load_pretrained>
train_df[['SibSp','Survived']].groupby(['SibSp'],as_index=False ).mean().sort_values(by='Survived',ascending=False )
group.to_pickle('/kaggle/working/group.pkl' )<load_from_csv>
train_df[['Parch','Survived']].groupby(['Parch'],as_index=False ).mean().sort_values(by='Survived',ascending=False )
full_train = pd.read_pickle(".. /input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip")
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<drop_column>
dataset["Fsize"] = dataset["SibSp"] + dataset["Parch"] + 1
dataset['Single'] = dataset['Fsize'].map(lambda s: 1 if s == 1 else 0)
 dataset['SmallF'] = dataset['Fsize'].map(lambda s: 1 if s == 2 else 0)
 dataset['MedF'] = dataset['Fsize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 dataset['LargeF'] = dataset['Fsize'].map(lambda s: 1 if s >= 5 else 0)
full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]
train = full_train.groupby('user_id' ).tail(400)
 test = full_train.groupby('user_id' ).tail(4)
 train = train.drop(test.index )<merge>
dataset["Fare"] = dataset["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
question_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean() ).rename(columns={'answered_correctly':'question_average'})
 question_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_average.join(question_count)
 question_df['question_sum'] = question_df['question_average'] * question_df['question_count']
question_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]
question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 del question_average,question_count
gc.collect()<merge>
dataset['Fare']=dataset['Fare'].fillna(dataset['Fare'].dropna().median())
train = train.join(question_df,on=['content_id'], rsuffix='_question' )<drop_column>
dataset.loc[dataset['Fare'] <=2.06,'Fare']=0
dataset.loc[(dataset['Fare'] <=2.67)&(dataset['Fare'] > 2.06),'Fare']=1
dataset.loc[(dataset['Fare'] <=3.44)&(dataset['Fare'] > 2.67),'Fare']=2
dataset.loc[(dataset['Fare'] <=6.2)&(dataset['Fare'] > 3.44),'Fare']=3
dataset.loc[dataset['Fare'] > 6.2 ,'Fare']=4
dataset['Fare']=dataset['Fare'].astype(int )
del full_train
gc.collect()<data_type_conversions>
index_NaN_age = list(dataset["Age"][dataset["Age"].isnull() ].index)
 for i in index_NaN_age:
age_med = dataset["Age"].median()
age_pred = dataset["Age"][(( dataset['SibSp'] == dataset.iloc[i]["SibSp"])& 
(dataset['Parch'] == dataset.iloc[i]["Parch"])& 
(dataset['Pclass'] == dataset.iloc[i]["Pclass"])) ].median()
if not np.isnan(age_pred):
dataset['Age'].iloc[i] = age_pred
else :
dataset['Age'].iloc[i] = age_med
def reduce_memory_usage(train_data):
train_data['prior_question_had_explanation'] = train_data['prior_question_had_explanation'].fillna(False ).astype('bool')
 return train_data<groupby>
dataset['AgeBand']=pd.cut(dataset['Age'],5)
mean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()
mean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()<groupby>
dataset[['AgeBand','Survived']].groupby(['AgeBand'],as_index=False ).mean()
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()<feature_engineering>
dataset.loc[dataset['Age'] <=16 , 'Age' ] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <=32), 'Age' ] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <=48), 'Age' ] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <=64), 'Age' ] = 3
dataset.loc[(dataset['Age'] > 64), 'Age' ] = 4
dataset['Age']=dataset['Age'].astype(int)
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
cumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])
 train.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] /(cumulated_question['cumcount'] + 1)
 train.drop(columns=['user_shift_question'],inplace=True)
 train['average_past_questions'].fillna(mean_question,inplace=True)
 del cumulated_question<feature_engineering>
dataset=dataset.drop(['Name','Parch','PassengerId','SibSp','Fsize','FareBand','AgeBand'],axis=1 )
train['user_shift'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].shift()
cumulated = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift'].agg(['cumsum', 'cumcount'])
 train.loc[train['content_type_id'] == False,'answered_correctly_user_average'] = cumulated['cumsum'] / cumulated['cumcount']
train['answered_correctly_user_average'].fillna(mean_user,inplace=True)
 train.drop(columns=['user_shift'], inplace=True)
 del cumulated<create_dataframe>
dataset=pd.get_dummies(dataset)
user_average = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['answered_correctly_user_average'].last() ).rename(columns={'answered_correctly_user_average':'user_average'})
 user_count = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'] ).size() - 1,columns=['user_count'] )<create_dataframe>
dataset=pd.get_dummies(dataset,prefix=['Pclass','Age','Fare'],columns=['Pclass','Age','Fare'] )
tmp = train.loc[train['content_type_id'] == False].groupby(['user_id'] ).mean()
train['performance_before'] = train['answered_correctly_user_average'] - train['average_past_questions']
user_performance = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['performance_before'].last() ).rename(columns={'performance_before':'performance'})
 del tmp<set_options>
train = dataset[:train_len]
test = dataset[train_len:]
test.drop(labels=["Survived"],axis = 1,inplace=True )
gc.collect()<merge>
train["Survived"] = train["Survived"].astype(int)
 X_train = train.drop(labels = ["Survived"],axis = 1)
 Y_train = train["Survived"]
user_df = user_performance.join(user_average ).join(user_count)
 user_df['user_sum'] = user_df['user_average'] * user_df['user_count']<groupby>
X_test=test
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'] )<prepare_output>
kfold= StratifiedKFold(n_splits=10 )
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_<create_dataframe>
random_state=2
classifiers=[]
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state=random_state))
classifiers.append(LinearDiscriminantAnalysis())
 classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(XGBClassifier(random_state=random_state))
cv_results=[]
for classifier in classifiers:
cv_results.append(cross_val_score(estimator=classifier,X=X_train,y=Y_train,cv=kfold,scoring='accuracy',n_jobs=-1))
cv_means=[]
cv_std=[]
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res=pd.DataFrame({'CrossValMeans':cv_means , 'CrossValErrors':cv_std , 'Algorithm':['SVC','DTC','RFC','KNN','LR','LDA','ADA','XT','GBC','XGB']})
 sns.barplot('CrossValMeans','Algorithm',data=cv_res)
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df<create_dataframe>
svc_classifier=SVC(probability=True)
 svc_param_grid = [{'C': [1], 'kernel': ['rbf'],
'gamma': [0.1],
'cache_size':[100],
'coef0':[0.1],
'degree':[1],
'tol':[0.001]}]
gs_SVC = GridSearchCV(estimator = svc_classifier,
param_grid = svc_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_SVC = gs_SVC.fit(X_train, Y_train)
 svc_best_params = gs_SVC.best_params_
svc_best_score = gs_SVC.best_score_
svc_best=gs_SVC.best_estimator_
svc_best_score
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df<create_dataframe>
dtc_classifier=DecisionTreeClassifier()
dtc_param_grid = [{'criterion': ['gini'],
"min_samples_split": [2],
"max_depth": [None],
"min_samples_leaf": [5],
"max_leaf_nodes": [10],
'splitter': ['best']}]
gs_DTC = GridSearchCV(estimator = dtc_classifier,
param_grid = dtc_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_DTC = gs_DTC.fit(X_train, Y_train)
 dtc_best_params = gs_DTC.best_params_
dtc_best_score = gs_DTC.best_score_
dtc_best=gs_DTC.best_estimator_
dtc_best_score
prior_df = pd.DataFrame()
current_df = pd.DataFrame()
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df )<define_variables>
rfc_classifier=RandomForestClassifier()
rfc_param_grid = [{'n_estimators':[1200] ,'criterion': ['entropy'],
'max_features':['auto'] ,'min_samples_split':[9],'min_samples_leaf':[2],
'bootstrap' : [True], 'n_jobs':[-1] ,'oob_score':[True]}]
gs_RFC = GridSearchCV(estimator = rfc_classifier,
param_grid = rfc_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_RFC = gs_RFC.fit(X_train, Y_train)
 rfc_best_params = gs_RFC.best_params_
rfc_best_score = gs_RFC.best_score_
rfc_best=gs_RFC.best_estimator_
rfc_best_score
TARGET_COL = ['answered_correctly']
FEATURE_COLS = ['row_id', 'performance', 'question_average']<merge>
knn_classifier=KNeighborsClassifier()
knn_param_grid = [{'n_neighbors':[10],'weights':['uniform'],'algorithm':['brute'],
}]
gs_KNN = GridSearchCV(estimator = knn_classifier,
param_grid = knn_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_KNN = gs_KNN.fit(X_train, Y_train)
 knn_best_params = gs_KNN.best_params_
knn_best_score = gs_KNN.best_score_
knn_best=gs_KNN.best_estimator_
knn_best_score
def data_transform(df, is_training = True, is_validation = True):
df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df['is_beginning'] = df['user_count'] < 20
df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS + TARGET_COL]
else:
df = df[FEATURE_COLS]
return df<categorify>
lr_classifier=LogisticRegression()
lr_param_grid = [{'penalty':['l1','l2'] , 'C':[1]}]
gs_LR = GridSearchCV(estimator = lr_classifier,
param_grid = lr_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_LR = gs_LR.fit(X_train, Y_train)
 lr_best_params = gs_LR.best_params_
lr_best_score = gs_LR.best_score_
lr_best=gs_LR.best_estimator_
lr_best_score
%%time
train = data_transform(train )<categorify>
adadtc_classifier=AdaBoostClassifier(dtc_best)
 adadtc_param_grid = [{'n_estimators':[500],
"learning_rate": [0.1],
"algorithm" : ["SAMME"],
}]
gs_ADADTC = GridSearchCV(estimator = adadtc_classifier,
param_grid = adadtc_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_ADADTC = gs_ADADTC.fit(X_train, Y_train)
 adadtc_best_params = gs_ADADTC.best_params_
adadtc_best_score = gs_ADADTC.best_score_
adadtc_best=gs_ADADTC.best_estimator_
adadtc_best_score
test = data_transform(test,False )<prepare_x_and_y>
xt_classifier=ExtraTreesClassifier()
xt_param_grid = [{'n_estimators':[100],
'criterion':['entropy'],
'max_features':[None],
'max_depth':[None],
'min_samples_split':[2],
'min_samples_leaf':[10]
}]
gs_XT = GridSearchCV(estimator = xt_classifier,
param_grid = xt_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_XT = gs_XT.fit(X_train, Y_train)
 xt_best_params = gs_XT.best_params_
xt_best_score = gs_XT.best_score_
xt_best=gs_XT.best_estimator_
xt_best_score
X_train = train[FEATURE_COLS]
y_train = train[TARGET_COL]
X_test = test[FEATURE_COLS]
y_test = test[TARGET_COL]<create_dataframe>
votingC = VotingClassifier(estimators=[('svc', svc_best),('dtc', dtc_best), 
('rfc', rfc_best),('knn',knn_best),('lr',lr_best),('adadtc',adadtc_best),('xt',xt_best)], voting='soft', n_jobs=-1)
 votingC = votingC.fit(X_train, Y_train )
lgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)
 lgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test )<set_options>
test_Survived = pd.Series(votingC.predict(test), name="Survived")
 results = pd.concat([IDtest,test_Survived],axis=1)
 results.to_csv("ensemble_python_voting.csv",index=False )