Kaggle competition: Titanic - Machine Learning from Disaster
class SimpleReader() :
def __init__(self, file, y_train, batch_size, maxlen, index_filter=lambda x : True):
self.fn = file
self.file = open(file, 'r')
 self.k = 0
self.y_train = y_train
self.batch_size = batch_size
self.maxlen = maxlen
self.index_filter = index_filter
def flow(self):
X = []
y = []
self.k = 0
while True:
line = self.file.readline().replace('
', '')
 if line is not None and len(line)> 0:
if self.index_filter(self.k):
X.append(list(map(int, line.split(' '))))
y.append(self.y_train[self.k])
 self.k += 1
else:
self.file.close()
self.file = open(self.fn, 'r')
 self.k = 0
if len(X)== self.batch_size:
X = pad_sequences(X, maxlen=self.maxlen, truncating='post', padding='post')
 y = np.array(y)
 yield(( X, y))
X = []
y = []<choose_model_class>
path = '/kaggle/input/titanic/train.csv'
train_data = pd.read_csv(path)
 train_data
def create_model(embedding_matrix):
input_tensor = Input(shape=(None,))
output_tensor = input_tensor
output_tensor = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False )(output_tensor)
 trainable_embed = Embedding(embedding_matrix.shape[0], TRAINABLE_EMBED_SIZE, trainable=True )(input_tensor)
 output_tensor = Concatenate(-1 )([output_tensor, trainable_embed])
 output_tensor = SpatialDropout1D(0.3 )(output_tensor)
 output_tensor = Bidirectional(CuDNNLSTM(384, return_sequences=True))(output_tensor)
 output_tensor = Add()([Bidirectional(CuDNNGRU(384, return_sequences=True))(output_tensor), output_tensor])
 output_tensor = Add()([Bidirectional(CuDNNGRU(384, return_sequences=True))(output_tensor), output_tensor])
 output_tensor = Concatenate()([GlobalMaxPooling1D()(output_tensor), GlobalAveragePooling1D()(output_tensor)])
 output_tensor = Add()([Dense(384*4, activation='elu' )(output_tensor), output_tensor])
 output_tensor = Dense(CATEGORY_CNT, activation='softmax' )(output_tensor)
 model = Model(inputs=input_tensor, outputs=output_tensor)
 return model<train_model>
path = '/kaggle/input/titanic/test.csv'
test_data = pd.read_csv(path)
 test_data
%%time
train_generator = SimpleReader(".. /input/bonus-x-txt-fasttext-unk/X_train_fasttext_unk.txt", y_train, BATCH_SIZE, MAXLEN, index_filter=lambda x : x % 30 != 7)
 val_generator = SimpleReader(".. /input/bonus-x-txt-fasttext-unk/X_train_fasttext_unk.txt", y_train, BATCH_SIZE, MAXLEN, index_filter=lambda x : x % 30 == 7)
 lr_scheldure = LearningRateScheduler(lambda epoch, lr: EXP_DECAY_COEF ** epoch * 1e-3, verbose=VERBOSE)
 model = create_model(embedding_matrix)
 model.compile(optimizer=Adam(lr=1e-3), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 model.fit_generator(
 train_generator.flow() ,
validation_data=val_generator.flow() ,
steps_per_epoch=STEPS_PER_EPOCH,
validation_steps=VAL_STEPS_PER_EPOCH,
epochs=EPOCHS,
verbose=VERBOSE,
callbacks=[lr_scheldure]
)<predict_on_test>
path = '/kaggle/input/titanic/gender_submission.csv'
gender_submission_data = pd.read_csv(path)
 gender_submission_data
predictions = np.ones(( len(itemid),), dtype=np.int)* -1
gen = SimpleReader(".. /input/bonus-x-txt-fasttext-unk/X_test_fasttext_unk.txt", np.arange(0, len(itemid), 1), PREDICT_BATCH_SIZE, MAXLEN)
 flow = gen.flow()
for i in tqdm(range(( len(itemid)+ PREDICT_BATCH_SIZE - 1)// PREDICT_BATCH_SIZE), disable=not(VERBOSE)) :
batch = next(flow)
 batch_predictions = model.predict_on_batch(batch[0])
 predictions[batch[1]] = batch_predictions.argmax(-1 )<load_from_csv>
df = pd.concat([train_data, test_data], ignore_index=True)
 df.reset_index(drop=True, inplace=True)
 df
submission = pd.read_csv(".. /input/texts-classification-ml-hse-2019/sample_submission.csv")
 submission['Category'] = predictions
submission['Id'] = itemid<feature_engineering>
submission['Category'] = submission['Category'].apply(lambda x : b_cat_map[x] )<save_to_csv>
def NaN_info(df):
global null_view
try:
null_view = df[[col for col in df.columns if df[col].isna().sum() > 0]].isna().sum().sort_values(ascending = True)
 null_view = pd.DataFrame(null_view, columns=['NANs'])
 null_view[['PERCENT']] = null_view.NANs.apply(lambda x: round(( x/len(df)) *100, 2))
null_view[['TYPE']] = df.dtypes
except:
return null_view
return null_view
NaN_info(df )
submission.to_csv("submission.csv", index=False )<save_to_csv>
df['Survived'].value_counts(dropna=False )
submission.to_csv("submission.csv", index=False )<load_from_csv>
df['Survived'].replace(to_replace=0, value='no', inplace=True)
 df['Survived'].replace(to_replace=1, value='yes', inplace=True)
 df['Survived'] = df['Survived'].astype('object')
 df['Survived'].value_counts(dropna=False )
train_data = pd.read_csv('/kaggle/input/nmlo-contest-3/train.csv')
 test_data = pd.read_csv('/kaggle/input/nmlo-contest-3/test.csv')
 train = train_data.copy()
test = test_data.copy()<count_missing_values>
def NaN_predict(df,
skip_features_with_missing_data_percentage = 90,
include_features_as_predictors_where_pec_miss_data_less = 10,
apply_fast_predict_where_missing_data_less_than_percent = 1 
):
if not sys.warnoptions:
warnings.simplefilter("ignore")
 global counter_all_predicted_values
counter_all_predicted_values = 0
global numeric_features
numeric_features = []
PARAMS = {'num_leaves': np.arange(50, 1000, step=50),
'max_depth': np.arange(10, 26, step=2),
'learning_rate': [0.001, 0.01, 0.1],
'n_estimators': np.arange(500, 2000, step=100),
'subsample': [0.1, 0.2, 0.3, 0.4, 0.5],
'feature_fraction': [0.1, 0.2, 0.3, 0.4, 0.5],
'bagging_fraction': [0.1, 0.2, 0.3, 0.4, 0.5],
'bagging_seed': np.arange(1, 20, step=1),
'lambda_l1': [0.1, 0.2, 0.3],
'lambda_l2': [0.1, 0.2, 0.3],
'min_child_samples': np.arange(5, 50, step=5),
'min_split_gain': [0.00001, 0.0001, 0.001]}
def NaN_info(df):
global null_view
try:
null_view = df[[col for col in df.columns if df[col].isna().sum() > 0]].isna().sum().sort_values(ascending = True)
 null_view = pd.DataFrame(null_view, columns=['NANs'])
 null_view[['PERCENT']] = null_view.NANs.apply(lambda x: round(( x/len(df)) *100, 2))
null_view[['TYPE']] = df.dtypes
except:
return null_view
return null_view
def encoding(work_predictors, df):
feature_power = 0.5
for j in work_predictors:
el_type = df[j].dtype
if el_type == 'object':
df[j].replace(np.nan, '0', inplace=True)
 labelencoder = LabelEncoder()
df.loc[:, j] = labelencoder.fit_transform(df.loc[:, j])
 else:
df[j] = df[j]**feature_power
return df, work_predictors
def imput_missing_value_to_main_df(df, miss_indeces, pred_miss, el):
counter = 0
for idx in miss_indeces:
df.loc[idx, el] = pred_miss[counter]
counter += 1
return df
def hyperparms_tuning_regressor(X_train, X_test, y_train, y_test, n_iter_for_RandomizedSearchCV, PARAMS):
global best_params
lgbm = lgb.LGBMRegressor(n_jobs = -1)
 lgbm_randomized_mse = RandomizedSearchCV(estimator=lgbm,
param_distributions=PARAMS,
n_iter=n_iter_for_RandomizedSearchCV,
scoring='neg_mean_squared_error',
cv=3,
verbose=0,
n_jobs = -1)
 lgbm_randomized_mse.fit(X_train, y_train)
 best_params = lgbm_randomized_mse.best_params_
print(best_params)
 pred_test_lgb = lgbm_randomized_mse.predict(X_test)
 MAE = mean_absolute_error(y_test,pred_test_lgb)
 y_te = list(round(y_test[:10], 1))
y_pred = list(np.round(pred_test_lgb[:10], 1))
print(f'first 10 y_test: {y_te}')
 print(f'first 10 y_pred: {y_pred}')
 print(f'mean_absolute_error: {MAE}')
 return best_params
def predict_regressor(best_params, X, y, miss_df):
global pred_miss
lgbm = lgb.LGBMRegressor(**best_params, n_jobs = -1)
 lgbm.fit(X, y)
 pred_miss = list(lgbm.predict(miss_df))
print('-------------------------------')
 print(f"The first 100 predicted missing values:
{pred_miss[:100]}")
 return pred_miss
def hyperparms_tuning_classifier(X_train, X_test, y_train, y_test, n_iter_for_RandomizedSearchCV, PARAMS):
global best_params
lgbm = lgb.LGBMClassifier(n_jobs = -1)
 lgbm_randomized_mse = RandomizedSearchCV(estimator=lgbm,
param_distributions=PARAMS,
n_iter=n_iter_for_RandomizedSearchCV,
scoring='f1_weighted',
cv=3,
verbose=0,
n_jobs = -1)
 lgbm_randomized_mse.fit(X_train, y_train)
 best_params = lgbm_randomized_mse.best_params_
print(best_params)
 pred_test_lgb = lgbm_randomized_mse.predict(X_test)
 accuracy = accuracy_score(y_test, pred_test_lgb)
 print(f'first 10 y_test: {y_test[:10]}')
 print(f'first 10 y_pred: {pred_test_lgb[:10]}')
 f1 = f1_score(y_test, pred_test_lgb, average='weighted')
 print(f'accuracy_score: {accuracy}')
 print(f'f1_score(weighted): {f1}')
 return best_params
def predict_classifier(best_params, X, y, miss_df):
global pred_miss
lgbm = lgb.LGBMClassifier(**best_params, n_jobs = -1)
 lgbm.fit(X, y)
 pred_miss = list(lgbm.predict(miss_df))
print('-------------------------------')
 print(f"The first 100 predicted missing values:
{pred_miss[:100]}")
 return pred_miss
def light_predict_regressor(X, y, miss_df):
global pred_miss
lgbm = lgb.LGBMRegressor()
lgbm.fit(X, y)
 pred_miss = list(lgbm.predict(miss_df))
print('-------------------------------')
 print(f"The first 100 predicted missing values:
{pred_miss[:100]}")
 return pred_miss
def light_predict_classifier(X, y, miss_df):
global pred_miss
lgbm = lgb.LGBMClassifier()
lgbm.fit(X, y)
 pred_miss = list(lgbm.predict(miss_df))
print('-------------------------------')
 print(f"The first 100 predicted missing values:
{pred_miss[:100]}")
 return pred_miss
def numeric_features(df):
num_features = [feature for feature in df.columns if df[feature].dtype in ['int64', 'float64']]
return num_features
def integer_features(df):
global int_features
int_features = []
int_features = [feature for feature in df.columns if df[feature].dtype in ['int64']]
return int_features
print(NaN_info(df))
print('
')
 all_features = list(df.columns)
 df_indeces = list(df.index)
 integer_features(df)
 delete_miss_features = list(( null_view.loc[null_view['PERCENT'] > skip_features_with_missing_data_percentage] ).index)
 print(f'Exclude from the prediction, because missing data more than \
{skip_features_with_missing_data_percentage}% :
{delete_miss_features}')
 print('')
 all_miss_features = list(null_view.index)
 for delete_feature in delete_miss_features:
all_miss_features.remove(delete_feature)
 for el in all_miss_features:
print('
')
 NaN_info(df)
 lot_of_miss_features = list(( null_view.loc[null_view['PERCENT'] > include_features_as_predictors_where_pec_miss_data_less] ).index)
 now_predictors = list(set(all_features)-set(lot_of_miss_features))
work_predictors = list(set(now_predictors)- set([el]))
miss_indeces = list(( df[pd.isnull(df[el])] ).index)
 miss_df = df.iloc[miss_indeces][:]
miss_df = miss_df[work_predictors]
encoding(work_predictors, df=miss_df)
 work_indeces = list(set(df_indeces)- set(miss_indeces))
work_df = df.iloc[work_indeces][:]
encoding(work_predictors, df=work_df)
 X = work_df[work_predictors]
y = work_df[el]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0)
 feature_type = df[el].dtypes
percent = null_view['PERCENT'][el]
print(f'feature: {el}, type: {feature_type}, missing values: {percent}%')
 print(f'X.shape: {X.shape}, y.shape: {y.shape}')
 print(f'Unused for prediction, because missing data more than \
{include_features_as_predictors_where_pec_miss_data_less}% :')
 print(lot_of_miss_features)
 print('')
 if percent < apply_fast_predict_where_missing_data_less_than_percent:
print('light prediction without tuning hyperparameters, because missing data less than 1 %')
 if feature_type == 'object' or feature_type == 'bool':
print('FAST_classifier:')
 light_predict_classifier(X, y, miss_df)
 counter_all_predicted_values += len(miss_indeces)
 imput_missing_value_to_main_df(df, miss_indeces, pred_miss, el)
 print(f'miss_indexes:
{miss_indeces}')
 elif feature_type == 'float64' or feature_type == 'int64':
print('FAST_regressor:')
 light_predict_regressor(X, y, miss_df)
 counter_all_predicted_values += len(miss_indeces)
 imput_missing_value_to_main_df(df, miss_indeces, pred_miss, el)
 print(f'miss_indexes:
{miss_indeces}')
 else:
print(f"unprocessed feature: {el} - {feature_type} type")
 else:
n_iter_for_RandomizedSearchCV = int(300 + percent * 5)
 print(f'Iteration for RandomizedSearchCV: {n_iter_for_RandomizedSearchCV}')
 if feature_type == 'object' or feature_type == 'bool':
print('ADVANCED classifier evaluation:')
 labelencoder = LabelEncoder()
y_train = labelencoder.fit_transform(y_train)
 y_test = labelencoder.fit_transform(y_test)
 hyperparms_tuning_classifier(X_train, X_test, y_train, y_test, n_iter_for_RandomizedSearchCV, PARAMS)
 predict_classifier(best_params, X, y, miss_df)
 counter_all_predicted_values += len(miss_indeces)
 imput_missing_value_to_main_df(df, miss_indeces, pred_miss, el)
 elif feature_type == 'float64' or feature_type == 'int64':
print('ADVANCED regressor evaluation:')
 hyperparms_tuning_regressor(X_train, X_test, y_train, y_test, n_iter_for_RandomizedSearchCV, PARAMS)
 print(f'mean for {el}: {df[el].mean() }')
 predict_regressor(best_params, X, y, miss_df)
 counter_all_predicted_values += len(miss_indeces)
 imput_missing_value_to_main_df(df, miss_indeces, pred_miss, el)
 else:
print(f"unprocessed feature: {el} - {feature_type} type")
 for feature in int_features:
df[[feature]] = df[[feature]].astype('int64')
 print('
')
 print(f'These features have not been processed, because missing data more than {skip_features_with_missing_data_percentage}%')
 print(NaN_info(df))
print('
')
 print(f'{counter_all_predicted_values} values have been predicted and replaced')
 print('
')
 return df
train.isnull().sum()<count_missing_values>
NaN_predict(df,
skip_features_with_missing_data_percentage = 70 )
train.isnull().sum()<import_modules>
target_column = ['Survived']
predictors = list(set(list(df.columns)) -set(target_column))
predictors
from sklearn.preprocessing import StandardScaler<normalization>
df['Age'] = df['Age'].astype('int64')
 df['Fare'] = df['Fare'].astype('int64' )
cases_scaled = StandardScaler().fit_transform(train['cases'][:,np.newaxis]);
low_range = cases_scaled[cases_scaled[:,0].argsort() ][:10]
high_range= cases_scaled[cases_scaled[:,0].argsort() ][-10:]
print('outer range(low)of the distribution:')
 print(low_range)
 print('
outer range(high)of the distribution:')
 print(high_range )<import_modules>
for el in predictors:
print(f'======================= {el} =======================')
 print(df[el].value_counts(dropna=False))
from scipy.stats import norm, skew
from scipy import stats<count_values>
df['AgeCategory'] = df['Age']/5
df['AgeCategory'] = df['AgeCategory'].astype('int64')
 df['AgeCategory']
train['cases'].replace([0], 1, inplace = True)
 train['ed'].replace([0], 1, inplace = True)
 train['inc'].replace([0], 1, inplace = True)
 train['pop'].replace([0], 1, inplace = True)
 print(( train['cases']==0 ).value_counts())
 test['ed'].replace([0], 1, inplace = True)
 test['inc'].replace([0], 1, inplace = True)
 test['pop'].replace([0], 1, inplace = True )<feature_engineering>
indices_1 = list(np.where([df['Ticket'].str.contains(r'^\w.+\s', regex=True)])[1])
 indices_2 = list(np.where([df['Ticket'].str.contains(r'^\w.\s', regex=True)])[1])
 indices_3 = list(np.where([df['Ticket'].str.contains(r'^\w\s', regex=True)])[1])
 indices = list(set(indices_1 + indices_2 + indices_3))
print(len(indices))
print(indices )
train['cases'] = np.log(train['cases'])
<feature_engineering>
for el in indices[:20]:
ticket = df.loc[el, 'Ticket']
print(ticket )
res = stats.probplot(train['inc'], plot=plt)
 train['inc'] = np.log(train['inc'])
 test['inc'] = np.log(test['inc'])
<feature_engineering>
for el in indices:
ticket = df.loc[el, 'Ticket']
ticket = ticket.split(sep=' ', maxsplit=2)
 if len(ticket)== 3:
ticket[0] = ticket[0] + ticket[1]
ticket[1] = ticket[2]
del ticket[2]
print(f'{ticket[0]} {ticket[1]}')
 df.loc[el, 'Ticket'] = ticket[1]
df.loc[el, 'TicketSeries'] = ticket[0]
res = stats.probplot(train['pop'], plot=plt)
 train['pop'] = np.log(train['pop'])
 test['pop'] = np.log(test['pop'])
<categorify>
df['TicketSeries'].replace(np.nan, 'without', inplace=True)
 df['Ticket'] = df['Ticket'].astype('int64' )
train = pd.get_dummies(train)
 print(train.shape )<import_modules>
values = df['Ticket'].value_counts()
print(values[:40] )
from sklearn.model_selection import train_test_split<prepare_x_and_y>
LINE = df.loc[df['Ticket'] == 'LINE']
indeses = list(LINE.index)
 indeses
train_y = train['cases']
train.drop('id', axis = 1, inplace = True)
 train.drop('cases', axis = 1, inplace = True )<split>
for el in indeses:
print(el, 'LINE')
 df.loc[el, 'Ticket'] = 1000
df.loc[el, 'TicketSeries'] = 'LINE'
X_train, X_test, y_train, y_test = train_test_split(train, train_y, test_size=0.2, shuffle=True )<import_modules>
df['TicketSeries'].replace(np.nan, 'without', inplace=True)
 df['Ticket'] = df['Ticket'].astype('int64' )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
indexes = list(df.index)
 lastName = []
for el in indexes:
last_name = df.loc[el, 'Name']
last_name = last_name.split(sep=',', maxsplit=1)
 last_name = last_name[0]
lastName.append(last_name)
 df.loc[el, 'LastName'] = last_name
lastName = list(set(lastName))
print(f'Total lastName: {len(lastName)}')
 print('')
 print(f'{lastName[:10]}...' )
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X_train)
 rmse= np.sqrt(-cross_val_score(model, X_train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
LastName = df['LastName'].value_counts()
LastName[:20]
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1)) 
<choose_model_class>
b = df['Cabin'].value_counts()
b[:10]
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3)) 
<choose_model_class>
indexes = list(df.index)
 for el in indexes:
Cabin = df.loc[el, 'Cabin']
level = Cabin[0]
df.loc[el, 'CabinLevel'] = level
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
<choose_model_class>
df['CabinLevel'].value_counts(dropna=False )
GBoost = GradientBoostingRegressor(n_estimators=500, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )<choose_model_class>
target_column = ['Survived']
predictors = list(set(list(df.columns)) -set(target_column))
predictors
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )<choose_model_class>
for el in predictors:
el_type = df[el].dtype
if el_type == 'object':
labelencoder = LabelEncoder()
df.loc[:, el] = labelencoder.fit_transform(df.loc[:, el])
 df[el] = df[el]**0.5
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<train_model>
df['Survived'].replace(to_replace='no', value=0, inplace=True)
 df['Survived'].replace(to_replace='yes', value=1, inplace=True)
 df['Survived'] = df['Survived'].astype('int64')
 df['Survived'].head(3 )
model_xgb.fit(X_train, y_train )<choose_model_class>
NaN_predict(df )
xg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.4603, gamma=0.0468,
gpu_id=-1, importance_type='gain', interaction_constraints='',
learning_rate=0.05, max_delta_step=0, max_depth=3,
min_child_weight=1.7817, missing=np.nan, monotone_constraints='() ',
n_estimators=2200, n_jobs=-1, nthread=-1, num_parallel_tree=1,
random_state=7, reg_alpha=0.464, reg_lambda=0.8571,
scale_pos_weight=1, silent=1, subsample=0.5213,
tree_method='exact', validate_parameters=1, verbosity=None)
<compute_test_metric>
target_column = ['Survived']
predictors = list(set(list(df.columns)) -set(target_column))
predictors
score_lasso = rmsle_cv(lasso)
 score_lasso.mean()
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std())) 
<compute_test_metric>
X = df.loc[:, predictors]
X
score = rmsle_cv(model_xgb)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std())) 
<compute_test_metric>
y = df.loc[:,target_column]
y
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def features_selection(X, y):

 if not sys.warnoptions:
warnings.simplefilter("ignore")
 global droped_features
droped_features = []
def cross_val(X, y, features):
CV = ShuffleSplit(n_splits=5, test_size=0.25, random_state=0)
 test_df = X[features]
model = lgb.LGBMClassifier(random_state=0)
 model.fit(test_df, y)
 cv_score = cross_val_score(model,
test_df,
y,
cv=CV,
scoring='f1')
 cv_score = cv_score.mean()
return cv_score
features = list(X.columns)
 SCORES = pd.DataFrame(index = features, columns=['cv_score'])
 score_with_all_features = cross_val(X, y, features)
 print(f'{score_with_all_features} General cv_score with all features')
 cv_score_of_monstrous_feature = 1
while score_with_all_features <= cv_score_of_monstrous_feature:
print('
')
 features = list(X.columns)
 SCORES = pd.DataFrame(index = features, columns=['cv_score'])
 score_with_all_features = cross_val(X, y, features)
 print(f'{len(features)} number of features')
 print(f'{score_with_all_features} score with all features')
 ')
 for without_feature in features:
without_feature = [without_feature]
fit_faetures = features[:]
fit_faetures = list(set(fit_faetures)- set(without_feature))
score = cross_val(X, y, fit_faetures)
 SCORES.loc[without_feature] = score
SCORES = SCORES.sort_values(by=['cv_score'], ascending=False)
 print('
__SORTED SCORES__')
 print(SCORES)
 monstrous_feature = SCORES.index[0]
cv_score_of_monstrous_feature = SCORES.iloc[0][0]
if score_with_all_features <= cv_score_of_monstrous_feature:
X.drop([monstrous_feature], axis=1, inplace=True)
 droped_features.append(monstrous_feature)
 print('--------------------------------------------')
 print(f' DROP ==== {monstrous_feature}')
 print('--------------------------------------------')
 else:
print('
')
 print(f'These features have been droped:
{droped_features}')
 print('
')
 return X, droped_features
score = rmsle_cv(model_lgb)
 print(" score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
features_selection(X, y )
score = rmsle_cv(lasso)
 print(" score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
df.drop(droped_features, axis=1, inplace=True )
score = rmsle_cv(GBoost)
 GBoost.fit(X_train, y_train)
 sample = GBoost.predict(test)
 sample = np.exp(sample)
 print(" score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier
from catboost import CatBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
def constriction_hyperparameters_space(X,
y,
algorithm,
params,
n_iter_RandomizedSearchCV=100,
times_for_estimate=10):
global best_parameters
print('scores:
')
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0)
 CV = ShuffleSplit(n_splits=4, test_size=0.25, random_state=0)
 estimator = algorithm()
RESULTS_DF = None
PARAMS_DICT = None
PARAMS_DICT = {}
for i in range(0, times_for_estimate):
RANDOMIZED_MODEL = RandomizedSearchCV(estimator=estimator,
param_distributions=params,
n_iter=n_iter_RandomizedSearchCV,
scoring='f1',
cv=CV,
verbose=0,
n_jobs = -1)
 RANDOMIZED_FITED_BEST_MODEL = RANDOMIZED_MODEL.fit(X_train, y_train)
 best_parameters = RANDOMIZED_FITED_BEST_MODEL.best_params_
pred_test = RANDOMIZED_FITED_BEST_MODEL.predict(X_test)
 f1 = f1_score(y_test,pred_test)
 print(f' f1: {f1}')
 model = algorithm(**best_parameters)
 cv_score = cross_val_score(model, X, y, cv=CV)
 std_per =(cv_score.std())* 2
cv_score = cv_score.mean()
print(f'cv_score: {cv_score}')
 print(f' params: {best_parameters}')
 PARAMS_DICT[cv_score] = best_parameters
columns = ['cv_score', 'std_per', 'f1',]
values = []
values.append(cv_score)
 values.append(std_per)
 values.append(f1)
 for k,v in best_parameters.items() :
columns.append(k)
 values.append(v)
 if RESULTS_DF is None:
RESULTS_DF = pd.DataFrame(columns=columns)
 else:
pass
RESULTS_DF.loc[i] = values
print('
')
 RESULTS_DF = RESULTS_DF.sort_values(by='cv_score', ascending = False)
 RESULTS_DF.reset_index(drop=True, inplace=True)
 return RESULTS_DF, best_parameters
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
averaged_models.fit(X_train, y_train)
 submission = averaged_models.predict(test)
 submission = np.exp(submission)
<train_model>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self<compute_test_metric>
params = {'num_leaves': np.arange(600, 750, step=10),
'max_depth': [4, 5, 6, 7, 8],
'learning_rate': [0.01],
'n_estimators': np.arange(2000, 2300, step=20),
'subsample': [0.9, 1.0],
'feature_fraction': [0.3, 0.4],
'bagging_fraction': [0.7, 0.8],
'bagging_seed': np.arange(16, 19, step=1),
'lambda_l1': [0.1],
'lambda_l2': [0.1],
'min_child_samples': [2, 5],
'min_split_gain': [0.01],
'n_jobs':[-1],
}
constriction_hyperparameters_space(X,
y,
LGBMClassifier,
params,
n_iter_RandomizedSearchCV=100,
times_for_estimate=1)
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<choose_model_class>
I decided to use the stock lasso regression.Xgboost and Lgb were both banned for this competition .<save_to_csv>
target_column = ['Survived']
predictors = list(set(list(df.columns)) -set(target_column))
predictors
sub = pd.DataFrame()
sub['id'] = range(0,993)
 sub['cases'] = sample
sub.to_csv('submission.csv',index=False)
 print(sub )<import_modules>
X = df.loc[:890, predictors]
X
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import KFold
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn.naive_bayes import GaussianNB<load_from_csv>
y = df.loc[:890,target_column]
y
file = '.. /input/iris-train.csv'
df = pd.read_csv(file, delimiter = ',', index_col='Id')
 X = df[['SepalLengthCm','SepalWidthCm','PetalLengthCm','PetalWidthCm']]
Y = df['Species']<choose_model_class>
test_X = df.loc[891:, predictors]
test_X
models = []
models.append(( 'Support Vector Machines - linear', svm.SVC(kernel='linear', random_state=0, gamma=.10, C=2.0)))
 models.append(( 'Logistic Regression - lbfgs', LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=1000)))
 models.append(( 'KNeighborsClassifier', KNeighborsClassifier()))
models.append(( 'DecisionTreeClassifier', DecisionTreeClassifier()))
models.append(( 'RandomForestClassifier', RandomForestClassifier(n_estimators=100)))
 models.append(( 'Gaussian Naïve Bayes', GaussianNB())) 
<import_modules>
from sklearn.metrics import f1_score
from sklearn.model_selection import cross_val_score<load_from_csv>
BAGGING_DF = pd.DataFrame()
model = models[0][1]
filename = models[0][0]
X_train = df.drop(['Species'], axis=1)
 Y_train = df['Species']
model.fit(X_train, Y_train)
 dfPredict = pd.read_csv('.. /input/iris-test.csv', delimiter=',', index_col='Id')
 xPredict = dfPredict[['SepalLengthCm','SepalWidthCm','PetalLengthCm','PetalWidthCm']]
yPredict = model.predict(xPredict)
 submission = pd.DataFrame({
'Id': xPredict.index,
'Species': yPredict
})
 submission.set_index('Id', inplace=True)
 submission.to_csv(filename+'.csv' )<import_modules>
result = BAGGING_DF.mode(axis='columns')
 result.tail(10 )
import numpy as np
import pandas as pd<load_from_csv>
predictions = np.array(result[0])
 predictions
train_data = pd.read_csv('.. /input/iris-train.csv', index_col='Id')
 train_data.head()<load_from_csv>
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Submission was successfully saved!" )