Kaggle competition: Titanic - Machine Learning from Disaster
sales, prices, calendar, submission = read_data()
NUM_ITEMS = sales.shape[0]
DAYS_PRED = submission.shape[1] - 1<categorify>
plt.style.use('seaborn-whitegrid' )
def encode_categorical(df, cols):
for col in cols:
le = LabelEncoder()
not_null = df[col][df[col].notnull() ]
df[col] = pd.Series(le.fit_transform(not_null), index=not_null.index)
 return df
calendar = encode_categorical(
 calendar, ["event_name_1", "event_type_1", "event_name_2", "event_type_2"]
).pipe(reduce_mem_usage)
 sales = encode_categorical(
 sales, ["item_id", "dept_id", "cat_id", "store_id", "state_id"],
).pipe(reduce_mem_usage)
 prices = encode_categorical(prices, ["item_id", "store_id"] ).pipe(reduce_mem_usage )<categorify>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.info()
def extract_num(ser):
return ser.str.extract(r"(\d+)" ).astype(np.int16)
 def reshape_sales(sales, submission, d_thresh=0, verbose=True):
id_columns = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
product = sales[id_columns]
sales = sales.melt(id_vars=id_columns, var_name="d", value_name="demand",)
 sales = reduce_mem_usage(sales)
 vals = submission[submission["id"].str.endswith("validation")]
evals = submission[submission["id"].str.endswith("evaluation")]
vals.columns = ["id"] + [f"d_{d}" for d in range(1914, 1914 + DAYS_PRED)]
evals.columns = ["id"] + [f"d_{d}" for d in range(1942, 1942 + DAYS_PRED)]
evals["id"] = evals["id"].str.replace("_evaluation", "_validation")
 vals = vals.merge(product, how="left", on="id")
 evals = evals.merge(product, how="left", on="id")
 evals["id"] = evals["id"].str.replace("_validation", "_evaluation")
 if verbose:
print("validation")
 display(vals)
 print("evaluation")
 display(evals)
 vals = vals.melt(id_vars=id_columns, var_name="d", value_name="demand")
 evals = evals.melt(id_vars=id_columns, var_name="d", value_name="demand")
 sales["part"] = "train"
vals["part"] = "validation"
evals["part"] = "evaluation"
data = pd.concat([sales, vals, evals], axis=0)
 del sales, vals, evals
data["d"] = extract_num(data["d"])
 data = data[data["d"] >= d_thresh]
data = data[data["part"] != "evaluation"]
gc.collect()
if verbose:
print("data")
 display(data)
 return data
def merge_calendar(data, calendar):
calendar = calendar.drop(["weekday", "wday", "month", "year"], axis=1)
 return data.merge(calendar, how="left", on="d")
 def merge_prices(data, prices):
return data.merge(prices, how="left", on=["store_id", "item_id", "wm_yr_wk"] )<categorify>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.info()
def add_demand_features(df):
for diff in [0, 1, 2]:
shift = DAYS_PRED + diff
df[f"shift_t{shift}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(shift)
)
 for window in [7, 30, 60, 90, 180]:
df[f"rolling_std_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).std() 
)
 for window in [7, 30, 60, 90, 180]:
df[f"rolling_mean_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).mean() 
)
 for window in [7, 30, 60]:
df[f"rolling_min_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).min() 
)
 for window in [7, 30, 60]:
df[f"rolling_max_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).max() 
)
 df["rolling_skew_t30"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(30 ).skew() 
)
 df["rolling_kurt_t30"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(30 ).kurt() 
)
 return df
def add_price_features(df):
df["shift_price_t1"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.shift(1)
)
 df["price_change_t1"] =(df["shift_price_t1"] - df["sell_price"])/(
 df["shift_price_t1"] 
)
 df["rolling_price_max_t365"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.shift(1 ).rolling(365 ).max() 
)
 df["price_change_t365"] =(df["rolling_price_max_t365"] - df["sell_price"])/(
 df["rolling_price_max_t365"] 
)
 df["rolling_price_std_t7"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.rolling(7 ).std() 
)
 df["rolling_price_std_t30"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.rolling(30 ).std() 
)
 return df.drop(["rolling_price_max_t365", "shift_price_t1"], axis=1)
 def add_time_features(df, dt_col):
df[dt_col] = pd.to_datetime(df[dt_col])
 attrs = [
"year",
"quarter",
"month",
"week",
"day",
"dayofweek",
]
for attr in attrs:
dtype = np.int16 if attr == "year" else np.int8
df[attr] = getattr(df[dt_col].dt, attr ).astype(dtype)
 df["is_weekend"] = df["dayofweek"].isin([5, 6] ).astype(np.int8)
 return df<split>
train_data[train_data['Fare'] > 500]
class CustomTimeSeriesSplitter:
def __init__(self, n_splits=5, train_days=80, test_days=20, day_col="d"):
self.n_splits = n_splits
self.train_days = train_days
self.test_days = test_days
self.day_col = day_col
def split(self, X, y=None, groups=None):
SEC_IN_DAY = 3600 * 24
sec =(X[self.day_col] - X[self.day_col].iloc[0])* SEC_IN_DAY
duration = sec.max()
train_sec = self.train_days * SEC_IN_DAY
test_sec = self.test_days * SEC_IN_DAY
total_sec = test_sec + train_sec
if self.n_splits == 1:
train_start = duration - total_sec
train_end = train_start + train_sec
train_mask =(sec >= train_start)&(sec < train_end)
 test_mask = sec >= train_end
yield sec[train_mask].index.values, sec[test_mask].index.values
else:
step = DAYS_PRED * SEC_IN_DAY
for idx in range(self.n_splits):
shift =(self.n_splits -(idx + 1)) * step
train_start = duration - total_sec - shift
train_end = train_start + train_sec
test_end = train_end + test_sec
train_mask =(sec > train_start)&(sec <= train_end)
 if idx == self.n_splits - 1:
test_mask = sec > train_end
else:
test_mask =(sec > train_end)&(sec <= test_end)
 yield sec[train_mask].index.values, sec[test_mask].index.values
def get_n_splits(self):
return self.n_splits<define_variables>
test_data[test_data['Fare'] > 500]
def show_cv_days(cv, X, dt_col, day_col):
for ii,(tr, tt)in enumerate(cv.split(X)) :
print(f"----- Fold:({ii + 1} / {cv.n_splits})-----")
 tr_start = X.iloc[tr][dt_col].min()
tr_end = X.iloc[tr][dt_col].max()
tr_days = X.iloc[tr][day_col].max() - X.iloc[tr][day_col].min() + 1
tt_start = X.iloc[tt][dt_col].min()
tt_end = X.iloc[tt][dt_col].max()
tt_days = X.iloc[tt][day_col].max() - X.iloc[tt][day_col].min() + 1
df = pd.DataFrame(
 {
"start": [tr_start, tt_start],
"end": [tr_end, tt_end],
"days": [tr_days, tt_days],
},
index=["train", "test"], 
)
 display(df)
 def plot_cv_indices(cv, X, dt_col, lw=10):
n_splits = cv.get_n_splits()
_, ax = plt.subplots(figsize=(20, n_splits))
for ii,(tr, tt)in enumerate(cv.split(X)) :
indices = np.array([np.nan] * len(X))
indices[tt] = 1
indices[tr] = 0
ax.scatter(
 X[dt_col],
[ii + 0.5] * len(indices),
c=indices,
marker="_",
lw=lw,
cmap=plt.cm.coolwarm,
vmin=-0.2,
vmax=1.2, 
)
 MIDDLE = 15
LARGE = 20
ax.set_xlabel("Datetime", fontsize=LARGE)
 ax.set_xlim([X[dt_col].min() , X[dt_col].max() ])
 ax.set_ylabel("CV iteration", fontsize=LARGE)
 ax.set_yticks(np.arange(n_splits)+ 0.5)
 ax.set_yticklabels(list(range(n_splits)))
 ax.invert_yaxis()
ax.tick_params(axis="both", which="major", labelsize=MIDDLE)
 ax.set_title("{}".format(type(cv ).__name__), fontsize=LARGE)
 return ax<define_variables>
train_data["Fare"].replace({ 512.3292 : 7.25}, inplace=True)
 test_data["Fare"].replace({ 512.3292 : 7.25}, inplace=True )
features = [
"item_id",
"dept_id",
"cat_id",
"store_id",
"state_id",
"event_name_1",
"event_type_1",
"event_name_2",
"event_type_2",
"snap_CA",
"snap_TX",
"snap_WI",
"sell_price",
"shift_t28",
"shift_t29",
"shift_t30",
"rolling_std_t7",
"rolling_std_t30",
"rolling_std_t60",
"rolling_std_t90",
"rolling_std_t180",
"rolling_mean_t7",
"rolling_mean_t30",
"rolling_mean_t60",
"rolling_mean_t90",
"rolling_mean_t180",
"rolling_min_t7",
"rolling_min_t30",
"rolling_min_t60",
"rolling_max_t7",
"rolling_max_t30",
"rolling_max_t60",
"rolling_skew_t30",
"rolling_kurt_t30",
"price_change_t1",
"price_change_t365",
"rolling_price_std_t7",
"rolling_price_std_t30",
"year",
"quarter",
"month",
"week",
"day",
"dayofweek",
"is_weekend",
]
is_train = data["d"] < 1914
X_train = data[is_train][[day_col] + features].reset_index(drop=True)
 y_train = data[is_train]["demand"].reset_index(drop=True)
 X_test = data[~is_train][features].reset_index(drop=True)
 id_date = data[~is_train][["id", "date"]].reset_index(drop=True)
 del data
gc.collect()
print("X_train shape:", X_train.shape)
 print("X_test shape:", X_test.shape )<train_model>
train_data = train_data[(train_data.PassengerId != 259)&(train_data.PassengerId != 680)&(train_data.PassengerId != 738)]
def train_lgb(bst_params, fit_params, X, y, cv, drop_when_train=None):
models = []
if drop_when_train is None:
drop_when_train = []
for idx_fold,(idx_trn, idx_val)in enumerate(cv.split(X, y)) :
print(f"
----- Fold:({idx_fold + 1} / {cv.get_n_splits() })-----
")
 X_trn, X_val = X.iloc[idx_trn], X.iloc[idx_val]
y_trn, y_val = y.iloc[idx_trn], y.iloc[idx_val]
train_set = lgb.Dataset(
 X_trn.drop(drop_when_train, axis=1),
label=y_trn,
categorical_feature=["item_id"], 
)
 val_set = lgb.Dataset(
 X_val.drop(drop_when_train, axis=1),
label=y_val,
categorical_feature=["item_id"], 
)
 model = lgb.train(
 bst_params,
train_set,
valid_sets=[train_set, val_set],
valid_names=["train", "valid"],
**fit_params, 
)
 models.append(model)
 del idx_trn, idx_val, X_trn, X_val, y_trn, y_val
gc.collect()
return models<train_model>
train_data.isna().sum()
bst_params = {
"boosting_type": "gbdt",
"metric": "rmse",
"objective": "regression",
"n_jobs": -1,
"seed": 42,
"learning_rate": 0.1,
"bagging_fraction": 0.75,
"bagging_freq": 10,
"colsample_bytree": 0.75,
}
fit_params = {
"num_boost_round": 100_000,
"early_stopping_rounds": 50,
"verbose_eval": 100,
}
models = train_lgb(
 bst_params, fit_params, X_train, y_train, cv, drop_when_train=[day_col] 
)
 del X_train, y_train
gc.collect()<compute_test_metric>
test_data.isna().sum()
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))<features_selection>
train_data['Age'] = train_data['Age'].replace(np.nan, 29)
 test_data['Age'] = test_data['Age'].replace(np.nan, 30 )
imp_type = "gain"
importances = np.zeros(X_test.shape[1])
 preds = np.zeros(X_test.shape[0])
 for model in models:
preds += model.predict(X_test)
 importances += model.feature_importance(imp_type)
 preds = preds / cv.get_n_splits()
importances = importances / cv.get_n_splits()<save_to_csv>
train_data = train_data.drop(['Cabin'], axis=1)
 test_data = test_data.drop(['Cabin'], axis=1 )
def make_submission(test, submission):
preds = test[["id", "date", "demand"]]
preds = preds.pivot(index="id", columns="date", values="demand" ).reset_index()
preds.columns = ["id"] + ["F" + str(d + 1)for d in range(DAYS_PRED)]
vals = submission[["id"]].merge(preds, how="inner", on="id")
 evals = submission[submission["id"].str.endswith("evaluation")]
final = pd.concat([vals, evals])
 assert final.drop("id", axis=1 ).isnull().sum().sum() == 0
assert final["id"].equals(submission["id"])
 final.to_csv("submission.csv", index=False )<prepare_output>
train_data['Embarked'] = train_data['Embarked'].replace(np.nan, 'Q')
 test_data['Fare'] = test_data['Fare'].replace(np.nan, 36.0 )
make_submission(id_date.assign(demand=preds), submission )<import_modules>
train_data.isna().sum()
import os
import sys
import time
import numpy as np
import pandas as pd
import seaborn as sns
from math import sqrt
from sklearn import metrics
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error<load_from_csv>
test_data.isna().sum()
df1 = pd.read_csv(".. /input/elo-blending/3.695.csv")
 df2 = pd.read_csv(".. /input/elo-blending/3.696.csv")
 df3 = pd.read_csv(".. /input/submit/submit.csv")
 df4 = pd.read_csv(".. /input/combined/combining_submission.csv" )<load_from_csv>
train_data['Title'] = train_data.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 train_data.Title.value_counts()
df_base0 = pd.read_csv('.. /input/elo-blending/3.695.csv', names=["card_id","target0"], skiprows=[0],header=None)
 df_base1 = pd.read_csv('.. /input/elo-blending/3.696.csv', names=["card_id","target1"], skiprows=[0],header=None)
 df_base2 = pd.read_csv('.. /input/elo-blending/3.6999.csv', names=["card_id","target2"], skiprows=[0],header=None)
 df_base3 = pd.read_csv('.. /input/elo-blending/3.69991.csv', names=["card_id","target3"], skiprows=[0],header=None)
 df_base4 = pd.read_csv('.. /input/elo-blending/3.699992.csv', names=["card_id","target4"], skiprows=[0],header=None)
 df_base5 = pd.read_csv('.. /input/elo-blending/3.70.csv', names=["card_id","target5"], skiprows=[0],header=None)
 df_base6 = pd.read_csv('.. /input/elo-blending/3.701.csv', names=["card_id","target6"], skiprows=[0],header=None)
 df_base7 = pd.read_csv('.. /input/elo-blending/3.702.csv', names=["card_id","target7"], skiprows=[0],header=None)
 df_base8 = pd.read_csv('.. /input/elo-blending/3.703.csv', names=["card_id","target8"], skiprows=[0],header=None)
 df_base9 = pd.read_csv('.. /input/elo-blending/3.704.csv', names=["card_id","target9"], skiprows=[0],header=None)
 df_base10 = pd.read_csv('.. /input/elo-blending/Blending.csv', names=["card_id","target10"], skiprows=[0],header=None)
 df_base11 = pd.read_csv('.. /input/elo-blending/BlendingRLS.csv',names=["card_id","target11"], skiprows=[0],header=None )<merge>
test_data['Title'] = test_data.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 test_data.Title.value_counts()
df_base = pd.merge(df_base0,df_base1,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base2,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base3,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base4,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base5,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base6,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base7,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base8,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base9,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base10,how='inner',on='card_id')
 df_base = pd.merge(df_base,df_base11,how='inner',on='card_id' )<compute_test_metric>
normalized_titles = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royal",
"Don": "Royal",
"Sir" : "Royal",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Royal",
"Dona": "Royal",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royal"
}
train_data.Title = train_data.Title.map(normalized_titles)
 test_data.Title = test_data.Title.map(normalized_titles )
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<load_from_csv>
test_data.Title.value_counts()
df_base0 = pd.read_csv('.. /input/elo-blending/3.695.csv', names=["card_id","target0"], skiprows=[0],header=None)
 df_base1 = pd.read_csv('.. /input/elo-blending/3.696.csv', names=["card_id","target1"], skiprows=[0],header=None)
 df_base10 = pd.read_csv('.. /input/elo-blending/Blending.csv', names=["card_id","target10"], skiprows=[0],header=None)
 df_base11 = pd.read_csv('.. /input/elo-blending/BlendingRLS.csv',names=["card_id","target11"], skiprows=[0],header=None)
 df_base = pd.merge(df_base0,df_base1,how='inner',on='card_id')
 df_base = pd.merge(df_base0,df_base10,how='inner',on='card_id')
 df_base = pd.merge(df_base0,df_base11,how='inner',on='card_id')
 plt.figure(figsize=(12,8))
sns.heatmap(df_base.iloc[:,1:].corr() ,annot=True,fmt=".2f" )<compute_test_metric>
train_data.Title.value_counts()
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<feature_engineering>
le = LabelEncoder()
train_data["Sex"] = le.fit_transform(train_data["Sex"].values)
 test_data["Sex"] = le.fit_transform(test_data["Sex"].values )
df_base['target'] = df_base_median
df_base['target4'] = df4['target']<count_values>
train_data = train_data.drop(['Ticket'], axis=1)
 test_data = test_data.drop(['Ticket'], axis=1 )
df_final = np.zeros(len(df_base))
a=-10*np.log2(10)
 thresh = -14
for i in range(len(df3)-1):
if df3['target'][i]< thresh:
df_final[i]=a
else:
df_final[i]=df_base['target'][i]
pd.Series(df_final ).value_counts().head(1 )<feature_engineering>
train_data["Embarked"].value_counts()
for i in range(len(df_final)-1):
if df_final[i] > magic:
df_final[i] = df_final[i] - abs(np.median(df_final)) /8 +0.001<save_to_csv>
train_data["Embarked"] = le.fit_transform(train_data["Embarked"].values)
 test_data["Embarked"] = le.fit_transform(test_data["Embarked"].values )
df_finall=pd.DataFrame(df_base['card_id'])
 df_finall['target'] = df_final
df_finall[['card_id','target']].to_csv("good_output.csv",index=False )<import_modules>
train_data["Name"] = le.fit_transform(train_data["Name"].values)
 test_data["Name"] = le.fit_transform(test_data["Name"].values )
warnings.simplefilter(action='ignore', category=FutureWarning)
 plt.style.use('seaborn')
 sns.set(font_scale=1 )<load_from_csv>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier
import lightgbm as gbm
from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold, cross_val_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC, LinearSVC
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
from sklearn.metrics import r2_score
df_train = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 X = np.load(".. /input/folk-base-neural-network-using-lstm/X.npy")
 y = np.load(".. /input/folk-base-neural-network-using-lstm/y.npy")
 features = np.load(".. /input/folk-base-neural-network-using-lstm/features.npy" )<categorify>
features = ["Pclass","Sex","Age","SibSp","Fare"]
X = train_data[features]
y = train_data['Survived']
X_train,X_test, y_train,y_test = train_test_split(X,y,test_size=0.15)
 sc_X = MinMaxScaler()
X_train = sc_X.fit_transform(X_train)
 X_test = sc_X.transform(X_test )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<train_model>
pipelineone = Pipeline([('RandomForest', RandomForestClassifier())])
 param_grid = {'RandomForest__max_features': ['auto'],
'RandomForest__min_samples_leaf': [1, 2, 4],
'RandomForest__min_samples_split': [2,3,4],
'RandomForest__n_estimators': [100,200,300]}
model = GridSearchCV(pipelineone, param_grid, cv =None)
 model.fit(X_train, y_train)
 y_pred_sub = model.predict(test_data[features])
 y_pred_random = model.predict(X_test)
 forest = accuracy_score(y_test, y_pred_random)
 print(forest )
%%time
X, y = augment(X,y)
 print(X.shape, y.shape )<load_pretrained>
pipelinetwo = Pipeline([('logisticregression', LogisticRegression(max_iter=100)) ])
 param_grid = {'logisticregression__penalty' : ['l2'],
'logisticregression__C' : [0.001, 0.01, 0.1, 1, 10, 100, 1000],
'logisticregression__solver' : ['liblinear']}
model = GridSearchCV(pipelinetwo, param_grid, cv =None)
 model.fit(X_train, y_train)
 model = LogisticRegression()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 logistic = accuracy_score(y_test, y_pred)
 print(logistic )
features = np.load('.. /input/vsb-aug-features/aug_features.npy')
 features.shape<set_options>
xg = XGBClassifier()
xg.fit(X_train, y_train)
 y_pred = xg.predict(X_test)
 XGB = accuracy_score(y_test, y_pred)
 print(XGB )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<compute_test_metric>
KN = KNeighborsClassifier(n_neighbors=3)
 KN.fit(X_train, y_train)
 y_pred = KN.predict(X_test)
 knn = accuracy_score(y_test, y_pred)
 print(knn )
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<compute_test_metric>
svc = SVC()
svc.fit(X_train, y_train)
 y_pred = svc.predict(X_test)
 SVC = accuracy_score(y_test, y_pred)
 print(SVC )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result<choose_model_class>
lsvc = LinearSVC(random_state=0)
 lsvc.fit(X_train, y_train)
 y_pred = lsvc.predict(X_test)
 LSVC = accuracy_score(y_test, y_pred)
 print(LSVC )
def model_lstm(input_shape, feat_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
feat = Input(shape=(feat_shape[1],))
bi_lstm_1 = Bidirectional(CuDNNLSTM(128, return_sequences=True), merge_mode='concat' )(inp)
 bi_lstm_2 = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat' )(bi_lstm_1)
 attention = Attention(input_shape[1] )(bi_lstm_2)
 x = concatenate([attention, feat], axis=1)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp, feat], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model 
<split>
finalleaderboard = {
"Random Forest": forest,
"logistic regression": logistic,
"XGB": XGB,
"KNeighborsClassifier": knn,
"SVC": SVC,
"Linear SVC": LSVC
}
finalleaderboard = pd.DataFrame.from_dict(finalleaderboard, orient='index', columns=['Accuracy'])
 print(finalleaderboard )
N_SPLITS = 5
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_feat, train_y, val_X, val_feat, val_y = X[train_idx], features[train_idx], y[train_idx], X[val_idx], features[val_idx], y[val_idx]
model = model_lstm(train_X.shape, features.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit([train_X, train_feat], train_y, batch_size=128, epochs=50, validation_data=([val_X, val_feat], val_y), callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict([val_X, val_feat], batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape 
<compute_test_metric>
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': y_pred_sub})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )
def matthews_correlation(y_true, y_pred):

 y_pred = tf.convert_to_tensor(y_pred, np.float64)
 y_true = tf.convert_to_tensor(y_true, np.float64)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<compute_test_metric>
print(np.count_nonzero(y_pred))