Kaggle competition: University of Liverpool - Ion Switching
train_title = [i.split(",")[1].split(".")[0].strip() for i in train_data["Name"]]
train_data["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip() for i in test_data["Name"]]
test_data["Title"] = pd.Series(test_title )<drop_column>
def plot_cm(y_true, y_pred, title):
figsize=(14,14)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )
train_data = train_data.drop(labels = "Name", axis = "columns")
 test_data = test_data.drop(labels = "Name", axis = "columns" )<data_type_conversions>
PATH = '/kaggle/input/data-without-drift/'
train = pd.read_csv(PATH + 'train_clean.csv')
 test = pd.read_csv(PATH + 'test_clean.csv')
 train.head()
train_data["Title"] = train_data["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train_data["Title"] = train_data["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train_data["Title"] = train_data["Title"].astype(int)
 test_data["Title"] = test_data["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test_data["Title"] = test_data["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test_data["Title"] = test_data["Title"].astype(int )<prepare_output>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
pred = train_data['Survived']<import_modules>
WINDOW_SIZES = [3, 5, 10, 50, 100, 500]
import statsmodels.formula.api as smf
import statsmodels.api as sm<train_on_grid>
%%time
def gen_roll_features(full, win_sizes = WINDOW_SIZES):
for window in tqdm(win_sizes):
full["rolling_mean_" + str(window)] = full['signal'].rolling(window=window ).mean()
full["rolling_std_" + str(window)] = full['signal'].rolling(window=window ).std()
full["rolling_var_" + str(window)] = full['signal'].rolling(window=window ).var()
full["rolling_min_" + str(window)] = full['signal'].rolling(window=window ).min()
full["rolling_max_" + str(window)] = full['signal'].rolling(window=window ).max()
a =(full['signal'] - full['rolling_min_' + str(window)])/(full['rolling_max_' + str(window)] - full['rolling_min_' + str(window)])
 full["norm_" + str(window)] = a *(np.floor(full['rolling_max_' + str(window)])- np.ceil(full['rolling_min_' + str(window)]))
full = full.replace([np.inf, -np.inf], np.nan)
 full.fillna(0, inplace=True)
 return full
train = gen_roll_features(train)
 test = gen_roll_features(test )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_A4 + Ticket_A5 +Ticket_C + Ticket_CA+Ticket_FC + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SC + Ticket_SCA4 + Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOPP+Ticket_SOTONO2+Ticket_SOTONOQ + Ticket_STONO + Ticket_STONO2 + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
beta1 = reg_log.params[0]
beta2 = reg_log.params[1]
print(reg_log.summary() )<train_on_grid>
%%time
GROUP_BATCH_SIZE = 8000
def batching(df, batch_size, gr_name='group'):
df[gr_name] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df[gr_name] = df[gr_name].astype(np.uint16)
 return df
def run_feat_engineering(df, batch_size, gr_name='group'):
df = batching(df, batch_size = batch_size, gr_name=gr_name)
 df['signal_2'] = df['signal'] ** 2
df['signal_2-7500-mean'] = df['signal_2'] - df['signal_2'].rolling(window=7500 ).mean()
return df
train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE, gr_name='group')
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE, gr_name='group' )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_A4 + Ticket_A5 +Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOPP+Ticket_SOTONO2+Ticket_SOTONOQ + Ticket_STONO + Ticket_STONO2 + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
beta1 = reg_log.params[0]
beta2 = reg_log.params[1]
print(reg_log.summary() )<train_on_grid>
%%time
STD = 0.01
old_data = train['signal']
new_data = old_data + np.random.normal(0,STD,size=len(train))
train['signal'] = new_data
old_data = test['signal']
new_data = old_data + np.random.normal(0,STD,size=len(test))
test['signal'] = new_data
del old_data, new_data
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_A4 + Ticket_A5 +Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOPP+Ticket_SOTONOQ + Ticket_STONO + Ticket_STONO2 + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
%%time
def gen_roll_features(full, win_sizes = WINDOW_SIZES):
for window in tqdm(win_sizes):
full["rolling_mean_" + str(window)] = full['signal'].rolling(window=window ).mean()
full["rolling_std_" + str(window)] = full['signal'].rolling(window=window ).std()
full["rolling_var_" + str(window)] = full['signal'].rolling(window=window ).var()
full["rolling_min_" + str(window)] = full['signal'].rolling(window=window ).min()
full["rolling_max_" + str(window)] = full['signal'].rolling(window=window ).max()
a =(full['signal'] - full['rolling_min_' + str(window)])/(full['rolling_max_' + str(window)] - full['rolling_min_' + str(window)])
 full["norm_" + str(window)] = a *(np.floor(full['rolling_max_' + str(window)])- np.ceil(full['rolling_min_' + str(window)]))
return full
train = gen_roll_features(train)
 test = gen_roll_features(test )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_A5 +Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOPP+Ticket_SOTONOQ + Ticket_STONO + Ticket_STONO2 + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
%%time
def gen_sig_features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in tqdm(['batch','batch_slices2']):
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df = reduce_mem_usage(df)
 gc.collect()
return df
train = gen_sig_features(train)
 test = gen_sig_features(test )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_A5 +Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_STONO2 + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
%%time
def gen_shift_features(df):
df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices2'], inplace=True)
 gc.collect()
for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels', 'group' 'category', 'index']]:
df[c+'_msignal'] = df[c] - df['signal']
df = df.replace([np.inf, -np.inf], np.nan)
 df.fillna(0, inplace=True)
 df = reduce_mem_usage(df)
 gc.collect()
return df
train = gen_shift_features(train)
 test = gen_shift_features(test )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_A5 +Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
%%time
vt = VarianceThreshold(0.5)
 y = train['open_channels'].values
X = train[selected_columns].values
vt.fit(X, y)
 top_idx = np.argpartition(vt.variances_, -25)[-25:]
SELECTED_FEATURES = [selected_columns[i] for i in top_idx]
print(SELECTED_FEATURES )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCPARIS + Ticket_SCParis + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
X_train = train[SELECTED_FEATURES]
y_train = train['open_channels'].values
print(f'Original sizes: train: {train.shape}, test: {test.shape}')
 print(f'Reduced train sizes: X_train: {X_train.shape}, y_train: {y_train.shape}' )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SCParis + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP+ Ticket_SCAH + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
NUM_BOOST_ROUND = 2000
EARLY_STOPPING_ROUNDS = 40
VERBOSE_EVAL = 100
RANDOM_SEED = 13
LEARNING_RATE = 0.02
MAX_DEPTH = -1
NUM_LEAVES = 200
X_train1, X_valid1, y_train1, y_valid1 = train_test_split(X_train, y_train, test_size=0.3, random_state=RANDOM_SEED)
 params = {
'learning_rate': LEARNING_RATE,
'max_depth': MAX_DEPTH,
'num_leaves': NUM_LEAVES,
'metric': 'logloss',
'random_state': RANDOM_SEED,
'n_jobs':-1,
'sample_fraction':0.33
}
evals_result = {}
model = lgb.train(
 params,
train_set=lgb.Dataset(X_train1, y_train1),
num_boost_round = NUM_BOOST_ROUND,
valid_sets = lgb.Dataset(X_valid1, y_valid1),
verbose_eval = VERBOSE_EVAL,
evals_result = evals_result,
early_stopping_rounds = EARLY_STOPPING_ROUNDS,
feval = MacroF1Metric)
 gc.collect()
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_WC + Ticket_WEP + Ticket_X +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
y_pred_train_lgb = model.predict(X_train, num_iteration=model.best_iteration)
 print('LGB score {0:.4f}'.format(np.mean(f1_score(y_train, np.round(np.clip(y_pred_train_lgb,0,10)).astype(int), average="macro"))))
gc.collect()
reg_log = smf.glm('Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Ticket_C + Ticket_CA + Ticket_FCC + Ticket_PC + Ticket_PP + Ticket_SOC+Ticket_SOTONOQ + Ticket_STONO + Ticket_WC + Ticket_WEP +Title', data=train_data, family=sm.families.Binomial() ).fit()
print(reg_log.summary() )<train_on_grid>
y_hat = model.predict(test[SELECTED_FEATURES], num_iteration=model.best_iteration)
 y_pred = np.round(np.clip(y_hat,0,10)).astype(int )
<train_on_grid><EOS> 
sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 sub['open_channels'] = y_pred
sub.to_csv('submission.csv', index=False, float_format='%.4f')
 sub.head(20 )