Kaggle competition: New York City Taxi Fare Prediction
test['open_channels'] = np.round(np.clip(final_preds, 0, 10)).astype(int)
 test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )<init_hyperparams>
train = pd.read_csv(".. /input/train.csv", nrows = 1000000)
 test = pd.read_csv(".. /input/test.csv" )
seed_random = 42
window_sizes = [10, 50]
lr_lgb = 0.05
num_leaves = 200
num_iterations = 1500
lr_wn = 0.0015
EPOCHS_wn = 15
SPLITS = 2
w_lgb = 0.5
w_wnet = 1 - w_lgb
print(w_wnet)
<import_modules>
train.isnull().sum().sort_values(ascending=False )
!pip install tensorflow_addons==0.9.1
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore' )<load_from_csv>
train = train.drop(train[train.isnull().any(1)].index, axis = 0 )
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv' )<feature_engineering>
Counter(train['fare_amount']<0 )
%%time
for window in window_sizes:
train["rolling_mean_" + str(window)] = train['signal'].rolling(window=window ).mean()
train["rolling_std_" + str(window)] = train['signal'].rolling(window=window ).std()
train["rolling_var_" + str(window)] = train['signal'].rolling(window=window ).var()
train["rolling_min_" + str(window)] = train['signal'].rolling(window=window ).min()
train["rolling_max_" + str(window)] = train['signal'].rolling(window=window ).max()
a =(train['signal'] - train['rolling_min_' + str(window)])/(train['rolling_max_' + str(window)] - train['rolling_min_' + str(window)])
 train["norm_" + str(window)] = a *(np.floor(train['rolling_max_' + str(window)])- np.ceil(train['rolling_min_' + str(window)]))
train = train.replace([np.inf, -np.inf], np.nan)
 train.fillna(0, inplace=True )<feature_engineering>
train = train.drop(train[train['fare_amount']<0].index, axis=0)
 train.shape
%%time
for window in window_sizes:
test["rolling_mean_" + str(window)] = test['signal'].rolling(window=window ).mean()
test["rolling_std_" + str(window)] = test['signal'].rolling(window=window ).std()
test["rolling_var_" + str(window)] = test['signal'].rolling(window=window ).var()
test["rolling_min_" + str(window)] = test['signal'].rolling(window=window ).min()
test["rolling_max_" + str(window)] = test['signal'].rolling(window=window ).max()
a =(test['signal'] - test['rolling_min_' + str(window)])/(test['rolling_max_' + str(window)] - test['rolling_min_' + str(window)])
 test["norm_" + str(window)] = a *(np.floor(test['rolling_max_' + str(window)])- np.ceil(test['rolling_min_' + str(window)]))
test = test.replace([np.inf, -np.inf], np.nan)
 test.fillna(0, inplace=True )<feature_engineering>
train['fare_amount'].sort_values(ascending=False )
%%time
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices2'])
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels']]:
df[c+'_msignal'] = df[c] - df['signal']
df = df.replace([np.inf, -np.inf], np.nan)
 df.fillna(0, inplace=True)
 gc.collect()
return df
train = features(train)
 test = features(test )<drop_column>
train[train['passenger_count']>8]
train = reduce_mem_usage(train )<compute_test_metric>
train = train.drop(train[train['passenger_count']==208].index, axis = 0 )
test = reduce_mem_usage(test )<prepare_x_and_y>
train[train['pickup_latitude']<-90]
y = train['open_channels']
col = [c for c in train.columns if c not in ['time', 'open_channels', 'group', 'medianbatch', 'abs_avgbatch', 'abs_maxbatch']]<categorify>
train[train['pickup_latitude']>90]
<data_type_conversions>
train = train.drop(((train[train['pickup_latitude']<-90])|(train[train['pickup_latitude']>90])).index, axis=0 )
<data_type_conversions>
train[train['pickup_longitude']<-180]
<choose_model_class>
train[train['pickup_longitude']>180]
<compute_test_metric>
train = train.drop(((train[train['pickup_longitude']<-180])|(train[train['pickup_longitude']>180])).index, axis=0 )
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )<train_model>
train[train['dropoff_latitude']<-90]
%%time
X_train, X_valid, y_train, y_valid = train_test_split(train[col], y, test_size=0.3, random_state=seed_random)
 params = {'learning_rate': lr_lgb,
'max_depth': -1,
'num_leaves': num_leaves,
'metric': 'logloss',
'random_state': seed_random,
'n_jobs':-1,
'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(X_train, y_train), num_iterations, lgb.Dataset(X_valid, y_valid), verbose_eval=100, early_stopping_rounds=200, feval=MacroF1Metric)
 gc.collect()<predict_on_test>
train[train['dropoff_latitude']>90]
%%time
y_lgb_pred = model.predict(test[col], num_iteration=model.best_iteration)
 y_pred_train_lgb = model.predict(train[col], num_iteration=model.best_iteration)
 gc.collect()<compute_test_metric>
train = train.drop(((train[train['dropoff_latitude']<-90])|(train[train['dropoff_latitude']>90])).index, axis=0 )
print('LGB score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_lgb,0,10)).astype(int), average="macro"))))<predict_on_test>
train[train['dropoff_latitude']<-180]|train[train['dropoff_latitude']>180]
<compute_test_metric>
train['key'] = pd.to_datetime(train['key'],infer_datetime_format = True)
 train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'],infer_datetime_format=True )
<define_variables>
test['key'] = pd.to_datetime(test['key'],infer_datetime_format = True)
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'],infer_datetime_format = True )
EPOCHS = EPOCHS_wn
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = lr_wn<load_from_csv>
def haversine_distance(lat1,long1, lat2,long2):
data = [train,test]
for i in data:
r = 6371
phi1 = np.radians(i[lat1])
 phi2 = np.radians(i[lat2])
 delta_phi = np.radians(i[lat2]-i[lat1])
 delta_lambda = np.radians(i[long2]-i[long1])
 a = np.sin(delta_phi /2.0)**2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(r * c)
 i['H_Distance'] = d
return d
%%time
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index'])
 return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 3:
lr = LR
elif epoch < 8:
lr = LR / 3
elif epoch < 30:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
preds_train = np.zeros(( len(train), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_train_preds = model.predict(train)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
te_train_preds = te_train_preds.reshape(-1, te_train_preds.shape[-1])
 preds_train += te_train_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 y_wnet_pred = np.argmax(preds_, axis = 1)
 y_pred_train_wnet = np.argmax(preds_train, axis = 1)
 return y_wnet_pred, y_pred_train_wnet
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 y_wnet_pred, y_pred_train_wnet = run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training Wavenet model completed...' )<compute_test_metric>
haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude' )
print('Wavenet score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_wnet,0,10)).astype(int), average="macro"))))<compute_test_metric>
data = [train,test]
for i in data:
i['Year'] = i['pickup_datetime'].dt.year
i['Month'] = i['pickup_datetime'].dt.month
i['Date'] = i['pickup_datetime'].dt.day
i['Day of Week'] = i['pickup_datetime'].dt.dayofweek
i['Hour'] = i['pickup_datetime'].dt.hour
def plot_cm(y_true, y_pred, title):
figsize=(14,14)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )<set_options>
train.loc[(( train['pickup_latitude']==0)&(train['pickup_longitude']==0)) &(( train['dropoff_latitude']!=0)&(train['dropoff_longitude']!=0)) &(train['fare_amount']==0)]
gc.collect()<drop_column>
train = train.drop(train.loc[(( train['pickup_latitude']==0)&(train['pickup_longitude']==0)) &(( train['dropoff_latitude']!=0)&(train['dropoff_longitude']!=0)) &(train['fare_amount']==0)].index, axis=0 )
del y_pred_train_lgb, y_pred_train_wnet
gc.collect()<define_variables>
train.loc[(( train['pickup_latitude']!=0)&(train['pickup_longitude']!=0)) &(( train['dropoff_latitude']==0)&(train['dropoff_longitude']==0)) &(train['fare_amount']==0)]
y_preds = w_lgb*y_lgb_pred + w_wnet*y_wnet_pred<drop_column>
train = train.drop(train.loc[(( train['pickup_latitude']!=0)&(train['pickup_longitude']!=0)) &(( train['dropoff_latitude']==0)&(train['dropoff_longitude']==0)) &(train['fare_amount']==0)].index, axis=0 )
del y_lgb_pred, y_wnet_pred
gc.collect()<categorify>
high_distance = train.loc[(train['H_Distance']>200)&(train['fare_amount']!=0)]
def pred_proc(pred):
pred = np.round(np.clip(pred, 0, 10))
return pred.astype(int )<predict_on_test>
high_distance['H_Distance'] = high_distance.apply(
 lambda row:(row['fare_amount'] - 2.50)/1.56,
axis=1
)
y_preds = pred_proc(y_preds)
 y_train_preds = pred_proc(y_train_preds )<compute_test_metric>
train.update(high_distance )
print('The main solution score {0:.4f}'.format(np.mean(f1_score(y, y_train_preds, average="macro"))))<set_options>
train[train['H_Distance']==0]
del y_train_preds
gc.collect()<save_to_csv>
train[(train['H_Distance']==0)&(train['fare_amount']==0)]
test['open_channels'] = y_preds
test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )<init_hyperparams>
train = train.drop(train[(train['H_Distance']==0)&(train['fare_amount']==0)].index, axis = 0 )
seed_random = 42
window_sizes = [10, 50]
lr_lgb = 0.05
num_leaves = 200
num_iterations = 1500
lr_wn = 0.0015
EPOCHS_wn = 15
SPLITS = 2
w_lgb = 0.5
w_wnet = 1 - w_lgb
print(w_wnet)
<import_modules>
rush_hour = train.loc[(((train['Hour']>=6)&(train['Hour']<=20)) &(( train['Day of Week']>=1)&(train['Day of Week']<=5)) &(train['H_Distance']==0)&(train['fare_amount'] < 2.5)) ]
rush_hour
!pip install tensorflow_addons==0.9.1
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore' )<load_from_csv>
train=train.drop(rush_hour.index, axis=0 )
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv' )<feature_engineering>
non_rush_hour = train.loc[(((train['Hour']<6)|(train['Hour']>20)) &(( train['Day of Week']>=1)&(train['Day of Week']<=5)) &(train['H_Distance']==0)&(train['fare_amount'] < 3.0)) ]
non_rush_hour
%%time
for window in window_sizes:
train["rolling_mean_" + str(window)] = train['signal'].rolling(window=window ).mean()
train["rolling_std_" + str(window)] = train['signal'].rolling(window=window ).std()
train["rolling_var_" + str(window)] = train['signal'].rolling(window=window ).var()
train["rolling_min_" + str(window)] = train['signal'].rolling(window=window ).min()
train["rolling_max_" + str(window)] = train['signal'].rolling(window=window ).max()
a =(train['signal'] - train['rolling_min_' + str(window)])/(train['rolling_max_' + str(window)] - train['rolling_min_' + str(window)])
 train["norm_" + str(window)] = a *(np.floor(train['rolling_max_' + str(window)])- np.ceil(train['rolling_min_' + str(window)]))
train = train.replace([np.inf, -np.inf], np.nan)
 train.fillna(0, inplace=True )<feature_engineering>
weekends = train.loc[(( train['Day of Week']==0)|(train['Day of Week']==6)) &(train['H_Distance']==0)&(train['fare_amount'] < 3.0)]
weekends
%%time
for window in window_sizes:
test["rolling_mean_" + str(window)] = test['signal'].rolling(window=window ).mean()
test["rolling_std_" + str(window)] = test['signal'].rolling(window=window ).std()
test["rolling_var_" + str(window)] = test['signal'].rolling(window=window ).var()
test["rolling_min_" + str(window)] = test['signal'].rolling(window=window ).min()
test["rolling_max_" + str(window)] = test['signal'].rolling(window=window ).max()
a =(test['signal'] - test['rolling_min_' + str(window)])/(test['rolling_max_' + str(window)] - test['rolling_min_' + str(window)])
 test["norm_" + str(window)] = a *(np.floor(test['rolling_max_' + str(window)])- np.ceil(test['rolling_min_' + str(window)]))
test = test.replace([np.inf, -np.inf], np.nan)
 test.fillna(0, inplace=True )<feature_engineering>
train.loc[(train['H_Distance']!=0)&(train['fare_amount']==0)]
%%time
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index', 'batch_slices', 'batch_slices2'])
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels']]:
df[c+'_msignal'] = df[c] - df['signal']
df = df.replace([np.inf, -np.inf], np.nan)
 df.fillna(0, inplace=True)
 gc.collect()
return df
train = features(train)
 test = features(test )<drop_column>
scenario_3 = train.loc[(train['H_Distance']!=0)&(train['fare_amount']==0)]
train = reduce_mem_usage(train )<compute_test_metric>
scenario_3.sort_values('H_Distance', ascending=False )
test = reduce_mem_usage(test )<prepare_x_and_y>
scenario_3['fare_amount'] = scenario_3.apply(
 lambda row:(( row['H_Distance'] * 1.56)+ 2.50), axis=1
)
y = train['open_channels']
col = [c for c in train.columns if c not in ['time', 'open_channels', 'group', 'medianbatch', 'abs_avgbatch', 'abs_maxbatch']]<categorify>
train.loc[(train['H_Distance']==0)&(train['fare_amount']!=0)]
<data_type_conversions>
scenario_4 = train.loc[(train['H_Distance']==0)&(train['fare_amount']!=0)]
<data_type_conversions>
scenario_4.loc[(scenario_4['fare_amount']<=3.0)&(scenario_4['H_Distance']==0)]
<choose_model_class>
scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]
<compute_test_metric>
scenario_4_sub = scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )<train_model>
scenario_4_sub = scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]
%%time
X_train, X_valid, y_train, y_valid = train_test_split(train[col], y, test_size=0.3, random_state=seed_random)
 params = {'learning_rate': lr_lgb,
'max_depth': -1,
'num_leaves': num_leaves,
'metric': 'logloss',
'random_state': seed_random,
'n_jobs':-1,
'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(X_train, y_train), num_iterations, lgb.Dataset(X_valid, y_valid), verbose_eval=100, early_stopping_rounds=200, feval=MacroF1Metric)
 gc.collect()<predict_on_test>
scenario_4_sub['H_Distance'] = scenario_4_sub.apply(
 lambda row:(( row['fare_amount']-2.50)/1.56), axis=1
)
%%time
y_lgb_pred = model.predict(test[col], num_iteration=model.best_iteration)
 y_pred_train_lgb = model.predict(train[col], num_iteration=model.best_iteration)
 gc.collect()<compute_test_metric>
train = train.drop(['key','pickup_datetime'], axis = 1)
 test = test.drop(['key','pickup_datetime'], axis = 1 )
print('LGB score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_lgb,0,10)).astype(int), average="macro"))))<predict_on_test>
x_train = train.iloc[:,train.columns!='fare_amount']
y_train = train['fare_amount'].values
x_test = test
<compute_test_metric>
rf = RandomForestRegressor()
rf.fit(x_train, y_train)
 rf_predict = rf.predict(x_test)
<define_variables>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = rf_predict
submission.to_csv('submission_1.csv', index=False)
 submission.head(20 )
EPOCHS = EPOCHS_wn
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = lr_wn<load_from_csv>
lr = linear_model.LinearRegression()
lr.fit(x_train, y_train)
 lr_predict = lr.predict(x_test )
%%time
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['signal_shift_+2'] = [0,] + [1,] + list(df['signal'].values[:-2])
 df['signal_shift_-2'] = list(df['signal'].values[2:])+ [0] + [1]
for i in df[df['batch_index']==0].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==1].index:
df['signal_shift_+2'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-2'][i] = np.nan
for i in df[df['batch_index']==49998].index:
df['signal_shift_-2'][i] = np.nan
df = df.drop(columns=['batch', 'batch_index'])
 return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 3:
lr = LR
elif epoch < 8:
lr = LR / 3
elif epoch < 30:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
preds_train = np.zeros(( len(train), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_train_preds = model.predict(train)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
te_train_preds = te_train_preds.reshape(-1, te_train_preds.shape[-1])
 preds_train += te_train_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 y_wnet_pred = np.argmax(preds_, axis = 1)
 y_pred_train_wnet = np.argmax(preds_train, axis = 1)
 return y_wnet_pred, y_pred_train_wnet
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 y_wnet_pred, y_pred_train_wnet = run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training Wavenet model completed...' )<compute_test_metric>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = lr_predict
submission.to_csv('submission_2.csv', index=False)
 submission.head(20 )