Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
<SOS> metric: LogLoss Kaggle data source: statoil-iceberg-classifier-challenge<prepare_x_and_y>
np.random.seed(98643)
 tf.set_random_seed(683)
 denoise_wavelet, estimate_sigma, denoise_tv_bregman, denoise_nl_means)
 %matplotlib inline
play_col = train.drop(players_col+['Yards'], axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = train[col][::22]<prepare_x_and_y>
train = pd.read_json(".. /input/train.json")
 train.inc_angle = train.inc_angle.replace('na', 0)
 train.inc_angle = train.inc_angle.astype(float ).fillna(0.0)
 train_all = True
train_b = True or train_all
train_img = True or train_all
train_total = True or train_all
predict_submission = True and train_all
clean_all = True
clean_b = True or clean_all
clean_img = True or clean_all
load_all = False
load_b = False or load_all
load_img = False or load_all
X_train = np.concatenate([X_train, X_play_col], axis=1)
 y_train = np.zeros(shape=(X_train.shape[0], 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))<import_modules>
def create_dataset(frame, labeled, smooth_rgb=0.2, smooth_gray=0.5,
weight_rgb=0.05, weight_gray=0.05):
band_1, band_2, images = frame['band_1'].values, frame['band_2'].values, color_composite(frame)
 to_arr = lambda x: np.asarray([np.asarray(item)for item in x])
 band_1 = to_arr(band_1)
 band_2 = to_arr(band_2)
 band_3 =(band_1 + band_2)/ 2
gray_reshape = lambda x: np.asarray([item.reshape(75, 75)for item in x])
 band_1 = gray_reshape(band_1)
 band_2 = gray_reshape(band_2)
 band_3 = gray_reshape(band_3)
 print('Denoising and reshaping')
 if train_b and clean_b:
band_1 = smooth(denoise(band_1, weight_gray, False), smooth_gray)
 print('Gray 1 done')
 band_2 = smooth(denoise(band_2, weight_gray, False), smooth_gray)
 print('Gray 2 done')
 band_3 = smooth(denoise(band_3, weight_gray, False), smooth_gray)
 print('Gray 3 done')
 if train_img and clean_img:
images = smooth(denoise(images, weight_rgb, True), smooth_rgb)
 print('RGB done')
 tf_reshape = lambda x: np.asarray([item.reshape(75, 75, 1)for item in x])
 band_1 = tf_reshape(band_1)
 band_2 = tf_reshape(band_2)
 band_3 = tf_reshape(band_3)
 band = np.concatenate([band_1, band_2, band_3], axis=3)
 X_angle = np.array(frame.inc_angle)
 if labeled:
y = np.array(frame["is_iceberg"])
 else:
y = None
return y, X_angle, band, images
from keras.callbacks import EarlyStopping<init_hyperparams>
y_train, X_angles, X_b, X_images = create_dataset(train, True )
__all__ = ['RAdam']
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<train_model>
def get_model_notebook(angle, lr, decay, channels, relu_type='relu'):
input_1 = Input(shape=(75, 75, channels))
input_2 = Input(shape=[1])
 fcnn = Conv2D(32, kernel_size=(3, 3), activation=relu_type )(input_1)
 fcnn = MaxPooling2D(( 3, 3))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(64, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(128, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(128, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Flatten()(fcnn)
 if angle:
local_input = [input_1, input_2]
else:
local_input = input_1
dense = Dropout(0.2 )(fcnn)
 dense = Dense(256, activation=relu_type )(dense)
 partial_model = Model(input_1, fcnn)
 dense = Dropout(0.2 )(dense)
 dense = Dense(128, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 dense = Dense(64, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 if angle:
dense = Concatenate()([dense, input_2])
 else:
dense = dense
output = Dense(1, activation="sigmoid" )(dense)
 model = Model(local_input, output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model, partial_model
class Mish(Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 def build(self, input_shape):
super(Mish, self ).build(input_shape)
 def call(self, x):
return x * K.tanh(K.softplus(x))
def compute_output_shape(self, input_shape):
return input_shape<import_modules>
def combined_model(m_b, m_img, lr, decay):
input_b = Input(shape=(75, 75, 3))
input_img = Input(shape=(75, 75, 3))
input_angular = Input(shape=[1])
 m1 = m_b(input_b)
 m2 = m_img(input_img)
 common = Concatenate()([m1, m2])
 common = BatchNormalization()(common)
 common = Dropout(0.3 )(common)
 common = Dense(2048, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 common = Dense(1024, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 common = Dense(512, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 common = Concatenate()([common, BatchNormalization()(input_angular)])
 output = Dense(1, activation="sigmoid" )(common)
 model = Model([input_b, input_img, input_angular], output)
 optimizer = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )<import_modules>
def train_model(model, batch_size, epochs, checkpoint_name, X_train, y_train, verbose=2, val_data=None, val_split=0.15):
callbacks = [ModelCheckpoint(checkpoint_name, save_best_only=True, monitor='val_loss')]
try:
if val_data is None:
model.fit(X_train, y_train, epochs=epochs, validation_split=val_split,
batch_size=batch_size, callbacks=callbacks, verbose=verbose, shuffle=True)
 else:
x_val, y_val = val_data
model.fit(X_train, y_train, epochs=epochs, validation_data=[x_val, y_val],
batch_size=batch_size, callbacks=callbacks, verbose=verbose, shuffle=True)
 except KeyboardInterrupt:
if verbose > 0:
print('Interrupted')
 if verbose > 0:
print('Loading model')
 model.load_weights(filepath=checkpoint_name)
 return model
import tensorflow as tf<choose_model_class>
def get_angular_status(angle, x, x_a):
if angle:
result = [x, x_a]
else:
result = x
return result
def train_model(x_tr, y_tr, x_vl, y_vl):
inp = keras.layers.Input([X_train.shape[1]])
 x = keras.layers.Dense(units=324 )(inp)
 x = keras.layers.BatchNormalization(momentum=0.8, axis=1 )(x)
 x = Mish()(x)
 x = keras.layers.Dropout(0.35 )(x)
 x = keras.layers.Dense(units=512 )(x)
 x = keras.layers.BatchNormalization(momentum=0.8, axis=1 )(x)
 x = Mish()(x)
 x = keras.layers.Dropout(0.5 )(x)
 x = keras.layers.Dense(units=324 )(x)
 x = keras.layers.BatchNormalization(momentum=0.8, axis=1 )(x)
 x = Mish()(x)
 x = keras.layers.Dropout(0.35 )(x)
 x = keras.layers.Dense(units=199, activation='sigmoid' )(x)
 model = keras.Model(inp, x)
 er = EarlyStopping(patience=15, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 clr = CyclicLR(base_lr=1e-3, max_lr=5*1e-3, step_size = 1000, gamma = 0.99)
 model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5),
loss='mse',
metrics = ['mse'] 
)
 model.fit(x_tr, y_tr, epochs=100, callbacks=[clr, er], validation_data=[x_vl, y_vl])
 return model<normalization>
def gen_model_weights(angle, lr, decay, channels, relu, batch_size, epochs, path_name, data, only_load=False, verbose=2):
X_train, X_angle_train, y_train, X_val, X_angles_val, y_val = data
X_train, X_angle_train, y_train = shuffle(X_train, X_angle_train, y_train, random_state=np.random.randint(1, 123))
model, partial_model = get_model_notebook(angle, lr, decay, channels, relu)
 if only_load:
model.load_weights(path_name)
 return model, partial_model
model = train_model(model, batch_size, epochs, path_name,
get_angular_status(angle, X_train, X_angle_train), y_train, verbose=verbose)
 if verbose > 0:
loss_val, acc_val = model.evaluate(get_angular_status(angle, X_val, X_angles_val), y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = model.evaluate(get_angular_status(angle, X_train, X_angle_train), y_train,
verbose=0, batch_size=batch_size)
 print('Val/Train Loss:', str(loss_val)+ '/' + str(loss_train), \
'Val/Train Acc:', str(acc_val)+ '/' + str(acc_train))
return model, partial_model
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train )<categorify>
def train_models(dataset, lr, batch_size, max_epoch, verbose=2, return_model=False):
X_angles, y_train, X_b, X_images = dataset
angle_b = True
angle_images = True
X_angles, X_angles_val,\
y_train, y_val,\
X_b, X_b_val,\
X_images, X_images_val = train_test_split(X_angles, y_train, X_b, X_images, random_state=687, train_size=0.9)
 if train_b:
if verbose > 0:
print('Training bandwidth network')
 data_b1 =(X_b, X_angles, y_train, X_b_val, X_angles_val, y_val)
 model_b, model_b_cut = gen_model_weights(angle_b, lr, 0, 3, 'relu', batch_size, max_epoch, 'model_b',
data_b1, only_load=load_b, verbose=verbose)
 if train_img:
if verbose > 0:
print('Training image network')
 data_images =(X_images, X_angles, y_train, X_b_val, X_angles_val, y_val)
 model_images, model_images_cut = gen_model_weights(angle_images, lr, 0, 3, 'relu', batch_size, max_epoch, 'model_img',
data_images, only_load=load_img, verbose=verbose)
 if train_total:
common_model = combined_model(model_b_cut, model_images_cut, lr, 0)
 common_x_train = [X_b, X_images, X_angles]
common_y_train = y_train
common_x_val = [X_b_val, X_images_val, X_angles_val]
common_y_val = y_val
if verbose > 0:
print('Training common network')
 common_model = train_model(common_model, batch_size, max_epoch, 'common_check', common_x_train,
common_y_train, verbose=verbose, val_split=0.2)
 loss_val, acc_val = common_model.evaluate(common_x_val, common_y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = common_model.evaluate(common_x_train, common_y_train,
verbose=0, batch_size=batch_size)
 if verbose > 0:
print('Loss:', loss_val, 'Acc:', acc_val)
 if return_model:
return common_model
else:
return(loss_train, acc_train),(loss_val, acc_val )
def make_pred(df, sample, env, models):
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(test.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df.drop(['Yards'], axis=1, inplace=True)
 df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 df['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df = df.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'WindDirection', 'NflId', 'NflIdRusher', 'GameId', 'PlayId', 'index', 'IsRusher', 'Team', 'Season'], axis=1)
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 X = np.array(df[players_col] ).reshape(-1, 11*22)
 play_col = df.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = df[col][::22]
X = scaler.transform(np.concatenate([X, X_play_col], axis=1))
y_pred = np.mean([model.predict(X)for model in models], axis=0)
 for pred in y_pred:
prev = 0
for i in range(len(pred)) :
if pred[i]<prev:
pred[i]=prev
prev=pred[i]
env.predict(pd.DataFrame(data=y_pred,columns=sample.columns))
return y_pred<choose_model_class>
common_model = train_models(( X_angles, y_train, X_b, X_images), 5e-04, 32, 50, 1, return_model=True )
rkf = RepeatedKFold(n_splits=5, n_repeats=5 )<import_modules>
if predict_submission:
print('Reading test dataset')
 test = pd.read_json(".. /input/test.json")
 test.inc_angle = test.inc_angle.replace('na', 0)
 test.inc_angle = test.inc_angle.astype(float ).fillna(0.0)
 y_fin, X_angle_fin, X_fin_b, X_fin_img = create_dataset(test, False)
 print('X shape:', X_fin_img.shape)
 print('X angle shape:', X_angle_fin.shape)
 print('Predicting')
 prediction = common_model.predict([X_fin_b, X_fin_img, X_angle_fin], verbose=1, batch_size=32)
 print('Submitting')
 submission = pd.DataFrame({'id': test["id"], 'is_iceberg': prediction.reshape(( prediction.shape[0])) })
 submission.to_csv("./submission.csv", index=False)
 print('Done' )