Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<choose_model_class>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
def generate_kfolds(n_folds=10, image_names=None, categorical_labels=None):
return KFold(n_splits=n_folds, random_state=333, shuffle=True )<compute_test_metric>
df= pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv")
 sell_prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv")
 calendar = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv" )
gamma = 2.0
epsilon = tf.keras.backend.epsilon()
def focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = tf.keras.backend.clip(pt, epsilon, 1-epsilon)
 CE = -tf.log(pt)
 FL = tf.pow(1-pt, gamma)* CE
loss = tf.keras.backend.sum(FL, axis=1)
 return loss
def f2_score(y_true, y_pred):
beta = 2
true_positives = tf.keras.backend.sum(tf.round(tf.keras.backend.clip(y_true * y_pred, 0, 1)) , axis=1)
 predicted_positives = tf.keras.backend.sum(tf.round(tf.keras.backend.clip(y_pred, 0, 1)) , axis=1)
 possible_positives = tf.keras.backend.sum(tf.round(tf.keras.backend.clip(y_true, 0, 1)) , axis=1)
 precision = true_positives /(predicted_positives + tf.keras.backend.epsilon())
 recall = true_positives /(possible_positives + tf.keras.backend.epsilon())
 return tf.keras.backend.mean(((1+beta**2)*precision*recall)/(( beta**2)*precision+recall+tf.keras.backend.epsilon()))<define_variables>
del df,store_agg,cat_agg,agg_sales,one,x
gc.collect()
custom_weight = True
training = False<choose_model_class>
warnings.filterwarnings('ignore' )
def create_model(input_shape, n_out):
input_tensor = tf.keras.layers.Input(shape=input_shape)
 base_model = tf.keras.applications.Xception(include_top=False,
weights=None,
input_tensor=input_tensor)
 if(not custom_weight):
base_model.load_weights('.. /input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)
 x = tf.keras.layers.Dense(1024, activation='relu' )(x)
 final_output = tf.keras.layers.Dense(n_out, activation='sigmoid', name='final_output' )(x)
 model = tf.keras.Model(input_tensor, final_output)
 if(custom_weight):
model.load_weights('.. /input/imetpretrained/imet_xception_val_f2_0.686829-f2_0.611568.h5')
 return model<choose_model_class>
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar = reduce_mem_usage(calendar)
 sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 sell_prices = reduce_mem_usage(sell_prices)
 sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 submission = reduce_mem_usage(submission )
checkpoint = tf.keras.callbacks.ModelCheckpoint('.. /working/imet_xception_val_f2_{val_f2_score:3f}-f2_{f2_score:3f}.h5',
monitor='val_loss', verbose=1, save_best_only=True, mode='min',
save_weights_only = True)
 reduceLROnPlat = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = tf.keras.callbacks.EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = tf.keras.callbacks.CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]<choose_model_class>
sales_train_validation = pd.melt(sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation = reduce_mem_usage(sales_train_validation )
if(training and not custom_weight):
n_folds = 10
for layer in model.layers:
layer.trainable = False
for i in range(-5,0):
model.layers[i].trainable = True
model.compile(
 loss=focal_loss,
optimizer=tf.keras.optimizers.Adam(1e-3),
metrics=[f2_score])
 kfolder = generate_kfolds(n_folds, image_names=train_val_df['id'], categorical_labels=categorical_labels)
 list_loss = []
list_f2 = []
for i,(train_indices, val_indices)in enumerate(kfolder.split(train_val_df['id'], categorical_labels)) :
print("============== Fold",(i+1), "================")
 train_dataset = create_dataset(train_val_df.loc[train_indices, 'id'], categorical_labels[train_indices, :], preproc_fn=tr_preprocessing_fn, threads=2)
 val_dataset = create_dataset(train_val_df.loc[val_indices, 'id'], categorical_labels[val_indices, :], preproc_fn=val_preprocessing_fn, threads=2)
 history = model.fit_generator(
 train_dataset,
steps_per_epoch=int(np.ceil(float(len(train_indices)) / float(BATCH_SIZE))),
validation_data=val_dataset,
validation_steps=int(np.ceil(float(len(val_indices)) / float(BATCH_SIZE))),
epochs=1,
verbose=1,
callbacks=callbacks_list)
 list_loss.append(history.history['val_loss'][0])
 list_f2.append(history.history['val_f2_score'][0])
 print("Val loss of fold %d = %f"%(i+1, history.history['val_loss'][0]))
print("F2 score of fold %d = %f"%(i+1, history.history['val_f2_score'][0]))
print("
Mean val loss over all: ", np.mean(np.array(list_loss)))
 print("Mean f2_score over all: ", np.mean(np.array(list_f2)) )<choose_model_class>
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test2 = submission[submission['id'].isin(test2_rows)]
test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959',
'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']
product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
test2 = test2.merge(product, how = 'left', on = 'id')
 test2 = pd.melt(test2,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation['part'] = 'train'
test2['part'] = 'test'
if(training):
n_folds = 15
for layer in model.layers:
layer.trainable = True
model.compile(loss=focal_loss,
optimizer=tf.keras.optimizers.Adam(lr=0.0003),
metrics=[f2_score])
 checkpoint = tf.keras.callbacks.ModelCheckpoint('.. /working/imet_xception_val_f2_{val_f2_score:3f}-f2_{f2_score:3f}.h5',
monitor='val_loss', verbose=1, save_best_only=True,
mode='min', save_weights_only = True)
 reduceLROnPlat = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1,
verbose=1, mode='auto', epsilon=0.0001)
 early = tf.keras.callbacks.EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
print("Start training...")
 kfolder = generate_kfolds(n_folds, image_names=train_val_df['id'], categorical_labels=categorical_labels)
 list_loss = []
list_f2 = []
for i,(train_indices, val_indices)in enumerate(kfolder.split(train_val_df['id'], categorical_labels)) :
print("============== Fold",(i+1), "================")
 train_dataset = create_dataset(train_val_df.loc[train_indices, 'id'], categorical_labels[train_indices, :], preproc_fn=tr_preprocessing_fn, threads=2)
 val_dataset = create_dataset(train_val_df.loc[val_indices, 'id'], categorical_labels[val_indices, :], preproc_fn=val_preprocessing_fn, threads=2)
 history = model.fit_generator(
 train_dataset,
steps_per_epoch=int(np.ceil(float(len(train_indices)) / float(BATCH_SIZE))),
validation_data=val_dataset,
validation_steps=int(np.ceil(float(len(val_indices)) / float(BATCH_SIZE))),
epochs=1,
verbose=1,
callbacks=callbacks_list)
 list_loss.append(history.history['val_loss'][0])
 list_f2.append(history.history['val_f2_score'][0])
 print("Val loss of fold %d = %f"%(i+1, history.history['val_loss'][0]))
print("F2 score of fold %d = %f"%(i+1, history.history['val_f2_score'][0]))
print("
Mean val loss over all: ", np.mean(np.array(list_loss)))
 print("Mean f2_score over all: ", np.mean(np.array(list_f2)) )<load_from_csv>
data = pd.concat([sales_train_validation,test2], axis = 0)
 del sales_train_validation,test2
gc.collect()
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 predicted = []<load_pretrained>
data.reset_index(drop=True,inplace=True)
 data = data.loc[30000000:]
calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)
 data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1 )
def file_filter(filename):
return filename.startswith('imet_xception')
 if(training):
model_files = list(filter(file_filter, os.listdir('.. /working/')))
 model_files = sorted(model_files)
 weight_path = os.path.join(".. /working/", model_files[-1])
 print("Current weight path for prediction: ", weight_path)
 model.load_weights(weight_path )<predict_on_test>
data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 del calendar,sell_prices,product
gc.collect()
BATCH = 512
check_dataset = create_dataset(train_val_df['id'], categorical_labels, preproc_fn=val_preprocessing_fn, threads=2)
 counter = 0
n_val = round(len(train_val_df)*0.5//BATCH)
 print(n_val)
 lastFullValPred = np.empty(( 0, N_CLASSES))
lastFullValLabels = np.empty(( 0, N_CLASSES))
for X, y in tqdm(check_dataset):
if(counter >= n_val):
break
else:
counter += 1
y_pred = model.predict(X.numpy())
 lastFullValPred = np.append(lastFullValPred, y_pred, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, y, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape )<compute_train_metric>
nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in nan_features:
data[feature].fillna('unknown', inplace = True)
 cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in cat:
encoder = LabelEncoder()
data[feature] = encoder.fit_transform(data[feature] )
beta_f2 = 2
def my_f2(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1))
tn = np.sum(( y_true == 0)&(y_pred == 0))
fp = np.sum(( y_true == 0)&(y_pred == 1))
fn = np.sum(( y_true == 1)&(y_pred == 0))
p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + 1e-15)
 return f2
def find_best_fixed_threshold(preds, targs, do_plot=True):
score = []
thrs = np.arange(0, 0.5, 0.01)
 for thr in tqdm(thrs):
score.append(my_f2(targs,(preds > thr ).astype(int)))
 score = np.array(score)
 pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')
 if do_plot:
plt.plot(thrs, score)
 plt.vlines(x=best_thr, ymin=score.min() , ymax=score.max())
 plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);
plt.show()
return best_thr, best_score<compute_train_metric>
data = reduce_mem_usage(data )
best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True )<normalization>
data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))
data['price_change_t1'] =(data['lag_price_t1'] - data['sell_price'])/(data['lag_price_t1'])
 data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1 ).rolling(365 ).max())
 data['price_change_t365'] =(data['rolling_price_max_t365'] - data['sell_price'])/(data['rolling_price_max_t365'])
 data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7 ).std())
 data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30 ).std())
 data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1 )
def load_img(img_name):
path = '.. /input/imet-2019-fgvc6/test/'+ img_name +".png"
img_string = tf.read_file(path)
 img = tf.image.decode_png(img_string)
 shape = tf.shape(img)
 tfwidth = tf.to_float(shape[1])
 tfheight = tf.to_float(shape[0])
 img = tf.cond(tf.math.logical_or(tfwidth/tfheight > 3.0, tfwidth/tfheight < 1/3.0),
lambda: tf.image.random_crop(img, size=[300, 300, 3]),
lambda: img)
 img = tf.image.resize(img,(SIZE, SIZE))
img = tf.to_float(img)/ 255.0
return img<categorify>
data['date'] = pd.to_datetime(data['date'])
 data['year'] = data['date'].dt.year
data['month'] = data['date'].dt.month
data['week'] = data['date'].dt.week
data['day'] = data['date'].dt.day
data['dayofweek'] = data['date'].dt.dayofweek
submit_ds = tf.data.Dataset.from_tensor_slices(submit['id'])
 submit_ds = submit_ds.apply(tf.data.experimental.map_and_batch(map_func=load_img, batch_size=BATCH_SIZE))<predict_on_test>
def weekend(arg):
if arg==5 or arg==6:
return 1
else:
return 0
data['isweekend'] = data['dayofweek'].apply(weekend )
predictions = model.predict_generator(submit_ds, steps = int(np.ceil(len(submit['id'])/ BATCH_SIZE)) , verbose=1 )<define_variables>
data['revenue'] = data['demand'] * data['sell_price']
data['lag_revenue_t1'] = data.groupby(['id'])['revenue'].transform(lambda x: x.shift(28))
data['rolling_revenue_std_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).std())
 data['rolling_revenue_mean_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).mean())
 data.drop(['revenue'],axis=1,inplace=True )
label_indices = []
for raw_pred in predictions:
label_predict = np.arange(N_CLASSES)[raw_pred>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 label_indices.append(str_predict_label )<save_to_csv>
features = ['item_id', 'cat_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2',
'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90',
'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30', 'rolling_skew_t30', 'rolling_kurt_t30',
'isweekend','lag_revenue_t1','rolling_revenue_std_t28','rolling_revenue_mean_t28'
]
data = reduce_mem_usage(data )
submit['attribute_ids'] = label_indices
submit.to_csv('submission.csv', index=False )<count_values>
x_train = data[data['part'] == 'train']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-04-24')&(data['part'] == 'train')]
y_val = x_val['demand']
test = data[data['date'] > '2016-04-24']
test.loc[test['part']=='train','id'] = test.loc[test['part']=='train','id'].str.replace('_evaluation','_validation')
 x_train.drop(['demand','part',],inplace=True,axis=1)
 x_val.drop(['demand','part',],inplace=True,axis=1)
 test.drop(['demand','part',],inplace=True,axis=1 )
submit['attribute_ids'].map(lambda row: len(row.split(" ")) ).value_counts()<set_options>
del data
gc.collect()
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 beta_f2=2<define_variables>
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.1}
nb_classes = 1103
img_size = 224
nb_epochs = 25<define_variables>
TEST = None
for i in category:
for j in category2:
x_t,x_v,y_t,y_v,t = dataset(i,j)
 train_set = Dataset(x_t[features], y_t)
 val_set = Dataset(x_v[features], y_v)
 del x_t, y_t
gc.collect()
model = train(params, train_set, num_boost_round = 150, early_stopping_rounds = 40, valid_sets = [train_set, val_set], verbose_eval = 1000)
 y_pred = model.predict(t[features])
 t['demand'] = y_pred
TEST = pd.concat([TEST,t],axis=0)
 del x_v,y_v,t,train_set,val_set,y_pred,model
gc.collect()
imet_path = '.. /input/imet-2019-fgvc6/'
model_weights_path = '.. /input/densenet121weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5'
train_df_path = imet_path + 'train.csv'
label_df_path = imet_path + 'labels.csv'
sub_df_path = imet_path + 'sample_submission.csv'
train_img_path = imet_path + 'train/'
test_img_path = imet_path + 'test/'<init_hyperparams>
predictions1 = TEST[TEST['id'].apply(lambda x: "validation" in x)][['id', 'date', 'demand']]
predictions2 = TEST[TEST['id'].apply(lambda x: "evaluation" in x)][['id', 'date', 'demand']]
class AdamAccumulate_v1(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):
super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, dtype='int64')
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update(self.iterations, self.iterations + 1)]
flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)
 flag = K.cast(flag, K.floatx())
 self.updates.append(K.update(self.effective_iterations,
self.effective_iterations + K.cast(flag, 'int64')))
 lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.effective_iterations,
K.dtype(self.decay))))
t = K.cast(self.effective_iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):
gg_t =(1 - flag)*(gg + g)
 m_t =(self.beta_1 * m)+(1.- self.beta_1)*(gg + flag * g)/ K.cast(self.accum_iters, K.floatx())
 v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(
(gg + flag * g)/ K.cast(self.accum_iters, K.floatx()))
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - flag * lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
p_t = p - flag * lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(( m, flag * m_t +(1 - flag)* m))
self.updates.append(( v, flag * v_t +(1 - flag)* v))
self.updates.append(( gg, gg_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
class AdamAccumulate(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):
if accum_iters < 1:
raise ValueError('accum_iters must be >= 1')
 super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))
self.accum_iters_float = K.cast(self.accum_iters, K.floatx())
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())
 if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t)))
 update_switch = K.equal(( self.iterations + 1)% self.accum_iters, 0)
 update_switch = K.cast(update_switch, K.floatx())
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t =(self.beta_1 * m)+(1.- self.beta_1)* avg_grad
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(avg_grad)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat,(1 - update_switch)* vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m,(1 - update_switch)* m + update_switch * m_t))
self.updates.append(K.update(v,(1 - update_switch)* v + update_switch * v_t))
self.updates.append(K.update(tg,(1 - update_switch)* sum_grad))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p,(1 - update_switch)* p + update_switch * new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<load_from_csv>
predictions = pd.concat([prediction1,prediction2],axis=0)
 predictions.to_csv("submissionstoreanddeptwise.csv",index=False )
<compute_test_metric><EOS> 
!gzip submissionstoreanddeptwise.csv