Kaggle competition: Rossmann Store Sales
TEST_PATH = ".. /input/plant-pathology-2020-fgvc7/test.csv"
SUB_PATH = ".. /input/plant-pathology-2020-fgvc7/sample_submission.csv"
sub = pd.read_csv(SUB_PATH)
 test_data = pd.read_csv(TEST_PATH )<feature_engineering>
%matplotlib inline
test_paths = test_data.image_id.apply(format_path_gcs ).values<categorify>
def to_weight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(y, yhat):
w = to_weight(y)
 rmspe = np.sqrt(np.mean(w *(y - yhat)**2))
return rmspe
def str_date_to_ordinal(date):
return datetime.strptime(date, '%Y-%m-%d' ).date().toordinal()
def date_to_ordinal(date):
return date.toordinal()
def str_to_date(date):
return datetime.strptime(date, '%Y-%m-%d' ).date()
SEED = 42
test_dataset =(
 tf.data.Dataset.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<save_to_csv>
train_df = pd.read_csv('.. /input/train.csv', sep=',', parse_dates=['Date'], date_parser=str_to_date,
low_memory=False)
 test_df = pd.read_csv('.. /input/test.csv', sep=',', parse_dates=['Date'], date_parser=str_to_date,
low_memory=False )
prediction = model_Eff_B7.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = prediction
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
test_df[pd.isnull(test_df.Open)]
print(tf.__version__)
 print(tf.keras.__version__ )<install_modules>
test_df.loc[pd.isnull(test_df.Open), 'Open'] = 1
!pip install efficientnet 
<set_options>
all_df = pd.concat([train_df, test_df], axis=0)
 all_df.head()
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<train_model>
all_df['StateHoliday'] = all_df['StateHoliday'].astype('category' ).cat.codes
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_0.jpg')
 print(img.shape)
 plt.imshow(img )<load_from_csv>
feature_name = 'Date'
dispatcher = {'DayOfMonth': pd.Index(all_df[feature_name] ).day,
'WeekOfYear': pd.Index(all_df[feature_name] ).week,
'MonthOfYear': pd.Index(all_df[feature_name] ).month,
'Year': pd.Index(all_df[feature_name] ).year,
'DayOfYear': pd.Index(all_df[feature_name] ).dayofyear
}
for new_feat_suffx, mapping in dispatcher.items() :
all_df[feature_name + new_feat_suffx] = mapping
all_df[feature_name] = all_df[feature_name].apply(date_to_ordinal)
 all_df.head()
path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<define_variables>
store_df = pd.read_csv('.. /input/store.csv', sep=',')
 store_df.head()
nb_classes = 4
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 40<normalization>
def CompetitionOpenSince_to_ordinal(row):
try:
date = '%d-%d' %(int(row['CompetitionOpenSinceYear']), int(row['CompetitionOpenSinceMonth']))
return datetime.strptime(date, '%Y-%m' ).date().toordinal()
except:
return np.nan
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
store_df['CompetitionOpenSinceDate'] = store_df.apply(CompetitionOpenSince_to_ordinal, axis=1)
 mean_CompetitionOpenSince = store_df['CompetitionOpenSinceDate'].mean()
store_df['CompetitionOpenSinceDate'] = store_df['CompetitionOpenSinceDate'].fillna(
 mean_CompetitionOpenSince ).astype(np.int64)
 store_df.head()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<categorify>
def Promo2Since_to_ordinal(row):
try:
date = '%d-W%d' %(int(row['Promo2SinceYear']), int(row['Promo2SinceWeek']))
return datetime.strptime(date + '-1', '%Y-W%W-%w' ).date().toordinal()
except:
return np.nan
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
store_df['Promo2SinceDate'] = store_df.apply(Promo2Since_to_ordinal, axis=1)
 mean_Promo2SinceDate = store_df['Promo2SinceDate'].mean()
store_df['Promo2SinceDate'] = store_df['Promo2SinceDate'].fillna(mean_Promo2SinceDate ).astype(np.int64)
 store_df.loc[store_df['Promo2'] == 0, 'Promo2SinceDate'] = 0
store_df.head()
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
mode_PromoInterval = store_df[store_df['PromoInterval'].notnull() ]['PromoInterval'].mode() [0]
promo_intervals = {'Jan,Apr,Jul,Oct':(1,4,7,10),
'Feb,May,Aug,Nov':(2,5,8,11),
'Mar,Jun,Sept,Dec':(3,6,9,12)
 }
promo_intervals[np.nan] = promo_intervals[mode_PromoInterval]
store_df['PromoInterval'] = store_df['PromoInterval'].apply(lambda x: promo_intervals[x])
 store_df.head()
def get_model() :
base_model = efn.EfficientNetB7(weights='imagenet', include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(nb_classes, activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions )<choose_model_class>
columns_to_drop = ['CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear',
'Promo2SinceWeek', 'Promo2SinceYear']
store_df.drop(columns_to_drop, axis=1, inplace=True)
 store_df.head()
with strategy.scope() :
model = get_model()
model.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy'] )<train_model>
mean_CompetitionDistance = store_df.CompetitionDistance.mean()
store_df['CompetitionDistance'] = store_df['CompetitionDistance'].fillna(mean_CompetitionDistance)
 store_df.info()
%%time
model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[lr_callback],
epochs=EPOCHS
)<predict_on_test>
df = pd.merge(all_df, store_df, how='left', on=['Store'])
 df.head().T
%%time
probs = model.predict(test_dataset )<save_to_csv>
def is_promo2_today(row):
return int(row['Promo2'] == 1 and row['Promo2SinceDate'] <= row['Date'] \
and row['DateMonthOfYear'] in row['PromoInterval'])
 def is_competition_open(row):
return int(row['CompetitionOpenSinceDate'] <= row['Date'] )
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()<load_from_csv>
df['Promo2Today'] = df.apply(is_promo2_today, axis=1)
 df['CompetitionIsOpen'] = df.apply(is_competition_open, axis=1)
 avg_checks = pd.DataFrame(df.groupby('Store')['Sales'].sum().astype(np.float64)\
/ df.groupby('Store')['Customers'].sum().astype(np.float64),
columns=['AverageCheck'] ).reset_index()
df = df.merge(avg_checks, on='Store', how='left')
 df.head().T
print(check_output(["ls", ".. /input"] ).decode("utf8"))
data = pd.read_csv('.. /input/train.csv')
 test_ = pd.read_csv('.. /input/test.csv')
 data['val']='train'
test_['val']='test'
all_data=pd.concat([data,test_])
 cat_cols=[i for i in all_data.columns if 'cat' in i]
new_data=pd.get_dummies(all_data,columns=cat_cols,drop_first=True)
 X=[i for i in new_data.columns if i not in ['id','target','val']]
Y=['target']
train_data=new_data[new_data['val']=='train']
test_data=new_data[new_data['val']=='test']
del all_data
del data
del new_data
print('Data process Complete' )<compute_train_metric>
columns_to_drop = ['PromoInterval', 'Date', 'CompetitionOpenSinceDate', 'Promo2SinceDate']
df.drop(columns_to_drop, axis=1, inplace=True )
def gini(list_of_values):
sorted_list = sorted(list(list_of_values))
height, area = 0, 0
for value in sorted_list:
height += value
area += height - value / 2.
 fair_area = height * len(list_of_values)/ 2
return(fair_area - area)/ fair_area
def normalized_gini(y_pred, y):
normalized_gini = gini(y_pred)/gini(y)
 return normalized_gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = gini_normalized(labels, preds)
 return [('gini', gini_score)]
K = 5
kf = cross_validation.KFold(train_data.shape[0],n_folds= K, random_state = 3228, shuffle = True )<train_model>
X_train = train_df[train_df.columns.drop(['Customers', 'Sales'])].values
y_train = train_df['Sales'].values
print(X_train.shape, y_train.shape )
xgb_preds=[]
for train_index, test_index in kf:
print('A')
 train_X, valid_X = train_data[X].loc[train_index], train_data[X].loc[test_index]
train_y, valid_y = train_data[Y].loc[train_index], train_data[Y].loc[test_index]
xgb_params = {
'eta': 0.02,
'max_depth': 6,
'subsample': 0.9,
'objective': 'binary:logistic',
'silent': 1,
'colsample_bytree': 0.9
}
d_train = xgb.DMatrix(train_X, train_y)
 d_valid = xgb.DMatrix(valid_X, valid_y)
 d_test = xgb.DMatrix(test_data[X])
 model = xgb.train(xgb_params, d_train, num_boost_round = 400)
 xgb_pred = model.predict(d_test)
 xgb_preds.append(list(xgb_pred))<save_to_csv>
columns = train_df.columns.drop(['Sales', 'Customers'])
 del train_df
gc.collect()
preds=[]
for i in range(len(xgb_preds[0])) :
sum=0
for j in range(K):
sum+=xgb_preds[j][i]
preds.append(sum / K)
 output = pd.DataFrame({'id': test_['id'], 'target': preds})
 output.to_csv("sol_sub.csv".format(K), index=False)
 print('process done' )<set_options>
color = sns.color_palette()
py.init_notebook_mode(connected=True)
 init_notebook_mode(connected=True)
 offline.init_notebook_mode()
cf.go_offline() 
<load_from_csv>
rf = RandomForestRegressor(n_estimators=50, random_state=SEED)
 rf.fit(X_train, y_train )
application_train = pd.read_csv('/kaggle/input/home-credit-default-risk/application_train.csv')
 POS_CASH_balance = pd.read_csv('/kaggle/input/home-credit-default-risk/POS_CASH_balance.csv')
 bureau_balance = pd.read_csv('/kaggle/input/home-credit-default-risk/bureau_balance.csv')
 previous_application = pd.read_csv('/kaggle/input/home-credit-default-risk/previous_application.csv')
 installments_payments = pd.read_csv('/kaggle/input/home-credit-default-risk/installments_payments.csv')
 credit_card_balance = pd.read_csv('/kaggle/input/home-credit-default-risk/credit_card_balance.csv')
 bureau = pd.read_csv('/kaggle/input/home-credit-default-risk/bureau.csv')
 application_test = pd.read_csv('/kaggle/input/home-credit-default-risk/application_test.csv' )<sort_values>
application_train.isnull().mean().sort_values(ascending = False )<sort_values>
test_predicted = rf.predict(X_test)
 test_predicted.shape
POS_CASH_balance.isnull().mean().sort_values(ascending = False )<sort_values>
submission = pd.concat([test_df.Id.astype(int), pd.DataFrame(test_predicted,
columns=['Sales'], index=test_df.index)], axis=1)
 submission.head()
<sort_values><EOS> 
submission[['Id', 'Sales']].to_csv('submission_6.csv', index=False )