Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_from_csv>
pd.set_option('display.max_rows', 500)
 %matplotlib inline
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values 
<normalization>
def downcast_dtypes(df):
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
img_size = 768
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
RAW_DATA_FOLDER = '/kaggle/input/competitive-data-science-predict-future-sales/'
DATA_FOLDER = '/kaggle/working/'
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
loaded=%who_ls
loaded.append('loaded' )
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
all_vars=%who_ls
all_vars.append('all_vars')
 for var in list(set(all_vars)-set(loaded)) :
exec('del '+var)
 del var
def get_model() :
base_model = InceptionResNetV2(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(train_labels.shape[1], activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model1 = get_model()
model1.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['categorical_accuracy'])
 model1.load_weights("/kaggle/input/tf-zoo-models-on-tpu-inceptionresnetv2/InceptionResNetV2.h5" )<choose_model_class>
%macro reset_variable_space 6
loaded.append('reset_variable_space' )
def get_model() :
base_model = EfficientNetB7(weights='noisy-student',
include_top=False, pooling='avg',
input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(train_labels.shape[1], activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model2 = get_model()
model2.compile(optimizer='nadam', loss='categorical_crossentropy',metrics=['categorical_accuracy'])
 model2.load_weights("/kaggle/input/tf-zoo-models-on-tpu-efficientnetb7/EfficientNetB7.h5" )<save_to_csv>
train_df = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'sales_train.csv'))
items = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'items.csv'))
item_categories = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'item_categories.csv'))
shops = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'shops.csv'))
test_df = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'test.csv'))
probs =(model1.predict(test_dataset)+model2.predict(test_dataset)) /2
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
print('train_df :' + str(train_df.shape))
print()
train_df.info(null_counts=True)
 print('_'*40)
 print()
print('test set :' + str(test_df.shape))
print()
test_df.info(null_counts=True)
 print('_'*40)
 print('_'*40)
 print()
print('number of different items in training set : '+str(train_df['item_id'].nunique())+ ' / ' + str(items.item_id.nunique()))
print('number of different items in test set : '+str(test_df['item_id'].nunique())+ ' / ' + str(items.item_id.nunique()))
print('number of different items ONLY in test set : '+str(test_df.loc[~test_df['item_id'].isin(train_df['item_id']),'item_id'].nunique()))
print()
print('number of different shops in training set : '+str(train_df['shop_id'].nunique())+ ' / ' + str(shops.shop_id.nunique()))
print('number of different shops in test set : '+str(test_df['shop_id'].nunique())+ ' / ' + str(shops.shop_id.nunique()))
from sklearn.model_selection import train_test_split
import cv2
import matplotlib.pyplot as plt
import keras
from keras.preprocessing.image import img_to_array<load_from_csv>
train_df.rename({'date_block_num':'month_id', 'item_cnt_day':'item_quantity'},axis=1,inplace=True )
train_df=pd.read_csv('.. /input/plant-pathology-2020-fgvc7/train.csv')
 test_df=pd.read_csv('.. /input/plant-pathology-2020-fgvc7/test.csv' )<define_variables>
train_df['date']=pd.DataFrame(pd.to_datetime(train_df['date'],format="%d.%m.%Y"))
target_df = train_df[['healthy', 'multiple_diseases', 'rust', 'scab']]
test_ids = test_df['image_id']<train_model>
train_df['day_id']=(train_df['date']-train_df['date'].min() ).dt.days
img_size = 300
train_imgs = []
for name in train_df['image_id']:
path = '.. /input/plant-pathology-2020-fgvc7/images/' + name + '.jpg'
img = cv2.imread(path)
 image = cv2.resize(img,(img_size,img_size),interpolation=cv2.INTER_AREA)
 train_imgs.append(image )<train_model>
print('percentage of realisations that represent returned articles : ' +str(round(( train_df['item_quantity']<0 ).sum() /(train_df['item_quantity']>=0 ).sum() *100,2)) + ' %')
 train_df.drop(train_df[train_df['item_quantity']<0].index,axis=0,inplace=True )
X_train = np.ndarray(shape=(len(train_imgs), img_size, img_size, 3),dtype = np.float32)
 for i, image in enumerate(train_imgs):
X_train[i] = img_to_array(image)
 X_train[i] = train_imgs[i]
X_train = X_train/255
print('Train Shape: {}'.format(X_train.shape))<train_model>
print('shop 39 : ' + shops.shop_name[39])
 print('shop 40 : ' + shops.shop_name[40])
 print()
tmp=train_df[(train_df['shop_id']==39)|(train_df['shop_id']==40)].drop('month_id',axis=1)
 print('
print(tmp['shop_id'].value_counts())
 print()
print('
tmpa=train_df[(train_df['shop_id']==39)].drop('month_id',axis=1)
 tmpb=train_df[(train_df['shop_id']==40)].drop('month_id',axis=1)
 print('duplicates date/item(shop 39): ' + str(tmpa.duplicated(subset=['date','item_id'],keep=False ).sum()))
print('duplicates date/item(shop 40): ' + str(tmpb.duplicated(subset=['date','item_id'],keep=False ).sum()))
print('...but the reunion of both has many...')
 print('duplicates date/item : ' + str(tmp.duplicated(subset=['date','item_id'],keep=False ).sum()))
print('duplicates date/item/price : ' + str(tmp.duplicated(subset=['date','item_id','item_price'],keep=False ).sum()))
print('duplicates date/item/price/quantity : ' + str(tmp.duplicated(subset=['date','item_id','item_price','item_quantity'],keep=False ).sum()))
print('...so all duplicates are exactly paired between the two shops(39,40)!
print()
print('
print('shop 39 : ' + tmp[tmp.shop_id==39].date.min().strftime("%d.%m.%Y")+ ' --- ' + tmp[tmp.shop_id==39].date.max().strftime("%d.%m.%Y"))
print('shop 40 : ' + tmp[tmp.shop_id==40].date.min().strftime("%d.%m.%Y")+ ' --- ' + tmp[tmp.shop_id==40].date.max().strftime("%d.%m.%Y"))
print()
print('
print('
tmpr=tmp[tmp['date']<tmpb['date'].max() ]
print(tmpr['shop_id'].value_counts())
 print()
print('
pa=np.zeros(22170)
 pa[tmpa['item_id'].unique() ]=1
pa.astype(int)
 pb=np.zeros(22170)
 pb[tmpb['item_id'].unique() ]=1
pb.astype(int)
 print('number of different items sold in shop 39 : ' + str(sum(pa ).astype(int)))
 print('number of different items sold in shop 40 : ' + str(sum(pb ).astype(int)))
 print('number of different items sold in both shops : ' + str(sum(pa*pb ).astype(int)))
 print()
print("
print("
print()
del tmp,tmpa,tmpb,tmpr,pa,pb
X_test = np.ndarray(shape=(len(test_imgs), img_size, img_size, 3),dtype = np.float32)
 for i, image in enumerate(test_imgs):
X_test[i] = img_to_array(image)
 X_test[i] = test_imgs[i]
X_test = X_test/255
print('Test Shape: {}'.format(X_test.shape))<drop_column>
shops.drop(0,axis=0,inplace=True)
 shops.drop(1,axis=0,inplace=True)
 shops.drop(11,axis=0,inplace=True)
 shops.drop(40,axis=0,inplace=True)
 train_df.loc[train_df['shop_id']==0,'shop_id']=57
train_df.loc[train_df['shop_id']==1,'shop_id']=58
train_df.loc[train_df['shop_id']==11,'shop_id']=10
train_df.loc[train_df['shop_id']==40,'shop_id']=39
y_train = train_df.copy()
del y_train['image_id']
y_train.head()<split>
train_df=train_df.groupby(list(train_df.columns.drop('item_quantity')) ,as_index=False ).sum()
train_df.info(null_counts=True )
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
 X_train.shape, X_val.shape<choose_model_class>
print('Remaining duplicates in the training set : ' + str(train_df.duplicated(subset=['date','shop_id','item_id','item_price'],keep=False ).sum()))
lr_reduce=ReduceLROnPlateau(monitor='val_accuracy',
factor=.25,
patience=3,
min_lr=.000001,
verbose=1)
 es_monitor=EarlyStopping(monitor='val_loss',
patience=5)
 mdl_check = ModelCheckpoint('best_model.h5',
monitor='accuracy',
verbose=0,
save_best_only=True,
mode='max' )<install_modules>
shops['city'] = shops['shop_name'].str.extract('(\S+)\s', expand=False)
 print('number of cities in the dataset : ' + str(shops.city.nunique()))
print('number of null cities : ' + str(shops.city.isnull().sum()))
shops.city.unique()
!pip install efficientnet<import_modules>
shops['city_id']=pd.factorize(shops['city'])[0]
import efficientnet.keras as efn
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D<choose_model_class>
tmp=train_df[['month_id','shop_id']].groupby('shop_id' ).agg({'month_id':['min','max','nunique']})
 tmp[('month_id','laps')]=tmp[('month_id','max')]-tmp[('month_id','min')]+1
tmp=tmp.join(shops[['shop_id','shop_name','city','city_id']].set_index('shop_id'))
tmp.sort_values(by='city_id',inplace=True)
 tmp
net =efn.EfficientNetB3(weights='imagenet', include_top=False, input_shape=(img_size,img_size,3))
x = net.output
x = GlobalAveragePooling2D()(x)
 x = Dense(128, activation="relu" )(x)
 x = Dense(64, activation="relu" )(x)
 predictions = Dense(4, activation= 'softmax' )(x)
 model = Model(inputs = net.input, outputs = predictions )<choose_model_class>
train_df.loc[(train_df['shop_id']==33)&(train_df['month_id']==19)]
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<set_options>
del tmp, tmp_test
gc.collect()
datagen = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
rotation_range=25,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=.1,
rescale=1/255,
fill_mode='nearest',
shear_range=0.1,
brightness_range=[0.5, 1.5] )<train_model>
print(shops.loc[shops['shop_id']==9,'shop_name'].values)
 print('shop 9 is for "out-bound trade", and it is only open on months 9,21 and 33')
 print()
print(shops.loc[shops['shop_id']==20,'shop_name'].values)
 print('shop 20 is for "Moscow sell-out" and is only open on months 21 and 33' )
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=16),
epochs = 14,
steps_per_epoch = X_train.shape[0] // 16,
verbose = 1,
callbacks = [es_monitor,lr_reduce, mdl_check],
validation_data = datagen.flow(X_val, y_val,batch_size=16),
validation_steps = X_val.shape[0] // 16 )<predict_on_test>
print(shops.loc[shops['shop_id']==12,'shop_name'].values)
 print('shop 12 is "emergency online shop"(appears in the test set as well)')
 print()
print(shops.loc[shops['shop_id']==55,'shop_name'].values)
 print('shop 55 is "digital warehouse 1C online"(appears in the test set as well)' )
y_pred = model.predict(X_test )<save_to_csv>
shops.drop(9,axis=0,inplace=True)
 shops.drop(20,axis=0,inplace=True)
 train_df.drop(train_df.loc[train_df['shop_id']==9].index,axis=0,inplace=True)
 train_df.drop(train_df.loc[train_df['shop_id']==20].index,axis=0,inplace=True )
sub_df = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub_df.loc[:, 'healthy':] = y_pred
sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<install_modules>
shops.drop(33,axis=0,inplace=True)
 train_df.drop(train_df.loc[train_df['shop_id']==33].index,axis=0,inplace=True )
!pip install -q efficientnet<import_modules>
train_df.drop(train_df.loc[(train_df['shop_id']==34)&(train_df['month_id']==18)].index,axis=0,inplace=True )
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import keras
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.applications import InceptionResNetV2
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<choose_model_class>
del tmp
gc.collect()
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<define_variables>
item_categories.loc[0,'item_category_name']='Аксессуары - PC(Гарнитуры/Наушники)'
item_categories.loc[8,'item_category_name']='Билеты - Билеты(Цифра)'
item_categories.loc[9,'item_category_name']='Доставка товара - Доставка товара'
item_categories.loc[26,'item_category_name']='Игры - Android(Цифра)'
item_categories.loc[27,'item_category_name']='Игры - MAC(Цифра)'
item_categories.loc[28,'item_category_name']='Игры - PC(Дополнительные издания)'
item_categories.loc[29,'item_category_name']='Игры - PC(Коллекционные издания)'
item_categories.loc[30,'item_category_name']='Игры - PC(Стандартные издания)'
item_categories.loc[31,'item_category_name']='Игры - PC(Цифра)'
item_categories.loc[32,'item_category_name']='Карты оплаты - Кино, Музыка, Игры'
item_categories.loc[79,'item_category_name']='Прием денежных средств для 1С-Онлайн - Прием денежных средств для 1С-Онлайн'
item_categories.loc[80,'item_category_name']='Билеты - Билеты'
item_categories.loc[81,'item_category_name']='Misc - Чистые носители(шпиль)'
item_categories.loc[82,'item_category_name']='Misc - Чистые носители(штучные)'
item_categories.loc[83,'item_category_name']='Misc - Элементы питания'
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
item_categories['item_supercategory_name'] = item_categories['item_category_name'].str.extract('([\S\s]+)\s\-', expand=False)
 item_categories['item_category_is_digital']=(item_categories['item_category_name'].str.find('(Цифра)')>=0)
 consoles=['PS2','PS3','PS4','PSP','PSVita','XBOX 360','XBOX ONE','PC','MAC','Android']
item_categories['item_category_console']=''
for console in consoles:
item_categories['item_category_console']+=item_categories['item_category_name'].str.extract('('+console+')',expand=False ).fillna('')
 item_categories.loc[item_categories['item_category_console']=='','item_category_console']='None'
print('cardinality of the supercategories : ' + str(item_categories['item_supercategory_name'].nunique()))
print('items in the null category : ' + str(item_categories['item_supercategory_name'].isnull().sum()))
print()
print('cardinality of the consoles : ' + str(item_categories['item_category_console'].nunique()))
print('items associated with no consoles : ' + str(item_categories['item_category_console'].isnull().sum()))
print()
print('cardinality of digital items : '+ str(item_categories['item_category_is_digital'].sum()))
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values 
<define_variables>
train_categories=sorted(train_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(train_categories)) + ' categories in the training set :')
 print(train_categories)
 print()
test_categories=sorted(test_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(test_categories)) + ' categories in the test set :')
 print(test_categories)
 print()
print('number of supercategories : '+str(item_categories['item_supercategory_name'].nunique()))
print()
for supercat in item_categories['item_supercategory_name'].unique() :
print(supercat + ' : ' +str(item_categories.loc[item_categories['item_supercategory_name']==supercat,'item_category_id'].nunique()))
print(item_categories.loc[item_categories['item_supercategory_name']==supercat,['item_category_id','item_category_name']])
 print()
item_categories
del train_categories, test_categories
image_size = 850<normalization>
train_df['item_category_id']=train_df['item_id'].map(items['item_category_id'] )
def decode_image(filename, label=None, image_size=(image_size, image_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
print('category 8 : ' + str(item_categories.loc[8,'item_category_name']))
print('category 80 : ' + str(item_categories.loc[80,'item_category_name']))
print('category 81 : ' + str(item_categories.loc[81,'item_category_name']))
print('category 82 : ' + str(item_categories.loc[82,'item_category_name']))
train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==8)].index.values,axis=0,inplace=True)
 train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==80)].index.values,axis=0,inplace=True)
 train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==81)].index.values,axis=0,inplace=True)
 train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==82)].index.values,axis=0,inplace=True)
 item_categories.drop(8,axis=0,inplace=True)
 item_categories.drop(80,axis=0,inplace=True)
 item_categories.drop(81,axis=0,inplace=True)
 item_categories.drop(82,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==8].index.values,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==80].index.values,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==81].index.values,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==82].index.values,axis=0,inplace=True )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_dataset_1 =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(64)
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
item_categories['item_supercategory_id']=item_categories['item_supercategory_name'].map({'Игровые консоли':0,'Игры':1,'Аксессуары':2,'Доставка товара':3,'Прием денежных средств для 1С-Онлайн':4,'Карты оплаты':5,'Кино':6,'Книги':7,'Музыка':8,'Подарки':9,'Программы':10,'Misc':11})
 item_categories['item_category_console_id']=item_categories['item_category_console'].map({console:i for i,console in enumerate(consoles+['None'])} )
LR_START = 0.0001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.0001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<categorify>
original_cols=['item_category_name','item_supercategory_name','item_category_console','item_category_is_digital']
label_cols=['item_category_id','item_supercategory_id','item_category_console_id']
item_categories=item_categories[original_cols+label_cols]
class SoftProbField(Layer):
def __init__(self, **kwargs):
super(SoftProbField, self ).__init__(**kwargs)
 def build(self, input_shape):
super(SoftProbField, self ).build(input_shape)
 def call(self, x):
h = x[:, 0]
s = x[:, 1]
r = x[:, 2]
m = s*r*(1-h)
 s = s*(1-h)*(1-m)
 r = r*(1-h)*(1-m)
 return tf.stack([h, m, r, s], axis=-1)
 def compute_output_shape(self, input_shape):
return(input_shape[0], 4 )<choose_model_class>
for col in original_cols:
train_df[col]=train_df['item_category_id'].map(item_categories[col])
 del original_cols
del label_cols
with strategy.scope() :
model = tf.keras.Sequential([
InceptionResNetV2(
 input_shape=(image_size, image_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(3, activation='sigmoid'),
SoftProbField()
])
 model.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()<train_model>
train_categories=sorted(train_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(train_categories)) + ' categories in the training set :')
 print(train_categories)
 print()
test_categories=sorted(test_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(test_categories)) + ' categories in the test set :')
 print(test_categories)
 print()
print('number of supercategories : '+str(item_categories['item_supercategory_name'].nunique()))
print()
for supercat in item_categories['item_supercategory_name'].unique() :
print(supercat + ' : ' +str(item_categories.loc[item_categories['item_supercategory_name']==supercat,'item_category_id'].nunique()))
print(item_categories.loc[item_categories['item_supercategory_name']==supercat,['item_category_id','item_category_name']])
 print()
print('number of consoles : '+str(item_categories['item_category_console'].nunique()))
print()
for console in item_categories['item_category_console'].unique() :
print(console + ' : ' +str(item_categories.loc[item_categories['item_category_console']==console,'item_category_id'].nunique()))
print(item_categories.loc[item_categories['item_category_console']==console,['item_category_id','item_category_name']])
 print()
item_categories
del train_categories, test_categories
STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
)<choose_model_class>
gc.collect()
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(image_size, image_size, 3),
weights='noisy-student',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(3, activation='sigmoid'),
SoftProbField()
])
 model2.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model2.summary()<train_model>
print('number of realisation with negative prices : '+str(( train_df['item_price']<0 ).sum()))
train_df.drop(train_df['item_price'].idxmin() ,axis=0,inplace=True)
 print('number of realisation with negative prices : '+str(( train_df['item_price']<0 ).sum()))
STEPS_PER_EPOCH = train_labels.shape[0] // 64
history = model2.fit(
 train_dataset_1,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
)<save_to_csv>
gc.collect()
probs1 = model.predict(test_dataset, verbose=1)
 probs2 = model2.predict(test_dataset, verbose=1)
 probs_avg =(probs1+probs2)/2
sub.loc[:, 'healthy':] = probs_avg
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
downcast_dtypes(train_df)
 train_df['month_id']=train_df['month_id'].astype(np.int8)
 train_df['shop_id']=train_df['shop_id'].astype(np.int8)
 downcast_dtypes(shops)
 shops['shop_id']=shops['shop_id'].astype(np.int8)
 downcast_dtypes(items)
 items['item_category_id']=items['item_category_id'].astype(np.int8)
 downcast_dtypes(item_categories)
 item_categories['item_category_id']=item_categories['item_category_id'].astype(np.int8)
 gc.collect()
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
import xgboost as xgb<compute_train_metric>
create_directory(os.path.join(DATA_FOLDER, 'cleaned'))
train_df.to_pickle(os.path.join(DATA_FOLDER, 'cleaned/train.pkl'))
shops.to_pickle(os.path.join(DATA_FOLDER,'cleaned/shops.pkl'))
items.to_pickle(os.path.join(DATA_FOLDER,'cleaned/items.pkl'))
item_categories.to_pickle(os.path.join(DATA_FOLDER,'cleaned/item_categories.pkl'))
def gini(actual, pred, cmpcol = 0, sortcol = 1):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(a, p):
return gini(a, p)/ gini(a, a)
 def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = gini_normalized(labels, preds)
 return [('gini', gini_score)] 
<load_from_csv>
del train_df
del shops
del items
del item_categories
del test_df
gc.collect()
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 target_train = df_train['target'].values
id_test = df_test['id'].values
train = np.array(df_train.drop(['target','id'], axis = 1))
test = np.array(df_test.drop(['id'], axis = 1))
xgb_preds = []<choose_model_class>
reset_variable_space
K = 5
kf = KFold(n_splits = K, random_state = 3228, shuffle = True )<split>
train_df=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/train.pkl'))
shops=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/shops.pkl'))
items=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/items.pkl'))
item_categories=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/item_categories.pkl'))
for train_index, test_index in kf.split(train):
train_X, valid_X = train[train_index], train[test_index]
train_y, valid_y = target_train[train_index], target_train[test_index]
xgb_params = {'eta': 0.02, 'max_depth': 4, 'subsample': 0.9, 'colsample_bytree': 0.9, 'objective': 'binary:logistic', 'eval_metric': 'auc', 'seed': 99, 'silent': True}
d_train = xgb.DMatrix(train_X, train_y)
 d_valid = xgb.DMatrix(valid_X, valid_y)
 d_test = xgb.DMatrix(test)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(xgb_params, d_train, 5000, watchlist, feval=gini_xgb, maximize=True, verbose_eval=50, early_stopping_rounds=100)
 xgb_pred = model.predict(d_test)
 xgb_preds.append(list(xgb_pred))<save_to_csv>
train_df.drop('day_id',axis=1,inplace=True )
preds=[]
for i in range(len(xgb_preds[0])) :
sum=0
for j in range(K):
sum+=xgb_preds[j][i]
preds.append(sum / K)
 output = pd.DataFrame({'id': id_test, 'target': preds})
 output.to_csv("{}-foldCV_avg_sub.csv".format(K), index=False )<set_options>
col_agg = ['month_id','shop_id','item_id']
train_agg=train_df.groupby(col_agg ).agg(item_quantity=pd.NamedAgg(column='item_quantity',aggfunc='sum'))
train_agg.reset_index(inplace=True)
 train_agg['month_id'] = train_agg['month_id'].astype(np.int8)
 train_agg['shop_id'] = train_agg['shop_id'].astype(np.int8)
 train_agg['item_id'] = train_agg['item_id'].astype(np.int16)
 train_agg['item_quantity'].clip(0,20,inplace=True)
 train_agg
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=UserWarning )<define_variables>
test_df=pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'test.csv'))
test_df['ID']=test_df['ID'].astype(np.int32)
 test_df['shop_id']=test_df['shop_id'].astype(np.int8)
 test_df['item_id']=test_df['item_id'].astype(np.int16)
 test_df['month_id']=34
test_df=test_df[col_agg]
print('
'.join([''.join([(' I_Love_Data_Science_'[(x-y)% len('I_Love_Data_Science_')] if(( x*0.05)**2+(y*0.1)**2-1)**3-(x*0.05)**2*(y*0.1)**3 <= 0 else ' ')for x in range(-30, 30)])for y in range(15, -15, -1)]))<load_from_csv>
ts = time.time()
train_X = []
for i in range(0,34):
sales = train_df.loc[train_df['month_id']==i]
train_X.append(np.array(list(product([i], sales['shop_id'].unique() , sales['item_id'].unique())) , dtype='int16'))
train_X = pd.DataFrame(np.vstack(train_X), columns=col_agg)
 train_X = pd.concat([train_X,test_df],ignore_index=True)
 train_X['month_id'] = train_X['month_id'].astype(np.int8)
 train_X['shop_id'] = train_X['shop_id'].astype(np.int8)
 train_X['item_id'] = train_X['item_id'].astype(np.int16)
 train_X.sort_values(by=col_agg,inplace=True)
 print('time : ' +str(time.time() - ts))
print()
print(train_X.info(null_counts=True))
del sales
gc.collect()
def application_train() :
df = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 test_df = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv')
 df = df.append(test_df ).reset_index()
df = df[df['CODE_GENDER'] != 'XNA']
lbe = LabelEncoder()
for col in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[col] = lbe.fit_transform(df[col])
 df = pd.get_dummies(df, dummy_na = True)
 df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace = True)
 df['NEW_DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['NEW_ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
df.drop("index", axis = 1, inplace = True)
 df.columns = pd.Index(["APP_" + col for col in df.columns.tolist() ])
 df.rename(columns={"APP_SK_ID_CURR":"SK_ID_CURR"}, inplace = True)
 df.rename(columns={"APP_TARGET":"TARGET"}, inplace = True)
 return df<load_from_csv>
ts = time.time()
train_X=train_X.join(train_agg.set_index(col_agg),on=col_agg)
 train_X['item_quantity'].fillna(0,inplace=True)
 print('time : ' +str(time.time() - ts))
def bureau_bb() :
bb = pd.read_csv('.. /input/home-credit-default-risk/bureau_balance.csv')
 bb = pd.get_dummies(bb, dummy_na = True)
 agg_list = {"MONTHS_BALANCE":"count",
"STATUS_0":["sum","mean"],
"STATUS_1":["sum"],
"STATUS_2":["sum"],
"STATUS_3":["sum"],
"STATUS_4":["sum"],
"STATUS_5":["sum"],
"STATUS_C":["sum","mean"],
"STATUS_X":["sum","mean"] }
bb_agg = bb.groupby("SK_ID_BUREAU" ).agg(agg_list)
 bb_agg.columns = pd.Index([col[0] + "_" + col[1].upper() for col in bb_agg.columns.tolist() ])
 bb_agg['NEW_STATUS_SCORE'] = bb_agg['STATUS_1_SUM'] + bb_agg['STATUS_2_SUM']^2 + bb_agg['STATUS_3_SUM']^3 + bb_agg['STATUS_4_SUM']^4 + bb_agg['STATUS_5_SUM']^5
bb_agg.drop(['STATUS_1_SUM','STATUS_2_SUM','STATUS_3_SUM','STATUS_4_SUM','STATUS_5_SUM'], axis=1,inplace=True)
 bureau = pd.read_csv('.. /input/home-credit-default-risk/bureau.csv')
 bureau_and_bb = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau_and_bb['CREDIT_TYPE'] = bureau_and_bb['CREDIT_TYPE'].replace(['Car loan',
'Mortgage',
'Microloan',
'Loan for business development',
'Another type of loan',
'Unknown type of loan',
'Loan for working capital replenishment',
"Loan for purchase of shares(margin lending)",
'Cash loan(non-earmarked)',
'Real estate loan',
"Loan for the purchase of equipment",
"Interbank credit",
"Mobile operator loan"], 'Rare')
 bureau_and_bb['CREDIT_ACTIVE'] = bureau_and_bb['CREDIT_ACTIVE'].replace(['Bad debt','Sold'], 'Active')
 bureau_and_bb = pd.get_dummies(bureau_and_bb, columns = ["CREDIT_TYPE","CREDIT_ACTIVE"])
 bureau_and_bb.drop(["SK_ID_BUREAU","CREDIT_CURRENCY"], inplace = True, axis = 1)
 bureau_and_bb["NEW_MONTHS_CREDIT"]= round(( bureau_and_bb.DAYS_CREDIT_ENDDATE - bureau_and_bb.DAYS_CREDIT)/30)
 agg_list = {
"SK_ID_CURR":["count"],
"DAYS_CREDIT":["min","max"],
"CREDIT_DAY_OVERDUE":["sum","mean","max"],
"DAYS_CREDIT_ENDDATE":["max","min"],
"DAYS_ENDDATE_FACT":["max","min"],
"AMT_CREDIT_MAX_OVERDUE":["mean","max","min"],
"CNT_CREDIT_PROLONG":["sum","mean","max","min"],
"AMT_CREDIT_SUM":["mean","max","min"],
"AMT_CREDIT_SUM_DEBT":["sum","mean","max"],
"AMT_CREDIT_SUM_LIMIT":["sum","mean","max"],
'AMT_CREDIT_SUM_OVERDUE':["sum","mean","max"],
'DAYS_CREDIT_UPDATE':["max","min"],
'AMT_ANNUITY':["sum","mean"],
'MONTHS_BALANCE_COUNT':["sum"],
'STATUS_0_SUM':["sum"],
'STATUS_0_MEAN':["mean"],
'STATUS_C_SUM':["sum"],
'STATUS_C_MEAN':["mean"],
'CREDIT_ACTIVE_Active':["sum","mean"],
'CREDIT_ACTIVE_Closed':["sum","mean"],
'CREDIT_TYPE_Rare':["sum","mean"],
'CREDIT_TYPE_Consumer credit':["sum","mean"],
'CREDIT_TYPE_Credit card':["sum","mean"],
"NEW_MONTHS_CREDIT":["count","sum","mean","max","min"]}
bureau_and_bb_agg = bureau_and_bb.groupby("SK_ID_CURR" ).agg(agg_list ).reset_index()
bureau_and_bb_agg.columns = pd.Index(["BB_" + col[0] + "_" + col[1].upper() for col in bureau_and_bb_agg.columns.tolist() ])
 bureau_and_bb_agg["BB_NEW_AMT_CREDIT_SUM_RANGE"] = bureau_and_bb_agg["BB_AMT_CREDIT_SUM_MAX"] - bureau_and_bb_agg["BB_AMT_CREDIT_SUM_MIN"]
bureau_and_bb_agg["BB_NEW_DAYS_CREDIT_RANGE"]= round(( bureau_and_bb_agg["BB_DAYS_CREDIT_MAX"] - bureau_and_bb_agg["BB_DAYS_CREDIT_MIN"])/(30 * bureau_and_bb_agg["BB_SK_ID_CURR_COUNT"]))
agg_list = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean'],
'CNT_CREDIT_PROLONG': ['sum']
}
active = bureau_and_bb[bureau_and_bb['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(agg_list)
 active_agg.columns = pd.Index(['BB_NEW_ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_and_bb_agg.rename(columns = {'BB_SK_ID_CURR_': 'SK_ID_CURR'}, inplace = True)
 bureau_and_bb_agg = bureau_and_bb_agg.join(active_agg, how='left', on='SK_ID_CURR')
 closed = bureau_and_bb[bureau_and_bb['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(agg_list)
 closed_agg.columns = pd.Index(['BB_NEW_CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_and_bb_agg = bureau_and_bb_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 return bureau_and_bb_agg<load_from_csv>
del col_agg
gc.collect()
def installments_payments() :
ins = pd.read_csv('.. /input/home-credit-default-risk/installments_payments.csv')
 ins['NEW_DAYS_PAID_EARLIER'] = ins['DAYS_INSTALMENT']-ins['DAYS_ENTRY_PAYMENT']
ins['NEW_NUM_PAID_LATER'] = ins['NEW_DAYS_PAID_EARLIER'].map(lambda x: 1 if x<0 else 0)
 agg_list = {'NUM_INSTALMENT_VERSION':['nunique'],
'NUM_INSTALMENT_NUMBER':'max',
'DAYS_INSTALMENT':['min','max'],
'DAYS_ENTRY_PAYMENT':['min','max'],
'AMT_INSTALMENT':['min','max','sum','mean'],
'AMT_PAYMENT':['min','max','sum','mean'],
'NEW_DAYS_PAID_EARLIER':'mean',
'NEW_NUM_PAID_LATER':'sum'}
ins_agg = ins.groupby('SK_ID_PREV' ).agg(agg_list)
 ins_agg.columns = pd.Index(["INS_" + e[0] + '_' + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg.drop(['INS_DAYS_INSTALMENT_MIN',
'INS_DAYS_INSTALMENT_MAX',
'INS_DAYS_ENTRY_PAYMENT_MIN',
'INS_DAYS_ENTRY_PAYMENT_MAX'],axis=1,inplace=True)
 ins_agg['INS_NEW_PAYMENT_PERC'] = ins_agg['INS_AMT_PAYMENT_SUM'] / ins_agg['INS_AMT_INSTALMENT_SUM']
ins_agg['INS_NEW_PAYMENT_DIFF'] = ins_agg['INS_AMT_INSTALMENT_SUM'] - ins_agg['INS_AMT_PAYMENT_SUM']
agg_list_previous_application = {}
for col in ins_agg.columns:
agg_list_previous_application[col] = ['mean',"min","max","sum"]
ins_agg.reset_index(inplace = True)
 return agg_list_previous_application, ins_agg<load_from_csv>
train_X['month']=train_X['month_id']%12+1
def pos_cash_balance(agg_list_previous_application):
pos = pd.read_csv('.. /input/home-credit-default-risk/POS_CASH_balance.csv')
 pos = pd.get_dummies(pos, columns=['NAME_CONTRACT_STATUS'], dummy_na = True)
 agg_list = {'MONTHS_BALANCE':['min','max'],
'CNT_INSTALMENT':['min','max'],
'CNT_INSTALMENT_FUTURE':['min','max'],
'SK_DPD':['max','mean'],
'SK_DPD_DEF':['max','mean'],
'NAME_CONTRACT_STATUS_Active':'sum',
'NAME_CONTRACT_STATUS_Amortized debt':'sum',
'NAME_CONTRACT_STATUS_Approved':'sum',
'NAME_CONTRACT_STATUS_Canceled':'sum',
'NAME_CONTRACT_STATUS_Completed':'sum',
'NAME_CONTRACT_STATUS_Demand':'sum',
'NAME_CONTRACT_STATUS_Returned to the store':'sum',
'NAME_CONTRACT_STATUS_Signed':'sum',
'NAME_CONTRACT_STATUS_XNA':'sum',
'NAME_CONTRACT_STATUS_nan':'sum'
}
pos_agg = pos.groupby('SK_ID_PREV' ).agg(agg_list)
 pos_agg.columns= pd.Index(["POS_" + e[0] + '_' + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_NEW_IS_CREDIT_NOT_COMPLETED_ON_TIME']=(pos_agg['POS_CNT_INSTALMENT_FUTURE_MIN']==0)&(pos_agg['POS_NAME_CONTRACT_STATUS_Completed_SUM']==0)
 pos_agg['POS_NEW_IS_CREDIT_NOT_COMPLETED_ON_TIME']=pos_agg['POS_NEW_IS_CREDIT_NOT_COMPLETED_ON_TIME'].astype(int)
 pos_agg.drop(['POS_NAME_CONTRACT_STATUS_Approved_SUM',
'POS_NAME_CONTRACT_STATUS_Amortized debt_SUM',
'POS_NAME_CONTRACT_STATUS_Canceled_SUM',
'POS_NAME_CONTRACT_STATUS_Returned to the store_SUM',
'POS_NAME_CONTRACT_STATUS_Signed_SUM',
'POS_NAME_CONTRACT_STATUS_XNA_SUM',
'POS_NAME_CONTRACT_STATUS_nan_SUM'],axis=1,inplace=True)
 for col in pos_agg.columns:
agg_list_previous_application[col] = ['mean',"min","max","sum"]
pos_agg.reset_index(inplace = True)
 return agg_list_previous_application, pos_agg<load_from_csv>
train_X['shop_months_since_opening']=train_X['month_id']-train_X['shop_id'].map(train_X[['month_id','shop_id']].groupby('shop_id' ).min() ['month_id'])
 train_X['shop_opening']=(train_X['shop_months_since_opening']==0 )
def credit_card_balance() :
CCB = pd.read_csv('.. /input/home-credit-default-risk/credit_card_balance.csv')
 CCB = pd.get_dummies(CCB, columns= ['NAME_CONTRACT_STATUS'])
 dropthis = ['NAME_CONTRACT_STATUS_Approved', 'NAME_CONTRACT_STATUS_Demand',
'NAME_CONTRACT_STATUS_Refused', 'NAME_CONTRACT_STATUS_Sent proposal',
'NAME_CONTRACT_STATUS_Signed' ]
CCB = CCB.drop(dropthis, axis=1)
 grp = CCB.groupby(by = ['SK_ID_CURR'])['SK_ID_PREV'].nunique().reset_index().rename(index = str, columns = {'SK_ID_PREV': 'NUMBER_OF_LOANS_PER_CUSTOMER'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 grp = CCB.groupby(by = ['SK_ID_CURR', 'SK_ID_PREV'])['CNT_INSTALMENT_MATURE_CUM'].max().reset_index().rename(index = str, columns = {'CNT_INSTALMENT_MATURE_CUM': 'NUMBER_OF_INSTALMENTS'})
 grp1 = grp.groupby(by = ['SK_ID_CURR'])['NUMBER_OF_INSTALMENTS'].sum().reset_index().rename(index = str, columns = {'NUMBER_OF_INSTALMENTS': 'TOTAL_INSTALMENTS_OF_ALL_LOANS'})
 CCB = CCB.merge(grp1, on = ['SK_ID_CURR'], how = 'left')
 CCB['INSTALLMENTS_PER_LOAN'] =(CCB['TOTAL_INSTALMENTS_OF_ALL_LOANS']/CCB['NUMBER_OF_LOANS_PER_CUSTOMER'] ).astype('uint32')
 def geciken_gun_hesapla(DPD):
x = DPD.tolist()
c = 0
for i,j in enumerate(x):
if j != 0:
c += 1
return c
grp = CCB.groupby(by = ['SK_ID_CURR', 'SK_ID_PREV'] ).apply(lambda x: geciken_gun_hesapla(x.SK_DPD)).reset_index().rename(index = str, columns = {0: 'NUMBER_OF_DPD'})
 grp1 = grp.groupby(by = ['SK_ID_CURR'])['NUMBER_OF_DPD'].mean().reset_index().rename(index = str, columns = {'NUMBER_OF_DPD' : 'DPD_COUNT'})
 CCB = CCB.merge(grp1, on = ['SK_ID_CURR'], how = 'left')
 def f(min_pay, total_pay):
M = min_pay.tolist()
T = total_pay.tolist()
P = len(M)
 c = 0
for i in range(len(M)) :
if T[i] < M[i]:
c += 1
return(100*c)/P
grp = CCB.groupby(by = ['SK_ID_CURR'] ).apply(lambda x: f(x.AMT_INST_MIN_REGULARITY, x.AMT_PAYMENT_CURRENT)).reset_index().rename(index = str, columns = { 0 : 'PERCENTAGE_MIN_MISSED_PAYMENTS'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 grp = CCB.groupby(by = ['SK_ID_CURR'])['AMT_DRAWINGS_ATM_CURRENT'].sum().reset_index().rename(index = str, columns = {'AMT_DRAWINGS_ATM_CURRENT' : 'DRAWINGS_ATM'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 grp = CCB.groupby(by = ['SK_ID_CURR'])['AMT_DRAWINGS_CURRENT'].sum().reset_index().rename(index = str, columns = {'AMT_DRAWINGS_CURRENT' : 'DRAWINGS_TOTAL'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 CCB['CASH_CARD_RATIO1'] =(CCB['DRAWINGS_ATM']/CCB['DRAWINGS_TOTAL'])*100
del CCB['DRAWINGS_ATM']
del CCB['DRAWINGS_TOTAL']
grp = CCB.groupby(by = ['SK_ID_CURR'])['CASH_CARD_RATIO1'].mean().reset_index().rename(index = str, columns ={ 'CASH_CARD_RATIO1' : 'CASH_CARD_RATIO'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 grp = CCB.groupby(by = ['SK_ID_CURR'])['AMT_DRAWINGS_CURRENT'].sum().reset_index().rename(index = str, columns = {'AMT_DRAWINGS_CURRENT' : 'TOTAL_DRAWINGS'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 grp = CCB.groupby(by = ['SK_ID_CURR'])['CNT_DRAWINGS_CURRENT'].sum().reset_index().rename(index = str, columns = {'CNT_DRAWINGS_CURRENT' : 'NUMBER_OF_DRAWINGS'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 CCB['DRAWINGS_RATIO1'] =(CCB['TOTAL_DRAWINGS']/CCB['NUMBER_OF_DRAWINGS'])*100
del CCB['TOTAL_DRAWINGS']
del CCB['NUMBER_OF_DRAWINGS']
grp = CCB.groupby(by = ['SK_ID_CURR'])['DRAWINGS_RATIO1'].mean().reset_index().rename(index = str, columns ={ 'DRAWINGS_RATIO1' : 'DRAWINGS_RATIO'})
 CCB = CCB.merge(grp, on = ['SK_ID_CURR'], how = 'left')
 del CCB['DRAWINGS_RATIO1']
CCB['CC_COUNT'] = CCB.groupby('SK_ID_CURR' ).size()
CCB_agg = CCB.groupby('SK_ID_CURR' ).agg({
'MONTHS_BALANCE':["sum","mean"],
'AMT_BALANCE':["sum","mean","min","max"],
'AMT_CREDIT_LIMIT_ACTUAL':["sum","mean"],
'AMT_DRAWINGS_ATM_CURRENT':["sum","mean","min","max"],
'AMT_DRAWINGS_CURRENT':["sum","mean","min","max"],
'AMT_DRAWINGS_OTHER_CURRENT':["sum","mean","min","max"],
'AMT_DRAWINGS_POS_CURRENT':["sum","mean","min","max"],
'AMT_INST_MIN_REGULARITY':["sum","mean","min","max"],
'AMT_PAYMENT_CURRENT':["sum","mean","min","max"],
'AMT_PAYMENT_TOTAL_CURRENT':["sum","mean","min","max"],
'AMT_RECEIVABLE_PRINCIPAL':["sum","mean","min","max"],
'AMT_RECIVABLE':["sum","mean","min","max"],
'AMT_TOTAL_RECEIVABLE':["sum","mean","min","max"],
'CNT_DRAWINGS_ATM_CURRENT':["sum","mean"],
'CNT_DRAWINGS_CURRENT':["sum","mean","max"],
'CNT_DRAWINGS_OTHER_CURRENT':["mean","max"],
'CNT_DRAWINGS_POS_CURRENT':["sum","mean","max"],
'CNT_INSTALMENT_MATURE_CUM':["sum","mean","max","min"],
'SK_DPD':["sum","mean","max"],
'SK_DPD_DEF':["sum","mean","max"],
'NAME_CONTRACT_STATUS_Active':["sum","mean","min","max"],
'INSTALLMENTS_PER_LOAN':["sum","mean","min","max"],
'NUMBER_OF_LOANS_PER_CUSTOMER':["mean"],
'DPD_COUNT':["mean"],
'PERCENTAGE_MIN_MISSED_PAYMENTS':["mean"],
'CASH_CARD_RATIO':["mean"],
'DRAWINGS_RATIO':["mean"]})
 CCB_agg.columns = pd.Index(['CCB_' + e[0] + "_" + e[1].upper() for e in CCB_agg.columns.tolist() ])
 CCB_agg.reset_index(inplace = True)
 return CCB_agg<load_from_csv>
train_X['item_month_id_of_release']=train_X['item_id'].map(train_X[['month_id','item_id']].groupby('item_id' ).min() ['month_id'])
 train_X['item_month_of_release']=train_X['item_month_id_of_release']%12+1
train_X['item_months_since_release']=train_X['month_id']-train_X['item_month_id_of_release']
train_X['item_months_since_release'].clip(0,12,inplace=True)
 train_X['item_new']=(train_X['item_months_since_release']==0 )
def previous_application(agg_list_previous_application):
df_prev = pd.read_csv('.. /input/home-credit-default-risk/previous_application.csv')
 df_prev["WEEKDAY_APPR_PROCESS_START"] = df_prev["WEEKDAY_APPR_PROCESS_START"].replace(['MONDAY','TUESDAY', 'WEDNESDAY','THURSDAY','FRIDAY'], 'WEEK_DAY')
 df_prev["WEEKDAY_APPR_PROCESS_START"] = df_prev["WEEKDAY_APPR_PROCESS_START"].replace(['SATURDAY', 'SUNDAY'], 'WEEKEND')
 a = [8,9,10,11,12,13,14,15,16,17]
df_prev["HOUR_APPR_PROCESS_START"] = df_prev["HOUR_APPR_PROCESS_START"].replace(a, 'working_hours')
 b = [18,19,20,21,22,23,0,1,2,3,4,5,6,7]
df_prev["HOUR_APPR_PROCESS_START"] = df_prev["HOUR_APPR_PROCESS_START"].replace(b, 'off_hours')
 df_prev["DAYS_DECISION"] = [1 if abs(i/(12*30)) <=1 else 0 for i in df_prev.DAYS_DECISION]
df_prev["NAME_TYPE_SUITE"] = df_prev["NAME_TYPE_SUITE"].replace('Unaccompanied', 'alone')
 b = ['Family', 'Spouse, partner', 'Children', 'Other_B', 'Other_A', 'Group of people']
df_prev["NAME_TYPE_SUITE"] = df_prev["NAME_TYPE_SUITE"].replace(b, 'not_alone')
 a = ['Auto Accessories', 'Jewelry', 'Homewares', 'Medical Supplies', 'Vehicles', 'Sport and Leisure',
'Gardening', 'Other', 'Office Appliances', 'Tourism', 'Medicine', 'Direct Sales', 'Fitness', 'Additional Service',
'Education', 'Weapon', 'Insurance', 'House Construction', 'Animals']
df_prev["NAME_GOODS_CATEGORY"] = df_prev["NAME_GOODS_CATEGORY"].replace(a, 'others')
 a = ['Auto technology', 'Jewelry', 'MLM partners', 'Tourism']
df_prev["NAME_SELLER_INDUSTRY"] = df_prev["NAME_SELLER_INDUSTRY"].replace(a, 'others')
 df_prev["LOAN_RATE"] = df_prev.AMT_APPLICATION/df_prev.AMT_CREDIT
df_prev["NEW_LOAN_RATE"] = df_prev.AMT_APPLICATION/df_prev.AMT_CREDIT
k = df_prev.DAYS_LAST_DUE_1ST_VERSION - df_prev.DAYS_LAST_DUE
df_prev["NEW_CHURN_PREV"] = [1 if i >= 0 else(0 if i < 0 else "NaN")for i in k]
df_prev[(df_prev['AMT_CREDIT'] == 0)|(df_prev['AMT_GOODS_PRICE'] == 0)]['NEW_INSURANCE'] = np.nan
df_prev['sigorta_miktari'] = df_prev['AMT_CREDIT'] - df_prev['AMT_GOODS_PRICE']
df_prev["NEW_INSURANCE"] = df_prev['sigorta_miktari'].apply(lambda x: 1 if x > 0 else(0 if x <= 0 else np.nan))
df_prev.drop('sigorta_miktari', axis=1, inplace=True)
 drop_list = ['AMT_DOWN_PAYMENT', 'SELLERPLACE_AREA', 'CNT_PAYMENT', 'PRODUCT_COMBINATION', 'DAYS_FIRST_DRAWING', 'DAYS_FIRST_DUE',
'DAYS_LAST_DUE_1ST_VERSION', 'DAYS_LAST_DUE','DAYS_TERMINATION','NFLAG_INSURED_ON_APPROVAL']
df_prev.drop(drop_list, axis = 1, inplace = True)
 category_columns=[]
for i in df_prev.columns:
if df_prev[i].dtypes == "O":
category_columns.append(i)
 df_prev = pd.get_dummies(df_prev, columns = category_columns)
 prev_agg_list = {"SK_ID_CURR":["count"],
"AMT_ANNUITY":["max"],
"AMT_APPLICATION":["min","mean","max"],
"AMT_CREDIT":["max"],
"AMT_GOODS_PRICE":["sum", "mean"],
"NFLAG_LAST_APPL_IN_DAY":["sum","mean"],
"RATE_DOWN_PAYMENT":["sum", "mean"],
"RATE_INTEREST_PRIMARY":["sum", "mean"],
"RATE_INTEREST_PRIVILEGED":["sum", "mean"],
"DAYS_DECISION":["sum"],
"NEW_LOAN_RATE":["sum", "mean", "min", "max"],
"NEW_INSURANCE":["sum", "mean"],
"NAME_CONTRACT_TYPE_Cash loans":["sum", "mean"],
"NAME_CONTRACT_TYPE_Consumer loans":["sum", "mean"],
"NAME_CONTRACT_TYPE_Revolving loans":["sum", "mean"],
"NAME_CONTRACT_TYPE_XNA":["sum", "mean"],
"WEEKDAY_APPR_PROCESS_START_WEEKEND":["sum", "mean"],
"WEEKDAY_APPR_PROCESS_START_WEEK_DAY":["sum", "mean"],
"HOUR_APPR_PROCESS_START_off_hours":["sum", "mean"],
"HOUR_APPR_PROCESS_START_working_hours":["sum", "mean"],
"FLAG_LAST_APPL_PER_CONTRACT_N":["sum", "mean"],
"FLAG_LAST_APPL_PER_CONTRACT_Y":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Building a house or an annex":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Business development":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Buying a garage":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Buying a holiday home / land":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Buying a home":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Buying a new car":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Buying a used car":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Car repairs":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Education":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Everyday expenses":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Furniture":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Gasification / water supply":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Hobby":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Journey":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Medicine":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Money for a third person":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Other":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Payments on other loans":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Purchase of electronic equipment":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Refusal to name the goal":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Repairs":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Urgent needs":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_Wedding / gift / holiday":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_XAP":["sum", "mean"],
"NAME_CASH_LOAN_PURPOSE_XNA":["sum", "mean"],
"NAME_CONTRACT_STATUS_Approved":["sum", "mean"],
"NAME_CONTRACT_STATUS_Canceled":["sum", "mean"],
"NAME_CONTRACT_STATUS_Refused":["sum", "mean"],
"NAME_CONTRACT_STATUS_Unused offer":["sum", "mean"],
"NAME_PAYMENT_TYPE_Cash through the bank":["sum", "mean"],
"NAME_PAYMENT_TYPE_Cashless from the account of the employer":["sum", "mean"],
"NAME_PAYMENT_TYPE_Non-cash from your account":["sum", "mean"],
"NAME_PAYMENT_TYPE_XNA":["sum", "mean"],
"CODE_REJECT_REASON_CLIENT":["sum", "mean"],
"CODE_REJECT_REASON_HC":["sum", "mean"],
"CODE_REJECT_REASON_LIMIT":["sum", "mean"],
"CODE_REJECT_REASON_SCO":["sum", "mean"],
"CODE_REJECT_REASON_SCOFR":["sum", "mean"],
"CODE_REJECT_REASON_SYSTEM":["sum", "mean"],
"CODE_REJECT_REASON_VERIF":["sum", "mean"],
"CODE_REJECT_REASON_XAP":["sum", "mean"],
"CODE_REJECT_REASON_XNA":["sum", "mean"],
"NAME_TYPE_SUITE_alone":["sum", "mean"],
"NAME_TYPE_SUITE_not_alone":["sum", "mean"],
"NAME_CLIENT_TYPE_New":["sum", "mean"],
"NAME_CLIENT_TYPE_Refreshed":["sum", "mean"],
"NAME_CLIENT_TYPE_Repeater":["sum", "mean"],
"NAME_CLIENT_TYPE_XNA":["sum", "mean"],
"NAME_GOODS_CATEGORY_Audio/Video":["sum", "mean"],
"NAME_GOODS_CATEGORY_Clothing and Accessories":["sum", "mean"],
"NAME_GOODS_CATEGORY_Computers":["sum", "mean"],
"NAME_GOODS_CATEGORY_Construction Materials":["sum", "mean"],
"NAME_GOODS_CATEGORY_Consumer Electronics":["sum", "mean"],
"NAME_GOODS_CATEGORY_Furniture":["sum", "mean"],
"NAME_GOODS_CATEGORY_Mobile":["sum", "mean"],
"NAME_GOODS_CATEGORY_Photo / Cinema Equipment":["sum", "mean"],
"NAME_GOODS_CATEGORY_XNA":["sum", "mean"],
"NAME_GOODS_CATEGORY_others":["sum", "mean"],
"NAME_PORTFOLIO_Cards":["sum", "mean"],
"NAME_PORTFOLIO_Cars":["sum", "mean"],
"NAME_PORTFOLIO_Cash":["sum", "mean"],
"NAME_PORTFOLIO_POS":["sum", "mean"],
"NAME_PORTFOLIO_XNA":["sum", "mean"],
"NAME_PRODUCT_TYPE_XNA":["sum", "mean"],
"NAME_PRODUCT_TYPE_walk-in":["sum", "mean"],
"NAME_PRODUCT_TYPE_x-sell":["sum", "mean"],
"CHANNEL_TYPE_AP+(Cash loan)":["sum", "mean"],
"CHANNEL_TYPE_Car dealer":["sum", "mean"],
"CHANNEL_TYPE_Channel of corporate sales":["sum", "mean"],
"CHANNEL_TYPE_Contact center":["sum", "mean"],
"CHANNEL_TYPE_Country-wide":["sum", "mean"],
"CHANNEL_TYPE_Credit and cash offices":["sum", "mean"],
"CHANNEL_TYPE_Regional / Local":["sum", "mean"],
"CHANNEL_TYPE_Stone":["sum", "mean"],
"NAME_SELLER_INDUSTRY_Clothing":["sum", "mean"],
"NAME_SELLER_INDUSTRY_Connectivity":["sum", "mean"],
"NAME_SELLER_INDUSTRY_Construction":["sum", "mean"],
"NAME_SELLER_INDUSTRY_Consumer electronics":["sum", "mean"],
"NAME_SELLER_INDUSTRY_Furniture":["sum", "mean"],
"NAME_SELLER_INDUSTRY_Industry":["sum", "mean"],
"NAME_SELLER_INDUSTRY_XNA":["sum", "mean"],
"NAME_SELLER_INDUSTRY_others":["sum", "mean"],
"NAME_YIELD_GROUP_XNA":["sum", "mean"],
"NAME_YIELD_GROUP_high":["sum", "mean"],
"NAME_YIELD_GROUP_low_action":["sum", "mean"],
"NAME_YIELD_GROUP_low_normal":["sum", "mean"],
"NAME_YIELD_GROUP_middle":["sum", "mean"],
"NEW_CHURN_PREV_0":["sum", "mean"],
"NEW_CHURN_PREV_1":["sum", "mean"],
"NEW_CHURN_PREV_NaN":["sum", "mean"]}
prev_agg_list.update(agg_list_previous_application)
 return prev_agg_list, df_prev<train_model>
train_X=train_X.join(train_agg[['shop_id','month_id','item_id']].groupby(['shop_id','item_id'] ).min().rename({'month_id':'item_month_id_of_first_sale_in_shop'},axis=1),on=['shop_id','item_id'])
 train_X['item_month_of_first_sale_in_shop']=train_X['item_month_id_of_first_sale_in_shop']%12+1
train_X['item_months_since_first_sale_in_shop']=(train_X['month_id']-train_X['item_month_id_of_first_sale_in_shop'])
 train_X['item_months_since_first_sale_in_shop'].clip(0,12,inplace=True)
 train_X['item_never_sold_in_shop_before']=~(train_X['item_months_since_first_sale_in_shop']>0)
 train_X.loc[train_X['item_never_sold_in_shop_before'],'item_months_since_first_sale_in_shop']=-1
train_X.loc[train_X['item_never_sold_in_shop_before'],'item_month_id_of_first_sale_in_shop']=-1
train_X.loc[train_X['item_never_sold_in_shop_before'],'item_month_of_first_sale_in_shop']=-1
train_X['item_months_since_first_sale_in_shop']=train_X['item_months_since_first_sale_in_shop'].astype(np.int8)
 train_X['item_month_id_of_first_sale_in_shop']=train_X['item_month_id_of_first_sale_in_shop'].astype(np.int8)
 train_X['item_month_of_first_sale_in_shop']=train_X['item_month_of_first_sale_in_shop'].astype(np.int8 )
def pre_processing_and_combine() :
with timer("Process application train"):
df = application_train()
print("application train & test shape:", df.shape)
 with timer("Bureau and Bureau Balance"):
bureau_and_bb_agg = bureau_bb()
print("Bureau and Bureau Balance:", bureau_and_bb_agg.shape)
 with timer("Installment Payments"):
agg_list_previous_application, ins_agg = installments_payments()
print("Installment Payments:", ins_agg.shape)
 with timer("Pos Cash Balance"):
agg_list_previous_application, pos_agg = pos_cash_balance(agg_list_previous_application)
 print("Pos Cash Balance:", pos_agg.shape)
 with timer("Credit Card Balance"):
CCB_agg = credit_card_balance()
print("Credit Card Balance:", CCB_agg.shape)
 with timer("previous_application"):
prev_agg_list, df_prev = previous_application(agg_list_previous_application)
 print("previous_application:", df_prev.shape)
 with timer("All tables are combining"):
df_prev_ins = df_prev.merge(ins_agg, how = 'left', on = 'SK_ID_PREV')
 df_prev_ins_pos = df_prev_ins.merge(pos_agg, how = 'left', on = 'SK_ID_PREV')
 df_prev_ins_pos_agg = df_prev_ins_pos.groupby("SK_ID_CURR" ).agg(prev_agg_list ).reset_index()
df_prev_ins_pos_agg.columns = pd.Index(["PREV_" + col[0] + "_" + col[1].upper() for col in df_prev_ins_pos_agg.columns.tolist() ])
 df_prev_ins_pos_agg.rename(columns={"PREV_SK_ID_CURR_":"SK_ID_CURR"}, inplace = True)
 df_prev_others = df.merge(df_prev_ins_pos_agg, how = 'left',on = 'SK_ID_CURR')
 df_prev_ins_pos_ccb = df_prev_others.merge(CCB_agg, how = 'left',on = 'SK_ID_CURR')
 all_data = df_prev_ins_pos_ccb.merge(bureau_and_bb_agg, how = 'left',on = 'SK_ID_CURR')
 print("all_data process:", all_data.shape)
 return all_data 
<train_on_grid>
train_X['item_never_sold_in_shop_before_but_not_new']=(( 1-train_X['item_new'])*train_X['item_never_sold_in_shop_before'] ).astype(bool )
<choose_model_class>
train_X['item_seniority']=(2-train_X['item_new'].astype(int)-train_X['item_never_sold_in_shop_before'].astype(int)).astype(np.int8 )
def modeling(all_data):
all_data.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in all_data.columns]
train_df = all_data[all_data['TARGET'].notnull() ]
test_df = all_data[all_data['TARGET'].isnull() ]
folds = KFold(n_splits = 10, shuffle = True, random_state = 1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 n_jobs = -1,
n_estimators=10000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1,)
 clf.fit(train_x, train_y, eval_set = [(train_x, train_y),(valid_x, valid_y)],
eval_metric = 'auc', verbose = 200, early_stopping_rounds = 200)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv("submission_lightgbm.csv", index= False)
 display_importances(feature_importance_df)
 return feature_importance_df 
<train_model>
tmp_list=[]
for mid in train_X['month_id'].unique() :
tmp=train_agg.loc[train_agg['month_id']<mid,['month_id','shop_id','item_id']].groupby(['shop_id','item_id'] ).last().rename({'month_id':'item_month_id_of_last_sale_in_shop'},axis=1 ).astype(np.int16)
 tmp['month_id']=mid
tmp.reset_index(inplace=True)
 tmp_list.append(tmp)
 tmp=pd.concat(tmp_list)
 del tmp_list
train_X=train_X.join(tmp.set_index(['month_id','shop_id','item_id']),on=['month_id','shop_id','item_id'])
 del tmp
train_X['item_month_id_of_last_sale_in_shop']=train_X['item_month_id_of_last_sale_in_shop'].astype(np.float16)
 train_X['item_months_since_last_sale_in_shop']=train_X['month_id']-train_X['item_month_id_of_last_sale_in_shop']
def main() :
with timer("Preprocessing Time"):
all_data = pre_processing_and_combine()
with timer("Modeling"):
feat_importance = modeling(all_data)
<train_model>
tmp_list=[]
for mid in train_X['month_id'].unique() :
tmp=train_agg.loc[train_agg['month_id']<mid,['month_id','item_id']].groupby('item_id' ).last().rename({'month_id':'item_month_id_of_last_sale'},axis=1)
 tmp['month_id']=mid
tmp.reset_index(inplace=True)
 tmp_list.append(tmp)
 tmp=pd.concat(tmp_list)
 del tmp_list
train_X=train_X.join(tmp.set_index(['month_id','item_id']),on=['month_id','item_id'])
 del tmp
train_X['item_month_id_of_last_sale']=train_X['item_month_id_of_last_sale'].astype(np.float16)
 train_X['item_months_since_last_sale']=train_X['month_id']-train_X['item_month_id_of_last_sale']
if __name__ == "__main__":
with timer("Full model run"):
main()<set_options>
train_X['item_sold_in_shop']=(train_X['item_quantity']>0 )
%matplotlib inline<load_from_disk>
ts=time.time()
train_X=train_X.join(shops.set_index('shop_id'),on='shop_id')
 train_X=train_X.join(items.set_index('item_id'),on='item_id')
 train_X=train_X.join(item_categories.set_index('item_category_id'),on='item_category_id')
 print('time : ' +str(time.time() - ts))
gc.collect()
train = pd.read_json(".. /input/train.json" )<data_type_conversions>
ts=time.time()
train_X=train_X.join(train_X.loc[:,['month_id','item_category_id']].groupby('month_id')['item_category_id'].value_counts(normalize=True ).rename('item_category_freq',axis=1 ).astype(np.float32),on=['month_id','item_category_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_supercategory_id']].groupby('month_id')['item_supercategory_id'].value_counts(normalize=True ).rename('item_supercategory_freq',axis=1 ).astype(np.float32),on=['month_id','item_supercategory_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_category_console_id']].groupby('month_id')['item_category_console_id'].value_counts(normalize=True ).rename('item_category_console_freq',axis=1 ).astype(np.float32),on=['month_id','item_category_console_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_category_is_digital']].groupby('month_id')['item_category_is_digital'].value_counts(normalize=True ).rename('item_category_digital_freq',axis=1 ).astype(np.float32),on=['month_id','item_category_is_digital'])
 print('time : ' +str(time.time() - ts))
gc.collect()
def standardise_data(data):

 out =(np.array(data)- np.mean(data)) / np.std(data)
 return out.tolist()
def process_data(data, predict=False):
data["band_1"] = data["band_1"].apply(standardise_data)
 data["band_2"] = data["band_2"].apply(standardise_data)
 band1 = np.array([np.array(band ).astype(np.float32 ).reshape(75,75)for band in data["band_1"]])
 band2 = np.array([np.array(band ).astype(np.float32 ).reshape(75,75)for band in data["band_2"]])
 X = np.concatenate([band1[:, :, :, np.newaxis],
band2[:, :, :, np.newaxis], 
(( band1+band2)/2)[:, :, :, np.newaxis]],
axis=-1)
 if predict==False:
y = np.array(data['is_iceberg'])
 return X, y
return X<prepare_x_and_y>
ts=time.time()
train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_id']].groupby(['month_id','item_seniority'])['item_id'].value_counts(normalize=True ).rename('item_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','shop_id']].groupby(['month_id','item_seniority'])['shop_id'].value_counts(normalize=True ).rename('shop_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','shop_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_category_id']].groupby(['month_id','item_seniority'])['item_category_id'].value_counts(normalize=True ).rename('item_category_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_category_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_supercategory_id']].groupby(['month_id','item_seniority'])['item_supercategory_id'].value_counts(normalize=True ).rename('item_supercategory_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_supercategory_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_category_console_id']].groupby(['month_id','item_seniority'])['item_category_console_id'].value_counts(normalize=True ).rename('item_category_console_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_category_console_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_category_is_digital']].groupby(['month_id','item_seniority'])['item_category_is_digital'].value_counts(normalize=True ).rename('item_category_digital_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_category_is_digital'])
 print('time : ' +str(time.time() - ts))
gc.collect()
X_train, y_train = process_data(train )<import_modules>
train_df_expand=train_df.set_index(['month_id','shop_id','item_id'])['item_price']
train_df_expand=pd.DataFrame(train_df_expand.repeat(train_df['item_quantity']))
train_df_expand.reset_index(inplace=True)
 train_df_expand=pd.concat([train_df_expand,test_df],sort=False)
 train_df_expand['item_category_id']=train_df_expand['item_id'].map(items['item_category_id'])
 train_df_expand['item_supercategory_id']=train_df_expand['item_category_id'].map(item_categories['item_supercategory_id'])
 train_df_expand['item_category_console_id']=train_df_expand['item_category_id'].map(item_categories['item_category_console_id'] )
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten, Activation, BatchNormalization
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping<choose_model_class>
def encode_prices(groupby_labels,name):
df_prices=train_df_expand.groupby(groupby_labels ).agg({'item_price':['min','max','median','mean']})
 df_prices.columns=[name+'_price_'+col for col in df_prices.columns.get_level_values(1)]
df_prices.reset_index(inplace=True)
 df_prices=downcast_dtypes(df_prices)
 return df_prices
def compare_to_super(df,df_super,join_labels,label,label_super,name,name_super,df_mapping):
col_list=['price_min','price_max','price_median','price_mean']
df[label_super]=df[label].map(df_mapping[label_super])
 df=df.join(df_super[join_labels+[label_super]+[name_super+'_'+col for col in col_list]].set_index(join_labels+[label_super]),on=join_labels+[label_super])
 df.drop(label_super,axis=1,inplace=True)
 for col in col_list:
df[name_super+'_'+col]=df[name+'_'+col]/df[name_super+'_'+col]
df.rename({name_super+'_'+col:name+'_'+col+'_compared_to_'+name_super+'_'+col},axis=1,inplace=True)
 df=downcast_dtypes(df)
 return df
def compare_to_lower(df,df_lower,label_lower,name,name_lower):
col_list=['price_min','price_max','price_median','price_mean']
df=df.join(df_lower[['month_id',label_lower]+[name_lower+'_'+col for col in col_list]].set_index(['month_id',label_lower]),on=['month_id',label_lower])
 for col in col_list:
df[name_lower+'_'+col]=df[name+'_'+col]/df[name_lower+'_'+col]
df.rename({name_lower+'_'+col:name+'_'+col+'_compared_to_'+name_lower+'_'+col},axis=1,inplace=True)
 df=downcast_dtypes(df)
 return df
def getModel() :
drop = 0.2
gmodel=Sequential()
gmodel.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(75, 75, 3)))
 gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))
 gmodel.add(Dropout(drop))
gmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 gmodel.add(Dropout(drop))
gmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
gmodel.add(Dropout(drop))
gmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
gmodel.add(Dropout(drop))
gmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 gmodel.add(Dropout(drop))
gmodel.add(Flatten())
 gmodel.add(Dense(512))
gmodel.add(BatchNormalization())
 gmodel.add(Activation('relu'))
gmodel.add(Dropout(drop))
gmodel.add(Dense(256))
gmodel.add(BatchNormalization())
 gmodel.add(Activation('relu'))
gmodel.add(Dropout(drop))
gmodel.add(Dense(1))
gmodel.add(Activation('sigmoid'))
mypotim=Adam(lr=0.0008, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0, amsgrad=True)
 gmodel.compile(loss='binary_crossentropy',
optimizer=mypotim,
metrics=['accuracy'])
 gmodel.summary()
return gmodel
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
file_path = "model_weights3.hdf5"
callbacks = get_callbacks(filepath=file_path, patience=5)
<split>
supercategory_prices=encode_prices(['month_id','item_supercategory_id'],'supercategory')
 supercategory_prices['month_id'] = supercategory_prices['month_id'].astype(np.int8)
 print(supercategory_prices.loc[supercategory_prices['month_id']<34,:].info(null_counts=True))
supercategory_prices
X_train_cv, X_valid, y_train_cv, y_valid = train_test_split(X_train, y_train, random_state=1, test_size=0.25 )<train_model>
category_prices=encode_prices(['month_id','item_category_id'],'category')
 category_prices=compare_to_super(category_prices,supercategory_prices,['month_id'],'item_category_id','item_supercategory_id','category','supercategory',item_categories)
 category_prices['month_id'] = category_prices['month_id'].astype(np.int8)
 print(category_prices.loc[category_prices['month_id']<34,:].info(null_counts=True))
category_prices
gen = ImageDataGenerator(rotation_range=10, width_shift_range=0.3, height_shift_range=0.3,
zoom_range=0.2, horizontal_flip=True, vertical_flip=True)
 gen.fit(X_train_cv )<train_model>
item_prices=encode_prices(['month_id','item_id'],'item')
 item_prices=compare_to_super(item_prices,category_prices,['month_id'],'item_id','item_category_id','item','category',items)
 item_prices['month_id'] = item_prices['month_id'].astype(np.int8)
 print(item_prices.loc[item_prices['month_id']<34,:].info(null_counts=True))
item_prices
gmodel=getModel()
history = gmodel.fit_generator(gen.flow(X_train_cv, y_train_cv, batch_size=32),steps_per_epoch=256,
epochs=50, verbose=1, validation_data=(X_valid, y_valid), callbacks=callbacks )<compute_test_metric>
shop_supercategory_prices=encode_prices(['month_id','shop_id','item_supercategory_id'],'shop_supercategory')
 shop_supercategory_prices=compare_to_lower(shop_supercategory_prices,supercategory_prices,'item_supercategory_id','shop_supercategory','supercategory')
 shop_supercategory_prices['month_id'] = shop_supercategory_prices['month_id'].astype(np.int8)
 shop_supercategory_prices['shop_id'] = shop_supercategory_prices['shop_id'].astype(np.int8)
 print(shop_supercategory_prices.loc[shop_supercategory_prices['month_id']<34,:].info(null_counts=True))
shop_supercategory_prices
gmodel.load_weights(filepath=file_path)
 score = gmodel.evaluate(X_valid, y_valid, verbose=1)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )<predict_on_test>
shop_category_prices=encode_prices(['month_id','shop_id','item_category_id'],'shop_category')
 shop_category_prices=compare_to_lower(shop_category_prices,category_prices,'item_category_id','shop_category','category')
 shop_category_prices=compare_to_super(shop_category_prices,shop_supercategory_prices,['month_id','shop_id'],'item_category_id','item_supercategory_id','shop_category','shop_supercategory',item_categories)
 shop_category_prices['month_id'] = shop_category_prices['month_id'].astype(np.int8)
 shop_category_prices['shop_id'] = shop_category_prices['shop_id'].astype(np.int8)
 print(shop_category_prices.loc[shop_category_prices['month_id']<34,:].info(null_counts=True))
shop_category_prices
pred = np.round(gmodel.predict_proba(X_valid[0:3,:,:,:]))
for i in range(3):
print('real:', y_valid[i], 'predicted:', pred[i])
 plt.imshow(X_valid[i,:,:,0])
 plt.show() 
<save_to_csv>
shop_item_prices=encode_prices(['month_id','shop_id','item_id'],'shop_item')
 shop_item_prices=compare_to_lower(shop_item_prices,item_prices,'item_id','shop_item','item')
 shop_item_prices=compare_to_super(shop_item_prices,shop_category_prices,['month_id','shop_id'],'item_id','item_category_id','shop_item','shop_category',items)
 shop_item_prices['month_id'] = shop_item_prices['month_id'].astype(np.int8)
 shop_item_prices['shop_id'] = shop_item_prices['shop_id'].astype(np.int8)
 print(shop_item_prices.loc[shop_item_prices['month_id']<34,:].info(null_counts=True))
shop_item_prices
gmodel=getModel()
gmodel.load_weights(filepath=file_path)
 test = pd.read_json(".. /input/test.json")
 X_test = process_data(test, predict=True)
 predicted_test=gmodel.predict_proba(X_test)
 submission = pd.DataFrame()
submission['id'] = test['id']
submission['is_iceberg'] = predicted_test.reshape(( predicted_test.shape[0]))
submission.to_csv('sub3.csv', index=False)
 del test
del X_test<import_modules>
create_directory(os.path.join(DATA_FOLDER, 'processed'))
create_directory(os.path.join(DATA_FOLDER, 'processed/price_features'))
train_X.to_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
shop_item_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_item_prices.pkl'))
item_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/item_prices.pkl'))
shop_category_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_category_prices.pkl'))
category_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/category_prices.pkl'))
shop_supercategory_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_supercategory_prices.pkl'))
supercategory_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/supercategory_prices.pkl'))
ANGLE = "inc_angle"
ICEBERG = "is_iceberg"
BAND1 = "band_1"
BAND2 = "band_2"
ID = "id"
initial_model_path = "lgbm" 
<load_from_disk>
del items
del item_categories
del shops
del train_df
del train_df_expand
del train_agg
del train_X
del shop_item_prices
del item_prices
del shop_category_prices
del category_prices
del shop_supercategory_prices
del supercategory_prices
gc.collect()
train = pd.read_json('.. /input/train.json')
 print("rows, cols = " + str(train.shape))
train.head()<data_type_conversions>
reset_variable_space
train[ANGLE] = pd.to_numeric(train[ANGLE],errors='coerce')
 nullPD = pd.DataFrame(train.isnull().sum() , columns=['TotalNull'])
 nullPD<groupby>
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
train_X=train_X[train_X['month_id']<34]
P = train.groupby(ICEBERG)[ID].count().reset_index()
P['Percentage'] = 100*P[ID]/P[ID].sum()
P<define_variables>
tmp_sales=train_X.loc[:,['month_id','shop_id','item_id','item_quantity']]
tmp_sales['item_quantity']=tmp_sales['item_quantity'].astype(np.float32)
 tmp=pd.DataFrame()
tmp['total_sales_per_month']=tmp_sales[['month_id','item_quantity']].groupby('month_id' ).sum() ['item_quantity']
tmp['shop_count']=tmp_sales[['month_id','shop_id']].groupby('month_id' ).nunique() ['shop_id']
tmp['item_count']=tmp_sales[['month_id','item_id']].groupby('month_id' ).nunique() ['item_id']
tmp['avg_sales_per_shop']=tmp['total_sales_per_month']/tmp['shop_count']
tmp['avg_sales_per_item']=tmp['total_sales_per_month']/tmp['item_count']
tmp['pair_count']=tmp['shop_count']*tmp['item_count']
tmp['avg_sales_per_pair']=tmp['total_sales_per_month']/tmp['pair_count']
fig,axes=plt.subplots(4,2,figsize=(15,20))
axes[0,1].plot(tmp['total_sales_per_month'],'-o')
 axes[0,1].set_xlim(0,34)
 axes[0,1].set_ylim(0,170000)
 axes[0,1].grid(True)
 axes[0,1].set_xlabel('month_id')
 axes[0,1].set_ylabel('total_sales')
 axes[1,0].plot(tmp['shop_count'],'-o')
 axes[1,0].set_xlim(0,34)
 axes[1,0].set_ylim(0,60)
 axes[1,0].grid(True)
 axes[1,0].set_xlabel('month_id')
 axes[1,0].set_ylabel('shop_count')
 axes[1,1].plot(tmp['avg_sales_per_shop'],'-o')
 axes[1,1].set_xlim(0,34)
 axes[1,1].set_ylim(0,3700)
 axes[1,1].grid(True)
 axes[1,1].set_xlabel('month_id')
 axes[1,1].set_ylabel('avg_sales_per_shop')
 axes[2,0].plot(tmp['item_count'],'-o')
 axes[2,0].set_xlim(0,34)
 axes[2,0].set_ylim(0,9000)
 axes[2,0].grid(True)
 axes[2,0].set_xlabel('month_id')
 axes[2,0].set_ylabel('item_count')
 axes[2,1].plot(tmp['avg_sales_per_item'],'-o')
 axes[2,1].set_xlim(0,34)
 axes[2,1].set_ylim(0,25)
 axes[2,1].grid(True)
 axes[2,1].set_xlabel('month_id')
 axes[2,1].set_ylabel('avg_sales_per_item')
 axes[3,0].plot(tmp['pair_count'],'-o')
 axes[3,0].set_xlim(0,34)
 axes[3,0].set_ylim(0,400000)
 axes[3,0].grid(True)
 axes[3,0].set_xlabel('month_id')
 axes[3,0].set_ylabel('pair_count')
 axes[3,1].plot(tmp['avg_sales_per_pair'],'-o')
 axes[3,1].set_xlim(0,34)
 axes[3,1].set_ylim(0,0.5)
 axes[3,1].grid(True)
 axes[3,1].set_xlabel('month_id')
 axes[3,1].set_ylabel('avg_sales_per_pair')
 del tmp_sales,tmp,fig,axes
MIN="min"
MAX="max"
MEAN="mean"
STD="std"
LAPLACE="laplacestd"
SOBEL0 = "sobelstd_x"
SOBEL1 = "sobelstd_y"
KURTOSIS = "kurtosis"
SKEW = "skew"
CORR = "pearson"
HYP = "hypstd"
AGG_COLS = [
"%s_%s"%(BAND1,MAX), "%s_%s"%(BAND2,MAX),
"%s_%s"%(BAND1,MIN), "%s_%s"%(BAND2,MIN),
"%s_%s"%(BAND1,MEAN), "%s_%s"%(BAND2,MEAN),
"%s_%s"%(BAND1,STD), "%s_%s"%(BAND2,STD),
"%s_%s"%(BAND1,KURTOSIS), "%s_%s"%(BAND2,KURTOSIS),
"%s_%s"%(BAND1,SKEW), "%s_%s"%(BAND2,SKEW),
"%s_%s"%(BAND1,SOBEL0), "%s_%s"%(BAND2,SOBEL0),
"%s_%s"%(BAND1,SOBEL1), "%s_%s"%(BAND2,SOBEL1),
"%s_%s"%(BAND1,LAPLACE), "%s_%s"%(BAND2,LAPLACE),
CORR,
HYP,
]
ISO = "iso"
ISO_COLS = [
"%s_%s"%(BAND1,ISO), "%s_%s"%(BAND2,ISO)
 ]
VOLUME = "vol"
FEATURES = [ANGLE, VOLUME] + AGG_COLS
def iso(arr):
image = img_as_float(np.reshape(np.array(arr), [75,75]))
image = gaussian_filter(image,2.5)
 seed = np.copy(image)
 seed[1:-1, 1:-1] = image.min()
mask = image
dilated = reconstruction(seed, mask, method='dilation')
 return image-dilated
def sobelstd(arr, axis=0):
image = img_as_float(np.reshape(np.array(arr), [75,75]))
sobelstd = sobel(image, axis=axis, mode='reflect', cval=0.0 ).ravel()
return [sobelstd.std() , sobelstd.max() , sobelstd.mean() ]
def lapacestd(arr):
image = img_as_float(np.reshape(np.array(arr), [75,75]))
lapacestd = laplace(image, mode='reflect', cval=0.0 ).ravel()
return [lapacestd.std() , lapacestd.max() , lapacestd.mean() ]
def volume(arr):
return np.sum(arr)
 def hypot(arr1, arr2):
hyp = np.hypot(arr1, arr2)
 return [np.std(hyp), np.max(hyp), np.median(hyp)]
def computeAdditionalFeatures(df):
df["%s_%s"%(BAND1,MAX)] = df[BAND1].apply(lambda x: np.max(x))
df["%s_%s"%(BAND2,MAX)] = df[BAND2].apply(lambda x: np.max(x))
df["%s_%s"%(BAND1,MIN)] = df[BAND1].apply(lambda x: np.min(x))
df["%s_%s"%(BAND2,MIN)] = df[BAND2].apply(lambda x: np.min(x))
df["%s_%s"%(BAND1,MEAN)] = df[BAND1].apply(lambda x: np.mean(x))
df["%s_%s"%(BAND2,MEAN)] = df[BAND2].apply(lambda x: np.mean(x))
df["%s_%s"%(BAND1,STD)] = df[BAND1].apply(lambda x: np.std(x))
df["%s_%s"%(BAND2,STD)] = df[BAND2].apply(lambda x: np.std(x))
df["%s_%s"%(BAND1,KURTOSIS)] = df[BAND1].apply(lambda x: kurtosis(x))
df["%s_%s"%(BAND2,KURTOSIS)] = df[BAND2].apply(lambda x: kurtosis(x))
df["%s_%s"%(BAND1,SKEW)] = df[BAND1].apply(lambda x: skew(x))
df["%s_%s"%(BAND2,SKEW)] = df[BAND2].apply(lambda x: skew(x))
df["%s_%s"%(BAND1,SOBEL0)] = df[BAND1].apply(lambda x: sobelstd(x, axis=0)[0])
 df["%s_%s"%(BAND1,SOBEL1)] = df[BAND1].apply(lambda x: sobelstd(x, axis=1)[0])
 df["%s_%s"%(BAND2,SOBEL0)] = df[BAND2].apply(lambda x: sobelstd(x, axis=0)[0])
 df["%s_%s"%(BAND2,SOBEL1)] = df[BAND2].apply(lambda x: sobelstd(x, axis=1)[0])
 df["%s_%s"%(BAND1,LAPLACE)] = df[BAND1].apply(lambda x: lapacestd(x)[0])
 df["%s_%s"%(BAND2,LAPLACE)] = df[BAND2].apply(lambda x: lapacestd(x)[0])
 df[CORR] = df.apply(lambda row: np.corrcoef(x=row[BAND1], y=row[BAND2])[1,0], axis=1)
 df[HYP] = df.apply(lambda row: hypot(row[BAND1], row[BAND2])[0], axis=1)
 df["%s_%s"%(BAND1,ISO)] = df[BAND1].apply(lambda x: iso(x))
df["%s_%s"%(BAND2,ISO)] = df[BAND2].apply(lambda x: iso(x))
df[VOLUME] =(df["%s_%s"%(BAND1,ISO)] + df["%s_%s"%(BAND2,ISO)] ).apply(volume)
 cleanDF = df.dropna()
ret = cleanDF[FEATURES].copy(deep=True)
 ret_labels = None
if ICEBERG in cleanDF.columns:
ret_labels = cleanDF[[ICEBERG]]
ret_ids = cleanDF[[ID]]
ret_cols = ret.columns
return ret, ret_labels, ret_ids<data_type_conversions>
del a,b,b_over,b_under,df_count,df_percentage
gc.collect()
def read_and_normalize_train_data(train_df):
featuresDF, labelsDF, idsDF = computeAdditionalFeatures(train_df.copy(deep=True))
train_features = featuresDF.as_matrix()
train_target = labelsDF.as_matrix()
train_id = idsDF.as_matrix()
print("Features size: %s/%s"%(str(train_features.shape), str(train_target.shape)))
 return train_features, train_target, train_id, featuresDF[FEATURES].columns<train_model>
tmp=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
tmp=tmp[['month_id','item_seniority']].groupby('month_id')['item_seniority'].value_counts(normalize=True ).unstack()
print(tmp)
 del tmp
def fit_evaluate_model_lgbm(X_train, Y_train, X_valid, Y_valid, train_data_columns, model_path, num_fold, importance=False):
X_trainDF = pd.DataFrame(X_train, columns=train_data_columns)
 X_validDF = pd.DataFrame(X_valid, columns=train_data_columns)
 train_dataset = lgb.Dataset(X_trainDF, Y_train.reshape(Y_train.shape[0]))
test_dataset = lgb.Dataset(X_validDF, Y_valid.reshape(Y_valid.shape[0]))
evals_result = {}
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'boosting': 'gbdt',
'learning_rate': 0.1,
'num_rounds': 150,
'early_stopping_rounds': 100
}
gbm = lgb.train(params, train_dataset,
valid_sets=test_dataset,
evals_result=evals_result,
verbose_eval=50)
 predict_y_proba_gbm = gbm.predict(X_valid, num_iteration=gbm.best_iteration)
 predict_y_gbm = np.where(predict_y_proba_gbm.reshape(( predict_y_proba_gbm.shape[0])) > 0.5, 1, 0)
 score_ll = metrics.log_loss(Y_valid, predict_y_proba_gbm)
 score_ac = metrics.accuracy_score(Y_valid, predict_y_gbm)
 score_pr = metrics.precision_score(Y_valid, predict_y_gbm)
 score_re = metrics.recall_score(Y_valid, predict_y_gbm)
 score = [score_ll, score_ac, score_pr, score_re]
if(importance == True):
ax = lgb.plot_importance(gbm, max_num_features=20, figsize=(16, 5))
plt.show()
gbmDF = pd.DataFrame([tuple(gbm.feature_importance())], columns= gbm.feature_name())
 gbmDF.sort_index(axis=1, inplace=True)
 return score, gbmDF, gbm<find_best_model_class>
tmp=train_X.loc[:,['month_id','item_seniority','item_sold_in_shop']]
tmp['all']=1
tmp['seniority_0']=(tmp['item_seniority']==0)
 tmp['seniority_1']=(tmp['item_seniority']==1)
 tmp['seniority_2']=(tmp['item_seniority']==2)
 tmp['sold_0']=tmp['item_sold_in_shop']&tmp['seniority_0']
tmp['sold_1']=tmp['item_sold_in_shop']&tmp['seniority_1']
tmp['sold_2']=tmp['item_sold_in_shop']&tmp['seniority_2']
tmp_count=tmp[['month_id','seniority_0','seniority_1','seniority_2','sold_0','sold_1','sold_2','item_sold_in_shop','all']].groupby('month_id' ).sum()
fig,axes=plt.subplots(2,2,figsize=(15,10))
axes[0,0].plot(tmp_count['seniority_0']/tmp_count['all'],'-o')
 axes[0,0].plot(tmp_count['seniority_1']/tmp_count['all'],'-o')
 axes[0,0].plot(tmp_count['seniority_2']/tmp_count['all'],'-o')
 axes[0,0].grid(True)
 axes[0,0].set_ylim(0,1)
 axes[0,0].set_title('proportion of items of each seniority in the dataset')
 axes[0,1].plot(tmp_count['item_sold_in_shop']/tmp_count['all'],'-o')
 axes[0,1].grid(True)
 axes[0,1].set_ylim(0,1)
 axes[0,1].set_title('fraction of items sold overall')
 axes[1,0].plot(tmp_count['sold_0']/tmp_count['item_sold_in_shop'],'-o')
 axes[1,0].plot(tmp_count['sold_1']/tmp_count['item_sold_in_shop'],'-o')
 axes[1,0].plot(tmp_count['sold_2']/tmp_count['item_sold_in_shop'],'-o')
 axes[1,0].grid(True)
 axes[1,0].set_ylim(0,1)
 axes[1,0].set_title('contribution of seniority to items sold in shop')
 axes[1,1].plot(tmp_count['sold_0']/tmp_count['seniority_0'],'-o')
 axes[1,1].plot(tmp_count['sold_1']/tmp_count['seniority_1'],'-o')
 axes[1,1].plot(tmp_count['sold_2']/tmp_count['seniority_2'],'-o')
 axes[1,1].grid(True)
 axes[1,1].set_ylim(0,1)
 axes[1,1].set_title('fraction of items sold per seniority')
 axes[1,1].legend(['seniority 0','seniority 1','seniority 2'])
 del fig,axes,tmp,tmp_count
def run_cross_validation_create_models(train_df, model_path, nfolds=4, break_fold=-1, importance=False):
train_data, train_target, train_id, train_data_columns = read_and_normalize_train_data(train_df)
 train_target = train_target.ravel()
kf = StratifiedKFold(n_splits=nfolds, random_state=None, shuffle=True)
 num_fold = 0
sum_score_ll = 0
sum_score_ac = 0
sum_score_pr = 0
sum_score_re = 0
scores = []
models = []
importanceDF = pd.DataFrame()
for train_index, test_index in kf.split(train_data, train_target):
num_fold += 1
print('
==> Start KFold number {} from {}'.format(num_fold, nfolds))
X_train = train_data[train_index]
Y_train = train_target[train_index]
X_valid = train_data[test_index]
Y_valid = train_target[test_index]
print("Train size: %s/%s"%(str(X_train.shape), str(Y_train.shape)))
 print("Valid size: %s/%s"%(str(X_valid.shape), str(Y_valid.shape)))
 score, impDF, m = fit_evaluate_model_lgbm(X_train,Y_train, X_valid, Y_valid, train_data_columns,
model_path, num_fold, importance=False)
 models.append(m)
 if len(importanceDF)== 0:
importanceDF = impDF
else:
importanceDF = pd.concat([importanceDF, impDF])
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])
 sum_score_ll += score[0]*len(test_index)
 sum_score_ac += score[1]*len(test_index)
 sum_score_pr += score[2]*len(test_index)
 sum_score_re += score[3]*len(test_index)
 scores.append(score)
 if(break_fold > 0)&(break_fold == num_fold):
break
score_ll = sum_score_ll/len(train_data)
 score_ac = sum_score_ac/len(train_data)
 score_pr = sum_score_pr/len(train_data)
 score_re = sum_score_re/len(train_data)
 print('
CV average scores:')
 print('log_loss: %s
accuracy: %s
precision: %s
recall: %s
'%(score_ll, score_ac, score_pr, score_re))
return scores, importanceDF, models<load_from_disk>
train_X.drop(train_X.loc[train_X['month_id']<12].index,axis=0,inplace=True )
test = pd.read_json('.. /input/test.json')
 print("rows, cols = " + str(test.shape))<data_type_conversions>
tmp_avg=train_X[['month_id','item_quantity']].groupby('month_id' ).mean()
tmp_avg.rename({'item_quantity':'overall'},axis=1,inplace=True)
 idxbool=(train_X['item_quantity']>0)
 tmp_avg['over_sold']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
idxbool=(train_X['item_new'])
 tmp_avg['over_new']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
for i in range(1,13):
idxbool=(train_X['item_months_since_release']==i)
 tmp_avg['over_released_since_'+str(i)+'_months']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
idxbool=(train_X['item_never_sold_in_shop_before'])
 tmp_avg['over_never_sold_in_shop_before']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
for i in range(1,13):
idxbool=(train_X['item_months_since_first_sale_in_shop']==i)
 tmp_avg['over_first_sold_in_shop_since_'+str(i)+'_months']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
plt.figure(figsize=(15,10))
sns.heatmap(tmp_avg.T)
 plt.title('average sales')
 del tmp_avg
def read_and_normalize_test_data(test_df):
featuresDF, labelsDF, idsDF = computeAdditionalFeatures(test_df.copy(deep=True))
test_features = featuresDF.as_matrix()
test_id = idsDF.as_matrix()
print("Features size: %s/%s"%(str(test_features.shape), str(test_id.shape)))
 return test_features, test_id, featuresDF[FEATURES].columns<load_from_csv>
del train_X
gc.collect()
X_test, X_test_id, X_test_columns = read_and_normalize_test_data(test )<predict_on_test>
reset_variable_space
yfull_proba_train = []
yfull_proba_test = []
yfull_label_test = []
num_fold = 0
for model in models:
num_fold += 1
print('==> Start KFold number {} from {}'.format(num_fold, NFOLDS))
predicted_test = model.predict(X_test, num_iteration=model.best_iteration)
 predicted_test_label = np.where(predicted_test.reshape(( predicted_test.shape[0])) > 0.5, 1, 0)
 yfull_proba_test.append(predicted_test)
 yfull_label_test.append(predicted_test_label )<create_dataframe>
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
items=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/items.pkl'))
item_categories=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/item_categories.pkl'))
def merge_several_folds_mean(data, nfolds):
a = np.array(data[0])
 aPD = pd.DataFrame(a, columns = ["%s_%d"%(ICEBERG, 1)])
 for i in range(1, nfolds):
a = a + np.array(data[i])
 aPD["%s_%d"%(ICEBERG, i+1)] = np.array(data[i])
 a = a / nfolds*1.0
return a, aPD<find_best_params>
def encoder(agg_func,target_label,groupby_labels,idxbool=None,target_dtype=None):
if target_dtype is None:
target_dtype=train_X[target_label].dtype
if agg_func=='std':
if idxbool is None:
return train_X.loc[:,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func,ddof=0)[target_label].astype(target_dtype)
 return train_X.loc[idxbool,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func,ddof=0)[target_label].astype(target_dtype)
 else:
if idxbool is None:
return train_X.loc[:,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func)[target_label].astype(target_dtype)
 return train_X.loc[idxbool,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func)[target_label].astype(target_dtype )
kfold_cols = ["%s_%d"%(ICEBERG, i)for i in range(1, NFOLDS + 1)]
predicted_test_mean, predicted_test_pd = merge_several_folds_mean(yfull_proba_test, NFOLDS)
 predicted_test_pd.head()<save_to_csv>
def encode_comparison(df,df_super,join_labels,target_labels,comparison_labels,new_labels):
df=df.join(df_super[join_labels+comparison_labels].set_index(join_labels),on=join_labels)
 for target_label,cp_label in zip(target_labels,comparison_labels):
df[cp_label]=df[target_label]/df[cp_label]
return df.rename({cp_label:new_label for(cp_label,new_label)in zip(comparison_labels,new_labels)},axis=1 )
submission = pd.DataFrame()
submission[ID]=test[ID]
submission[ICEBERG]=predicted_test_mean
submission.to_csv("submissionlgbmv1.csv", index=False, sep=",", decimal="." )<define_variables>
all_items=(train_X['item_quantity']>=0)
 def encoding_0(idxbool,groupby_labels,prefix,suffix):
idxbool_sold=idxbool&train_X['item_sold_in_shop']
df=pd.DataFrame(encoder('mean','item_quantity',groupby_labels,idxbool)).rename({'item_quantity':prefix+'_avg_sales'+suffix},axis=1)
 df[prefix+'_fraction_of_items_sold'+suffix]=encoder('mean','item_sold_in_shop',groupby_labels,idxbool,target_dtype=np.float32)
 df[prefix+'_avg_sales_over_sold'+suffix]=encoder('mean','item_quantity',groupby_labels,idxbool_sold)
 df.reset_index(inplace=True)
 return df
def encoding_1(idxbool,groupby_labels,prefix,suffix):
idxbool_sold=idxbool&train_X['item_sold_in_shop']
df=pd.DataFrame(encoder('mean','item_quantity',groupby_labels,idxbool)).rename({'item_quantity':prefix+'_avg_sales'+suffix},axis=1)
 df[prefix+'_fraction_of_items_sold'+suffix]=encoder('mean','item_sold_in_shop',groupby_labels,idxbool,target_dtype=np.float32)
 df[prefix+'_avg_sales_over_sold'+suffix]=encoder('mean','item_quantity',groupby_labels,idxbool_sold)
 df[prefix+'_min_quantity_over_sold'+suffix]=encoder('min','item_quantity',groupby_labels,idxbool_sold)
 df[prefix+'_max_quantity'+suffix]=encoder('max','item_quantity',groupby_labels,idxbool)
 df.reset_index(inplace=True)
 return df
def add_comparison_super(df,idxbool,groupby_labels,prefix,suffix,df_super,label_super,prefix_super,df_mapping):
compared_features=['_avg_sales','_fraction_of_items_sold','_avg_sales_over_sold']
df[label_super]=df[groupby_labels[1]].map(df_mapping[label_super])
 df=encode_comparison(df,df_super,['month_id',label_super],[prefix+feature+suffix for feature in compared_features],[prefix_super+feature+suffix for feature in compared_features],[prefix+feature+'_compared_to_'+prefix_super+suffix for feature in compared_features])
 df.drop(label_super,axis=1,inplace=True)
 return df
def add_comparison_lower(df,idxbool,groupby_labels,prefix_1,prefix_2,suffix,df_compare_1,df_compare_2):
prefix=prefix_1+'_'+prefix_2
compared_features=['_avg_sales','_fraction_of_items_sold','_avg_sales_over_sold']
df=encode_comparison(df,df_compare_1,['month_id',groupby_labels[1]],[prefix+feature+suffix for feature in compared_features],[prefix_1+feature+suffix for feature in compared_features],[prefix+feature+'_compared_to_'+prefix_1+suffix for feature in compared_features])
 df=encode_comparison(df,df_compare_2,['month_id',groupby_labels[2]],[prefix+feature+suffix for feature in compared_features],[prefix_2+feature+suffix for feature in compared_features],[prefix+feature+'_compared_to_'+prefix_2+suffix for feature in compared_features])
 return df
def encoding_2(idxbool,groupby_labels,prefix_1,prefix_2,suffix,df_compare_1,df_compare_2):
prefix=prefix_1+'_'+prefix_2
df=encoding_1(idxbool,groupby_labels,prefix,suffix)
 return add_comparison_lower(df,idxbool,groupby_labels,prefix_1,prefix_2,suffix,df_compare_1,df_compare_2)
 def encoding_3(idxbool,groupby_labels,prefix_1,prefix_2,prefix_3,suffix):
prefix=prefix_1+'_'+prefix_2+'_'+prefix_3
df=encoding_1(idxbool,groupby_labels,prefix,suffix)
 return df
sub_path = ".. /input/statoil-iceberg-submissions"
all_files = os.listdir(sub_path)
 all_files = all_files[1:3]
all_files.append('submission38.csv')
 all_files.append('submission43.csv')
 all_files<load_from_csv>
month_encoding=encoding_0(all_items,['month_id'],'month','')
 month_encoding_seniority_0=encoding_0(( train_X['item_seniority']==0),['month_id'],'month','_seniority_0')
 month_encoding_seniority_1=encoding_0(( train_X['item_seniority']==1),['month_id'],'month','_seniority_1')
 month_encoding_seniority_2=encoding_0(( train_X['item_seniority']==2),['month_id'],'month','_seniority_2')
 print_infos(month_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(month_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(month_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(month_encoding_seniority_2)
 print()
fig,axes=plt.subplots(1,3,figsize=(15,4))
axes[0].plot(month_encoding['month_avg_sales'].values[:-1],'-o')
 axes[0].plot(month_encoding_seniority_0['month_avg_sales_seniority_0'].values[:-1],'-o')
 axes[0].plot(month_encoding_seniority_1['month_avg_sales_seniority_1'].values[:-1],'-o')
 axes[0].plot(month_encoding_seniority_2['month_avg_sales_seniority_2'].values[:-1],'-o')
 axes[0].grid(True)
 axes[0].set_ylim(0,1.5)
 axes[0].legend(['overall','seniority 0','seniority 1','seniority 2'])
 axes[0].set_title('average sales')
 axes[0].set_xlabel('month_id')
 axes[1].plot(month_encoding['month_fraction_of_items_sold'].values[:-1],'-o')
 axes[1].plot(month_encoding_seniority_0['month_fraction_of_items_sold_seniority_0'].values[:-1],'-o')
 axes[1].plot(month_encoding_seniority_1['month_fraction_of_items_sold_seniority_1'].values[:-1],'-o')
 axes[1].plot(month_encoding_seniority_2['month_fraction_of_items_sold_seniority_2'].values[:-1],'-o')
 axes[1].grid(True)
 axes[1].set_ylim(0,0.3)
 axes[1].set_title('fraction of items sold')
 axes[1].set_xlabel('month_id')
 axes[2].plot(month_encoding['month_avg_sales_over_sold'].values[:-1],'-o')
 axes[2].plot(month_encoding_seniority_0['month_avg_sales_over_sold_seniority_0'].values[:-1],'-o')
 axes[2].plot(month_encoding_seniority_1['month_avg_sales_over_sold_seniority_1'].values[:-1],'-o')
 axes[2].plot(month_encoding_seniority_2['month_avg_sales_over_sold_seniority_2'].values[:-1],'-o')
 axes[2].grid(True)
 axes[2].set_ylim(0,5)
 axes[2].set_title('average sales over sold')
 axes[2].set_xlabel('month_id' )
out1 = pd.read_csv(".. /input/statoil-iceberg-submissions/sub_200_ens_densenet.csv", index_col=0)
 out2 = pd.read_csv(".. /input/statoil-iceberg-submissions/sub_TF_keras.csv", index_col=0)
 out3 = pd.read_csv(".. /input/submission38-lb01448/submission38.csv", index_col=0)
 out4 = pd.read_csv(".. /input/submission38-lb01448/submission43.csv", index_col=0)
 concat_sub = pd.concat([out1, out2, out3, out4], axis=1)
 cols = list(map(lambda x: "is_iceberg_" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head() 
<feature_engineering>
months_since_release_encoding=encoding_1(all_items,['month_id','item_months_since_release'],'months_since_release','')
 months_since_release_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_months_since_release'],'months_since_release','_seniority_0')
 months_since_release_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_months_since_release'],'months_since_release','_seniority_1')
 months_since_release_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_months_since_release'],'months_since_release','_seniority_2')
 print_infos(months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(months_since_release_encoding_seniority_2)
 print()
display_1(months_since_release_encoding,'months_since_release_avg_sales' )
concat_sub['is_iceberg_max'] = concat_sub.iloc[:, 1:6].max(axis=1)
 concat_sub['is_iceberg_min'] = concat_sub.iloc[:, 1:6].min(axis=1)
 concat_sub['is_iceberg_mean'] = concat_sub.iloc[:, 1:6].mean(axis=1)
 concat_sub['is_iceberg_median'] = concat_sub.iloc[:, 1:6].median(axis=1 )<define_variables>
display_1(months_since_release_encoding_seniority_1,'months_since_release_avg_sales_seniority_1' )
cutoff_lo = 0.8
cutoff_hi = 0.2<save_to_csv>
month_of_release_encoding=encoding_1(all_items,['month_id','item_month_of_release'],'month_of_release','')
 month_of_release_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_month_of_release'],'month_of_release','_seniority_0')
 month_of_release_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_month_of_release'],'month_of_release','_seniority_1')
 month_of_release_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_month_of_release'],'month_of_release','_seniority_2')
 print_infos(month_of_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(month_of_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(month_of_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(month_of_release_encoding_seniority_2)
 print()
display_1(month_of_release_encoding,'month_of_release_avg_sales' )
<load_from_csv>
display_1(month_of_release_encoding_seniority_1,'month_of_release_avg_sales_seniority_1' )
sub_base = pd.read_csv('.. /input/submission38-lb01448/submission43.csv' )<feature_engineering>
display_1(month_of_release_encoding_seniority_2,'month_of_release_avg_sales_seniority_2' )
concat_sub['is_iceberg_base'] = sub_base['is_iceberg']
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:4] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:4] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_base']))
concat_sub['is_iceberg'] = np.clip(concat_sub['is_iceberg'].values, 0.001, 0.999)
 concat_sub[['id', 'is_iceberg']].to_csv('submission54.csv',
index=False, float_format='%.6f' )<define_variables>
months_since_first_sale_in_shop_encoding=encoding_1(all_items,['month_id','item_months_since_first_sale_in_shop'],'months_since_first_sale_in_shop','')
 months_since_first_sale_in_shop_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_months_since_first_sale_in_shop'],'months_since_first_sale_in_shop','_seniority_0')
 months_since_first_sale_in_shop_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_months_since_first_sale_in_shop'],'months_since_first_sale_in_shop','_seniority_1')
 months_since_first_sale_in_shop_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_months_since_first_sale_in_shop'],'months_since_first_sale_in_shop','_seniority_2')
 months_since_first_sale_in_shop_encoding.loc[months_since_first_sale_in_shop_encoding['item_months_since_first_sale_in_shop']==-1,months_since_first_sale_in_shop_encoding.columns[2:]]=-1
months_since_first_sale_in_shop_encoding_seniority_0.loc[months_since_first_sale_in_shop_encoding_seniority_0['item_months_since_first_sale_in_shop']==-1,months_since_first_sale_in_shop_encoding_seniority_0.columns[2:]]=-1
months_since_first_sale_in_shop_encoding_seniority_1.loc[months_since_first_sale_in_shop_encoding_seniority_1['item_months_since_first_sale_in_shop']==-1,months_since_first_sale_in_shop_encoding_seniority_1.columns[2:]]=-1
months_since_first_sale_in_shop_encoding_seniority_2.loc[months_since_first_sale_in_shop_encoding_seniority_2['item_months_since_first_sale_in_shop']==-1,months_since_first_sale_in_shop_encoding_seniority_2.columns[2:]]=-1
print_infos(months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(months_since_first_sale_in_shop_encoding_seniority_2)
 print()
display_1(months_since_first_sale_in_shop_encoding_seniority_2,'months_since_first_sale_in_shop_avg_sales_seniority_2' )
sub_path = ".. /input/statoil-iceberg-submissions"
all_files = os.listdir(sub_path)
 all_files = all_files[1:3]
all_files.append('submission38.csv')
 all_files<load_from_csv>
month_of_first_sale_in_shop_encoding=encoding_1(all_items,['month_id','item_month_of_first_sale_in_shop'],'month_of_first_sale_in_shop','')
 month_of_first_sale_in_shop_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_month_of_first_sale_in_shop'],'month_of_first_sale_in_shop','_seniority_0')
 month_of_first_sale_in_shop_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_month_of_first_sale_in_shop'],'month_of_first_sale_in_shop','_seniority_1')
 month_of_first_sale_in_shop_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_month_of_first_sale_in_shop'],'month_of_first_sale_in_shop','_seniority_2')
 month_of_first_sale_in_shop_encoding.loc[month_of_first_sale_in_shop_encoding['item_month_of_first_sale_in_shop']==-1,month_of_first_sale_in_shop_encoding.columns[2:]]=-1
month_of_first_sale_in_shop_encoding_seniority_0.loc[month_of_first_sale_in_shop_encoding_seniority_0['item_month_of_first_sale_in_shop']==-1,month_of_first_sale_in_shop_encoding_seniority_0.columns[2:]]=-1
month_of_first_sale_in_shop_encoding_seniority_1.loc[month_of_first_sale_in_shop_encoding_seniority_1['item_month_of_first_sale_in_shop']==-1,month_of_first_sale_in_shop_encoding_seniority_1.columns[2:]]=-1
month_of_first_sale_in_shop_encoding_seniority_2.loc[month_of_first_sale_in_shop_encoding_seniority_2['item_month_of_first_sale_in_shop']==-1,month_of_first_sale_in_shop_encoding_seniority_2.columns[2:]]=-1
print_infos(month_of_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(month_of_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(month_of_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(month_of_first_sale_in_shop_encoding_seniority_2)
 print()
display_1(month_of_first_sale_in_shop_encoding_seniority_2,'month_of_first_sale_in_shop_avg_sales_seniority_2' )
out1 = pd.read_csv(".. /input/statoil-iceberg-submissions/sub_200_ens_densenet.csv", index_col=0)
 out2 = pd.read_csv(".. /input/statoil-iceberg-submissions/sub_TF_keras.csv", index_col=0)
 out3 = pd.read_csv(".. /input/submission38-lb01448/submission38.csv", index_col=0)
 concat_sub = pd.concat([out1, out2, out3], axis=1)
 cols = list(map(lambda x: "is_iceberg_" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head() 
<feature_engineering>
shop_encoding=encoding_1(all_items,['month_id','shop_id'],'shop','')
 shop_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','shop_id'],'shop','_seniority_0')
 shop_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','shop_id'],'shop','_seniority_1')
 shop_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','shop_id'],'shop','_seniority_2')
 print_infos(shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_encoding_seniority_2)
 print()
display_2(shop_encoding,'shop_avg_sales' )
concat_sub['is_iceberg_max'] = concat_sub.iloc[:, 1:6].max(axis=1)
 concat_sub['is_iceberg_min'] = concat_sub.iloc[:, 1:6].min(axis=1)
 concat_sub['is_iceberg_mean'] = concat_sub.iloc[:, 1:6].mean(axis=1)
 concat_sub['is_iceberg_median'] = concat_sub.iloc[:, 1:6].median(axis=1 )<init_hyperparams>
supercategory_encoding=encoding_1(all_items,['month_id','item_supercategory_id'],'supercategory','')
 supercategory_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_supercategory_id'],'supercategory','_seniority_0')
 supercategory_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_supercategory_id'],'supercategory','_seniority_1')
 supercategory_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_supercategory_id'],'supercategory','_seniority_2')
 print_infos(supercategory_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_encoding_seniority_2)
 print()
display_3(supercategory_encoding,'supercategory_avg_sales' )
cutoff_lo = 0.75
cutoff_hi = 0.25<save_to_csv>
category_encoding=encoding_1(all_items,['month_id','item_category_id'],'category','')
 category_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_category_id'],'category','_seniority_0')
 category_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_category_id'],'category','_seniority_1')
 category_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_category_id'],'category','_seniority_2')
 category_encoding=add_comparison_super(category_encoding,all_items,['month_id','item_category_id'],'category','',supercategory_encoding,'item_supercategory_id','supercategory',item_categories)
 category_encoding_seniority_0=add_comparison_super(category_encoding_seniority_0,(train_X['item_seniority']==0),['month_id','item_category_id'],'category','_seniority_0',supercategory_encoding_seniority_0,'item_supercategory_id','supercategory',item_categories)
 category_encoding_seniority_1=add_comparison_super(category_encoding_seniority_1,(train_X['item_seniority']==1),['month_id','item_category_id'],'category','_seniority_1',supercategory_encoding_seniority_1,'item_supercategory_id','supercategory',item_categories)
 category_encoding_seniority_2=add_comparison_super(category_encoding_seniority_2,(train_X['item_seniority']==2),['month_id','item_category_id'],'category','_seniority_2',supercategory_encoding_seniority_2,'item_supercategory_id','supercategory',item_categories)
 print_infos(category_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(category_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(category_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(category_encoding_seniority_2)
 print()
display_3(category_encoding,'category_avg_sales' )
<load_from_csv>
item_encoding=encoding_1(all_items,['month_id','item_id'],'item','')
 item_encoding_seniority_0=encoding_1(( train_X['item_seniority']==0),['month_id','item_id'],'item','_seniority_0')
 item_encoding_seniority_1=encoding_1(( train_X['item_seniority']==1),['month_id','item_id'],'item','_seniority_1')
 item_encoding_seniority_2=encoding_1(( train_X['item_seniority']==2),['month_id','item_id'],'item','_seniority_2')
 item_encoding=add_comparison_super(item_encoding,all_items,['month_id','item_id'],'item','',category_encoding,'item_category_id','category',items)
 item_encoding_seniority_0=add_comparison_super(item_encoding_seniority_0,(train_X['item_seniority']==0),['month_id','item_id'],'item','_seniority_0',category_encoding_seniority_0,'item_category_id','category',items)
 item_encoding_seniority_1=add_comparison_super(item_encoding_seniority_1,(train_X['item_seniority']==1),['month_id','item_id'],'item','_seniority_1',category_encoding_seniority_1,'item_category_id','category',items)
 item_encoding_seniority_2=add_comparison_super(item_encoding_seniority_2,(train_X['item_seniority']==2),['month_id','item_id'],'item','_seniority_2',category_encoding_seniority_2,'item_category_id','category',items)
 print_infos(item_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(item_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(item_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(item_encoding_seniority_2)
 print()
sub_base = pd.read_csv('.. /input/submission38-lb01448/submission38.csv' )<feature_engineering>
shop_category_encoding=encoding_2(all_items,['month_id','shop_id','item_category_id'],'shop','category','',shop_encoding,category_encoding)
 shop_category_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','shop_id','item_category_id'],'shop','category','_seniority_0',shop_encoding_seniority_0,category_encoding_seniority_0)
 shop_category_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','shop_id','item_category_id'],'shop','category','_seniority_1',shop_encoding_seniority_1,category_encoding_seniority_1)
 shop_category_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','shop_id','item_category_id'],'shop','category','_seniority_2',shop_encoding_seniority_2,category_encoding_seniority_2)
 print_infos(shop_category_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_encoding_seniority_2)
 print()
display_4(shop_category_encoding,'shop_category_avg_sales' )
concat_sub['is_iceberg_base'] = sub_base['is_iceberg']
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_base']))
concat_sub['is_iceberg'] = np.clip(concat_sub['is_iceberg'].values, 0.001, 0.999)
 concat_sub[['id', 'is_iceberg']].to_csv('submission39.csv',
index=False, float_format='%.6f' )<load_from_disk>
shop_supercategory_encoding=encoding_2(all_items,['month_id','shop_id','item_supercategory_id'],'shop','supercategory','',shop_encoding,supercategory_encoding)
 shop_supercategory_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','shop_id','item_supercategory_id'],'shop','supercategory','_seniority_0',shop_encoding_seniority_0,supercategory_encoding_seniority_0)
 shop_supercategory_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','shop_id','item_supercategory_id'],'shop','supercategory','_seniority_1',shop_encoding_seniority_1,supercategory_encoding_seniority_1)
 shop_supercategory_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','shop_id','item_supercategory_id'],'shop','supercategory','_seniority_2',shop_encoding_seniority_2,supercategory_encoding_seniority_2)
 print_infos(shop_supercategory_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_encoding_seniority_2)
 print()
display_5(shop_supercategory_encoding,'shop_supercategory_avg_sales' )
train = pd.read_json(".. /input/train.json")
 test = pd.read_json(".. /input/test.json" )<prepare_x_and_y>
shop_item_encoding=train_X[['month_id','shop_id','item_id','item_quantity']]
shop_item_encoding=encode_comparison(shop_item_encoding,shop_encoding,['month_id','shop_id'],['item_quantity','item_quantity'],['shop_avg_sales','shop_avg_sales_over_sold'],['item_quantity_compared_to_shop_avg_sales','item_quantity_compared_to_shop_avg_sales_over_sold'])
 shop_item_encoding=encode_comparison(shop_item_encoding,item_encoding,['month_id','item_id'],['item_quantity','item_quantity'],['item_avg_sales','item_avg_sales_over_sold'],['item_quantity_compared_to_item_avg_sales','item_quantity_compared_to_item_avg_sales_over_sold'])
 shop_item_encoding['item_category_id']=shop_item_encoding['item_id'].map(items['item_category_id'])
 shop_item_encoding=encode_comparison(shop_item_encoding,shop_category_encoding,['month_id','shop_id','item_category_id'],['item_quantity','item_quantity'],['shop_category_avg_sales','shop_category_avg_sales_over_sold'],['item_quantity_compared_to_shop_category_avg_sales','item_quantity_compared_to_shop_category_avg_sales_over_sold'])
 shop_item_encoding.drop('item_category_id',axis=1,inplace=True)
 print_infos(shop_item_encoding )
X_band_1=np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in train["band_1"]])
 X_band_2=np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in train["band_2"]])
 X_train = np.concatenate([X_band_1[:, :, :, np.newaxis], X_band_2[:, :, :, np.newaxis],(( X_band_1+X_band_2)/2)[:, :, :, np.newaxis]], axis=-1)
 Y_train=train['is_iceberg'] 
<import_modules>
category_months_since_release_encoding=encoding_2(all_items,['month_id','item_category_id','item_months_since_release'],'category','months_since_release','',category_encoding,months_since_release_encoding)
 category_months_since_release_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_category_id','item_months_since_release'],'category','months_since_release','_seniority_0',category_encoding_seniority_0,months_since_release_encoding_seniority_0)
 category_months_since_release_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_category_id','item_months_since_release'],'category','months_since_release','_seniority_1',category_encoding_seniority_1,months_since_release_encoding_seniority_1)
 category_months_since_release_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_category_id','item_months_since_release'],'category','months_since_release','_seniority_2',category_encoding_seniority_2,months_since_release_encoding_seniority_2)
 print_infos(category_months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_release_encoding_seniority_2)
 print()
display_6(category_months_since_release_encoding,'category_months_since_release_avg_sales' )
from matplotlib import pyplot
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten
from keras.layers import GlobalMaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.merge import Concatenate
from keras.models import Model
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping, ReduceLROnPlateau<choose_model_class>
supercategory_months_since_release_encoding=encoding_2(all_items,['month_id','item_supercategory_id','item_months_since_release'],'supercategory','months_since_release','',supercategory_encoding,months_since_release_encoding)
 supercategory_months_since_release_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_supercategory_id','item_months_since_release'],'supercategory','months_since_release','_seniority_0',supercategory_encoding_seniority_0,months_since_release_encoding_seniority_0)
 supercategory_months_since_release_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_supercategory_id','item_months_since_release'],'supercategory','months_since_release','_seniority_1',supercategory_encoding_seniority_1,months_since_release_encoding_seniority_1)
 supercategory_months_since_release_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_supercategory_id','item_months_since_release'],'supercategory','months_since_release','_seniority_2',supercategory_encoding_seniority_2,months_since_release_encoding_seniority_2)
 print_infos(supercategory_months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_release_encoding_seniority_2)
 print()
display_6(supercategory_months_since_release_encoding,'supercategory_months_since_release_avg_sales' )
def getModel() :
model=Sequential()
model.add(Conv2D(8, kernel_size=(3, 3),activation='relu', input_shape=(75, 75, 3)))
 model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(16, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation="sigmoid"))
optimizer = Adam(lr=0.001)
 model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 return model
model = getModel()
model.summary()<choose_model_class>
category_month_of_release_encoding=encoding_2(all_items,['month_id','item_category_id','item_month_of_release'],'category','month_of_release','',category_encoding,month_of_release_encoding)
 category_month_of_release_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_category_id','item_month_of_release'],'category','month_of_release','_seniority_0',category_encoding_seniority_0,month_of_release_encoding_seniority_0)
 category_month_of_release_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_category_id','item_month_of_release'],'category','month_of_release','_seniority_1',category_encoding_seniority_1,month_of_release_encoding_seniority_1)
 category_month_of_release_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_category_id','item_month_of_release'],'category','month_of_release','_seniority_2',category_encoding_seniority_2,month_of_release_encoding_seniority_2)
 print_infos(category_month_of_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(category_month_of_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(category_month_of_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(category_month_of_release_encoding_seniority_2)
 print()
display_6(category_month_of_release_encoding,'category_month_of_release_avg_sales' )
batch_size = 32
earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
 mcp_save = ModelCheckpoint('.mdl_wts.hdf5', save_best_only=True, monitor='val_loss', mode='min')
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='min' )<train_model>
supercategory_month_of_release_encoding=encoding_2(all_items,['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','',supercategory_encoding,month_of_release_encoding)
 supercategory_month_of_release_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','_seniority_0',supercategory_encoding_seniority_0,month_of_release_encoding_seniority_0)
 supercategory_month_of_release_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','_seniority_1',supercategory_encoding_seniority_1,month_of_release_encoding_seniority_1)
 supercategory_month_of_release_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','_seniority_2',supercategory_encoding_seniority_2,month_of_release_encoding_seniority_2)
 print_infos(supercategory_month_of_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_month_of_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_month_of_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_month_of_release_encoding_seniority_2)
 print()
display_6(supercategory_month_of_release_encoding,'supercategory_month_of_release_avg_sales' )
X_train_cv, X_valid, y_train_cv, y_valid = train_test_split(X_train, Y_train, random_state=1, train_size=0.75)
 model=getModel()
model.fit(X_train_cv, y_train_cv, batch_size=batch_size, epochs=100, verbose=1, validation_data=(X_valid, y_valid), callbacks=[earlyStopping, mcp_save, reduce_lr_loss], validation_split=0.25 )<train_model>
category_months_since_first_sale_in_shop_encoding=encoding_2(all_items,['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','',category_encoding,months_since_first_sale_in_shop_encoding)
 category_months_since_first_sale_in_shop_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','_seniority_0',category_encoding_seniority_0,months_since_first_sale_in_shop_encoding_seniority_0)
 category_months_since_first_sale_in_shop_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','_seniority_1',category_encoding_seniority_1,months_since_first_sale_in_shop_encoding_seniority_1)
 category_months_since_first_sale_in_shop_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','_seniority_2',category_encoding_seniority_2,months_since_first_sale_in_shop_encoding_seniority_2)
 category_months_since_first_sale_in_shop_encoding.loc[category_months_since_first_sale_in_shop_encoding['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding.columns[3:]]=-1
category_months_since_first_sale_in_shop_encoding_seniority_0.loc[category_months_since_first_sale_in_shop_encoding_seniority_0['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding_seniority_0.columns[3:]]=-1
category_months_since_first_sale_in_shop_encoding_seniority_1.loc[category_months_since_first_sale_in_shop_encoding_seniority_1['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding_seniority_1.columns[3:]]=-1
category_months_since_first_sale_in_shop_encoding_seniority_2.loc[category_months_since_first_sale_in_shop_encoding_seniority_2['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding_seniority_2.columns[3:]]=-1
print_infos(category_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_first_sale_in_shop_encoding_seniority_2)
 print()
display_6(category_months_since_first_sale_in_shop_encoding_seniority_2,'category_months_since_first_sale_in_shop_avg_sales_seniority_2' )
model.load_weights(filepath = '.mdl_wts.hdf5')
 score = model.evaluate(X_train, Y_train, verbose=1)
 print('Train score:', score[0])
 print('Train accuracy:', score[1] )<predict_on_test>
supercategory_months_since_first_sale_in_shop_encoding=encoding_2(all_items,['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','',supercategory_encoding,months_since_first_sale_in_shop_encoding)
 supercategory_months_since_first_sale_in_shop_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','_seniority_0',supercategory_encoding_seniority_0,months_since_first_sale_in_shop_encoding_seniority_0)
 supercategory_months_since_first_sale_in_shop_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','_seniority_1',supercategory_encoding_seniority_1,months_since_first_sale_in_shop_encoding_seniority_1)
 supercategory_months_since_first_sale_in_shop_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','_seniority_2',supercategory_encoding_seniority_2,months_since_first_sale_in_shop_encoding_seniority_2)
 supercategory_months_since_first_sale_in_shop_encoding.loc[supercategory_months_since_first_sale_in_shop_encoding['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding.columns[3:]]=-1
supercategory_months_since_first_sale_in_shop_encoding_seniority_0.loc[supercategory_months_since_first_sale_in_shop_encoding_seniority_0['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding_seniority_0.columns[3:]]=-1
supercategory_months_since_first_sale_in_shop_encoding_seniority_1.loc[supercategory_months_since_first_sale_in_shop_encoding_seniority_1['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding_seniority_1.columns[3:]]=-1
supercategory_months_since_first_sale_in_shop_encoding_seniority_2.loc[supercategory_months_since_first_sale_in_shop_encoding_seniority_2['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding_seniority_2.columns[3:]]=-1
print_infos(supercategory_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_first_sale_in_shop_encoding_seniority_2)
 print()
display_6(supercategory_months_since_first_sale_in_shop_encoding_seniority_2,'supercategory_months_since_first_sale_in_shop_avg_sales_seniority_2' )
X_band_test_1=np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in test["band_1"]])
 X_band_test_2=np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in test["band_2"]])
 X_test = np.concatenate([X_band_test_1[:, :, :, np.newaxis]
, X_band_test_2[:, :, :, np.newaxis]
,(( X_band_test_1+X_band_test_2)/2)[:, :, :, np.newaxis]], axis=-1)
 pred_test = model.predict(X_test )<prepare_output>
shop_category_months_since_release_encoding=encoding_3(all_items,['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','')
 shop_category_months_since_release_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','_seniority_0')
 shop_category_months_since_release_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','_seniority_1')
 shop_category_months_since_release_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','_seniority_2')
 print_infos(shop_category_months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_release_encoding_seniority_2)
 print()
submission = pd.DataFrame({'id': test["id"], 'is_iceberg': pred_test.reshape(( pred_test.shape[0])) })
 submission.head(10 )<save_to_csv>
shop_category_months_since_first_sale_in_shop_encoding=encoding_3(all_items,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','')
 shop_category_months_since_first_sale_in_shop_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','_seniority_0')
 shop_category_months_since_first_sale_in_shop_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','_seniority_1')
 shop_category_months_since_first_sale_in_shop_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','_seniority_2')
 print_infos(shop_category_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_first_sale_in_shop_encoding_seniority_2)
 print()
submission.to_csv("./submission.csv", index=False )<set_options>
shop_supercategory_months_since_release_encoding=encoding_3(all_items,['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','')
 shop_supercategory_months_since_release_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','_seniority_0')
 shop_supercategory_months_since_release_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','_seniority_1')
 shop_supercategory_months_since_release_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','_seniority_2')
 print_infos(shop_supercategory_months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_release_encoding_seniority_2)
 print()
np.random.seed(98643)
 tf.set_random_seed(683)
 denoise_wavelet, estimate_sigma, denoise_tv_bregman, denoise_nl_means)
 %matplotlib inline 
<data_type_conversions>
shop_supercategory_months_since_first_sale_in_shop_encoding=encoding_3(all_items,['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','')
 shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','_seniority_0')
 shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','_seniority_1')
 shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','_seniority_2')
 print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_2)
 print()
train = pd.read_json(".. /input/train.json")
 train.inc_angle = train.inc_angle.replace('na', 0)
 train.inc_angle = train.inc_angle.astype(float ).fillna(0.0)
 train_all = True
train_b = True or train_all
train_img = True or train_all
train_total = True or train_all
predict_submission = True and train_all
clean_all = False
clean_b = False or clean_all
clean_img = False or clean_all
load_all = False
load_b = False or load_all
load_img = False or load_all<create_dataframe>
dfs=['month_encoding',
'months_since_release_encoding',
'month_of_release_encoding',
'months_since_first_sale_in_shop_encoding',
'month_of_first_sale_in_shop_encoding',
'shop_encoding',
'item_encoding',
'category_encoding',
'supercategory_encoding',
'shop_category_encoding',
'shop_supercategory_encoding',
'category_months_since_release_encoding',
'supercategory_months_since_release_encoding',
'category_month_of_release_encoding',
'supercategory_month_of_release_encoding',
'category_months_since_first_sale_in_shop_encoding',
'supercategory_months_since_first_sale_in_shop_encoding',
'shop_category_months_since_release_encoding',
'shop_supercategory_months_since_release_encoding',
'shop_category_months_since_first_sale_in_shop_encoding',
'shop_supercategory_months_since_first_sale_in_shop_encoding'
]
def create_dataset(frame, labeled, smooth_rgb=0.2, smooth_gray=0.5,
weight_rgb=0.05, weight_gray=0.05):
band_1, band_2, images = frame['band_1'].values, frame['band_2'].values, color_composite(frame)
 to_arr = lambda x: np.asarray([np.asarray(item)for item in x])
 band_1 = to_arr(band_1)
 band_2 = to_arr(band_2)
 band_3 =(band_1 + band_2)/ 2
gray_reshape = lambda x: np.asarray([item.reshape(75, 75)for item in x])
 band_1 = gray_reshape(band_1)
 band_2 = gray_reshape(band_2)
 band_3 = gray_reshape(band_3)
 print('Denoising and reshaping')
 if train_b and clean_b:
band_1 = smooth(denoise(band_1, weight_gray, False), smooth_gray)
 print('Gray 1 done')
 band_2 = smooth(denoise(band_2, weight_gray, False), smooth_gray)
 print('Gray 2 done')
 band_3 = smooth(denoise(band_3, weight_gray, False), smooth_gray)
 print('Gray 3 done')
 if train_img and clean_img:
images = smooth(denoise(images, weight_rgb, True), smooth_rgb)
 print('RGB done')
 tf_reshape = lambda x: np.asarray([item.reshape(75, 75, 1)for item in x])
 band_1 = tf_reshape(band_1)
 band_2 = tf_reshape(band_2)
 band_3 = tf_reshape(band_3)
 band = np.concatenate([band_1, band_2, band_3], axis=3)
 if labeled:
y = np.array(frame["is_iceberg"])
 else:
y = None
return y, band, images<prepare_x_and_y>
create_directory(os.path.join(DATA_FOLDER,'processed/target_encodings'))
for df in dfs+['shop_item_encoding']:
exec(df+".to_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+".pkl')) ")
 exec("del "+df)
 for df in dfs:
for seniority in range(0,3):
exec(df+"_seniority_"+str(seniority)+".to_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+"_seniority_"+str(seniority)+".pkl')) ")
 exec("del "+df+"_seniority_"+str(seniority))
y_train, X_b, X_images = create_dataset(train, True )<choose_model_class>
del train_X
del items, item_categories
gc.collect()
def get_model_notebook(lr, decay, channels, relu_type='relu'):
input_1 = Input(shape=(75, 75, channels))
fcnn = Conv2D(32, kernel_size=(3, 3), activation=relu_type )(
 BatchNormalization()(input_1))
fcnn = MaxPooling2D(( 3, 3))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(64, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(128, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(128, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = BatchNormalization()(fcnn)
 fcnn = Flatten()(fcnn)
 local_input = input_1
partial_model = Model(input_1, fcnn)
 dense = Dropout(0.2 )(fcnn)
 dense = Dense(256, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 dense = Dense(128, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 dense = Dense(64, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 output = Dense(1, activation="sigmoid" )(dense)
 model = Model(local_input, output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model, partial_model 
<define_search_model>
reset_variable_space
def combined_model(m_b, m_img, lr, decay):
input_b = Input(shape=(75, 75, 3))
input_img = Input(shape=(75, 75, 3))
m1 = m_b(input_b)
 m2 = m_img(input_img)
 common = Concatenate()([m1, m2])
 common = BatchNormalization()(common)
 common = Dropout(0.3 )(common)
 common = Dense(1024, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 common = Dense(512, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 output = Dense(1, activation="sigmoid" )(common)
 model = Model([input_b, input_img], output)
 optimizer = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=decay)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model<concatenate>
def lag_features(df,df_features,col_agg,col_lag,lagging_values,fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_lag)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
df_features_lag=df.loc[:,col_agg]
tmp=df_features.loc[:,col_agg+col_lag]
for lag in lagging_values:
df_features_lag=df_features_lag.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 if fill_value is not None:
for col in col_lag:
for lag in lagging_values:
df_features_lag[col+'_lag_'+str(lag)].fillna(fill_value,inplace=True)
 return df_features_lag.drop(col_agg,axis=1 )
def gen_flow_multi_inputs(I1, I2, y, batch_size):
gen1 = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.,
height_shift_range=0.,
channel_shift_range=0,
zoom_range=0.2,
rotation_range=10)
 gen2 = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.,
height_shift_range=0.,
channel_shift_range=0,
zoom_range=0.2,
rotation_range=10)
 genI1 = gen1.flow(I1, y, batch_size=batch_size, seed=57, shuffle=False)
 genI2 = gen2.flow(I1, I2, batch_size=batch_size, seed=57, shuffle=False)
 while True:
I1i = genI1.next()
I2i = genI2.next()
np.testing.assert_array_equal(I2i[0], I1i[0])
 yield [I1i[0], I2i[1]], I1i[1]<train_model>
def moving_statistics(df,df_features,col_agg,col_avg,windowsize,func,suffix='',fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
df_features_lag=df.loc[:,col_agg]
tmp=df_features.loc[:,col_agg+col_lag]
for lag in range(1,windowsize+1):
df_features_lag=df_features_lag.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_avg=df.loc[:,col_agg]
for col in col_avg:
if func=='mean':
if suffix=='':
suffix='_movavg_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].mean(axis=1)
 elif func=='max':
if suffix=='':
suffix='_movmax_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].max(axis=1)
 elif func=='min':
if suffix=='':
suffix='_movmin_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].min(axis=1)
 elif func=='std':
if suffix=='':
suffix='_movstd_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].std(ddof=0,axis=1)
 elif func=='rsd':
if suffix=='':
suffix='_movrsd_'+str(windowsize)
 denom_avg=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].mean(axis=1)
 num_std=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].std(ddof=0,axis=1)
 df_features_avg[col+suffix]=num_std/denom_avg
df_features_avg.loc[num_std==0,col+suffix]=0
del denom_avg
del num_std
if fill_value is not None:
for col in col_lag:
df_features_avg[col+suffix].fillna(fill_value,inplace=True)
 return df_features_avg.drop(col_agg,axis=1 )
def train_model(model, batch_size, epochs, checkpoint_name, X_train, y_train, val_data, verbose=2):
callbacks = [ModelCheckpoint(checkpoint_name, save_best_only=True, monitor='val_loss')]
datagen = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.,
height_shift_range=0.,
channel_shift_range=0,
zoom_range=0.2,
rotation_range=10)
 x_test, y_test = val_data
try:
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size), epochs=epochs,
steps_per_epoch=len(X_train)/ batch_size,
validation_data=(x_test, y_test), verbose=1,
callbacks=callbacks)
 except KeyboardInterrupt:
if verbose > 0:
print('Interrupted')
 if verbose > 0:
print('Loading model')
 model.load_weights(filepath=checkpoint_name)
 return model<train_model>
def temporal_compare(df,df_features,col_agg,col_avg,lags_num,lags_denom,suffix,fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
tmp=df_features.loc[:,col_agg+col_lag]
df_features_lag_num=df.loc[:,col_agg]
for lag in lags_num:
df_features_lag_num=df_features_lag_num.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_lag_denom=df.loc[:,col_agg]
for lag in lags_denom:
df_features_lag_denom=df_features_lag_denom.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_comp=df.loc[:,col_agg]
for col in col_avg:
df_features_lag_num[col+'_avg']=df_features_lag_num.loc[:,[col+'_lag_'+str(lag)for lag in lags_num]].mean(axis=1)
 df_features_lag_denom[col+'_avg']=df_features_lag_denom.loc[:,[col+'_lag_'+str(lag)for lag in lags_denom]].mean(axis=1)
 df_features_comp[col+suffix]=df_features_lag_num[col+'_avg']/df_features_lag_denom[col+'_avg']
if fill_value is not None:
for col in col_lag:
df_features_comp[col+suffix].fillna(fill_value,inplace=True)
 return df_features_comp.drop(col_agg,axis=1 )
def gen_model_weights(lr, decay, channels, relu, batch_size, epochs, path_name, data, verbose=2):
X_train, y_train, X_test, y_test, X_val, y_val = data
model, partial_model = get_model_notebook(lr, decay, channels, relu)
 model = train_model(model, batch_size, epochs, path_name,
X_train, y_train,(X_test, y_test), verbose=verbose)
 if verbose > 0:
loss_val, acc_val = model.evaluate(X_val, y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = model.evaluate(X_test, y_test,
verbose=0, batch_size=batch_size)
 print('Val/Train Loss:', str(loss_val)+ '/' + str(loss_train), \
'Val/Train Acc:', str(acc_val)+ '/' + str(acc_train))
return model, partial_model<split>
def rational_fraction(df,df_features,col_agg,col_avg,weights_num,weights_denom,suffix,fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 windowsize=len(weights_num)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
tmp=df_features.loc[:,col_agg+col_lag]
df_features_lag_num=df.loc[:,col_agg]
for lag in range(1,windowsize+1):
df_features_lag_num=df_features_lag_num.join(lag_dataframe(tmp,lag ).set_index(col_agg)*weights_num[lag-1],on=col_agg)
 df_features_lag_denom=df.loc[:,col_agg]
for lag in range(1,windowsize+1):
df_features_lag_denom=df_features_lag_denom.join(lag_dataframe(tmp,lag ).set_index(col_agg)*weights_denom[lag-1],on=col_agg)
 df_features_comp=df.loc[:,col_agg]
for col in col_avg:
df_features_lag_num[col+'_avg']=df_features_lag_num.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].sum(axis=1)
 df_features_lag_denom[col+'_avg']=df_features_lag_denom.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].sum(axis=1)
 df_features_comp[col+suffix]=df_features_lag_num[col+'_avg']/df_features_lag_denom[col+'_avg']
if fill_value is not None:
for col in col_lag:
df_features_comp[col+suffix].fillna(fill_value,inplace=True)
 return df_features_comp.drop(col_agg,axis=1 )
def train_models(dataset, lr, batch_size, max_epoch, verbose=2, return_model=False):
y_train, X_b, X_images = dataset
y_train_full, y_val,\
X_b_full, X_b_val,\
X_images_full, X_images_val = train_test_split(y_train, X_b, X_images, random_state=687, train_size=0.9)
 y_train, y_test, \
X_b_train, X_b_test, \
X_images_train, X_images_test = train_test_split(y_train_full, X_b_full, X_images_full, random_state=576, train_size=0.85)
 if train_b:
if verbose > 0:
print('Training bandwidth network')
 data_b1 =(X_b_train, y_train, X_b_test, y_test, X_b_val, y_val)
 model_b, model_b_cut = gen_model_weights(lr, 1e-6, 3, 'relu', batch_size, max_epoch, 'model_b',
data=data_b1, verbose=verbose)
 if train_img:
if verbose > 0:
print('Training image network')
 data_images =(X_images_train, y_train, X_images_test, y_test, X_images_val, y_val)
 model_images, model_images_cut = gen_model_weights(lr, 1e-6, 3, 'relu', batch_size, max_epoch, 'model_img',
data_images, verbose=verbose)
 if train_total:
common_model = combined_model(model_b_cut, model_images_cut, lr/2, 1e-7)
 common_x_train = [X_b_full, X_images_full]
common_y_train = y_train_full
common_x_val = [X_b_val, X_images_val]
common_y_val = y_val
if verbose > 0:
print('Training common network')
 callbacks = [ModelCheckpoint('common', save_best_only=True, monitor='val_loss')]
try:
common_model.fit_generator(gen_flow_multi_inputs(X_b_full, X_images_full, y_train_full, batch_size),
epochs=30,
steps_per_epoch=len(X_b_full)/ batch_size,
validation_data=(common_x_val, common_y_val), verbose=1,
callbacks=callbacks)
 except KeyboardInterrupt:
pass
common_model.load_weights(filepath='common')
 loss_val, acc_val = common_model.evaluate(common_x_val, common_y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = common_model.evaluate(common_x_train, common_y_train,
verbose=0, batch_size=batch_size)
 if verbose > 0:
print('Loss:', loss_val, 'Acc:', acc_val)
 if return_model:
return common_model
else:
return(loss_train, acc_train),(loss_val, acc_val )<train_model>
def time_mapping(df,df_features,join_labels,col_labels,map_to_label,suffix):
df_mapped=df.loc[:,[map_to_label]+join_labels]
shifted_df=df_features.loc[:,['month_id']+join_labels+col_labels].rename({'month_id':map_to_label},axis=1 ).set_index([map_to_label]+join_labels)
 shifted_df.rename({col:col+suffix for col in col_labels},axis=1,inplace=True)
 return df_mapped.join(shifted_df,on=[map_to_label]+join_labels ).drop([map_to_label]+join_labels,axis=1 )
common_model = train_models(( y_train, X_b, X_images), 7e-04, 32, 50, 1, return_model=True )<save_to_csv>
def feature_correlation_analysis(train_df,feature_df,feature_df_seniority,feature_name,groupby_labels,seniority):
tmp=feature_df_seniority.loc[(feature_df_seniority['month_id']>12)&(feature_df_seniority['month_id']<34),groupby_labels+[feature_name+'_avg_sales_seniority_'+str(seniority),feature_name+'_avg_sales_over_sold_seniority_'+str(seniority)]]
tmp2=feature_df.loc[(feature_df['month_id']>12)&(feature_df['month_id']<34),groupby_labels+[feature_name+'_avg_sales',feature_name+'_avg_sales_over_sold']]
tmp=tmp.join(tmp2.set_index(groupby_labels),on=groupby_labels)
 tmp_full=train_df[groupby_labels+['item_quantity']].join(tmp.set_index(groupby_labels),on=groupby_labels)
 fig=plt.figure(figsize=(5,5))
sns.heatmap(tmp_full.drop(groupby_labels,axis=1 ).corr() ,annot=True)
 def compare_features(df,df_seniority,name,labels,seniority,avg_sales_max=20,avg_sales_seniority_max=20,avg_sales_over_sold_max=20):
n=len(labels)
 tmp=df.join(df_seniority.set_index(labels),on=labels)
 labels.remove('month_id')
 labels.append('month')
 tmp['month']=tmp['month_id']%12+1
if name=='item':
fig,axes=plt.subplots(1,3,figsize=(15,5))
sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_seniority_'+str(seniority),hue='month',ax=axes[0])
 axes[0].plot([0,20],[0,20],'k')
 axes[0].grid(True)
 axes[0].set_xlim(0,avg_sales_max)
 axes[0].set_ylim(0,avg_sales_seniority_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_over_sold',hue='month',ax=axes[1])
 axes[1].plot([0,20],[0,20],'k')
 axes[1].grid(True)
 axes[1].set_xlim(0,avg_sales_max)
 axes[1].set_ylim(0,avg_sales_over_sold_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales_over_sold',y=name+'_avg_sales_seniority_'+str(seniority),hue='month',ax=axes[2])
 axes[2].plot([0,20],[0,20],'k')
 axes[2].grid(True)
 axes[2].set_xlim(0,avg_sales_over_sold_max)
 axes[2].set_ylim(0,avg_sales_seniority_max)
 else:
fig,axes=plt.subplots(n,3,figsize=(15,n*5))
for i,label in enumerate(labels):
sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_seniority_'+str(seniority),hue=label,ax=axes[i,0])
 axes[i,0].plot([0,20],[0,20],'k')
 axes[i,0].grid(True)
 axes[i,0].set_xlim(0,avg_sales_max)
 axes[i,0].set_ylim(0,avg_sales_seniority_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_over_sold',hue=label,ax=axes[i,1])
 axes[i,1].plot([0,20],[0,20],'k')
 axes[i,1].grid(True)
 axes[i,1].set_xlim(0,avg_sales_max)
 axes[i,1].set_ylim(0,avg_sales_over_sold_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales_over_sold',y=name+'_avg_sales_seniority_'+str(seniority),hue=label,ax=axes[i,2])
 axes[i,2].plot([0,20],[0,20],'k')
 axes[i,2].grid(True)
 axes[i,2].set_xlim(0,avg_sales_over_sold_max)
 axes[i,2].set_ylim(0,avg_sales_seniority_max )
if predict_submission:
print('Reading test dataset')
 test = pd.read_json(".. /input/test.json")
 y_fin, X_fin_b, X_fin_img = create_dataset(test, False)
 print('Predicting')
 prediction = common_model.predict([X_fin_b, X_fin_img], verbose=1, batch_size=32)
 print('Submitting')
 submission = pd.DataFrame({'id': test["id"], 'is_iceberg': prediction.reshape(( prediction.shape[0])) })
 submission.to_csv("./submission.csv", index=False)
 print('Done' )<set_options>
loaded=%who_ls
loaded.append('loaded' )
%matplotlib inline
master = pd.read_csv(".. /input/invasive-species-monitoring/train_labels.csv")
 master.head()<define_variables>
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
img_path = ".. /input/invasive-species-monitoring/train/"
y = []
file_paths = []
for i in range(len(master)) :
file_paths.append(img_path + str(master.iloc[i][0])+'.jpg')
 y.append(master.iloc[i][1])
 y = np.array(y )<load_from_csv>
def moving_statistics_0(df,df_features,col_agg,col_avg,windowsize,func,suffix='',fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
df_features_lag=df.loc[:,col_agg]
tmp=df_features.loc[:,col_agg+col_lag]
for lag in range(0,windowsize):
df_features_lag=df_features_lag.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_avg=df.loc[:,col_agg]
for col in col_avg:
if func=='mean':
if suffix=='':
suffix='_movavg_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].mean(axis=1)
 elif func=='max':
if suffix=='':
suffix='_movmax_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].max(axis=1)
 elif func=='min':
if suffix=='':
suffix='_movmin_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].min(axis=1)
 elif func=='std':
if suffix=='':
suffix='_movstd_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].std(ddof=0,axis=1)
 elif func=='rsd':
if suffix=='':
suffix='_movrsd_'+str(windowsize)
 denom_avg=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].mean(axis=1)
 num_std=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].std(ddof=0,axis=1)
 df_features_avg[col+suffix]=num_std/denom_avg
df_features_avg.loc[num_std==0,col+suffix]=0
del denom_avg
del num_std
if fill_value is not None:
for col in col_lag:
df_features_avg[col+suffix].fillna(fill_value,inplace=True)
 return df_features_avg.drop(col_agg,axis=1)
 def categorical_variability_encoding(agg_label,name,windowsize_inner,suffix,fill_value=None,windowsize_outer=34):
df=train_X[['month_id','shop_id','item_id',agg_label,'item_quantity']]
tmp=moving_statistics_0(df,df,['month_id','shop_id','item_id'],['item_quantity'],windowsize_inner,'rsd',suffix,fill_value=None)
 tmp=pd.concat([df,tmp],axis=1,sort=False)
 tmp=tmp.groupby(['month_id',agg_label] ).agg({'item_quantity'+suffix:'mean'} ).rename({'item_quantity'+suffix:name+suffix},axis=1)
 tmp=moving_statistics(df,tmp.reset_index() ,['month_id',agg_label],[name+suffix],windowsize_outer,'mean','_',fill_value)
 tmp.rename({name+suffix+'_':name+suffix},axis=1,inplace=True)
 return tmp
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 img_path = ".. /input/invasive-species-monitoring/test/"
test_names = []
file_paths = []
for i in range(len(sample_submission)) :
test_names.append(sample_submission.iloc[i][0])
 file_paths.append(img_path + str(int(sample_submission.iloc[i][0])) +'.jpg')
 test_names = np.array(test_names )<randomize_order>
category_variability_rs=categorical_variability_encoding('item_category_id','category_semiannual_avg',3,'_recent_rsd',fill_value=None,windowsize_outer=6)
 category_variability_as=categorical_variability_encoding('item_category_id','category_semiannual_avg',12,'_annual_rsd',fill_value=None,windowsize_outer=6)
 train_X=pd.concat([train_X,category_variability_rs,category_variability_as],axis=1,sort=False)
 del category_variability_rs, category_variability_as
data_num = len(y)
 random_index = np.random.permutation(data_num)
 x_shuffle = []
y_shuffle = []
for i in range(data_num):
x_shuffle.append(x[random_index[i]])
 y_shuffle.append(y[random_index[i]])
 x = np.array(x_shuffle)
 y = np.array(y_shuffle )<data_type_conversions>
train_X.drop(train_X.loc[train_X['month_id']<18].index,axis=0,inplace=True )
x_train = x_train.astype('float32')
 x_test = x_test.astype('float32')
 x_train /= 255
x_test /= 255<choose_model_class>
train_0=train_X.loc[train_X['item_seniority']==0,:]
train_1=train_X.loc[train_X['item_seniority']==1,:]
train_2=train_X.loc[train_X['item_seniority']==2,:]
pd.DataFrame([train_X.count().rename('global'),train_0.count().rename('seniority_0'),train_1.count().rename('seniority_1'),train_2.count().rename('seniority_2')] ).T
img_rows, img_cols, img_channel = 224, 224, 3
base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(img_rows, img_cols, img_channel))<choose_model_class>
os.remove(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
train_0.to_pickle(os.path.join(DATA_FOLDER,'processed/train_0.pkl'))
train_1.to_pickle(os.path.join(DATA_FOLDER,'processed/train_1.pkl'))
train_2.to_pickle(os.path.join(DATA_FOLDER,'processed/train_2.pkl'))
add_model = Sequential()
add_model.add(Flatten(input_shape=base_model.output_shape[1:]))
add_model.add(Dense(256, activation='relu'))
add_model.add(Dense(1, activation='sigmoid'))
model = Model(inputs=base_model.input, outputs=add_model(base_model.output))
model.compile(loss='binary_crossentropy', optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
 model.summary()<train_model>
del train_X
del train_0
del train_1
del train_2
gc.collect()
batch_size = 32
epochs = 50
train_datagen = ImageDataGenerator(
 rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True)
 train_datagen.fit(x_train)
 history = model.fit_generator(
 train_datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
)<train_model>
reset_variable_space
print("Training loss: {:.2f} / Validation loss: {:.2f}".\
format(history.history['loss'][-1], history.history['val_loss'][-1]))
print("Training accuracy: {:.2f}% / Validation accuracy: {:.2f}%".\
format(100*history.history['acc'][-1], 100*history.history['val_acc'][-1]))<data_type_conversions>
train_0=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_0.pkl'))
print(train_0.info(null_counts=True))
test_images = test_images.astype('float32')
 test_images /= 255<predict_on_test>
split_col='month_id'
target_col='item_quantity'
raw_features=[]
label_encoding=['shop_id',
'item_id',
'item_category_id',
'item_supercategory_id',
'item_category_console_id',
'item_category_is_digital'
]
mapping_columns=[]
features_to_discard=['item_id',
'shop_id',
'item_supercategory_id',
'item_category_console_id',
'item_category_is_digital'
]
features_to_discard+=mapping_columns
raw_features+=['shop_months_since_opening']
raw_features+=['shop_opening']
raw_features+=['item_category_freq']
raw_features+=['item_supercategory_freq']
raw_features+=['item_category_console_freq']
raw_features+=['item_category_digital_freq']
raw_features+=['item_category_freq_in_seniority']
raw_features+=['item_supercategory_freq_in_seniority']
raw_features+=['item_category_console_freq_in_seniority']
raw_features+=['item_category_digital_freq_in_seniority']
raw_features+=['category_semiannual_avg_recent_rsd']
raw_features+=['category_semiannual_avg_annual_rsd']
train_0=train_0[[split_col]+label_encoding+mapping_columns+[target_col]+raw_features]
print(train_0.info(null_counts=True))
gc.collect()
predictions = model.predict(test_images )<save_to_csv>
shop_category_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_category_prices.pkl'))
category_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/category_prices.pkl'))
supercategory_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/supercategory_prices.pkl'))
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 for i, name in enumerate(test_names):
sample_submission.loc[sample_submission['name'] == name, 'invasive'] = predictions[i]
sample_submission.to_csv("submit.csv", index=False )<import_modules>
ts = time.time()
price_features_df=[]
price_features_df.append(moving_statistics(train_0,shop_category_prices,['month_id','shop_id','item_category_id'],['shop_category_price_median'],34,'mean',suffix='_absolute_mean'))
price_features_df.append(moving_statistics(train_0,shop_category_prices,['month_id','shop_id','item_category_id'],['shop_category_price_median'],3,'mean',suffix='_recent_mean'))
price_features_df.append(lag_features(train_0,shop_category_prices,['month_id','shop_id','item_category_id'],['shop_category_price_median'],[12]))
price_features_df.append(moving_statistics(train_0,category_prices,['month_id','item_category_id'],['category_price_min'],34,'min',suffix='_absolute_min'))
price_features_df.append(moving_statistics(train_0,category_prices,['month_id','item_category_id'],['category_price_max'],34,'max',suffix='_absolute_max'))
price_features_df.append(moving_statistics(train_0,category_prices,['month_id','item_category_id'],['category_price_min'],12,'min',suffix='_annual_min'))
price_features_df.append(moving_statistics(train_0,category_prices,['month_id','item_category_id'],['category_price_max'],12,'max',suffix='_annual_max'))
price_features_df.append(moving_statistics(train_0,category_prices,['month_id','item_category_id'],['category_price_median'],3,'mean',suffix='_recent_mean'))
price_features_df.append(lag_features(train_0,category_prices,['month_id','item_category_id'],['category_price_median'],[12]))
price_features_df.append(moving_statistics(train_0,category_prices,['month_id','item_category_id'],['category_price_median_compared_to_supercategory_price_median'],34,'mean',suffix='_absolute_mean'))
price_features_df.append(moving_statistics(train_0,supercategory_prices,['month_id','item_supercategory_id'],['supercategory_price_median'],3,'mean',suffix='_recent_mean'))
price_features_df.append(lag_features(train_0,supercategory_prices,['month_id','item_supercategory_id'],['supercategory_price_median'],[12]))
print('time : ' +str(time.time() - ts))
train_0=pd.concat([train_0]+price_features_df,axis=1,sort=False)
 del price_features_df
gc.collect()
print('time : ' +str(time.time() - ts))
train_0=downcast_dtypes(train_0)
 print('time : ' +str(time.time() - ts))
print(train_0.info(null_counts=True))
train_0
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
import datashader as ds
from datashader import transfer_functions as tf<load_from_disk>
del shop_category_prices
del category_prices
del supercategory_prices
gc.collect()
train = pd.read_json(".. /input/two-sigma-connect-rental-listing-inquiries/train.json")
 train = train.reset_index()
train.pop("index")
 train.shape<load_from_disk>
dfs=['shop_encoding',
'category_encoding',
'supercategory_encoding',
'shop_category_encoding',
'shop_supercategory_encoding'
]
for df in dfs:
exec(df+"=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+".pkl')) ")
 exec(df+"_seniority_0=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+"_seniority_0.pkl')) " )
test = pd.read_json(".. /input/two-sigma-connect-rental-listing-inquiries/test.json")
 test = test.reset_index()
test.pop("index");
test.shape<load_from_csv>
feature_correlation_analysis(train_0,supercategory_encoding,supercategory_encoding_seniority_0,'supercategory',['month_id','item_supercategory_id'],0)
 compare_features(supercategory_encoding,supercategory_encoding_seniority_0,'supercategory',['month_id','item_supercategory_id'],0,20,6,20 )
submission = pd.read_csv(".. /input/two-sigma-connect-rental-listing-inquiries/sample_submission.csv")
 submission.head()<concatenate>
feature_correlation_analysis(train_0,shop_supercategory_encoding,shop_supercategory_encoding_seniority_0,'shop_supercategory',['month_id','shop_id','item_supercategory_id'],0)
 compare_features(shop_supercategory_encoding,shop_supercategory_encoding_seniority_0,'shop_supercategory',['month_id','shop_id','item_supercategory_id'],0,20,20,20 )
train["train_test"] = 'train'
test["train_test"] = 'test'
train = pd.concat([train, test],axis=0,sort=True)
 train.shape<feature_engineering>
ts = time.time()
target_features_df=[]
col_labels=['shop_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(moving_statistics(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,[1]))
target_features_df.append(lag_features(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['supercategory_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(moving_statistics(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,[1]))
target_features_df.append(lag_features(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_supercategory_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding_seniority_0,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_0,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding_seniority_0,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_0,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding_seniority_0,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_0,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding_seniority_0,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_0,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,shop_supercategory_encoding_seniority_0,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_0,[1]))
target_features_df.append(lag_features(train_0,shop_supercategory_encoding_seniority_0,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_0,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['category_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(moving_statistics(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,[1]))
target_features_df.append(lag_features(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(moving_statistics(train_0,shop_category_encoding_seniority_0,['month_id','shop_id','item_category_id'],col_labels_seniority_0,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,shop_category_encoding_seniority_0,['month_id','shop_id','item_category_id'],col_labels_seniority_0,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,shop_category_encoding_seniority_0,['month_id','shop_id','item_category_id'],col_labels_seniority_0,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,shop_category_encoding_seniority_0,['month_id','shop_id','item_category_id'],col_labels_seniority_0,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,shop_category_encoding_seniority_0,['month_id','shop_id','item_category_id'],col_labels_seniority_0,[1]))
target_features_df.append(lag_features(train_0,shop_category_encoding_seniority_0,['month_id','shop_id','item_category_id'],col_labels_seniority_0,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_avg_sales_compared_to_category']
target_features_df.append(moving_statistics(train_0,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,[1]))
target_features_df.append(lag_features(train_0,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,[12]))
col_labels=['shop_supercategory_avg_sales_compared_to_supercategory']
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding,['month_id','shop_id','item_supercategory_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding,['month_id','shop_id','item_supercategory_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding,['month_id','shop_id','item_supercategory_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_0,shop_supercategory_encoding,['month_id','shop_id','item_supercategory_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,shop_supercategory_encoding,['month_id','shop_id','item_supercategory_id'],col_labels,[1]))
target_features_df.append(lag_features(train_0,shop_supercategory_encoding,['month_id','shop_id','item_supercategory_id'],col_labels,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(rational_fraction(train_0,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(rational_fraction(train_0,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['supercategory_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(rational_fraction(train_0,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales_compared_to_supercategory']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(moving_statistics(train_0,category_encoding,['month_id','item_category_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_0,category_encoding,['month_id','item_category_id'],col_labels,[1]))
target_features_df.append(lag_features(train_0,category_encoding,['month_id','item_category_id'],col_labels,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(rational_fraction(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,shop_encoding_seniority_0,['month_id','shop_id'],col_labels_seniority_0,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['supercategory_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(rational_fraction(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,supercategory_encoding_seniority_0,['month_id','item_supercategory_id'],col_labels_seniority_0,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales']
col_labels_seniority_0=[col+'_seniority_0' for col in col_labels]
target_features_df.append(rational_fraction(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_0,category_encoding_seniority_0,['month_id','item_category_id'],col_labels_seniority_0,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
print('time : ' +str(time.time() - ts))
train_0=pd.concat([train_0]+target_features_df,axis=1,sort=False)
 del target_features_df
gc.collect()
print('time : ' +str(time.time() - ts))
print(train_0.info(null_counts=True,verbose=True))
train_0
train["price_type"] = np.log(train["price"])/(train["bedrooms"]+1)
 train["room_sum"] = train["bedrooms"]+train["bathrooms"]
train["num_features"] = train["features"].apply(len)
 train["num_description_words"] = train["description"].apply(lambda x: len(x.split(" ")) )<count_unique_values>
for df in dfs:
exec("del "+df)
 exec("del "+df+"_seniority_0")
 gc.collect()
ft = []
train['features'].apply(lambda x: ft.append(x))
ft=list(it.chain.from_iterable(ft))
print("Nombre d'éléments: " + str(len(ft)))
 uniq_ft = set(ft)
 print("Nombre d'éléments uniques: " + str(len(uniq_ft)) )<load_from_csv>
train_0.drop(features_to_discard,axis=1,inplace=True)
 train_0.info(null_counts=True,verbose=True )
image_timestamp = pd.read_csv('.. /input/hugoboum-renthop-image-timestamp/listing_image_time.csv')
 image_timestamp.columns = ['listing_id', 'timestamp']
image_timestamp.head()<merge>
create_directory(os.path.join(DATA_FOLDER, 'training'))
train_0.to_pickle(os.path.join(DATA_FOLDER,'training/train_0_pred.pkl'))
train = pd.merge(train,image_timestamp,on='listing_id',how='left' )<define_variables>
del train_0
gc.collect()
latitude_median = np.median(train["latitude"])
 longitude_median = np.median(train["longitude"])
 print(latitude_median)
 print(longitude_median )<feature_engineering>
reset_variable_space
train['latitude'].values[train['latitude'] < 39] = latitude_median
train['latitude'].values[train['latitude'] > 41] = latitude_median
train['longitude'].values[train['longitude'] <-80] = longitude_median
train['longitude'].values[train['longitude'] > -70] = longitude_median
print('Valeurs abberantes ')
 print(train.query("latitude == 0")["latitude"].agg("count"))
print(train.query("longitude == 0")["longitude"].agg("count"))<compute_train_metric>
train_1=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_1.pkl'))
print(train_1.info(null_counts=True))
jb = jarque_bera(np.log(train["price"]))
print("p-value Jarque-Bera normality test: " + str(jb[1]))<groupby>
split_col='month_id'
target_col='item_quantity'
raw_features=[]
label_encoding=['shop_id',
'item_id',
'item_category_id',
'item_supercategory_id',
'item_category_console_id',
'item_category_is_digital'
]
mapping_columns=['item_month_id_of_last_sale',
'item_month_id_of_release',
'item_month_of_release'
]
features_to_discard=['item_id',
'shop_id',
'item_supercategory_id',
'item_category_console_id',
'item_category_is_digital'
]
features_to_discard+=mapping_columns
raw_features+=['shop_months_since_opening']
raw_features+=['shop_opening']
raw_features+=['item_freq_in_seniority']
raw_features+=['item_category_freq']
raw_features+=['item_supercategory_freq']
raw_features+=['item_category_console_freq']
raw_features+=['item_category_digital_freq']
raw_features+=['item_category_freq_in_seniority']
raw_features+=['item_supercategory_freq_in_seniority']
raw_features+=['item_category_console_freq_in_seniority']
raw_features+=['item_category_digital_freq_in_seniority']
raw_features+=['category_semiannual_avg_recent_rsd']
raw_features+=['category_semiannual_avg_annual_rsd']
raw_features+=['item_months_since_release']
train_1=train_1[[split_col]+label_encoding+mapping_columns+[target_col]+raw_features]
print(train_1.info(null_counts=True))
gc.collect()
train.groupby("price_cat" ).agg("count" ).iloc[:,1]<data_type_conversions>
item_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/item_prices.pkl'))
category_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/category_prices.pkl'))
train["price_cat"] = train["price_cat"].astype("str" )<categorify>
ts = time.time()
price_features_df=[]
col_labels=['item_price_median']
col_labels+=['item_price_median_compared_to_category_price_median']
price_features_df.append(time_mapping(train_1,item_prices,['item_id'],col_labels,'item_month_id_of_last_sale',suffix='_last_sale'))
price_features_df.append(moving_statistics(train_1,item_prices,['month_id','item_id'],col_labels,34,'mean',suffix='_absolute_mean'))
time_window=34
price_features_df.append(moving_statistics(train_1,item_prices,['month_id','item_id'],['item_price_min'],34,'min',suffix='_absolute_min'))
price_features_df.append(moving_statistics(train_1,item_prices,['month_id','item_id'],['item_price_max'],34,'max',suffix='_absolute_max'))
print('time : ' +str(time.time() - ts))
price_features_df.append(moving_statistics(train_1,category_prices,['month_id','item_category_id'],['category_price_median'],3,'mean',suffix='_recent_mean'))
print('time : ' +str(time.time() - ts))
train_1=pd.concat([train_1]+price_features_df,axis=1,sort=False)
 del price_features_df
gc.collect()
print('time : ' +str(time.time() - ts))
train_1=downcast_dtypes(train_1)
 print('time : ' +str(time.time() - ts))
print(train_1.info(null_counts=True))
train_1
le = preprocessing.LabelEncoder()
train["price_cat"] = le.fit_transform(train["price_cat"] )<groupby>
del item_prices
del category_prices
gc.collect()
streets = train.groupby(train["display_address"] ).agg(["count"] ).iloc[:,1]<concatenate>
dfs=['shop_encoding',
'item_encoding',
'category_encoding',
'supercategory_encoding',
'shop_category_encoding',
'shop_supercategory_encoding',
'category_months_since_release_encoding',
'supercategory_months_since_release_encoding',
'category_month_of_release_encoding',
'supercategory_month_of_release_encoding',
'shop_category_months_since_release_encoding',
'shop_supercategory_months_since_release_encoding'
]
for df in dfs:
exec(df+"=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+".pkl')) ")
 exec(df+"_seniority_1=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+"_seniority_1.pkl')) " )
k=100
topstreets = streets.sort_values(ascending=False ).iloc[:k]
topstreets = pd.DataFrame(topstreets)
 topstreets["street"] = topstreets.index
topstreets = topstreets.unstack().iloc[:k]
topstreets = topstreets.reset_index().drop(["level_0","level_1"],axis=1)
 topstreets.columns = ["display_address","display_address_count"]
topstreets.head()<merge>
feature_correlation_analysis(train_1,item_encoding,item_encoding_seniority_1,'item',['month_id','item_id'],1)
 compare_features(item_encoding,item_encoding_seniority_1,'item',['month_id','item_id'],1,20,20,20 )
train = pd.merge(train,topstreets,on='display_address',how='left' )<concatenate>
feature_correlation_analysis(train_1,supercategory_encoding,supercategory_encoding_seniority_1,'supercategory',['month_id','item_supercategory_id'],1)
 compare_features(supercategory_encoding,supercategory_encoding_seniority_1,'supercategory',['month_id','item_supercategory_id'],1,20,3,20 )
managers = train.groupby(train["manager_id"] ).agg(["count"] ).iloc[:,1]
managers = pd.DataFrame(managers)
 managers = managers.unstack()
managers["manager_id"] = managers.index
managers = managers.reset_index().drop(["level_0","level_1"],axis=1)
 managers.columns = ["manager_id","manager_id_count"]
managers = managers.iloc[:-1,:]
managers.head()<merge>
feature_correlation_analysis(train_1,shop_supercategory_encoding,shop_supercategory_encoding_seniority_1,'shop_supercategory',['month_id','shop_id','item_supercategory_id'],1)
 compare_features(shop_supercategory_encoding,shop_supercategory_encoding_seniority_1,'shop_supercategory',['month_id','shop_id','item_supercategory_id'],1,20,20,20 )
train = pd.merge(train,managers,on='manager_id',how='left' )<data_type_conversions>
feature_correlation_analysis(train_1,category_months_since_release_encoding,category_months_since_release_encoding_seniority_1,'category_months_since_release',['month_id','item_category_id','item_months_since_release'],1)
 compare_features(category_months_since_release_encoding,category_months_since_release_encoding_seniority_1,'category_months_since_release',['month_id','item_category_id','item_months_since_release'],1,20,20,20 )
train["created"] = train["created"].astype("datetime64" )<feature_engineering>
feature_correlation_analysis(train_1,shop_category_months_since_release_encoding,shop_category_months_since_release_encoding_seniority_1,'shop_category_months_since_release',['month_id','shop_id','item_category_id','item_months_since_release'],1 )