Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<merge>
! pip install git+https://www.github.com/keras-team/keras-contrib.git
! wget https://github.com/jganzabal/santander_kaggle_solutions_tests/raw/master/santander_helper.py
train=train.join(total_cnt_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_item_categories"]=train["total_cnt_for_item_categories"].astype(np.float16)
 train.head().T<drop_column>
warnings.filterwarnings("ignore" )
%%time
train,n_col=create_lag_features(train,"total_cnt_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_item_categories"],axis=1)
 numeric=numeric+n_col<groupby>
test_path = '.. /input/test.csv'
train_path = '.. /input/train.csv'
df_test = pd.read_csv(test_path)
 df_test.drop(['ID_code'], axis=1, inplace=True)
 df_test = df_test.values
unique_samples = []
unique_count = np.zeros_like(df_test)
 for feature in range(df_test.shape[1]):
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print('Found',len(real_samples_indexes),'real test')
 print('Found',len(synthetic_samples_indexes),'fake test')
 d = {}
for i in range(200): d['var_'+str(i)] = 'float32'
d['target'] = 'uint8'
d['ID_code'] = 'object'
train = pd.read_csv('.. /input/train.csv', dtype=d)
 test = pd.read_csv('.. /input/test.csv', dtype=d)
 print('Loaded',len(train),'rows of train')
 print('Loaded',len(test),'rows of test')
 print('Found',len(real_samples_indexes),'real test')
 print('Found',len(synthetic_samples_indexes),'fake test')
 d = {}
for i in range(200): d['var_'+str(i)] = 'float32'
d['target'] = 'uint8'
d['ID_code'] = 'object'
train = pd.read_csv(train_path, dtype=d)
 test = pd.read_csv(test_path, dtype=d)
 print('Loaded',len(train),'rows of train')
 print('Loaded',len(test),'rows of test' )
mean_price=sales_train_df[["date_block_num","shop_id","item_id","item_price"]].groupby(["date_block_num","shop_id","item_id"] ).mean()
mean_price.columns=["mean_price"]
mean_price.head()<merge>
def encode_FE(df,col,test):
cv = df[col].value_counts()
nm = col+'_FE'
df[nm] = df[col].map(cv)
 test[nm] = test[col].map(cv)
 test[nm].fillna(0,inplace=True)
 if cv.max() <=255:
df[nm] = df[nm].astype('uint8')
 test[nm] = test[nm].astype('uint8')
 else:
df[nm] = df[nm].astype('uint16')
 test[nm] = test[nm].astype('uint16')
 return
test['target'] = -1
comb = pd.concat([train,test.loc[real_samples_indexes]],axis=0,sort=True)
 for i in range(200):
encode_FE(comb,'var_'+str(i),test)
 train = comb[:len(train)]; del comb
print('Added 200 new magic features!' )
train=train.join(mean_price,on=["date_block_num","shop_id","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price"]=train["mean_price"].astype(np.float16)
 train.head().T<drop_column>
del df_test, real_samples_indexes, synthetic_samples_indexes, unique_count, unique_samples, d
gc.collect()
%%time
train,n_col=create_lag_features(train,"mean_price")
 train=train.fillna(0)
 train=train.drop(["mean_price"],axis=1)
 numeric=numeric+n_col<groupby>
df_train_data = train.drop(columns=['ID_code'])
 y = df_train_data['target'].values
df_train_X = df_train_data.drop(columns=['target'] )
mean_price_for_items=sales_train_df[["date_block_num","item_id","item_price"]].groupby(["date_block_num","item_id"] ).mean()
mean_price_for_items.columns=["mean_price_for_items"]
mean_price_for_items.head()<merge>
reverse_columns = True
if reverse_columns:
reverse_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 15, 16, 18, 19, 22, 24, 25, 26,
27, 29, 32, 35, 37, 40, 41, 47, 48, 49, 51, 52, 53, 55, 60, 61,
62, 65, 66, 67, 69, 70, 71, 74, 78, 79, 82, 84, 89, 90, 91, 94,
95, 96, 97, 99, 103, 105, 106, 110, 111, 112, 118, 119, 125, 128,
130, 133, 134, 135, 137, 138, 140, 144, 145, 147, 151, 155, 157,
159, 161, 162, 163, 164, 167, 168, 170, 171, 173, 175, 176, 179,
180, 181, 184, 185, 187, 189, 190, 191, 195, 196, 199,
]
for j in reverse_list:
df_train_X[f'var_{j}'] *= -1
train=train.join(mean_price_for_items,on=["date_block_num","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_items"]=train["mean_price_for_items"].astype(np.float16)
 train.head().T<drop_column>
means = df_train_X.mean(axis=0)
 stds = df_train_X.std(axis=0)
 df_train_X_normalized =(df_train_X - means)/stds
%%time
train,n_col=create_lag_features(train,"mean_price_for_items")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_items"],axis=1)
 numeric=numeric+n_col<groupby>
X_train_normalized = np.zeros(( df_train_X_normalized.shape[0], 400, 1))
for i in range(200):
X_train_normalized[:, 2*i] = df_train_X_normalized[[f'var_{i}']].values
X_train_normalized[:, 2*i+1] = df_train_X_normalized[[f'var_{i}_FE']].values
mean_price_for_item_categories=sales_train_df[["date_block_num","item_category_id","item_price"]].groupby(["date_block_num","item_category_id"] ).mean()
mean_price_for_item_categories.columns=["mean_price_for_item_categories"]
mean_price_for_item_categories.head()<merge>
def get_model(N_units = 600, kernel_size=2, strides=2):
model = Sequential()
model.add(Conv1D(N_units, kernel_size=kernel_size, strides=strides, padding='valid',
activation='relu', input_shape=(X_train_normalized.shape[1], 1)))
 model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
return model
train=train.join(mean_price_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_item_categories"]=train["mean_price_for_item_categories"].astype(np.float16)
 train.head().T<drop_column>
del df_train_data, df_train_X
gc.collect()
%%time
train,n_col=create_lag_features(train,"mean_price_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_item_categories"],axis=1)
 numeric=numeric+n_col<feature_engineering>
df_test = test.drop(columns=['ID_code', 'target'])
 if reverse_columns:
for j in reverse_list:
df_test[f'var_{j}'] *= -1
df_test_X_normalized =(df_test - means)/stds
sales_train_df["revenue"]=sales_train_df["item_price"]*sales_train_df["item_cnt_day"]
total_revenue=sales_train_df[["date_block_num","shop_id","revenue"]].groupby(["date_block_num","shop_id"] ).sum()
total_revenue.columns=["total_shop_revenue"]
total_revenue.head()<merge>
X_test_normalized = np.zeros(( df_test_X_normalized.shape[0], 400, 1))
for i in range(200):
X_test_normalized[:, 2*i] = df_test_X_normalized[[f'var_{i}']].values
X_test_normalized[:, 2*i+1] = df_test_X_normalized[[f'var_{i}_FE']].values
train=train.join(total_revenue,on=["date_block_num","shop_id"],how="left")
 train["total_shop_revenue"]=train["total_shop_revenue"].astype(np.float32)
 train.fillna(0,inplace=True)
 train.head().T<drop_column>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 full_val_preds = np.zeros(( len(df_train_X_normalized), 1))
model = get_model(N_units)
 test_predictions = 0
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train_X_normalized, y)) :
print('
print(f'
print('
X_train = X_train_normalized[trn_idx]
X_val = X_train_normalized[val_idx]
filename = f'CNN_generator_fold_{fold}_cl1_{class_1_aug}_cl0_{class_0_aug}_{N_units}.hdf5'
model.load_weights(filename)
 full_val_preds[val_idx] = model.predict(X_val, verbose=1)
 print(roc_auc_score(y[val_idx], full_val_preds[val_idx]))
test_predictions = test_predictions + model.predict(X_test_normalized, verbose=1)/5
%%time
train,n_col=create_lag_features(train,"total_shop_revenue")
 train=train.fillna(0.0)
 train=train.drop(["total_shop_revenue"],axis=1)
 numeric=numeric+n_col<drop_column>
roc_auc_score(y, full_val_preds )
numeric=numeric+["date_block_num","item_cnt_day"]<data_type_conversions>
def save_submit_file(predictions, filename, test_filename=test_path, index_column='ID_code', target_column = 'target'):
df_test_submit = pd.read_csv(test_filename ).set_index(index_column)
 df_test_submit[target_column] = predictions
df_test_submit[[target_column]].to_csv(filename)
 return
X_train_categorical=train[categorical]
X_test_categorical=test_df[categorical]
X_train_categorical["subcategory"]=X_train_categorical["subcategory"].astype(str)
 X_test_categorical["subcategory"]=X_test_categorical["subcategory"].astype(str)
 X_test_categorical["year"]=X_test_categorical["year"].astype(np.int16)
 X_test_categorical["month"]=X_test_categorical["month"].astype(np.int8)
 X_test_categorical.loc[X_test_categorical.category=="PC ","category"]="Игры PC "
X_test_categorical.loc[X_test_categorical.subcategory==" Гарнитуры/Наушники","subcategory"]=" Аксессуары для игр"<categorify>
save_submit_file(test_predictions,
f'submit_CNN_generator_cl1_{class_1_aug}_cl0_{class_0_aug}_{N_units}.csv' )