Kaggle competition: Predict Future Sales
gc.collect()<load_pretrained>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from itertools import product
from sklearn.feature_extraction.text import TfidfVectorizer
import re
from xgboost import XGBRegressor
from sklearn.ensemble import RandomForestClassifier
from xgboost import plot_importance
x_test = np.load(tokenized_path+'x_test.npy',mmap_mode='r')
 test_ids = np.load(tokenized_path+'test_df_ids.npy',mmap_mode='r' )<create_dataframe>
train = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 item_category = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv")
 item = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shop = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv" )
test_dataset = ArrayDataset(test_ids,x_test )<load_pretrained>
trained_state_dict = torch.load("xlm_roberta_model.bin")
 mx.load_state_dict(trained_state_dict)
 del trained_state_dict
gc.collect()<save_to_csv>
def gather_submissions() :
ids = []
outputs = []
for core in range(8):
ids.extend(np.load(f'ids_core{core}.npy'))
outputs.extend(np.load(f'outputs_core{core}.npy'))
submission_df = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 print(len(ids))
submission_df['toxic'].iloc[ids] = np.array(outputs)[:,0]
submission_df.to_csv('submission.csv',index=False )<categorify>
def test_loop_fn(data_loader, model, device):
fin_id = []
fin_outputs = []
for bi, d in enumerate(data_loader):
text_id = d[0]
ids = d[1]
ids = ids.to(device, dtype=torch.long)
 with torch.no_grad() :
outputs = model(
 input_ids=ids, 
)
 outputs_cpu = outputs.data.cpu()
fin_id.append(text_id)
 fin_outputs.append(outputs_cpu)
 del text_id, ids, outputs_cpu
gc.collect()
xm.master_print('done with inference, saving...')
 fin_id_cat = torch.cat(fin_id)
 fin_outputs_cat = torch.cat(fin_outputs)
 np.save(f'ids_core{xm.get_ordinal() }.npy',fin_id_cat.cpu().numpy())
 np.save(f'outputs_core{xm.get_ordinal() }.npy',fin_outputs_cat.numpy())
<load_pretrained>
def _run_test() :
BATCH_SIZE=64
test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
drop_last=False,
num_workers=0, 
)
 device = xm.xla_device()
model = mx.to(device)
 xm.master_print('done loading model')
 para_loader = pl.ParallelLoader(test_data_loader, [device])
 test_loop_fn(para_loader.per_device_loader(device),model,device)
<set_options>
train['date'] = pd.to_datetime(train['date'],format = '%d.%m.%Y' )
gc.collect()<save_to_csv>
train[train.item_price > 100000]
sub = pd.read_csv('submission.csv')
 test = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub.loc[test["lang"] == "es", "toxic"] *= 1.06
sub.loc[test["lang"] == "fr", "toxic"] *= 1.04
sub.loc[test["lang"] == "it", "toxic"] *= 0.97
sub.loc[test["lang"] == "pt", "toxic"] *= 0.96
sub.loc[test["lang"] == "tr", "toxic"] *= 0.98
sub.toxic -= sub.toxic.min()
sub.toxic /= sub.toxic.max()
sub.toxic.hist(bins=100, log=True, alpha=0.6)
 sub.to_csv('submission.csv', index=False )<load_from_csv>
train[train.item_price < 0]
%matplotlib inline
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<groupby>
train[train.item_cnt_day >= 1000]
train.groupby('sub_area' ).sub_area.count()<groupby>
train[train.item_cnt_day < 0]
train.groupby('ecology' ).sub_area.count()<groupby>
shop["city"] = shop.shop_name.str.split(" " ).map(lambda x: x[0])
 shop["type"] = shop.shop_name.str.split(" " ).map(lambda x: x[1] )
train.groupby('state' ).sub_area.count()<sort_values>
shop["shop_type"] = LabelEncoder().fit_transform(shop.type)
 shop["shop_city"] = LabelEncoder().fit_transform(shop.city)
 shop.head()
Data_stat=train.groupby('state' ).sub_area.count().sort_values()
Data_stat<count_missing_values>
item_category['item_type'] = item_category['item_category_name'].str.split('-' ).map(lambda x: x[0])
 item_category['item_name'] = item_category['item_category_name'].str.split('-' ).map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 item_category.head()
train=train.fillna(-999)
 pd.isnull(train ).any()<count_missing_values>
item_category['item_type_code'] = LabelEncoder().fit_transform(item_category.item_type)
 item_category['item_name_code'] = LabelEncoder().fit_transform(item_category.item_name)
 item_category.head()
pd.isnull(test ).any()<correct_missing_values>
item_category = item_category[['item_category_id', 'item_type_code', 'item_name_code']]
item_category.head()
test=test.fillna(-999 )<prepare_x_and_y>
def clean_text(item_name):
item_name = item_name.lower()
item_name = re.sub(r'[^\w\s]', '', item_name)
 item_name = re.sub(r'\d+', '', item_name)
 item_name = re.sub(' +', ' ', item_name)
 item_name = item_name.strip()
return item_name
y_train = train["price_doc"]
x_train = train.drop(["id", "price_doc","timestamp" ], axis=1)
 x_test = test.drop(["id","timestamp"], axis=1)
 id_test = test['id']<count_values>
item['clean_item_name'] = item['item_name'].apply(clean_text)
 item.head()
x_train.dtypes.value_counts()<categorify>
item['sub_name1'] = item['clean_item_name'].str.split(' ' ).map(lambda x: x[0])
 item['sub_name2'] = item['clean_item_name'].str.split(' ' ).map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())

item.head()
for c in x_train.columns:
if x_train[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_train[c].values))
x_train[c] = lbl.transform(list(x_train[c].values)) 
<categorify>
item.sub_name1.unique().shape, item.sub_name2.unique().shape
for c in x_test.columns:
if x_test[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_test[c].values))
x_test[c] = lbl.transform(list(x_test[c].values)) 
<train_model>
item.item_id.unique().shape, item.item_category_id.unique().shape
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
<train_model>
item['sub_name_1'] = LabelEncoder().fit_transform(item.sub_name1)
 item['sub_name_2'] = LabelEncoder().fit_transform(item.sub_name2)

item.head()
num_boost_rounds = 384
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round= num_boost_rounds )<save_to_csv>
len(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)) )
y_predict = model.predict(dtest)
 output = pd.DataFrame({'id': id_test, 'price_doc': y_predict})
 output.head()
output.to_csv('xgbSb.csv', index=False)
<load_from_csv>
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True )
pd.read_csv('.. /output/xgbSb.csv' )<load_from_csv>
train['revenue'] = train['item_price'] * train['item_cnt_day']
df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 df2 = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print(df.shape)
 print(df2.shape )<normalization>
cols = ['date_block_num','shop_id','item_id']
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True )
scaler = MinMaxScaler()
x = scaler.fit_transform(x)
 x<normalization>
matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.astype(np.float32)
.fillna(0))
x1 = np.array(df2)
 x1 = scaler.fit_transform(x1)
 x1 = x1.reshape(( 28000,28,28,1))
x1.shape 
<categorify>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 test.head()
y = np.array(y)
 enc = OneHotEncoder(sparse=False)
 y= y.reshape(( -1,1))
y = enc.fit_transform(y)
 y.shape<train_model>
matrix = pd.concat([matrix, test], ignore_index = True, keys = cols, sort = False)
 matrix.fillna(0, inplace = True)
 matrix.head()
x_train,x_test,y_train,y_test = tts(x,y,test_size = 0.2, random_state=42)
 print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape)
 print(y_test.shape )<import_modules>
matrix = pd.merge(matrix, shop, on = 'shop_id', how = 'left')
 matrix = pd.merge(matrix, item, on = 'item_id', how = 'left')
 matrix = pd.merge(matrix, item_category, on = 'item_category_id', how = 'left')
 matrix.head()
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau<choose_model_class>
matrix.drop(['ID'], axis=1, inplace=True)
 matrix.head()
model = keras.Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.summary()<init_hyperparams>
matrix['shop_city'] = matrix['shop_city'].astype(np.int8)
 matrix['shop_type'] = matrix['shop_type'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['sub_name_1'] = matrix['sub_name_1'].astype(np.int16)
 matrix['sub_name_2'] = matrix['sub_name_2'].astype(np.int16)

matrix['item_type_code'] = matrix['item_type_code'].astype(np.int8)
 matrix['item_name_code'] = matrix['item_name_code'].astype(np.int8)
 matrix.dtypes
callbacks = [
keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5,
patience=25,
verbose=1)
 ]<predict_on_test>
matrix['month'] = matrix['date_block_num'] % 12
matrix.head()
predictions=model.predict(x1)
 pre=predictions.argmax(axis=-1 )<save_to_csv>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8)
 matrix.head()
submission = pd.Series(pre,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),submission],axis = 1)
 submission.to_csv("final_submission_v1.csv",index=False)
 submission.head()<import_modules>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted[col+'_lag_'+str(i)] = shifted[col+'_lag_'+str(i)].astype(np.float16)
 shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
matrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month')
 matrix.head()
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras import layers, Sequential, optimizers
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.utils import to_categorical
import matplotlib.pyplot as plt<set_options>
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 matrix.head(10 )
print("GPUs Available: ", tf.config.experimental.list_physical_devices('GPU'))<load_from_csv>
raw_csv = "/kaggle/input/digit-recognizer/train.csv"
test_csv = "/kaggle/input/digit-recognizer/test.csv"<load_from_csv>
group = matrix.groupby(['date_block_num','shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_avg_item_cnt')
 matrix.head(10 )
raw_df = pd.read_csv(raw_csv)
 test_df = pd.read_csv(test_csv )<create_dataframe>
def get_image_and_label(data_frame):
IMGs = data_frame.drop(["label"], axis=1 ).values if 'label' in data_frame.columns else data_frame.values
IMGs = np.array([image.reshape(( 28, 28)) for image in IMGs])
 IMGs = np.expand_dims(IMGs, axis=3)
 labels = data_frame['label'].values if 'label' in data_frame.columns else None
return IMGs, labels<categorify>
group = matrix.groupby(['date_block_num','item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_avg_item_cnt')
 matrix.head(10 )
raw_IMGs, raw_labels = get_image_and_label(raw_df)
 test_IMGs, _ = get_image_and_label(test_df )<define_variables>
classes = len(set(raw_labels))
classes<categorify>
raw_labels = to_categorical(raw_labels, num_classes=classes)
 raw_labels<split>
train_IMGs, validation_IMGs, trian_labels, validation_labels = train_test_split(raw_IMGs, raw_labels, test_size=0.1, random_state=42 )<choose_model_class>
model = Sequential([
layers.Conv2D(32,(3,3), activation="relu", input_shape=(28,28,1)) ,
layers.BatchNormalization() ,
layers.MaxPooling2D(( 2,2)) ,
layers.Conv2D(64,(3,3), activation="relu"),
layers.BatchNormalization() ,
layers.MaxPooling2D(( 2,2)) ,
layers.Conv2D(128,(3,3), activation="relu"),
layers.BatchNormalization() ,
layers.Flatten() ,
layers.Dense(1024, activation="relu"),
layers.Dropout(0.2),
layers.Dense(256, activation="relu"),
layers.Dropout(0.2),
layers.Dense(64, activation="relu"),
layers.Dropout(0.2),
layers.Dense(32, activation="relu"),
layers.Dropout(0.2),
layers.Dense(16, activation="relu"),
layers.Dropout(0.2),
layers.Dense(int(classes), activation="softmax")
 ] )<choose_model_class>
model.compile(loss="categorical_crossentropy",
optimizer=optimizers.Adam(learning_rate=1e-4),
metrics=['accuracy'] )<define_variables>
train_datagen = ImageDataGenerator(
 rescale=1/255,
rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
shear_range=0.1 
)
 validation_datagen = ImageDataGenerator(rescale=1/255)
 test_datagen = ImageDataGenerator(rescale=1/255 )<define_variables>
train_generator = train_datagen.flow(train_IMGs, trian_labels, batch_size=32)
 validation_generator = train_datagen.flow(validation_IMGs, validation_labels, batch_size=32)
 test_generator = test_datagen.flow(test_IMGs, batch_size=32, shuffle=False )<train_model>
history = model.fit_generator(train_generator, epochs=100, validation_data=validation_generator, verbose=1 )<compute_test_metric>
model.evaluate(validation_IMGs, validation_labels )<find_best_params>
accuracy = history.history["accuracy"]
val_accuracy = history.history["val_accuracy"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(len(accuracy))<predict_on_test>
pred_labels = model.predict_generator(test_generator )<prepare_output>
X_train = matrix[matrix.date_block_num <= 33].drop(['item_cnt_month'], axis=1)
 Y_train = matrix[matrix.date_block_num <= 33]['item_cnt_month']
X_valid = matrix[matrix.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = matrix[matrix.date_block_num == 33]['item_cnt_month']
X_test = matrix[matrix.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
pred_labels = np.argmax(pred_labels, axis=-1)
 pred_labels<prepare_output>
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10 )
my_submission = pd.DataFrame({'ImageId': test_df.index + 1, 'Label': pred_labels})
 my_submission.head()<save_to_csv>
Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('submission.csv', index=False )