Kaggle competition: Tobigs13_7week_competition
for cat in shops.category.unique() :
print(cat, len(shops[shops.category == cat]))<categorify>
sample_submission = pd.read_csv(".. /input/tobigs13nn/sample_submission.csv")
 test = pd.read_csv(".. /input/tobigs13nn/test_df.csv")
 train = pd.read_csv(".. /input/tobigs13nn/train_df.csv")
 print(f"Train data shape {train.shape}")
 print(f"Test data shape {test.shape}" )
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city )<define_variables>
X = train.iloc[:,1:].values
y = train.iloc[:,0].values
shops = shops[["shop_id", "shop_category", "shop_city"]]<feature_engineering>
y = tf.keras.utils.to_categorical(y)
 y.shape
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"<categorify>
X = X / 255
X
cats.type_code = cats.type_code.apply(lambda x: x if(x in category)else "etc" )<count_unique_values>
x_train,x_val,y_train,y_val = train_test_split(X, y, train_size=0.95,random_state=10 )
for cat in cats.type_code.unique() :
print(cat, len(cats[cats.type_code == cat]))<categorify>
print(x_train.shape, y_train[0].argmax())
 print(y_train[0])
 plt.imshow(x_train[0].reshape(28, 28, 1)[:,:,0] )
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
 cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats = cats[["item_category_id", "subtype_code", "type_code"]]<categorify>
num_calsses = 10
model = tf.keras.models.Sequential([
tf.keras.layers.Dense(256,input_shape=(784,)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128,activation="selu",bias_initializer=tf.keras.initializers.he_normal(seed=None)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(64,activation="selu",bias_initializer=tf.keras.initializers.he_normal(seed=None)) ,
tf.keras.layers.Dense(num_calsses,activation="softmax")
 ])
 model.compile(optimizer="adam",loss="categorical_crossentropy",metrics=['accuracy'])
 model.summary()
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
history = model.fit(x_train,y_train,
batch_size=512,
epochs=5,
validation_data=[x_val,y_val] )
items = pd.read_csv(items_path)
 items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )<feature_engineering>
x_test = test.iloc[:,1:].values
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3" 
<groupby>
x_test = x_test / 255
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 print(group_sum.reset_index())
 group_sum = group_sum.reset_index()<drop_column>
predictions = model.predict_classes(x_test )
items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )<categorify>
sample_submission['Category'] = pd.Series(predictions)
 sample_submission.head()
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()<data_type_conversions>
sample_submission.to_csv("submission.csv",index=False )
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts<feature_engineering>
def cv_ensemble(X, y, x_test):
preds = []
folds = KFold(n_splits=10, shuffle=True, random_state=2020)
 for train_idx, val_idx in folds.split(X):
x_train, x_val = X[train_idx], X[val_idx]
y_train, y_val = y[train_idx], y[val_idx]
num_calsses = 10
model = tf.keras.models.Sequential([
tf.keras.layers.Dense(256,input_shape=(784,)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128,activation="selu",bias_initializer=tf.keras.initializers.he_normal(seed=None)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(64,activation="selu",bias_initializer=tf.keras.initializers.he_normal(seed=None)) ,
tf.keras.layers.Dense(num_calsses,activation="softmax")
 ])
 model.compile(optimizer="adam",loss="categorical_crossentropy",metrics=['accuracy'])
 model.fit(x_train,y_train,
batch_size=512,
epochs=30,
validation_data=[x_val,y_val])
 predictions = model.predict(x_test)
 preds.append(predictions)
 return preds
train["revenue"] = train["item_cnt_day"] * train["item_price"]<merge>
preds = cv_ensemble(X, y, x_test )
ts = time.time()
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).clip(0,20 ).astype(np.float16)
 time.time() - ts 
<data_type_conversions>
pred = np.mean(preds,axis=0)
 print(pred.shape )
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16)
<concatenate>
sample_submission['Category'] = pd.Series(predictions)
 sample_submission.head()
ts = time.time()
matrix = pd.concat([matrix, test.drop(["ID"],axis = 1)], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace = True)
 time.time() - ts<data_type_conversions>
sample_submission.to_csv("submission_cv.csv",index=False )