Kaggle competition: Eval Lab 2 F464
<SOS> metric: categorizationaccuracy Kaggle data source: eval-lab-2-f464<feature_engineering>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
shops.category = shops.category.apply(lambda x: x if(x in category)else "etc" )<count_unique_values>
dataset = pd.read_csv('.. /input/eval-lab-2-f464/train.csv' )
for cat in shops.category.unique() :
print(cat, len(shops[shops.category == cat]))<categorify>
test = pd.read_csv('.. /input/eval-lab-2-f464/test.csv' )
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city )<define_variables>
dataset.isnull().sum()
shops = shops[["shop_id", "shop_category", "shop_city"]]<feature_engineering>
dataset.drop(['id'], axis = 1, inplace = True )
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"<categorify>
X = dataset[dataset.drop(['class'], axis = 1 ).columns].values
y = dataset['class'].values
cats.type_code = cats.type_code.apply(lambda x: x if(x in category)else "etc" )<count_unique_values>
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size = 20/120 )
for cat in cats.type_code.unique() :
print(cat, len(cats[cats.type_code == cat]))<categorify>
clf = DecisionTreeClassifier(random_state = 42)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
 cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats = cats[["item_category_id", "subtype_code", "type_code"]]<categorify>
clf = RandomForestClassifier(random_state = 42)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
clf = GradientBoostingClassifier(random_state = 42)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
items = pd.read_csv(items_path)
 items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )<feature_engineering>
clf = AdaBoostClassifier(random_state = 42)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3" 
<groupby>
clf = ExtraTreesClassifier(random_state = 42, max_depth = 10, n_estimators = 1000, min_samples_split = 2)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)))
 scoring = ['accuracy']
scores = cross_validate(clf, X, y, cv=6, scoring = scoring)
 print(np.mean(scores['test_accuracy']))
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 print(group_sum.reset_index())
 group_sum = group_sum.reset_index()<drop_column>
clf = BernoulliNB()
clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )<categorify>
clf = LogisticRegression(random_state = 42, penalty='l1')
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()<data_type_conversions>
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts<feature_engineering>
clf = SGDClassifier()
clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)))
 print(accuracy_score(train_y, clf.predict(train_x)) )
train["revenue"] = train["item_cnt_day"] * train["item_price"]<merge>
clf = KNeighborsClassifier(n_neighbors = 5, leaf_size = 10, weights = 'distance', algorithm = 'auto')
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)))
 print(accuracy_score(train_y, clf.predict(train_x)) )
ts = time.time()
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).clip(0,20 ).astype(np.float16)
 time.time() - ts 
<data_type_conversions>
clf = XGBClassifier()
clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16)
<concatenate>
ts = time.time()
matrix = pd.concat([matrix, test.drop(["ID"],axis = 1)], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace = True)
 time.time() - ts<data_type_conversions>
clf = ExtraTreesClassifier(random_state = 42)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)))
 print(accuracy_score(train_y, clf.predict(train_x)) )
ts = time.time()
matrix = pd.merge(matrix, shops, on = ["shop_id"], how = "left")
 matrix = pd.merge(matrix, items, on = ["item_id"], how = "left")
 matrix = pd.merge(matrix, cats, on = ["item_category_id"], how = "left")
 matrix["shop_city"] = matrix["shop_city"].astype(np.int8)
 matrix["shop_category"] = matrix["shop_category"].astype(np.int8)
 matrix["item_category_id"] = matrix["item_category_id"].astype(np.int8)
 matrix["subtype_code"] = matrix["subtype_code"].astype(np.int8)
 matrix["name2"] = matrix["name2"].astype(np.int8)
 matrix["name3"] = matrix["name3"].astype(np.int16)
 matrix["type_code"] = matrix["type_code"].astype(np.int8)
 time.time() - ts<merge>
clf = ExtraTreesClassifier(random_state = 0)
 clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)))
 print(accuracy_score(train_y, clf.predict(train_x)) )
def lag_feature(df,lags, cols):
for col in cols:
print(col)
 tmp = df[["date_block_num", "shop_id","item_id",col ]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_"+str(i)]
shifted.date_block_num = shifted.date_block_num + i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
clf = GaussianNB()
clf.fit(train_x, train_y)
 print(accuracy_score(test_y, clf.predict(test_x)) )
ts = time.time()
group = matrix.groupby(["date_block_num"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1], ["date_avg_item_cnt"])
 matrix.drop(["date_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts<merge>
test_X = test[test.drop('id', axis = 1 ).columns].values
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix.date_item_avg_item_cnt = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ['date_item_avg_item_cnt'])
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
gsc = RandomForestClassifier(n_estimators=1000)
 gsc.fit(X, y)
 print(accuracy_score(train_y, gsc.predict(train_x)))
 print(accuracy_score(test_y, gsc.predict(test_x)))
 test_Y = gsc.predict(test_X )
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_avg_item_cnt"])
 matrix.drop(["date_shop_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts<merge>
data = {'id':test['id'],'class':test_Y}
df = pd.DataFrame(data )
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id","item_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_item_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id","item_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_item_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_item_avg_item_cnt"])
 matrix.drop(["date_shop_item_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts<merge>
df.to_csv(path_or_buf ='output.csv', index = False )