Kaggle competition: Market Basket - ID NDSC 2020
<SOS> metric: categorizationaccuracy Kaggle data source: market-basket-id-ndsc-2020<filter>
import pandas as pd
from math import floor
%%time
matrix = matrix[matrix.date_block_num > 11]<correct_missing_values>
association = pd.read_csv('.. /input/market-basket-id-ndsc-2020/association_order.csv')
 rules = pd.read_csv('.. /input/market-basket-id-ndsc-2020/rules.csv' )
%%time
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix )<sort_values>
group_itemid = association.groupby('itemid')['orderid'].apply(set )
numerical_features = [col for col in matrix.columns if(col.find('cnt')!=-1 or col.find('cnt')!=-1 or col.find('price')!=-1 or col.find('revenue')!=-1 or col.find('sale')!=-1)]
matrix[numerical_features].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )<sort_values>
def confidence(data):
data = [i.split('&')for i in data.split('>')]
denominator = group_itemid[int(data[0][0])]
if len(data[0])> 1:
denominator = denominator.intersection(group_itemid[int(data[0][1])])
 numerator = set(denominator)
 for i in data[1]:
numerator = numerator.intersection(group_itemid[int(i)])
 return floor(len(numerator)/len(denominator)*1000 )
matrix[numerical_features].apply(lambda x: kurt(x.dropna())).sort_values(ascending=False )<load_pretrained>
rules['confidence'] = rules['rule'].apply(confidence )
matrix.to_pickle('data.pkl')
 del matrix, cache, group, train_data, shop_data, item_data, item_category_data
gc.collect() ;<feature_engineering>
rules.to_csv('submission.csv',index=False )