Kaggle competition: IIITB ML Project: SFO Crime Classification
<SOS> metric: multiclassloss Kaggle data source: iiitb-ml-project:-sfo-crime-classification<load_from_csv>
%matplotlib inline
warnings.filterwarnings('ignore')
 warnings.filterwarnings('ignore', category=DeprecationWarning)
 pd.options.display.max_columns = 100
sns.set(rc={'figure.figsize':(12,9)})
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
from xgboost import XGBClassifier
import xgboost as xgb
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
from sklearn.preprocessing import StandardScaler
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
data = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
df=data
datetime=df.Dates.str.split(pat=" ",expand=True)
 datetime.columns=['Date','Time']
df_test=test
datetime_test=df_test.Dates.str.split(pat=" ",expand=True)
 datetime_test.columns=['Date','Time']
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
Date=datetime.Date.str.split(pat="-",expand=True)
 Date.columns=['Year','Month','Day']
Time=datetime.Time.str.split(pat=":",expand=True)
 Time.columns=['Hour','Minute','Second']
Date_test=datetime_test.Date.str.split(pat="-",expand=True)
 Date_test.columns=['Year','Month','Day']
Time_test=datetime_test.Time.str.split(pat=":",expand=True)
 Time_test.columns=['Hour','Minute','Second']
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
df=pd.concat([df,Date,Time],axis=1)
 df_test=pd.concat([df_test,Date_test,Time_test],axis=1)
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
df=df.drop(labels=['Dates'],axis=1)
 df_test=df_test.drop(labels=['Dates'],axis=1)
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
le = preprocessing.LabelEncoder()
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
le_res=le.fit_transform(df['Category'])
 y=pd.DataFrame(le_res)
 y.columns=['Category']
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
df["rot60_X"]=(0.5)* df["Y"] +(1.732/2)* df["X"]
df["rot60_Y"]=0.5 * df["Y"] -(1.732/2)* df["X"]
df_test["rot60_X"]=(0.5)* df_test["Y"] +(1.732/2)* df_test["X"]
df_test["rot60_Y"]=0.5 * df_test["Y"] -(1.732/2)* df_test["X"]
df["rot45_X"]=0.707 * df["Y"] + 0.707 * df["X"]
df["rot45_Y"]=0.707 * df["Y"] - 0.707 * df["X"]
df_test["rot45_X"]=0.707 * df_test["Y"] + 0.707 * df_test["X"]
df_test["rot45_Y"]=0.707 * df_test["Y"] - 0.707 * df_test["X"]
df["rot30_X"]=(1.732/2)* df["Y"] + 0.5 * df["X"]
df["rot30_Y"]=(1.732/2)* df["Y"] - 0.5 * df["X"]
df_test["rot30_X"]=(1.732/2)* df_test["Y"] + 0.5 * df_test["X"]
df_test["rot30_Y"]=(1.732/2)* df_test["Y"] - 0.5 * df_test["X"]
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
df["radial60"]=np.sqrt(np.power(df['rot60_X'],2)+ np.power(df['rot60_Y'],2))
df_test["radial60"]=np.sqrt(np.power(df_test['rot60_X'],2)+ np.power(df_test['rot60_Y'],2))
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
df=df.drop(labels='rot60_X',axis=1)
 df_test=df_test.drop(labels='rot60_X',axis=1 )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
df=df.drop(labels='rot60_Y',axis=1)
 df_test=df_test.drop(labels='rot60_Y',axis=1 )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df=df.drop(labels='Second',axis=1)
 df_test=df_test.drop(labels='Second',axis=1 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df['Minute']=df['Minute'].apply(lambda x:int(x))
df['Minute']=df['Minute'].apply(lambda x : 'low' if x <31 else 'high')
 df_test['Minute']=df_test['Minute'].apply(lambda x:int(x))
df_test['Minute']=df_test['Minute'].apply(lambda x : 'low' if x <31 else 'high')
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df['DayOfWeek']= df['DayOfWeek'].apply(lambda x : 'WeekHigh' if x in('Wednesday','Friday')else('WeekMed' if x in('Tuesday','Thursday','Saturday')else 'WeekLow'))
df_test['DayOfWeek']= df_test['DayOfWeek'].apply(lambda x : 'WeekHigh' if x in('Wednesday','Friday')else('WeekMed' if x in('Tuesday','Thursday','Saturday')else 'WeekLow'))
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df['Intersection']=df['Address'].apply(lambda x : 1 if '/' in x else 0)
 df['Block']=df['Address'].apply(lambda x : 1 if 'Block' in x else 0)
 df_test['Intersection']=df_test['Address'].apply(lambda x : 1 if '/' in x else 0)
 df_test['Block']=df_test['Address'].apply(lambda x : 1 if 'Block' in x else 0 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
address=pd.DataFrame(df['Address'],columns=['Address'])
 address=address.Address.str.split(pat=" /",expand=True)
 address.columns=['Address','Intr2']
address=address.Address.str.split(pat=" /",expand=True)
 address.columns=['Address']
string=address.iloc[:,0]
string=string.str.strip()
address_fram=string.to_frame()
temp=address_fram['Address'].astype(str ).str[-2:]
address=temp.to_frame()
address['Address']=address['Address'].apply(lambda x :(x if x in("ST","AV","LN","DR","BL","HY","CT","RD","PL","PZ","TR","AL","CR","WK","EX","RW")else(( "I-80" if x in("80")else("HWY" if x in("WY")else("WAY" if x in("AY")else("TER" if x in("ER")else("ALMS" if x in("MS")else("MAR" if x in("AR")else("PARK" if x in("RK")else("STWY" if x in("WY")else("VIA" if x in("NO")else("BLOCK")))))))))))))
 df=df.drop(labels=['Address'],axis=1)
 df=pd.concat([address,df],axis=1)
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
address_test=pd.DataFrame(df_test['Address'],columns=['Address'])
 address_test=address_test.Address.str.split(pat=" /",expand=True)
 address_test.columns=['Address','Intr2']
address_test=address_test.Address.str.split(pat=" /",expand=True)
 address_test.columns=['Address']
string_test=address_test.iloc[:,0]
string_test=string_test.str.strip()
address_fram_test=string_test.to_frame()
temp_test=address_fram_test['Address'].astype(str ).str[-2:]
address_test=temp_test.to_frame()
address_test['Address']=address_test['Address'].apply(lambda x :(x if x in("ST","AV","LN","DR","BL","HY","CT","RD","PL","PZ","TR","AL","CR","WK","EX","RW")else(( "I-80" if x in("80")else("HWY" if x in("WY")else("WAY" if x in("AY")else("TER" if x in("ER")else("ALMS" if x in("MS")else("MAR" if x in("AR")else("PARK" if x in("RK")else("STWY" if x in("WY")else("VIA" if x in("NO")else("BLOCK")))))))))))))
 df_test=df_test.drop(labels=['Address'],axis=1)
 df_test=pd.concat([address_test,df_test],axis=1 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
Id=df['Id']
df=df.drop(['Descript','Resolution','Id'],axis=1)
 Id_test=df_test['Id']
df_test=df_test.drop(['Descript','Resolution','Id'],axis=1 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
le = preprocessing.LabelEncoder()
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
le_res=le.fit_transform(df['DayOfWeek'])
 Day=pd.DataFrame(le_res)
 Day.columns=['DayOfWeek']
df=df.drop(labels=['DayOfWeek'],axis=1)
 df=pd.concat([Day,df],axis=1)
 le_res_test=le.fit_transform(df_test['DayOfWeek'])
 Day_test=pd.DataFrame(le_res_test)
 Day_test.columns=['DayOfWeek']
df_test=df_test.drop(labels=['DayOfWeek'],axis=1)
 df_test=pd.concat([Day_test,df_test],axis=1 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
le_res=le.fit_transform(df['PdDistrict'])
 District=pd.DataFrame(le_res)
 District.columns=['District']
df=df.drop(labels=['PdDistrict'],axis=1)
 df=pd.concat([District,df],axis=1)
 le_res_test=le.fit_transform(df_test['PdDistrict'])
 District_test=pd.DataFrame(le_res_test)
 District_test.columns=['District']
df_test=df_test.drop(labels=['PdDistrict'],axis=1)
 df_test=pd.concat([District_test,df_test],axis=1)
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
le_res=le.fit_transform(df['Year'])
 Year=pd.DataFrame(le_res)
 Year.columns=['Year']
df=df.drop(labels=['Year'],axis=1)
 df=pd.concat([Year,df],axis=1)
 le_res_test=le.fit_transform(df_test['Year'])
 Year_test=pd.DataFrame(le_res_test)
 Year_test.columns=['Year']
df_test=df_test.drop(labels=['Year'],axis=1)
 df_test=pd.concat([Year_test,df_test],axis=1)
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
le_res=le.fit_transform(df['Month'])
 Month=pd.DataFrame(le_res)
 Month.columns=['Month']
df=df.drop(labels=['Month'],axis=1)
 df=pd.concat([Month,df],axis=1)
 le_res_test=le.fit_transform(df_test['Month'])
 Month_test=pd.DataFrame(le_res_test)
 Month_test.columns=['Month']
df_test=df_test.drop(labels=['Month'],axis=1)
 df_test=pd.concat([Month_test,df_test],axis=1)
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
le_res=le.fit_transform(df['Day'])
 Day=pd.DataFrame(le_res)
 Day.columns=['Day']
df=df.drop(labels=['Day'],axis=1)
 df=pd.concat([Day,df],axis=1)
 le_res_test=le.fit_transform(df_test['Day'])
 Day_test=pd.DataFrame(le_res_test)
 Day_test.columns=['Day']
df_test=df_test.drop(labels=['Day'],axis=1)
 df_test=pd.concat([Day_test,df_test],axis=1)
matrix['month'] = matrix['date_block_num'] % 12<categorify>
le_res=le.fit_transform(df['Hour'])
 Hour=pd.DataFrame(le_res)
 Hour.columns=['Hour']
df=df.drop(labels=['Hour'],axis=1)
 df=pd.concat([Hour,df],axis=1)
 le_res_test=le.fit_transform(df_test['Hour'])
 Hour_test=pd.DataFrame(le_res_test)
 Hour_test.columns=['Hour']
df_test=df_test.drop(labels=['Hour'],axis=1)
 df_test=pd.concat([Hour_test,df_test],axis=1)
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
le_res=le.fit_transform(df['Minute'])
 Minute=pd.DataFrame(le_res)
 Minute.columns=['Minute']
df=df.drop(labels=['Minute'],axis=1)
 df=pd.concat([Minute,df],axis=1)
 le_res_test=le.fit_transform(df_test['Minute'])
 Minute_test=pd.DataFrame(le_res_test)
 Minute_test.columns=['Minute']
df_test=df_test.drop(labels=['Minute'],axis=1)
 df_test=pd.concat([Minute_test,df_test],axis=1)
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<data_type_conversions>
df["raw_radial"]=np.sqrt(np.power(df['X'],2)+ np.power(df['Y'],2))
df_test["raw_radial"]=np.sqrt(np.power(df_test['X'],2)+ np.power(df_test['Y'],2))
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<feature_engineering>
le_res=le.fit_transform(df['Category'])
 cat=pd.DataFrame(le_res)
 cat.columns=['Category']
df=df.drop(labels=['Category'],axis=1)
 df=pd.concat([cat,df],axis=1)
 df.columns
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<filter>
xy_scaler = StandardScaler()
xy_scaler.fit(df.loc[:,['X','Y']])
 xy_scaled = xy_scaler.transform(df.loc[:,['X','Y']])
 kmeans = KMeans(n_clusters=26, init='k-means++')
 kmeans.fit(xy_scaled);
xy_scaler_test = StandardScaler()
xy_scaler_test.fit(df_test.loc[:,['X','Y']])
 xy_scaled_test = xy_scaler_test.transform(df_test.loc[:,['X','Y']])
 kmeans = KMeans(n_clusters=26, init='k-means++')
 kmeans.fit(xy_scaled_test);
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
geoData = df.loc[:,['X','Y']]
df['closest_centers_f'] = kmeans.predict(geoData)
 id_label=kmeans.labels_
df.loc[:,'label'] = pd.Series(kmeans.labels_)
 geoData_test = df_test.loc[:,['X','Y']]
df_test['closest_centers_f'] = kmeans.predict(geoData_test)
 id_label_test=kmeans.labels_
df_test.loc[:, 'label'] = pd.Series(kmeans.labels_ )
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
le_res=le.fit_transform(df['Address'])
 Address=pd.DataFrame(le_res)
 Address.columns=['Address']
df=df.drop(labels=['Address'],axis=1)
 df=pd.concat([Address,df],axis=1)
 le_res=le.fit_transform(df_test['Address'])
 Address_test=pd.DataFrame(le_res)
 Address_test.columns=['Address']
df_test=df_test.drop(labels=['Address'],axis=1)
 df_test=pd.concat([Address_test,df_test],axis=1 )
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
df=df[['Address', 'Minute', 'Hour', 'Day', 'Month', 'Year',
'District', 'DayOfWeek', 'X', 'Y', 'rot45_X', 'rot45_Y', 'rot30_X',
'rot30_Y', 'radial60', 'Intersection', 'Block', 'raw_radial',
'closest_centers_f', 'label']]
df_test=df_test[['Address', 'Minute', 'Hour', 'Day', 'Month', 'Year', 'District',
'DayOfWeek', 'X', 'Y', 'rot45_X', 'rot45_Y', 'rot30_X', 'rot30_Y',
'radial60', 'Intersection', 'Block', 'raw_radial', 'closest_centers_f',
'label']]
data = pd.read_pickle('data.pkl' )<drop_column>
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.20,shuffle=False )
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
HYPER_PARAMS = {
'learning_rate': 0.02,
'n_estimators':800,
'max_depth': 6,
'subsample': 0.8,
'colsample_bytree': 0.8,
'max_delta_step': 1,
'objective': 'multi:softmax',
'nthread': 4,
'seed': 1747
}
model = xgb.XGBClassifier(**HYPER_PARAMS)
 model.fit(X,y)
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
y_pred=model.predict_proba(df_test )
del data
gc.collect() ;<train_model>
temp = data['Category']
le.fit_transform(temp)
 le.classes_
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
y_pred= pd.DataFrame(y_pred, index=Id_test,columns = le.classes_ )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<set_options>
y_pred.to_csv("submit.csv", float_format = '%.5F' )
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
root_path = '/kaggle/input/competitive-data-science-predict-future-sales/'
df_test = pd.read_csv(root_path + 'test.csv')
 df_item_categories = pd.read_csv(root_path + 'item_categories.csv')
 df_sales_train = pd.read_csv(root_path + 'sales_train.csv')
 df_sample_submission = pd.read_csv(root_path + 'sample_submission.csv')
 df_items = pd.read_csv(root_path + 'items.csv')
 df_shops = pd.read_csv(root_path + 'shops.csv')
 df_categories_translated = pd.read_csv('/kaggle/input/translated/item_categories-translated.csv')
 df_shops_translated = pd.read_csv('/kaggle/input/translated/shops-translated.csv')
 print('The files have been loaded!')
 print('df_test')
 print(df_test.shape)
 display(df_test.head(3))
print('df_item_categories')
 print(df_item_categories.shape)
 display(df_item_categories.head(3))
print('df_sales_train')
 print(df_sales_train.shape)
 display(df_sales_train.head(3))
print('df_sample_submission')
 print(df_sample_submission.shape)
 display(df_sample_submission.head(3))
print('df_items')
 print(df_items.shape)
 display(df_items.head(3))
print('df_shops')
 print(df_shops.shape)
 display(df_shops.head(3))<count_unique_values>