Kaggle competition: Tap30 Challenge
<count_unique_values><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: rmse Kaggle data source: tap30-challenge<remove_duplicates>
import pandas as pd
import numpy as np
train_comb = train[["shop_id", "item_id"]].drop_duplicates()
print(train_comb )<remove_duplicates>
t = 732
m=8
n=8
test_comb = test[["shop_id", "item_id"]].drop_duplicates()
print(test_comb )<count_duplicates>
df = pd.read_csv('.. /input/data.txt',skiprows=2,sep=' ',names=list(map(str,(list(range(n))))))
unseen_shops = test_comb.shop_id[~test_comb["shop_id"].isin(train_comb["shop_id"])].drop_duplicates()
unseen_items = test_comb.item_id[~test_comb["item_id"].isin(train_comb["item_id"])].drop_duplicates()
print("In the test set:", unseen_shops.count() , "unseen shops","and", unseen_items.count() , "unseen items" )<count_unique_values>
arr = np.array(df )
len(test["item_id"].unique())- unseen_items.count()<filter>
df_r = pd.DataFrame(columns=['id','m1','m2','m3','m4'])
 for i in range(m):
for j in range(n):
for k in range(732):
temp = td[::24,i,j]
temp = temp[temp!=-1]
temp2 = td[k,:,:]
temp2 = temp2[temp2!=-1]
temp3 = td[::24*7,i,j]
temp3 = temp3[temp3!=-1]
x = i//4
y = j//4
temp4 = td[k,x*4:(x+1)*4,y*4:(y+1)*4]
temp4 = temp4[temp4!=-1]
st = str(k)+':'+str(i)+':'+str(j)
 df_r = df_r.append({'id':st,'m1':temp.mean() ,'m2':temp2.mean() ,'m3':temp3.mean() ,'m4':temp4.mean() },ignore_index=True )
unseen_combs = []
seen_others = []
for shop in test_comb["shop_id"].unique() :
test_items = test_comb.loc[test_comb["shop_id"] == shop, ["item_id"]].values.reshape(-1)
 train_items = train_comb.loc[train_comb["shop_id"] == shop, ["item_id"]].values.reshape(-1)
 train_items_others = train_comb.loc[train_comb["shop_id"] != shop, ["item_id"]].values.reshape(-1)
 for item in test_items:
if item not in train_items:
unseen_combs.append(( shop, item))<count_unique_values>
new_df = df.stack()
unseen_comb_items = unseen_combs_df["item_id"].unique()
len(unseen_comb_items )<define_variables>
new_df = pd.DataFrame(new_df )
214200 - 102697<groupby>
new_df.columns=['value']
last_months_shops_sales = train.query('date_block_num > 27' ).groupby("shop_id")["item_cnt_day"].sum()
print(last_months_shops_sales > 0 )<define_variables>
new_df.columns= ['t','n','val']
last_months_sales_shops = last_months_shops_sales.index.tolist()<groupby>
new_df['m'] = new_df['t'].apply(lambda x : int(x)%8 )
last_months_items_sales = train.query('date_block_num > 27' ).groupby("item_id")["item_cnt_day"].sum()
last_months_items_sales = last_months_items_sales[last_months_items_sales > 0]
print(last_months_items_sales )<data_type_conversions>
new_df['t'] = new_df['t'].apply(lambda x:x//8 )
last_months_sales_items = last_months_items_sales.index.tolist()<define_variables>
new_df['hour'] = new_df['t'].apply(lambda x : x%24 )
len(set(no_sales_items)& set(unseen_items))<define_variables>
new_df['day'] = new_df['t'].apply(lambda x : x//24 )
outdated_items = list(set(no_sales_items)- set(unseen_items))
len(outdated_items )<feature_engineering>
new_df['mm']=new_df['m']
new_df['nn']= new_df['n']
no_split = item_categories.item_category_name.str.split(" ", expand=True, n = 2)
 no_split = no_split.fillna("" ).apply(lambda x: x.replace("-", ""))
no_split[0] = no_split[0] + " " + no_split[1]
item_category_df = no_split.drop(columns = 1)
 item_category_df.columns = ["big_category_name", "sub_category_name"]
big_category_name = item_category_df.big_category_name.str.strip()
sub_category_name = item_category_df.sub_category_name.apply(lambda x: x.replace("-", "")).str.strip()
item_category_df["big_category_name"] = big_category_name
item_category_df["sub_category_name"] = sub_category_name
item_category_df["big_category_id"] = item_category_df.big_category_name.astype("category" ).cat.codes.to_frame(name = "big_category_id")
 item_category_df["sub_category_id"] = item_category_df.sub_category_name.astype("category" ).cat.codes.to_frame(name = "sub_category_id")
 item_categories = item_categories.join(item_category_df)
 print(item_categories)
 item_categories = downcast(item_categories)
 del no_split
del item_category_df<categorify>
new_df['id'] = new_df['t'].map(str)+':'+new_df['mm'].map(str)+':'+new_df['nn'].map(str )
split = shops.shop_name
split = split.apply(lambda x:x.replace("!", ""))
without_split = split.str.split(expand=True)
 to_correct = without_split.iloc[[34, 35, 42, 43, 46]].drop(columns = [3,4,5,6,7])
 to_correct[0] = to_correct[0] + " " + to_correct[1]
to_correct[1] = to_correct[2]
to_correct.drop(columns = 2, inplace=True)
 without_split_new = without_split.drop(index= [34, 35, 42, 43, 46] ).drop(columns = [2,3,4,5,6,7])
 without_split_new = to_correct.join(without_split_new, how="outer", rsuffix="_" ).fillna("")
 without_split_new["0"] = without_split_new["0"] + without_split_new["0_"]
without_split_new["1"] = without_split_new["1"] + without_split_new["1_"]
without_split_new.drop(columns = ["0_","1_"], inplace=True)
 without_split_new.columns = ["city_name", "shop_category_name"]
without_split_new.city_name = without_split_new.city_name.str.strip()
without_split_new.shop_category_name = without_split_new.shop_category_name.str.strip()
city_name_ids = without_split_new.city_name.astype("category" ).cat.codes.to_frame(name = "city_id")
 shop_category_ids = without_split_new.shop_category_name.astype("category" ).cat.codes.to_frame(name = "shop_category_id")
 shop_df = without_split_new.join(city_name_ids ).join(shop_category_ids)
 shops = shops.join(shop_df)
 print(shops)
 shops = downcast(shops)
 del to_correct
del shop_df<count_missing_values>
new_df = df_r.set_index('id' ).join(new_df.set_index('id'))
print("train")
 print(train.isna().sum())
 print("test")
 print(test.isna().sum())
 print("item_categories")
 print(item_categories.isna().sum())
 print("items")
 print(items.isna().sum())
 print("shops")
 print(shops.isna().sum())
 print("train")
 print(train.isnull().sum())
 print("test")
 print(test.isnull().sum())
 print("item_categories")
 print(item_categories.isnull().sum())
 print("items")
 print(items.isnull().sum())
 print("shops")
 print(shops.isnull().sum() )<create_dataframe>
scaler = StandardScaler()
scaler.fit(new_df.drop(['val','t','mm','nn'],axis=1))
train_extended = pd.DataFrame(np.vstack(train_extended), columns = cols )<categorify>
scaler.transform(new_df.drop(['val','t','mm','nn'],axis=1))
train_extended = downcast(train_extended )<feature_engineering>
new_df[['m1','m2','m3','m4','n','m','hour','day']]=scaler.transform(new_df.drop(['val','t','mm','nn'],axis=1))
train["revenue"] = train["item_price"] * train["item_cnt_day"]<categorify>
train = new_df[new_df['val'] !=-1]
train_monthly = train.groupby(['date_block_num', "shop_id", "item_id"], as_index = False)\ 
.agg({"item_cnt_day" : ["sum"]})
 train_monthly.columns = ['date_block_num',"shop_id", "item_id", "item_cnt_month"]
train_monthly = downcast(train_monthly )<remove_duplicates>
X_train = train.drop(['val','t','mm','nn'], axis=1)
 y_train = train['val'].values
train_monthly["item_cnt_month"] = train_monthly["item_cnt_month"].clip(0,20)
<merge>
test = new_df[new_df['val'] == -1]
train_new = train_extended.merge(train_monthly, on=["date_block_num", "shop_id", "item_id"], how="left" ).fillna(0 )<data_type_conversions>
X_test = test.drop(['val','t','mm','nn'], axis=1 )
train_new = downcast(train_new)
 train_new["item_cnt_month"] = train_new["item_cnt_month"].astype(np.int8 )<drop_column>
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import adam
test = test.drop("ID", axis = 1)
 test.insert(0, "date_block_num", 34 )<categorify>
model = Sequential()
model.add(Dense(128, activation='relu', input_shape=(8,)))
 model.add(Dense(256, activation='relu'))
model.add(Dense(256, activation='relu'))
model.add(Dense(1))
model.summary()
test = downcast(test )<correct_missing_values>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
train_new.fillna(0, inplace=True )<categorify>
model.compile(loss=root_mean_squared_error, optimizer='adam' )
train_new = downcast(train_new )<merge>
model.fit(X_train, y_train, batch_size=128, epochs=240, verbose=1,validation_split=0.2 )
items_and_item_categories = items.drop(columns=["item_name"])\ 
.merge(item_categories.drop(columns=["item_category_name", "big_category_name", "sub_category_name"]))
items_and_item_categories.info()<merge>
model_XGB = XGBRegressor()
model_XGB.fit(X_train,y_train )
train_new = train_new.merge(items_and_item_categories, on = ["item_id"])
 test = test.merge(items_and_item_categories, on = ["item_id"] )<merge>
def make_positive(x):
if x<0:
return 0
else:
return x
train_new = train_new.merge(shops.drop(columns=["shop_name", "city_name", "shop_category_name"]), on = ["shop_id"])
 test = test.merge(shops.drop(columns=["shop_name", "city_name", "shop_category_name"]), on = ["shop_id"] )<train_model>
predict = model.predict(X_test )
train_new.fillna(0, inplace=True)
 test.fillna(0, inplace=True )<merge>
X_test['pred']= predict
lags = [1,2,3,6,12]
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<categorify>
X_test['demand'] = X_test['pred'].apply(make_positive )
<groupby><EOS> 
X_test[['id','demand']].to_csv('result.csv',index=False )