Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<save_to_csv>
sns.set_style('whitegrid')
 %matplotlib inline
df_train = pd.read_csv(".. /input/train.csv")
 df_store = pd.read_csv(".. /input/store.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df_train['Year'] = df_train['Date'].apply(lambda x: int(x[:4]))
df_train['Month'] = df_train['Date'].apply(lambda x: int(x[5:7]))
df_train.head()
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 for i, name in enumerate(test_names):
sample_submission.loc[sample_submission['name'] == name, 'invasive'] = predictions[i]
sample_submission.to_csv("submit.csv", index=False )<define_search_space>
avg = df_train.groupby('Month')["Customers"].mean()
avg_sales = df_train.groupby('Month')['Sales'].mean()
total_sales_customers = df_train.groupby('Store')['Sales', 'Customers'].sum()
total_sales_customers.head()
index=list(range(train_df.shape[0]))
	
random.shuffle(index)
 a=[np.nan]*len(train_df)
 b=[np.nan]*len(train_df)
 c=[np.nan]*len(train_df)
 for i in range(5):
building_level={}
	
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
	
test_index=index[int(( i*train_df.shape[0])/5):int(((i+1)*train_df.shape[0])/5)]
train_index=list(set(index ).difference(test_index))
for j in train_index:
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for j in test_index:
temp=train_df.iloc[j]
if sum(building_level[temp['manager_id']])!=0:
a[j]=building_level[temp['manager_id']][0]*1.0/sum(building_level[temp['manager_id']])
 b[j]=building_level[temp['manager_id']][1]*1.0/sum(building_level[temp['manager_id']])
 c[j]=building_level[temp['manager_id']][2]*1.0/sum(building_level[temp['manager_id']])
 train_df['manager_level_low']=a
train_df['manager_level_medium']=b
train_df['manager_level_high']=c
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high')
 categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f)
 print(train_df["features"].head())
 train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["manager_level_low"].head() )<feature_engineering>
df_total_sales_customers = pd.DataFrame({'Sales': total_sales_customers['Sales'],
'Customers': total_sales_customers['Customers']},
index = total_sales_customers.index)
 df_total_sales_customers = df_total_sales_customers.reset_index()
df_total_sales_customers.head()
tfidf = CountVectorizer(min_df=5)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
 print(te_sparse )<prepare_x_and_y>
avg_sales_customers = df_train.groupby('Store')['Sales', 'Customers'].mean()
avg_sales_customers.head()
print(test_df[features_to_use].head())
 print(train_df[features_to_use].head())
 train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
df_avg_sales_customers = pd.DataFrame({'Sales': avg_sales_customers['Sales'],
'Customers': avg_sales_customers['Customers']},
index = avg_sales_customers.index)
 df_avg_sales_customers = df_avg_sales_customers.reset_index()
df_stores_avg = df_avg_sales_customers.join(df_store.set_index('Store'), on='Store')
 df_stores_avg.head()
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
df_stores_new = df_total_sales_customers.join(df_store.set_index('Store'), on='Store')
 df_stores_new.head()
preds, model = runXGB(train_X, train_y, test_X, num_rounds=1500)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )<load_from_csv>
average_storetype = df_stores_new.groupby('StoreType')['Sales', 'Customers', 'CompetitionDistance'].mean()
fig,(axis1,axis2,axis3)= plt.subplots(1,3,figsize=(15,4))
sns.barplot(average_storetype.index, average_storetype['Sales'], ax=axis1)
 sns.barplot(average_storetype.index, average_storetype['Customers'], ax=axis2)
 sns.barplot(average_storetype.index, average_storetype['CompetitionDistance'], ax=axis3 )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_y_train = df_train['deal_probability']
df_x_train = df_train.drop(['deal_probability'], axis=1 )<count_unique_values>
df_train = pd.read_csv(".. /input/train.csv")
 df_store = pd.read_csv(".. /input/store.csv")
 df_test = pd.read_csv(".. /input/test.csv" )
for col in df_x_train.columns:
print(col, len(df_x_train[col].unique()))<train_model>
closed_store_ids = df_test["Id"][df_test["Open"] == 0].values
df_train['Year'] = df_train['Date'].apply(lambda x: int(x[:4]))
df_train['Month'] = df_train['Date'].apply(lambda x: int(x[5:7]))
df_train["HolidayBin"] = df_train.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1})
 del df_train['Date']
del df_train['StateHoliday']
df_train.head()
df_x_train['image_top_1'].fillna(value=3067, inplace=True)
 df_test['image_top_1'].fillna(value=3067, inplace=True )<categorify>
df_test['Year'] = df_test['Date'].apply(lambda x: int(x[:4]))
df_test['Month'] = df_test['Date'].apply(lambda x: int(x[5:7]))
df_test["HolidayBin"] = df_test.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1})
 del df_test['Date']
del df_test['StateHoliday']
df_test.head()
df_x_train['param_1'].fillna(value='_NA_', inplace=True)
 df_test['param_1'].fillna(value='_NA_', inplace=True)
 df_x_train['param_2'].fillna(value='_NA_', inplace=True)
 df_test['param_2'].fillna(value='_NA_', inplace=True)
 df_x_train['param_3'].fillna(value='_NA_', inplace=True)
 df_test['param_3'].fillna(value='_NA_', inplace=True )<data_type_conversions>
a = list()
for i in df_test['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_test['CompetitionDistance'] = a
df_test.head()
df_x_train['description'].fillna(value='_NA_', inplace=True)
 df_test['description'].fillna(value='_NA_', inplace=True )<set_options>
a = list()
for i in df_train['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_train['CompetitionDistance'] = a
df_train['CompetitionDistance'] = df_train['CompetitionDistance'].fillna(df_train['CompetitionDistance'].mean())
 df_train.head()
<train_model><EOS> 
df_train['CompetitionDistance'] = np.log(df_train['CompetitionDistance'])
 df_test ['CompetitionDistance'] = np.log(df_test ['CompetitionDistance'] )