Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<count_duplicates>
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
import scipy
from tqdm import tqdm_notebook as tqdm
print('NaN summary
',data_prep.isna().sum() /len(data_prep)*100,'
')
 print('Number of absoulute dublicates:',data_prep.duplicated().sum())
 print('Number of Store - Date dublicates:',data_prep.duplicated(subset = ['Date','Store'] ).sum() )<prepare_x_and_y>
dd0=pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 ddtest0=pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 ddall=dd0.append(ddtest0, sort=False)
 num_train=len(dd0)
 ddall.head()
X = data_prep.drop(['Date','Sales','Customers','Store'],axis = 1)
 y = data_prep.Sales
X_train,X_val = X.loc[X.year < 2015,:],X.loc[X.year == 2015,:]
y_train,y_val = y[:X_train.index[-1]+1], y[X_train.index[-1]+1:]<feature_engineering>
drop_cols=["bin_0"]
ddall["ord_5a"]=ddall["ord_5"].str[0]
ddall["ord_5b"]=ddall["ord_5"].str[1]
drop_cols.append("ord_5" )
transformer = make_column_transformer(
(StandardScaler() ,['CompetitionDistance', 'Customers_enc_mean']), 
(OneHotEncoder() ,['StateHoliday', 'StoreType', 'Assortment', 'DayOfWeek', 'month']),
remainder = 'passthrough' 
)
 regressor = xgb.XGBRegressor(n_estimators = 200,
max_depth = 10 
)
 pipeline = make_pipeline(transformer,
regressor)
 pipeline.fit(X_train,y_train)
 print('TRAIN RMSPE = ',rmspe(np.expm1(pipeline.predict(X_train)) ,np.expm1(y_train)))
 print('VAL RMSPE = ',rmspe(np.expm1(pipeline.predict(X_val)) ,np.expm1(y_val)) )<merge>
X=ddall[ddall.columns.difference(["id", "target"] + drop_cols)]
test_prep =(test 
.copy() 
.pipe(timeseries_features)
.drop(['Open','Date'],axis=1)
)
 test_prep = pd.merge(test_prep,store_prep,how='left',on='Store')
 test_prep['is_promo'] = test_prep.apply(lambda x: 1 if x['month'] in x['PromoInterval'] else 0,axis = 1)
 test_prep = pd.merge(test_prep,dict_for_test,how='left',on='Store')
 test_prep = test_prep.drop(['PromoInterval','Store'],axis=1 ).reset_index(drop=True)
<predict_on_test>
X_oh=X[X.columns.difference(["ord_1", "ord_4", "ord_5a", "ord_5b", "day", "month"])]
oh1=pd.get_dummies(X_oh, columns=X_oh.columns, drop_first=True, sparse=True)
 ohc1=oh1.sparse.to_coo()
test_id = test_prep.Id
test_prep.drop('Id',axis=1,inplace = True)
 predict = np.expm1(pipeline.predict(test_prep))<save_to_csv>
class ThermometerEncoder(TransformerMixin):

 def __init__(self, sort_key=None):
self.sort_key = sort_key
self.value_map_ = None
def fit(self, X, y=None):
self.value_map_ = {val: i for i, val in enumerate(sorted(X.unique() , key=self.sort_key)) }
return self
def transform(self, X, y=None):
values = X.map(self.value_map_)
 possible_values = sorted(self.value_map_.values())
 idx1 = []
idx2 = []
all_indices = np.arange(len(X))
for idx, val in enumerate(possible_values[:-1]):
new_idxs = all_indices[values > val]
idx1.extend(new_idxs)
 idx2.extend(repeat(idx, len(new_idxs)))
 result = scipy.sparse.coo_matrix(( [1] * len(idx1),(idx1, idx2)) , shape=(len(X), len(possible_values)) , dtype="int8")
 return result
sub = pd.DataFrame({'Id':test_id,'Sales':predict} ).sort_values('Id' ).reset_index(drop=True)
 sub.to_csv('submission.csv',index=False )<load_from_csv>
thermos=[]
for col in ["ord_1", "ord_2", "ord_3", "ord_4", "ord_5a", "day", "month"]:
if col=="ord_1":
sort_key=['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'].index
elif col=="ord_2":
sort_key=['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot'].index
elif col in ["ord_3", "ord_4", "ord_5a"]:
sort_key=str
elif col in ["day", "month"]:
sort_key=int
else:
raise ValueError(col)
 enc=ThermometerEncoder(sort_key=sort_key)
 thermos.append(enc.fit_transform(X[col]))
train_file = '.. /input/rossmann-store-sales/train.csv'
test_file = '.. /input/rossmann-store-sales/test.csv'
output_file = 'predictions.csv'
train = pd.read_csv(train_file, low_memory=False)
 test = pd.read_csv(test_file, low_memory=False)
 print('train データ
'
'shape', train.shape)
 display(train.head())
 print('test データ：Sales だけ欠けているのでこれを予測する（回帰）．
'
'shape', test.shape)
 display(test.head())
 train_means = train.groupby([ 'Store', 'DayOfWeek', 'Promo' ])['Sales'].mean().reset_index()
print("とりあえず適当にグルーピングした平均値を予測値とすることにする．
"
"train データを店，曜日，プロモーション有無で集計
",
'shape', train_means.shape)
 display(train_means.head())
 print("test データに結合")
 test = pd.merge(test, train_means,
on = ['Store','DayOfWeek','Promo'], how='left')
 test.fillna(train.Sales.mean() , inplace=True)
 display(test.head())
 print('提出用に Id, Sales だけ抜き出す')
 submission = test[[ 'Id', 'Sales' ]]
display(submission.head())
 submission.to_csv(output_file, index = False )<load_from_csv>
ohc=scipy.sparse.hstack([ohc1] + thermos ).tocsr()
display(ohc)
 X_train = ohc[:num_train]
X_test = ohc[num_train:]
y_train = dd0["target"].values
train=pd.read_csv('/kaggle/input/rossmann-store-sales/train.csv')
 test=pd.read_csv('/kaggle/input/rossmann-store-sales/test.csv')
 store=pd.read_csv('/kaggle/input/rossmann-store-sales/store.csv' )<count_values>
clf=LogisticRegression(C=0.123456789, solver="lbfgs", max_iter=5000)
 clf.fit(X_train, y_train)
 pred=clf.predict_proba(X_test)[:,1]
pd.DataFrame({"id": ddtest0["id"], "target": pred} ).to_csv("submission.csv", index=False )
train['StateHoliday'].value_counts()<count_values>
score=cross_validate(clf, X_train, y_train, cv=3, scoring="roc_auc")["test_score"].mean()
print(f"{score:.6f}" )