Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<filter>
warnings.filterwarnings("ignore")
 os.environ['KMP_DUPLICATE_LIB_OK']='True'
all_df[all_df["BsmtFinSF1"] > 5000]<filter>
f_train = pd.read_csv('.. /input/X_train.csv')
 t_train = pd.read_csv('.. /input/y_train.csv')
 test = pd.read_csv('.. /input/X_test.csv')
 train = pd.merge(t_train, f_train, how = 'outer', left_on = 'series_id', right_on = 'series_id' )
all_df[all_df["TotalBsmtSF"] > 6000]<filter>
def RMS(signal):
signal = np.array(signal)
 return np.sqrt(( signal**2 ).mean() )
all_df[all_df["GrLivArea"] > 5000]<filter>
def psd(signal):
_,PSD = welch(signal, nperseg = 128, scaling = 'spectrum')
 return PSD
all_df[all_df["1stFlrSF"] > 4000]<filter>
def find_matches(current_signal, df, in_front = True, threshold = 0.01):
orientations = ['orientation_X','orientation_Y','orientation_Z','orientation_W']
selection = ['series_id'] + orientations
if in_front:
end = current_signal.iloc[127][selection]
front = df.loc[df.measurement_number ==0][selection]
else:
front = current_signal.iloc[0][selection]
end = df.loc[df.measurement_number ==127][selection]
compare =(front-end ).abs()
if in_front:
compare.series_id = front.series_id
else:
compare.series_id = end.series_id
compare['sums'] = compare[orientations].sum(axis=1)
 compare['in_front'] = in_front
compare.drop(labels = orientations, axis = 1, inplace = True)
 compare.sort_values(by = 'sums', inplace = True)
 if any(compare.sums < threshold):
return compare.loc[compare.sums < threshold], True
else:
return 0,False
all_df = all_df[(all_df['BsmtFinSF1'] < 2000)|(all_df['SalePrice'].isnull())]
all_df = all_df[(all_df['TotalBsmtSF'] < 3000)|(all_df['SalePrice'].isnull())]
all_df = all_df[(all_df['GrLivArea'] < 4500)|(all_df['SalePrice'].isnull())]
all_df = all_df[(all_df['1stFlrSF'] < 2500)|(all_df['SalePrice'].isnull())]
all_df = all_df[(all_df['LotArea'] < 100000)|(all_df['SalePrice'].isnull())]<drop_column>
def continuity(current_signal, compare_signal, in_front = True, merge_threshold = 4, slope_threshold = 6):
orientations = ['orientation_X','orientation_Y','orientation_Z','orientation_W']
select = ['series_id'] + orientations
if in_front:
end = current_signal[select]
front = compare_signal[select]
else:
front = current_signal[select]
end = compare_signal[select]
stitched = pd.concat([end,front])
 diff = stitched.diff()
mean = diff[orientations].iloc[120:127].mean()
std = diff[orientations].iloc[120:127].std()
test =(diff[orientations].iloc[128]-mean)/std
if any(test.abs() > merge_threshold):
return False
slope_front = diff[orientations].iloc[120:127].mean()
std_front = diff[orientations].iloc[120:127].std()
var_front = std_front.apply(np.square)
 slope_end = diff[orientations].iloc[129:136].mean()
std_end = diff[orientations].iloc[129:136].std()
var_end = std_end.apply(np.square)
 t_value =(slope_front-slope_end)/(var_front/8+var_end/8 ).apply(np.sqrt)
 if any(t_value.abs() > slope_threshold):
return False
return True
categories = categories.drop(["PoolQC","MiscFeature","Alley"] )<categorify>
def recast_df(df, match_threshold = 0.01, continuity_thresholds =(4,6)) :
lookup_dict = group_series(df,
match_threshold=match_threshold,
continuity_thresholds=continuity_thresholds)
 sub_dfs = []
for key,group in lookup_dict.items() :
temp_df = stitch_series(df,group)
 temp_df['new_group_id'] = key
sub_dfs.append(temp_df)
 stitched = pd.concat(sub_dfs)
 return stitched
for cat in categories:
le = LabelEncoder()
print(cat)
 all_df[cat].fillna("missing", inplace=True)
 le = le.fit(all_df[cat])
 all_df[cat] = le.transform(all_df[cat])
 all_df[cat] = all_df[cat].astype("category" )<prepare_x_and_y>
stitched_train = recast_df(train, match_threshold=0.05, continuity_thresholds=(6,9))
train_df_le = all_df[~all_df["SalePrice"].isnull() ]
test_df_le = all_df[all_df["SalePrice"].isnull() ]
train_df_le["SalePrice_log"] = np.log(train_df_le["SalePrice"])
 train_X = train_df_le.drop(["SalePrice","SalePrice_log", "Id"], axis=1)
 train_Y = train_df_le["SalePrice_log"]<train_model>
g = stitched_train.groupby('new_group_id')
 surface_count = []
sample_count = []
for name, group in g:
surfaces = group.surface.unique()
surface_count.append(len(surfaces))
sample_count.append(len(group))
print('Max Surface Count:{0}'.format(max(surface_count)))
 print('Min Sample Count:{0}'.format(min(sample_count)))
 print('Max Sample Count:{0}'.format(max(sample_count)) )
models = []
rmses = []
oof = np.zeros(len(train_X))
for train_index, val_index in kf.split(train_X):
X_train = train_X.iloc[train_index]
X_valid = train_X.iloc[val_index]
y_train = train_Y.iloc[train_index]
y_valid = train_Y.iloc[val_index]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 model_lgb = lgb.train(lgbm_params,
lgb_train,
valid_sets=lgb_eval,
num_boost_round=100,
early_stopping_rounds=20,
verbose_eval=10, 
)
 y_pred = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)
 tmp_rmse = np.sqrt(mean_squared_error(y_valid, y_pred))
print(tmp_rmse)
 models.append(model_lgb)
 rmses.append(tmp_rmse)
 oof[val_index] = y_pred<count_values>
g = stitched_train.groupby('group_id')
 sample_count = []
for name, group in g:
surfaces = group.surface.unique()
surface_count.append(len(surfaces))
sample_count.append(len(group))
print('Max sample count in original groups:{0}'.format(max(sample_count)) )
sum(rmses)/len(rmses )<feature_engineering>
train.drop(['row_id','group_id'], axis = 1, inplace = True)
 train['orig_id'] = train.series_id
train['test'] = 0
test.drop(['row_id'], axis = 1, inplace = True)
 test['surface'] = 'no_surface'
test['orig_id'] = test.series_id
test['test'] = 1
test.series_id += train.series_id.max() +1
all_df["Age"] = all_df["YrSold"] - all_df["YearBuilt"]<prepare_x_and_y>
all_data = pd.concat([train,test], sort=False, ignore_index=True )
train_df_le = all_df[~all_df["SalePrice"].isnull() ]
test_df_le = all_df[all_df["SalePrice"].isnull() ]
train_df_le["SalePrice_log"] = np.log(train_df_le["SalePrice"])
 train_X = train_df_le.drop(["SalePrice","SalePrice_log","Id"], axis=1)
 train_Y = train_df_le["SalePrice_log"]<train_model>
all_stitched = recast_df(all_data, match_threshold=0.05, continuity_thresholds=(9,9))
models = []
rmses = []
oof = np.zeros(len(train_X))
for train_index, val_index in kf.split(train_X):
X_train = train_X.iloc[train_index]
X_valid = train_X.iloc[val_index]
y_train = train_Y.iloc[train_index]
y_valid = train_Y.iloc[val_index]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 model_lgb = lgb.train(lgbm_params,
lgb_train,
valid_sets=lgb_eval,
num_boost_round=100,
early_stopping_rounds=20,
verbose_eval=10, 
)
 y_pred = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)
 tmp_rmse = np.sqrt(mean_squared_error(y_valid, y_pred))
print(tmp_rmse)
 models.append(model_lgb)
 rmses.append(tmp_rmse)
 oof[val_index] = y_pred<count_values>
g = all_stitched.groupby('new_group_id')
 surface_count = []
sample_count = []
for name, group in g:
surfaces = group.surface.unique()
surface_count.append(len(surfaces))
sample_count.append(len(group))
print('Max Surface Count:{0}'.format(max(surface_count)))
 print('Min Sample Count:{0}'.format(min(sample_count)))
 print('Max Sample Count:{0}'.format(max(sample_count)) )
sum(rmses)/len(rmses )<feature_engineering>
g = all_stitched.groupby('new_group_id')
 new_dfs = []
error_surfaces = []
error_group_ids = []
matched = 0
unmatched = 0
unmatched_ids = []
for name, group in g:
surfaces = list(group.surface.unique())
 if(len(surfaces)==2)&('no_surface' in surfaces):
surfaces.remove('no_surface')
 group['surface'] = surfaces[0]
new_dfs.append(group)
 matched +=1
elif(len(surfaces)==2)&('no_surface' not in surfaces):
error_surfaces.append(surfaces)
 new_dfs.append(group)
 elif(len(surfaces)>2):
error_surfaces.append(surfaces)
 error_group_ids.append(name)
 new_dfs.append(group)
 else:
new_dfs.append(group)
 if surfaces[0] == 'no_surface':
unmatched += 1
unmatched_ids.append(name)
 sorted_data = pd.concat(new_dfs)
 sorted_data.reset_index(inplace=True, drop = True )
all_df["TotalSF"] = all_df["TotalBsmtSF"] + all_df["1stFlrSF"] + all_df["2ndFlrSF"]
all_df["Total_Bathrooms"] = all_df["FullBath"] + all_df["HalfBath"] + all_df["BsmtFullBath"] + all_df["BsmtHalfBath"]<feature_engineering>
print('Number of unmatched groups:{0}'.format(unmatched))
all_df["Total_PorchSF"] = all_df["WoodDeckSF"] + all_df["OpenPorchSF"] + all_df["EnclosedPorch"] + all_df["3SsnPorch"] + all_df["ScreenPorch"]<drop_column>
def reassign_surfaces(new_surface_list, group_id, df):
error_group = df[sorted_data.new_group_id == group_id]
reassign = {}
start = 1
end = 1
segment = 0
escape = 0
while escape == 0:
temp_dict = {}
surface = error_group.surface.iloc[0]
start = error_group.first_valid_index()
if len(error_group.surface.unique())> 1:
end = error_group.loc[error_group.surface != surface].first_valid_index()
else:
end = error_group.last_valid_index()
escape = 1
error_group.drop(range(start,end),inplace = True)
 if surface == 'no_surface':
temp_dict['surface'] = new_surface_list[segment]
temp_dict['start'] = start
temp_dict['end'] = end
reassign[segment] = temp_dict
segment += 1
for key,value in reassign.items() :
df.iloc[value['start']:value['end'],1] = value['surface']
fixed_group = df[sorted_data.new_group_id == group_id]
plt.figure(figsize =(12,4))
sns.lineplot(x = 'measurement_number',
y = 'orientation_X',
data = fixed_group,
hue='surface');
all_df["hasPorch"] = all_df["Total_PorchSF"].apply(lambda x: 1 if x > 0 else 0)
 all_df = all_df.drop("Total_PorchSF",axis=1 )<prepare_x_and_y>
surface_2 = ['concrete','wood','wood']
reassign_surfaces(surface_2,2,sorted_data )
train_df_le = all_df[~all_df["SalePrice"].isnull() ]
test_df_le = all_df[all_df["SalePrice"].isnull() ]
train_df_le["SalePrice_log"] = np.log(train_df_le["SalePrice"])
 train_X = train_df_le.drop(["SalePrice","SalePrice_log","Id"], axis=1)
 train_Y = train_df_le["SalePrice_log"]<train_model>
surface_9 = ['concrete', 'concrete']
reassign_surfaces(surface_9, 9, sorted_data )
models = []
rmses = []
oof = np.zeros(len(train_X))
for train_index, val_index in kf.split(train_X):
X_train = train_X.iloc[train_index]
X_valid = train_X.iloc[val_index]
y_train = train_Y.iloc[train_index]
y_valid = train_Y.iloc[val_index]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 model_lgb = lgb.train(lgbm_params,
lgb_train,
valid_sets=lgb_eval,
num_boost_round=100,
early_stopping_rounds=20,
verbose_eval=10, 
)
 y_pred = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)
 tmp_rmse = np.sqrt(mean_squared_error(y_valid, y_pred))
print(tmp_rmse)
 models.append(model_lgb)
 rmses.append(tmp_rmse)
 oof[val_index] = y_pred<count_values>
output = sorted_data[sorted_data.test == 1][['orig_id','surface']].groupby('orig_id' ).max().reset_index()
output.to_csv(path_or_buf = 'preClassification.csv',
header = ['series_id','surface'],
index = False)
 estimated_score = sum(output.surface!='no_surface')/len(output )
sum(rmses)/len(rmses )<import_modules>
real_score = 0.55*0.6878 + 0.45*0.8175
print('Estimated Score: {0:1.3f}'.format(estimated_score))
print('Real Score: {0:1.3f}'.format(real_score))
print('Accuracy: {0:1.3f}'.format(real_score/estimated_score))
import optuna<import_modules>
extract_from = sorted_data.drop(['surface','test','new_group_id', 'orig_id']+orientations, axis = 1)
 extract_from.head()
from sklearn.model_selection import train_test_split<import_modules>
params = {'abs_energy':None,
'absolute_sum_of_changes':None,
'agg_autocorrelation':[{'f_agg':'var','maxlag':32}],
'change_quantiles':[{'ql':0.25,'qh':0.75,'isabs':True, 'f_agg':'mean'},
{'ql':0.25,'qh':0.75,'isabs':True, 'f_agg':'std'}],
'cid_ce':[{'normalize':True},{'normalize':False}],
'fft_aggregated':[{'aggtype': 'centroid'},
{'aggtype': 'variance'},
{'aggtype': 'skew'},
{'aggtype': 'kurtosis'}],
'c3': [{'lag': 1}, {'lag': 2}, {'lag': 3}],
'standard_deviation': None,
'variance': None,
'skewness': None,
'kurtosis': None,
'maximum': None,
'minimum': None,
'sample_entropy':None,
'mean_abs_change':None,
'sum_values':None,
'quantile': [{'q': 0.1},
{'q': 0.2},
{'q': 0.3},
{'q': 0.4},
{'q': 0.6},
{'q': 0.7},
{'q': 0.8},
{'q': 0.9}],
'large_standard_deviation': [{'r': 0.25},{'r':0.35}],
'fft_coefficient': [{'coeff': 0, 'attr': 'real'},
{'coeff': 1, 'attr': 'real'},
{'coeff': 2, 'attr': 'real'},
{'coeff': 3, 'attr': 'real'},
{'coeff': 4, 'attr': 'real'},
{'coeff': 5, 'attr': 'real'},
{'coeff': 6, 'attr': 'real'},
{'coeff': 7, 'attr': 'real'},
{'coeff': 8, 'attr': 'real'},
{'coeff': 9, 'attr': 'real'},
{'coeff': 10, 'attr': 'real'},
{'coeff': 11, 'attr': 'real'},
{'coeff': 12, 'attr': 'real'},
{'coeff': 13, 'attr': 'real'},
{'coeff': 14, 'attr': 'real'},
{'coeff': 15, 'attr': 'real'},
{'coeff': 16, 'attr': 'real'},
{'coeff': 17, 'attr': 'real'},
{'coeff': 18, 'attr': 'real'},
{'coeff': 19, 'attr': 'real'},
{'coeff': 20, 'attr': 'real'},
{'coeff': 21, 'attr': 'real'},
{'coeff': 22, 'attr': 'real'},
{'coeff': 23, 'attr': 'real'},
{'coeff': 24, 'attr': 'real'},
{'coeff': 25, 'attr': 'real'},
{'coeff': 26, 'attr': 'real'},
{'coeff': 27, 'attr': 'real'},
{'coeff': 28, 'attr': 'real'},
{'coeff': 29, 'attr': 'real'},
{'coeff': 30, 'attr': 'real'},
{'coeff': 31, 'attr': 'real'},
{'coeff': 32, 'attr': 'real'},
{'coeff': 33, 'attr': 'real'},
{'coeff': 34, 'attr': 'real'},
{'coeff': 35, 'attr': 'real'},
{'coeff': 36, 'attr': 'real'},
{'coeff': 37, 'attr': 'real'},
{'coeff': 38, 'attr': 'real'},
{'coeff': 39, 'attr': 'real'},
{'coeff': 40, 'attr': 'real'},
{'coeff': 41, 'attr': 'real'},
{'coeff': 42, 'attr': 'real'},
{'coeff': 43, 'attr': 'real'},
{'coeff': 44, 'attr': 'real'},
{'coeff': 45, 'attr': 'real'},
{'coeff': 46, 'attr': 'real'},
{'coeff': 47, 'attr': 'real'},
{'coeff': 48, 'attr': 'real'},
{'coeff': 49, 'attr': 'real'},
{'coeff': 50, 'attr': 'real'},
{'coeff': 51, 'attr': 'real'},
{'coeff': 52, 'attr': 'real'},
{'coeff': 53, 'attr': 'real'},
{'coeff': 54, 'attr': 'real'},
{'coeff': 55, 'attr': 'real'},
{'coeff': 56, 'attr': 'real'},
{'coeff': 57, 'attr': 'real'},
{'coeff': 58, 'attr': 'real'},
{'coeff': 59, 'attr': 'real'},
{'coeff': 60, 'attr': 'real'},
{'coeff': 61, 'attr': 'real'},
{'coeff': 62, 'attr': 'real'},
{'coeff': 63, 'attr': 'real'},
{'coeff': 64, 'attr': 'real'}],
}
from sklearn.model_selection import train_test_split<split>
extracted_features = tsfresh.extract_features(extract_from,
column_id='series_id',
column_sort='measurement_number',
n_jobs = 2,
default_fc_parameters = params )
X_train, X_valid, y_train, y_valid = train_test_split(train_X, train_Y, test_size=0.2, random_state=1234, shuffle=False, stratify=None )<find_best_params>
efc = extracted_features.copy()
tsfresh.utilities.dataframe_functions.impute(efc)
 y = all_stitched.groupby('series_id' ).max() ['surface']
matched_y = y[y!='no_surface']
matched_ef = efc.iloc[matched_y.index]
def objective(trial):
params = {
"objective":"regression",
"random_seed":1234,
"learning_rate":0.05,
"n_estimators":1000,
"num_leaves":trial.suggest_int("num_leaves",4,64),
"max_bin":trial.suggest_int("max_bin",50,200),
"bagging_fraction":trial.suggest_uniform("bagging_fraction",0.4,0.9),
"bagging_freq":trial.suggest_int("bagging_freq",1,10),
"feature_fraction":trial.suggest_uniform("feature_fraction",0.4,0.9),
"min_data_in_leaf":trial.suggest_int("min_data_in_leaf",2,16),
"min_sum_hessian_in_leaf":trial.suggest_int("min_sum_hessian_in_leaf",1,10),
}
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 model_lgb = lgb.train(params, lgb_train,
valid_sets=lgb_eval,
num_boost_round=100,
early_stopping_rounds=20,
verbose_eval=10,)
 y_pred = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)
 score = np.sqrt(mean_squared_error(y_valid, y_pred))
return score<find_best_params>
features_filtered = tsfresh.select_features(matched_ef, matched_y )
study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
study.optimize(objective, n_trials=50)
 study.best_params<init_hyperparams>
metadata = all_stitched.groupby('series_id' ).max() [['surface','test','new_group_id', 'orig_id']]
lgbm_params = {
"objective":"regression",
"random_seed":1234,
"learning_rate":0.05,
"n_estimators":1000,
"num_leaves":12,
"bagging_fraction": 0.8319278029616157,
"bagging_freq": 5,
"feature_fraction": 0.4874544371547538,
"max_bin":189,
"min_data_in_leaf":13,
"min_sum_hessian_in_leaf":4
}<train_model>
filtered_extracted = metadata.join(extracted_features[feature_columns] )
models = []
rmses = []
oof = np.zeros(len(train_X))
for train_index, val_index in kf.split(train_X):
X_train = train_X.iloc[train_index]
X_valid = train_X.iloc[val_index]
y_train = train_Y.iloc[train_index]
y_valid = train_Y.iloc[val_index]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 model_lgb = lgb.train(lgbm_params,
lgb_train,
valid_sets=lgb_eval,
num_boost_round=100,
early_stopping_rounds=20,
verbose_eval=10, 
)
 y_pred = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)
 tmp_rmse = np.sqrt(mean_squared_error(y_valid, y_pred))
print(tmp_rmse)
 models.append(model_lgb)
 rmses.append(tmp_rmse)
 oof[val_index] = y_pred<count_values>
train_filtered = filtered_extracted[filtered_extracted.test == 0]
test_filtered = filtered_extracted[filtered_extracted.test == 1]
sum(rmses)/len(rmses )<drop_column>
X_filtered = train_filtered.drop(labels = ['surface','test','new_group_id', 'orig_id'],
axis = 1)
 y_filtered = train_filtered.surface
X_f_train, X_f_test, y_f_train, y_f_test = train_test_split(X_filtered, y_filtered, test_size=0.25, random_state=42, stratify = y_filtered )
test_X = test_df_le.drop(["SalePrice", "Id"], axis=1 )<predict_on_test>
scaler_f = StandardScaler()
X_f_train_scaled = scaler_f.fit_transform(X_f_train)
 X_f_test_scaled = scaler_f.transform(X_f_test )
preds = []
for model in models:
pred = model.predict(test_X)
 preds.append(pred )<prepare_output>
RFC = RandomForestClassifier(n_estimators=50, random_state = 42, n_jobs=-1)
 RFC.fit(X_f_train, y_f_train)
 y_f_test_RFC = RFC.predict(X_f_test)
 print(classification_report(y_f_test, y_f_test_RFC))
preds_array = np.array(preds)
 preds_mean = np.mean(preds_array, axis=0 )<prepare_output>
NB = GaussianNB()
NB.fit(X_f_train_scaled, y_f_train)
 y_test_NB = NB.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_NB))
preds_exp = np.exp(preds_mean )<prepare_output>
C_range = np.logspace(-4, 0, 4)
 param_grid = dict(C=C_range)
 SVM = GridSearchCV(SVC(kernel = 'linear'),
param_grid=param_grid,
n_jobs=-1,
pre_dispatch=2,
cv = 3)
 SVM.fit(X_f_train_scaled, y_f_train)
 y_test_svc = SVM.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_svc))
submission["SalePrice"] = preds_exp<save_to_csv>
KNN = KNeighborsClassifier(10)
 KNN.fit(X_f_train_scaled,y_f_train)
 y_test_knn = KNN.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_knn))
submission.to_csv("./houseprices_submit01.csv",index=False )<import_modules>
DT = DecisionTreeClassifier()
DT.fit(X_f_train_scaled,y_f_train)
 y_test_dt = DT.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_dt))
from sklearn.ensemble import RandomForestRegressor as rf<count_missing_values>
param_grid = {
'max_depth': [3,4,5,6],
'min_child_weight':np.linspace(0.8,1.2,4),
'gamma': np.linspace(0,0.2,4),
}
hasnan_cat = []
for col in all_df.columns:
tmp_null_count = all_df[col].isnull().sum()
if(tmp_null_count > 0)&(col != "SalePrice"):
print(col, tmp_null_count)
 hasnan_cat.append(col )<data_type_conversions>
XGB = xgb.sklearn.XGBClassifier(learning_rate = 0.025,
objective = 'multi:softmax',
n_estimators = 150,
max_depth = 5,
min_child_weight = 1.2,
subsample=0.8,
colsample_bytree = 0.8,
gamma = 0.066,
n_jobs = 4,
nthreads = 1,
silent = True,
seed = 42)
 XGB.fit(X_f_train_scaled,y_f_train)
 y_test_XGB = XGB.predict(X_f_test_scaled)
 print(classification_report(y_f_test, y_test_svc))
for col in all_df.columns:
tmp_null_count = all_df[col].isnull().sum()
if(tmp_null_count > 0)&(col != "SalePrice"):
print(col, tmp_null_count)
 all_df[col] = all_df[col].fillna(all_df[col].median() )<feature_engineering>
def get_predictions(filtered_data, classifiers, scaler):
warnings.filterwarnings("ignore")
 df = filtered_data[['surface','test','new_group_id', 'orig_id']]
X = filtered_data.drop(labels = ['surface','test','new_group_id','orig_id'],
axis = 1 ).values
X_scaled = scaler.transform(X)
 for name, classifier_dict in classifiers.items() :
clf = classifier_dict['classifier']
if classifier_dict['scale']:
prediction = clf.predict(X_scaled)
 else:
prediction = clf.predict(X)
 df[name] = prediction
warnings.resetwarnings()
return df
train_df_le = all_df[~all_df["SalePrice"].isnull() ]
test_df_le = all_df[all_df["SalePrice"].isnull() ]
train_df_le["SalePrice_log"] = np.log(train_df_le["SalePrice"] )<prepare_x_and_y>
classifiers = {'RFC':{'classifier':RFC,'scale': False},
'NB':{'classifier':NB,'scale':True},
'SVM':{'classifier':SVM,'scale':True},
'KNN':{'classifier':KNN,'scale':True},
'DT':{'classifier':DT,'scale':True},
'XGB':{'classifier':XGB,'scale':True}}
train_X = train_df_le.drop(["SalePrice","SalePrice_log","Id"], axis=1)
 train_Y = train_df_le["SalePrice_log"]<choose_model_class>
multi_predictions = get_predictions(filtered_data=train_filtered,
classifiers=classifiers,
scaler=scaler_f )
folds = 3
kf = KFold(n_splits=folds )<train_model>
def classifier_ensemble(predictions, classifier_weights, groupby = 'new_group_id'):
warnings.filterwarnings("ignore")
 df = predictions[['test','new_group_id','orig_id']]
df['surface'] = 'None'
g = predictions.groupby(groupby)
 for name, group in g:
index = group.index
weighted_value_counts = []
for classifier,weight in classifier_weights.items() :
value_counts = group[classifier].value_counts()
weighted_value_counts.append(value_counts*weight)
 all_counts = pd.DataFrame(weighted_value_counts ).fillna(value=0)
 surface = all_counts.sum().sort_values(ascending=False ).index[0]
df.loc[index,'surface'] = surface
warnings.resetwarnings()
return df
models_rf = []
rmses_rf = []
oof_rf = np.zeros(len(train_X))
for train_index, val_index in kf.split(train_X):
X_train = train_X.iloc[train_index]
X_valid = train_X.iloc[val_index]
y_train = train_Y.iloc[train_index]
y_valid = train_Y.iloc[val_index]
model_rf = rf(
 n_estimators=50,
random_state=1234 
)
 model_rf.fit(X_train, y_train)
 y_pred = model_rf.predict(X_valid)
 tmp_rmse = np.sqrt(mean_squared_error(y_valid, y_pred))
print(tmp_rmse)
 models_rf.append(model_rf)
 rmses_rf.append(tmp_rmse)
 oof_rf[val_index] = y_pred<count_values>
classifier_weights = {classifier:1 for classifier in classifiers.keys() }
del classifier_weights['NB']
ensemble_prediction = classifier_ensemble(predictions=multi_predictions,
classifier_weights=classifier_weights )
sum(rmses_rf)/len(rmses_rf )<drop_column>
print(classification_report(train_filtered.surface, ensemble_prediction.surface))
test_X = test_df_le.drop(["SalePrice","Id"], axis=1 )<predict_on_test>
def CEParamSearch(y_true, multi_predictions, classifier_names,
groupby = 'new_group_id', generations = 20,
seed = 1, mutation_rate = 0.35):
np.random.seed(seed)
 weights = np.random.rand(len(classifiers))
classifier_weights = {classifier:weight for classifier,weight in zip(classifier_names,weights)}
ensemble_prediction = classifier_ensemble(multi_predictions,classifier_weights,groupby=groupby)
 y_pred = ensemble_prediction.surface
score = sum(y_pred == y_true)/len(y_pred)
 scores = [score]
start = time.time()
gene = random.choice(classifier_names)
 orig_mut_rate = mutation_rate
for generation in range(1,generations+1):
new_weights = classifier_weights
mutation =(np.random.rand(1)-0.5)*mutation_rate
new_weights[gene] += mutation
if new_weights[gene] < 0:
new_weights[gene] = 0
if new_weights[gene] > 1:
new_weights[gene] = 1
ensemble_prediction = classifier_ensemble(multi_predictions,new_weights,groupby=groupby)
 y_pred = ensemble_prediction.surface
new_score = sum(y_pred == y_true)/len(y_pred)
 change = new_score - score
if change < 0:
scores.append(score)
 gene = random.choice(classifier_names)
 mutation_rate = orig_mut_rate
elif change == 0:
scores.append(score)
 classifier_weights = new_weights
elif change > 0:
mutation_rate /= 0.75
score = new_score
scores.append(score)
 classifier_weights = new_weights
progress = generation/generations
time_elapsed =(time.time() -start)
 ETA =(time_elapsed/progress-time_elapsed)/60
print('Score:{0:1.5f}\t|\tChange:{1:1.5f}\t|\tTime Remaining: {2:1.2f} min'.format(score,change,ETA), end = '\r')
 plt.plot(scores)
 return classifier_weights
preds_rf = []
for model in models_rf:
pred = model.predict(test_X)
 preds_rf.append(pred )<prepare_output>
classifier_weights = CEParamSearch(train_filtered.surface,
multi_predictions,
classifier_names,
generations = 25,
mutation_rate = 0.5,
seed = 40 )
preds_array_rf = np.array(preds_rf)
 preds_mean_rf = np.mean(preds_array_rf, axis=0)
 preds_exp_rf = np.exp(preds_mean_rf)
 submission["SalePrice"] = preds_exp_rf<save_to_csv>
classifier_weights = {'RFC': 1,
'SVM': 1,
'KNN': 0,
'DT': 0.22006725,
'XGB': 0.38377276}
submission.to_csv("./houseprices_submit02.csv",index=False )<import_modules>
multi_predictions_test = get_predictions(test_filtered,classifiers,scaler_f )
import xgboost as xgb<define_variables>
ensemble_prediction_test = classifier_ensemble(predictions=multi_predictions_test,
classifier_weights=classifier_weights )
categories = train_X.columns[train_X.dtypes == "category"]<data_type_conversions>
output = ensemble_prediction_test[['orig_id','surface']].reset_index().drop('series_id', axis =1)
 output.to_csv(path_or_buf = 'Ensemble_Classified.csv',
header = ['series_id','surface'],
index = False )
for col in categories:
train_X[col] = train_X[col].astype("int8")
 test_X[col] = test_X[col].astype("int8" )<split>
metadata = sorted_data.groupby('series_id' ).max() [['surface','test','new_group_id', 'orig_id']]
filtered_extracted_matched = metadata.join(extracted_features[feature_columns])
 unmatched_filtered = filtered_extracted_matched[filtered_extracted_matched.surface == 'no_surface']
X_train, X_valid, y_train, y_valid = train_test_split(train_X, train_Y, test_size=0.2, random_state=1234, shuffle=False, stratify=None )<train_model>
multi_predictions_unmatched = get_predictions(unmatched_filtered, classifiers, scaler_f)
 ensemble_prediction_unmatched = classifier_ensemble(multi_predictions_unmatched, classifier_weights)
 unmatched_output = ensemble_prediction_unmatched[['orig_id','surface']].reset_index().drop('series_id', axis =1 )
def objective(trial):
xgb_params = {
"learning_rate":0.05,
"seed":1234,
"max_depth":trial.suggest_int("max_depth",3,16),
"colsample_bytree":trial.suggest_uniform("colsample_bytree",0.2,0.9),
"sublsample":trial.suggest_uniform("sublsample",0.2,0.9),
}
xgb_train = xgb.DMatrix(X_train, label=y_train)
 xgb_eval = xgb.DMatrix(X_valid, label=y_valid)
 evals = [(xgb_train, "train"),(xgb_eval, "eval")]
model_xgb = xgb.train(xgb_params, xgb_train,
evals=evals,
num_boost_round=1000,
early_stopping_rounds=20,
verbose_eval=10,)
 y_pred = model_xgb.predict(xgb_eval)
 score = np.sqrt(mean_squared_error(y_valid, y_pred))
return score<find_best_params>
matched_output = metadata[(metadata.test == 1)&(metadata.surface != 'no_surface')][['orig_id','surface']].reset_index().drop('series_id', axis =1 )
study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
study.optimize(objective, n_trials=50)
 study.best_params<init_hyperparams>
output = pd.concat([matched_output,unmatched_output])
 output.to_csv('Hybrid_Classified.csv',
header = ['series_id','surface'],
index = False )