Kaggle competition: CareerCon 2019 - Help Navigate Robots
columns = ['MSZoning', 'MSSubClass', 'PavedDrive', 'GarageFinish', 'Foundation', 'Functional', 'LandContour', 'Condition1',
'Condition2', 'Street', 'LotShape', 'ExterQual', 'ExterCond', 'LotConfig', 'LandSlope', 'Neighborhood','BldgType',
'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st','Exterior2nd', 'MasVnrType', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'GarageType', 'SaleType', 'SaleCondition',
'MoSold']
one_hot = pd.get_dummies(all_data.loc[:, columns], drop_first=True)
 all_data.drop(columns=columns, inplace=True)
 for i in range(len(one_hot.columns.values)) :
all_data[one_hot.columns.values[i]] = one_hot[one_hot.columns.values[i]].values<split>
X_train=pd.read_csv('.. /input/X_train.csv')
 y_train=pd.read_csv('.. /input/y_train.csv')
 X_test=pd.read_csv('.. /input/X_test.csv')
 y_test=pd.read_csv('.. /input/sample_submission.csv' )
predict_data = all_data.loc[np.isnan(all_data['SalePrice'].values)]
model_data = all_data.loc[~np.isnan(all_data['SalePrice'].values)]
predict_ids = predict_data['Id'].values
predict_data.drop(columns=['Id'], inplace=True)
 model_data.drop(columns=['Id'], inplace=True )<feature_engineering>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(model_data,1,["SalePrice", 'LotArea', 'GarageArea'])
 model_data = model_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<prepare_x_and_y>
def perform_euler_factors_calculation(df):
df['total_angular_velocity'] = np.sqrt(np.square(df['angular_velocity_X'])+ np.square(df['angular_velocity_Y'])+ np.square(df['angular_velocity_Z']))
df['total_linear_acceleration'] = np.sqrt(np.square(df['linear_acceleration_X'])+ np.square(df['linear_acceleration_Y'])+ np.square(df['linear_acceleration_Z']))
df['total_xyz'] = np.sqrt(np.square(df['orientation_X'])+ np.square(df['orientation_Y'])+
np.square(df['orientation_Z']))
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
x, y, z, w = df['orientation_X'].tolist() , df['orientation_Y'].tolist() , df['orientation_Z'].tolist() , df['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 df['euler_x'] = nx
df['euler_y'] = ny
df['euler_z'] = nz
df['total_angle'] = np.sqrt(np.square(df['euler_x'])+ np.square(df['euler_y'])+ np.square(df['euler_z']))
df['angle_vs_acc'] = df['total_angle'] / df['total_linear_acceleration']
df['angle_vs_vel'] = df['total_angle'] / df['total_angular_velocity']
return df
scaler = MinMaxScaler()
scaler.fit(model_data.drop(columns=['SalePrice']))
X_train = scaler.transform(model_data.drop(columns=['SalePrice']))
y_train = model_data['SalePrice']
X_predict_data_scaled = scaler.transform(predict_data.drop(columns=['SalePrice']))<import_modules>
def perform_feature_engineering(df):
df_out = pd.DataFrame()
for col in df.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
df_out[col + '_mean'] = df.groupby(['series_id'])[col].mean()
df_out[col + '_min'] = df.groupby(['series_id'])[col].min()
df_out[col + '_max'] = df.groupby(['series_id'])[col].max()
df_out[col + '_std'] = df.groupby(['series_id'])[col].std()
df_out[col + '_mad'] = df.groupby(['series_id'])[col].mad()
df_out[col + '_med'] = df.groupby(['series_id'])[col].median()
df_out[col + '_skew'] = df.groupby(['series_id'])[col].skew()
df_out[col + '_range'] = df_out[col + '_max'] - df_out[col + '_min']
df_out[col + '_max_to_min'] = df_out[col + '_max'] / df_out[col + '_min']
df_out[col + '_mean_abs_change'] = df.groupby('series_id')[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df_out[col + '_mean_change_of_abs_change'] = df.groupby('series_id')[col].apply(lambda x: np.mean(np.diff(np.abs(np.diff(x)))))
 df_out[col + '_abs_max'] = df.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 df_out[col + '_abs_min'] = df.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 df_out[col + '_abs_mean'] = df.groupby('series_id')[col].apply(lambda x: np.mean(np.abs(x)))
 df_out[col + '_abs_std'] = df.groupby('series_id')[col].apply(lambda x: np.std(np.abs(x)))
 df_out[col + '_abs_avg'] =(df_out[col + '_abs_min'] + df_out[col + '_abs_max'])/2
df_out[col + '_abs_range'] = df_out[col + '_abs_max'] - df_out[col + '_abs_min']
return df_out
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV<train_on_grid>
X_test = perform_euler_factors_calculation(X_test )
param_grid = {
'criterion' : ['mse'],
'n_estimators': [90, 100, 110],
'max_features': ['auto', 'log2'],
'max_depth' : [7, 9, 11, 13]
}
randomForest_CV = GridSearchCV(estimator = RandomForestRegressor() , param_grid = param_grid, cv = 3)
 randomForest_CV.fit(X_train, y_train)
 print(randomForest_CV.best_params_)
 print(randomForest_CV.best_score_)
 criterion = randomForest_CV.best_params_.get('criterion')
 nRF = randomForest_CV.best_params_.get('n_estimators')
 max_features = randomForest_CV.best_params_.get('max_features')
 max_depth = randomForest_CV.best_params_.get('max_depth')
 bestRF = RandomForestRegressor(n_estimators=nRF, max_depth=max_depth, max_features=max_features, criterion=criterion)
 bestRF.fit(X_train, y_train )<train_model>
X_train = perform_feature_engineering(X_train[X_train.columns.values[:13]] )
xgboost = xgb.XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3)
 xgboost.fit(X_train,y_train )<save_to_csv>
X_test = perform_feature_engineering(X_test[X_test.columns.values[:13]] )
testPredictionsRF = bestRF.predict(X_predict_data_scaled)
 testPredictionsXGB = xgboost.predict(X_predict_data_scaled)
 outputRF = pd.DataFrame({'Id': predict_ids, 'SalePrice': testPredictionsRF})
 outputRF.to_csv('my_submissionRF.csv', index=False)
 outputXGB = pd.DataFrame({'Id': predict_ids, 'SalePrice': testPredictionsXGB})
 outputXGB.to_csv('my_submissionXGB.csv', index=False )<load_from_csv>
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))
warnings.filterwarnings('ignore')
 dataset = pandas.read_csv(".. /input/train.csv")
 dataset_test = pandas.read_csv(".. /input/test.csv")
 ID = dataset_test['id']
dataset_test.drop('id',axis=1,inplace=True)
 pandas.set_option('display.max_rows', None)
 pandas.set_option('display.max_columns', None)
 print(dataset.head(5))<filter>
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )
dataset = dataset.iloc[:,1:]<statistical_test>
scaler = StandardScaler()
X_train_scaled = pd.DataFrame(scaler.fit_transform(X_train))
X_test_scaled = pd.DataFrame(scaler.transform(X_test))
def categories(series):
return range(int(series.min()), int(series.max())+ 1)
 def chi_square_of_df_cols(df, col1, col2):
df_col1, df_col2 = df[col1], df[col2]
le1 = LE()
df_col1 = le1.fit_transform(df_col1)
 df_col2 = le1.fit_transform(df_col2)
 result = [[sum(( df_col1 == cat1)&(df_col2 == cat2))
for cat2 in categories(df_col2)]
for cat1 in categories(df_col1)]
return scs.chi2_contingency(result)
<statistical_test>
folds = StratifiedKFold(n_splits=49, shuffle=True, random_state=41 )
<prepare_x_and_y>
sub_preds_rf = np.zeros(( X_test_scaled.shape[0], 9))
oof_preds_rf = np.zeros(( X_train_scaled.shape[0]))
score = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train_scaled, y_train['surface'])) :
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X_train_scaled.iloc[trn_idx], y_train['surface'][trn_idx])
 oof_preds_rf[val_idx] = clf.predict(X_train_scaled.iloc[val_idx])
 sub_preds_rf += clf.predict_proba(X_test_scaled)/ folds.n_splits
score += clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])
 print('Fold: {} score: {}'.format(fold_,clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])))
 print('Avg Accuracy', score / folds.n_splits )
y = dataset.loss
dataset.drop(["loss"], axis=1, inplace=True )<define_variables>
sub_preds_xgboost = np.zeros(( X_test_scaled.shape[0], 9))
oof_preds_xgboost = np.zeros(( X_train_scaled.shape[0]))
score = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train_scaled, y_train['surface'])) :
xgb_clf = XGBClassifier(n_jobs = -1)
 xgb_clf.fit(X_train_scaled.iloc[trn_idx], y_train['surface'][trn_idx])
 oof_preds_xgboost[val_idx] = xgb_clf.predict(X_train_scaled.iloc[val_idx])
 sub_preds_xgboost += xgb_clf.predict_proba(X_test_scaled)/ folds.n_splits
score += xgb_clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])
 print('Fold: {} score: {}'.format(fold_,xgb_clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])))
 print('Avg Accuracy', score / folds.n_splits )
shapeTrain = dataset.shape<concatenate>
y_test_pred_final = np.array(sub_preds_rf.argmax(axis=1)*0.5+sub_preds_xgboost.argmax(axis=1)*0.5 )
data = dataset.append(dataset_test )<count_values>
y_test_pred_final = np.array([int(val)for val in y_test_pred_final] )
<drop_column><EOS> 
y_test['surface'] = le.inverse_transform(y_test_pred_final)
 y_test.to_csv('submission_18.csv', index=False )