Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<feature_engineering>
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
find = re.compile(r"^[^.]*")
 train_df['netloc'] = train_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test_df['netloc'] = test_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 count_columns = ['question_title', 'question_user_name', 'answer_user_name', 'category', 'netloc']
for col in count_columns:
value = train_df[col].value_counts()
train_feature[col+'_count'] = train_df[col].map(value)
 test_feature[col+'_count'] = test_df[col].map(value ).fillna(1 )<data_type_conversions>
train_X_0 = pd.read_csv(".. /input/feature1/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/feature2/train_X_features_865_1.csv")
 y_0 = pd.read_csv(".. /input/feature1/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/feature2/train_y1.csv", index_col=False, header=None )
for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))<normalization>
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 print(train_X_0.shape)
 train_X.head()
scaler = MinMaxScaler()
scaler.fit(train_feature)
 train_feature = pd.DataFrame(scaler.transform(train_feature), columns=train_feature.columns)
 test_feature = pd.DataFrame(scaler.transform(test_feature), columns=test_feature.columns)
 del scaler
gc.collect()<categorify>
y = pd.concat([y_0, y_1], axis=0)
 y = y.reset_index(drop=True)
 y[0].shape
def convert_label(df, result_df, column, count=10):
labels = [(df[column].values>=(rate/count)).astype(np.int)for rate in range(1, count+1)]
columns = ['{}_{}'.format(column, i)for i in range(count)]
labels = np.array(labels ).T
label_df = pd.DataFrame(labels, columns=columns)
 result_df = pd.concat(( result_df, label_df), axis=1)
 return result_df<create_dataframe>
test_X = pd.read_csv(".. /input/feature2/test_X_features_10.csv")
 test_X.shape
label_convert_count = 12
train_feature2 = pd.DataFrame()
test_feature2 = pd.DataFrame()
for column in train_feature.columns:
train_feature2 = convert_label(train_feature, train_feature2, column, label_convert_count)
 test_feature2 = convert_label(test_feature, test_feature2, column, label_convert_count)
 train_feature = train_feature2.copy()
test_feature = test_feature2.copy()
del train_feature2, test_feature2<categorify>
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )
features = ['question_title', 'question_user_name', 'answer_user_name']
limits = [6, 8, 8]
for col, limit in zip(features, limits):
value = train_df[col].value_counts()
train_df['item_count'] = train_df[col].map(value)
 train_df['item_value'] = train_df[col].copy()
train_df.loc[train_df.item_count < limit, 'item_value'] = "___
test_df['item_count'] = test_df[col].map(value ).fillna(1)
 test_df['item_value'] = test_df[col].copy()
test_df.loc[test_df.item_count < limit, 'item_value'] = "___
lb = LabelBinarizer()
lb.fit(train_df['item_value'])
 encode_train = lb.transform(train_df['item_value'])
 encode_test = lb.transform(test_df['item_value'])
 columns = ['LabelBinarizer_{}'.format(i)for i in range(encode_train.shape[1])]
print("{}: {}".format(col, len(train_df['item_value'].value_counts())))
 encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del lb
train_df.drop(['item_count', 'item_count'], axis=1, inplace=True)
 test_df.drop(['item_count', 'item_count'], axis=1, inplace=True )<categorify>
train_columns = train_X.columns
n_fold = 3
features = ['url_0', 'category']
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(train_df[features])
 encode_train = enc.transform(train_df[features] ).toarray()
encode_test = enc.transform(test_df[features] ).toarray()
columns = ['encode_{}'.format(i)for i in range(encode_train.shape[1])]
encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del encode_train, encode_test, enc<data_type_conversions>
%%time
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 oof = np.zeros(len(train_X))
train_score = []
fold_idxs = []
predictions = np.zeros(len(test_X))
xgb_params = {'eta': 0.05,
'max_depth': 10,
'subsample': 0.8,
'objective': 'reg:linear',
'eval_metric': 'mae',
'nthread': 4}
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 fold_idxs.append(val_idx)
 X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
train_data = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 valid_data = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_tr.columns)
 watchlist = [(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=10000, evals=watchlist, early_stopping_rounds=40,params=xgb_params)
 oof[val_idx] = model.predict(xgb.DMatrix(X_val, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 predictions += model.predict(xgb.DMatrix(test_X[train_columns], feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)/folds.n_splits
cv_score = mean_absolute_error(train_y, oof)
 print(cv_score )
for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))<define_variables>
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 cat_saved=pd.DataFrame(oof,columns=['oof'])
 cat_saved.to_csv('xgb_oof.csv',index=False)
 submission["time_to_failure"] = predictions
submission.to_csv(f'xgb_submission_{cv_score:.3f}.csv', index=False)
 submission.head()