Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<drop_column>
df = pd.read_csv(".. /input/remove-trends-giba-explained/train_clean_giba.csv" ).sort_values("time" ).reset_index(drop=True)
 test_df = pd.read_csv(".. /input/remove-trends-giba-explained/test_clean_giba.csv" ).sort_values("time" ).reset_index(drop=True )
idx_to_drop = list(train[(train['meter'] == "Electricity")&(train['meter_reading'] == 0)].index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )<drop_column>
multiprocessing.cpu_count()
idx_to_drop = list(( train[(train['building_id']==1099)&(train['meter_reading'] > 30000)&(train['meter'] == "Steam")] ).index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )<feature_engineering>
df["group"] = np.arange(df.shape[0])//500_000
df["mg"] = np.arange(df.shape[0])//100_000
df["group"].value_counts()
train['meter_reading'] = np.log1p(train['meter_reading'] )<categorify>
df["category"] = 0
test_df["category"] = 0
df.loc[2_000_000:2_500_000-1, 'category'] = 1
df.loc[4_500_000:5_000_000-1, 'category'] = 1
test_df.loc[500_000:600_000-1, "category"] = 1
test_df.loc[700_000:800_000-1, "category"] = 1
%%time
mean_meter_reading_per_building = train.groupby('building_id')['meter_reading'].mean()
train['mean_meter_reading_per_building'] = train['building_id'].map(mean_meter_reading_per_building)
 median_meter_reading_per_building = train.groupby('building_id')['meter_reading'].median()
train['median_meter_reading_per_building'] = train['building_id'].map(median_meter_reading_per_building)
 std_meter_reading_per_building = train.groupby('building_id')['meter_reading'].std()
train['std_meter_reading_per_building'] = train['building_id'].map(std_meter_reading_per_building)
 mean_meter_reading_per_dayofweek = train.groupby('DayOfWeek')['meter_reading'].mean()
train['mean_meter_reading_per_dayofweek'] = train['DayOfWeek'].map(mean_meter_reading_per_dayofweek)
 median_meter_reading_per_dayofweek = train.groupby('DayOfWeek')['meter_reading'].median()
train['median_meter_reading_per_dayofweek'] = train['DayOfWeek'].map(median_meter_reading_per_dayofweek)
 std_meter_reading_per_dayofweek = train.groupby('DayOfWeek')['meter_reading'].std()
train['std_meter_reading_per_dayofweek'] = train['DayOfWeek'].map(std_meter_reading_per_dayofweek)
 mean_meter_reading_per_meter = train.groupby('meter')['meter_reading'].mean()
train['mean_meter_reading_per_meter'] = train['meter'].map(mean_meter_reading_per_meter)
 median_meter_reading_per_meter = train.groupby('meter')['meter_reading'].median()
train['median_meter_reading_per_meter'] = train['meter'].map(median_meter_reading_per_meter)
 std_meter_reading_per_meter = train.groupby('meter')['meter_reading'].std()
train['std_meter_reading_per_meter'] = train['meter'].map(std_meter_reading_per_meter)
 mean_meter_reading_per_month = train.groupby('Month')['meter_reading'].mean()
train['mean_meter_reading_per_month'] = train['Month'].map(mean_meter_reading_per_month)
 median_meter_reading_per_month = train.groupby('Month')['meter_reading'].median()
train['median_meter_reading_per_month'] = train['Month'].map(median_meter_reading_per_month)
 std_meter_reading_per_month = train.groupby('Month')['meter_reading'].std()
train['std_meter_reading_per_month'] = train['Month'].map(std_meter_reading_per_month)
 test['mean_meter_reading_per_building'] = test['building_id'].map(mean_meter_reading_per_building)
 test['median_meter_reading_per_building'] = test['building_id'].map(median_meter_reading_per_building)
 test['std_meter_reading_per_building'] = test['building_id'].map(std_meter_reading_per_building)
 test['mean_meter_reading_per_dayofweek'] = test['DayOfWeek'].map(mean_meter_reading_per_dayofweek)
 test['median_meter_reading_per_dayofweek'] = test['DayOfWeek'].map(median_meter_reading_per_dayofweek)
 test['std_meter_reading_per_dayofweek'] = test['DayOfWeek'].map(std_meter_reading_per_dayofweek)
 test['mean_meter_reading_per_meter'] = test['meter'].map(mean_meter_reading_per_meter)
 test['median_meter_reading_per_meter'] = test['meter'].map(median_meter_reading_per_meter)
 test['std_meter_reading_per_meter'] = test['meter'].map(std_meter_reading_per_meter)
 test['mean_meter_reading_per_month'] = test['Month'].map(mean_meter_reading_per_month)
 test['median_meter_reading_per_month'] = test['Month'].map(median_meter_reading_per_month)
 test['std_meter_reading_per_month'] = test['Month'].map(std_meter_reading_per_month )<data_type_conversions>
TARGET = "open_channels"
aug_df = df[df["group"] == 5].copy()
aug_df["category"] = 1
aug_df["group"] = 10
for col in ["signal", TARGET]:
aug_df[col] += df[df["group"] == 8][col].values
df = df.append(aug_df, sort=False )
%%time
for df in [train, test]:
df['mean_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['median_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['std_meter_reading_per_building'] = df['std_meter_reading_per_building'].astype("float16")
 df['mean_meter_reading_per_meter'] = df['mean_meter_reading_per_meter'].astype("float16")
 df['median_meter_reading_per_meter'] = df['median_meter_reading_per_meter'].astype("float16")
 df['std_meter_reading_per_meter'] = df['std_meter_reading_per_meter'].astype("float16")
 df['mean_meter_reading_per_dayofweek'] = df['mean_meter_reading_per_dayofweek'].astype("float16")
 df['median_meter_reading_per_dayofweek'] = df['median_meter_reading_per_dayofweek'].astype("float16")
 df['std_meter_reading_per_dayofweek'] = df['std_meter_reading_per_dayofweek'].astype("float16")
 df['mean_meter_reading_per_month'] = df['mean_meter_reading_per_month'].astype("float16")
 df['median_meter_reading_per_month'] = df['median_meter_reading_per_month'].astype("float16")
 df['std_meter_reading_per_month'] = df['std_meter_reading_per_month'].astype("float16")
 df['Age'] = df['Age'].astype('uint8')
 gc.collect()<drop_column>
df.groupby("group")["signal"].agg({"mean", "std"} )
train.drop(['timestamp','year_built'],axis=1,inplace=True)
 test.drop(['timestamp','year_built'],axis=1,inplace=True )<data_type_conversions>
%%time
le = LabelEncoder()
train['meter']= le.fit_transform(train['meter'] ).astype("uint8")
 test['meter']= le.fit_transform(test['meter'] ).astype("uint8")
 train['primary_use']= le.fit_transform(train['primary_use'] ).astype("uint8")
 test['primary_use']= le.fit_transform(test['primary_use'] ).astype("uint8" )<data_type_conversions>
%%time
NUM_FOLDS = 5
skf = StratifiedKFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 oof_preds = np.zeros(( len(df), 11))
y_test = np.zeros(( len(test_df), 11))
for fold,(train_ind, val_ind)in enumerate(skf.split(df, df["group"])) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
print(fold, len(train_df), len(val_df))
for cat in range(2):
fit_df = train_df[train_df["category"] == cat]
y = fit_df[TARGET].values
y[0] = 0
model = xgb.XGBRFClassifier(
 n_estimators = 40,
learning_rate=1,
subsample=0.50,
colsample_bynode=0.25,
reg_lambda=1e-05,
objective= 'multi:softmax',
num_class= len(np.unique(y)) ,
max_depth = 17,
num_parallel_tree = 1,
tree_method = 'gpu_hist',
n_jobs = 2,
verbosity = 0,
predictor = 'gpu_predictor',
).fit(fit_df[features].values, y)
 pred = model.predict_proba(val_df[val_df["category"] == cat][features].values)
 oof_preds[val_ind[np.where(val_df["category"].values == cat)[0]], :pred.shape[1]] = pred
y_test[np.where(test_df["category"].values == cat)[0], :pred.shape[1]] += model.predict_proba(test_df[test_df["category"] == cat][features].values)/NUM_FOLDS
del model; _=gc.collect()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper<drop_column>
f1_score(df["open_channels"], oof_preds.argmax(axis=1), average="macro" )
to_drop = [column for column in upper.columns if any(upper[column] > threshold)]
print('There are %d columns to remove.' %(len(to_drop)))
 print("Following columns can be dropped {}".format(to_drop))<drop_column>
test_df[TARGET] = y_test.argmax(axis=1)
 test_df.iloc[:600_000][TARGET].value_counts() /600_000
<prepare_x_and_y><EOS> 
np.save("y_oof.npy", oof_preds)
 np.save("y_test.npy", y_test)
 test_df.to_csv('submission.csv', index=False, float_format='%.4f', columns=["time", TARGET] )