Kaggle competition: University of Liverpool - Ion Switching
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()<split>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import mode
from sklearn.metrics import f1_score, accuracy_score
from cuml.neighbors import KNeighborsClassifier, NearestNeighbors
import cuml; cuml.__version__
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()<feature_engineering>
%%time
KNN = 100
batch = 1000
test_pred = np.zeros(( test.shape[0]),dtype=np.int8)
 for g in [0,1,2,3,4]:
print('Infering group %i'%g)
 data = train.loc[train.group==g]
X_train = np.zeros(( len(data)-6,7))
X_train[:,0] = 0.25*data.signal[:-6]
X_train[:,1] = 0.5*data.signal[1:-5]
X_train[:,2] = 1.0*data.signal[2:-4]
X_train[:,3] = 4.0*data.signal[3:-3]
X_train[:,4] = 1.0*data.signal[4:-2]
X_train[:,5] = 0.5*data.signal[5:-1]
X_train[:,6] = 0.25*data.signal[6:]
y_train = data.open_channels[3:].values
data = test.loc[test.group==g]
X_test = np.zeros(( len(data)-6,7))
X_test[:,0] = 0.25*data.signal[:-6]
X_test[:,1] = 0.5*data.signal[1:-5]
X_test[:,2] = 1.0*data.signal[2:-4]
X_test[:,3] = 4.0*data.signal[3:-3]
X_test[:,4] = 1.0*data.signal[4:-2]
X_test[:,5] = 0.5*data.signal[5:-1]
X_test[:,6] = 0.25*data.signal[6:]
model = NearestNeighbors(n_neighbors=KNN)
 model.fit(X_train)
 distances, indices = model.kneighbors(X_test)
 ct = indices.shape[0]
pred = np.zeros(( ct+6),dtype=np.int8)
 it = ct//batch + int(ct%batch!=0)
 for k in range(it):
a = batch*k; b = batch*(k+1); b = min(ct,b)
 pred[a+3:b+3] = mode(y_train[ indices[a:b].astype(int)], axis=1)[0][:,0]
test_pred[test.group==g] = pred
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_mean'] = test_df['building_id'].map(building_mean)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 test_df['building_min'] = test_df['building_id'].map(building_min)
 test_df['building_max'] = test_df['building_id'].map(building_max)
 test_df['building_std'] = test_df['building_id'].map(building_std)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_from_disk>
sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 sub.open_channels = test_pred
sub.to_csv('submission.csv',index=False,float_format='%.4f')
 res=200
plt.figure(figsize=(20,5))
plt.plot(sub.time[::res],sub.open_channels[::res])
 plt.show()