Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<sort_values>
print(os.listdir(".. /input"))
style.use('ggplot')
 warnings.filterwarnings('ignore')
 py.init_notebook_mode(connected=True)
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
X_train = pd.read_csv('.. /input/X_train.csv')
 X_train.head(3 )
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.25*df1['Confidence'] + 0.75*df2['Confidence']
df.head()<save_to_csv>
y_train = pd.read_csv('.. /input/y_train.csv')
 y_train.head(3 )
df.to_csv('submission.csv', index=False )<define_variables>
X_test = pd.read_csv('.. /input/X_test.csv')
 X_test.head(3 )
MODEL_DIR = '.. /input/ranzcr-resnext50-starter-models/'
OUTPUT_DIR = './'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TEST_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test'<define_variables>
target = y_train['surface'].value_counts().reset_index().rename(columns = {'index' : 'target'})
 target
class CFG:
debug=False
num_workers=4
model_name='resnext50_32x4d'
size=600
batch_size=64
seed=42
target_size=11
target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',
'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',
'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',
'Swan Ganz Catheter Present']
n_fold=4
trn_fold=[0, 1, 2, 3]<compute_test_metric>
X_train.isnull().sum()
def get_score(y_true, y_pred):
scores = []
for i in range(y_true.shape[1]):
score = roc_auc_score(y_true[:,i], y_pred[:,i])
 scores.append(score)
 avg_score = np.mean(scores)
 return avg_score, scores
def get_result(result_df):
preds = result_df[[f'pred_{c}' for c in CFG.target_cols]].values
labels = result_df[CFG.target_cols].values
score, scores = get_score(labels, preds)
 LOGGER.info(f'Score: {score:<.4f} Scores: {np.round(scores, decimals=4)}')
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def init_logger(log_file=OUTPUT_DIR+'inference.log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
LOGGER = init_logger()
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed )<load_from_csv>
X_train['is_duplicate'] = X_train.duplicated()
X_train['is_duplicate'].value_counts()
train = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/train.csv')
 folds = train.copy()
Fold = GroupKFold(n_splits=CFG.n_fold)
 groups = folds['PatientID'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[CFG.target_cols], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 display(folds.groupby('fold' ).size() )<feature_engineering>
X_train = X_train.drop(['is_duplicate'], axis = 1 )
oof_df = pd.DataFrame()
for fold in CFG.trn_fold:
valid_folds = folds[folds['fold']==fold].reset_index(drop=True)
 check_point = torch.load(MODEL_DIR+f'{CFG.model_name}_fold{fold}_best.pth', map_location=device)
 for c in [f'pred_{c}' for c in CFG.target_cols]:
valid_folds[c] = np.nan
valid_folds[[f'pred_{c}' for c in CFG.target_cols]] = check_point['preds']
LOGGER.info(f"========== fold: {fold} result ==========")
 get_result(valid_folds)
 oof_df = pd.concat([oof_df, valid_folds])
 oof_df = oof_df.reset_index(drop=True)
 LOGGER.info(f"========== CV ==========")
 get_result(oof_df )<load_from_csv>
X_train_sort = X_train.sort_values(by = ['series_id', 'measurement_number'], ascending = True)
 X_train_sort.head()
test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 print(test.shape)
 test.head()<categorify>
df = X_train.merge(y_train, on = 'series_id', how = 'inner')
 targets =(y_train['surface'].value_counts() ).index
def get_transforms(*, data):
if data == 'train':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
])
 elif data == 'valid':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
] )<choose_model_class>
series_dict = {}
for series in(X_train['series_id'].unique()):
series_dict[series] = X_train[X_train['series_id'] == series]
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x<predict_on_test>
def filter_signal(signal, threshold=1e3):
fourier = rfft(signal)
 frequencies = rfftfreq(signal.size, d=20e-3/signal.size)
 fourier[frequencies > threshold] = 0
return irfft(fourier )
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.sigmoid().to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<save_to_csv>
X_train_denoised = X_train.copy()
X_test_denoised = X_test.copy()
model = CustomResNext(CFG.model_name, pretrained=False)
 states = [torch.load(MODEL_DIR+f'{CFG.model_name}_fold{fold}_best.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 test[CFG.target_cols] = predictions
test[['StudyInstanceUID'] + CFG.target_cols].to_csv(OUTPUT_DIR+'submission.csv', index=False)
 test.head()<install_modules>
for col in X_train.columns:
if col[0:3] == 'ang' or col[0:3] == 'lin':
denoised_data = X_train.groupby(['series_id'])[col].apply(lambda x: filter_signal(x))
list_denoised_data = []
for arr in denoised_data:
for val in arr:
list_denoised_data.append(val)
 X_train_denoised[col] = list_denoised_data
for col in X_test.columns:
if col[0:3] == 'ang' or col[0:3] == 'lin':
denoised_data = X_test.groupby(['series_id'])[col].apply(lambda x: filter_signal(x))
list_denoised_data = []
for arr in denoised_data:
for val in arr:
list_denoised_data.append(val)
 X_test_denoised[col] = list_denoised_data
!pip install /kaggle/input/kerasapplications -q
!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps<import_modules>
series_dict = {}
for series in(X_train_denoised['series_id'].unique()):
series_dict[series] = X_train_denoised[X_train_denoised['series_id'] == series]
import os
import efficientnet.tfkeras as efn
import numpy as np
import pandas as pd
import tensorflow as tf<categorify>
def quaternion_to_euler(qx,qy,qz,qw):
sinr_cosp = +2.0 *(qw * qx + qy + qz)
 cosr_cosp = +1.0 - 2.0 *(qx * qx + qy * qy)
 roll = math.atan2(sinr_cosp, cosr_cosp)
 sinp = +2.0 *(qw * qy - qz * qx)
 if(math.fabs(sinp)>= 1):
pitch = copysign(M_PI/2, sinp)
 else:
pitch = math.asin(sinp)
 siny_cosp = +2.0 *(qw * qz + qx * qy)
 cosy_cosp = +1.0 - 2.0 *(qy * qy + qz * qz)
 yaw = math.atan2(siny_cosp, cosy_cosp)
 return roll, pitch, yaw
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset<define_variables>
def eular_angle(data):
x, y, z, w = data['orientation_X'].tolist() , data['orientation_Y'].tolist() , data['orientation_Z'].tolist() , data['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 data['euler_x'] = nx
data['euler_y'] = ny
data['euler_z'] = nz
return data
COMPETITION_NAME = "ranzcr-clip-catheter-line-classification"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16<load_from_csv>
data = eular_angle(X_train_denoised)
 test = eular_angle(X_test_denoised)
 print(data.shape, test.shape )
IMSIZE =(224, 240, 260, 300, 380, 456, 528, 600)
 load_dir = f"/kaggle/input/{COMPETITION_NAME}/"
sub_df = pd.read_csv(load_dir + 'sample_submission.csv')
 test_paths = load_dir + "test/" + sub_df['StudyInstanceUID'] + '.jpg'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[7], IMSIZE[7]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder
)<save_to_csv>
def fe_eng1(data):
data['total_angular_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['total_linear_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['total_orientation'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['total_linear_acc'] / data['total_angular_vel']
data['total_angle'] =(data['euler_x'] ** 2 + data['euler_y'] ** 2 + data['euler_z'] ** 2)** 5
data['angle_vs_acc'] = data['total_angle'] / data['total_linear_acc']
data['angle_vs_vel'] = data['total_angle'] / data['total_angular_vel']
return data
sub_df[label_cols] = model.predict(dtest, verbose=1)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<set_options>
def fe_eng2(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mad'] = data.groupby(['series_id'])[col].apply(lambda x: np.median(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
pd.set_option('max_columns', 50)
 pd.options.mode.chained_assignment = None<split>
data.fillna(0, inplace = True)
 data.replace(-np.inf, 0, inplace = True)
 data.replace(np.inf, 0, inplace = True)
 test.fillna(0, inplace = True)
 test.replace(-np.inf, 0, inplace = True)
 test.replace(np.inf, 0, inplace = True )
env = twosigmanews.make_env()
market_train, news_train = env.get_training_data()<set_options>
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )
del news_train
gc.collect()<categorify>
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=60)
 predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( data.shape[0]))
score = 0
class InitialPreprocessing(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
print("Initial Preprocessing")
 return self
def transform(self, X, y = None):
if len(X)== 1:
X[0]['date'] = X[0]['time'].dt.date
X[0] = X[0].loc[X[0]['date'] >= date(2010, 1, 1)]
else:
X[0]['date'] = X[0]['time'].dt.date
X[1]['date'] = X[1]['time'].dt.date
return X<define_search_space>
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,y_train['surface'].values)) :
model = RandomForestClassifier(n_estimators=700, n_jobs = -1)
 model.fit(data.iloc[trn_idx],y_train['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],y_train['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],y_train['surface'][val_idx])))
 gc.collect()
class AddingQuantFeatures(BaseEstimator, TransformerMixin):
def __init__(self):
self.feature_cols = ['close', 'open', 'volume']
self.moving_average_number_of_days = [10, 30, 90]
self.exponential_moving_average_span = [10, 30, 90]
self.bollinger_band_number_of_days = [7]
self.ewma = pd.Series.ewm
self.no_of_std = 2
self.returns_features = ['returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
self.drop_features = ['time', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
self.final_drop_features = []
def _compute_moving_average(self, X):
for column in self.feature_cols:
for window_period in self.moving_average_number_of_days:
feature_column_name = 'moving_average_{0}_{1}_days'.format(column, window_period)
 X[feature_column_name] = X.groupby('assetCode')[column].apply(lambda x: x.rolling(window_period ).mean())
 return X
def _compute_exponential_moving_average(self, X):
for column in self.feature_cols:
for span_period in self.exponential_moving_average_span:
feature_column_name = 'exponential_moving_average_{0}_{1}_days'.format(column, span_period)
 X[feature_column_name] = X.groupby('assetCode')[column].apply(lambda x: self.ewma(x, span = span_period ).mean())
 return X
def _compute_macd(self, X):
for column in self.feature_cols[1:3]:
feature_column_name = 'macd_{0}'.format(column)
 X[feature_column_name] =(X.groupby('assetCode')[column].apply(lambda x: self.ewma(x, span = 12 ).mean())-
X.groupby('assetCode')[column].apply(lambda x: self.ewma(x, span = 26 ).mean()))
return X
def _compute_bollinger_band(self, X):
for column in self.feature_cols[1:3]:
for window_period in self.bollinger_band_number_of_days:
bb_high_feature_column_name = 'bollinger_band_{0}_{1}_days_high'.format(column, window_period)
 bb_low_feature_column_name = 'bollinger_band_{0}_{1}_days_low'.format(column, window_period)
 std_feature_column_name = 'moving_average_std_{0}_{1}_days'.format(column, window_period)
 moving_average_feature_column_name = 'moving_average_{0}_{1}_days'.format(column, window_period)
 X[std_feature_column_name] = X.groupby('assetCode')[column].apply(lambda x: x.rolling(window_period ).std())
 X[bb_high_feature_column_name] = X[moving_average_feature_column_name] + self.no_of_std * X[std_feature_column_name]
X[bb_low_feature_column_name] = X[moving_average_feature_column_name] - self.no_of_std * X[std_feature_column_name]
return X
def _compute_rsi(self, X):
return
def _compute_miscellaneous_features(self, X):
for column in self.returns_features:
feature_column_name = '{0}_average'.format(column)
 mean_by_date = X.groupby('date')[column].agg({feature_column_name: nanmean} ).reset_index()
X = pd.merge(X, mean_by_date, how = 'left', on = ['date'])
 X['close_open_ratio'] = X['close']/X['open']
mean_by_date = X.groupby('date')['close_open_ratio'].agg({'close_open_ratio_average': nanmean} ).reset_index()
X = pd.merge(X, mean_by_date, how = 'left', on = ['date'])
 return X
def fit(self, X, y = None):
print("Adding Quant Features")
 self.quant_features_functions = [
self._compute_miscellaneous_features
]
return self
def transform(self, X, y = None):
for quant_function in self.quant_features_functions:
X[0] = quant_function(X[0])
 if len(X)== 1:
X = X[0]
else:
X[0] = X[0].drop(self.drop_features, 1)
 X = pd.merge(X[1], X[0], how = 'left', on = ['date', 'assetCode'])
 gc.collect()
return X<feature_engineering>
print('Average score', score / folds.n_splits )
class AddingLagFeatures(BaseEstimator, TransformerMixin):
def __init__(self):
self.shift_size = 1
self.n_lag = [3, 7, 14]
self.return_features = ['returnsClosePrevMktres10', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'open', 'close']
self.drop_features = [ 'time', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
def fit(self, X, y = None):
print("Adding Lag Features")
 return self
def _create_lag_features(self, X):
for col in self.return_features:
for window in self.n_lag:
rolled = X[col].shift(self.shift_size ).rolling(window = window)
 lag_mean = rolled.mean()
lag_max = rolled.max()
lag_min = rolled.min()
X['%s_lag_%s_mean'%(col, window)] = lag_mean
X['%s_lag_%s_max'%(col, window)] = lag_max
X['%s_lag_%s_min'%(col, window)] = lag_min
gc.collect()
return X.fillna(-1)
 def transform(self, X, y = None):
assetCodes = X[0]['assetCode'].unique()
all_df = []
df_codes = X[0].groupby('assetCode')
 df_codes = [df_code[1][['date', 'assetCode'] + self.return_features] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(self._create_lag_features, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(self.return_features, axis = 1, inplace = True)
 X[0] = pd.merge(X[0], new_df, how = 'left', on = ['date', 'assetCode'])
 if len(X)== 1:
X = X[0]
else:
X[0] = X[0].drop(self.drop_features, 1)
 X = pd.merge(X[1], X[0], how = 'left', on = ['date', 'assetCode'])
 pool.close()
del all_df, new_df, df_codes, assetCodes
gc.collect()
return X<categorify>
confusion_matrix(measured,y_train['surface'] )
class MissingValuesImputer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
print("Missing Values Imputer")
 return self
def transform(self, X, y = None):
for i in X.columns:
if X[i].dtype == "object":
X[i] = X[i].fillna("other")
 elif(X[i].dtype == "int64" or X[i].dtype == "float64"):
X[i] = X[i].fillna(X[i].mean())
 else:
pass
return X<categorify>
less_important_features = feature_importances.loc[feature_importances['importance'] < 0.0025]
print('There are {0} features their importance value is less then 0.0025'.format(less_important_features.shape[0]))
class AssetcodeEncoder(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
print("Encoding AssetCode")
 return self
def transform(self, X, y = None):
lbl = {k: v for v, k in enumerate(X['assetCode'].unique())}
X['assetCodeT'] = X['assetCode'].map(lbl)
 return X<choose_model_class>
for i, col in enumerate(less_important_features.index):
data = data.drop(columns = [col], axis = 1)
 test = test.drop(columns = [col], axis = 1)
 data.shape, test.shape
class LGBClassifierCV(BaseEstimator, RegressorMixin):
def __init__(self,
axis = 0,
lgb_params = None,
fit_params = None,
cv = 5,
perform_bayes_search = False,
perform_random_search = False,
use_train_test_split = False,
use_full_data = True,
use_kfold_split = True):
self.axis = axis
self.lgb_params = lgb_params
self.fit_params = fit_params
self.fit_params = fit_params
self.cv = cv
self.perform_random_search = perform_random_search
self.perform_bayes_search = perform_bayes_search
self.use_train_test_split = use_train_test_split
self.use_kfold_split = use_kfold_split
self.use_full_data = use_full_data
def _get_random_search_params(self, X, y):
lgb_search_params = {
'learning_rate': [0.15, 0.1, 0.05, 0.02, 0.01],
'num_leaves': [i for i in range(12, 90, 6)],
'n_estimators': [100, 200, 300, 400, 500, 600, 800],
'min_child_samples': [i for i in range(10, 100, 10)],
'colsample_bytree': [0.8, 0.9, 0.95, 1],
'subsample': [0.8, 0.9, 0.95, 1],
'reg_alpha': [0.1, 0.2, 0.4, 0.6, 0.8],
'reg_lambda': [0.1, 0.2, 0.4, 0.6, 0.8],
}
x_train, x_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, shuffle = False)
 best_eval_score = 0
for i in range(100):
print("random search iteration - " + str(i+1))
params = {k: np.random.choice(v)for k, v in lgb_search_params.items() }
params['n_jobs'] = cpu_count() - 1
score = self._evaluate_model(x_train, x_val, y_train, y_val, params)
 if score < best_eval_score or best_eval_score == 0:
best_eval_score = score
best_params = params
print("Best evaluation logloss", best_eval_score)
 print(best_params)
 return best_params
def _custom_metric(self, dates, pred_proba, num_target, universe):
y = 2*pred_proba[:, 1] - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : dates, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t)
 def _evaluate_model(self, x_train, x_val, y_train, y_val, params):
model = LGBMClassifier(**params)
 model.fit(x_train, y_train)
 return log_loss(y_val, model.predict_proba(x_val))
def find_best_params_(self, X, y):
if self.perform_random_search:
self.lgb_optimal_params = self._get_random_search_params(X, y)
 def fit(self, X, y = None, **fit_params):
print("LGBClassifierCV")
 dates = X['date']
num_target = X['returnsOpenNextMktres10'].astype('float32')
 y =(X['returnsOpenNextMktres10'] >= 0 ).astype('int8')
 universe = X['universe'].astype('int8')
 X.drop(['returnsOpenNextMktres10', 'universe', 'assetCode', 'assetName', 'time', 'date'], axis = 1, inplace = True)
 print(X.columns)
 self.mins = np.min(X, axis = 0)
 self.maxs = np.max(X, axis = 0)
 self.rng = self.maxs - self.mins
X = 1 -(( self.maxs - X)/self.rng)
 gc.collect()
if self.perform_random_search:
self.find_best_params_(X, y)
 else:
self.lgb_optimal_params = {'learning_rate': 0.01,
'num_leaves': 66,
'n_estimators': 200,
'min_child_samples': 40,
'colsample_bytree': 0.9,
'subsample': 0.9,
'reg_alpha': 0.4,
'reg_lambda': 0.2,
'n_jobs': 3}
x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
self.lgb_optimal_params_1 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': 239,
'max_bin': x_1[4],
'verbose': 1
}
self.lgb_optimal_params_2 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': 172,
'max_bin': x_2[4],
'verbose': 1
}
self.lgb_optimal_params["objective"] = "binary"
self.lgb_optimal_params["boosting_type"] = "gbdt"
if self.use_full_data:
train_data = lgb.Dataset(X, label = y)
 self.lgb_model_1 = lgb.train(self.lgb_optimal_params_1, train_set = train_data, num_boost_round = 100)
 self.lgb_model_2 = lgb.train(self.lgb_optimal_params_2, train_set = train_data, num_boost_round = 100)
 if self.use_train_test_split:
x_train, x_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, shuffle = False)
 train_data = lgb.Dataset(x_train, label = y_train)
 val_data = lgb.Dataset(x_val, label = y_val)
 self.lgb_model = lgb.train(self.lgb_optimal_params,
train_set = train_data,
num_boost_round = 100,
valid_sets = [val_data],
early_stopping_rounds = 5)
 if self.use_kfold_split:
kf = StratifiedKFold(n_splits = self.cv, shuffle = False)
 for fold_index,(train_data, val_data)in enumerate(kf.split(X, y)) :
print("Train Fold Index - " + str(fold_index))
lgb_model = lgb.LGBMClassifier(**self.lgb_optimal_params)
 lgb_model = lgb.train(self.lgb_optimal_params,
train_set = train_data,
num_boost_round = 100,
valid_sets = [val_data],
early_stopping_rounds = 5)
 self.estimators_.append(lgb_model)
 return self
def predict(self, X):
X.drop(['assetCode', 'assetName', 'time', 'date'], axis = 1, inplace = True)
 X = 1 -(( self.maxs - X)/self.rng)
 predictions =(self.lgb_model_1.predict(X)+ self.lgb_model_2.predict(X)) /2
predictions =(predictions - predictions.min())/(predictions.max() - predictions.min())
 predictions = predictions * 2 - 1
return predictions<categorify>
predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( data.shape[0]))
score = 0
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,y_train['surface'].values)) :
model = RandomForestClassifier(n_estimators=700, n_jobs = -1)
 model.fit(data.iloc[trn_idx],y_train['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],y_train['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],y_train['surface'][val_idx])))
 gc.collect()
data_pipeline = Pipeline([ 
('initial_preprocessor', InitialPreprocessing()), 
('quant_features_generator', AddingQuantFeatures()), 
('data_imputer', MissingValuesImputer()), 
('asset_encoder', AssetcodeEncoder()), 
('lgb', LGBClassifierCV(cv = 5,
perform_random_search = False,
use_train_test_split = False,
use_full_data = True,
use_kfold_split = False)
)
 ])
 data_pipeline.fit([market_train] )<define_variables>
print('Average score', score / folds.n_splits )
<train_model><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['surface'] = le.inverse_transform(predicted.argmax(axis=1))
submission.to_csv('rs_surface_submission6.csv', index=False)
 submission.head(10 )