Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<create_dataframe>
py.init_notebook_mode(connected=True )
lgb_train = lgb.Dataset(df.iloc[train_index], bin_target.iloc[train_index])
 lgb_eval = lgb.Dataset(df.iloc[test_index], bin_target.iloc[test_index], reference=lgb_train )<train_model>
def metric(y_true, y_pred):
return mean_squared_error(np.log1p(y_true), np.log1p(y_pred)) **(0.5 )
clf = lgb.train(
 best_params,
lgb_train,
valid_sets=lgb_eval,
early_stopping_rounds=5
)<predict_on_test>
USE_STACKING = False
USE_HYPEROPT = False
USE_GENETIC_ALG = True
EXCLUDE_LIST =[
'bland-nn-on-pp-leaks-train-fe_version3',
'bland-lgbt-on-leaks_version1',
'bland-lgbt-on-leaks_version2',
'ashrae-exploiting-leak-site-5',
'ashrae-1-1-to-1-06-with-ucl',
'ashrae-divide-and-conquer-fix0'
]
ADD_LIST = []
pred_y = clf.predict(df.iloc[test_index])
 pred_y[pred_y >= 0.5] = 1
pred_y[pred_y < 0.5] = 0
print(( pred_y == bin_target.iloc[test_index] ).sum() / len(pred_y))<feature_engineering>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
test_df_columns = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
base_df = market_train_df[market_train_df['time'] >= '2016-10-01']
base_df = base_df[test_df_columns]
base_df['id'] = -1
base_df.shape<categorify>
leak_df = reduce_mem_usage(pd.read_csv('/kaggle/input/leakaggregator/leaked_test_target.csv'))
not_nan_values_in_leak_df = ~leak_df['meter_reading'].isna()
def calc_target(df):
df['open_next10'] = df.groupby('assetCode')['open'].shift(-10)
 df['return_next10'] = df['open_next10'] / df['open'] - 1.0
df['target'] =(df.return_next10 >= 0 ).astype('int8')
 return df<prepare_output>
not_nan_values_in_leak_df.value_counts(normalize=True )
target_test_df = calc_target(market_train_df)
 target_test_df.head()<train_model>
datasets_to_take = [
'/kaggle/input/ashrae-submissions/*.csv',
'/kaggle/input/ashrae-submissions-1/*.csv',
'/kaggle/input/ashrae-submissions-2/*.csv',
'/kaggle/input/ashrae-submissions-3/*.csv',
'/kaggle/input/ashrae-submissions-4/*.csv',
'/kaggle/input/ashrae-submissions-5/*.csv',
'/kaggle/input/ashrae-submissions-6/*.csv'
]
files_to_take = []
for dtt in datasets_to_take:
files_to_take += glob(dtt )
def online_train(model, df, train_day_id, feats):
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 500,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}
train_df = calc_target(df)
 train_df = train_df[train_df['id'] == train_day_id]
train_df.reset_index(drop=True)
 target_df = train_df[['target']]
train_df = train_df[feats]
model.refit(train_df, target_df['target'])
 return model

<categorify>
all_sumbissions = pd.DataFrame()
all_sumbissions['row_id'] = leak_df['row_id']
if ADD_LIST:
print('ADD MODE')
 for df_name in tqdm(files_to_take):
if os.path.basename(df_name ).split('.')[0] in ADD_LIST:
print(df_name)
 all_sumbissions[os.path.basename(df_name ).split('.')[0]] = reduce_mem_usage(pd.read_csv(df_name)).sort_values('row_id')['meter_reading']
gc.collect()
else:
print('EXCLUDE MODE')
 for df_name in tqdm(files_to_take):
if os.path.basename(df_name ).split('.')[0] in EXCLUDE_LIST:
continue
print(df_name)
 all_sumbissions[os.path.basename(df_name ).split('.')[0]] = reduce_mem_usage(pd.read_csv(df_name)).sort_values('row_id')['meter_reading']
gc.collect()
def write_submission(model, env):
days = env.get_prediction_days()
day_id = 0
market_obs_df_append = None
online_train_df_append = None
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['id'] = day_id
if market_obs_df_append is None:
market_obs_df_append = base_df
market_obs_df_append = pd.concat([market_obs_df_append,market_obs_df],
ignore_index=True,
sort=False)
 market_obs_process = process_date(market_obs_df_append)
 market_obs_process = process_ma(market_obs_process)
 market_obs_df = market_obs_process[market_obs_process['id']==day_id]
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time', 'id']]
if online_train_df_append is None:
online_train_df_append = obs_df
else:
online_train_df_append = pd.concat([online_train_df_append, obs_df],
ignore_index=True,
sort=False)
 if day_id >= 10:
model = online_train(model, online_train_df_append, day_id - 10, feats)
 preds = model.predict(obs_df[feats])* 2 - 1
preds[np.where(abs(preds)< 0.05)] = 0.0
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 if day_id == 59:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==-1],
inplace=True)
 elif day_id >= 60:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==day_id-60],
inplace=True)
 day_id += 1
del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
print('day_count',day_id )<save_to_csv>
gc.collect()
write_submission(clf, env )<import_modules>
for col in all_sumbissions.columns[all_sumbissions.min() < 0]:
all_sumbissions.loc[all_sumbissions[col] < 0, col] = 0
import pandas as pd
import numpy as np
import gc<categorify>
gc.collect()
env = twosigmanews.make_env()<prepare_x_and_y>
for col in all_sumbissions.columns[1:]:
print(col)
 leak_score = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'],
all_sumbissions.loc[not_nan_values_in_leak_df, col])
 print('score1=', leak_score)
 gc.collect()
( market_train_df, news_train_df)= env.get_training_data()<count_missing_values>
leak_score = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,1:].mean(axis=1))
print('mean score=', leak_score )
print(market_train_df.isna().sum())
 print("---------------")
 print(market_train_df.shape[0] )<filter>
leak_score = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,1:].median(axis=1))
print('mean score=', leak_score )
market_train_df = market_train_df[market_train_df['universe'] == 1]
market_train_df = market_train_df[market_train_df['time'].dt.year >= 2012]
market_train_df.reset_index(drop=True, inplace=True )<count_missing_values>
class GenerativeAlgorithm(object):
def __init__(self,
x_shape,
amout_ind_to_mutate,
population_size,
target_function,
num_generations,
populations_to_take,
available_indices,
mode='min'):
self.amout_ind_to_mutate = amout_ind_to_mutate
self.x_shape = x_shape
self.population_size = population_size
self.target_function = target_function
self.num_generations = num_generations
self.populations_to_take = populations_to_take
self.available_indices = available_indices
self.mode = mode
self.gen_initial_generation()
if self.mode == 'min':
self.best_score = 1000000
if self.mode == 'max':
self.best_score = -1000000
def gen_initial_generation(self):
self.initial_generation = []
for i, el in enumerate(permutations(self.available_indices, self.x_shape)) :
if i == self.population_size:
break
self.initial_generation.append(list(el))
self.initial_generation = np.array(self.initial_generation)
 def select_mating_pool(self, population, fitness):
parents = np.empty(( self.populations_to_take, population.shape[1]))
for parent_num in range(self.populations_to_take):
if self.mode == 'min':
fitness_idx = np.argmin(fitness)
 elif self.mode == 'max':
fitness_idx = np.argmax(fitness)
 else:
raise ValueError('Not implemented!')
 parents[parent_num, :] = population[fitness_idx, :]
if self.mode == 'min':
fitness[fitness_idx] = 1000000
elif self.mode == 'max':
fitness[fitness_idx] = - 1000000
return parents
def crossover(self, parents, offspring_size):
offspring = np.empty(offspring_size)
 crossover_point = np.uint8(offspring_size[1]/2)
 for k in range(offspring_size[0]):
parent1_idx = np.random.randint(0, parents.shape[0])
 parent2_idx = np.random.randint(0, parents.shape[0])
 offspring[k, :crossover_point] = parents[parent1_idx, :crossover_point]
offspring[k, crossover_point:] = parents[parent2_idx, crossover_point:]
return offspring
def mutation(self, offspring_crossover):
for idx in range(offspring_crossover.shape[0]):
for i in range(self.amout_ind_to_mutate):
cur_available_idices = list(set(self.available_indices)- set(offspring_crossover[idx,:]))
value = np.random.choice(cur_available_idices)
 idx_1 = np.random.randint(0,offspring_crossover.shape[1])
 offspring_crossover[idx, idx_1] = value
return offspring_crossover
def apply_function_on_array(self, ar):
return np.array([self.target_function(ar[i,:])for i in range(ar.shape[0])])
 def fit(self):
cur_population = self.initial_generation
history = {'best_points':[], 'best_fits':[]}
generation = 0
while generation < self.num_generations:
fitness = self.apply_function_on_array(cur_population)
 if self.mode == 'min':
best_idx = np.argmin(fitness)
 elif self.mode == 'max':
best_idx = np.argmax(fitness)
 history['best_points'].append(cur_population[best_idx,:])
 history['best_fits'].append(fitness[best_idx])
 self.best_score = fitness[best_idx]
print('Itteration {} best score: {}'.format(generation, self.best_score))
parents = self.select_mating_pool(cur_population, fitness)
 offspring_crossover = self.crossover(parents,
offspring_size=(self.population_size-parents.shape[0], cur_population.shape[1]))
offspring_mutation = self.mutation(offspring_crossover)
 cur_population[:parents.shape[0], :] = parents
cur_population[parents.shape[0]:, :] = offspring_mutation
generation+=1
fitness = self.apply_function_on_array(cur_population)
 if self.mode == 'min':
best_idx = np.argmin(fitness)
 elif self.mode == 'max':
best_idx == np.argmax(fitness)
 history['best_points'].append(cur_population[best_idx,:])
 history['best_fits'].append(fitness[best_idx])
 return history
print(market_train_df.isna().sum())
 print("---------------")
 print(market_train_df.shape[0] )<data_type_conversions>
if USE_GENETIC_ALG:
def tgt_f(col_idx):
sc = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'],
all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,col_idx].median(axis=1))
return sc
a_ind = np.array(range(1, all_sumbissions.shape[1]-1))
alg = GenerativeAlgorithm(
 amout_ind_to_mutate=6,
population_size=20,
x_shape=11,
target_function=tgt_f,
num_generations=2,
populations_to_take=6,
available_indices=a_ind 
)
 h = alg.fit()
market_train_df['returnsClosePrevMktres1'].fillna(market_train_df['returnsClosePrevMktres1'].mean() , inplace=True)
 market_train_df['returnsOpenPrevMktres1'].fillna(market_train_df['returnsOpenPrevMktres1'].mean() , inplace=True)
 market_train_df['returnsClosePrevMktres10'].fillna(market_train_df['returnsClosePrevMktres10'].mean() , inplace=True)
 market_train_df['returnsOpenPrevMktres10'].fillna(market_train_df['returnsOpenPrevMktres10'].mean() , inplace=True)
 print(market_train_df.isna().sum())
 print("---------------")
 print(market_train_df.shape[0] )<count_missing_values>
if USE_GENETIC_ALG:
best_cols = h['best_points'][-1]
m = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,best_cols].median(axis=1))
print('best_comb: {}
best_metric: {}'.format(all_sumbissions.columns[best_cols], m))
print(news_train_df.isna().sum())
 print("---------------")
 print(news_train_df.shape[0] )<feature_engineering>
if USE_HYPEROPT:
score_comb = []
def objective(x):
cols_idx = [el[1] for el in x]
if len(set(cols_idx)) != len(cols_idx):
return 100
m = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions[not_nan_values_in_leak_df].iloc[:,cols_idx].median(axis=1))
gc.collect()
score_comb.append([cols_idx,m])
 return m
space = [(str(i),1 + hp.hp.randint(str(i), all_sumbissions.shape[1]-1)) for i in range(10)]
best = hp.fmin(objective, space, algo=hp.tpe.suggest, max_evals=1000 )
news_var = ['time','assetName', 'bodySize','companyCount','sentenceCount','wordCount',
'firstMentionSentence','relevance','sentimentClass','sentimentNegative',
'sentimentNeutral','sentimentPositive','sentimentWordCount','noveltyCount12H',
'noveltyCount24H','noveltyCount3D','noveltyCount5D','noveltyCount7D',
'volumeCounts12H','volumeCounts24H','volumeCounts3D','volumeCounts5D','volumeCounts7D'
]
news_train_df = news_train_df[news_var]
news_train_df = news_train_df[news_train_df.time.dt.year >= 2012]
news_train_df['date'] = news_train_df.time.dt.date
news_train_df.info()<merge>
if USE_HYPEROPT:
best_cols = pd.DataFrame(score_comb ).sort_values(1 ).iloc[0,0]
m = metric(leak_df.loc[not_nan_values_in_leak_df, 'meter_reading'], all_sumbissions.loc[not_nan_values_in_leak_df].iloc[:,best_cols].median(axis=1))
print('best_comb: {}
best_metric: {}'.format(all_sumbissions.columns[best_cols], m))
market_train_df['date'] = market_train_df.time.dt.date
market_train_df = pd.merge(market_train_df, news_train_df,how='left',on = ['assetName','date'])
 train_df = market_train_df
train_df.head()<set_options>
if USE_STACKING:
for col in all_sumbissions.columns[1:]:
all_sumbissions[col] = np.log1p(all_sumbissions[col])
 leak_df['meter_reading'] = np.log1p(leak_df['meter_reading'] )
del news_train_df
del market_train_df
gc.collect()<count_missing_values>
gc.collect()
print(train_df.isna().sum() / train_df.shape[0])
 print("-------------")
 print(train_df.shape[0] )<data_type_conversions>
if USE_STACKING:
all_sumbissions['target'] = leak_df['meter_reading']
del leak_df
gc.collect()
train_df['bodySize'].fillna(train_df['bodySize'].mean() , inplace=True)
 train_df['companyCount'].fillna(train_df['companyCount'].mean() , inplace=True)
 train_df['sentenceCount'].fillna(train_df['sentenceCount'].mean() , inplace=True)
 train_df['wordCount'].fillna(train_df['wordCount'].mean() , inplace=True)
 train_df['firstMentionSentence'].fillna(train_df['firstMentionSentence'].mean() , inplace=True)
 train_df['relevance'].fillna(train_df['relevance'].mean() , inplace=True)
 train_df['sentimentClass'].fillna(train_df['sentimentClass'].mean() , inplace=True)
 train_df['sentimentNegative'].fillna(train_df['sentimentNegative'].mean() , inplace=True)
 train_df['sentimentNeutral'].fillna(train_df['sentimentNeutral'].mean() , inplace=True)
 train_df['sentimentPositive'].fillna(train_df['sentimentPositive'].mean() , inplace=True)
 train_df['sentimentWordCount'].fillna(train_df['sentimentWordCount'].mean() , inplace=True)
 train_df['noveltyCount12H'].fillna(train_df['noveltyCount12H'].mean() , inplace=True)
 train_df['noveltyCount24H'].fillna(train_df['noveltyCount24H'].mean() , inplace=True)
 train_df['noveltyCount3D'].fillna(train_df['noveltyCount3D'].mean() , inplace=True)
 train_df['noveltyCount5D'].fillna(train_df['noveltyCount5D'].mean() , inplace=True)
 train_df['noveltyCount7D'].fillna(train_df['noveltyCount7D'].mean() , inplace=True)
 train_df['volumeCounts12H'].fillna(train_df['volumeCounts12H'].mean() , inplace=True)
 train_df['volumeCounts24H'].fillna(train_df['volumeCounts24H'].mean() , inplace=True)
 train_df['volumeCounts3D'].fillna(train_df['volumeCounts3D'].mean() , inplace=True)
 train_df['volumeCounts5D'].fillna(train_df['volumeCounts5D'].mean() , inplace=True)
 train_df['volumeCounts7D'].fillna(train_df['volumeCounts7D'].mean() , inplace=True)
 print(train_df.isna().sum() / train_df.shape[0])
 print("-------------")
 print(train_df.shape[0] )<filter>
if USE_STACKING:
train = all_sumbissions[not_nan_values_in_leak_df]
test = all_sumbissions[~not_nan_values_in_leak_df]
del all_sumbissions
gc.collect()
id_train = train_df.time.dt.year != 2016
id_test = train_df.time.dt.year == 2016
dep_var = 'returnsOpenNextMktres10'
ind_var = ['volume', 'close', 'open', 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','bodySize', 'companyCount', 'wordCount',
'firstMentionSentence', 'relevance', 'sentimentClass',
'sentimentWordCount', 'noveltyCount7D', 'volumeCounts7D']
df_train = train_df.loc[id_train,ind_var]
df_test = train_df.loc[id_test,ind_var]
print("{0} training rows and {1} testing rows".format(df_train.shape[0],df_test.shape[0]))
y_train = train_df.loc[id_train,dep_var]
y_test = train_df.loc[id_test,dep_var]<train_model>
if USE_STACKING:
train = train.set_index('row_id')
 test = test.set_index('row_id' )
xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
max_depth = 5, alpha = 10, n_estimators = 100)
 xg_reg.fit(df_train,y_train )<compute_train_metric>
gc.collect()
pred_train = xg_reg.predict(df_train)
 rms_train = sqrt(mean_squared_error(y_train, pred_train))
pred_test = xg_reg.predict(df_test)
 rms_test = sqrt(mean_squared_error(y_test, pred_test))
print('Train RMSE: {0} Test RMSE: {1}'.format(rms_train,rms_test))<feature_engineering>
class NNModel(object):
def __init__(self, model, target_variable='target'):
self.model = model
self.es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=EARLY_STOPPING, verbose=False, mode='auto', restore_best_weights=True)
 self.rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-6, mode='auto', verbose=False)
 self.target_variable = target_variable
self.scaling_stats = {}
self.predictors = ['bland-lgbt-folds_version_1', 'bland-lgbt-folds_version_2',
'pp-lgbt-refactored_version_7',
'bland-lgbt-on-pp-leaks-train-fe_version1',
'bland-nn-on-pp-leaks-train-fe_version1',
'Bland LGBT on PP Leaks Train FE_version4',
'bland-nn-on-pp-leaks-train-fe_version2',
'Bland LGBT on PP Leaks Train FE_version2']
def train_preprocessing(self, data):
y = data[self.target_variable]
data = data[self.predictors]
print(data.shape)
 for col in self.predictors:
self.scaling_stats[col] = {'mean':data[col].mean() , 'std':data[col].std() }
data[col] =(data[col] - self.scaling_stats[col]['mean'])/ self.scaling_stats[col]['std']
print(data.isna().sum().sum())
 print('Scaling completed!')
 gc.collect()
return data.values, y.values
def test_preprocessing(self, data, is_val=False):
if is_val:
y = data[self.target_variable]
data = data[self.predictors]
gc.collect()
for col in self.predictors:
data[col] =(data[col] - self.scaling_stats[col]['mean'])/ self.scaling_stats[col]['std']
gc.collect()
if is_val:
return data.values, y.values
else:
return data.values
def fit(self, data, data_val):
data, y_train = self.train_preprocessing(data)
 data_val, y_val = self.test_preprocessing(data_val, is_val=True)
 self.model.fit(
 data, y_train, epochs=N_EPOCHS, batch_size=BATCH_SIZE, validation_data=(data_val, y_val), verbose=True, callbacks=[self.es, self.rlr] 
)
 def predict(self, data):
data = self.test_preprocessing(data, is_val=False)
 return self.model.predict(data, batch_size=BATCH_SIZE, verbose=True ).flatten()
pred_test_df = train_df.loc[id_test,['time','assetCode','universe','returnsOpenNextMktres10']]
pred_test_df['dayofyear'] = pred_test_df.time.dt.dayofyear
pred_test_df['confidence'] = [1 if pred >=0 else -1 for pred in pred_test]
pred_test_df['score'] = pred_test_df.universe * pred_test_df.returnsOpenNextMktres10 * pred_test_df.confidence
print(pred_test_df.confidence.value_counts())
 score_1 = pred_test_df.groupby(['dayofyear'] ).score.sum()
score_2 = score_1.mean() / score_1.std()
print("
Competition Score: ",np.round(score_2,4))<merge>
def time_val(data, model_create_func, metric_to_use=mean_squared_error, target_var_name='target', test_to_predict=None):
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 print('Starting Validation')
 results = []
data['pred'] = 0
if test_to_predict is not None:
test_prediction = []
for train_idx, test_idx in kf.split(data):
print('New Itter')
 model = model_create_func()
model.fit(data.iloc[train_idx].reset_index(drop=True),
data.iloc[test_idx].reset_index(drop=True))
gc.collect()
data['pred'].iloc[test_idx] = model.predict(data.iloc[test_idx].reset_index(drop=True))
gc.collect()
itter_metric = metric_to_use(data.iloc[test_idx][target_var_name], data['pred'].iloc[test_idx])
 print('Itter metric: '+str(itter_metric))
results.append(itter_metric)
 gc.collect()
if test_to_predict is not None:
test_prediction.append(model.predict(test_to_predict))
gc.collect()
if test_to_predict is not None:
return results, sum(test_prediction)/NUM_FOLDS
else:
return results
def make_predictions(market_obs_df,news_obs_df,predictions_df,ind_var=ind_var,news_var=news_var,xg_reg=xg_reg):
news_obs_df = news_obs_df.loc[:,news_var]
news_obs_df['date'] = news_obs_df.time.dt.date
news_train_df_grp = news_obs_df.groupby(['date','assetName'] ).mean().reset_index()
market_obs_df['date'] = market_obs_df.time.dt.date
market_obs_df = pd.merge(market_obs_df,news_train_df_grp,how='left',on = ['assetName','date'])
 market_obs_df.fillna(0,inplace = True)
 test = market_obs_df.loc[:,ind_var]
predictions_df.confidenceValue = [1 if pred >=0 else -1 for pred in xg_reg.predict(test)]<predict_on_test>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
 def create_model(inp_dim):
inps = Input(shape=(inp_dim,))
x = Dense(1 )(inps)
 model = Model(inputs=inps, outputs=x)
 model.compile(
 optimizer=Nadam(lr=1e-3),
loss=root_mean_squared_error 
)
 return model
days = env.get_prediction_days()
for(market_obs_df, news_obs_df, predictions_template_df)in days:
make_predictions(market_obs_df, news_obs_df, predictions_template_df)
 env.predict(predictions_template_df)
 print('Done!' )<save_to_csv>
gc.collect()
env.write_submission_file()<compute_test_metric>
if USE_STACKING:
NUM_FOLDS = 5
N_EPOCHS = 3
BATCH_SIZE = 256
EARLY_STOPPING = 2
my_model_create_f = lambda : NNModel(model = create_model(8))
def vela(a,b):
return(a-b)/a
def ema(vela2, vela1, vela0):
ema0 = 2/4 *(vela0)
 ema1 = ema0 + 2/4 *(vela1 - ema0)
 ema2 = ema1 + 2/4 *(vela2 - ema1)
 return ema2 
<drop_column>
if USE_STACKING:
rf_res, test['target'] = time_val(train, my_model_create_f, test_to_predict=test )
def get_x(market_train_df):
x = market_train_df
try:
x.drop(columns=['assetCode'], inplace=True)
 except:
pass
try:
x.drop(columns=['assetName'], inplace=True)
 except:
pass
try:
x.drop(columns=['returnsOpenNextMktres10'], inplace=True)
 except:
pass
try:
x.drop(columns=['universe'], inplace=True)
 except:
pass
try:
x.drop(columns=['returnsClosePrevRaw1'], inplace=True)
 except:
pass
try:
x.drop(columns=['returnsOpenPrevRaw1'], inplace=True)
 except:
pass
try:
x.drop(columns=['returnsOpenPrevRaw10'], inplace=True)
 except:
pass
try:
x.drop(columns=['returnsClosePrevRaw10'], inplace=True)
 except:
pass
x['vela'] = vela(market_train_df['close'],market_train_df['open'])
 x['vela1'] = vela(market_train_df['returnsClosePrevMktres1'],market_train_df['returnsOpenPrevMktres1'])
 x['vela10'] = vela(market_train_df['returnsClosePrevMktres10'],market_train_df['returnsOpenPrevMktres10'])
 try:
x.drop(columns='time', inplace=True)
 except:
pass
try:
x.drop(columns='volume', inplace=True)
 except:
pass
x.fillna(-1000,inplace=True)
 return x
def get_y(market_train_df):
y2 = market_train_df
try:
y = y2['returnsOpenNextMktres10']
except:
y = None
pass
return y
def get_xy(market_train_df):
y = get_y(market_train_df)
 x = get_x(market_train_df)
 return x, y		
def make_predictions(predictions_template_df, market_obs_df):
			
x = get_x(market_obs_df)
 x = x[train_cols]
data = xgb.DMatrix(x, nthread=-1)
 predictions_template_df.confidenceValue = np.clip(m.predict(data, ntree_limit=0),-1,1)
<train_model>
if USE_STACKING:
print('Result: {} +/- {}'.format(round(np.mean(rf_res),5), round(np.std(rf_res),5)))
 print(mean_squared_error(train['pred'], train['target']))
( market_train_df, news_train_df)= env.get_training_data()
print('Datos y News Recuperados!')
 x, y = get_xy(market_train_df)
 data = xgb.DMatrix(x, label=y, feature_names=x.columns,nthread=-1)
 print('Get Matrix!')
 train_cols = x.columns
params = {'max_depth':10,'min_child_weight': 1,'eta':0.8,'objective': 'reg:linear', 'eval_metric:' : "mae",}
m = xgb.train(params, data, 240, evals=[(data, 'Test')])
 print('Train!')
 for(market_obs_df, news_obs_df, predictions_template_df)in env.get_prediction_days() :
make_predictions(predictions_template_df, market_obs_df)
 env.predict(predictions_template_df)
 print('Ok!')
 xgb.plot_importance(m)
 env.write_submission_file() 
<define_variables>
if USE_STACKING:
train = train.rename(columns={'pred':'meter_reading'})
 test = test.rename(columns={'target':'meter_reading'})
 gc.collect()
sub = pd.concat([train['meter_reading'], test['meter_reading']], axis=0)
 sub = sub.reset_index()
sub['meter_reading'] = np.expm1(sub['meter_reading'])
 sub.loc[sub['meter_reading'] < 0, 'meter_reading'] = 0
sub = sub.sort_values('row_id' )
<define_variables><EOS> 
if USE_GENETIC_ALG or USE_HYPEROPT:
all_sumbissions['meter_reading'] = all_sumbissions.iloc[:,best_cols].median(axis=1)
 all_sumbissions = all_sumbissions[['row_id','meter_reading']]
gc.collect()
all_sumbissions[['row_id','meter_reading']].to_csv('submission_blend.csv', index=False)
 elif USE_STACKING:
sub.to_csv('submission_blend.csv', index=False)
 else:
raise ValueError('Not ready' )