Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
%load_ext autoreload
%autoreload 2
%matplotlib inline
fix_time_col(market_train_df, news_train_df)
 clean_assetName_col(market_train_df, news_train_df)
 dic_assetName, dic_assetNameSNo, dic_assetCodes = set_assetNameCodes_SNo(market_train_df, news_train_df,dic_assetName, dic_assetNameSNo, dic_assetCodes)
 replace_assetName(market_train_df, news_train_df)
 sector_name_df, asset_df, sector_df, all_df, news_feature_df = gen_news_features(market_train_df, news_train_df)
<drop_column>
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore' )
drop_cols = list(set(["companyCount","marketCommentary","assetCodesSNo","num_rows"])| setSector)
 news_feature_df.drop(drop_cols, axis=1, inplace=True)
 news_feature_df.tail(1)
<drop_column>
PATH = ".. /input/cleaned-data/"
drop_cols = ["sentimentNegative","sentimentPositive","sentimentNeutral"]
news_feature_df.drop(drop_cols, axis=1, inplace=True )<feature_engineering>
df_train=pd.read_csv(f'{PATH}train_clean.csv')
 df_test=pd.read_csv(f'{PATH}test_clean.csv' )
apple_df = news_feature_df[news_feature_df["assetName"]=="apple"]
apple_df["target"] = apple_df["returnsOpenNextMktres10"].apply(lambda x: 1 if x>0 else 2 if x<0 else 3)
 apple_df["asset_sector_SenSR"] = apple_df["smooth_asset_SenSR"] * apple_df["smooth_sector_SenSR"]
apple_df["asset_sector_all_SenSR"] = apple_df["smooth_asset_SenSR"] * apple_df["smooth_sector_SenSR"] * apple_df["smooth_all_SenSR"]
col_list1 = ["returnsClosePrevMktres10","returnsOpenNextMktres10",\
"asset_SenSR","rolling_asset_SenSR","smooth_asset_SenSR"]
col_list2 = ["returnsClosePrevMktres10","returnsOpenNextMktres10",\
"sector_SenSR","rolling_sector_SenSR","smooth_sector_SenSR"]
col_list3 = ["returnsClosePrevMktres10","returnsOpenNextMktres10",\
"asset_sector_SenSR","asset_sector_all_SenSR"]
for col in list(set(col_list1)|set(col_list2)|set(col_list3)) :
scale_col(apple_df, col)
 scale_col(apple_df, "smooth_all_SenSR")
<data_type_conversions>
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['SalePrice'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
env = twosigmanews.make_env() 
(marketdf, newsdf)= env.get_training_data()
print('preparing data...')
 def prepare_data(marketdf, newsdf):
marketdf['time'] = marketdf.time.dt.strftime("%Y%m%d" ).astype(int)
 marketdf['bartrend'] = marketdf['close'] / marketdf['open']
marketdf['average'] =(marketdf['close'] + marketdf['open'])/2
marketdf['pricevolume'] = marketdf['volume'] * marketdf['close']
newsdf['time'] = newsdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['assetCode'] = newsdf['assetCodes'].map(lambda x: list(eval(x)) [0])
 newsdf['position'] = newsdf['firstMentionSentence'] / newsdf['sentenceCount']
newsdf['coverage'] = newsdf['sentimentWordCount'] / newsdf['wordCount']
marketdf = marketdf.loc[marketdf['time'] > 20120000]
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'assetCodes','urgency','wordCount','sentimentWordCount']
newsdf.drop(droplist, axis=1, inplace=True)
 marketdf.drop(['assetName', 'volume'], axis=1, inplace=True)
 newsgp = newsdf.groupby(['time','assetCode'], sort=False ).aggregate(np.mean ).reset_index()
return pd.merge(marketdf, newsgp, how='left', on=['time', 'assetCode'], copy=False)
 cdf = prepare_data(marketdf, newsdf)
 del marketdf, newsdf<prepare_x_and_y>
ntrain = df_train.shape[0]
ntest = df_test.shape[0]
print('building training set...')
 targetcols = ['returnsOpenNextMktres10']
traincols = [col for col in cdf.columns if col not in ['time', 'assetCode', 'universe'] + targetcols]
dates = cdf['time'].unique()
train = range(len(dates)) [:int(0.85*len(dates)) ]
val = range(len(dates)) [int(0.85*len(dates)) :]
cdf[targetcols[0]] =(cdf[targetcols[0]] > 0 ).astype(int)
 Xt = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Yt = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Xv = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
Yv = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
print(Xt.shape, Xv.shape )<train_model>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
df_all.shape
print('Training XGBoost')
 xgbmodel = xgb.XGBClassifier(scale_pos_weight=100,
n_estimators=200,
max_depth=100, objective='binary:logistic')
 xgbmodel.fit(Xt, Yt[:,0],
eval_set=[(Xt, Yt[:,0]),(Xv, Yv[:,0])],
eval_metric='logloss',early_stopping_rounds=10)
<init_hyperparams>
y_train_full = df_train.SalePrice.values
df_all.drop(['SalePrice'], axis=1, inplace=True)
<predict_on_test>
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
skewness = df_all.select_dtypes(include=numerics ).apply(lambda x: skew(x))
skew_index = skewness[abs(skewness)>= 0.85].index
skewness[skew_index].sort_values(ascending=False )
print("generating predictions...")
 preddays = env.get_prediction_days()
for marketdf, newsdf, predtemplatedf in preddays:
cdf = prepare_data(marketdf, newsdf)
 Xp = cdf[traincols].fillna(0 ).values
preds = xgbmodel.predict(Xp)* 2-1
predsdf = pd.DataFrame({'ast':cdf['assetCode'],'conf':preds})
 predtemplatedf['confidenceValue'][predtemplatedf['assetCode'].isin(predsdf.ast)] = predsdf['conf'].values
env.predict(predtemplatedf)
 env.write_submission_file()<import_modules>

 lam = 0.15
for column in skew_index:
df_all[column] = boxcox1p(df_all[column], lam)
import os
import json
import cv2
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pydicom
from keras import layers
from keras.applications import DenseNet121
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import Callback, ModelCheckpoint
from keras.preprocessing.image import ImageDataGenerator
from keras.initializers import Constant
from keras.models import Sequential
from keras.optimizers import Adam
from tensorflow.python.ops import array_ops
from tqdm import tqdm
from keras import backend as K
import tensorflow as tf<define_variables>
df_all=pd.get_dummies(df_all )
BASE_PATH = '/kaggle/input/rsna-intracranial-hemorrhage-detection/'
TRAIN_DIR = 'stage_1_train_images/'
TEST_DIR = 'stage_1_test_images/'<feature_engineering>

 X_test=df_all.iloc[ntrain:]
X_train_full=df_all.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full )
train_df = pd.read_csv(BASE_PATH + 'stage_1_train.csv')
 sub_df = pd.read_csv(BASE_PATH + 'stage_1_sample_submission.csv')
 train_df['filename'] = train_df['ID'].apply(lambda st: "ID_" + st.split('_')[1] + ".png")
 train_df['type'] = train_df['ID'].apply(lambda st: st.split('_')[2])
 sub_df['filename'] = sub_df['ID'].apply(lambda st: "ID_" + st.split('_')[1] + ".png")
 sub_df['type'] = sub_df['ID'].apply(lambda st: st.split('_')[2])
 print(train_df.shape)
 train_df.head()<prepare_output>
all_features = df_all.keys()
df_all = df_all.drop(df_all.loc[:,(df_all==0 ).sum() >=(df_all.shape[0]*0.984)],axis=1)
 df_all = df_all.drop(df_all.loc[:,(df_all==1 ).sum() >=(df_all.shape[0]*0.984)],axis=1)
 remain_features = df_all.keys()
remov_features = [st for st in all_features if st not in remain_features]
print(len(remov_features), 'features were removed:', remov_features )
test_df = pd.DataFrame(sub_df.filename.unique() , columns=['filename'])
 print(test_df.shape)
 test_df.head()<define_variables>
scaler = RobustScaler()
df_all = pd.DataFrame(scaler.fit_transform(df_all))
np.random.seed(1749)
 sample_files = np.random.choice(os.listdir(BASE_PATH + TRAIN_DIR), 4000)
 sample_df = train_df[train_df.filename.apply(lambda x: x.replace('.png', '.dcm')).isin(sample_files)]<count_duplicates>

 X_test=df_all.iloc[ntrain:]
X_train_full=df_all.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full )
pivot_df = sample_df[['Label', 'filename', 'type']].drop_duplicates().pivot(
 index='filename', columns='type', values='Label' ).reset_index()
print(pivot_df.shape)
 pivot_df.head()<categorify>
def rmse(x,y): return math.sqrt(((x-y)**2 ).mean())
 def print_score(m,X_train=X_train, X_valid=X_valid, y_train=y_train, y_valid=y_valid):
res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )
def window_image(img, window_center,window_width, intercept, slope, rescale=True):
img =(img*slope +intercept)
 img_min = window_center - window_width//2
img_max = window_center + window_width//2
img[img<img_min] = img_min
img[img>img_max] = img_max
if rescale:
img =(img - img_min)/(img_max - img_min)
 return img
def get_first_of_dicom_field_as_int(x):
if type(x)== pydicom.multival.MultiValue:
return int(x[0])
 else:
return int(x)
 def get_windowing(data):
dicom_fields = [data[('0028','1050')].value,
data[('0028','1051')].value,
data[('0028','1052')].value,
data[('0028','1053')].value]
return [get_first_of_dicom_field_as_int(x)for x in dicom_fields]<train_model>
print('ElasticNet Lasso')
 def lasso_score(X,y):
lasso = ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
lasso = GridSearchCV(lasso, param, cv=5, scoring='neg_mean_squared_error')
 lasso.fit(X,y)
 print('Lasso:', np.sqrt(lasso.best_score_*-1))
return lasso
lasso_score(X_train, y_train)
 print('XGBoost')
 m_xgb = XGBRegressor(n_estimators=160, learning_rate=0.05)
 m_xgb.fit(X_train, y_train)
 print_score(m_xgb,X_train, X_valid, y_train, y_valid)
 print('Random Forest')
 m_rf = RandomForestRegressor(n_estimators=160, min_samples_leaf=1, max_features=0.5, n_jobs=-1, oob_score=True)
 m_rf.fit(X_train, y_train)
 print_score(m_rf,X_train, X_valid, y_train, y_valid)
save_and_resize(filenames=sample_files, load_dir=BASE_PATH + TRAIN_DIR)
 save_and_resize(filenames=os.listdir(BASE_PATH + TEST_DIR), load_dir=BASE_PATH + TEST_DIR )<prepare_x_and_y>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X_train_full=X_train_full):
rmse = np.sqrt(-cross_val_score(model, X_train_full, y_train_full,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
BATCH_SIZE = 64
def create_datagen() :
return ImageDataGenerator(validation_split=0.15)
 def create_test_gen() :
return ImageDataGenerator().flow_from_dataframe(
 test_df,
directory='/kaggle/tmp/',
x_col='filename',
class_mode=None,
target_size=(224, 224),
batch_size=BATCH_SIZE,
shuffle=False 
)
 def create_flow(datagen, subset):
return datagen.flow_from_dataframe(
 pivot_df,
directory='/kaggle/tmp/',
x_col='filename',
y_col=['any', 'epidural', 'intraparenchymal',
'intraventricular', 'subarachnoid', 'subdural'],
class_mode='multi_output',
target_size=(224, 224),
batch_size=BATCH_SIZE,
subset=subset 
)
 data_generator = create_datagen()
train_gen = create_flow(data_generator, 'training')
 val_gen = create_flow(data_generator, 'validation')
 test_gen = create_test_gen()<compute_train_metric>
print('StackingCVRegressor')
 stack_gen_model = stack_gen.fit(np.array(X_train_full), np.array(y_train_full))
elastic_model_full_data = elasticnet.fit(X_train_full, y_train_full)
 lasso_model_full_data = lasso.fit(X_train_full, y_train_full)
 ridge_model_full_data = ridge.fit(X_train_full, y_train_full)
 svr_model_full_data = svr.fit(X_train_full, y_train_full)
 gbr_model_full_data = gbr.fit(X_train_full, y_train_full)
 xgb_model_full_data = xgboost.fit(X_train_full, y_train_full)
 lgb_model_full_data = lightgbm.fit(X_train_full, y_train_full )
def focal_loss(prediction_tensor, target_tensor, weights=None, alpha=0.25, gamma=2):
r
sigmoid_p = tf.nn.sigmoid(prediction_tensor)
 zeros = array_ops.zeros_like(sigmoid_p, dtype=sigmoid_p.dtype)
 pos_p_sub = array_ops.where(target_tensor > zeros, target_tensor - sigmoid_p, zeros)
 neg_p_sub = array_ops.where(target_tensor > zeros, zeros, sigmoid_p)
 per_entry_cross_ent = - alpha *(pos_p_sub ** gamma)* tf.log(tf.clip_by_value(sigmoid_p, 1e-8, 1.0)) \
-(1 - alpha)*(neg_p_sub ** gamma)* tf.log(tf.clip_by_value(1.0 - sigmoid_p, 1e-8, 1.0))
return tf.reduce_sum(per_entry_cross_ent )<choose_model_class>
def blend_models_predict(X_train_full):
return(( 0.1 * elastic_model_full_data.predict(X_train_full)) + \ 
(0.05 * lasso_model_full_data.predict(X_train_full)) + \ 
(0.1 * ridge_model_full_data.predict(X_train_full)) + \ 
(0.1 * svr_model_full_data.predict(X_train_full)) + \ 
(0.1 * gbr_model_full_data.predict(X_train_full)) + \ 
(0.15 * xgb_model_full_data.predict(X_train_full)) + \ 
(0.1 * lgb_model_full_data.predict(X_train_full)) + \ 
(0.3 * stack_gen_model.predict(np.array(X_train_full))))
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
 )<choose_model_class>
print('RMSLE score on train data:')
 print(rmsle(y_train_full, blend_models_predict(X_train_full)) )
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(6, activation='sigmoid'))
model.compile(
 loss='categorical_crossentropy',
optimizer=Adam(lr=0.001),
metrics=['accuracy'] 
)
 return model<train_model>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_test)) )
<predict_on_test><EOS> 
submission.to_csv("submission.csv", index=False)
 print('Save submission', datetime.now() , )