Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
%matplotlib inline
warnings.filterwarnings("ignore")
 %matplotlib inline
from keras.models import Model, load_model
from keras.layers import Input, BatchNormalization, Activation, Add, Multiply, Dot
from keras.layers import Embedding, Permute, Reshape
from keras.layers.core import Dropout, Lambda, Dense, Flatten
from keras.layers.convolutional import Conv1D, Conv2D
from keras.layers.pooling import GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers.merge import Concatenate
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.optimizers import Adam, SGD, Nadam
from keras import backend as K
from keras.engine.topology import Layer
import tensorflow as tf<normalization>
train_data=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test_data['SalePrice']=np.nan
test_data['tmpPrice']=np.nan
class ScaleLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(ScaleLayer, self ).__init__(**kwargs)
 def build(self, input_shape):
super(ScaleLayer, self ).build(input_shape)
 def call(self, x):
xx = K.arange(-99, 100, dtype=tf.float32)
 mu = y_mean + tf.reshape(x[:, 0],(-1, 1))
sigma_minus = tf.identity(K.exp(0.5 * tf.reshape(x[:, 1],(-1, 1))), name="sigma")
 sigma_plus = tf.identity(K.exp(0.5 * tf.reshape(x[:, 2],(-1, 1))), name="sigma")
 xx = tf.subtract(xx, mu)
 pcf = tf.where(xx >= 0, tf.divide(xx, sigma_plus), tf.divide(xx, sigma_minus))
return pcf
def compute_output_shape(self, input_shape):
return(input_shape[0], self.output_dim )<compute_train_metric>
def cramers_v(x, y):
confusion_matrix = pd.crosstab(x,y)
 chi2 = scipy.stats.chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum().sum()
phi2 = chi2/n
r,k = confusion_matrix.shape
phi2corr = max(0, phi2-(( k-1)*(r-1)) /(n-1))
rcorr = r-(( r-1)**2)/(n-1)
 kcorr = k-(( k-1)**2)/(n-1)
 return np.sqrt(phi2corr/min(( kcorr-1),(rcorr-1)) )
def dist_mult(dist, players):
res = Lambda(lambda c: K.batch_dot(c[0], c[1]))([dist, players])
 return res
def dist_attention(dist, players, dropout):
if 1:
dist1 = Reshape(( 22, 22, 1))(dist)
 dist1 = Conv2D(16, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(dist1)
 dist1 = Conv2D(1, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(dist1)
 dist1 = Reshape(( 22, 22))(dist1)
 dist = Add()([dist, dist1])
 dist = LayerNormalization()(dist)
 att = dist_mult(dist, players,)
 x_player = Add()([players, att])
 x_player = LayerNormalization()(x_player)
 if dropout > 0:
x_player = Dropout(dropout )(x_player)
 return x_player 
<choose_model_class>
def eta_squared(x,y):
moyenne_y = y.mean()
classes = []
for classe in x.unique() :
yi_classe = y[x==classe]
classes.append({'ni': len(yi_classe),
'moyenne_classe': yi_classe.mean() })
 SCT = sum([(yj-moyenne_y)**2 for yj in y])
 SCE = sum([c['ni']*(c['moyenne_classe']-moyenne_y)**2 for c in classes])
 return SCE/SCT
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = Permute(( 2, 1))(x_K)
 res = Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_self_attention(x, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, x, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, x, n_factor_head, dropout)for i in range(n_head)]
att = Concatenate()(heads)
 att = Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = Add()([x, att])
 x = LayerNormalization()(x)
 if dropout > 0:
x = Dropout(dropout )(x)
 return x
def multi_head_outer_attention(x_inner, x_outer, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x_inner, x_outer, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x_inner, x_outer, n_factor_head, dropout)for i in range(n_head)]
att = Concatenate()(heads)
 att = Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x_inner = Add()([x_inner, att])
 x_inner = LayerNormalization()(x_inner)
 if dropout > 0:
x = Dropout(dropout )(x_inner)
 return x<define_search_model>
def adaptative_correlation(x,y):
x = x.reset_index(drop=True)
 y = y.reset_index(drop=True)
 non_na = np.logical_and(~x.isna() ,~y.isna())
 x=x[non_na]
y=y[non_na]
if(x.dtype.name=='object' or x.dtype.name=='category')and(y.dtype.name=='object' or y.dtype.name=='category'):
return cramers_v(x,y)
 elif(x.dtype.name=='object' or x.dtype.name=='category'):
return abs(eta_squared(x,y))
elif(y.dtype.name=='object' or y.dtype.name=='category'):
return abs(eta_squared(y,x))
else:
return abs(x.corr(y))
def se_bloc(in_bloc, ch, ratio):
x = GlobalAveragePooling1D()(in_bloc)
 x = Dense(ch//ratio, activation='relu' )(x)
 x = Dense(ch, activation='sigmoid' )(x)
 x = Multiply()([in_bloc, x])
 return Add()([x, in_bloc])
 def conv_bloc(players, n_factor, n_hidden, se_ratio, dropout):
players0 = players
players = Conv1D(n_hidden, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(players)
 players = Conv1D(n_factor, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(players)
 players = Add()([players0, players])
 players = se_bloc(players, n_factor, se_ratio)
 players = LayerNormalization()(players)
 if dropout > 0:
players = Dropout(dropout )(players)
 return players<define_search_model>
imputer = KNNImputer(n_neighbors=3)
 n_folds = 5
model = RandomForestRegressor(n_jobs=-1)
 def quick_predict(X,y):
X = pd.get_dummies(X)
 X = pd.DataFrame(imputer.fit_transform(X), columns = X.columns)
 kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X)
 rmse= np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv = kf))
print("Quality score: {:.4f} standard deviation({:.4f})
 ".format(rmse.mean() , rmse.std()))
def get_model(n_player, n_factor, n_loop, n_head, n_hidden, se_ratio, dropout, n_player_cols, n_play_cols):
input_players = Input(( n_player, n_player_cols), name="players")
 input_dmats = Input(( n_player, n_player), name="inv_dist")
 input_play = Input(( n_play_cols,), name="plays")
 x_player = input_players
x_player = Conv1D(n_factor, 1 )(x_player)
 x_player = LayerNormalization()(x_player)
 for l in range(n_loop):
x_player = dist_attention(input_dmats, x_player, dropout)
 x_player = conv_bloc(x_player, n_factor, n_hidden, se_ratio, dropout)
 x_player = multi_head_self_attention(x_player, n_factor, n_head, dropout)
 x_player = conv_bloc(x_player, n_factor, n_hidden, se_ratio, dropout)
 x_play = Dense(n_factor )(input_play)
 x_play = Reshape(( 1, -1))(x_play)
 readout = multi_head_outer_attention(x_play, x_player, n_factor, n_head, dropout)
 readout = Flatten()(readout)
 out1 = Dense(199, activation='sigmoid' )(readout)
 readout = Dense(4 )(readout)
 readout = ScaleLayer(output_dim=199 )(readout)
 out2 = keras.layers.Activation('sigmoid' )(readout)
 return Model(inputs=[input_players, input_dmats, input_play], outputs=[out1, out2])
<prepare_x_and_y>
quick_predict(train_data.drop(['SalePrice'],axis=1),train_data.SalePrice )
simulate_test = False
if simulate_test:
len_data = X_players.shape[0]
len_train = int(0.9 * len_data)
 else:
len_data = X_players.shape[0]
len_train = len_data
X_players_train, X_players_test = X_players[:len_train], X_players[len_train:]
inv_dmats_train, inv_dmats_test = inv_dmats[:len_train], inv_dmats[len_train:]
X_play_train, X_play_test = X_play[:len_train], X_play[len_train:]
y_mae_train, y_mae_test = y_mae[:len_train], y_mae[len_train:]
y_crps_train, y_crps_test = y_crps[:len_train], y_crps[len_train:]
yardline_train, yardline_test = yardline[:len_train], yardline[len_train:]<split>
def mean_encoder(df,columns):
for col in columns:
if col not in df.columns:
print("Error, ",col," not in list of columns")
 else:
mean_encode = df.groupby(col)['tmpPrice'].mean()
mean_encode =(mean_encode - mean_encode.min())/(mean_encode.max() - mean_encode.min())
 new_col_name = col + '_enc'
df.loc[:,new_col_name] = df[col].map(mean_encode )
kf = KFold(n_splits=12, shuffle=False)
 ind_2017 = np.arange(len_2017)
 def add_2017(x,y):
x = np.concatenate(( ind_2017, x + len_2017))
y = y + len_2017
return x,y
indices = [add_2017(x, y)for x, y in kf.split(X_players_train[len_2017:len_train_reorient],
y_mae_train[len_2017:len_train_reorient])]
n_steps = len(time_steps)
 def augment_indices(x, y, time_steps):
x_a = np.concatenate([x + i*len_train_reorient for i in range(n_steps)])
 y_a = np.concatenate([y + i*len_train_reorient for i in range(n_steps)])
 return x_a, y_a
indices = [augment_indices(x, y, time_steps)for x,y in indices ]
indices = [(i, ind)for i,ind in enumerate(indices)]
w = np.ones(len_train_reorient)
 w[:len_2017] = 0.5
w = np.concatenate([w for i in range(n_steps)] )<feature_engineering>
len(train_data[train_data.SalePrice>600000] )
def post_process(y_pred, yardline_train, time_steps):
upper = 100 - yardline_train
lower = - yardline_train
for i in range(y_pred.shape[0]):
y_pred[i, 99 + upper[i]:] = 1
y_pred[i, :99 + lower[i]] = 0
y_pred[i] = nondecreasing(y_pred[i].ravel())
 if 1:
n_steps = len(time_steps)
 len_y_pred = len(y_pred)// n_steps
y_pred = np.mean([y_pred[len_y_pred * i: len_y_pred *(i+1)] for i in range(n_steps)], axis=0)
 return y_pred<define_variables>
train_data = train_data[train_data.SalePrice<600000]
name = 'nn'
results[name] = {'crps': [], 'y_test': [], 'imp': [], 'models': []}
try:
del sess
except:
pass
sess = init_seeds(0)
 pred_test = []
for index in indices:
if 1 and index[0] < 10:
continue
train_idx, val_idx = index[1]
train_players, train_inv_dmats, train_play, train_y, =(X_players_train[train_idx],
inv_dmats_train[train_idx],
X_play_train[train_idx],
y_crps_train[train_idx],)
 val_players, val_inv_dmats, val_play, val_y, =(X_players_train[val_idx],
inv_dmats_train[val_idx],
X_play_train[val_idx],
y_crps_train[val_idx],)
 w_train = w[train_idx]
n_player = 22
n_factor = 64
se_ratio = 2
n_loop = 1
n_head = 4
n_hidden = 2*n_factor
dropout = 0.25
n_player_cols = len(player_cols)
 n_play_cols = len(play_cols)
 model = get_model(n_player, n_factor, n_loop, n_head, n_hidden, se_ratio, dropout, n_player_cols, n_play_cols)
 if 1:
opm = keras.optimizers.Adam(lr=1e-3)
 model.compile(loss='mse', optimizer=opm, metrics=[])
 es = keras.callbacks.EarlyStopping(monitor='val_loss', mode='min',
restore_best_weights=True, verbose=0, patience=21)
 lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=10, verbose=0, mode='min', min_delta=0.00001)
 history = model.fit([train_players, train_inv_dmats, train_play], [train_y, train_y],
verbose=2, batch_size=256, epochs=500,
callbacks=[es,lr],
validation_data=[[val_players, val_inv_dmats, val_play], [val_y, val_y]],
sample_weight=[w_train, w_train])
 X_ir = np.arange(199 ).astype('float64')
 y_pred = model.predict([val_players, val_inv_dmats, val_play])
 y_pred =(y_pred[0]+y_pred[1])/2.
 score0 = crps(y_mae_train[val_idx], y_pred)
 y_pred = post_process(y_pred, yardline_train[val_idx], time_steps)
 score1 = crps(y_mae_train[val_idx][:len(y_pred)], y_pred)
 print(' val fold:', index[0], score0)
 print(' val fold:', index[0], score1)
 if score0 < score1:
break
results[name]['models'].append(model)
 results[name]['crps'].append(score1)
 if simulate_test:
y_pred = model.predict([X_players_test, inv_dmats_test, X_play_test])
 y_pred =(y_pred[0]+y_pred[1])/2.
 score0 = crps(y_mae_test, y_pred)
 y_pred = post_process(y_pred, yardline_test, time_steps)
 score1 = crps(y_mae_test[:len(y_pred)], y_pred)
 print('test fold:', index[0], score0)
 print('test fold:', index[0], score1)
 pred_test.append(y_pred)
 print('*' * 40)
 print('Folds:', results[name]['crps'])
 print('Average:', np.mean(results[name]['crps']))<drop_column>
scipy.stats.skew(train_data.SalePrice.dropna() )
del train_reorient<split>
train_data['tmpPrice']=np.log1p(train_data.SalePrice )
env = nflrush.make_env()
iter_test = env.iter_test() 
<prepare_x_and_y>
tmpPriceMean = train_data['tmpPrice'].mean()
train_data['tmpPrice'] = train_data['tmpPrice']
for(test_df, sample_prediction_df)in tqdm(iter_test):
test_reorient = reorient(test_df.copy() , flip_left=True)
 test_reorient = augment_data(test_reorient, time_steps)
 df = test_reorient[player_cols]
X_players = df.values.astype('float32')
 X_players = ss_players.transform(df)
 X_players = X_players.reshape(( -1, 22, len(player_cols)))
 X,Y = test_reorient.X.values, test_reorient.Y.values
dmats = [get_dmat(X[i:i+22], Y[i:i+22])for i in range(0, test_reorient.shape[0], 22)]
dmats = np.vstack(dmats ).reshape(( -1, 22, 22))
inv_dmats = 1./(1e-2 + dmats)**2
inv_dmats /= inv_dmats.sum(axis=2, keepdims=True)
 X_play = test_reorient[play_cols].values[::22]
X_play = ss_play.transform(X_play)
 yardline = test_reorient['YardLine_std'].values[::22]
y_preds = [model.predict([X_players, inv_dmats, X_play] 
)for model in results[name]['models']]
y_preds = [(pred[0] + pred[1])/ 2.for pred in y_preds]
y_pred = np.mean(y_preds, axis=0)
 y_pred = post_process(y_pred, yardline, time_steps)
 sample_prediction_df.iloc[0, :] = y_pred.ravel()
env.predict(sample_prediction_df)
 env.write_submission_file()<prepare_x_and_y>
quick_predict(train_data.drop(['SalePrice','tmpPrice'],axis=1),train_data.tmpPrice )
X,Y = test_reorient.X.values, test_reorient.Y.values
dmats = [get_dmat(X[i:i+22], Y[i:i+22])for i in range(0, test_reorient.shape[0], 22)]
dmats = np.vstack(dmats ).reshape(( -1, 22, 22))
dmats.shape<set_options>
train_data.set_index(train_data.Id,drop=True, inplace=True)
 train_data = train_data.drop(['Id'],axis=1)
 test_data.set_index(test_data.Id,drop=True, inplace=True)
 test_data = test_data.drop(['Id'],axis=1 )
warnings.simplefilter('ignore')
 tf.disable_v2_behavior()
np.random.seed(42)
 random.seed(42 )<predict_on_test>
data=pd.concat([train_data,test_data],keys=['train','test'], join='inner' )
def convert_pred(yards):
p = 99 + yards
gap = 10
y_pred = np.zeros(199)
 y_pred[p-gap:p+gap] = np.cumsum(norm.pdf(np.arange(-gap,gap,1),0,3))
y_pred[p+gap:] = 1
return pd.Series(y_pred )<feature_engineering>
data[['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond',
'PoolQC','Fence','MiscFeature']]=data[['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond',
'PoolQC','Fence','MiscFeature']].fillna(value="NA" )
def convert_y(yards):
p = 99 + yards
y_ans = np.zeros(199)
 y_ans[p:] = 1
return pd.Series(y_ans )<predict_on_test>
data.isna().sum() [lambda x: x>0]
def predict_models(models, df):
preds = []
for model in models:
pred = model.predict(df)
 preds.append(pred)
 return np.column_stack(preds )<categorify>
data[data.MSZoning.isna() ]
class GenerateData:
def __init__(self, cat_features, drop_cols=[]):
self.cat_features = list(set(cat_features)- set(drop_cols))
self.drop_cols = drop_cols
self.players_score = None
self.category_dict = None
def make_train_score(self, df):
df['MaxHomeScore'] = df.groupby('GameId')['HomeScoreBeforePlay'].transform('max')
 df['MaxVisitorScore'] = df.groupby('GameId')['VisitorScoreBeforePlay'].transform('max')
 df['Score'] = 0
df.loc[df['Team'] == 'home', 'Score'] = df[df['Team'] == 'home']['MaxHomeScore'] - df[df['Team'] == 'home']['MaxVisitorScore']
df.loc[df['Team'] == 'away', 'Score'] = df[df['Team'] == 'away']['MaxVisitorScore'] - df[df['Team'] == 'away']['MaxHomeScore']
df['Score'] = df['Score'].map(lambda x: 1 if x>0 else 0 if x<0 else 0.5)
 players_score = df[['GameId', 'NflId', 'Score']].drop_duplicates()
players_score['GeneralScore'] = players_score.groupby('NflId')['Score'].transform('sum')
 players_score = players_score[['NflId', 'GeneralScore']].drop_duplicates()
df = df.merge(players_score, on='NflId', how='left')
 df.loc[df['Team'] == 'away', 'GeneralScore'] = df[df['Team'] == 'away'].groupby('GameId')['GeneralScore'].transform(lambda x: np.round(x.mode().mean()))
df.loc[df['Team'] == 'home', 'GeneralScore'] = df[df['Team'] == 'home'].groupby('GameId')['GeneralScore'].transform(lambda x: np.round(x.mode().mean()))
df.drop(['MaxHomeScore', 'MaxVisitorScore', 'Score'], axis=1, inplace=True)
 return df, players_score
def make_test_score(self, df, players_score):
df = df.merge(players_score, on='NflId', how='left')
 df.loc[df['Team'] == 'away', 'GeneralScore'] = df[df['Team'] == 'away'].groupby('GameId')['GeneralScore'].transform(lambda x: np.round(x.mode().mean()))
df.loc[df['Team'] == 'home', 'GeneralScore'] = df[df['Team'] == 'home'].groupby('GameId')['GeneralScore'].transform(lambda x: np.round(x.mode().mean()))
return df
def to_category_train(self, df):
df = df.fillna(-999)
 category_dict = {}
for col in self.cat_features:
f = df[col].factorize() [0]
d = dict(zip(df[col], f))
category_dict[col] = d
df[col] = df[col].map(d)
 return df, category_dict
def to_category_test(self, df, category_dict):
df = df.fillna(-999)
 for col, d in category_dict.items() :
df[col + '_orig'] = df[col]
df[col] = df[col].map(d)
 mask = df[col].isna()
if mask.any() :
increment = max(d.values())+ 1
df.loc[mask, col] = df.loc[mask, col + '_orig'].factorize() [0] + increment
df[col] = df[col].astype('int64')
 del_cols = [i for i in df.columns if '_orig' in i]
df.drop(del_cols, axis=1, inplace=True)
 return df
def convert_train(self, df):
df, self.players_score = self.make_train_score(df)
 df = self.make_new_features(df)
 df.drop(df.index[(df['YardsLeft']<df['Yards'])|(df['YardsLeft']-100>df['Yards'])], inplace=True)
 df, self.category_dict = self.to_category_train(df)
 return df
def convert_test(self, df):
df = self.make_test_score(df, self.players_score)
 df = self.make_new_features(df)
 df = self.to_category_test(df, self.category_dict)
 return df
def make_new_features(self, df):
def convert_to_milliseconds(t):
t = t.split(':')
 t = list(map(int, t))
t = t[0]*60*60 + t[1]*60 + t[2]
return t
df['GameClock'] = df['GameClock'].apply(convert_to_milliseconds)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
outdoor = ['Outdoor', 'Outdoors', 'Cloudy', 'Heinz Field', 'Outdor', 'Ourdoor',
'Outside', 'Outddors','Outdoor Retr Roof-Open', 'Oudoor', 'Bowl']
indoor_closed = ['Indoors', 'Indoor', 'Indoor, Roof Closed', 'Indoor, Roof Closed', 'Retractable Roof',
'Retr.Roof-Closed', 'Retr.Roof - Closed', 'Retr.Roof Closed']
indoor_open = ['Indoor, Open Roof', 'Open', 'Retr.Roof-Open', 'Retr.Roof - Open']
dome_closed = ['Dome', 'Domed, closed', 'Closed Dome', 'Domed', 'Dome, closed']
dome_open = ['Domed, Open', 'Domed, open']
df['StadiumType'] = df['StadiumType'].replace(outdoor,'outdoor')
 df['StadiumType'] = df['StadiumType'].replace(indoor_closed,'indoor_closed')
 df['StadiumType'] = df['StadiumType'].replace(indoor_open,'indoor_open')
 df['StadiumType'] = df['StadiumType'].replace(dome_closed,'dome_closed')
 df['StadiumType'] = df['StadiumType'].replace(dome_open,'dome_open')
 natural_grass = ['natural grass','Naturall Grass','Natural Grass']
grass = ['Grass']
fieldturf = ['FieldTurf','Field turf','FieldTurf360','Field Turf']
artificial = ['Artificial','Artifical']
df['Turf'] = df['Turf'].replace(natural_grass,'natural_grass')
 df['Turf'] = df['Turf'].replace(grass,'grass')
 df['Turf'] = df['Turf'].replace(fieldturf,'fieldturf')
 df['Turf'] = df['Turf'].replace(artificial,'artificial')
 map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in df['PossessionTeam'].unique() :
map_abbr[abb] = abb
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] =(703*df['PlayerWeight'])/(df['PlayerHeight']**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ")).astype(int)// 10**9
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ")).astype(int)// 10**9
df['TimeDelta'] = df['TimeHandoff'] - df['TimeSnap']
df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y")).astype(int)// 10**9
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df['PlayerBirthDate'] / seconds_in_year
df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 def str_to_float(txt):
try:
return float(txt)
 except:
return -999
df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 north = ['N','From S','North']
south = ['S','From N','South','s']
west = ['W','From E','West']
east = ['E','From W','from W','EAST','East']
north_east = ['FROM SW','FROM SSW','FROM WSW','NE','NORTH EAST','North East','East North East','NorthEast','Northeast','ENE','From WSW','From SW']
north_west = ['E','From ESE','NW','NORTHWEST','N-NE','NNE','North/Northwest','W-NW','WNW','West Northwest','Northwest','NNW','From SSE']
south_east = ['E','From WNW','SE','SOUTHEAST','South Southeast','East Southeast','Southeast','SSE','From SSW','ESE','From NNW']
south_west = ['E','From ENE','SW','SOUTHWEST','W-SW','South Southwest','West-Southwest','WSW','SouthWest','Southwest','SSW','From NNE']
no_wind = ['clear','Calm']
nan = ['1','8','13']
df['WindDirection'] = df['WindDirection'].replace(north,'north')
 df['WindDirection'] = df['WindDirection'].replace(south,'south')
 df['WindDirection'] = df['WindDirection'].replace(west,'west')
 df['WindDirection'] = df['WindDirection'].replace(east,'east')
 df['WindDirection'] = df['WindDirection'].replace(north_east,'north_east')
 df['WindDirection'] = df['WindDirection'].replace(north_west,'north_west')
 df['WindDirection'] = df['WindDirection'].replace(south_east,'clear')
 df['WindDirection'] = df['WindDirection'].replace(south_west,'south_west')
 df['WindDirection'] = df['WindDirection'].replace(no_wind,'no_wind')
 df['WindDirection'] = df['WindDirection'].replace(nan,np.nan)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 rain = ['Rainy', 'Rain Chance 40%', 'Showers','Cloudy with periods of rain, thunder possible.Winds shifting to WNW, 10-20 mph.',
'Scattered Showers', 'Cloudy, Rain', 'Rain shower', 'Light Rain', 'Rain']
overcast = ['Cloudy, light snow accumulating 1-3"', 'Party Cloudy', 'Cloudy, chance of rain',
'Coudy', 'Cloudy, 50% change of rain', 'Rain likely, temps in low 40s.',
'Cloudy and cold', 'Cloudy, fog started developing in 2nd quarter',
'Partly Clouidy', '30% Chance of Rain', 'Mostly Coudy', 'Cloudy and Cool',
'cloudy', 'Partly cloudy', 'Overcast', 'Hazy', 'Mostly cloudy', 'Mostly Cloudy',
'Partly Cloudy', 'Cloudy']
clear = ['Partly clear', 'Sunny and clear', 'Sun & clouds', 'Clear and Sunny',
'Sunny and cold', 'Sunny Skies', 'Clear and Cool', 'Clear and sunny',
'Sunny, highs to upper 80s', 'Mostly Sunny Skies', 'Cold',
'Clear and warm', 'Sunny and warm', 'Clear and cold', 'Mostly sunny',
'T: 51; H: 55; W: NW 10 mph', 'Clear Skies', 'Clear skies', 'Partly sunny',
'Fair', 'Partly Sunny', 'Mostly Sunny', 'Clear', 'Sunny', 'Sunny, Windy']
snow = ['Heavy lake effect snow', 'Snow']
none = ['N/A Indoor', 'Indoors', 'Indoor', 'N/A(Indoors)', 'Controlled Climate']
df['GameWeather'] = df['GameWeather'].replace(rain,'rain')
 df['GameWeather'] = df['GameWeather'].replace(overcast,'overcast')
 df['GameWeather'] = df['GameWeather'].replace(clear,'clear')
 df['GameWeather'] = df['GameWeather'].replace(snow,'snow')
 df['GameWeather'] = df['GameWeather'].replace(none,'none')
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df.loc[df['PlayDirection'] == 'left', 'X'] = 120 - df.loc[df['PlayDirection'] == 'left', 'X']
df.loc[df['PlayDirection'] == 'left', 'Y'] =(160 / 3)- df.loc[df['PlayDirection'] == 'left', 'Y']
df.loc[df['PlayDirection'] == 'left', 'Orientation'] = np.mod(180 + df.loc[df['PlayDirection'] == 'left', 'Orientation'], 360)
 df.loc[df['PlayDirection'] == 'left', 'Dir'] = np.mod(180 + df.loc[df['PlayDirection'] == 'left', 'Dir'], 360)
 df['FieldPosition'].fillna('', inplace=True)
 df.loc[df['PossessionTeam'] != df['FieldPosition'], 'YardLine'] = 100 - df.loc[df['PossessionTeam'] != df['FieldPosition'], 'YardLine']
df.loc[df['Season'] == 2017, 'Orientation'] = np.mod(90 + df.loc[df['Season'] == 2017, 'Orientation'], 360)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df['DiffScore'] =(df['HomeScoreBeforePlay'] - df['VisitorScoreBeforePlay'] ).abs()
df['MaxHomeScore'] = df.groupby('GameId')['HomeScoreBeforePlay'].transform('max')
 df['MaxVisitorScore'] = df.groupby('GameId')['VisitorScoreBeforePlay'].transform('max')
 df.drop(self.drop_cols, axis=1, inplace=True)
 return df<load_from_csv>
data[data.Neighborhood=="Mitchel"].MSZoning.value_counts()
df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False )<define_variables>
data[data.Neighborhood=="IDOTRR"].MSZoning.value_counts()
cat_features = ['PlayId', 'Team', 'NflId', 'Quarter', 'PossessionTeam', 'FieldPosition', 'NflIdRusher', 'OffenseFormation', 'OffensePersonnel', 'DefendersInTheBox', 'DefensePersonnel', 'PlayDirection', 'PlayerCollegeName', 'Position', 'HomeTeamAbbr', 'VisitorTeamAbbr', 'Week', 'Stadium', 'Location', 'StadiumType', 'Turf', 'GameWeather', 'WindDirection', 'HomePossesion', 'Field_eq_Possession', 'HomeField', 'IsRusher']
drop_cols = ['DisplayName', 'JerseyNumber', 'GameId', 'PlayId', 'NflId', 'NflIdRusher'] + ['HomePossesion', 'PlayerAge', 'Temperature', 'Turf']
drop_cols += ['YardLine', 'MaxVisitorScore', 'DiffScore', 'DefendersInTheBox', 'WindSpeed']<concatenate>
data.loc[(data.MSZoning.isna())&(data.Neighborhood=="Mitchel"),"MSZoning"]="RL"
data.loc[(data.MSZoning.isna())&(data.Neighborhood=="IDOTRR"),"MSZoning"]="RM"
gd = GenerateData(cat_features, drop_cols)
 df = gd.convert_train(df )<prepare_x_and_y>
data['LotAreaSqr']=np.sqrt(data.LotArea )
train = df[df['Season'] == 2017]
y_train = train['Yards']
train.drop(['Yards', 'Season'], axis=1, inplace=True)
 valid = df[df['Season'] == 2018]
y_valid = valid['Yards']
valid.drop(['Yards', 'Season'], axis=1, inplace=True)
 del df
gc.collect()<define_variables>
data.LotShape.value_counts()
models = []<choose_model_class>
data['LotFrontage']=data.LotFrontage.fillna(data.LotAreaSqr )
class KerasCustom:
def __init__(self):
self.model = None
self.input_shape = None
def create_model(self, input_shape):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv1D(96, 2, activation='tanh', input_shape=(input_shape, 1), kernel_initializer=tf.keras.initializers.glorot_uniform(seed=42)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(96, activation='tanh', kernel_initializer=tf.keras.initializers.glorot_uniform(seed=42)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(1, activation='linear', kernel_initializer=tf.keras.initializers.glorot_uniform(seed=42)))
 model.compile(loss=['mae'], optimizer=tf.keras.optimizers.Adam(lr=0.001))
return model
def fit(self, train, y_train):
es = tf.keras.callbacks.EarlyStopping(monitor='loss', min_delta=0.0001, patience=5, verbose=1, mode='min', restore_best_weights=True)
 self.input_shape = train.shape[1]
self.model = self.create_model(self.input_shape)
 self.model.fit(train.values.reshape(-1, self.input_shape, 1), y_train,
epochs=100,
batch_size=1024,
callbacks=[es],
verbose=1)
 def predict(self, df):
return self.model.predict(df.values.reshape(-1, self.input_shape, 1))
model = KerasCustom()
model.fit(train, y_train)
 models.append(model)
 class KerasCustom:
def __init__(self):
self.model = None
self.input_shape = None
def create_model(self, input_shape):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv1D(96, 2, activation='tanh', input_shape=(input_shape, 1), kernel_initializer=tf.keras.initializers.glorot_uniform(seed=42)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(1, activation='linear', kernel_initializer=tf.keras.initializers.glorot_uniform(seed=42)))
 model.compile(loss=['mae'], optimizer=tf.keras.optimizers.Adam(lr=0.001))
return model
def fit(self, train, y_train):
es = tf.keras.callbacks.EarlyStopping(monitor='loss', min_delta=0.0001, patience=3, verbose=1, mode='min', restore_best_weights=True)
 self.input_shape = train.shape[1]
self.model = self.create_model(self.input_shape)
 self.model.fit(train.values.reshape(-1, self.input_shape, 1), y_train,
epochs=100,
batch_size=1024,
callbacks=[es],
verbose=1)
 def predict(self, df):
return self.model.predict(df.values.reshape(-1, self.input_shape, 1))
model = KerasCustom()
model.fit(train, y_train)
 models.append(model )<choose_model_class>
data = data.drop(['LotAreaSqr'],axis=1 )
cat_features_ids = [c for c, col in enumerate(train.columns)if col in cat_features]
model = CatBoostRegressor(loss_function='MAE',
n_estimators=359,
depth=6,
one_hot_max_size=255,
random_seed=42)
 model.fit(train, y_train,
cat_features=cat_features_ids,
verbose=200)
 models.append(model)
 model = CatBoostRegressor(loss_function='MAE',
n_estimators=968,
depth=3,
one_hot_max_size=255,
random_seed=42)
 model.fit(train, y_train,
cat_features=cat_features_ids,
verbose=200)
 models.append(model)
 model = CatBoostRegressor(loss_function='MAE',
n_estimators=136,
depth=9,
one_hot_max_size=255,
random_seed=42)
 model.fit(train, y_train,
cat_features=cat_features_ids,
verbose=200)
 models.append(model )<train_model>
data = data.drop(['Utilities'],axis=1 )
model = lgb.LGBMRegressor(n_estimators=194,
random_state=42,
learning_rate=0.005,
importance_type = 'gain',
n_jobs = -1,
metric='mae')
 model.fit(train, y_train,
categorical_feature = cat_features_ids,
verbose=10)
 models.append(model )<train_model>
data[data.Exterior1st.isna() ]
model=ExtraTreesRegressor(n_estimators=400, n_jobs=-1, bootstrap=True)
 model.fit(train, y_train)
 models.append(model )<predict_on_test>
data[(data.YearBuilt>1935)&(data.YearBuilt<1945)&(data.Neighborhood=='Edwards')].Exterior1st.value_counts()
pred = predict_models(models, valid )<train_model>
data['Exterior1st']=data.Exterior1st.fillna('MetalSd')
 data['Exterior2nd']=data.Exterior2nd.fillna('MetalSd' )
meta_model=LinearRegression(n_jobs=-1)
 meta_model.fit(pred, y_valid )<predict_on_test>
data[data.BsmtFinSF1.isna() ]
env = nflrush.make_env()
for(test, sub)in tqdm_notebook(env.iter_test()):
df = gd.convert_test(test)
 df.drop('Season', axis=1, inplace=True)
 test_pred = predict_models(models, df)
 final_pred=meta_model.predict(test_pred)
 final_pred = np.mean(final_pred ).astype('int')
 final_pred = convert_pred(final_pred)
 sub[sub.columns] = final_pred.values
env.predict(sub)
 env.write_submission_file()<set_options>
data.loc[data.BsmtFinSF1.isna() ,'BsmtFinSF2' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtUnfSF' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtFullBath' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'TotalBsmtSF' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtHalfBath' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtFinSF1' ]=0.0
data.loc[data.BsmtQual=="NA",'BsmtExposure' ]="NA"
data.loc[data.BsmtQual=="NA",'BsmtFullBath' ]=0.0
data.loc[data.BsmtQual=="NA",'BsmtHalfBath' ]=0.0
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<load_from_csv>
data[data.Electrical.isna() ]
if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
<remove_duplicates>
data.Electrical.value_counts()
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<categorify>
data[['Electrical']]=data[['Electrical']].fillna(value="SBrkr" )
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<prepare_x_and_y>
data['KitchenQual']=data['KitchenQual'].fillna(value="TA" )
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS','RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
for feat1 in ['RusherS','RusherA','RusherX', 'RusherY']:
for feat2 in ['RusherS','RusherA','RusherX', 'RusherY']:
if feat1 == feat2:
continue
new_feature = feat1 + '_' + feat2
new_feature_reversed = feat2 + '_' + feat1
if new_feature in rusher.columns or new_feature_reversed in rusher.columns:
continue
rusher[new_feature] = rusher[feat1].div(rusher[feat2] + 10)
 rusher['v_horizontal_vertical'] = rusher['v_horizontal'].div(rusher['v_vertical'] + 10)
 return rusher
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense,rushing, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,rushing,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 rush_feats = rusher_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats,rush_feats,static_feats, deploy=deploy)
 return basetable<feature_engineering>
data['GarageYrBlt']=data['GarageYrBlt'].fillna(data.YearBuilt )
%time train_basetable = create_features(train, False )<prepare_x_and_y>
data[data.GarageCars.isna() ]
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )<normalization>
data.loc[data.GarageCars.isna() ,data.columns.str.contains('Gar')]
scaler = StandardScaler()
X = scaler.fit_transform(X )<split>
data['GarageArea']=data['GarageArea'].fillna(value=300)
 data['GarageCars']=data['GarageCars'].fillna(value=1.0 )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )<set_options>
data.Functional.value_counts()
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<choose_model_class>
data['Functional']=data['Functional'].fillna(value="Typ" )
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
data.SaleType.value_counts()
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<compute_test_metric>
data['SaleType']=data['SaleType'].fillna(value="WD" )
print("mean crps is %f"%np.mean(crps_csv)) 
<predict_on_test>
X = data.drop(['SalePrice', 'tmpPrice','MasVnrType','MasVnrArea'],axis=1)
 X = X.reset_index(level=0, drop=True)
 X = pd.get_dummies(X)
 Y = data[['MasVnrType']]
Y = Y.reset_index(level=0, drop=True)
 Y_source = Y.dropna()
X_source = X.loc[Y_source.index,:]
X_target = X.loc[set(X.index)- set(Y_source.index),:]
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<set_options>
le = preprocessing.LabelEncoder()
le.fit(Y_source.values )
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<load_from_csv>
Y_source = le.transform(Y_source )
if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
<remove_duplicates>
clf = RandomForestClassifier(n_estimators=300, max_depth=None,min_samples_split=2, random_state=0)
 scores = cross_val_score(clf, X_source, Y_source, cv=5)
 scores.mean()
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<categorify>
_=clf.fit(X_source, Y_source )
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<prepare_x_and_y>
Y_target = pd.DataFrame(le.inverse_transform(clf.predict(X_target)) ).set_index(X_target.index )
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS','RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
return rusher
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense,rushing, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,rushing,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 rush_feats = rusher_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats,rush_feats,static_feats, deploy=deploy)
 return basetable<feature_engineering>
Y_target.columns = ['MasVnrType']
%time train_basetable = create_features(train, False )<prepare_x_and_y>
X = data.drop(['SalePrice', 'tmpPrice','MasVnrArea'],axis=1)
 X = X.reset_index(level=0, drop=True)
 X = pd.get_dummies(X)
 Y = data[['MasVnrArea']]
Y = Y.reset_index(level=0, drop=True)
 Y_source = Y.dropna()
X_source = X.loc[Y_source.index,:]
X_target = X.loc[set(X.index)- set(Y_source.index),:]
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )<normalization>
clf = RandomForestRegressor(n_estimators=100, min_samples_split=3, random_state=0)
 scores = cross_val_score(clf, X_source, Y_source, cv=5, scoring="neg_mean_squared_error",n_jobs=-1)
 scores.mean()
scaler = StandardScaler()
X = scaler.fit_transform(X )<split>
_=clf.fit(X_source, Y_source)
 Y_target = pd.DataFrame(clf.predict(X_target)).set_index(X_target.index)
 Y_target.columns = ['MasVnrArea']
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )<set_options>
data.isna().sum() [lambda x: x>0]
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<choose_model_class>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
data.dtypes.value_counts()
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<compute_test_metric>
pd.set_option('display.max_columns', 100)
 data[data.select_dtypes(['int64'] ).columns].head()
print("mean crps is %f"%np.mean(crps_csv)) 
<predict_on_test>
data[['MSSubClass', 'MoSold','OverallQual','OverallCond','YrSold']] = data[['MSSubClass', 'MoSold','OverallQual','OverallCond','YrSold']].astype(str )
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<import_modules>
data[['LotFrontage', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath',
'BsmtHalfBath', 'GarageYrBlt', 'GarageCars',
'GarageArea']] = data[['LotFrontage', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath',
'BsmtHalfBath', 'GarageYrBlt', 'GarageCars', 'GarageArea']].astype(int)
warnings.filterwarnings('ignore')
 sns.set()
%matplotlib inline
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<load_from_csv>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )<remove_duplicates>
env_cols =['MSZoning','Street','Alley','Neighborhood','Condition1','Condition2']
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<categorify>
mean_encoder(data,['MSZoning','Street','Alley','Neighborhood'])
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<prepare_x_and_y>
data['ConditionFeedr'] =(data.Condition2=="Feedr")*1+(data.Condition1=="Feedr")*1
data['ConditionArtery'] =(data.Condition2=="Artery")*1+(data.Condition1=="Artery")*1
data['ConditionPosA'] =(data.Condition2=="PosA")*1+(data.Condition1=="PosA")*1
data['ConditionPosN'] =(data.Condition2=="PosN")*1+(data.Condition1=="PosN")*1
data['ConditionRRAn'] =(data.Condition2=="RRAn")*1+(data.Condition1=="RRAn")*1
data['ConditionRRAe'] =(data.Condition2=="RRAe")*1+(data.Condition1=="RRAe")*1
data['ConditionRRNn'] =(data.Condition2=="RRNn")*1+(data.Condition1=="RRNn")*1
data['ConditionRRNe'] =(data.Condition2=="RRNe")*1+(data.Condition1=="RRNe")*1
data = data.drop(['Condition1','Condition2'],axis=1 )
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS','RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
return rusher
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense, rushing, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,rushing,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 rush_feats = rusher_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, rush_feats, static_feats, deploy=deploy)
 return basetable<feature_engineering>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
%time train_basetable = create_features(train, False )<prepare_x_and_y>
lot_cols =['LotFrontage','LotArea','LotShape','LandContour','LotConfig','LandSlope','Fence','Fireplaces','FireplaceQu','GarageType','GarageYrBlt','GarageFinish','GarageCars','GarageArea','GarageQual','GarageCond','PoolArea','PoolQC']
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )<feature_engineering>
data['LotShape'].dtype=='object'
columns = np.full(( corr.shape[0],), True, dtype=bool)
 for i in range(corr.shape[0]):
for j in range(i+1, corr.shape[0]):
if corr.iloc[i,j] >= 0.99:
if columns[j]:
columns[j] = False
feature_columns = X.columns[columns].values
drop_columns = X.columns[columns == False].values
print(feature_columns)
 print('-'*73)
 print(drop_columns )<count_unique_values>
mean_encoder(data,['LotShape','LandContour','LotConfig','LandSlope','Fence','FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond'])
cat_feats = X.columns[(X.nunique() <= 50)].values.tolist()
cat_feats = list(set(cat_feats)& set(feature_columns))
print(cat_feats )<normalization>
data.PoolQC.isna().sum()
scaler = StandardScaler()
scaled_features = scaler.fit_transform(X[feature_columns].values)
 X = pd.DataFrame(scaled_features, columns=feature_columns)
 print(X.shape )<train_model>
data['Pool'] =(data.PoolQC!="NA")*1
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s<compute_test_metric>
data = data.drop(['PoolQC'],axis=1 )
def crps_score(y_prediction, y_valid, shape=X.shape[0]):
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_prediction, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * shape)
 crps = np.round(val_s, 6)
 return crps<compute_train_metric>
data[data.Pool==0].PoolArea.value_counts()
def get_nn(x_tr, y_tr, x_val, y_val, shape):
K.clear_session()
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz, verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
crps = crps_score(y_pred, y_valid, shape=shape)
 return model,crps<init_hyperparams>
data.loc[(data.Pool==0)&(data.PoolArea>0),'Pool']=1
metric = "multi_logloss"
param = {'num_leaves': 50,
'min_data_in_leaf': 30,
'objective':'multiclass',
'num_class': 199,
'max_depth': 6,
'learning_rate': 0.01,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.4,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": metric,
"lambda_l1": 0.1,
"verbosity": -1,
'n_jobs': -1,
"seed":1234}<predict_on_test>
data[data.LotFrontage>300]
def get_lgbm(x_tr, y_tr, x_val, y_val, shape):
y_valid = y_val
y_tr = np.argmax(y_tr, axis=1)
 y_val = np.argmax(y_val, axis=1)
 trn_data = lgb.Dataset(x_tr, label=y_tr, categorical_feature=cat_feats)
 val_data = lgb.Dataset(x_val, label=y_val, categorical_feature=cat_feats)
 model = lgb.train(param, trn_data, 10000, valid_sets = [val_data], verbose_eval = 100, early_stopping_rounds = 200)
 y_pred = model.predict(x_val, num_iteration=model.best_iteration)
 crps = crps_score(y_pred, y_valid, shape=shape)
 return model, crps<compute_train_metric>
data = data[data.LotFrontage<300]
def get_rf(x_tr, y_tr, x_val, y_val, shape):
model = RandomForestRegressor(bootstrap=False, max_features=0.3, min_samples_leaf=15,
min_samples_split=8, n_estimators=50, n_jobs=-1, random_state=42)
 model.fit(x_tr, y_tr)
 y_pred = model.predict(x_val)
 y_valid = y_val
crps = crps_score(y_pred, y_valid, shape=shape)
 return model, crps<init_hyperparams>
data['LotFrontage']=np.power(data.LotFrontage,0.25 )
loop = 2
fold = 5
oof_nn = np.zeros([loop, y.shape[0], y.shape[1]])
 oof_lgbm = np.zeros([loop, y.shape[0], y.shape[1]])
 oof_rf = np.zeros([loop, y.shape[0], y.shape[1]])
 models_nn = []
crps_csv_nn = []
models_lgbm = []
crps_csv_lgbm = []
models_rf = []
crps_csv_rf = []
feature_importance = np.zeros([loop, fold, X.shape[1]])
 s_time = time.time()
for k in range(loop):
kfold = KFold(fold, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print(f'Loop {k+1}/{loop}' + f' Fold {k_fold+1}/{fold}')
 print("-----------")
 tr_x, tr_y = X.loc[tr_inds], y[tr_inds]
val_x, val_y = X.loc[val_inds], y[val_inds]
nn, crps_nn = get_nn(tr_x, tr_y, val_x, val_y, shape=val_x.shape[0])
 models_nn.append(nn)
 print("the %d fold crps(NN)is %f"%(( k_fold+1), crps_nn))
crps_csv_nn.append(crps_nn)
 lgbm, crps_lgbm = get_lgbm(tr_x, tr_y, val_x, val_y, shape=val_x.shape[0])
 models_lgbm.append(lgbm)
 print("the %d fold crps(LGBM)is %f"%(( k_fold+1), crps_lgbm))
crps_csv_lgbm.append(crps_lgbm)
 feature_importance[k, k_fold, :] = lgbm.feature_importance()
oof_nn[k, val_inds, :] = nn.predict(val_x)
 oof_lgbm[k, val_inds, :] = lgbm.predict(val_x, num_iteration=lgbm.best_iteration)
<predict_on_test>
data = data[data.LotArea<100000]
a = lgbm.predict(val_x, num_iteration=lgbm.best_iteration )<compute_test_metric>
data['LotArea'] = np.log1p(data.LotArea )
crps_oof_nn = []
crps_oof_lgbm = []
crps_oof_rf = []
for k in range(loop):
crps_oof_nn.append(crps_score(oof_nn[k,...], y))
crps_oof_lgbm.append(crps_score(oof_lgbm[k,...], y)) 
<compute_test_metric>
data.groupby('Fireplaces')['tmpPrice'].mean()
print("mean crps(NN)is %f"%np.mean(crps_csv_nn))
print("mean crps(LGBM)is %f"%np.mean(crps_csv_lgbm)) 
<compute_test_metric>
data.groupby('GarageCars')['tmpPrice'].mean()
print("mean OOF crps(NN)is %f"%np.mean(crps_oof_nn))
print("mean OOF crps(LGBM)is %f"%np.mean(crps_oof_lgbm)) 
<prepare_output>
data.GarageCars.value_counts()
feature_importances = pd.DataFrame(np.mean(feature_importance, axis=0 ).T, columns=[[f'fold_{fold_n}' for fold_n in range(fold)]])
 feature_importances['feature'] = feature_columns
feature_importances['average'] = feature_importances[[f'fold_{fold_n}' for fold_n in range(fold)]].mean(axis=1)
 feature_importances.sort_values(by=('average',), ascending=False ).head(10 )<compute_test_metric>
data['GarageCars']=data.GarageCars.replace([4,5],3 )
def weight_opt(oof_nn, oof_rf, y_true):
weight_nn = np.inf
best_crps = np.inf
for i in np.arange(0, 1.01, 0.05):
crps_blend = np.zeros(oof_nn.shape[0])
 for k in range(oof_nn.shape[0]):
crps_blend[k] = crps_score(i * oof_nn[k,...] +(1-i)* oof_rf[k,...], y_true)
 if np.mean(crps_blend)< best_crps:
best_crps = np.mean(crps_blend)
 weight_nn = round(i, 2)
 print(str(round(i, 2)) + ' : mean crps(Blend)is ', round(np.mean(crps_blend), 6))
print('-'*36)
 print('Best weight for NN: ', weight_nn)
 print('Best weight for LGBM: ', round(1-weight_nn, 2))
print('Best mean crps(Blend): ', round(best_crps, 6))
return weight_nn, round(1-weight_nn, 2 )<compute_test_metric>
data[data.GarageYrBlt>2010]
weight_nn, weight_lgbm = weight_opt(oof_nn, oof_lgbm, y)
<predict_on_test>
data.loc[data.GarageYrBlt>2010,'GarageYrBlt']=2007
def predict(x_te, models_nn, models_rf, weight_nn, weight_rf, iteration=False):
model_num_nn = len(models_nn)
 model_num_rf = len(models_rf)
 for k,m in enumerate(models_nn):
if k==0:
y_pred_nn = m.predict(x_te, batch_size=1024)
 if iteration:
y_pred_rf = models_rf[k].predict(x_te, num_iteration=models_rf[k].best_iteration)
 else:
y_pred_rf = models_rf[k].predict(x_te)
 else:
y_pred_nn += m.predict(x_te, batch_size=1024)
 if iteration:
y_pred_rf += models_rf[k].predict(x_te, num_iteration=models_rf[k].best_iteration)
 else:
y_pred_rf += models_rf[k].predict(x_te)
 y_pred_nn = y_pred_nn / model_num_nn
y_pred_rf = y_pred_rf / model_num_rf
return weight_nn * y_pred_nn + weight_rf * y_pred_rf<predict_on_test>
data['GarageYrBlt']=np.log1p(2011-data.GarageYrBlt )
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable_features = scaler.transform(basetable[feature_columns].values)
 scaled_basetable = pd.DataFrame(scaled_basetable_features, columns=feature_columns)
 y_pred = predict(scaled_basetable, models_nn, models_lgbm, weight_nn, weight_lgbm, iteration=True)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<import_modules>
data.groupby('GarageCars')['tmpPrice'].mean()
def default_tree_learner(depth=3):
return DecisionTreeRegressor(
 criterion='friedman_mse',
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_depth=depth,
splitter='best')
 class MLE:
def __init__(self, seed=123):
pass
def loss(self, forecast, Y):
return forecast.nll(Y.squeeze() ).mean()
def grad(self, forecast, Y, natural=True):
fisher = forecast.fisher_info()
grad = forecast.D_nll(Y)
 if natural:
grad = np.linalg.solve(fisher, grad)
 return grad
class CRPS:
def __init__(self, K=32):
self.K = K
def loss(self, forecast, Y):
return forecast.crps(Y.squeeze() ).mean()
def grad(self, forecast, Y, natural=True):
metric = forecast.crps_metric()
grad = forecast.D_crps(Y)
 if natural:
grad = np.linalg.solve(metric, grad)
 return grad
EPS = 1e-8
class Normal(object):
n_params = 2
def __init__(self, params, temp_scale = 1.0):
self.loc = params[0]
self.scale = np.exp(params[1] / temp_scale)+ 1e-8
self.var = self.scale
self.shp = self.loc.shape
self.dist = dist(loc=self.loc, scale=self.scale)
 def __getattr__(self, name):
if name in dir(self.dist):
return getattr(self.dist, name)
 return None
def nll(self, Y):
return -self.dist.logpdf(Y)
 def D_nll(self, Y_):
Y = Y_.squeeze()
D = np.zeros(( self.var.shape[0], 2))
D[:, 0] =(self.loc - Y)/ self.var
D[:, 1] = 1 -(( self.loc - Y)** 2)/ self.var
return D
def crps(self, Y):
Z =(Y - self.loc)/(self.scale + EPS)
 return(self.scale *(Z *(2 * sp.stats.norm.cdf(Z)- 1)+ \
2 * sp.stats.norm.pdf(Z)- 1 / np.sqrt(np.pi)))
 def D_crps(self, Y_):
Y = Y_.squeeze()
Z =(Y - self.loc)/(self.scale + EPS)
 D = np.zeros(( self.var.shape[0], 2))
D[:, 0] = -(2 * sp.stats.norm.cdf(Z)- 1)
 D[:, 1] = self.crps(Y)+(Y - self.loc)* D[:, 0]
return D
def crps_metric(self):
I = np.c_[2 * np.ones_like(self.var), np.zeros_like(self.var),
np.zeros_like(self.var), self.var]
I = I.reshape(( self.var.shape[0], 2, 2))
I = 1/(2*np.sqrt(np.pi)) * I
return I
def fisher_info(self):
FI = np.zeros(( self.var.shape[0], 2, 2))
FI[:, 0, 0] = 1/self.var + 1e-5
FI[:, 1, 1] = 2
return FI
def fisher_info_cens(self, T):
nabla = np.array([self.pdf(T), 
(T - self.loc)/ self.scale * self.pdf(T)])
 return np.outer(nabla, nabla)/(self.cdf(T)*(1 - self.cdf(T)))+ 1e-2 * np.eye(2)
 def fit(Y):
m, s = sp.stats.norm.fit(Y)
 return np.array([m, np.log(s)])
 class NGBoost(BaseEstimator):
def __init__(self, Dist=Normal, Score=MLE() ,
Base=default_tree_learner, natural_gradient=True,
n_estimators=500, learning_rate=0.01, minibatch_frac=1.0,
verbose=True, verbose_eval=100, tol=1e-4):
self.Dist = Dist
self.Score = Score
self.Base = Base
self.natural_gradient = natural_gradient
self.n_estimators = n_estimators
self.learning_rate = learning_rate
self.minibatch_frac = minibatch_frac
self.verbose = verbose
self.verbose_eval = verbose_eval
self.init_params = None
self.base_models = []
self.scalings = []
self.tol = tol
def pred_param(self, X, max_iter=None):
m, n = X.shape
params = np.ones(( m, self.Dist.n_params)) * self.init_params
for i,(models, s)in enumerate(zip(self.base_models, self.scalings)) :
if max_iter and i == max_iter:
break
resids = np.array([model.predict(X)for model in models] ).T
params -= self.learning_rate * resids * s
return params
def sample(self, X, Y, params):
if self.minibatch_frac == 1.0:
return np.arange(len(Y)) , X, Y, params
sample_size = int(self.minibatch_frac * len(Y))
idxs = np_rnd.choice(np.arange(len(Y)) , sample_size, replace=False)
 return idxs, X[idxs,:], Y[idxs], params[idxs, :]
def fit_base(self, X, grads):
models = [self.Base().fit(X, g)for g in grads.T]
fitted = np.array([m.predict(X)for m in models] ).T
self.base_models.append(models)
 return fitted
def line_search(self, resids, start, Y, scale_init=1):
S = self.Score
D_init = self.Dist(start.T)
 loss_init = S.loss(D_init, Y)
 scale = scale_init
while True:
scaled_resids = resids * scale
D = self.Dist(( start - scaled_resids ).T)
 loss = S.loss(D, Y)
 norm = np.mean(np.linalg.norm(scaled_resids, axis=1))
if not np.isnan(loss)and(loss < loss_init or norm < self.tol)and\
np.linalg.norm(scaled_resids, axis=1 ).mean() < 5.0:
break
scale = scale * 0.5
self.scalings.append(scale)
 return scale
def fit(self, X, Y, X_val = None, Y_val = None, train_loss_monitor = None, val_loss_monitor = None):
loss_list = []
val_loss_list = []
self.fit_init_params_to_marginal(Y)
 params = self.pred_param(X)
 if X_val is not None and Y_val is not None:
val_params = self.pred_param(X_val)
 S = self.Score
if not train_loss_monitor:
train_loss_monitor = S.loss
if not val_loss_monitor:
val_loss_monitor = S.loss
for itr in range(self.n_estimators):
_, X_batch, Y_batch, P_batch = self.sample(X, Y, params)
 D = self.Dist(P_batch.T)
 loss_list += [train_loss_monitor(D, Y_batch)]
loss = loss_list[-1]
grads = S.grad(D, Y_batch, natural=self.natural_gradient)
 proj_grad = self.fit_base(X_batch, grads)
 scale = self.line_search(proj_grad, P_batch, Y_batch)
 params -= self.learning_rate * scale * np.array([m.predict(X)for m in self.base_models[-1]] ).T
val_loss = 0
if X_val is not None and Y_val is not None:
val_params -= self.learning_rate * scale * np.array([m.predict(X_val)for m in self.base_models[-1]] ).T
val_loss = val_loss_monitor(self.Dist(val_params.T), Y_val)
 val_loss_list += [val_loss]
if len(val_loss_list)> 10 and np.mean(np.array(val_loss_list[-5:])) > \
np.mean(np.array(val_loss_list[-10:-5])) :
if self.verbose:
print(f"== Quitting at iteration / VAL {itr}(val_loss={val_loss:.4f})")
 break
if self.verbose and int(self.verbose_eval)> 0 and itr % int(self.verbose_eval)== 0:
grad_norm = np.linalg.norm(grads, axis=1 ).mean() * scale
print(f"[iter {itr}] loss={loss:.4f} val_loss={val_loss:.4f} scale={scale:.4f} "
f"norm={grad_norm:.4f}")
 if np.linalg.norm(proj_grad, axis=1 ).mean() < self.tol:
if self.verbose:
print(f"== Quitting at iteration / GRAD {itr}")
 break
return self
def fit_init_params_to_marginal(self, Y, iters=1000):
try:
E = Y['Event']
T = Y['Time'].reshape(( -1, 1)) [E == 1]
except:
T = Y
self.init_params = self.Dist.fit(T)
 return
def pred_dist(self, X, max_iter=None):
params = np.asarray(self.pred_param(X, max_iter))
dist = self.Dist(params.T)
 return dist
def predict(self, X):
dist = self.pred_dist(X)
 return list(dist.loc.flatten())
<categorify>
data = data.drop(['PoolArea'],axis=1 )
def clock_to_sec(s):
min_str, sec_str, msec_str = s.split(':')
 return 60 * int(min_str)+ int(sec_str)
 def featurize(rows):
featnames = []
features = []
for _, row in rows.iterrows() :
if row['NflId'] == row['NflIdRusher']:
rusher = row['NflId']
team = row['Team']
featnames += ['Home']
features += [int(team == 'home')]
featnames += ['Clock']
features += [clock_to_sec(row['GameClock'])]
featnames += ['Quarter']
features += [int(row['Quarter'])]
if team == 'home':
featnames += ['FriendScore']
features += [int(row['HomeScoreBeforePlay'])]
featnames += ['EnemyScore']
features += [int(row['VisitorScoreBeforePlay'])]
else:
featnames += ['FriendScore']
features += [int(row['VisitorScoreBeforePlay'])]
featnames += ['EnemyScore']
features += [int(row['HomeScoreBeforePlay'])]
featnames += ['RusherHeight']
ft, inch = row['PlayerHeight'].split('-')
 features += [int(ft)* 12 + int(inch)]
featnames += ['RusherWeight']
features += [int(row['PlayerWeight'])]
featnames += ['RusherAge']
features += [parse(row['TimeHandoff'] ).year - parse(row['PlayerBirthDate'] ).year]
featnames += ['Week']
features += [int(row['Week'])]
featnames += ['Temperature']
try:
features += [int(row['Temperature'])]
except:
features += [60]
featnames += ['Humidity']
try:
features += [int(row['Humidity'])]
except:
features += [55]
right = row['PlayDirection'] == 'right'
if right:
ball_X0, ball_Y0, ball_S, ball_A, ball_D = row['X'], row['Y'], row['S'], row['A'], np.radians(row['Dir'])
 else:
ball_X0, ball_Y0, ball_S, ball_A, ball_D = 120 - row['X'], 53.3 - row['Y'], row['S'], row['A'], np.radians(( row['Dir'] + 180)% 360)
 break
featnames += ['RusherSpeed']
features += [ball_S]
featnames += ['RusherDir']
features += [ball_D]
featnames += ['RusherAccel']
features += [row['A']]
friends, enemies = np.zeros(( 10, 5)) , np.zeros(( 11, 5))
fidx, eidx = 0, 0
for _, row in rows.iterrows() :
if row['NflId'] == rusher:
continue
if right:
vals = [row['X'], row['Y'], row['S'], row['A'], np.radians(row['Dir'])]
else:
vals = [120 - row['X'], 53.3 - row['Y'], row['S'], row['A'], np.radians(( row['Dir'] + 180)% 360)]
if team == row['Team']:
friends[fidx, :] = vals
fidx += 1
else:
enemies[eidx, :] = vals
eidx += 1
assert(fidx == 10)
 assert(eidx == 11)
 friends_rel_X0 = friends[:, 0] - ball_X0
friends_rel_Y0 = friends[:, 1] - ball_Y0
friends_R0 =(friends_rel_X0 ** 2 + friends_rel_Y0 ** 2)**.5
friends_Theta0 = np.arctan2(friends_rel_Y0, friends_rel_X0)
 enemies_rel_X0 = enemies[:, 0] - ball_X0
enemies_rel_Y0 = enemies[:, 1] - ball_Y0
enemies_R0 =(enemies_rel_X0 ** 2 + enemies_rel_Y0 ** 2)**.5
enemies_Theta0 = np.arctan2(enemies_rel_Y0, enemies_rel_X0)
 featnames += ['Ball_X0', 'Ball_Y0']
features += [ball_X0, ball_Y0]
Rs = [0, 3, 10, 30, 120]
for r in range(len(Rs)-1):
rl, rh = Rs[r], Rs[r+1]
for t in range(8):
tl, th = t * np.pi/8,(t+1)* np.pi/8
key = 'map0_r%dq%d' %(rh, t)
 featnames.append('F' + key)
 features.append(np.sum(
(rl <= friends_R0)&(friends_R0 < rh)
 &(tl <= friends_Theta0)&(friends_Theta0 < th)))
 featnames.append('E' + key)
 features.append(np.sum(
(rl <= enemies_R0)&(enemies_R0 < rh)
 &(tl <= enemies_Theta0)&(enemies_Theta0 < th)))
 return featnames, features
def fparse(fname):
df = pd.read_csv(fname, low_memory=False)
 Y = df[['PlayId', 'Yards']].groupby('PlayId' ).mean().values if 'Yards' in df.columns else None
X = []
for p in tqdm(range(int(df.shape[0] / 22))):
rows = df.iloc[p * 22: p*22 + 22]
names, features = featurize(rows)
 X.append(features)
 return np.array(X), Y<prepare_x_and_y>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
X, Y = fparse('/kaggle/input/nfl-big-data-bowl-2020/train.csv' )<train_model>
data.isna().sum() [lambda x: x>0]
np.random.seed(123)
 X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.2)
 ngb = NGBoost(Dist=Normal, Score=CRPS() , verbose=True, learning_rate=0.01, n_estimators=5000)
 ngb.fit(X_train, Y_train, X_val=X_val, Y_val=Y_val)
 Y_dists = ngb.pred_dist(X_val)
 test_RMSE = np.sqrt(mean_squared_error(Y_dists.mean() , Y_val))
print('Val RMSE', test_RMSE)
 test_CRPS = Y_dists.crps(Y_val.flatten() ).mean()
print('Val CRPS', test_CRPS )<predict_on_test>
const_cols = ['MSSubClass','BldgType','HouseStyle','OverallQual','OverallCond','YearBuilt','YearRemodAdd','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','MasVnrArea','ExterQual','ExterCond','Foundation','WoodDeckSF','PavedDrive']
env = nflrush.make_env()
iter_test = env.iter_test()
Q = list(range(-99, 100))
for(batch, sample)in tqdm(iter_test):
_, X_feats = featurize(batch)
 X_test = np.array([X_feats])
 Y_pred = ngb.pred_dist(X_test)
 sample.iloc[0] = Y_pred.cdf(Q)
 env.predict(sample)
 env.write_submission_file()<import_modules>
data['MSSubClass']=data.MSSubClass.replace('150','120' ).astype(object )
from sklearn.decomposition import PCA
from sklearn.preprocessing import normalize<import_modules>
adaptative_correlation(data.OverallQual,data.tmpPrice )
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb
import xgboost as xgb<load_from_csv>
data['OverallQual'].replace([1, 2], 3, inplace=True)
 data['OverallQual'].replace([10], 9, inplace=True )
train = pd.read_csv(".. /input/santander-value-prediction-challenge/train.csv")
 test = pd.read_csv(".. /input/santander-value-prediction-challenge/test.csv" )<prepare_x_and_y>
data['OverallQual'] = data.OverallQual.astype('int' )
all_zero_columns=[i for i in train.columns if train[i].nunique() ==1]
train=train[[i for i in train.columns if i not in all_zero_columns]]
test=test[[i for i in test.columns if i not in all_zero_columns]]
duplicte_columns=['d60ddde1b', 'acc5b709d', '912836770', 'f8d75792f', 'f333a5f60']
train=train[[i for i in train.columns if i not in duplicte_columns]]
test=test[[i for i in test.columns if i not in duplicte_columns]]
X = np.log1p(train.drop(["ID", "target"], axis=1))
y = np.log1p(train["target"].values)
 test = np.log1p(test.drop(["ID"], axis=1))<choose_model_class>
data = data.drop(['RoofMatl'],axis=1 )
lgbbest=lgb.LGBMRegressor(bagging_fraction=0.5, boosting_type='gbdt', class_weight=None,
colsample_bytree=1.0, feature_fraction=0.5,
importance_type='split', learning_rate=0.01, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=500, n_jobs=-1, num_leaves=130,
objective='regression', random_state=42, reg_alpha=0.0,
reg_lambda=1, silent=True, subsample=1.0,
subsample_for_bin=200000, subsample_freq=0 )<save_to_csv>
data.OverallCond.value_counts()
lgbbest.fit(X, y)
 y_pred = lgbbest.predict(test)
 sub = pd.read_csv('.. /input/santander-value-prediction-challenge/sample_submission.csv')
 sub["target"] = np.expm1(y_pred)
 sub.to_csv('submit_lgbbestpara_allfeatures.csv', index=False )<load_from_csv>
mean_encoder(data,['MSSubClass','BldgType','HouseStyle','OverallCond','RoofStyle','Exterior1st','Exterior2nd','MasVnrType','ExterQual','ExterCond','Foundation','PavedDrive'])
sub1 = pd.read_csv('.. /input/love-is-the-answer-new-blendwinner/blend_new1.csv')
 sub2 = pd.read_csv('.. /input/love-is-the-answer-new-blendwinner/blend_new1.csv')
 sub3 = pd.read_csv('.. /input/love-is-the-answer-new-blendwinner/blend_new1.csv')
 train = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv')
 print(train.shape, test.shape )<data_type_conversions>
data[data.YearBuilt>data.YearRemodAdd]
leak_col = []
for c in col:
leak1 = np.sum(( train[c]==train['target'] ).astype(int))
leak2 = np.sum(((( train[c] - train['target'])/ train['target'])< 0.05 ).astype(int))
if leak1 > 30 and leak2 > 3500:
leak_col.append(c)
 print(len(leak_col))<drop_column>
data.loc[data.YearBuilt>data.YearRemodAdd,'YearRemodAdd']=2002
col = list(leak_col)
 train = train[col + ['ID', 'target']]
test = test[col + ['ID']]<feature_engineering>
data['YearBuilt'] = np.power(np.abs(85-(2010 - data.YearBuilt)) ,2 )
train["nz_mean"] = train[col].apply(lambda x: x[x!=0].mean() , axis=1)
 train["nz_max"] = train[col].apply(lambda x: x[x!=0].max() , axis=1)
 train["nz_min"] = train[col].apply(lambda x: x[x!=0].min() , axis=1)
 train["ez"] = train[col].apply(lambda x: len(x[x==0]), axis=1)
 train["mean"] = train[col].apply(lambda x: x.mean() , axis=1)
 train["max"] = train[col].apply(lambda x: x.max() , axis=1)
 train["min"] = train[col].apply(lambda x: x.min() , axis=1)
 test["nz_mean"] = test[col].apply(lambda x: x[x!=0].mean() , axis=1)
 test["nz_max"] = test[col].apply(lambda x: x[x!=0].max() , axis=1)
 test["nz_min"] = test[col].apply(lambda x: x[x!=0].min() , axis=1)
 test["ez"] = test[col].apply(lambda x: len(x[x==0]), axis=1)
 test["mean"] = test[col].apply(lambda x: x.mean() , axis=1)
 test["max"] = test[col].apply(lambda x: x.max() , axis=1)
 test["min"] = test[col].apply(lambda x: x.min() , axis=1)
 col += ['nz_mean', 'nz_max', 'nz_min', 'ez', 'mean', 'max', 'min']<data_type_conversions>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
for i in range(2, 100):
train['index'+str(i)] =(( train.index + 2)% i == 0 ).astype(int)
 test['index'+str(i)] =(( test.index + 2)% i == 0 ).astype(int)
 col.append('index'+str(i))<merge>
basement_cols = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath']
test = pd.merge(test, sub3, how='left', on='ID', )<concatenate>
adaptative_correlation(data.BsmtFinSF2,data.tmpPrice )
train = train.replace(0, np.nan)
 test = test.replace(0, np.nan)
 train = pd.concat(( train, test), axis=0, ignore_index=True )<train_model>
data = data.drop(['BsmtFinSF2'],axis=1 )
test['target'] = 0.0
folds = 5
for fold in range(folds):
x1, x2, y1, y2 = model_selection.train_test_split(train[col], np.log1p(train.target.values), test_size=0.20, random_state=fold)
 params = {'learning_rate': 0.02, 'max_depth': 7, 'boosting': 'gbdt', 'objective': 'regression', 'metric': 'rmse', 'is_training_metric': True, 'feature_fraction': 0.9, 'bagging_fraction': 0.8, 'bagging_freq': 5, 'seed':fold}
model = lgb.train(params, lgb.Dataset(x1, label=y1), 3000, lgb.Dataset(x2, label=y2), verbose_eval=200, early_stopping_rounds=100)
 test['target'] += np.expm1(model.predict(test[col], num_iteration=model.best_iteration))
test['target'] /= folds
test[['ID', 'target']].to_csv('submission.csv', index=False )<save_to_csv>
mean_encoder(data,['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2'])
b1 = sub1.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('submission.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend01.csv', index=False )<save_to_csv>
for idx, feat in enumerate(['OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch']):
print(feat," : ",adaptative_correlation(data[feat],data.tmpPrice))
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend01.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend02.csv', index=False )<save_to_csv>
floors_cols = ['1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','KitchenQual','TotRmsAbvGrd']
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend02.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.5)+(b1['dp2'] * 0.5)
 b1[['ID','target']].to_csv('blend03.csv', index=False )<save_to_csv>
adaptative_correlation(data.LowQualFinSF,data.tmpPrice )
b1 = sub3.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend03.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.6)+(b1['dp2'] * 0.4)
 b1[['ID','target']].to_csv('blend04.csv', index=False )<load_from_csv>
data = data.drop(['LowQualFinSF'],axis=1 )
!rm submission.csv
!rm blend01.csv
!rm blend02.csv
!rm blend03.csv<import_modules>
mean_encoder(data,['KitchenQual'])
<load_from_csv>
data['LivingArea'] = data['TotalBsmtSF'] + data['1stFlrSF']+\
data['2ndFlrSF']+\
data['OpenPorchSF']+\
data['3SsnPorch']
data['LivingArea'] = np.power(data['LivingArea'],0.35 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
HalfBathfactor = 0.6
data['TotalBath'] = data.BsmtFullBath + HalfBathfactor * data.BsmtHalfBath +\
data.FullBath + HalfBathfactor * data.HalfBath
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
data['Rooms_Area'] =(data.TotRmsAbvGrd+data.TotalBath)/ data.GrLivArea
all_features = [c for c in test.columns if c not in ['ID']]
def has_ugly(row):
for v in row.values[row.values > 0]:
if str(v)[::-1].find('.')> 2:
return True
return False
test['has_ugly'] = test[all_features].apply(has_ugly, axis=1)
 test_og = test[['ID']].copy()
test_og['nonzero_mean'] = test[[c for c in test.columns if c not in ['ID', 'has_ugly']]].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 test = test[test.has_ugly == False]<drop_column>
for feature in ['GrLivArea','BsmtFinSF1','1stFlrSF','2ndFlrSF','BsmtUnfSF','TotalBsmtSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch']:
data[feature] = np.power(data[feature],0.35 )
train_t = train.drop(['target'], axis = 1, inplace=False)
 train_t.set_index('ID', inplace=True)
 train_t = train_t.T
test_t = test.set_index('ID', inplace=False)
 test_t = test_t.T<set_options>
comfort_cols=['Heating','HeatingQC','CentralAir','Electrical','Functional','MiscFeature','MiscVal']
gc.collect()<define_variables>
mean_encoder(data,['Heating','HeatingQC','CentralAir','Electrical','Functional','MiscFeature'])
features = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1',
'15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9',
'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b',
'62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992',
'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd',
'703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2',
'190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98'] 
<define_variables>
data['MiscVal'] = np.log1p(data.MiscVal )
extra_features = []<feature_engineering>
sale_cols = ['MoSold','YrSold','SaleType','SaleCondition']
def find_new_ordered_features(ordered_ids, data_t):
data = data_t.copy()
f1 = ordered_ids[0][:-1]
f2 = ordered_ids[0][1:]
for ef in ordered_ids[1:]:
f1 += ef[:-1]
f2 += ef[1:]
d1 = data[f1].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d1['ID'] = data.index
gc.collect()
d2 = data[f2].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2['ID'] = data.index
gc.collect()
d3 = d2[~d2.duplicated(['key'], keep=False)]
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 d_feat = d1.merge(d5, how='left', on='key')
 d_feat.fillna(0, inplace=True)
 ordered_features = list(d_feat[['ID_x', 'ID_y']][d_feat.ID_x != 0].apply(list, axis=1))
del d1,d2,d3,d4,d5,d_feat
gc.collect()
links_found = 1
while links_found > 0:
links_found, ordered_features = chain_pairs(ordered_features)
 ordered_features = prune_chain(ordered_features)
 found = {}
for ef in extra_features:
found[ef[0]] = ef
found [features[0]] = features
new_feature_sets = []
for of in ordered_features:
if len(of)>= 40:
if of[0] not in found:
new_feature_sets.append(of)
 return new_feature_sets<feature_engineering>
mean_encoder(data,['MoSold','YrSold','SaleType','SaleCondition'])
def add_new_feature_sets(data, data_t):
print('
Data Shape:', data.shape)
 f1 = features[:-1]
f2 = features[1:]
for ef in extra_features:
f1 += ef[:-1]
f2 += ef[1:]
d1 = data[f1].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d1['ID'] = data['ID']
gc.collect()
d2 = data[f2].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2['ID'] = data['ID']
gc.collect()
d3 = d2[~d2.duplicated(['key'], keep=False)]
del d2
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 del d4
d = d1.merge(d5, how='left', on='key')
 d.fillna(0, inplace=True)
 ordered_ids = list(d[['ID_x', 'ID_y']][d.ID_x != 0].apply(list, axis=1))
del d1,d3,d5,d
gc.collect()
links_found = 1
while links_found > 0:
links_found, ordered_ids = chain_pairs(ordered_ids)
 print('OrderedIds:', len(ordered_ids))
ordered_ids = prune_chain(ordered_ids)
 print('OrderedIds Pruned:', len(ordered_ids))
new_feature_sets = find_new_ordered_features(ordered_ids, data_t)
 extra_features.extend(new_feature_sets)
 print('New Feature Count:', len(new_feature_sets))
print('Extra Feature Count:', len(extra_features))<feature_engineering>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
%%time
add_new_feature_sets(train,train_t)
 add_new_feature_sets(test,test_t)
 add_new_feature_sets(train,train_t)
 add_new_feature_sets(test,test_t)
 add_new_feature_sets(train,train_t )<save_to_csv>
train_lenght = len(data.loc['train'] )
with open("extra_features_{}.txt".format(len(extra_features)) , "w")as text_file:
for ef in extra_features:
text_file.write(','.join(ef)+ '
' )<set_options>
y = data.tmpPrice.loc['train']
y_target = data.tmpPrice.loc['test']
del train_t, test_t, test
gc.collect()<load_from_csv>
X = data.drop(['SalePrice', 'tmpPrice'],axis=1)
 X = pd.get_dummies(X)
 X = RobustScaler().fit_transform(X)
 X_target = X[train_lenght:,:]
X = X[:train_lenght,:]
test = pd.read_csv('.. /input/test.csv')
 test['has_ugly'] = test[all_features].apply(has_ugly, axis=1)
 test[test.has_ugly == True] = 0<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0 )
def get_log_pred(data, feats, extra_feats, offset = 2):
f1 = feats[:(offset * -1)]
f2 = feats[offset:]
for ef in extra_feats:
f1 += ef[:(offset * -1)]
f2 += ef[offset:]
d1 = data[f1].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2 = data[f2].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2['pred'] = data[feats[offset-2]]
d2 = d2[d2['pred'] != 0]
d3 = d2[~d2.duplicated(['key'], keep=False)]
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 d = d1.merge(d5, how='left', on='key')
 return np.log1p(d.pred ).fillna(0)
<compute_test_metric>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score,cross_val_predict, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
end_offset = 39
pred_test = []
pred_train = []
efs = extra_features
for o in tqdm_notebook(list(range(2, end_offset))):
print('Offset:', o)
 log_pred = get_log_pred(train, features, extra_features, o)
 pred_train.append(np.expm1(log_pred))
have_data = log_pred != 0
train_count = have_data.sum()
score = sqrt(mean_squared_error(np.log1p(train.target[have_data]), log_pred[have_data]))
print(f'Score = {score} on {have_data.sum() } out of {train.shape[0]} training samples')
 log_pred_test = get_log_pred(test, features, efs, o)
 pred_test.append(np.expm1(log_pred_test))
have_data = log_pred_test != 0
test_count = have_data.sum()
print(f'Have predictions for {have_data.sum() } out of {test.shape[0]} test samples')
<feature_engineering>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X)
 rmse= np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
pred_train_final = pred_train[0].copy()
for r in range(1,len(pred_train)) :
pred_train_final[pred_train_final == 0] = pred_train[r][pred_train_final == 0]
train_leak_match_count =(pred_train_final!=0 ).sum() ;
no_match_count =(pred_train_final==0 ).sum() ;
print("Train leak count: ", train_leak_match_count, "Train no leak count: ", no_match_count)
 pred_train_temp = pred_train_final.copy()
train["nonzero_mean"] = train[[f for f in train.columns if f not in ["ID", "target","nonzero_mean"]]].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 pred_train_temp[pred_train_temp==0] = train['nonzero_mean'][pred_train_temp==0]
print(f'Baseline Train Score = {sqrt(mean_squared_error(np.log1p(train.target), np.log1p(pred_train_temp)))}' )<prepare_output>
model1 = linear_model.Ridge(alpha=30)
 score = rmsle_cv(model1)
 print("model score: {:.4f} standard deviation({:.4f})
 ".format(score.mean() , score.std()))
pred_test_final = pred_test[0].copy()
for r in range(1,len(pred_test)) :
pred_test_final[pred_test_final == 0] = pred_test[r][pred_test_final == 0] 
<feature_engineering>
model2 = linear_model.Lasso(alpha=.00075)
 score = rmsle_cv(model2)
 print("model score: {:.4f} standard deviation({:.4f})
 ".format(score.mean() , score.std()))
pred_test_final[(4e+07 < pred_test_final)] = 4e+07
pred_test_final[(( pred_test_final < 29000)&(pred_test_final > 0)) ] = 30000
pred_test_final[test.ID == 'd72fad286'] = 1560000
pred_test_final[test.ID == 'a304cde42'] = 320000.0
test_leak_match_count =(pred_test_final!=0 ).sum() ;
no_match_count =(pred_test_final==0 ).sum() ;
print("Test leak count: ", test_leak_match_count, "Test no leak count: ", no_match_count )<save_to_csv>
model3 = RandomForestRegressor(bootstrap=True, ccp_alpha=0.000001, criterion='mse',
max_depth=None, max_features=0.2, max_leaf_nodes=None,
max_samples=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
n_estimators=1000, n_jobs=-1, oob_score=False,
random_state=None, verbose=0, warm_start=False)
 score = rmsle_cv(model3)
 print("model score: {:.4f} standard deviation({:.4f})
 ".format(score.mean() , score.std()))
pred_test_temp = pred_test_final.copy()
test_og["nonzero_mean"] = test_og[[f for f in test_og.columns if f not in ["ID", "target", "nonzero_mean", "has_ugly"]]].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 pred_test_temp[pred_test_temp==0] = test_og['nonzero_mean'][pred_test_temp==0]
test_og['target']=pred_test_temp
test_og[['ID', 'target']].to_csv('leak_baseline_{}.csv'.format(test_leak_match_count), index=False)
<save_to_csv>
model4 = GradientBoostingRegressor(n_estimators=500,
learning_rate=0.027,
max_features=0.7,
criterion='mse',
min_samples_leaf=4,
min_samples_split=4,
loss='huber',
random_state =42)
 score = rmsle_cv(model4)
 print("gbr score: {:.4f} standard deviation({:.4f})
 ".format(score.mean() , score.std()))
test_leaks = pd.read_csv('.. /input/sample_submission.csv')
 del test_leaks['target']
test_leaks['target']=pred_test_final
test_leaks.to_csv('leak_only_{}.csv'.format(test_leak_match_count), index=False)
<concatenate>
model5 = xgb.XGBRegressor(learning_rate=0.04,n_estimators=1000,
max_depth=2, min_child_weight=0,
gamma=0, subsample=0.75,
colsample_bytree=0.6,
objective='reg:linear', nthread=-1,
scale_pos_weight=1.2, seed=27,
reg_alpha=0.0006 )
extra_features_list = []
for ef in extra_features:
extra_features_list.extend(ef)
 extra_features_list.extend(features)
 len(extra_features_list )<create_dataframe>
score = rmsle_cv(model5)
 print("xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
feats = pd.DataFrame(extra_features)
 time_features = []
for c in feats.columns[:]:
time_features.append([f for f in feats[c].values if f is not None])
 agg_features = []
all_cols = train.columns.drop(['ID', 'target', 'nonzero_mean'])
 agg_features.append(all_cols)
 agg_features.append([c for c in all_cols if c not in extra_features_list])
 agg_features.append(extra_features_list)
 agg_features.extend(time_features)
 agg_features.extend(extra_features)
<feature_engineering>
model1.fit(X, y)
 model2.fit(X, y)
 model3.fit(X, y)
 model4.fit(X, y)
 model5.fit(X, y)
 predictions1 = model1.predict(X_target)
 predictions2 = model2.predict(X_target)
 predictions3 = model3.predict(X_target)
 predictions4 = model4.predict(X_target)
 predictions5 = model5.predict(X_target)
 predictions = 0.2*predictions1+0.3*predictions2+0.1*predictions3+0.2*predictions4+0.2*predictions5
def add_new_features(source, dest, feats):
dest['mean_log_{}_{}'.format(feats[0], len(feats)) ] = np.log1p(source[feats].replace(0, np.nan ).mean(axis=1))
dest['first_nonZero_{}_{}'.format(feats[0], len(feats)) ] = np.log1p(source[feats].replace(0, np.nan ).bfill(1 ).iloc[:, 0])
 dest['last_nonZero_{}_{}'.format(feats[0], len(feats)) ] = np.log1p(source[feats[::-1]].replace(0, np.nan ).bfill(1 ).iloc[:, 0])
<load_from_csv>
predictions = np.expm1(predictions )
del test
gc.collect
test = pd.read_csv('.. /input/test.csv' )<create_dataframe>
submission = pd.DataFrame({'Id': data.loc['test'].index, 'SalePrice': predictions} )
train_feats = pd.DataFrame()
test_feats =pd.DataFrame()
for i, ef in tqdm_notebook(list(enumerate(agg_features))):
add_new_features(train, train_feats, ef)
 add_new_features(test, test_feats, ef )<predict_on_test>
submission.to_csv("submission.csv", index=False )
class MyModel() :
def __init__(self, X_tr, y_tr, X_val, y_val, X_test):
self.X_tr = X_tr
self.y_tr = y_tr
self.X_val = X_val
self.y_val = y_val
self.X_test = X_test
self.params = {}
def predict_val(self):
return self.model.predict(self.X_val)
 def predict_test(self):
return self.model.predict(self.X_test)
 class LgbBoostModel(MyModel):
def train(self):
self.params = { 'objective': 'regression', 'metric': 'rmse', 'boosting': 'gbdt', 'seed':seed, 'is_training_metric': True
,'max_bin': 350
,'learning_rate':.005
,'max_depth': -1
,'num_leaves': 48
,'feature_fraction': 0.1
,'reg_alpha': 0
,'reg_lambda': 0.2
,'min_child_weight': 10}
self.model = lgb.train(self.params, lgb.Dataset(self.X_tr, label=self.y_tr), 30000,
[lgb.Dataset(self.X_tr, label=self.y_tr), lgb.Dataset(self.X_val, label=self.y_val)],
verbose_eval=200, early_stopping_rounds=200)
<prepare_x_and_y>
submission.to_csv("submission.csv", index=False )
cols = train_feats.columns
train_feat_final = pd.concat([train_feats[cols], test_feats[cols][test_leaks.target != 0]], axis = 0)
 train_feat_id = pd.concat([train['ID'], test['ID'][test_leaks.target != 0]], axis = 0)
 test_feat_final = test_feats[cols]
y = np.array(list(np.log1p(train.target.values)) + list(np.log1p(test_leaks['target'][test_leaks.target != 0])))
 X = train_feat_final.values
X_test = test_feat_final.values
print(X.shape)
 print(X_test.shape )<feature_engineering>
%matplotlib inline
warnings.filterwarnings("ignore")
 %matplotlib inline
def mean(values):
return float(sum(values)) / max(len(values), 1)
 def sum_of_square_deviation(values, mean):
return float(1/len(values)* sum(( x - mean)** 2 for x in values))
def export_results(model_type):
subm = pd.read_csv('.. /input/sample_submission.csv')
 subm['target'] = np.expm1(sub_preds[model_type])
 oof = pd.DataFrame(train_feat_id.copy())
 oof['target'] = np.expm1(y)
 oof['prediction'] = np.expm1(oof_preds[model_type])
 mean_rmse = mean(rmse_scores[model_type])
 standard_deviation_rmse = math.sqrt(sum_of_square_deviation(rmse_scores[model_type],mean_rmse))
key = '{}'.format(model_type)
 print('{} Mean Squared Error {}'.format(model_type ,mean_rmse))
print('{} Stdev Squared Error {}'.format(model_type, standard_deviation_rmse))
file_name = 'subm_{}_ml_base.csv'.format(key)
 subm.to_csv(file_name, index=False, float_format="%.8f")
 file_name = 'submission.csv'.format(key)
 subm['target'][test_leaks.target != 0] = test_leaks['target'][test_leaks.target != 0]
subm.to_csv(file_name, index=False, float_format="%.8f")
 file_name = 'subm_{}_oof.csv'.format(key)
 oof.to_csv(file_name, index=False, float_format="%.8f")
 model_params[model_type]['cv_score'] = int(mean_rmse * 10000)
 model_params[model_type]['cv_stddev'] = int(standard_deviation_rmse * 10000)
 model_params[model_type]['train_row_count'] = X.shape[0]
model_params[model_type]['train_feature_count'] = X.shape[1]
model_params[model_type]['test_leak_count'] =(test_leaks.target != 0 ).sum()
with open('subm_{}_params.txt'.format(key), "w")as text_file:
params = str(model_params[model_type])
 print(f"{params}", file=text_file )<load_from_csv>
train_data=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test_data['SalePrice']=np.nan
test_data['tmpPrice']=np.nan
sub1 = pd.read_csv('.. /input/best-ensemble-score-made-available-0-68/SHAZ13_ENS_LEAKS.csv')
 sub3 = pd.read_csv('.. /input/feature-scoring-vs-zeros/leaky_submission.csv')
 sub2 = pd.read_csv('.. /input/feature-scoring-vs-zeros/leaky_submission.csv')
 train = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv')
 print(train.shape, test.shape )<data_type_conversions>
def cramers_v(x, y):
confusion_matrix = pd.crosstab(x,y)
 chi2 = scipy.stats.chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum().sum()
phi2 = chi2/n
r,k = confusion_matrix.shape
phi2corr = max(0, phi2-(( k-1)*(r-1)) /(n-1))
rcorr = r-(( r-1)**2)/(n-1)
 kcorr = k-(( k-1)**2)/(n-1)
 return np.sqrt(phi2corr/min(( kcorr-1),(rcorr-1)) )
leak_col = []
for c in col:
leak1 = np.sum(( train[c]==train['target'] ).astype(int))
leak2 = np.sum(((( train[c] - train['target'])/ train['target'])< 0.05 ).astype(int))
if leak1 > 30 and leak2 > 3500:
leak_col.append(c)
 print(len(leak_col))<drop_column>
def eta_squared(x,y):
moyenne_y = y.mean()
classes = []
for classe in x.unique() :
yi_classe = y[x==classe]
classes.append({'ni': len(yi_classe),
'moyenne_classe': yi_classe.mean() })
 SCT = sum([(yj-moyenne_y)**2 for yj in y])
 SCE = sum([c['ni']*(c['moyenne_classe']-moyenne_y)**2 for c in classes])
 return SCE/SCT
col = list(leak_col)
 train = train[col + ['ID', 'target']]
test = test[col + ['ID']]<feature_engineering>
def adaptative_correlation(x,y):
x = x.reset_index(drop=True)
 y = y.reset_index(drop=True)
 non_na = np.logical_and(~x.isna() ,~y.isna())
 x=x[non_na]
y=y[non_na]
if(x.dtype.name=='object' or x.dtype.name=='category')and(y.dtype.name=='object' or y.dtype.name=='category'):
return cramers_v(x,y)
 elif(x.dtype.name=='object' or x.dtype.name=='category'):
return abs(eta_squared(x,y))
elif(y.dtype.name=='object' or y.dtype.name=='category'):
return abs(eta_squared(y,x))
else:
return abs(x.corr(y))
train["nz_mean"] = train[col].apply(lambda x: x[x!=0].mean() , axis=1)
 train["nz_max"] = train[col].apply(lambda x: x[x!=0].max() , axis=1)
 train["nz_min"] = train[col].apply(lambda x: x[x!=0].min() , axis=1)
 train["ez"] = train[col].apply(lambda x: len(x[x==0]), axis=1)
 train["mean"] = train[col].apply(lambda x: x.mean() , axis=1)
 train["max"] = train[col].apply(lambda x: x.max() , axis=1)
 train["min"] = train[col].apply(lambda x: x.min() , axis=1)
 test["nz_mean"] = test[col].apply(lambda x: x[x!=0].mean() , axis=1)
 test["nz_max"] = test[col].apply(lambda x: x[x!=0].max() , axis=1)
 test["nz_min"] = test[col].apply(lambda x: x[x!=0].min() , axis=1)
 test["ez"] = test[col].apply(lambda x: len(x[x==0]), axis=1)
 test["mean"] = test[col].apply(lambda x: x.mean() , axis=1)
 test["max"] = test[col].apply(lambda x: x.max() , axis=1)
 test["min"] = test[col].apply(lambda x: x.min() , axis=1)
 col += ['nz_mean', 'nz_max', 'nz_min', 'ez', 'mean', 'max', 'min']<data_type_conversions>
imputer = KNNImputer(n_neighbors=3)
 n_folds = 5
model = RandomForestRegressor(n_jobs=-1)
 def quick_predict(X,y):
X = pd.get_dummies(X)
 X = pd.DataFrame(imputer.fit_transform(X), columns = X.columns)
 kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X)
 rmse= np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv = kf))
print("Quality score: {:.4f} standard deviation({:.4f})
 ".format(rmse.mean() , rmse.std()))
for i in range(2, 100):
train['index'+str(i)] =(( train.index + 2)% i == 0 ).astype(int)
 test['index'+str(i)] =(( test.index + 2)% i == 0 ).astype(int)
 col.append('index'+str(i))<merge>
quick_predict(train_data.drop(['SalePrice'],axis=1),train_data.SalePrice )
test = pd.merge(test, sub1, how='left', on='ID', )<concatenate>
def mean_encoder(df,columns):
for col in columns:
if col not in df.columns:
print("Error, ",col," not in list of columns")
 else:
mean_encode = df.groupby(col)['tmpPrice'].mean()
mean_encode =(mean_encode - mean_encode.min())/(mean_encode.max() - mean_encode.min())
 new_col_name = col + '_enc'
df.loc[:,new_col_name] = df[col].map(mean_encode )
train = train.replace(0, np.nan)
 test = test.replace(0, np.nan)
 train = pd.concat(( train, test), axis=0, ignore_index=True )<train_model>
len(train_data[train_data.SalePrice>600000] )
test['target'] = 0.0
folds = 5
for fold in range(folds):
x1, x2, y1, y2 = model_selection.train_test_split(train[col], np.log1p(train.target.values), test_size=0.20, random_state=fold)
 params = {'learning_rate': 0.02, 'max_depth': 7, 'boosting': 'gbdt', 'objective': 'regression', 'metric': 'rmse', 'is_training_metric': True, 'feature_fraction': 0.9, 'bagging_fraction': 0.8, 'bagging_freq': 5, 'seed':fold}
model = lgb.train(params, lgb.Dataset(x1, label=y1), 3000, lgb.Dataset(x2, label=y2), verbose_eval=200, early_stopping_rounds=100)
 test['target'] += np.expm1(model.predict(test[col], num_iteration=model.best_iteration))
test['target'] /= folds
test[['ID', 'target']].to_csv('submission.csv', index=False )<save_to_csv>
train_data = train_data[train_data.SalePrice<600000]
b1 = sub1.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('submission.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend01.csv', index=False )<save_to_csv>
scipy.stats.skew(train_data.SalePrice.dropna() )
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend01.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend02.csv', index=False )<save_to_csv>
train_data['tmpPrice']=np.log1p(train_data.SalePrice )
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend02.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.5)+(b1['dp2'] * 0.5)
 b1[['ID','target']].to_csv('blend03.csv', index=False )<save_to_csv>
tmpPriceMean = train_data['tmpPrice'].mean()
train_data['tmpPrice'] = train_data['tmpPrice']
b1 = sub3.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend03.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.6)+(b1['dp2'] * 0.4)
 b1[['ID','target']].to_csv('blend04.csv', index=False )<load_from_csv>
quick_predict(train_data.drop(['SalePrice','tmpPrice'],axis=1),train_data.tmpPrice )
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()<save_to_csv>
train_data.set_index(train_data.Id,drop=True, inplace=True)
 train_data = train_data.drop(['Id'],axis=1)
 test_data.set_index(test_data.Id,drop=True, inplace=True)
 test_data = test_data.drop(['Id'],axis=1 )
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
data=pd.concat([train_data,test_data],keys=['train','test'], join='inner' )
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<define_variables>
data[['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond',
'PoolQC','Fence','MiscFeature']]=data[['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond',
'PoolQC','Fence','MiscFeature']].fillna(value="NA" )
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
data.isna().sum() [lambda x: x>0]
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
data[data.MSZoning.isna() ]
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )<init_hyperparams>
data[data.Neighborhood=="Mitchel"].MSZoning.value_counts()
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
data[data.Neighborhood=="IDOTRR"].MSZoning.value_counts()
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
data.loc[(data.MSZoning.isna())&(data.Neighborhood=="Mitchel"),"MSZoning"]="RL"
data.loc[(data.MSZoning.isna())&(data.Neighborhood=="IDOTRR"),"MSZoning"]="RM"
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
data['LotAreaSqr']=np.sqrt(data.LotArea )
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<prepare_x_and_y>
data.LotShape.value_counts()
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values<concatenate>
data['LotFrontage']=data.LotFrontage.fillna(data.LotAreaSqr )
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
data = data.drop(['LotAreaSqr'],axis=1 )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )<train_model>
data = data.drop(['Utilities'],axis=1 )
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )<save_to_csv>
data[data.Exterior1st.isna() ]
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<set_options>
data[(data.YearBuilt>1935)&(data.YearBuilt<1945)&(data.Neighborhood=='Edwards')].Exterior1st.value_counts()
eng_stopwords = set(stopwords.words('english'))
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None<load_from_csv>
data['Exterior1st']=data.Exterior1st.fillna('MetalSd')
 data['Exterior2nd']=data.Exterior2nd.fillna('MetalSd' )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print(train_df.shape)
 print(test_df.shape )<feature_engineering>
data[data.BsmtFinSF1.isna() ]
all_ques_df = pd.DataFrame(pd.concat([train_df['question1'], train_df['question2']]))
all_ques_df.columns = ["questions"]
all_ques_df["num_of_words"] = all_ques_df["questions"].apply(lambda x : len(str(x ).split()))<feature_engineering>
data.loc[data.BsmtFinSF1.isna() ,'BsmtFinSF2' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtUnfSF' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtFullBath' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'TotalBsmtSF' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtHalfBath' ]=0.0
data.loc[data.BsmtFinSF1.isna() ,'BsmtFinSF1' ]=0.0
data.loc[data.BsmtQual=="NA",'BsmtExposure' ]="NA"
data.loc[data.BsmtQual=="NA",'BsmtFullBath' ]=0.0
data.loc[data.BsmtQual=="NA",'BsmtHalfBath' ]=0.0
def get_unigrams(que):
return [word for word in word_tokenize(que.lower())if word not in eng_stopwords]
def get_common_unigrams(row):
return len(set(row["unigrams_ques1"] ).intersection(set(row["unigrams_ques2"])))
 def get_common_unigram_ratio(row):
return float(row["unigrams_common_count"])/ max(len(set(row["unigrams_ques1"] ).union(set(row["unigrams_ques2"]))),1)
 train_df["unigrams_ques1"] = train_df['question1'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_ques2"] = train_df['question2'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_common_count"] = train_df.apply(lambda row: get_common_unigrams(row),axis=1)
 train_df["unigrams_common_ratio"] = train_df.apply(lambda row: get_common_unigram_ratio(row), axis=1 )<concatenate>
data[data.Electrical.isna() ]
ques = pd.concat([train_df[['question1', 'question2']], \
test_df[['question1', 'question2']]], axis=0 ).reset_index(drop='index')
 ques.shape<define_variables>
data.Electrical.value_counts()
q_dict = defaultdict(set)
 for i in range(ques.shape[0]):
q_dict[ques.question1[i]].add(ques.question2[i])
 q_dict[ques.question2[i]].add(ques.question1[i] )<feature_engineering>
data[['Electrical']]=data[['Electrical']].fillna(value="SBrkr" )
def q1_freq(row):
return(len(q_dict[row['question1']]))
def q2_freq(row):
return(len(q_dict[row['question2']]))
def q1_q2_intersect(row):
return(len(set(q_dict[row['question1']] ).intersection(set(q_dict[row['question2']]))))
train_df['q1_q2_intersect'] = train_df.apply(q1_q2_intersect, axis=1, raw=True)
 train_df['q1_freq'] = train_df.apply(q1_freq, axis=1, raw=True)
 train_df['q2_freq'] = train_df.apply(q2_freq, axis=1, raw=True )<import_modules>
data['KitchenQual']=data['KitchenQual'].fillna(value="TA" )
import tensorflow as tf
import os<compute_test_metric>
data['GarageYrBlt']=data['GarageYrBlt'].fillna(data.YearBuilt )
def apk(actual, predicted, k):
if actual not in predicted:
return 0.0
for i in range(k):
if actual in predicted[:i+1]:
return 1.0 / len(predicted[:i+1] )<load_from_csv>
data[data.GarageCars.isna() ]
sub_df = pd.read_csv(data_path+'sample_submission.csv')
 print("test data 수:",len(sub_df))
sub_df.head()<load_from_csv>
data.loc[data.GarageCars.isna() ,data.columns.str.contains('Gar')]
train_file_path = data_path + 'train_raw/'
eiffel_df = pd.read_csv(train_file_path + 'The Eiffel Tower.csv')
 eiffel_df.head()<load_from_csv>
data['GarageArea']=data['GarageArea'].fillna(value=300)
 data['GarageCars']=data['GarageCars'].fillna(value=1.0 )
train_simple_file_path = data_path + 'train_simplified/'
eiffel_simple_df = pd.read_csv(train_simple_file_path + 'The Eiffel Tower.csv')
 eiffel_simple_df.head()<define_variables>
data.Functional.value_counts()
raw_images = [json.loads(draw)for draw in eiffel_df.head() ['drawing'].values]
simple_images = [json.loads(draw)for draw in eiffel_simple_df.head() ['drawing'].values]<prepare_x_and_y>
data['Functional']=data['Functional'].fillna(value="Typ" )
print("======== 첫번째 raw drawing의 첫 획 Data 중 5개 Point 정보 =========")
 print("x좌표: ", json.loads(eiffel_df['drawing'][0])[0][0][:5])
 print("y좌표: ", json.loads(eiffel_df['drawing'][0])[0][1][:5])
 print("msec: ", json.loads(eiffel_df['drawing'][0])[0][2][:5])
 print("======== 첫번째 Simplified drawing의 첫 획 Data 중 5개 Point 정보 =========")
 print("x좌표: ", json.loads(eiffel_simple_df['drawing'][0])[0][0][:5])
 print("y좌표: ", json.loads(eiffel_simple_df['drawing'][0])[0][1][:5] )<define_variables>
data.SaleType.value_counts()
train_csvs= os.listdir(train_file_path)
 print("train_raw 폴더 내 파일 수:", len(train_csvs))
print(train_csvs[:5])
 file_size = 0
label_names = []
for csv_file in train_csvs:
file_size += os.path.getsize(train_file_path + csv_file)
 label_names.append(csv_file.replace('.csv',''))
print("파일 크기 : ", file_size//(1024*1024*1024),"GB")
 label_names = sorted(label_names,key=lambda x : str.lower(x+'.csv'))<load_from_csv>
data['SaleType']=data['SaleType'].fillna(value="WD" )
test_raw_df = pd.read_csv(data_path+"test_raw.csv")
 test_raw_df.head()
print(test_raw_df.shape )<train_model>
X = data.drop(['SalePrice', 'tmpPrice','MasVnrType','MasVnrArea'],axis=1)
 X = X.reset_index(level=0, drop=True)
 X = pd.get_dummies(X)
 Y = data[['MasVnrType']]
Y = Y.reset_index(level=0, drop=True)
 Y_source = Y.dropna()
X_source = X.loc[Y_source.index,:]
X_target = X.loc[set(X.index)- set(Y_source.index),:]
class DoodelGenerator(tf.keras.utils.Sequence):
def __init__(self, df_files, input_shape, batchsize,label_num=340, lw=3, state='Train', last_drop_r=0.0, mixup_r = 0.0):
self.df_files = df_files
self.file_sel = 0
self.batchsize = batchsize
self.input_shape = input_shape
self.label_num = label_num
self.lw = lw
self.state = state
self.last_drop_r = last_drop_r
self.mixup_r = mixup_r
self.on_epoch_end()
self.len = -(-len(self.df)//self.batchsize)
 def __len__(self):
return self.len
def __getitem__(self, index):
batch_idx = self.idx[index*self.batchsize:(index+1)*self.batchsize]
h,w,ch = self.input_shape
X = np.zeros(( len(batch_idx), h,w,ch))
y = np.zeros(( len(batch_idx), self.label_num))
df = self.df.loc[batch_idx]
mixup_num = int(self.batchsize*self.mixup_r)
 mixup_df = self.df.loc[np.random.randint(0,len(self.df),size=mixup_num)]
mixup_strokes=[]
mixup_labels=[]
for raw_strokes, label in mixup_df.values:
mixup_strokes.append(json.loads(raw_strokes))
mixup_labels.append(label)
 for i in range(self.batchsize):
raw_strokes = json.loads(df.drawing.values[i])
 if i < len(mixup_strokes):
X[i, :, :, ] = draw_raw_cv2(raw_strokes, size=h, lw=self.lw
, last_drop_r = self.last_drop_r,second_strokes=mixup_strokes[i])
 if self.state != 'Test':
ysm_mix = self.smooth_labels(to_categorical(mixup_labels[i], num_classes=self.label_num))
ysm_org = self.smooth_labels(to_categorical(df.y.values[i], num_classes=self.label_num))
y[i, :] =(ysm_mix*0.5)+(ysm_org*0.5)
 else:
X[i, :, :, ] = draw_raw_cv2(raw_strokes, size=h, lw=self.lw
, last_drop_r = self.last_drop_r)
 if self.state != 'Test':
y[i, :] = to_categorical(df.y.values[i], num_classes=self.label_num)
 if self.state != 'Test':
return X,y
else:
return X
def get_cur_df(self):
return self.df
def smooth_labels(self,labels, factor=0.1):
labels *=(1 - factor)
 labels +=(factor / labels.shape[0])
 return labels
def on_epoch_end(self):
self.df = pd.read_csv(self.df_files[self.file_sel])
 print('current step file : ', self.df_files[self.file_sel], 'state:', self.state, 'df_len:', self.df.shape[0])
 self.idx = np.tile(np.arange(len(self.df)) ,2)
 if self.state == 'Train':
np.random.shuffle(self.idx)
 self.file_sel =(self.file_sel+1)%len(self.df_files)
<define_variables>
le = preprocessing.LabelEncoder()
le.fit(Y_source.values )
hold_out_set= 'train_k99'<choose_model_class>
Y_source = le.transform(Y_source )
!pip install efficientnet
def build_model(backbone= efn.EfficientNetB0, input_shape =(128,128,3), use_imagenet = 'imagenet'):
base_model = backbone(input_shape=input_shape, weights=use_imagenet,include_top= False)
 x = base_model.output
x = tf.keras.layers.GlobalAvgPool2D(name='gap' )(x)
 predictions = tf.keras.layers.Dense(len(label_names), activation='softmax', name='prediction' )(x)
 model = tf.keras.Model(inputs=base_model.input, outputs=predictions)
 return model<compute_test_metric>
clf = RandomForestClassifier(n_estimators=300, max_depth=None,min_samples_split=2, random_state=0)
 scores = cross_val_score(clf, X_source, Y_source, cv=5)
 scores.mean()
def mapk(actual, predicted, k=3):
return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
 def map_at3(y_true, y_pred):
map3 = tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)*0.5
map3 += tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)*0.17
map3 += tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)*0.33
return map3<define_search_space>
_=clf.fit(X_source, Y_source )
recipes = []
recipes.append({'backbone':efn.EfficientNetB7, "batch_size":30, 'name':'Efb7','val_sel':7,'input_shape':(128,128,3),'lw':2} )<define_variables>
Y_target = pd.DataFrame(le.inverse_transform(clf.predict(X_target)) ).set_index(X_target.index )
ext_data_path = '/kaggle/input/doodle-model/'<load_pretrained>
Y_target.columns = ['MasVnrType']
for i, recipe in enumerate(recipes):
model_name = recipe["name"] + '_val_' + str(recipe["val_sel"])+ '_ep2'
best_save_model_file = model_name + '.h5'
print('best_save_model_file path : ',best_save_model_file)
 model = build_model(backbone= recipe['backbone'], input_shape = recipe['input_shape'], use_imagenet = None)
 model.load_weights(ext_data_path + best_save_model_file)
 test_datagen = DoodelGenerator([data_path+"test_raw.csv"], input_shape=recipe['input_shape'], lw=recipe['lw']
, batchsize=recipe['batch_size'],state='Test')
 test_predictions = model.predict(test_datagen, verbose=1)
 top3 = preds2catids(test_predictions)
 id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(label_names)}
top3cats = top3.replace(id2cat)
 sub_df['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = sub_df[['key_id', 'word']]
submission.to_csv('submission_'+model_name+'.csv', index=False )<init_hyperparams>
X = data.drop(['SalePrice', 'tmpPrice','MasVnrArea'],axis=1)
 X = X.reset_index(level=0, drop=True)
 X = pd.get_dummies(X)
 Y = data[['MasVnrArea']]
Y = Y.reset_index(level=0, drop=True)
 Y_source = Y.dropna()
X_source = X.loc[Y_source.index,:]
X_target = X.loc[set(X.index)- set(Y_source.index),:]
def balancing_predictions(test_prob, factor = 0.1, minfactor = 0.001, patient = 5, permit_cnt=332, max_search=10000, label_num=340):
maxk = float('inf')
 s_cnt = np.zeros(label_num)
 for i in range(max_search):
ctop1 = Counter(np.argmax(test_prob,axis=1))
ctop1 = sorted(ctop1.items() , key=itemgetter(1), reverse=True)
 if maxk > ctop1[0][1]:
maxk = ctop1[0][1]
else:
s_cnt[ctop1[0][0]]+=1
if np.max(s_cnt)>patient:
if factor< minfactor:
print('stop min factor')
 break
s_cnt=np.zeros(label_num)
 factor*=0.99
print('reduce factor: ', factor, ', current max category num: ', ctop1[0][1])
 if ctop1[0][1] <= permit_cnt:
print('idx: ',ctop1[0][0] ,', num: ', ctop1[0][1])
 break
test_prob[:,ctop1[0][0]] *=(1.0-factor)
 return test_prob<save_to_csv>
clf = RandomForestRegressor(n_estimators=100, min_samples_split=3, random_state=0)
 scores = cross_val_score(clf, X_source, Y_source, cv=5, scoring="neg_mean_squared_error",n_jobs=-1)
 scores.mean()
bal_test_prob = balancing_predictions(test_predictions)
 bal_top3 = preds2catids(bal_test_prob)
 bal_top3cats = bal_top3.replace(id2cat)
 sub_df['word'] = bal_top3cats['a'] + ' ' + bal_top3cats['b'] + ' ' + bal_top3cats['c']
bal_submission = sub_df[['key_id', 'word']]
bal_submission.to_csv('submission_bal_'+model_name+'.csv', index=False )<import_modules>
_=clf.fit(X_source, Y_source)
 Y_target = pd.DataFrame(clf.predict(X_target)).set_index(X_target.index)
 Y_target.columns = ['MasVnrArea']
DATA_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition'
BASE_SIZE = 256
num_csvs = 100
num_classes = 340
lw = 1
channel = 3
border = 2
def f2cat(filename):
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower)
 def preds2catids(predictions):
	return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def apk(actual, predicted, k=3):

 print('actual:', actual, 'predicted:', predicted)
 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3)
 shift_colors =(
(255, 0, 0), 
(255, 128, 0), 
(255, 255, 0), 
(128, 255, 0), 
(0, 255, 0), 
(0, 255, 128), 
(0, 255, 255), 
(0, 128, 255), 
(0, 0, 255), 
(128, 0, 255), 
(255, 0, 255), 
(255, 0, 128)
)
 def list2drawing(raw_strokes, img_size):
img = np.zeros(( img_size, img_size, channel), np.uint8)
 coef =(img_size - 2 * lw - 2 * border)/(BASE_SIZE - 1)
 nb_stokes = len(raw_strokes)
 for t, stroke in enumerate(raw_strokes[::-1]):
rgb = shift_colors[(nb_stokes-t-1)%12]
for i in range(len(stroke[0])- 1):
p1 =(int(coef * stroke[0][i] + lw + border), int(coef * stroke[1][i] + lw+ border))
p2 =(int(coef * stroke[0][i + 1] + lw + border), int(coef * stroke[1][i + 1] + lw + border))
_ = cv2.line(img, p1, p2, rgb, lw, cv2.LINE_AA)
 if img_size != BASE_SIZE:
return cv2.resize(img,(img_size, img_size))
else:
return img
def draw_cv2(raw_strokes, img_size=256, lw=lw, time_color=True):

 img = np.zeros(( BASE_SIZE, BASE_SIZE), np.uint8)
 for t, stroke in enumerate(raw_strokes):
for i in range(len(stroke[0])- 1):
color = 255 - min(t, 10)* 13 if time_color else 255
_ = cv2.line(img,(stroke[0][i], stroke[1][i]), 
(stroke[0][i + 1], stroke[1][i + 1]), color, lw)
 if img_size != BASE_SIZE:
return cv2.resize(img,(img_size, img_size))
else:
return img
def image_generator_test(img_size, batch_size, model_name, lw=lw, time_color=True):
while True:
filename = os.path.join(INPUT_DIR, 'test_simplified.csv')
 for df in pd.read_csv(filename, chunksize=batch_size):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), img_size, img_size, channel))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, :, :, 0] = draw_cv2(raw_strokes, img_size=img_size, lw=lw,
time_color=time_color)
 if model_name == 'mobilenet':
x = tf.keras.applications.mobilenet.preprocess_input(x ).astype(np.float32)
 elif model_name =='xception':
x = tf.keras.applications.xception.preprocess_input(x ).astype(np.float32)
 elif model_name == 'densenet201':
x = tf.keras.applications.densenet.preprocess_input(x ).astype(np.float32)
 else:
x /= 255.
 x.astype(np.float32)
 yield x
def image_generator_xd(img_size, batch_size, ks, model_name, lw=lw, time_color=True):
while True:
for k in np.random.permutation(ks):
filename = os.path.join(DATA_DIR, 'train_k{}.csv.gz'.format(k))
for df in pd.read_csv(filename, chunksize=batch_size):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), img_size, img_size, channel))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, :, :, 0] = draw_cv2(raw_strokes, img_size=img_size, lw=lw,
time_color=time_color)
 if model_name == 'mobilenet':
x = tf.keras.applications.mobilenet.preprocess_input(x ).astype(np.float32)
 elif model_name =='xception':
x = tf.keras.applications.xception.preprocess_input(x ).astype(np.float32)
 elif model_name == 'densenet201':
x = tf.keras.applications.densenet.preprocess_input(x ).astype(np.float32)
 else:
x /= 255.
 x.astype(np.float32)
 y = tf.keras.utils.to_categorical(df.y, num_classes=num_classes)
 yield x, y
def df_to_image_array_xd(df, img_size, model_name, lw=lw, time_color=True):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), img_size, img_size, channel))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, :, :, 0] = draw_cv2(raw_strokes, img_size=img_size, lw=lw, time_color=time_color)
 if model_name == 'mobilenet':
x = tf.keras.applications.mobilenet.preprocess_input(x ).astype(np.float32)
 elif model_name =='xception':
x = tf.keras.applications.xception.preprocess_input(x ).astype(np.float32)
 elif model_name == 'densenet201':
x = tf.keras.applications.densenet.preprocess_input(x ).astype(np.float32)
 else:
x /= 255.
 x.astype(np.float32)
 return x
class StanfordCNN(tf.keras.layers.Layer):

 def __init__(self, filters, kernel_size, pool_size, num_classes):
super(StanfordCNN, self ).__init__()
self.conv1 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')
 self.conv2 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')
 self.conv3 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')
 self.maxpool = tf.keras.layers.MaxPool2D(pool_size)
 self.fcn = FCN([700,500,400], num_classes)
 def call(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = self.conv3(x)
 x = self.maxpool(x)
 x = self.fcn(x)
 return x
class FCN(tf.keras.layers.Layer):
def __init__(self, num_unit, num_classes):
super(FCN, self ).__init__()
self.layers = []
num_layer = len(num_unit)
 self.flatten = tf.keras.layers.Flatten()
for i in range(num_layer):
W = tf.keras.layers.Dense(num_unit[i], activation='relu')
 self.layers.append(W)
 self.W_o = tf.keras.layers.Dense(num_classes, activation='softmax')
 def call(self, x):
x = self.flatten(x)
 for layer in self.layers:
x = layer(x)
 return self.W_o(x)
 def create_model(model_name, input_shape, num_gpu, learning_rate):
if model_name.lower() =='mobilenet':
model = MobileNet(input_shape=input_shape, weights=None, classes=num_classes)
 elif model_name.lower() == 'xception':
xcep = Xception(input_shape=input_shape, weights=None, include_top=False)
 fcn = FCN([512, 512], num_classes)
 output = fcn(xcep.output)
 model = tf.keras.Model(inputs=xcep.input, outputs=output)
 elif model_name.lower() =='densenet201':
dense = DenseNet201(input_shape=input_shape, weights='imagenet', include_top=False)
 x = dense.output
x = tf.keras.layers.GlobalAveragePooling2D()(x)
 fcn = FCN([1024, 512], num_classes)
 output = fcn(dense.output)
 model = tf.keras.Model(inputs=dense.input, outputs=output)
 elif model_name.lower() == 'resnet50':
resnet = tf.keras.applications.ResNet50(input_shape=input_shape, weights=None, include_top=False, pooling='avg')
 fcn = FCN([512, 512], num_classes)
 output = fcn(resnet.output)
 model = tf.keras.Model(inputs=resnet.input, outputs=output)
 elif model_name.lower() == 'base':
stanCNN = StanfordCNN(5,(3,3),(2,2), num_classes)
 input_img = tf.keras.layers.Input(shape=input_shape)
 output = stanCNN(input_img)
 model = tf.keras.Model(inputs=input_img, outputs=output)
 elif model_name.lower() == 'inception':
incep = tf.keras.applications.inception_resnet_v2.InceptionResNetV2(input_shape=input_shape, weights=None, include_top=False, pooling='avg')
 fcn = FCN([512], num_classes)
 output = fcn(incep.output)
 model = tf.keras.Model(inputs=incep.input, outputs=output)
 if num_gpu != 1:
model = tf.keras.utils.multi_gpu_model(model, gpus=num_gpu)
 model.compile(optimizer=Adam(lr=learning_rate), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary())
 return model
def test(args: Namespace, model=None):
img_size = args.img_size
input_shape =(img_size, img_size, channel)
 model_name = args.model_name
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test_datagen = image_generator_test(img_size=img_size, batch_size=128, model_name=model_name)
 if model == None:
print('Loading saved model')
 model = create_model(model_name, input_shape, args.num_gpu, args.learning_rate)
 model.load_weights('model.h5')
 test_predictions = model.predict_generator(test_datagen, verbose=1, steps=len(test)//128)
 top3 = preds2catids(test_predictions)
 cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
now_time = dt.datetime.now().strftime("%m%d%H%M")
 submission.to_csv('my_submission_{}.csv'.format(now_time), index=False)
 print('submission shape:', submission.shape)
 def train(args: Namespace):
start = dt.datetime.now()
num_gpu = args.num_gpu
epochs = args.epoch
batch_size = args.batch_size * num_gpu
if args.steps == -1:
steps_per_epoch =(49209839)// batch_size
else:
steps_per_epoch = args.steps
learning_rate = args.learning_rate
img_size = args.img_size
input_shape =(img_size, img_size, channel)
 model_name = args.model_name
model = create_model(model_name, input_shape, num_gpu, learning_rate)
 valid_df = pd.read_csv(os.path.join(DATA_DIR, 'train_k{}.csv.gz'.format(num_csvs - 1)) , nrows=3000)
 x_valid = df_to_image_array_xd(valid_df, img_size, model_name)
 y_valid = tf.keras.utils.to_categorical(valid_df.y, num_classes=num_classes)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 2.**30))
train_datagen = image_generator_xd(img_size=img_size, batch_size=batch_size, ks=range(num_csvs - 1), model_name=model_name)
 callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
model.fit_generator(
 train_datagen, steps_per_epoch=steps_per_epoch, epochs=epochs, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))
end = dt.datetime.now()
print('Total time {}s'.format(( end - start ).seconds))
return model
class Args() :

 def __init__(self):
self.mode = 'train'
self.model_name = 'densenet201'
self.img_size = 75
self.batch_size = 256
self.epoch = 15
self.steps = 500
self.learning_rate = 0.002
self.num_gpu = 1
def main() :


args = Args()
model = train(args)
 test(args, model)
 if __name__=='__main__':
main() 
<import_modules>
data.isna().sum() [lambda x: x>0]
import tensorflow as tf
import keras
from keras.layers import Conv2D, GlobalAveragePooling2D, MaxPooling2D, GlobalAveragePooling1D, Conv1D, MaxPooling1D
from keras.layers import Dense, Dropout, Flatten, Activation, Input, BatchNormalization
from keras.layers import CuDNNLSTM as LSTM
from keras.layers.merge import concatenate
from keras.metrics import categorical_crossentropy, top_k_categorical_accuracy, categorical_accuracy
from keras.models import Sequential, Model
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint,TensorBoard, LearningRateScheduler
from keras.optimizers import Adam,Adagrad,RMSprop,SGD
from keras.applications import Xception, MobileNet, MobileNetV2, xception, mobilenet
from keras.utils.np_utils import to_categorical
from keras.preprocessing.sequence import pad_sequences
from sklearn.preprocessing import LabelEncoder
from keras.metrics import top_k_categorical_accuracy
from tqdm import tqdm_notebook as tqdm
from ast import literal_eval
import glob
from multiprocessing import Pool
from functools import partial
from itertools import repeat
from itertools import product<define_variables>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
SH_DIR = '.. /input/shufflecsvs/shuffle-csvs/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'<define_variables>
data.dtypes.value_counts()
SEED = 42
np.random.seed(seed=SEED)
 tf.set_random_seed(seed=SEED )<compute_test_metric>
pd.set_option('display.max_columns', 100)
 data[data.select_dtypes(['int64'] ).columns].head()
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
data[['MSSubClass', 'MoSold','OverallQual','OverallCond','YrSold']] = data[['MSSubClass', 'MoSold','OverallQual','OverallCond','YrSold']].astype(str )
mobile_BASE_SIZE = 256
mobile_NCSVS = 100
mobile_NCATS = 340
mobile_STEPS = 800
mobile_EPOCHS = 70
mobile_SIZE = 64
mobile_BATCHSIZE = 128
lstm_batch_size = 128
lstm_STROKE_COUNT = 196
BATCHSIZE = 128<load_from_csv>
data[['LotFrontage', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath',
'BsmtHalfBath', 'GarageYrBlt', 'GarageCars',
'GarageArea']] = data[['LotFrontage', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath',
'BsmtHalfBath', 'GarageYrBlt', 'GarageCars', 'GarageArea']].astype(int)
def image_generator_xd(size, batchsize, ks, lw=6, time_color=True):
while True:
for k in np.random.permutation(ks):
filename = os.path.join(SH_DIR, 'train_k{}.csv.gz'.format(k))
for df in pd.read_csv(filename, chunksize=batchsize):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), size, size, 1))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw,
time_color=time_color)
 x = mobilenet.preprocess_input(x ).astype(np.float32)
 y = keras.utils.to_categorical(df.y, num_classes=NCATS)
 yield x, y<normalization>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
def df_to_image_array_xd(df, size, lw=6, time_color=True):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), size, size, 1))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)
 x = mobilenet.preprocess_input(x ).astype(np.float32)
 return x<feature_engineering>
env_cols =['MSZoning','Street','Alley','Neighborhood','Condition1','Condition2']
def df_to_sequence_array(df, time_size):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), time_size, 3))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, : ,: ] = _stack_it(raw_strokes, time_size)
 return x<categorify>
mean_encoder(data,['MSZoning','Street','Alley','Neighborhood'])
def _stack_it(raw_strokes, time_size):

 in_strokes = [(xi,yi,i)
 for i,(x,y)in enumerate(raw_strokes)
 for xi,yi in zip(x,y)]
c_strokes = np.stack(in_strokes)
 c_strokes[:,2] = [1]+np.diff(c_strokes[:,2] ).tolist()
c_strokes[:,2] += 1
return pad_sequences(c_strokes.swapaxes(0, 1),
maxlen=time_size,
padding='post' ).swapaxes(0, 1 )<load_from_csv>
data['ConditionFeedr'] =(data.Condition2=="Feedr")*1+(data.Condition1=="Feedr")*1
data['ConditionArtery'] =(data.Condition2=="Artery")*1+(data.Condition1=="Artery")*1
data['ConditionPosA'] =(data.Condition2=="PosA")*1+(data.Condition1=="PosA")*1
data['ConditionPosN'] =(data.Condition2=="PosN")*1+(data.Condition1=="PosN")*1
data['ConditionRRAn'] =(data.Condition2=="RRAn")*1+(data.Condition1=="RRAn")*1
data['ConditionRRAe'] =(data.Condition2=="RRAe")*1+(data.Condition1=="RRAe")*1
data['ConditionRRNn'] =(data.Condition2=="RRNn")*1+(data.Condition1=="RRNn")*1
data['ConditionRRNe'] =(data.Condition2=="RRNe")*1+(data.Condition1=="RRNe")*1
data = data.drop(['Condition1','Condition2'],axis=1 )
def read_batch(samples=5,
start_row=0,
max_rows = 1000):

 out_df_list = []
for c_path in ALL_TRAIN_PATHS:
c_df = pd.read_csv(c_path, nrows=max_rows, skiprows=start_row)
 c_df.columns=COL_NAMES
out_df_list += [c_df.sample(samples)[['drawing', 'word']]]
full_df = pd.concat(out_df_list)
 full_df['drawing'] = full_df['drawing'].\
map(_stack_it)
 return full_df<load_from_csv>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
def image_generator_xd(size,time_size, batchsize, ks, lw=6, time_color=True):
while True:
for k in np.random.permutation(ks):
filename = os.path.join(SH_DIR, 'train_k{}.csv.gz'.format(k))
for df in pd.read_csv(filename, chunksize=batchsize):
df['drawing'] = df['drawing'].apply(json.loads)
 x = np.zeros(( len(df), size, size, 1))
x_lstm = np.zeros(( len(df), time_size, 3))
for i, raw_strokes in enumerate(df.drawing.values):
x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw,
time_color=time_color)
 x_lstm[i, :, :] = _stack_it(raw_strokes, time_size)
 x = mobilenet.preprocess_input(x ).astype(np.float32)
 y = keras.utils.to_categorical(df.y, num_classes=mobile_NCATS)
 yield [x, x_lstm], y<prepare_x_and_y>
lot_cols =['LotFrontage','LotArea','LotShape','LandContour','LotConfig','LandSlope','Fence','Fireplaces','FireplaceQu','GarageType','GarageYrBlt','GarageFinish','GarageCars','GarageArea','GarageQual','GarageCond','PoolArea','PoolQC']
train_datagen = image_generator_xd(size = mobile_SIZE, time_size=lstm_STROKE_COUNT, batchsize=mobile_BASE_SIZE, ks= range(mobile_NCSVS-1))
x, y =next(train_datagen )<load_from_csv>
data['LotShape'].dtype=='object'
valid_df = pd.read_csv(os.path.join(SH_DIR, 'train_k{}.csv.gz'.format(mobile_NCSVS-1)) , nrows = 34000)
<split>
mean_encoder(data,['LotShape','LandContour','LotConfig','LandSlope','Fence','FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond'])
iterable_1 = product(np.array_split(valid_df, 100), [mobile_SIZE])
 iterable_2 = product(np.array_split(valid_df, 100), [lstm_STROKE_COUNT] )<prepare_x_and_y>
data.PoolQC.isna().sum()
with Pool(processes=NJOBS)as p:
x_valid = p.starmap(df_to_image_array_xd, iterable_1)
 with Pool(processes=NJOBS)as p:
x_lstm_valid = p.starmap(df_to_sequence_array, iterable_2 )<concatenate>
data['Pool'] =(data.PoolQC!="NA")*1
x_valid = np.vstack(np.array(x_valid))
x_lstm_valid = np.vstack(np.array(x_lstm_valid))
y_valid = keras.utils.to_categorical(valid_df.y, num_classes=mobile_NCATS)
 print(
 'mobile_valid size :{} \
RNN valid size : {} \
y_valid size : {}'.format(x_valid.shape, x_lstm_valid.shape, y_valid.shape))<choose_model_class>
data = data.drop(['PoolQC'],axis=1 )
def mobiel_net(size):
base_model = MobileNet(weights = None, input_shape=(size, size,1), include_top=False ,alpha=1.)
x= base_model.output
x= GlobalAveragePooling2D()(x)
 return x, base_model.input<choose_model_class>
data[data.Pool==0].PoolArea.value_counts()
def lstm_model(time_size ,f_size):
stroke_input = Input(shape=(time_size, f_size), name = 'lstm_input')
 batch_norm = BatchNormalization()(stroke_input)
 conv1 = Conv1D(48,(5,), activation='relu' )(batch_norm)
 conv2 = Conv1D(64,(5,), activation='relu' )(conv1)
 conv3 = Conv1D(96,(3,), activation='relu' )(conv2)

max_pool = MaxPooling1D(pool_size = 4 )(conv3)

lstm1 = LSTM(128, return_sequences=True )(max_pool)
 lstm2 = LSTM(128, return_sequences=False )(lstm1)

batch_norm_2 = BatchNormalization()(lstm2)

return batch_norm_2, stroke_input<choose_model_class>
data.loc[(data.Pool==0)&(data.PoolArea>0),'Pool']=1
mobile_model, mobile_input = mobiel_net(mobile_SIZE)
 sequence_model, sequence_input = lstm_model(lstm_STROKE_COUNT, 3 )<concatenate>
data[data.LotFrontage>300]
merge = concatenate([mobile_model, sequence_model] )<choose_model_class>
data = data[data.LotFrontage<300]
hidden_1 = Dense(512, activation='relu' )(merge)
 merge_batch_norm = BatchNormalization()(hidden_1)

output = Dense(mobile_NCATS, activation = 'softmax' )(merge_batch_norm)
 merge_model = Model(inputs = [mobile_input, sequence_input ], output = output, name = 'merge_model_fin_70' )<load_pretrained>
data['LotFrontage']=np.power(data.LotFrontage,0.25 )
def compile_and_train(model, num_epochs, BATCHSIZE, OPTIMIZER):
model.compile(optimizer=OPTIMIZER(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 directory_path = '.. /working/weights/{}/'.format(model.name)
 log_path = '.. /working/logs/{}/'.format(model.name)
 directory_check(directory_path)
 directory_check(log_path)
 file_name = model.name + '.{epoch:02d}-{loss:.2f}.h5'
filepath = os.path.join(directory_path, file_name)
 callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor = 0.75, patience=3, min_delta=0.001, mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint(filepath, monitor='val_top_3_accuracy', mode= 'max', save_best_only=True, save_weights_only=True),
TensorBoard(log_dir=log_path, histogram_freq=0, batch_size=BATCHSIZE)
 ]
hist = model.fit_generator(
 train_datagen, steps_per_epoch = mobile_STEPS, epochs = num_epochs, verbose = 1,
validation_data =([x_valid,x_lstm_valid], y_valid),
callbacks = callbacks 
)
 weight_files = glob.glob('{}*'.format(directory_path))
weight_file = max(weight_files, key = os.path.getctime)
 return hist, weight_file<train_model>
data = data[data.LotArea<100000]
model_his, model_weight = compile_and_train(merge_model, 60 , BATCHSIZE, Adam )<concatenate>
data['LotArea'] = np.log1p(data.LotArea )
history_df = pd.concat([pd.DataFrame(model_his.history)],sort = True )<predict_on_test>
data.groupby('Fireplaces')['tmpPrice'].mean()
valid_predictions = merge_model.predict([x_valid, x_lstm_valid], batch_size=BATCHSIZE, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('MAP3 : {:.3f}'.format(map3))<load_from_csv>
data.groupby('GarageCars')['tmpPrice'].mean()
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()<train_model>
data.GarageCars.value_counts()
x_test = df_to_image_array_xd(test.copy() , mobile_SIZE)
 x_lstm_test = df_to_sequence_array(test.copy() , lstm_STROKE_COUNT)
 print(test.shape, x_test.shape)
 print(test.shape, x_lstm_test.shape )<predict_on_test>
data['GarageCars']=data.GarageCars.replace([4,5],3 )
test_predictions = merge_model.predict([x_test,x_lstm_test], batch_size=BATCHSIZE, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()<categorify>
data[data.GarageYrBlt>2010]
cats = list_all_categories()
id2cat = {k : cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()<save_to_csv>
data.loc[data.GarageYrBlt>2010,'GarageYrBlt']=2007
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('{}-submission-fin.csv'.format(merge_model.name), index=False )<set_options>
data['GarageYrBlt']=np.log1p(2011-data.GarageYrBlt )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<define_variables>
data.groupby('GarageCars')['tmpPrice'].mean()
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
data = data.drop(['PoolArea'],axis=1 )
def apk(actual, predicted, k=3):
if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):
return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680<choose_model_class>
data.isna().sum() [lambda x: x>0]
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )<load_from_csv>
const_cols = ['MSSubClass','BldgType','HouseStyle','OverallQual','OverallCond','YearBuilt','YearRemodAdd','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','MasVnrArea','ExterQual','ExterCond','Foundation','WoodDeckSF','PavedDrive']
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<drop_column>
data['MSSubClass']=data.MSSubClass.replace('150','120' ).astype(object )
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<train_on_grid>
adaptative_correlation(data.OverallQual,data.tmpPrice )
callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=70, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<predict_on_test>
data['OverallQual'].replace([1, 2], 3, inplace=True)
 data['OverallQual'].replace([10], 9, inplace=True )
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<load_from_csv>
data['OverallQual'] = data.OverallQual.astype('int' )
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
data = data.drop(['RoofMatl'],axis=1 )
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
data.OverallCond.value_counts()
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('submission.csv', index=False)
 submission.head()
submission.shape<train_model>
mean_encoder(data,['MSSubClass','BldgType','HouseStyle','OverallCond','RoofStyle','Exterior1st','Exterior2nd','MasVnrType','ExterQual','ExterCond','Foundation','PavedDrive'])
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<set_options>
data[data.YearBuilt>data.YearRemodAdd]
%matplotlib inline
device = torch.device('cuda' if torch.cuda.is_available() else "cpu" )<define_variables>
data.loc[data.YearBuilt>data.YearRemodAdd,'YearRemodAdd']=2002
NROWS = 10000
VAL_NROWS = 500
path = '.. /input/train_simplified/'
filenames = glob.glob(os.path.join(path, '*.csv'))
NUM_CLASSES = len(filenames )<feature_engineering>
data['YearBuilt'] = np.power(np.abs(85-(2010 - data.YearBuilt)) ,2 )
encode_dict = {}
path = '.. /input/train_simplified/'
filenames = glob.glob(os.path.join(path, '*.csv'))
filenames = sorted(filenames)
 print(len(filenames))
for ix, filename in enumerate(filenames):
class_name = filename.split('/')[-1].split('.')[0].replace(' ', '_')
 encode_dict[class_name] = ix
decode_dict = {value:key for key, value in encode_dict.items() }<load_from_csv>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
class ClassDataset(Dataset):
def __init__(self, csv_file_path, mode='train', nrows=15000, skiprows=0, size=128):
super().__init__()
self.df = pd.read_csv(csv_file_path, usecols=['drawing'], nrows=nrows, skiprows=0)
 self.mode = mode
self.size = size
if self.mode == 'train':
self.class_name = csv_file_path.split('/')[-1].split('.')[0].replace(' ', '_')
 self.label = encode_dict[self.class_name]
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
raw_strokes = ast.literal_eval(self.df.drawing[index])
 image = draw_cv2(raw_strokes, size=self.size)
 image = image.transpose(2, 0, 1)
 if self.mode == 'train':
return(image/255 ).astype('float32'), self.label
else:
return(image/255 ).astype('float32')
<create_dataframe>
basement_cols = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath']
%%time
val_dset = ConcatDataset([ClassDataset(filename, nrows=VAL_NROWS, skiprows=0)for filename in filenames])
 dset = ConcatDataset([ClassDataset(filename, nrows=NROWS, skiprows=VAL_NROWS)for filename in filenames] )<load_pretrained>
adaptative_correlation(data.BsmtFinSF2,data.tmpPrice )
%%time
dloader = DataLoader(dset, batch_size=128, shuffle=True, num_workers=2)
 val_dloader = DataLoader(val_dset, batch_size=128, num_workers=2, shuffle=False )<choose_model_class>
data = data.drop(['BsmtFinSF2'],axis=1 )
class Conv(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):
super().__init__()
self.layers = nn.Sequential(
 nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, stride=stride, padding=padding, bias=False),
nn.BatchNorm2d(out_channel),
nn.ReLU(inplace=True)
)
 def forward(self, input):
return self.layers(input)
 class Conv_dw_Conv(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):
super().__init__()
self.layers = nn.Sequential(
 nn.Conv2d(in_channel, in_channel, kernel_size, stride, padding, bias=False, groups=in_channel),
nn.BatchNorm2d(in_channel),
nn.ReLU(inplace=True),
Conv(in_channel, out_channel, kernel_size=1, stride=1, padding=0)
)
 def forward(self, input):
return self.layers(input)
 class MobileNet(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.num_classes = num_classes
self.model = nn.Sequential(
 Conv(3, 32, stride=2),
Conv_dw_Conv(32, 64, kernel_size=3, stride=1),
Conv_dw_Conv(64, 128, kernel_size=3, stride=2),
Conv_dw_Conv(128, 128, kernel_size=3, stride=1),
Conv_dw_Conv(128, 256, kernel_size=3, stride=2),
Conv_dw_Conv(256, 256, kernel_size=3, stride=1),
Conv_dw_Conv(256, 512, kernel_size=3, stride=2),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 1024, kernel_size=3, stride=2),
Conv_dw_Conv(1024, 1024, kernel_size=3, stride=1)
)
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(1024, num_classes)
 def forward(self, input):
x = self.model(input)
 x = self.avg_pool(x)
 x = x.view(-1, 1024)
 out = self.fc(x)
 return out<compute_test_metric>
mean_encoder(data,['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2'])
def apk(actual, predicted, k=3):

 if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
for i,p in enumerate(predicted):
if p == actual:
score = 1 /(i+1.0)
 return score
def mapk(actual, predicted, k=3):

 return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)] )<train_model>
for idx, feat in enumerate(['OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch']):
print(feat," : ",adaptative_correlation(data[feat],data.tmpPrice))
def validation_score(model, val_data_loader):
model.eval()
sum_score = 0
count = 0
for images, labels in val_data_loader:
images = images.to(device)
 labels = labels.to(device)
 batch_size = images.size(0)
 output = model(images)
 topk = output.detach().topk(3, dim=1)[1]
sum_score += mapk(labels.cpu().numpy() , topk.cpu().numpy())* batch_size
count += batch_size
model.train()
return sum_score / count 
<choose_model_class>
floors_cols = ['1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','KitchenQual','TotRmsAbvGrd']
model = MobileNet(NUM_CLASSES ).to(device)
 criterion = nn.CrossEntropyLoss().to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.002)
 scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[60000, 130000, 160000, 190000], gamma=0.5 )<train_model>
adaptative_correlation(data.LowQualFinSF,data.tmpPrice )
%%time
print("Start training...")
 epochs = 8
print_every = 1000
validate_every = 10000
model.train()
best_val_score = 0
curr_iter = 0
avg_loss = 0
avg_score = 0
for epoch in range(epochs):
del dset
del dloader
dset = ConcatDataset([ClassDataset(filename, nrows=NROWS, skiprows=VAL_NROWS + NROWS * epoch)for filename in filenames])
 dloader = DataLoader(dset, batch_size=128, shuffle=True, num_workers=2)
 for ix,(images, labels)in enumerate(dloader):
images = images.to(device)
 labels = labels.to(device)
 model.zero_grad()
output = model(images)
 loss = criterion(output, labels)
 loss.backward()
optimizer.step()
avg_loss += loss.item()
topk = output.detach().topk(3, dim=1)[1]
avg_score += mapk(labels.cpu().numpy() , topk.cpu().numpy())
 curr_iter += 1
if(curr_iter)% print_every == 0:
print('Epoch {}, Iteration {} - Train Loss: {:.4f}, MAP@3: {:.3f}'.format(epoch + 1, curr_iter, avg_loss/print_every, avg_score/print_every))
avg_loss = 0
avg_score = 0
if curr_iter % validate_every == 0:
val_score = validation_score(model, val_dloader)
 print('Epoch {}, Iteration {}: validation map@3: {}'.format(epoch + 1, curr_iter, val_score))
if val_score > best_val_score:
print('New best validation score: {}, saving model...'.format(val_score))
best_val_score = val_score
torch.save(model.state_dict() , 'model_checkpoint_best_val.ckpt')
<load_from_csv>
data = data.drop(['LowQualFinSF'],axis=1 )
test_dset = ClassDataset('.. /input/test_simplified.csv', mode='test', nrows=None)
 test_dloader = DataLoader(test_dset, batch_size=128, shuffle=False, num_workers=0)
 model.eval()
labels = []
for images in tqdm.tqdm(test_dloader):
images = images.to(device)
 output = model(images)
 _, pred = output.topk(3, 1)
 for i in range(len(images)) :
labels.append(' '.join([decode_dict[pred[i][j].item() ]for j in range(3)])) 
<load_from_csv>
mean_encoder(data,['KitchenQual'])
%%time
submission = pd.read_csv('.. /input/test_simplified.csv', index_col='key_id' ,usecols=['key_id'])
 print(len(submission))
submission['word'] = labels 
<save_to_csv>
data['LivingArea'] = data['TotalBsmtSF'] + data['1stFlrSF']+\
data['2ndFlrSF']+\
data['OpenPorchSF']+\
data['3SsnPorch']
data['LivingArea'] = np.power(data['LivingArea'],0.35 )
submission.to_csv('submission_final.csv' )<save_to_csv>
HalfBathfactor = 0.6
data['TotalBath'] = data.BsmtFullBath + HalfBathfactor * data.BsmtHalfBath +\
data.FullBath + HalfBathfactor * data.HalfBath
model.load_state_dict(torch.load('model_checkpoint_best_val.ckpt'))
model.eval()
labels = []
for images in tqdm.tqdm(test_dloader):
images = images.to(device)
 output = model(images)
 _, pred = output.topk(3, 1)
 for i in range(len(images)) :
labels.append(' '.join([decode_dict[pred[i][j].item() ]for j in range(3)]))
print(len(labels))
submission = pd.read_csv('.. /input/test_simplified.csv', index_col='key_id' ,usecols=['key_id'])
 print(len(submission))
submission['word'] = labels
submission.to_csv('submission_best_val.csv' )<set_options>
data['Rooms_Area'] =(data.TotRmsAbvGrd+data.TotalBath)/ data.GrLivArea
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<define_variables>
for feature in ['GrLivArea','BsmtFinSF1','1stFlrSF','2ndFlrSF','BsmtUnfSF','TotalBsmtSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch']:
data[feature] = np.power(data[feature],0.35 )
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
comfort_cols=['Heating','HeatingQC','CentralAir','Electrical','Functional','MiscFeature','MiscVal']
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
mean_encoder(data,['Heating','HeatingQC','CentralAir','Electrical','Functional','MiscFeature'])
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680<choose_model_class>
data['MiscVal'] = np.log1p(data.MiscVal )
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )<load_from_csv>
sale_cols = ['MoSold','YrSold','SaleType','SaleCondition']
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<drop_column>
mean_encoder(data,['MoSold','YrSold','SaleType','SaleCondition'])
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<prepare_x_and_y>
quick_predict(data.loc['train'].drop(['SalePrice','tmpPrice'],axis=1),data.loc['train'].tmpPrice )
%%timeit
x, y = next(train_datagen )<train_on_grid>
train_lenght = len(data.loc['train'] )
callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=70, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<predict_on_test>
y = data.tmpPrice.loc['train']
y_target = data.tmpPrice.loc['test']
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<load_from_csv>
X = data.drop(['SalePrice', 'tmpPrice'],axis=1)
 X = pd.get_dummies(X)
 X = RobustScaler().fit_transform(X)
 X_target = X[train_lenght:,:]
X = X[:train_lenght,:]
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0 )
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score,cross_val_predict, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X)
 rmse= np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<import_modules>
model1 = linear_model.Ridge(alpha=30)
 score = rmsle_cv(model1)
 print("model score: {:.4f} standard deviation({:.4f})
 ".format(score.mean() , score.std()))