Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
warnings.filterwarnings("ignore")
 %matplotlib inline
train['WindSpeed'].value_counts()<feature_engineering>
pd.options.display.max_columns = None
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )<data_type_conversions>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
def str_to_float(txt):
try:
return float(txt)
 except:
return -1<feature_engineering>
test_ID = test['Id']
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )<count_values>
train_missing = train.isnull().sum().sort_values(ascending=False)
 train_missing.head(20 )
train['WindDirection'].value_counts()<categorify>
train['MiscFeature'].fillna('None', inplace=True)
 train['Fence'].fillna('No Fence', inplace=True)
 train['PoolQC'].fillna('No Pool', inplace=True)
 train['Alley'].fillna('No alley access', inplace=True)
 train['FireplaceQu'].fillna('No Fireplace', inplace=True)
 train['LotFrontage'].fillna(train['LotFrontage'].median() , inplace=True)
 train['GarageCond'].fillna('No Garage', inplace=True)
 train['GarageType'].fillna('No Garage', inplace=True)
 train['GarageYrBlt'].fillna(round(train['GarageYrBlt'].median() , 1), inplace=True)
 train['GarageFinish'].fillna('No Garage', inplace=True)
 train['GarageQual'].fillna('No Garage', inplace=True)
 train['BsmtExposure'].fillna('No Basement', inplace=True)
 train['BsmtFinType2'].fillna('No Basement', inplace=True)
 train['BsmtFinType1'].fillna('No Basement', inplace=True)
 train['BsmtCond'].fillna('No Basement', inplace=True)
 train['BsmtQual'].fillna('No Basement', inplace=True)
 train['MasVnrArea'].fillna(0.0, inplace=True)
 train['MasVnrType'].fillna('None', inplace=True)
 train['Electrical'].fillna('Mixed', inplace=True )
def clean_WindDirection(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt<feature_engineering>
best_columns = ['OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea',\
'TotalBsmtSF', '1stFlrSF', 'FullBath', 'TotRmsAbvGrd', \
'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'Fireplaces', \
'BsmtFinSF1', 'LotFrontage', 'WoodDeckSF', '2ndFlrSF', 'OpenPorchSF']
train['WindDirection'] = train['WindDirection'].apply(clean_WindDirection )<count_values>
test_missing = test.isnull().sum().sort_values(ascending=False)
 test_missing.head(35 )
train['WindDirection'].value_counts()<categorify>
test['MiscFeature'].fillna('None', inplace=True)
 test['Fence'].fillna('No Fence', inplace=True)
 test['PoolQC'].fillna('No Pool', inplace=True)
 test['Alley'].fillna('No alley access', inplace=True)
 test['FireplaceQu'].fillna('No Fireplace', inplace=True)
 test['LotFrontage'].fillna(train['LotFrontage'].median() , inplace=True)
 test['GarageCond'].fillna('No Garage', inplace=True)
 test['GarageType'].fillna('No Garage', inplace=True)
 test['GarageYrBlt'].fillna(round(test['GarageYrBlt'].median() , 1), inplace=True)
 test['GarageFinish'].fillna('No Garage', inplace=True)
 test['GarageQual'].fillna('No Garage', inplace=True)
 test['BsmtExposure'].fillna('No Basement', inplace=True)
 test['BsmtFinType2'].fillna('No Basement', inplace=True)
 test['BsmtFinType1'].fillna('No Basement', inplace=True)
 test['BsmtCond'].fillna('No Basement', inplace=True)
 test['BsmtQual'].fillna('No Basement', inplace=True)
 test['MasVnrArea'].fillna(0.0, inplace=True)
 test['MasVnrType'].fillna('None', inplace=True)
 test['MSZoning'].fillna('RL', inplace=True)
 test['Utilities'].fillna('AllPub', inplace=True)
 test['Functional'].fillna('Typ', inplace=True)
 test['BsmtFullBath'].fillna(0.0, inplace=True)
 test['BsmtHalfBath'].fillna(0.0, inplace=True)
 test['BsmtFinSF2'].fillna(0.0, inplace=True)
 test['BsmtUnfSF'].fillna(0.0, inplace=True)
 test['TotalBsmtSF'].fillna(test['TotalBsmtSF'].median() , inplace=True)
 test['SaleType'].fillna('WD', inplace=True)
 test['BsmtFinSF1'].fillna(0.0, inplace=True)
 test['GarageCars'].fillna(2.0, inplace=True)
 test['GarageArea'].fillna(0.0, inplace=True)
 test['KitchenQual'].fillna('TA', inplace=True)
 test['Exterior1st'].fillna('VinylSd', inplace=True)
 test['Exterior2nd'].fillna('VinylSd', inplace=True )
def transform_WindDirection(txt):
if pd.isna(txt):
return np.nan
if txt=='n':
return 0
if txt=='nne' or txt=='nen':
return 1/8
if txt=='ne':
return 2/8
if txt=='ene' or txt=='nee':
return 3/8
if txt=='e':
return 4/8
if txt=='ese' or txt=='see':
return 5/8
if txt=='se':
return 6/8
if txt=='ses' or txt=='sse':
return 7/8
if txt=='s':
return 8/8
if txt=='ssw' or txt=='sws':
return 9/8
if txt=='sw':
return 10/8
if txt=='sww' or txt=='wsw':
return 11/8
if txt=='w':
return 12/8
if txt=='wnw' or txt=='nww':
return 13/8
if txt=='nw':
return 14/8
if txt=='nwn' or txt=='nnw':
return 15/8
return np.nan<feature_engineering>
train_test = pd.concat([train,test], axis=0, sort=False )
train['WindDirection'] = train['WindDirection'].apply(transform_WindDirection )<count_values>
train.drop(['Id', 'Utilities', 'Street', 'LowQualFinSF', 'PoolArea'], axis=1, inplace=True)
 test.drop(['Id', 'Utilities', 'Street', 'LowQualFinSF', 'PoolArea'], axis=1, inplace=True)
 train_test.drop(['Id', 'Utilities', 'Street', 'LowQualFinSF', 'PoolArea'], axis=1, inplace=True )
train['PlayDirection'].value_counts()<feature_engineering>
train['MSSubClass'] = train['MSSubClass'].apply(str)
 train['OverallCond'] = train['OverallCond'].astype(str)
 train['YrSold'] = train['YrSold'].astype(str)
 train['MoSold'] = train['MoSold'].astype(str)
 test['MSSubClass'] = test['MSSubClass'].apply(str)
 test['OverallCond'] = test['OverallCond'].astype(str)
 test['YrSold'] = test['YrSold'].astype(str)
 test['MoSold'] = test['MoSold'].astype(str)
 train_test['MSSubClass'] = train_test['MSSubClass'].apply(str)
 train_test['OverallCond'] = train_test['OverallCond'].astype(str)
 train_test['YrSold'] = train_test['YrSold'].astype(str)
 train_test['MoSold'] = train_test['MoSold'].astype(str)
train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x.strip() == 'right' )<feature_engineering>
numeric = ['LotFrontage',
'LotArea',
'YearBuilt',
'YearRemodAdd',
'MasVnrArea',
'BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'1stFlrSF',
'2ndFlrSF',
'GrLivArea',
'GarageArea',
'WoodDeckSF',
'OpenPorchSF',
'EnclosedPorch',
'3SsnPorch',
'ScreenPorch',
'MiscVal',
'GarageYrBlt']
train['Team'] = train['Team'].apply(lambda x: x.strip() =='home' )<feature_engineering>
categorical = set(columns)- set(numeric )
train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x )<count_values>
sns.set_style('whitegrid' )
weather_count = Counter()
for weather in train['GameWeather']:
if pd.isna(weather):
continue
for word in weather.split() :
weather_count[word]+=1
weather_count.most_common() [:15]<categorify>
train.drop(['PoolQC', 'Condition2'], axis=1, inplace=True)
 test.drop(['PoolQC', 'Condition2'], axis=1, inplace=True)
 train_test.drop(['PoolQC', 'Condition2'], axis=1, inplace=True )
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0<feature_engineering>
categorical.remove('PoolQC')
 categorical.remove('Condition2')
 categorical.remove('SalePrice' )
train['GameWeather'] = train['GameWeather'].apply(map_weather )<feature_engineering>
LE = LabelEncoder()
train['IsRusher'] = train['NflId'] == train['NflIdRusher']<drop_column>
for col in categorical:
train[col] = LE.fit_transform(train[col] )
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )<feature_engineering>
rf_clf = RandomForestRegressor(random_state=42, n_estimators=500, max_depth=8, criterion='mse' )
train['X'] = train.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1 )<feature_engineering>
%time rf_clf.fit(train.drop(['SalePrice'], axis=1), train['SalePrice'] )
def new_orientation(angle, play_direction):
if play_direction == 0:
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
train['Orientation'] = train.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 train['Dir'] = train.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1 )<feature_engineering>
%time cross_val_score(rf_clf, train.drop(['SalePrice'], axis=1), train['SalePrice'], cv=5 )
train['YardsLeft'] = train.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 train['YardsLeft'] = train.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1 )<filter>
columns_to_models = list(columns_to_models )
((train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])).mean()<drop_column>
columns_to_models.append('SalePrice' )
train.drop(train.index[(train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])], inplace=True )<sort_values>
train_test = train_test[columns_to_models]
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()<drop_column>
scaler = StandardScaler()
train.drop(['GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1, inplace=True )<drop_column>
new_numeric = []
for col in numeric:
if col in columns_to_models:
new_numeric.append(col )
cat_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(col)
 train = train.drop(cat_features, axis=1 )<correct_missing_values>
new_cat = []
for col in categorical:
if col in columns_to_models:
new_cat.append(col )
train.fillna(-999, inplace=True )<prepare_x_and_y>
train_test[new_numeric] = scaler.fit_transform(train_test[new_numeric] )
X_train = np.array(train[players_col] ).reshape(-1, len(players_col)*22 )<prepare_x_and_y>
for col in new_cat:
if col != 'SalePrice':
train_test[col] = LE.fit_transform(train_test[col] )
play_col = train.drop(players_col+['Yards'], axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = train[col][::22]<prepare_x_and_y>
train_test_lin = train_test.copy()
X_train = np.concatenate([X_train, X_play_col], axis=1)
 y_train = np.zeros(shape=(X_train.shape[0], 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))<normalization>
train_test_lin = pd.concat([train_test_lin,
pd.get_dummies(train_test_lin['BedroomAbvGr'], prefix='BedroomAbvGr', drop_first=True),
pd.get_dummies(train_test_lin['GarageFinish'], prefix='GarageFinish', drop_first=True),
pd.get_dummies(train_test_lin['BsmtQual'], prefix='BsmtQual', drop_first=True),
pd.get_dummies(train_test_lin['HalfBath'], prefix='HalfBath', drop_first=True),
pd.get_dummies(train_test_lin['MoSold'], prefix='MoSold', drop_first=True),
pd.get_dummies(train_test_lin['SaleCondition'], prefix='SaleCondition', drop_first=True),
pd.get_dummies(train_test_lin['BsmtFinType1'], prefix='BsmtFinType1', drop_first=True),
pd.get_dummies(train_test_lin['KitchenQual'], prefix='KitchenQual', drop_first=True),
pd.get_dummies(train_test_lin['OverallQual'], prefix='OverallQual', drop_first=True),
pd.get_dummies(train_test_lin['LandContour'], prefix='LandContour', drop_first=True),
pd.get_dummies(train_test_lin['TotRmsAbvGrd'], prefix='TotRmsAbvGrd', drop_first=True),
pd.get_dummies(train_test_lin['ExterQual'], prefix='ExterQual', drop_first=True),
pd.get_dummies(train_test_lin['Exterior2nd'], prefix='Exterior2nd', drop_first=True),
pd.get_dummies(train_test_lin['OverallCond'], prefix='OverallCond', drop_first=True),
pd.get_dummies(train_test_lin['GarageType'], prefix='GarageType', drop_first=True),
pd.get_dummies(train_test_lin['Fireplaces'], prefix='Fireplaces', drop_first=True),
pd.get_dummies(train_test_lin['CentralAir'], prefix='CentralAir', drop_first=True),
pd.get_dummies(train_test_lin['LotShape'], prefix='LotShape', drop_first=True),
pd.get_dummies(train_test_lin['Exterior1st'], prefix='Exterior1st', drop_first=True),
pd.get_dummies(train_test_lin['BsmtExposure'], prefix='BsmtExposure', drop_first=True),
pd.get_dummies(train_test_lin['Neighborhood'], prefix='Neighborhood', drop_first=True),
pd.get_dummies(train_test_lin['FullBath'], prefix='FullBath', drop_first=True),
pd.get_dummies(train_test_lin['GarageCars'], prefix='GarageCars', drop_first=True),
pd.get_dummies(train_test_lin['MSZoning'], prefix='MSZoning', drop_first=True)
 ],
axis=1 )
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train )<define_variables>
train_test_lin.drop(new_cat, axis=1, inplace=True )
batch_size=64<choose_model_class>
train_to_model = train_test[train_test['SalePrice'].isna() == False]
test_to_model = train_test[train_test['SalePrice'].isna() == True]
train_to_model_lin = train_test_lin[train_test_lin['SalePrice'].isna() == False]
test_to_model_lin = train_test_lin[train_test_lin['SalePrice'].isna() == True]
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<compute_test_metric>
test_to_model.drop(['SalePrice'], axis=1, inplace=True)
 test_to_model_lin.drop(['SalePrice'], axis=1, inplace=True )
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(train_to_model.drop(['SalePrice'], axis=1), \
train_to_model['SalePrice'], test_size=0.1, random_state=42 )
def get_model() :
x = keras.layers.Input(shape=[X_train.shape[1]])
 fc1 = keras.layers.Dense(units=450, input_shape=[X_train.shape[1]] )(x)
 act1 = keras.layers.PReLU()(fc1)
 bn1 = keras.layers.BatchNormalization()(act1)
 dp1 = keras.layers.Dropout(0.55 )(bn1)
 gn1 = keras.layers.GaussianNoise(0.15 )(dp1)
 concat1 = keras.layers.Concatenate()([x, gn1])
 fc2 = keras.layers.Dense(units=600 )(concat1)
 act2 = keras.layers.PReLU()(fc2)
 bn2 = keras.layers.BatchNormalization()(act2)
 dp2 = keras.layers.Dropout(0.55 )(bn2)
 gn2 = keras.layers.GaussianNoise(0.15 )(dp2)
 concat2 = keras.layers.Concatenate()([concat1, gn2])
 fc3 = keras.layers.Dense(units=400 )(concat2)
 act3 = keras.layers.PReLU()(fc3)
 bn3 = keras.layers.BatchNormalization()(act3)
 dp3 = keras.layers.Dropout(0.55 )(bn3)
 gn3 = keras.layers.GaussianNoise(0.15 )(dp3)
 concat3 = keras.layers.Concatenate([concat2, gn3])
 output = keras.layers.Dense(units=199, activation='softmax' )(concat2)
 model = keras.models.Model(inputs=[x], outputs=[output])
 return model
def train_model(X_train, y_train, X_val, y_val):
model = get_model()
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-7), loss=crps)
 er = EarlyStopping(patience=20, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 model.fit(X_train, y_train, epochs=200, callbacks=[er], validation_data=[X_val, y_val], batch_size=batch_size)
 return model<split>
X_train_lin, X_test_lin, y_train_lin, y_test_lin = train_test_split(train_to_model_lin.drop(['SalePrice'], axis=1), \
train_to_model_lin['SalePrice'], test_size=0.1, random_state=42 )
rkf = RepeatedKFold(n_splits=5, n_repeats=5)
 models = []
for tr_idx, vl_idx in rkf.split(X_train, y_train):
x_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
x_vl, y_vl = X_train[vl_idx], y_train[vl_idx]
model = train_model(x_tr, y_tr, x_vl, y_vl)
 models.append(model )<categorify>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42 )
def make_pred(df, sample, env, models):
df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)
 df['StadiumType'] = df['StadiumType'].apply(transform_StadiumType)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(df.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df.drop(['Yards'], axis=1, inplace=True)
 df['Turf'] = df['Turf'].map(Turf)
 df['Turf'] = df['Turf'] == 'Natural'
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 df['WindDirection'] = df['WindDirection'].apply(clean_WindDirection)
 df['WindDirection'] = df['WindDirection'].apply(transform_WindDirection)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df = df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'NflId', 'NflIdRusher', 'GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1)
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 X = np.array(df[players_col] ).reshape(-1, len(players_col)*22)
 play_col = df.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = df[col][::22]
X = np.concatenate([X, X_play_col], axis=1)
 X = scaler.transform(X)
 y_pred = np.mean([np.cumsum(model.predict(X), axis=1)for model in models], axis=0)
 yardsleft = np.array(df['YardsLeft'][::22])
 for i in range(len(yardsleft)) :
y_pred[i, :yardsleft[i]-1] = 0
y_pred[i, yardsleft[i]+100:] = 1
env.predict(pd.DataFrame(data=y_pred.clip(0,1),columns=sample.columns))
return y_pred<predict_on_test>
%%time
lgb_reg = lgbm.LGBMRegressor(objective='regression', learning_rate=0.01, n_estimators=100, n_jobs=-1, subsample=0.5)
 lgb_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(lgb_reg, param_grid=lgb_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 lgb_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = lgb_reg.predict(X_test)
 print('-' * 10 + 'LGBM' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models )<save_to_csv>
%%time
xgb_reg = xgb.XGBRegressor(learning_rate=0.01, n_estimators=100, n_jobs=-1, booster='gbtree', random_state=42, subsample=0.5)
 xgb_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(xgb_reg, param_grid=xgb_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 xgb_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = xgb_reg.predict(X_test)
 print('-' * 10 + 'XGB' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))
env.write_submission_file()<import_modules>
%%time
cat_reg = cat.CatBoostRegressor(learning_rate=0.01, n_estimators=100, objective='RMSE', loss_function='R2', random_state=42, subsample=0.5)
 cat_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(cat_reg, param_grid=cat_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 cat_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = cat_reg.predict(X_test)
 print('-' * 10 + 'CatBoost' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))
print(keras.__version__)
 print(tf.__version__ )<load_from_csv>
%%time
gb_reg = GradientBoostingRegressor(n_estimators=100, learning_rate=0.01, random_state=42, subsample=0.5)
 gbdt_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(gb_reg, param_grid=gbdt_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 gb_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = gb_reg.predict(X_test)
 print('-' * 10 + 'GBM' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))
def nfl_read_train() :
if os.path.isfile('./input/nfl-big-data-bowl-2020/train.csv.gz'):
return pd.read_pickle('./input/nfl-big-data-bowl-2020/train.csv.gz')
 if os.path.isfile('./input/nfl-big-data-bowl-2020/train.csv'):
X = pd.read_csv('./input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 pd.to_pickle(X, './input/nfl-big-data-bowl-2020/train.csv.gz')
 return X
return pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 def nfl_read_test() :
if os.path.isfile('./input/nfl-big-data-bowl-2020/test.csv.gz'):
return pd.read_pickle('./input/nfl-big-data-bowl-2020/test.csv.gz')
 if os.path.isfile('./input/nfl-big-data-bowl-2020/test.csv'):
return pd.read_csv('./input/nfl-big-data-bowl-2020/test.csv', low_memory=False)
 return None
class ModelExtractionCallback(object):
def __init__(self):
self._model = None
def __call__(self, env):
self._model = env.model
def _assert_called_cb(self):
if self._model is None:
raise RuntimeError('callback has not called yet')
 @property
def boosters_proxy(self):
self._assert_called_cb()
return self._model
@property
def raw_boosters(self):
self._assert_called_cb()
return self._model.boosters
@property
def best_iteration(self):
self._assert_called_cb()
return self._model.best_iteration
def nfl_eval_crps(labels, predictions):
scaler = preprocessing.StandardScaler()
y_pred = np.zeros(( len(labels), 199))
y_ans = np.zeros(( len(labels), 199))
j = np.array(range(199))
for i,(p,t)in enumerate(zip(np.round(scaler.inverse_transform(predictions)) ,labels)) :
k2 = j[j>=p-10]
y_pred[i][k2]=(k2+10-p)*0.05
k1 = j[j>=p+10]
y_pred[i][k1]= 1.0
k3 = j[j>=t]
y_ans[i][k3]= 1.0
return 'CRPS', np.sum(( y_pred - y_ans)**2)/(199 * y_pred.shape[0]), False
def nfl_create_cdf(X):
print(X.shape)
 y_ = np.array([X["Yards"][i] for i in range(0, len(X), 22)])
 print(y_.shape)
 scaler = preprocessing.StandardScaler()
scaler.fit(y_.reshape(-1, 1))
return scaler.transform(y_.reshape(-1, 1)).flatten()
def nfl_create_yards_cdf(yard:int):
y_train = np.zeros(shape=(1, 199))
y_train[0, yard+99:] = np.ones(shape=(1, 100-yard))
return y_train
def nfl_calc_crps(y_true, y_pred):
y_true = np.clip(np.cumsum(y_true, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 return(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * y_true.shape[0])
 class NFL_NN_Metric(Callback):
def __init__(self, model, callbacks, data):
super().__init__()
self.model = model
self.callbacks = callbacks
self.data = data
def on_train_begin(self, logs=None):
for callback in self.callbacks:
callback.on_train_begin(logs)
 def on_train_end(self, logs=None):
for callback in self.callbacks:
callback.on_train_end(logs)
 def on_epoch_end(self, batch, logs=None):
X_train, y_train = self.data[0][0], self.data[0][1]
y_pred = self.model.predict(X_train)
 y_true = np.clip(np.cumsum(y_train, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 tr_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_train[-1].shape[0])
 tr_s = np.round(tr_s, 6)
 logs['tr_CRPS'] = tr_s
X_valid, y_valid = self.data[1][0], self.data[1][1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid[-1].shape[0])
 val_s = np.round(val_s, 6)
 logs['val_CRPS'] = val_s
print('tr CRPS', tr_s, 'val CRPS', val_s)
 for callback in self.callbacks:
callback.on_epoch_end(batch, logs)
 def nfl_show_play(X, play_id):
def create_football_field(linenumbers=True,
endzones=True,
highlight_line=True,
highlight_line_number=50,
highlighted_name='Line of Scrimmage',
fifty_is_los=False,
figsize=(12, 6.33)) :

 rect = patches.Rectangle(( 0, 0), 120, 53.3, linewidth=0.1,
edgecolor='r', facecolor='darkgreen', zorder=0)
 fig, ax = plt.subplots(1, figsize=figsize)
 ax.add_patch(rect)
 plt.plot([10, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 60, 60, 70, 70, 80,
80, 90, 90, 100, 100, 110, 110, 120, 0, 0, 120, 120],
[0, 0, 53.3, 53.3, 0, 0, 53.3, 53.3, 0, 0, 53.3, 53.3, 0, 0, 53.3,
53.3, 0, 0, 53.3, 53.3, 0, 0, 53.3, 53.3, 53.3, 0, 0, 53.3],
color='white')
 if fifty_is_los:
plt.plot([60, 60], [0, 53.3], color='gold')
 plt.text(62, 50, '<- Player Yardline at Snap', color='gold')
 if endzones:
ez1 = patches.Rectangle(( 0, 0), 10, 53.3,
linewidth=0.1,
edgecolor='r',
facecolor='blue',
alpha=0.2,
zorder=0)
 ez2 = patches.Rectangle(( 110, 0), 120, 53.3,
linewidth=0.1,
edgecolor='r',
facecolor='blue',
alpha=0.2,
zorder=0)
 ax.add_patch(ez1)
 ax.add_patch(ez2)
 plt.xlim(0, 120)
 plt.ylim(-5, 58.3)
 plt.axis('off')
 if linenumbers:
for x in range(20, 110, 10):
numb = x
if x > 50:
numb = 120 - x
plt.text(x, 5, str(numb - 10),
horizontalalignment='center',
fontsize=20,
color='white')
 plt.text(x - 0.95, 53.3 - 5, str(numb - 10),
horizontalalignment='center',
fontsize=20,
color='white', rotation=180)
 if endzones:
hash_range = range(11, 110)
 else:
hash_range = range(1, 120)
 for x in hash_range:
ax.plot([x, x], [0.4, 0.7], color='white')
 ax.plot([x, x], [53.0, 52.5], color='white')
 ax.plot([x, x], [22.91, 23.57], color='white')
 ax.plot([x, x], [29.73, 30.39], color='white')
 if highlight_line:
hl = highlight_line_number + 10
plt.plot([hl, hl], [0, 53.3], color='yellow')
 plt.text(hl + 2, 50, '<- {}'.format(highlighted_name),
color='yellow')
 return fig, ax
def get_dx_dy(angle, dist):
radian_angle = angle * math.pi/180.0
dx = dist * math.cos(radian_angle)
 dy = dist * math.sin(radian_angle)
 return dx, dy
df = X[X.PlayId == play_id]
df['ToLeft'] = df.PlayDirection == "left"
df['IsBallCarrier'] = df.NflId == df.NflIdRusher
df['Dir_std'] = np.mod(90 - df.Dir, 360)
 fig, ax = create_football_field()
ax.scatter(df.X, df.Y, cmap='rainbow', c=~(df.Team == 'home'), s=100)
 rusher_row = df[df.NflIdRusher == df.NflId]
ax.scatter(rusher_row.X, rusher_row.Y, color='black')
 yards_covered = rusher_row["Yards"].values[0]
x = rusher_row["X"].values[0]
y = rusher_row["Y"].values[0]
rusher_dir = rusher_row["Dir_std"].values[0]
rusher_speed = rusher_row["S"].values[0]
dx, dy = get_dx_dy(rusher_dir, rusher_speed)
 ax.arrow(x, y, dx, dy, length_includes_head=True, width=0.3, color='black')
 left = 'left' if df.ToLeft.sum() > 0 else 'right'
plt.title(f'Play
plt.legend()
plt.show()
class NFL_trn_BinsYard(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
bins = [0,2,4,6,8,10,15,20,30,40,50,60,70,80,90,100]
X['YardsToGo'] = X[['FieldPosition','PossessionTeam','YardLine']].apply(\
lambda x:(50-x['YardLine'])+50 if x['PossessionTeam']==x['FieldPosition'] else x['YardLine'],1)
 cut_YardsToGo = pd.cut(X['YardsToGo'],bins = bins)
 yard_bins = pd.get_dummies(cut_YardsToGo)
 yard_bins.columns = [str(i)for i in yard_bins.columns.tolist() ]
X = pd.merge(X,yard_bins,left_index = True, right_index = True)
 X.columns = [str(x ).replace(']','' ).replace('(','')for x in X.columns]
return X
class NFL_trn_CatOneHot(BaseEstimator, TransformerMixin):
def __init__(self, cat_list, cat_column, cat_onehot_prefix=None):
cat_list = pd.Series(cat_list ).unique()
cat_list.sort()
if cat_onehot_prefix==None:
cat_onehot_prefix = cat_column
rename_dic = {cat_list[i]: cat_onehot_prefix + '_' + str(i)for i in range(len(cat_list)) }
dummies = pd.get_dummies(cat_list)
 dummies = dummies.rename(columns=rename_dic)
 self.cat_column = cat_column
self.onehot_table = pd.DataFrame(cat_list, columns=[cat_column])
 self.onehot_table = pd.concat([self.onehot_table, dummies], axis=1)
 def fit(self, X, y=None):
return self
def transform(self, X):
return pd.merge(X, self.onehot_table, how='left', left_on=[self.cat_column], right_on=[self.cat_column])
 class NFL_trn_CatOneHot2(BaseEstimator, TransformerMixin):
def __init__(self, cat_series, cat_onehot_prefix=None, drop_org=True):
cat_se = cat_series.copy().drop_duplicates().dropna().sort_values()
cat_list = list(cat_se.values)
 cat_onehot_prefix = cat_se.name
rename_dic = {cat_list[i]: cat_onehot_prefix + '_' + str(i)for i in range(len(cat_list)) }
dummies = pd.get_dummies(cat_list)
 dummies = dummies.rename(columns=rename_dic)
 self.columns = dummies.columns
self.drop_org = drop_org
self.cat_column = cat_se.name
self.onehot_table = pd.DataFrame(cat_list, columns=[cat_se.name])
 self.onehot_table = pd.concat([self.onehot_table, dummies], axis=1)
 def fit(self, X, y=None):
return self
def transform(self, X):
X = pd.merge(X, self.onehot_table, how='left', left_on=[self.cat_column], right_on=[self.cat_column])
 if self.drop_org:
X.drop(self.cat_column, axis=1, inplace=True)
 for col in self.columns:
X[col].fillna(0, inplace=True)
 X[col] = X[col].astype('uint8')
 return X
class NFL_trn_FillNaForCat(BaseEstimator, TransformerMixin):
def __init__(self, cat_column=[]):
self.cat_column = cat_column
def fit(self, X, y=None):
return self
def transform(self, X):
for col in self.cat_column:
X[col] = X[col].astype('object')
 X[col].fillna('Unknown', inplace=True)
 X[col] = X[col].astype('category')
 return X
class NFL_trn_FillNaMean(BaseEstimator, TransformerMixin):
def __init__(self, cat_column=[]):
self.cat_column = cat_column
def fit(self, X, y=None):
return self
def transform(self, X):
for col in self.cat_column:
X[col] = X[col].astype('object')
 X[col].fillna('Unknown', inplace=True)
 X[col] = X[col].astype('category')
 return X
class NFL_trn_ChangeSameDirection(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
df = X.copy()
df['ToLeft'] = X['PlayDirection'] == "left"
X.loc[df['ToLeft'], 'X'] = 120 - df.loc[df['ToLeft'], 'X']
X.loc[df['ToLeft'], 'Y'] = 160/3 - df.loc[df['ToLeft'], 'Y']
X.loc[df['ToLeft'], 'Dir'] =(df.loc[df['ToLeft'], 'Dir']+180)% 360
X.loc[df['ToLeft'], 'Orientation'] =(df.loc[df['ToLeft'], 'Orientation']+180)% 360
X['PlayDirection'] = X['PlayDirection'].astype('object')
 X.loc[df['ToLeft'], 'PlayDirection'] = 'right'
X['PlayDirection'] = X['PlayDirection'].astype('category')
 X.loc[df['ToLeft'], 'YardLine'] = 100 - df.loc[df['ToLeft'], 'YardLine']
return X
def get_categorical_column(X, res_no_cat=""):
res = []
for col in X.columns:
if type(X[col].dtype)==pd.core.dtypes.dtypes.CategoricalDtype:
res.append(col)
 if len(res)<=0:
res = res_no_cat
return res
def get_numeric_column(X, res_no_num=""):
res = []
numerics = [ 'int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64',
'uint8', 'uint16', 'uint32', 'uint64']
for col in X.columns:
col_type = X[col].dtypes
if col_type in numerics:
res.append(col)
 return res
class NFL_trn_DataSelector(BaseEstimator, TransformerMixin):
def __init__(self, select_columns, number_only=False):
self.select_columns = select_columns
self.number_only = number_only
def fit(self, X, y=None):
return self
def transform(self, X):
df = X.copy()
for col in df.columns:
if not col in self.select_columns:
df.drop(col, axis=1, inplace=True)
 if self.number_only:
return df.select_dtypes(include='number')
 return df
class NFL_trn_DataDropper(BaseEstimator, TransformerMixin):
def __init__(self, drop_columns=[], remain_columns=[]):
self.drop_columns = drop_columns
self.remain_columns = remain_columns
def fit(self, X, y=None):
return self
def transform(self, X):
df = X.copy()
for attribute in self.drop_columns:
if attribute in df.columns:
df.drop(attribute, axis=1, inplace=True)
 for attribute in df.columns:
if df[attribute].dtype=='object' and not attribute in self.remain_columns:
df.drop(attribute, axis=1, inplace=True)
 return df
class NFL_trn_NumberSelector(BaseEstimator, TransformerMixin):
def __init__(self, remain_categories=True):
self.remain_categories = remain_categories
def fit(self, X, y=None):
return self
def transform(self, X):
df = X.copy()
for attribute in df.columns:
if not np.issubdtype(X[attribute].dtype, np.number):
if True==self.remain_categories and type(X[attribute].dtype)==pd.core.dtypes.dtypes.CategoricalDtype:
continue
df.drop(attribute, axis=1, inplace=True)
 return df
class NFL_trn_DataSelector_OneRowEachPlayId(BaseEstimator, TransformerMixin):
def __init__(self, remain_categories=True):
self.remain_categories = remain_categories
def fit(self, X, y=None):
return self
def transform(self, X):
df = X.copy()
return df[df['NflId'] == df['NflIdRusher']]
class NFL_trn_DataTypesCleaner(BaseEstimator, TransformerMixin):
def __init__(self, verbose=False):
self.verbose = verbose
def fit(self, X, y=None):
return self
def transform(self, X):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categories = [
'Team', 'JerseyNumber', 'Season', 'PossessionTeam',
'FieldPosition', 'OffenseFormation',
'PlayDirection', 'PlayerCollegeName',
'Position', 'HomeTeamAbbr', 'VisitorTeamAbbr',
'Stadium', 'Location', 'StadiumType',
'Turf', 'GameWeather', 'WindDirection',
'NflId', 'NflIdRusher',
'rusherPosition'
]
if self.verbose:
start_mem = X.memory_usage().sum() / 1024**2
for col in X.columns:
col_type = X[col].dtypes
if col_type in numerics:
c_min = X[col].min()
c_max = X[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
X[col] = X[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
X[col] = X[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
X[col] = X[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
X[col] = X[col].astype(np.int64)
 else:
c_prec = X[col].apply(lambda x: np.finfo(x ).precision ).max()
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max and c_prec == np.finfo(np.float16 ).precision:
X[col] = X[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max and c_prec == np.finfo(np.float32 ).precision:
X[col] = X[col].astype(np.float32)
 else:
X[col] = X[col].astype(np.float64)
 elif col in categories:
X[col] = X[col].astype('category')
 if self.verbose:
end_mem = X.memory_usage().sum() / 1024**2
print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return X
class NFL_trn_DefenseXspead(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def get_defense_x_spread(X):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team']]
rusher.columns = ['PlayId','RusherTeam']
defense = pd.merge(df, rusher,on=['PlayId'],how='inner')
 defense = defense[defense['Team']!=defense['RusherTeam']][['PlayId','X']]
defense = defense.groupby(['PlayId'] ).agg({'X':['min','max']} ).reset_index()
defense.columns = ['PlayId','def_min_X','def_max_X']
defense['defense_x_spread'] = defense['def_max_X'] - defense['def_min_X']
defense.drop(['def_min_X','def_max_X'], axis=1, inplace=True)
 return defense
res = get_defense_x_spread(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_DistanceToCentroid(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def euclidean_distance(x1, y1, x2, y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def get_centroid(df):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team','X','Y']]
rusher.columns = ['PlayId','RusherTeam','RusherX','RusherY']
base = pd.merge(df,rusher,on=['PlayId'],how='inner')
 defense = base[base['Team']!=base['RusherTeam']][['PlayId','X','Y','RusherX','RusherY']]
defense = defense.groupby(['PlayId'] ).agg({'X':['mean'], 'Y':['mean']} ).reset_index()
defense.columns = ['PlayId','cent_defence_X','cent_defence_Y']
offence = base[base['Team']==base['RusherTeam']][['PlayId','X','Y','RusherX','RusherY']]
offence = offence.groupby(['PlayId'] ).agg({'X':['mean'], 'Y':['mean']} ).reset_index()
offence.columns = ['PlayId','cent_offence_X','cent_offence_Y']
base = base[['PlayId', 'RusherX','RusherY']]
base = base.merge(defense, on=['PlayId'], how='inner')
 base = base.merge(offence, on=['PlayId'], how='inner')
 base.drop_duplicates('PlayId', inplace=True)
 base['distance_to_centroid'] = \
base[['RusherX','RusherY','cent_defence_X','cent_defence_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 base['distance_to_offence_centroid'] = \
base[['RusherX','RusherY','cent_offence_X','cent_offence_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 return base[['PlayId', 'distance_to_centroid', 'distance_to_offence_centroid']]
res = get_centroid(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_DistanceToQB(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def euclidean_distance(x1, y1, x2, y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def get_distance_to_qb(X):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team','X','Y']]
rusher.columns = ['PlayId','RusherTeam','RusherX','RusherY']
base = pd.merge(df,rusher,on=['PlayId'],how='inner')
 offence = base[base['Team']==base['RusherTeam']][['PlayId','X','Y','Position']]
offence = offence.query('Position == ["QB"]')
 offence = offence.groupby('PlayId' ).agg({'X':['mean'], 'Y':['mean']} ).reset_index()
offence.columns = ['PlayId', 'qbX', 'qbY']
res = pd.merge(rusher, offence, on='PlayId')
 res['distance_to_qb'] = res[['RusherX','RusherY','qbX','qbY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 return res[['PlayId', 'distance_to_qb']]
res = get_distance_to_qb(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_FillNa(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
for col in X.columns:
if X[col].isnull().any() :
if hasattr(X[col], 'cat'):
X[col] = X[col].astype('object')
 X[col].fillna('Unknown', inplace=True)
 X[col] = X[col].astype('category')
 elif np.issubdtype(X[col].dtypes, np.number):
X[col].fillna(X[col].mean() , inplace=True)
 return X
class NFL_trn_FillNa2(BaseEstimator, TransformerMixin):
def __init__(self):
self.tgt_cols = ['X', 'Y', 'S', 'A', 'Dis', 'Orientation',
'Dir', 'NflId', 'DisplayName', 'JerseyNumber', 'Season',
'Quarter', 'GameClock', 'PossessionTeam', 'Down', 'Distance',
'FieldPosition', 'HomeScoreBeforePlay', 'VisitorScoreBeforePlay',
'OffenseFormation', 'OffensePersonnel',
'DefendersInTheBox', 'DefensePersonnel', 'PlayDirection', 'TimeHandoff',
'TimeSnap', 'PlayerHeight', 'PlayerWeight', 'PlayerBirthDate',
'PlayerCollegeName', 'Position', 'HomeTeamAbbr', 'VisitorTeamAbbr',
'Week', 'Stadium', 'Location', 'StadiumType', 'Turf', 'GameWeather',
'Temperature', 'Humidity', 'WindSpeed', 'WindDirection']
self.fill_val = np.zeros(len(self.tgt_cols))
def fit(self, X, y=None):
return self
def transform(self, X):
res = X.copy()
values = np.vstack(( self.fill_val, X[self.tgt_cols].values))
df = pd.DataFrame(values, columns=self.tgt_cols)
 df.fillna(method='ffill', axis=0, inplace=True)
 self.fill_val = df.values[df.shape[0]-1, ]
res[self.tgt_cols] = df.iloc[1:,].values
return res
class NFL_trn_GameWeather(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
rain = ['Rainy', 'Rain Chance 40%', 'Showers',
'Cloudy with periods of rain, thunder possible.Winds shifting to WNW, 10-20 mph.',
'Scattered Showers', 'Cloudy, Rain', 'Rain shower', 'Light Rain', 'Rain']
overcast = ['Cloudy, light snow accumulating 1-3"', 'Party Cloudy',
'Cloudy, chance of rain', 'Coudy',
'Cloudy, 50% change of rain', 'Rain likely, temps in low 40s.',
'Cloudy and cold', 'Cloudy, fog started developing in 2nd quarter',
'Partly Clouidy', '30% Chance of Rain', 'Mostly Coudy', 'Cloudy and Cool',
'cloudy', 'Partly cloudy', 'Overcast', 'Hazy', 'Mostly cloudy',
'Mostly Cloudy', 'Partly Cloudy', 'Cloudy']
clear = ['Partly clear', 'Sunny and clear', 'Sun & clouds', 'Clear and Sunny',
'Sunny and cold', 'Sunny Skies', 'Clear and Cool', 'Clear and sunny',
'Sunny, highs to upper 80s', 'Mostly Sunny Skies', 'Cold',
'Clear and warm', 'Sunny and warm', 'Clear and cold', 'Mostly sunny',
'T: 51; H: 55; W: NW 10 mph', 'Clear Skies', 'Clear skies', 'Partly sunny',
'Fair', 'Partly Sunny', 'Mostly Sunny', 'Clear', 'Sunny', 'Sunny, Windy']
snow = ['Heavy lake effect snow', 'Snow']
none = ['N/A Indoor', 'Indoors', 'Indoor', 'N/A(Indoors)', 'Controlled Climate']
X['GameWeather'] = X['GameWeather'].replace(rain,'rain')
 X['GameWeather'] = X['GameWeather'].replace(overcast,'overcast')
 X['GameWeather'] = X['GameWeather'].replace(clear,'clear')
 X['GameWeather'] = X['GameWeather'].replace(snow,'snow')
 X['GameWeather'] = X['GameWeather'].replace(none,'none')
 X.fillna({'GameWeather': 'none'}, inplace=True)
 X['GameWeather'] = X['GameWeather'].astype('category')
 return X
class NFL_trn_NormalizedMovingX(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform_org(self, X):
def create_normalized_moving_X(df_rus):
try:
rusFieldPosition = df_rus['FieldPosition']
rusYardLine = df_rus['YardLine']
rusTeam = df_rus['Team']
rusHomeTeamAbbr = df_rus['HomeTeamAbbr']
rusPlayDirection = df_rus['PlayDirection']
if rusPlayDirection == 'left':
rusX = df_rus['X'] - 10
else:
rusX = 100 -(df_rus['X'] - 10)
 remaining_yardline_X = 0
if rusTeam == 'home':
if rusFieldPosition == rusHomeTeamAbbr:
remaining_yardline_X = 100 - rusYardLine
else:
remaining_yardline_X = rusYardLine
else:
if rusFieldPosition != rusHomeTeamAbbr:
remaining_yardline_X = 100 - rusYardLine
else:
remaining_yardline_X = rusYardLine
moving_X = rusX - remaining_yardline_X
normalized_moving_X =(moving_X / remaining_yardline_X)*100
if(normalized_moving_X > 100):
normalized_moving_X = 100
if(normalized_moving_X < 0):
normalized_moving_X =0
returned_df = df_rus[["PlayId"]]
returned_df["normalized_moving_X"] = normalized_moving_X
except Exception as e:
print(e)
 print("error")
 return returned_df
result_list = X[X['NflId'] == X['NflIdRusher']].apply(lambda x: create_normalized_moving_X(x),1)
 X = pd.merge(X,result_list,on=['PlayId'],how='inner')
 return X
def transform(self, X):
def get_rem_yard_line(rusTeam, rusFieldPosition, rusHomeTeamAbbr, rusYardLine):
if rusTeam == 'home':
if rusFieldPosition == rusHomeTeamAbbr:
remaining_yardline_X = 100 - rusYardLine
else:
remaining_yardline_X = rusYardLine
else:
if rusFieldPosition != rusHomeTeamAbbr:
remaining_yardline_X = 100 - rusYardLine
else:
remaining_yardline_X = rusYardLine
return remaining_yardline_X
def get_rus_x(x, PlayDirection):
if PlayDirection == 'left':
rusX = x - 10
else:
rusX = 100 -(x - 10)
 return rusX
def get_moving_x(x, yardline, dir):
if dir=='right':
return 110 - x
else:
return x - 10
def get_normalized_moving_X(X):
df = X.copy()
cols = ['PlayId','X','Team','YardLine','PlayDirection','FieldPosition','HomeTeamAbbr']
rusher = df[df['NflId'] == df['NflIdRusher']][cols]
rusher['remYardLine'] = rusher[['Team','FieldPosition','HomeTeamAbbr','YardLine']].apply(lambda x: get_rem_yard_line(x[0],x[1],x[2],x[3]), axis=1)
 rusher['rusX'] = rusher[['X','PlayDirection']].apply(lambda x: get_rus_x(x[0],x[1]), axis=1)
 rusher['moving_X'] = rusher[['rusX','remYardLine']].apply(lambda x: np.abs(x[0]-x[1]), axis=1)
 rusher['normalized_moving_X'] = rusher[['moving_X','remYardLine']].apply(lambda x:(x[0]/x[1])*100, axis=1)
 return rusher[['PlayId', 'normalized_moving_X']]
res = get_normalized_moving_X(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_OffenseLead(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform_org(self, X):
X['OffenseLead'] = X[['PossessionTeam','HomeTeamAbbr','HomeScoreBeforePlay','VisitorScoreBeforePlay']].apply(lambda x: x[2]-x[3] if x[0] == x[1] else x[3]-x[2], axis = 1)
 return X
def transform(self, X):
df = X.copy()
df = df[df['NflId'] == df['NflIdRusher']]
df = df[['PlayId', 'PossessionTeam','HomeTeamAbbr','HomeScoreBeforePlay','VisitorScoreBeforePlay']]
df['OffenseLead'] = df.apply(lambda x: x[3]-x[4] if x[1]==x[2] else x[4]-x[3], axis=1)
 return X.merge(df[['PlayId', 'OffenseLead']], on='PlayId')
 class NFL_trn_Orientation(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
X.loc[X['Season']==2017, 'Orientation'] = np.mod(90 + X.loc[X['Season']==2017, 'Orientation'], 360)
 return X
class NFL_trn_Orientation_std(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def std_orientation(x):
if x == None:
x = 0
x %= 360
if x>180:
x = 360 - x
return x
X['Orientation_std'] = X['Orientation'].apply(std_orientation)
 return X
class NFL_trn_PersonnelSplitter(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def split_personnel(s):
splits = s.split(',')
 for i in range(len(splits)) :
splits[i] = splits[i].strip()
return splits
def defense_formation(l):
dl = 0
lb = 0
db = 0
other = 0
for position in l:
sub_string = position.split(' ')
 if sub_string[1] == 'DL':
dl += int(sub_string[0])
 elif sub_string[1] in ['LB','OL']:
lb += int(sub_string[0])
 else:
db += int(sub_string[0])
 counts =(dl,lb,db,other)
 return counts
def offense_formation(l):
qb = 0
rb = 0
wr = 0
te = 0
ol = 0
sub_total = 0
qb_listed = False
for position in l:
sub_string = position.split(' ')
 pos = sub_string[1]
cnt = int(sub_string[0])
 if pos == 'QB':
qb += cnt
sub_total += cnt
qb_listed = True
elif pos in ['RB','LB']:
rb += cnt
sub_total += cnt
elif pos in ['WR','DB']:
wr += cnt
sub_total += cnt
elif pos == 'TE':
te += cnt
sub_total += cnt
else:
ol += cnt
sub_total += cnt
if sub_total < 11:
diff = 11 - sub_total
if not qb_listed:
qb += 1
diff -= 1
ol += diff
counts =(qb,rb,wr,te,ol)
 return counts
def personnel_features(df):
personnel = df[['GameId','PlayId','OffensePersonnel','DefensePersonnel']].drop_duplicates()
personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: split_personnel(x))
personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: defense_formation(x))
personnel['DefenceDL'] = personnel['DefensePersonnel'].apply(lambda x: x[0])
 personnel['DefenceLB'] = personnel['DefensePersonnel'].apply(lambda x: x[1])
 personnel['DefenceDB'] = personnel['DefensePersonnel'].apply(lambda x: x[2])
 personnel['DefenceOL'] = 0
personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: split_personnel(x))
personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: offense_formation(x))
personnel['OffenceQB'] = personnel['OffensePersonnel'].apply(lambda x: x[0])
 personnel['OffenceRB'] = personnel['OffensePersonnel'].apply(lambda x: x[1])
 personnel['OffenceWR'] = personnel['OffensePersonnel'].apply(lambda x: x[2])
 personnel['OffenceTE'] = personnel['OffensePersonnel'].apply(lambda x: x[3])
 personnel['OffenceOL'] = personnel['OffensePersonnel'].apply(lambda x: x[4])
 personnel['OffenceDB'] = 0
personnel['OffenceDL'] = 0
personnel['OffenceLB'] = 0
personnel['OL_diff'] = personnel['OffenceOL'] - personnel['DefenceDL']
personnel['OL_TE_diff'] =(personnel['OffenceOL'] + personnel['OffenceTE'])- personnel['DefenceDL']
personnel['run_def'] =(personnel['DefenceDL'] + personnel['DefenceLB'] > 6 ).astype(int)
 personnel.drop(['OffensePersonnel','DefensePersonnel'], axis=1, inplace=True)
 return personnel
res = personnel_features(X)
 res.drop('GameId', axis=1, inplace=True)
 return X.merge(res, on='PlayId')
 class NFL_trn_PlayerAge(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def calc_player_age(birth_day, season):
birth_year = birth_day.split('/')[2]
return season - int(birth_year)
 mapped_list = map(calc_player_age, X['PlayerBirthDate'], X['Season'])
 X['PlayerAge'] = list(mapped_list)
 return X
class NFL_trn_DistDefenders(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
res = defense_features(X)
 res.drop('GameId', axis=1, inplace=True)
 return X.merge(res, on='PlayId')
 class NFL_trn_DistRB2Defenders(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
def euclidean_distance(x1, y1, x2, y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def get_dist_rb2defense(X):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team']]
rusher.columns = ['PlayId','RusherTeam']
base = pd.merge(df,rusher,on=['PlayId'],how='inner')
 offence = base[base['Team']==base['RusherTeam']][['PlayId','X','Y','Position']]
offence = offence.query('Position == ["RB", "FB", "HB", "TB"]')
 if len(offence)<=0:
rusher['RB2Defence_avg'] = 10.218225670975324
rusher['RB2Defence_var'] = 36.048544281959956
return rusher[['PlayId', 'RB2Defence_avg', 'RB2Defence_var']]
offence = offence.groupby('PlayId' ).agg({'X':['mean'], 'Y':['mean']} ).reset_index()
offence.columns = ['PlayId', 'rbX', 'rbY']
defense = base[base['Team']!=base['RusherTeam']][['PlayId','X','Y']]
res = defense.merge(offence, on='PlayId')
 res['dist'] = res[['X','Y','rbX','rbY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 res = res.groupby(['PlayId'] ).agg({'dist':['mean', np.var]} ).reset_index()
res.columns = ['PlayId', 'RB2Defence_avg', 'RB2Defence_var']
return res
res = get_dist_rb2defense(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_PlayerHeight(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def calc_height_mm(heightYard):
h = heightYard.split('-')
 h_feet = float(h[0])
 h_inch = float(h[1])
 return(h_feet * 304.8)+(h_inch * 25.4)
 mapped_list = map(calc_height_mm, X['PlayerHeight'])
 X['PlayerHeight'] = list(mapped_list)
 return X
class NFL_trn_PlayerRelativeToBack(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']].copy()
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
df = X.copy()
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 res = features_relative_to_back(df, back_feats)
 res.drop('GameId', axis=1, inplace=True)
 X = X.merge(res, on='PlayId')
 X['back_oriented_down_field'] = X['back_oriented_down_field'].astype('category')
 X['back_moving_down_field'] = X['back_moving_down_field'].astype('category')
 return X
class NFL_trn_PlayerRusher(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
rusherCol = ['S', 'A', 'Dis', 'Orientation', 'Dir',
'PlayerHeight', 'PlayerWeight', 'PlayerAge', 'Position']
df = X[X['NflId'] == X['NflIdRusher']]
res = pd.DataFrame()
res['PlayId'] = df['PlayId']
for col in rusherCol:
if col in df.columns:
res[f'rusher{col}'] = df[col]
res['rusherS_horizontal'] = df['S'] * np.sin(np.deg2rad(res['rusherDir']))
res['rusherS_vertical'] = df['S'] * np.cos(np.deg2rad(res['rusherDir']))
res['rusherY_std'] = df['Y'] -(53.3 / 2)
 X = X.merge(res, on='PlayId')
 if 'rusherPosition' in X.columns:
X['rusherPosition'] = X['rusherPosition'].astype('category')
 return X
class NFL_trn_ProcessTwo(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def process_two(t_):
t_['fe1'] = pd.Series(np.sqrt(np.absolute(np.square(t_.X.values)- np.square(t_.Y.values))))
t_['fe5'] = np.square(t_['S'].values)+ 2 * t_['A'].values * t_['Dis'].values
t_['fe7'] = np.arccos(np.clip(t_['X'].values / t_['Y'].values, -1, 1))
t_['fe8'] = t_['S'].values / np.clip(t_['fe1'].values, 0.6, None)
 radian_angle =(90 - t_['Dir'])* np.pi / 180.0
t_['fe10'] = np.abs(t_['S'] * np.cos(radian_angle))
t_['fe11'] = np.abs(t_['S'] * np.sin(radian_angle))
return t_
return process_two(X)
 class NFL_trn_RunnerDistance2los(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def update_yardline(X):
df = X.copy()
new_yardline = df[df['NflId'] == df['NflIdRusher']].copy()
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 return new_yardline
def get_runner_dist_to_los(df):
rusher = df[['PlayId','X','YardLine']].copy()
rusher.columns = ['PlayId','RusherX','YardLine']
rusher['runner_distance_to_los'] = rusher[['RusherX','YardLine']].apply(lambda x: np.abs(x[0] - x[1]), axis=1)
 return rusher.drop(['RusherX', 'YardLine'], axis=1)
 df = update_yardline(X)
 res = get_runner_dist_to_los(df)
 return X.merge(res, on='PlayId')
 class NFL_trn_RunnerVs1stDefensorSpeed(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def get_runner_vs_1stdefensor_speed(X):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team','X','Y','S']]
rusher.columns = ['PlayId','RusherTeam','RusherX','RusherY','RusherS']
defense = pd.merge(df,rusher,on='PlayId',how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['PlayId','RusherX','RusherY','RusherS','X','Y','S']]
defense['dist'] = defense.apply(lambda x: euclidean_distance(x[1],x[2],x[4],x[5]), axis=1)
 dist_min = defense.groupby('PlayId' ).agg({'dist':['min']} ).reset_index()
dist_min.columns = ['PlayId', 'dist_min']
defense = defense.merge(dist_min, on='PlayId')
 defense = defense[defense['dist']==defense['dist_min']]
defense['runner_vs_1stdefensor_speed'] = defense['RusherS'] / defense['dist_min']
return defense[['PlayId', 'runner_vs_1stdefensor_speed']]
res = get_runner_vs_1stdefensor_speed(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_SecondsSinceGameStart(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
tmp = X.copy()
tmp['tmp'] = X['GameClock'].apply(lambda x: 15 * 60 -(int(x[0:2])*60 + int(x[3:5])))
 X['game_seconds_left'] = X['Quarter'].map({1:2700, 2:1800, 3:900, 4:0})+ tmp['tmp']
return X
class NFL_trn_SecondsSinceStart(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
X['seconds_since_start'] = X['GameClock'].apply(lambda x: 15 * 60 -(int(x[0:2])*60 + int(x[3:5])))
 return X
class NFL_trn_StadiumType(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
outdoor = ['Outdoor', 'Outdoors', 'Cloudy', 'Heinz Field', 'Outdor', 'Ourdoor',
'Outside', 'Outddors','Outdoor Retr Roof-Open', 'Oudoor', 'Bowl']
indoor_closed = ['Indoors', 'Indoor', 'Indoor, Roof Closed', 'Indoor, Roof Closed', 'Retractable Roof',
'Retr.Roof-Closed', 'Retr.Roof - Closed', 'Retr.Roof Closed']
indoor_open = ['Indoor, Open Roof', 'Open', 'Retr.Roof-Open', 'Retr.Roof - Open']
dome_closed = ['Dome', 'Domed, closed', 'Closed Dome', 'Domed', 'Dome, closed']
dome_open = ['Domed, Open', 'Domed, open']
X['StadiumType'] = X['StadiumType'].replace(outdoor,'outdoor')
 X['StadiumType'] = X['StadiumType'].replace(indoor_closed,'indoor_closed')
 X['StadiumType'] = X['StadiumType'].replace(indoor_open,'indoor_open')
 X['StadiumType'] = X['StadiumType'].replace(dome_closed,'dome_closed')
 X['StadiumType'] = X['StadiumType'].replace(dome_open,'dome_open')
 X.fillna({'StadiumType': 'Unknown'}, inplace=True)
 X['StadiumType'] = X['StadiumType'].astype('category')
 return X
class NFL_trn_TeamAbbr(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
X['VisitorTeamAbbr'] = X['VisitorTeamAbbr'].astype('object')
 X['HomeTeamAbbr'] = X['HomeTeamAbbr'].astype('object')
 X.loc[X.VisitorTeamAbbr == "ARI",'VisitorTeamAbbr'] = "ARZ"
X.loc[X.HomeTeamAbbr == "ARI",'HomeTeamAbbr'] = "ARZ"
X.loc[X.VisitorTeamAbbr == "BAL",'VisitorTeamAbbr'] = "BLT"
X.loc[X.HomeTeamAbbr == "BAL",'HomeTeamAbbr'] = "BLT"
X.loc[X.VisitorTeamAbbr == "CLE",'VisitorTeamAbbr'] = "CLV"
X.loc[X.HomeTeamAbbr == "CLE",'HomeTeamAbbr'] = "CLV"
X.loc[X.VisitorTeamAbbr == "HOU",'VisitorTeamAbbr'] = "HST"
X.loc[X.HomeTeamAbbr == "HOU",'HomeTeamAbbr'] = "HST"
X['VisitorTeamAbbr'] = X['VisitorTeamAbbr'].astype('category')
 X['HomeTeamAbbr'] = X['HomeTeamAbbr'].astype('category')
 return X
class NFL_trn_TeamStat(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def get_team_stat(df):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team']]
rusher.columns = ['PlayId','RusherTeam']
base = pd.merge(df,rusher,on=['PlayId'],how='inner')
 defense = base[base['Team']!=base['RusherTeam']][['PlayId','S','A']]
defense = defense.groupby(['PlayId'] ).agg({'S':['mean'], 'A':['mean']} ).reset_index()
defense.columns = ['PlayId','S_defense_mean','A_defense_mean']
offence = base[base['Team']==base['RusherTeam']][['PlayId','S','A']]
offence = offence.groupby(['PlayId'] ).agg({'S':['mean'], 'A':['mean']} ).reset_index()
offence.columns = ['PlayId','S_offence_mean','A_offence_mean']
res = pd.merge(offence, defense, on='PlayId')
 res['SpeedDifference'] = res['S_offence_mean'] - res['S_defense_mean']
return res
res = get_team_stat(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_TimeToTackle(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform_org(self, X):
def get_time_to_tackle(df):
rusher = df[df['NflIdRusher']==df['NflId']].iloc[0]
defense = df[df['Team']!=rusher['Team']]
tm = []
for i in range(len(defense)) :
dist = np.linalg.norm(rusher[['X', 'Y']].values - defense.iloc[i][['X', 'Y']].values, ord=2)
 speed = defense.iloc[i]['S']
if speed>0:
tm.append(dist/speed)
 return np.amin(tm), np.average(tm)
 res = X.groupby('PlayId' ).apply(get_time_to_tackle)
 res = pd.DataFrame(res ).reset_index()
res[['min_time_to_take_S', 'avg_time_to_take_S']] = res[0].apply(pd.Series)
 return X.merge(res.drop(0, axis=1), on='PlayId')
 def transform(self, X):
def get_tackle_time(x1, y1, x2, y2, speed):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)/ speed
def get_tackle_time_df(X):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team','X','Y','S']]
rusher.columns = ['PlayId','RusherTeam','RusherX','RusherY','RusherS']
base = pd.merge(df, rusher,on=['PlayId'],how='inner')
 defense = base[base['Team']!=base['RusherTeam']][['PlayId','X','Y', 'S','RusherX','RusherY']]
defense['tackle_time'] = defense.apply(lambda x: get_tackle_time(x[1],x[2],x[3],x[4],x[5]), axis=1)
 defense = defense.groupby('PlayId' ).agg({'tackle_time':['min','mean']} ).reset_index()
defense.columns = ['PlayId', 'min_time_to_take_S', 'avg_time_to_take_S']
return defense
res = get_tackle_time_df(X)
 return X.merge(res, on='PlayId')
 class NFL_trn_Turf(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
natural_grass = ['natural grass','Naturall Grass','Natural Grass']
grass = ['Grass']
fieldturf = ['FieldTurf','Field turf','FieldTurf360','Field Turf']
artificial = ['Artificial','Artifical']
X['Turf'] = X['Turf'].replace(natural_grass,'natural_grass')
 X['Turf'] = X['Turf'].replace(grass,'grass')
 X['Turf'] = X['Turf'].replace(fieldturf,'fieldturf')
 X['Turf'] = X['Turf'].replace(artificial,'artificial')
 X.fillna({'Turf': 'Unknown'}, inplace=True)
 X['Turf'] = X['Turf'].astype('category')
 return X
class NFL_trn_WindDirection(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def clean_wind_direction(windDirection):
if not isinstance(windDirection, str):
windDirection = '0'
if windDirection.isnumeric() :
windDirection = '0'
windDirection = windDirection.lower()
windDirection = windDirection.replace('calm' , '0')
 windDirection = windDirection.replace('north', 'n')
 windDirection = windDirection.replace('south', 's')
 windDirection = windDirection.replace('east' , 'e')
 windDirection = windDirection.replace('west' , 'w')
 windDirection = windDirection.replace(' ' , '')
 windDirection = windDirection.replace('-' , '')
 windDirection = windDirection.replace('/' , '')
 if 'from' in windDirection:
windDirection = windDirection.replace('from' , '')
 windDirection = windDirection.replace('n' , 's')
 windDirection = windDirection.replace('e' , 'w')
 windDirection = windDirection.replace('s' , 'n')
 windDirection = windDirection.replace('w' , 'e')
 return windDirection
X['WindDirection'] = X['WindDirection'].apply(clean_wind_direction)
 X['WindDirection'] = X['WindDirection'].astype('category')
 return X
class NFL_trn_WindSpeed(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
def clean_wind_speed(speed):
if not isinstance(speed, str):
return 0.
 speed = speed.lower()
speed = speed.replace(' ', '')
 speed = speed.replace('mph', '')
 speed = speed.replace('calm', '0')
 speed = speed.replace('gustsupto', '-')
 if '-' in speed:
tmp = speed.split('-')
 return(int(tmp[0])+ int(tmp[1])) / 2.
 if not speed.isnumeric() :
return 0.
 return int(speed)
 X['WindSpeed'] = X['WindSpeed'].apply(clean_wind_speed)
 X['WindSpeed'] = X['WindSpeed'].astype(np.float16)
 return X
class NFL_trn_Xstd(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
df = X.copy()
rusher = df[df['NflId'] == df['NflIdRusher']][['PlayId','Team',]]
rusher.columns = ['PlayId','RusherTeam']
defense = pd.merge(df,rusher,on=['PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['PlayId','X']]
defense = defense.groupby(['PlayId'] ).agg({'X':['std']} ).reset_index()
defense.columns = ['PlayId','defence_x_std']
X = X.merge(defense, on='PlayId')
 offence = pd.merge(df,rusher,on=['PlayId'],how='inner')
 offence = offence[offence['Team'] == offence['RusherTeam']][['PlayId','X']]
offence = offence.groupby(['PlayId'] ).agg({'X':['std']} ).reset_index()
offence.columns = ['PlayId','offence_x_std']
X = X.merge(offence, on='PlayId')
 return X
class Nfl_clf_EnsembleVoting(BaseEstimator, ClassifierMixin):
def __init__(self, estimators, voting='hard', weights=None, n_jobs=None, flatten_transform=True):
self.estimators = estimators
self.voting = voting
self.weights = weights
self.n_jobs = n_jobs
self.flatten_transform = flatten_transform
def softmax(self, x):
u = np.sum(np.exp(x))
return np.exp(x)/u
def fit(self, X, y, sample_weight=None):
for name, clf in self.estimators:
clf.fit(X, y)
 return self
def predict(self, X):
res = [clf.predict(X)for name, clf in self.estimators]
return np.mean(res, axis=0)
 class Nfl_clf_LightGBM(BaseEstimator, ClassifierMixin):
def __init__(self, verbose=False):
self.use_feat = [
'GameId', 'PlayId', 'Team', 'Season', 'YardLine', 'Quarter',
'PossessionTeam', 'Down', 'Distance', 'HomeScoreBeforePlay',
'VisitorScoreBeforePlay', 'NflIdRusher', 'OffenseFormation',
'DefendersInTheBox', 'PlayDirection',
'Yards', 'HomeTeamAbbr', 'VisitorTeamAbbr', 'Week', 'Stadium', 'Location',
'StadiumType', 'Turf', 'GameWeather', 'Temperature', 'Humidity', 'WindSpeed',
'OffenceDB', 'OffenceDL', 'OffenceLB', 'OffenceOL',
'OffenceQB', 'OffenceRB', 'OffenceTE', 'OffenceWR',
'DefenceDB', 'DefenceDL', 'DefenceLB', 'DefenceOL',
'OL_diff', 'OL_TE_diff', 'run_def',
'RB2Defence_avg', 'RB2Defence_var',
'rusherS', 'rusherA', 'rusherDis', 'rusherOrientation', 'rusherDir',
'rusherPlayerHeight', 'rusherPlayerWeight', 'rusherPlayerAge',
'rusherPlayerWeight', 'rusherPosition', 'min_time_to_take_S', 'avg_time_to_take_S',
'rusherS_horizontal', 'rusherS_vertical',
'seconds_since_start', 'distance_to_qb', 'offence_x_std', 'defence_x_std',
'A_offence_mean', 'A_defense_mean', 'S_offence_mean', 'S_defense_mean', 'SpeedDifference',
'normalized_moving_X', 'Orientation_std', 'runner_distance_to_los', 'runner_vs_1stdefensor_speed',
'defense_x_spread', 'rusherY_std', 'distance_to_centroid', 'distance_to_offence_centroid',
'0, 2', '2, 4', '4, 6', '6, 8', '8, 10', '10, 15', '15, 20', '20, 30',
'30, 40', '40, 50', '50, 60', '60, 70', '70, 80', '80, 90', '90, 100',
'OffenseLead', 'def_min_dist', 'def_max_dist', 'def_mean_dist', 'def_std_dist',
'back_from_scrimmage', 'back_oriented_down_field',
'back_moving_down_field', 'min_dist', 'max_dist', 'mean_dist', 'std_dist',
'game_seconds_left',
'fe1', 'fe5', 'fe7', 'fe8', 'fe10', 'fe11'
]
self.verbose = verbose
def fit(self, X, y):
X_train = X.copy()
self.pipeline_pre_predict_ = make_pipeline(
 NFL_trn_DataSelector_OneRowEachPlayId() ,
NFL_trn_DataSelector(select_columns=self.use_feat), 
)
 X_train = self.pipeline_pre_predict_.fit_transform(X_train)
 if self.verbose:
X_train.info(max_cols=X_train.shape[1])
 display(datetime.datetime.now().isoformat())
 X = NFL_trn_DataDropper(drop_columns=['GameId', 'PlayId', 'Yards'] ).fit_transform(X_train)
 y = np.zeros(( len(X_train['Yards']), 199))
for idx, target in enumerate(list(X_train['Yards'].values)) :
y[idx][99 + target] = 1
y = np.argmax(y, axis=1)
 model_extraction = ModelExtractionCallback()
param = {
'num_iterations': 1000,
'num_leaves': 50,
'min_data_in_leaf': 30,
'objective':'multiclass',
'num_class': 199,
'max_depth': -1,
'learning_rate': 0.01,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.7,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": "multi_logloss",
"lambda_l1": 0.1,
"verbosity": -1,
"seed":1234
}
args_common = {
"fobj": None,
"init_model": None,
"feature_name": 'auto',
"categorical_feature": 'auto',
"early_stopping_rounds": 10,
"callbacks": [model_extraction, ],
}
args_cv = {
"folds": None,
"nfold": 5,
"stratified": True,
"shuffle": True,
"fpreproc": None,
"verbose_eval": 10,
"show_stdv": True,
"seed": 0,
}
res = lgb.cv(param, lgb.Dataset(X, y), **{**args_common, **args_cv})
 self.model_ = model_extraction.boosters_proxy
if self.verbose:
feature_importance = pd.DataFrame()
feature_importance["feature"] = X.columns
feature_importance["importance"] = np.array(self.model_.feature_importance() ).mean(axis=0)
 feature_importance.sort_values("importance", inplace=True, ascending=False)
 plt.figure(figsize=(8, 12))
sns.barplot(x="importance", y="feature", data=feature_importance[:50])
 plt.show()
display(datetime.datetime.now().isoformat())
 return self
def predict(self, X):
X_test = X.copy()
X_test = self.pipeline_pre_predict_.fit_transform(X_test)
 X = NFL_trn_DataDropper(drop_columns=['GameId', 'PlayId', 'Yards'] ).fit_transform(X_test)
 y_pred = self.model_.predict(X, num_iteration = self.model_.best_iteration)
 y_pred = np.array(y_pred)
 y_pred = np.mean(y_pred, axis=0)
 return y_pred
class Nfl_clf_NN(BaseEstimator, ClassifierMixin):
def __init__(self, verbose=False):
self.use_feat = ['GameId', 'PlayId', 'back_from_scrimmage',
'back_oriented_down_field', 'back_moving_down_field', 'min_dist',
'max_dist', 'mean_dist', 'std_dist', 'def_min_dist',
'def_max_dist', 'def_mean_dist', 'def_std_dist',
'X', 'Y',
'rusherS',
'rusherA',
'rusherDis',
'rusherOrientation',
'rusherDir',
'YardLine', 'Quarter', 'Down',
'Distance', 'DefendersInTheBox',
'DefenceDL',
'DefenceLB',
'DefenceDB',
'OffenceQB',
'OffenceRB',
'OffenceWR',
'OffenceTE',
'OffenceOL',
'OL_diff',
'OL_TE_diff', 'run_def', 'Yards',
'fe1', 'fe5', 'fe7', 'fe8', 'fe10', 'fe11'
]
self.verbose = verbose
def get_numeric_column(self, X, res_no_num=""):
res = []
numerics = [ 'int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64',
'uint8', 'uint16', 'uint32', 'uint64']
for col in X.columns:
col_type = X[col].dtypes
if col_type in numerics:
res.append(col)
 return res
def prepare_misc(self, X, yards=None):
drop_feat = ['GameId', 'PlayId', 'Yards', 'game_seconds_left']
y = None
if yards is not None:
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
df = X.copy()
df = NFL_trn_DataDropper(drop_columns=drop_feat ).fit_transform(df)
 df['back_oriented_down_field'] = df['back_oriented_down_field'].astype('int')
 df['back_moving_down_field'] = df['back_moving_down_field'].astype('int')
 cat = ['back_oriented_down_field', 'back_moving_down_field']
num = self.get_numeric_column(df)
 return df, y, cat, num
def model_396_1(self, X, cat, num):
inputs = []
embeddings = []
for i in cat:
input_ = Input(shape=(1,))
embedding = Embedding(int(np.absolute(X[i] ).max() + 1), 10, input_length=1 )(input_)
 embedding = Reshape(target_shape=(10,))(embedding)
 inputs.append(input_)
 embeddings.append(embedding)
 input_numeric = Input(shape=(len(num),))
embedding_numeric = Dense(512, activation='relu' )(input_numeric)
 inputs.append(input_numeric)
 embeddings.append(embedding_numeric)
 x = Concatenate()(embeddings)
 x = Dense(256, activation='relu' )(x)
 x = Dense(128, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(199, activation='softmax' )(x)
 model = Model(inputs, output)
 return model
def fit(self, X, y):
X_train = X.copy()
self.pipeline_pre_predict_ = make_pipeline(
 NFL_trn_DataSelector_OneRowEachPlayId() ,
NFL_trn_CatOneHot2(X['Team']),
NFL_trn_CatOneHot2(X['PossessionTeam']),
NFL_trn_CatOneHot2(X['FieldPosition']),
NFL_trn_CatOneHot2(X['OffenseFormation']),
NFL_trn_CatOneHot2(X['PlayDirection']),
NFL_trn_CatOneHot2(X['HomeTeamAbbr']),
NFL_trn_CatOneHot2(X['VisitorTeamAbbr']),
NFL_trn_CatOneHot2(X['Stadium']),
NFL_trn_CatOneHot2(X['Location']),
NFL_trn_CatOneHot2(X['StadiumType']),
NFL_trn_CatOneHot2(X['Turf']),
NFL_trn_CatOneHot2(X['GameWeather']),
NFL_trn_CatOneHot2(X['Position']),
NFL_trn_CatOneHot2(X['rusherPosition']),
NFL_trn_DataSelector(select_columns=self.use_feat), 
)
 X_train = self.pipeline_pre_predict_.fit_transform(X)
 if self.verbose:
X_train.info(max_cols=X_train.shape[1])
 display(datetime.datetime.now().isoformat())
 self.scaler_ = preprocessing.StandardScaler()
X, y, cat, num = self.prepare_misc(X_train, X_train['Yards'])
 X[num] = self.scaler_.fit_transform(X[num])
 n_splits = 5
kf = GroupKFold(n_splits=n_splits)
 score = []
models = []
for i_369,(tdx, vdx)in enumerate(kf.split(X, y, X_train['GameId'])) :
if self.verbose:
print(f'Fold : {i_369}')
 X_train, X_val, y_train, y_val = X.iloc[tdx], X.iloc[vdx], y[tdx], y[vdx]
X_train = [np.absolute(X_train[i])for i in cat] + [X_train[num]]
X_val = [np.absolute(X_val[i])for i in cat] + [X_val[num]]
model = self.model_396_1(X, cat, num)
 model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='val_CRPS',
mode='min',
restore_best_weights=True,
verbose=2,
patience=5)
 es.set_model(model)
 metric = NFL_NN_Metric(model, [es], [(X_train,y_train),(X_val,y_val)])
 for i in range(1):
model.fit(X_train, y_train, verbose=self.verbose)
 for i in range(1):
model.fit(X_train, y_train, batch_size=64, verbose=self.verbose)
 for i in range(1):
model.fit(X_train, y_train, batch_size=128, verbose=self.verbose)
 for i in range(1):
model.fit(X_train, y_train, batch_size=256, verbose=self.verbose)
 model.fit(X_train, y_train, callbacks=[metric], epochs=100, batch_size=1024, verbose=self.verbose)
 score_ = nfl_calc_crps(y_val, model.predict(X_val))
models.append(model)
 score.append(score_)
 if self.verbose:
print(score_)
 self.models_ = models
self.score_ = np.mean(score)
 if self.verbose:
print(np.mean(self.score_))
display(datetime.datetime.now().isoformat())
 return self
def predict(self, X):
X_test = X.copy()
X_test = self.pipeline_pre_predict_.fit_transform(X_test)
 X, _, cat, num = self.prepare_misc(X_test)
 X[num] = self.scaler_.transform(X[num])
 X = [np.absolute(X[i])for i in cat] + [X[num]]
y_pred = np.mean([model.predict(X)for model in self.models_], axis=0)
 return y_pred<data_type_conversions>
alphas = [1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.1, 1, 10]
ridgecv_reg = make_pipeline(RidgeCV(alphas=alphas, cv=kfolds))
ridgecv_reg.fit(X_train_lin, y_train_lin)
 y_pred = ridgecv_reg.predict(X_test_lin)
 print('-' * 10 + 'RidgeCV' + '-' * 10)
 print('R square: ', r2_score(y_test_lin, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test_lin, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test_lin, y_pred))
display(datetime.datetime.now().isoformat() )<drop_column>
lassocv_reg = make_pipeline(LassoCV(alphas=alphas, cv=kfolds))
lassocv_reg.fit(X_train_lin, y_train_lin)
 y_pred = lassocv_reg.predict(X_test_lin)
 print('-' * 10 + 'LassoCV' + '-' * 10)
 print('R square: ', r2_score(y_test_lin, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test_lin, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test_lin, y_pred))
pipeline_fe = make_pipeline(
 NFL_trn_FillNa2() ,
NFL_trn_DataTypesCleaner() ,
NFL_trn_GameWeather() ,
NFL_trn_PlayerAge() ,
NFL_trn_PlayerHeight() ,
NFL_trn_StadiumType() ,
NFL_trn_TeamAbbr() ,
NFL_trn_Turf() ,
NFL_trn_WindDirection() ,
NFL_trn_WindSpeed() ,
NFL_trn_NormalizedMovingX() ,
NFL_trn_PlayerRelativeToBack() ,
NFL_trn_ChangeSameDirection() ,
NFL_trn_Orientation_std() ,
NFL_trn_RunnerDistance2los() ,
NFL_trn_RunnerVs1stDefensorSpeed() ,
NFL_trn_DefenseXspead() ,
NFL_trn_PersonnelSplitter() ,
NFL_trn_DistRB2Defenders() ,
NFL_trn_Orientation() ,
NFL_trn_PlayerRusher() ,
NFL_trn_TimeToTackle() ,
NFL_trn_DistanceToCentroid() ,
NFL_trn_SecondsSinceStart() ,
NFL_trn_DistanceToQB() ,
NFL_trn_Xstd() ,
NFL_trn_TeamStat() ,
NFL_trn_BinsYard() ,
NFL_trn_OffenseLead() ,
NFL_trn_DistDefenders() ,
NFL_trn_SecondsSinceGameStart() ,
NFL_trn_ProcessTwo() , 
)
 if not os.path.isfile('./res/nfl08_train.csv.gz'):
if os.path.isfile('/kaggle/input/masasuke-nfl/nfl08_train.pickle'):
X_train = pd.read_pickle('/kaggle/input/masasuke-nfl/nfl08_train.pickle', compression='gzip')
 else:
X_train = nfl_read_train()
X_train = pipeline_fe.fit_transform(X=X_train)
 if not os.path.isdir('/kaggle/working'):
X_train.to_pickle('./res/nfl08_train.csv.gz')
 else:
X_train = pd.read_pickle('./res/nfl08_train.csv.gz')
 X_train.info(max_cols=X_train.shape[1])
 display(datetime.datetime.now().isoformat() )<statistical_test>
alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008, 0.0009, 0.001]
l1ratio = [0.8, 0.83, 0.85, 0.87, 0.9, 0.92, 0.95, 0.97, 0.99, 1]
elasticv_reg = make_pipeline(ElasticNetCV(alphas=alphas, cv=kfolds, l1_ratio=l1ratio))
elasticv_reg.fit(X_train_lin, y_train_lin)
 y_pred = elasticv_reg.predict(X_test_lin)
 print('-' * 10 + 'ElasticNetCV' + '-' * 10)
 print('R square: ', r2_score(y_test_lin, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test_lin, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test_lin, y_pred))
clf_lgb = Nfl_clf_LightGBM()
clf_nn = Nfl_clf_NN()
eclf = Nfl_clf_EnsembleVoting(estimators=[('nn', clf_nn),('lgb', clf_lgb)] )<train_model>
ensemble = lgb_reg.predict(test_to_model)* 0.2 + xgb_reg.predict(test_to_model)* 0.2 + cat_reg.predict(test_to_model)* 0.2 \
+ gb_reg.predict(test_to_model)* 0.2 + lassocv_reg.predict(test_to_model_lin)* 0.05 + \
ridgecv_reg.predict(test_to_model_lin)* 0.05 + elasticv_reg.predict(test_to_model_lin)* 0.1
eclf.fit(X_train, y=X_train['Yards'])
 display(datetime.datetime.now().isoformat() )<predict_on_test>
import keras
from keras import models, layers, regularizers
env = nflrush.make_env()
for(test_df, sample_prediction_df)in env.iter_test() :
X_test = pipeline_fe.fit_transform(X=test_df)
 y_pred = eclf.predict(X_test)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()
display(datetime.datetime.now().isoformat() )<load_from_url>
model = models.Sequential()
model.add(layers.Dense(256, activation='relu', input_shape=(X_train.shape[1],)))
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dense(1))
optimizer = keras.optimizers.RMSprop(0.001)
 model.compile(optimizer=optimizer,
loss='mae',
metrics=['mae', 'mse'] )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py 
<import_modules>
history = model.fit(X_train, y_train,
epochs=250,
batch_size=5,
validation_data=(X_test, y_test))
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from nltk.corpus import stopwords
import string
import plotly.express as px
from collections import defaultdict
import operator
import re
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import precision_score,recall_score,f1_score
from tensorflow.keras.callbacks import Callback,EarlyStopping
import tensorflow as tf
from tensorflow.keras.layers import Dense,Dropout,BatchNormalization,Input
from tensorflow.keras.models import Model
import tensorflow_addons as tfa
from tensorflow.keras.losses import binary_crossentropy
from tensorflow_addons.optimizers import RectifiedAdam
import tensorflow_hub as hub
import tokenization
from tensorflow.keras.optimizers import Adam,SGD<load_from_csv>
y_pred = model.predict(X_test)
 print('-' * 10 + 'NN' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<create_dataframe>
model = models.Sequential()
model.add(layers.Dense(256, activation='relu', input_shape=(X_train.shape[1],)))
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dense(1))
optimizer = keras.optimizers.RMSprop(0.001)
 model.compile(optimizer=optimizer,
loss='mae',
metrics=['mae', 'mse'] )
missing_values = pd.DataFrame({c:[(train_df[c].isna().sum() /len(train_df)) *100,\ 
(test_df[c].isna().sum() /len(test_df)) *100] for c in \
["keyword","location"]},index=["train","test"])
 missing_values<categorify>
history = model.fit(train_to_model.drop(['SalePrice'], axis=1), train_to_model['SalePrice'],
epochs=300,
batch_size=5,
validation_split=0.05 )
train_df["keyword"].fillna("no_keywords",inplace = True)
 test_df["keyword"].fillna("no_keywords",inplace = True)
 train_df["location"].fillna("no_location",inplace=True)
 test_df["location"].fillna("no_location",inplace=True )<feature_engineering>
y_pred_model = model.predict(test_to_model )
train_df["word_count"] = train_df["text"].map(lambda x: len(str(x ).split()))
test_df["word_count"] = test_df["text"].map(lambda x: len(str(x ).split()))
train_df["unique_word_count"] = train_df["text"].map(lambda x:len(set(str(x ).split())))
 test_df["unique_word_count"] = test_df["text"].map(lambda x:len(set(str(x ).split())))
 train_df["stop_word_count"] = train_df["text"].map(lambda x: len([elt for elt in str(x ).lower().split() \
if elt in stopwords.words("english")]))
test_df["stop_word_count"] = test_df["text"].map(lambda x:len([elt for elt in str(x ).lower().split() \
if elt in stopwords.words("english")]))
train_df["url_count"] = train_df["text"].map(lambda x : len([w for w in str(x ).lower().split() \
if 'http' or 'https' in w]))
test_df["url_count"] = test_df["text"].map(lambda x : len([w for w in str(x ).lower().split() \
if "http" or "https" in w ]))
train_df["mean_word_length"] = train_df["text"].map(lambda x : np.mean([len(word)for word in x.split() ]))
test_df["mean_word_length"] = test_df["text"].map(lambda x : np.mean([len(word)for word in x.split() ]))
train_df["char_count"] = train_df["text"].map(lambda x:len(str(x)))
 test_df["char_count"] = test_df["text"].map(lambda x:len(str(x)))
 train_df["punctuation_count"] = train_df["text"].map(lambda x: len([elt for elt in str(x)if elt in string.punctuation]))
test_df["punctuation_count"] = test_df["text"].map(lambda x:len([elt for elt in str(x)if elt in string.punctuation]))
train_df["hashtag_count"] = train_df["text"].apply(lambda x:len([c for c in str(x)if c=="
test_df["hashtag_count"] = test_df["text"].apply(lambda x:len([c for c in str(x)if c=="
train_df["mention_count"] = train_df["text"].map(lambda x: len([c for c in str(x)if c=="@"]))
test_df["mention_count"] = test_df["text"].map(lambda x: len([c for c in str(x)if c=="@"]))<string_transform>
model_pred = []
for pred in range(len(y_pred_model.tolist())) :
model_pred.append(y_pred_model.tolist() [pred][0] )
def gen_n_grams(text,n_grams=1):

 tokens = [token for token in str(text ).lower().split() if token not in stopwords.words("english")]
ngrams = zip(*[tokens[i:] for i in range(n_grams)])
 return [" ".join(gram)for gram in ngrams]
def gen_df_ngrams(n_grams=1):

 mask = train_df["target"]==1
disaster_unigrams = defaultdict(int)
 non_disaster_unigrams = defaultdict(int)
 for tweet in train_df.loc[mask,"text"].values:
for gram in gen_n_grams(tweet,n_grams=n_grams):
disaster_unigrams[gram] +=1
for tweet in train_df.loc[~mask,"text"].values:
for gram in gen_n_grams(tweet,n_grams=n_grams):
non_disaster_unigrams[gram] +=1
df_disaster_n_grams = pd.DataFrame(sorted(disaster_unigrams.items() ,reverse=True,key=\
lambda item:item[1]))
df_non_disaster_n_grams = pd.DataFrame(sorted(non_disaster_unigrams.items() ,reverse=True,key=\
lambda item:item[1]))
return df_disaster_n_grams,df_non_disaster_n_grams
def plot_ngrams(df_disaster_n_grams,df_non_disaster_unigrams,N=100,n_grams=1):

 fig,ax = plt.subplots(1,2,figsize=(18,50))
sns.barplot(y=df_disaster_n_grams[0].values[:N],x=df_disaster_n_grams[1].values[:N],ax=ax[0],\
color="red")
 for i in range(2):
ax[i].tick_params(axis="x",labelsize=15)
 ax[i].tick_params(axis="y",labelsize=15)
 ax[i].set_xlabel("Occurences")
 ax[i].spines["right"].set_visible(False)
 sns.barplot(y=df_non_disaster_unigrams[0].values[:N],x=df_non_disaster_unigrams[1].values[:N],\
ax=ax[1],color="green")
 ax[0].set_title("Top most {} {}_grams for disaster tweets".format(N,n_grams),size=15)
 ax[1].set_title("Top most {} {}_grams for non disaster tweets".format(N,n_grams),size=15 )<load_pretrained>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = model_pred
sub.to_csv('submission_NN.csv',index=False )
%%time
glove_embeddings = np.load('.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', allow_pickle=True)
 fasttext_embeddings = np.load('.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl', allow_pickle=True )<string_transform>
model_pred = np.array(model_pred)
 model_pred
def build_vocab(X):

 tweets = X.apply(lambda x : x.split() ).values
vocab = {}
for tweet in tweets:
for word in tweet:
try:
vocab[word] +=1
except KeyError:
vocab[word] = 1
return vocab<categorify>
final_blend = ensemble * 0.3 + model_pred * 0.7
def check_embedding_coverage(X,embedding):

 vocab = build_vocab(X)
 covered = {}
oov ={}
n_covered = 0
n_oov = 0
for word in vocab :
try:
covered[word] = embedding[word]
n_covered += vocab[word]
except:
oov[word] = vocab[word]
n_oov += vocab[word]
coverage = len(covered)/ len(vocab)
 text_coverage = n_covered /(n_covered + n_oov)
 oov = sorted(oov.items() ,key = operator.itemgetter(1),reverse = True)
 return oov,coverage,text_coverage<categorify>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN7.csv',index=False )
train_glove_oov,train_glove_coverage,train_glove_text = check_embedding_coverage(train_df["text"],glove_embeddings)
 test_glove_oov,test_glove_coverage,test_glove_text = check_embedding_coverage(test_df["text"],glove_embeddings)
 print("Glove embedding cover {} of vocabulary and {} of text in the training dataset".format(train_glove_coverage,train_glove_text))
print("Glove embedding cover  {} of vocabularly and {} of text in the test dataset".format(test_glove_coverage,test_glove_text))<categorify>
final_blend = ensemble * 0.4 + model_pred * 0.6
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN6.csv',index=False )
train_fastext_oov,train_fastext_coverage,train_fastext_text = check_embedding_coverage(train_df["text"],fasttext_embeddings)
 test_fastext_oov,test_fastext_coverage,test_fastext_text = check_embedding_coverage(test_df["text"],fasttext_embeddings)
 print("FastText embedding cover {} of vocabulary and {} of text in the training dataset".format(train_fastext_coverage,train_fastext_text))
print("FastText embedding cover {} of vocabularly and {} of text in the test dataset".format(test_fastext_coverage,test_fastext_text))<categorify>
final_blend = ensemble * 0.5 + model_pred * 0.5
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN5.csv',index=False )
def clean(tweet):
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 tweet = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", tweet)
 punctuations = '@
for p in punctuations:
tweet = tweet.replace(p, f' {p} ')
 tweet = tweet.replace('...', '...')
 if '...' not in tweet:
tweet = tweet.replace('.. ', '...')
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 return tweet<feature_engineering>
final_blend = ensemble * 0.6 + model_pred * 0.4
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN4.csv',index=False )
<categorify><EOS> 
final_blend = ensemble * 0.2 + model_pred * 0.8
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN8.csv',index=False )