Kaggle competition: House Prices - Advanced Regression Techniques
all_stopwords = set(stopwords.words('english'))<count_values>
warnings.filterwarnings("ignore")
 %matplotlib inline
def stopword_count(data, column):
count_dict = dict.fromkeys(all_stopwords, 0)
 def row_count(row):
for word in row.split() :
if word in all_stopwords:
count_dict[str(word)] += 1
data[column].apply(lambda x: row_count(x))
return count_dict<sort_values>
train = train.drop(train[(train.GrLivArea > 4000)&(train.SalePrice < 300000)].index )
stopwords = stopword_count(train, 'text')
 stopwords = pd.Series(stopwords, index=stopwords.keys())
 stopwords = stopwords[stopwords > 0]
stopwords = stopwords.sort_values(ascending=False )<string_transform>
train_ID = train["Id"]
test_ID = test["Id"]
train.drop("Id", axis=1, inplace=True)
 test.drop("Id", axis=1, inplace=True )
train['text'] = train['text'].apply(lambda x: ' '.join([word for word in x.split() if word not in(all_stopwords)]))
test['text'] = test['text'].apply(lambda x: ' '.join([word for word in x.split() if word not in(all_stopwords)]))<feature_engineering>
train.SalePrice = np.log1p(train.SalePrice )
eng_words = set(nltk.corpus.words.words())
 train['text'] = train['text'].apply(lambda x: ' '.join(w for w in x.split() if not any(j.isdigit() for j in w)))
 test['text'] = test['text'].apply(lambda x: ' '.join(w for w in x.split() if not any(j.isdigit() for j in w)) )<categorify>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
vect = CountVectorizer(min_df=3, ngram_range=(1,1))
enc = OneHotEncoder()
full_data = pd.concat(( train, test))
enc.fit(full_data[['location', 'keyword']])
 vect.fit(full_data['text'])
 print(len(vect.get_feature_names()))
print(len(enc.get_feature_names()))<categorify>
total_na = all_data.isnull().sum()
total_na = total_na.drop(total_na[total_na == 0].index ).sort_values(ascending=False)
 missing = pd.DataFrame({"Total Missing data": total_na})
 missing.head(20 )
np.random.seed(15)
 shuffled_train = train.iloc[np.random.permutation(len(train)) ]
shuffled_test = test.iloc[np.random.permutation(len(test)) ]
n_estimators = [60, 80, 100]
X_vect = vect.transform(shuffled_train['text'] ).todense()
X_onehot = enc.transform(shuffled_train[['location', 'keyword']] ).todense()
X = np.concatenate(( X_vect, X_onehot), axis=1)
 Y = shuffled_train['target']
for n_est in n_estimators:
clf = RandomForestClassifier(n_estimators=n_est, max_features='sqrt', verbose=0, n_jobs=-1)
 score = cross_val_score(estimator=clf, X=X, y=Y, scoring='f1', cv=3)
 print('f1 scores from 3 validations')
 print(score)
 print('
' )<categorify>
all_data.loc[:, "Alley"] = all_data.loc[:, "Alley"].fillna("NoAl")
 all_data.loc[:, "BedroomAbvGr"] = all_data.loc[:, "BedroomAbvGr"].fillna(0)
 all_data.loc[:, "BsmtQual"] = all_data.loc[:, "BsmtQual"].fillna("NoBa")
 all_data.loc[:, "BsmtCond"] = all_data.loc[:, "BsmtCond"].fillna("NoBa")
 all_data.loc[:, "BsmtExposure"] = all_data.loc[:, "BsmtExposure"].fillna("NoBa")
 all_data.loc[:, "BsmtFinType1"] = all_data.loc[:, "BsmtFinType1"].fillna("NoBa")
 all_data.loc[:, "BsmtFinType2"] = all_data.loc[:, "BsmtFinType2"].fillna("NoBa")
 all_data.loc[:, "Electrical"] = all_data.loc[:, "Electrical"].fillna("NoEL")
 all_data.loc[:, "Fence"] = all_data.loc[:, "Fence"].fillna("NoFe")
 all_data.loc[:, "FireplaceQu"] = all_data.loc[:, "FireplaceQu"].fillna("NoFi")
 all_data.loc[:, "GarageType"] = all_data.loc[:, "GarageType"].fillna("NoGa")
 all_data.loc[:, "GarageFinish"] = all_data.loc[:, "GarageFinish"].fillna("NoGa")
 all_data.loc[:, "GarageQual"] = all_data.loc[:, "GarageQual"].fillna("NoGa")
 all_data.loc[:, "GarageCond"] = all_data.loc[:, "GarageCond"].fillna("NoGa")
 all_data.loc[:, "MasVnrType"] = all_data.loc[:, "MasVnrType"].fillna("None")
 all_data.loc[:, "MasVnrArea"] = all_data.loc[:, "MasVnrArea"].fillna(0)
 all_data.loc[:, "MiscFeature"] = all_data.loc[:, "MiscFeature"].fillna("NoFe")
 all_data.loc[:, "PoolQC"] = all_data.loc[:, "PoolQC"].fillna("NoPo")
vect = TfidfVectorizer(min_df=3, ngram_range=(1,1))
enc = OneHotEncoder()
full_data = pd.concat(( train, test))
enc.fit(full_data[['location', 'keyword']])
 vect.fit(full_data['text'])
 print(len(vect.get_feature_names()))
print(len(enc.get_feature_names()))<categorify>
all_data = all_data.replace({"MSSubClass" : {20 : "SC20", 30 : "SC30", 40 : "SC40", 45 : "SC45",
50 : "SC50", 60 : "SC60", 70 : "SC70", 75 : "SC75",
80 : "SC80", 85 : "SC85", 90 : "SC90", 120 : "SC120",
150 : "SC150", 160 : "SC160", 180 : "SC180", 190 : "SC190"},
"MoSold" : {1 : "Jan", 2 : "Feb", 3 : "Mar", 4 : "Apr",
5 : "May", 6 : "Jun", 7 : "Jul", 8 : "Aug",
9 : "Sep", 10 : "Oct", 11 : "Nov", 12 : "Dec"},
})
 all_data.OverallCond = all_data.OverallCond.astype(str)
 all_data.YrSold = all_data.YrSold.astype(str )
np.random.seed(15)
 shuffled_train = train.iloc[np.random.permutation(len(train)) ]
clf = BernoulliNB(fit_prior = False)
 X_vect = vect.transform(shuffled_train['text'] ).todense()
X_onehot = enc.transform(shuffled_train[['location', 'keyword']] ).todense()
X = np.concatenate(( X_vect, X_onehot), axis=1)
 Y = shuffled_train['target']
X_sparsed = sparse.csr_matrix(X)
 score = cross_val_score(estimator=clf, X=X_sparsed, y=Y, scoring='f1', cv=5)
 print('Best scores from 3 cross validations: ')
 print(score )<train_on_grid>
all_data = all_data.replace({
"BsmtCond": {"NoBa": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4,
"Ex": 5},
"BsmtExposure": {"NoBa": 0, "Mn": 1, "Av": 2, "Gd": 3},
"BsmtFinType1" : {"NoBa" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"NoBa" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"NoBa" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4,
"Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"NoGa" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"NoGa" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PoolQC" : {"NoPo" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}
} )
np.random.seed(15)
 shuffled_train = train.iloc[np.random.permutation(len(train)) ]
shuffled_test = test.iloc[np.random.permutation(len(test)) ]
max_iter = 400
X_vect = vect.transform(shuffled_train['text'] ).todense()
X_onehot = enc.transform(shuffled_train[['location', 'keyword']] ).todense()
X = np.concatenate(( X_vect, X_onehot), axis=1)
 Y = shuffled_train['target']
X_sparsed = sparse.csr_matrix(X)
 clf = LogisticRegressionCV(n_jobs = -1, max_iter = max_iter, scoring='f1')
 score = cross_val_score(estimator=clf, X=X_sparsed, y=Y, scoring='f1', cv=5)
 print('scores from 5 cross validations: ')
 print(score)
 print('
' )<prepare_x_and_y>
print("NAs for numerical features in train : " + str(all_data_num.isnull().values.sum()))
all_data_num = all_data_num.fillna(all_data_num.mean())
 print("Remaining NAs for numerical features in train : " + str(all_data_num.isnull().values.sum()))
np.random.seed(15)
 shuffled_train = train.iloc[np.random.permutation(len(train)) ]
shuffled_test = test.iloc[np.random.permutation(len(test)) ]
max_iter = 70000
X_vect = vect.transform(shuffled_train['text'] ).todense()
X_onehot = enc.transform(shuffled_train[['location', 'keyword']] ).todense()
X = np.concatenate(( X_vect, X_onehot), axis=1)
 Y = shuffled_train['target']
X_sparsed = sparse.csr_matrix(X)
 kernels = ['linear', 'rbf', 'poly']
for kernel in kernels:
clf = SVC(max_iter = max_iter, kernel=kernel)
 score = cross_val_score(estimator=clf, X=X_sparsed, y=Y, scoring='f1', cv=5)
 print(kernel)
 print(score)
 print('
' )<import_modules>
skewness = all_data_num.apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.5]
print(str(skewness.shape[0])+ " skewed numerical features to log transform")
 skewed_features = skewness.index
all_data_num[skewed_features] = np.log1p(all_data_num[skewed_features] )
from datetime import datetime<categorify>
print("NAs for categorical features in train : " + str(all_data_cat.isnull().values.sum()))
all_data_cat = pd.get_dummies(all_data_cat)
 print("Remaining NAs for categorical features in train : " + str(all_data_cat.isnull().values.sum()))
vect = CountVectorizer(min_df=3, ngram_range=(1,1))
enc = OneHotEncoder()
full_data = pd.concat(( train, test))
enc.fit(full_data[['location', 'keyword']])
 vect.fit(full_data['text'])
 print(len(vect.get_feature_names()))
print(len(enc.get_feature_names()))<categorify>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
max_iter = 400
X_vect_train = vect.transform(train['text'] ).todense()
X_onehot_train = enc.transform(train[['location', 'keyword']] ).todense()
X_train = np.concatenate(( X_vect_train, X_onehot_train), axis=1)
 Y_train = train['target']
X_vect_test = vect.transform(test['text'] ).todense()
X_onehot_test = enc.transform(test[['location', 'keyword']] ).todense()
X_test = np.concatenate(( X_vect_test, X_onehot_test), axis=1)
 X_sparsed_train = sparse.csr_matrix(X_train)
 X_sparsed_test = sparse.csr_matrix(X_test)
 start = datetime.now()
clf = LogisticRegressionCV(n_jobs = -1, max_iter = max_iter, scoring='f1')
 clf.fit(X_sparsed_train, Y_train)
 print('Training time')
 print(datetime.now() - start)
 start = datetime.now()
LogReg_predict = clf.predict_proba(X_sparsed_test)
 LogReg_predict = LogReg_predict[:, 1]
print('Predicting time')
 print(datetime.now() - start )<categorify>
train = all_data[:ntrain]
test = all_data[ntrain:]
X_vect_train = vect.transform(train['text'] ).todense()
X_onehot_train = enc.transform(train[['location', 'keyword']] ).todense()
X_train = np.concatenate(( X_vect_train, X_onehot_train), axis=1)
 Y_train = train['target']
X_vect_test = vect.transform(test['text'] ).todense()
X_onehot_test = enc.transform(test[['location', 'keyword']] ).todense()
X_test = np.concatenate(( X_vect_test, X_onehot_test), axis=1)
 X_sparsed_train = sparse.csr_matrix(X_train)
 X_sparsed_test = sparse.csr_matrix(X_test)
 start = datetime.now()
clf = BernoulliNB(fit_prior = False)
 clf.fit(X_sparsed_train, Y_train)
 print('Training time')
 print(datetime.now() - start)
 start = datetime.now()
NaiveBayes_predict = clf.predict_proba(X_sparsed_test)
 NaiveBayes_predict = NaiveBayes_predict[:, 1]
print('Predicting time')
 print(datetime.now() - start )<categorify>
all_data = pd.concat([all_data_num, all_data_cat], axis=1)
 print("New number of features : " + str(all_data.shape[1]))
train = all_data[:ntrain]
test = all_data[ntrain:]
print(len(train))
print(len(y_train))
max_iter = 70000
X_vect_train = vect.transform(train['text'] ).todense()
X_onehot_train = enc.transform(train[['location', 'keyword']] ).todense()
X_train = np.concatenate(( X_vect_train, X_onehot_train), axis=1)
 Y_train = train['target']
X_vect_test = vect.transform(test['text'] ).todense()
X_onehot_test = enc.transform(test[['location', 'keyword']] ).todense()
X_test = np.concatenate(( X_vect_test, X_onehot_test), axis=1)
 X_sparsed_train = sparse.csr_matrix(X_train)
 X_sparsed_test = sparse.csr_matrix(X_test)
 start = datetime.now()
clf = SVC(max_iter = max_iter, kernel='linear')
 clf.fit(X_sparsed_train, Y_train)
 print('Training time')
 print(datetime.now() - start)
 start = datetime.now()
SVM_predict = clf.predict(X_sparsed_test)
 print('Predicting time')
 print(datetime.now() - start )<categorify>
scorer = make_scorer(mean_squared_error, greater_is_better=False)
 def rmse_cv_train(model):
rmse = np.sqrt(-cross_val_score(model, train.values, y_train, scoring=scorer, cv=10))
return(rmse )
max_iter = 80
X_vect_train = vect.transform(train['text'] ).todense()
X_onehot_train = enc.transform(train[['location', 'keyword']] ).todense()
X_train = np.concatenate(( X_vect_train, X_onehot_train), axis=1)
 Y_train = train['target']
X_vect_test = vect.transform(test['text'] ).todense()
X_onehot_test = enc.transform(test[['location', 'keyword']] ).todense()
X_test = np.concatenate(( X_vect_test, X_onehot_test), axis=1)
 start = datetime.now()
clf = RandomForestClassifier(n_estimators=max_iter, max_features='sqrt', verbose=0, n_jobs=-1)
 clf.fit(X_train, Y_train)
 print('Training time')
 print(datetime.now() - start)
 start = datetime.now()
RF_predict = clf.predict_proba(X_test)
 RF_predict = RF_predict[:, 1]
print('Predicting time')
 print(datetime.now() - start )<compute_test_metric>
l1 = [0.00001, 0.00003, 0.00006, 0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6, 10, 30, 60, 100]
l2 = np.arange(100)+ 1
l3 = l2 * 0.1
l4 = l2 * 0.01
l5 = l2 * 0.001
l6 = l2 * 0.0001
l7 = l2 * 0.00001
l8 = l2 * 0.000001
print(l3 )
predict =(LogReg_predict + NaiveBayes_predict + RF_predict)/ 3<feature_engineering>
l2_1 = np.arange(1000)+ 1
l3_1 = l2_1 * 0.1
l4_1 = l2_1 * 0.01
predict[predict >= 0.5] = int(1)
 predict[predict < 0.5] = int(0 )<save_to_csv>
ridge = RidgeCV(alphas = [5.63])
 ridge.fit(train.values, y_train)
 alpha = ridge.alpha_
print("Best alpha: ", alpha)
 print("Ridge RMSE on Training set :", rmse_cv_train(ridge ).mean() )
result_df = pd.DataFrame(test['id'].values, columns=['id'])
 result_df['target'] = predict.astype(int)
 result_df.to_csv(path_or_buf=r'submission_Ensamble.csv', index=False )<save_to_csv>
lasso = LassoCV(alphas = [0.00031], max_iter=50000, cv=10)
 lasso.fit(train.values, y_train)
 print("Lasso RMSE on Training set: ", rmse_cv_train(lasso ).mean() )
for name, predict in [('LogReg', LogReg_predict), 
('NaiveBayes', NaiveBayes_predict),('RF', RF_predict),('SVM', SVM_predict)]:
predict[predict >= 0.5] = int(1)
 predict[predict < 0.5] = int(0)
 result_df = pd.DataFrame(test['id'].values, columns=['id'])
 result_df['target'] = predict.astype(int)
 result_df.to_csv(path_or_buf=r'submission_' + name + '.csv', index=False )<define_variables>
elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
 elasticNet.fit(train.values, y_train)
 alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("Try again for more precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(l1_ratio = [ratio *.85, ratio *.9, ratio *.95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
 elasticNet.fit(train.values, y_train)
 if(elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio)+
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio = ratio,
alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8, alpha *.85, alpha *.9,
alpha *.95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter = 50000, cv = 10)
 elasticNet.fit(train.values, y_train)
 if(elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("ElasticNet RMSE on Training set :", rmse_cv_train(elasticNet ).mean())
 y_train_ela = elasticNet.predict(train.values )
fname = "/kaggle/input/nfl-big-data-bowl-2020/train.csv" 
<categorify>
KRR = KernelRidge(alpha = [600], kernel="polynomial", degree=1.94, coef0=40)
 KRR.fit(train.values, y_train)
 print("Kernel Ridge Regression RMSE on Training set :", rmse_cv_train(KRR ).mean() )
def lookup(s):

 dates = {date:pd.to_datetime(date)for date in s.unique() }
return s.map(dates)
 def ageNow(d):
now = datetime.datetime.now()
difference = relativedelta(now, d)
 return(difference.years)
<train_model>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 GBoost.fit(train.values, y_train)
 print("XGB RMSE on Training set: ", rmse_cv_train(GBoost ).mean() )
clf = []
def randomForest(dfIn,cols):
global clf
np.random.seed(0)
 print("Starting 1")
 clf = RandomForestClassifier(n_jobs=1, random_state=0,n_estimators=10)
 training_df = dfIn[cols][::22]
y=training_df['Yards']
X=training_df[[i for i in training_df.columns if "Yards" not in i]]
clf.fit(X, y)
 print("Trained")
 def modifyXY(dfx):
dfIn = dfx
print('Doing ModifyXY')
 dfIn['XM'] = dfIn['X'] - 10
dfIn['XS'] = dfIn['AbsoluteYardLine']
T = 0.5
print('Doing ModifyXY ',T)
 dfIn['S05'] = dfIn['S'] + dfIn['A'] * T
dfIn['X05'] = dfIn[['X','S','A','Orientation']].apply(lambda x: x['X'] + np.sin(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 dfIn['X05'] = dfIn[['Y','S','A','Orientation']].apply(lambda x: x['Y'] + np.cos(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 T = 1
print('Doing ModifyXY',T)
 dfIn['S10'] = dfIn['S'] + dfIn['A'] * T
dfIn['Y10'] = dfIn[['X','S','A','Orientation']].apply(lambda x: x['X'] + np.sin(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 dfIn['Y10'] = dfIn[['Y','S','A','Orientation']].apply(lambda x: x['Y'] + np.cos(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 print('Done ModifyXY')
 print(dfIn.columns)
 dfOut = dfIn
return dfOut
def isRequired(strIn):
global flat_list,isTraining
str2 = ".*"+strIn+".*"
if(isTraining):
print("--> isRequired? ",str2,strIn,flat_list)
 r = re.compile(str2)
 l = list(filter(r.match, flat_list))
rc = len(l)
 if rc > 0:
if(isTraining):
print("Field Required = ",strIn)
 return rc
def addFeatures(dfy):
global flatlist,isTraining
dfIn = dfy
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in dfIn['PossessionTeam'].unique() :
map_abbr[abb] = abb
dfIn['PossessionTeam'] = dfIn['PossessionTeam'].map(map_abbr)
 dfIn['HomeTeamAbbr'] = dfIn['HomeTeamAbbr'].map(map_abbr)
 dfIn['VisitorTeamAbbr'] = dfIn['VisitorTeamAbbr'].map(map_abbr)
 dfIn['AbsoluteYardLine'] = dfIn[['YardLine','PlayDirection','HomeTeamAbbr','PossessionTeam','FieldPosition']].apply(lambda x: int(( 100-x['YardLine'])) if x['PossessionTeam'] == x['FieldPosition'] else int(int(x['YardLine'])) ,axis=1)
 dfIn['PlayerIsRusher'] = dfIn['NflIdRusher'] == dfIn['NflId']
if isRequired("XS"):
dfIn = modifyXY(dfIn)
 if isRequired("PlayerWeight"):
dfIn['PlayerWeightSumDefense'] = dfIn['PlayerWeight']
dfIn['PlayerWeightSumOffense'] = dfIn['PlayerWeight']
dfIn.loc[(dfIn['PossessionTeam'] == dfIn['VisitorTeamAbbr'])&(dfIn['Team'] == 'away'), 'PlayerWeightSumDefense'] = 0
dfIn.loc[(dfIn['PossessionTeam'] == dfIn['HomeTeamAbbr'])&(dfIn['Team'] == 'home'), 'PlayerWeightSumDefense'] = 0
dfIn.loc[(dfIn['PossessionTeam'] != dfIn['VisitorTeamAbbr'])&(dfIn['Team'] == 'away'), 'PlayerWeightSumOffense'] = 0
dfIn.loc[(dfIn['PossessionTeam'] != dfIn['HomeTeamAbbr'])&(dfIn['Team'] == 'home'), 'PlayerWeightSumOffense'] = 0
w = dfIn.groupby('PlayId')['PlayerWeightSumDefense','PlayerWeightSumOffense'].sum()
w.rename(columns={'PlayerWeightSumDefense' : 'PlayerWeightSumDefenseSum', 'PlayerWeightSumOffense' : 'PlayerWeightSumOffenseSum'},inplace=True)
 w = w.reset_index()
dfIn.drop(['PlayerWeightSumDefense','PlayerWeightSumOffense'],axis=1,inplace=True)
 dfIn = pd.merge(dfIn,w,how='left',left_on='PlayId',right_on='PlayId')
 dfIn['PlayerWeightSumDiff'] = dfIn['PlayerWeightSumDefenseSum'] - dfIn['PlayerWeightSumOffenseSum']
dfIn2 = dfIn
if isRequired("PlayerBirthDate"): dfIn2['PlayerBirthDate'] = lookup(dfIn2['PlayerBirthDate'])
 if isRequired("PlayerAge"): dfIn2['PlayerAge'] = dfIn2['PlayerBirthDate'].apply(lambda x: ageNow(x))
if isRequired("PlayerYearsInNFL"): dfIn2['PlayerYearsInNFL'] = dfIn2['PlayerAge'] - 22
dfIn2['GameClock2'] = dfIn2['GameClock'].apply(lambda x: int(x[0:2])+int(x[3:5])/60)
 dfIn2['AbsoluteGameClock'] = dfIn2[['GameClock','Quarter']].apply(lambda x: int(x['GameClock'][0:2])+float(x['GameClock'][3:5])/60+(( 4-x['Quarter'])*15),axis=1)
 dfIn2['AbsoluteGameClockRemaining'] = 60 - dfIn2['AbsoluteGameClock']
dfOut = dfIn2
return dfOut<predict_on_test>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([model.predict(X)for model in self.models_])
 return np.mean(predictions, axis=1 )
def predict_using_my_model_RandomForest(dfIn,sample_prediction_df):
global clfs
global dfME
global colsSet
test_dfIn = dfIn[dfIn['NflIdRusher'] == dfIn['NflId']].copy()
test_dfIn = addFeatures(test_dfIn)
 df3 = pd.DataFrame()
c = 0
AbsoluteYardLine = test_dfIn['AbsoluteYardLine'].values[0]
minYards = - AbsoluteYardLine
maxYards = 100 - AbsoluteYardLine
for cols in colsSet:
clf = clfs[c]
c = c + 1
y_df = pd.DataFrame(list(zip(clf.classes_,[x for x in clf.predict_proba(test_dfIn[cols])[0]])))
 i = range(-99,100)
 missingyards = set(i ).difference(sorted(y))
y_df = y_df.append(pd.DataFrame(list(zip(missingyards,[0]*len(missingyards)))))
 y_df.columns = ['Yards','probablity']
y_df = y_df.sort_values('Yards')
 df1 = pd.DataFrame(y_df.set_index('Yards' ).reset_index() ['probablity'] ).T
df1.columns = ["Yards"+str(x)for x in range(-99,100)]
df2 = df1.cumsum(axis=1)
 df3 = pd.concat([df3,df2])
 df4 = pd.DataFrame(df3.sum() ).T
sample_prediction_df = df4.div(df4.max(axis=1), axis=0)
 assert sample_prediction_df['Yards99'].values[0] == 1.0, "Values is not 1.0"
return sample_prediction_df 
<train_model>
averaged_models = AveragingModels(models =(lasso, elasticNet, KRR, GBoost))
print("Averaged Models RMSE on Training set :", rmse_cv_train(averaged_models ).mean())
isTraining = True
os.system(f'echo Training')
 clf = 0
colsSet = [
['Down','JerseyNumber']
]
i = 0
flat_list = list(set([item for sublist in colsSet for item in sublist]))
np.random.seed(0)
 print("Starting 1")
 train = True
if train == True:
filename = fname
dfIn = pd.read_csv(filename, low_memory=False)
 train_df = dfIn
test_df = dfIn
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train_df['PossessionTeam'].unique() :
map_abbr[abb] = abb
train_df['PossessionTeam'] = train_df['PossessionTeam'].map(map_abbr)
 train_df['HomeTeamAbbr'] = train_df['HomeTeamAbbr'].map(map_abbr)
 train_df['VisitorTeamAbbr'] = train_df['VisitorTeamAbbr'].map(map_abbr)
 train_df = addFeatures(train_df)
 isTraining = False
print('Done adding to train_df')
<train_model>
def rmse(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
i = 0
clfs = []
for cols in colsSet:
clf = RandomForestClassifier(n_jobs=1, random_state=0,n_estimators=10)
 Y = ['Yards']
training_df = train_df[train_df['PlayerIsRusher'] == True][cols+Y].copy()
y=training_df['Yards']
X=training_df[[i for i in training_df.columns if "Yards" not in i]]
clf = RandomForestClassifier(n_jobs=2, random_state=0,n_estimators=100)
 print("Training")
 os.system(f'echo training')
 clf.fit(X, y)
 print("Trained")
 os.system(f'echo trained')
 clfs.append(clf)
<predict_on_test>
averaged_models.fit(train.values, y_train)
 averaged_train_pred = averaged_models.predict(train.values)
 averaged_pred = np.expm1(averaged_models.predict(test.values))
print(rmse(y_train, averaged_train_pred))
env = nflrush.make_env()
i = 0
for(test_df, sample_prediction_df)in env.iter_test() :
dfPredict = predict_using_my_model_RandomForest(test_df,sample_prediction_df)
 os.system(f'echo {i} ')
 i = i + 1
env.predict(dfPredict)
 env.write_submission_file() 
<save_to_csv>
sub = pd.DataFrame()
sub["Id"] = test_ID
sub["SalePrice"] = averaged_pred
sub.to_csv("submission2.csv", index=False )