Kaggle competition: NFL Big Data Bowl
from IPython.display import HTML
from matplotlib.pyplot import *
from matplotlib import cm
from matplotlib import animation
import io
import base64 
<string_transform>
metric = "multi_logloss"
param = {'num_leaves': 50,
'min_data_in_leaf': 30,
'objective':'multiclass',
'num_class': 199,
'max_depth': 6,
'learning_rate': 0.01,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.4,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": metric,
"lambda_l1": 0.1,
"verbosity": -1,
'n_jobs': -1,
"seed":1234}
def unique_word_fraction(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
unique_count = list(set(text_splited)).__len__()
return(unique_count/word_count)
 eng_stopwords = set(stopwords.words("english"))
def stopwords_count(row):

 text = row['text'].lower()
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
stopwords_count = len([w for w in text_splited if w in eng_stopwords])
 return(stopwords_count/word_count)
 def punctuations_fraction(row):

 text = row['text']
char_count = len(text)
 punctuation_count = len([c for c in text if c in string.punctuation])
 return(punctuation_count/char_count)
 def char_count(row):

 return len(row['text'])
 def fraction_noun(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
pos_list = nltk.pos_tag(text_splited)
 noun_count = len([w for w in pos_list if w[1] in('NN','NNP','NNPS','NNS')])
 return(noun_count/word_count)
 def fraction_adj(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
pos_list = nltk.pos_tag(text_splited)
 adj_count = len([w for w in pos_list if w[1] in('JJ','JJR','JJS')])
 return(adj_count/word_count)
 def fraction_verbs(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
pos_list = nltk.pos_tag(text_splited)
 verbs_count = len([w for w in pos_list if w[1] in('VB','VBD','VBG','VBN','VBP','VBZ')])
 return(verbs_count/word_count)
<feature_engineering>
def get_lgbm(x_tr, y_tr, x_val, y_val, shape):
y_valid = y_val
y_tr = np.argmax(y_tr, axis=1)
 y_val = np.argmax(y_val, axis=1)
 trn_data = lgb.Dataset(x_tr, label=y_tr, categorical_feature=cat_feats)
 val_data = lgb.Dataset(x_val, label=y_val, categorical_feature=cat_feats)
 model = lgb.train(param, trn_data, 10000, valid_sets = [val_data], verbose_eval = 100, early_stopping_rounds = 200)
 y_pred = model.predict(x_val, num_iteration=model.best_iteration)
 crps = crps_score(y_pred, y_valid, shape=shape)
 return model, crps
train_df['unique_word_fraction'] = train_df.apply(lambda row: unique_word_fraction(row), axis =1)
 train_df['stopwords_count'] = train_df.apply(lambda row: stopwords_count(row), axis =1)
 train_df['punctuations_fraction'] = train_df.apply(lambda row: punctuations_fraction(row), axis =1)
 train_df['char_count'] = train_df.apply(lambda row: char_count(row), axis =1)
 train_df['fraction_noun'] = train_df.apply(lambda row: fraction_noun(row), axis =1)
 train_df['fraction_adj'] = train_df.apply(lambda row: fraction_adj(row), axis =1)
 train_df['fraction_verbs'] = train_df.apply(lambda row: fraction_verbs(row), axis =1)
 train_df.head()<feature_engineering>
def get_rf(x_tr, y_tr, x_val, y_val, shape):
model = RandomForestRegressor(bootstrap=False, max_features=0.3, min_samples_leaf=15,
min_samples_split=8, n_estimators=50, n_jobs=-1, random_state=42)
 model.fit(x_tr, y_tr)
 y_pred = model.predict(x_val)
 y_valid = y_val
crps = crps_score(y_pred, y_valid, shape=shape)
 return model, crps
test_df['unique_word_fraction'] = test_df.apply(lambda row: unique_word_fraction(row), axis =1)
 test_df['stopwords_count'] = test_df.apply(lambda row: stopwords_count(row), axis =1)
 test_df['punctuations_fraction'] = test_df.apply(lambda row: punctuations_fraction(row), axis =1)
 test_df['char_count'] = test_df.apply(lambda row: char_count(row), axis =1)
 test_df['fraction_noun'] = test_df.apply(lambda row: fraction_noun(row), axis =1)
 test_df['fraction_adj'] = test_df.apply(lambda row: fraction_adj(row), axis =1)
 test_df['fraction_verbs'] = test_df.apply(lambda row: fraction_verbs(row), axis =1)
 test_df.head()<define_variables>
loop = 2
fold = 5
oof_nn = np.zeros([loop, y.shape[0], y.shape[1]])
 oof_lgbm = np.zeros([loop, y.shape[0], y.shape[1]])
 oof_rf = np.zeros([loop, y.shape[0], y.shape[1]])
 models_nn = []
crps_csv_nn = []
models_lgbm = []
crps_csv_lgbm = []
models_rf = []
crps_csv_rf = []
feature_importance = np.zeros([loop, fold, X.shape[1]])
 s_time = time.time()
for k in range(loop):
kfold = KFold(fold, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print(f'Loop {k+1}/{loop}' + f' Fold {k_fold+1}/{fold}')
 print("-----------")
 tr_x, tr_y = X.loc[tr_inds], y[tr_inds]
val_x, val_y = X.loc[val_inds], y[val_inds]
nn, crps_nn = get_nn(tr_x, tr_y, val_x, val_y, shape=val_x.shape[0])
 models_nn.append(nn)
 print("the %d fold crps(NN)is %f"%(( k_fold+1), crps_nn))
crps_csv_nn.append(crps_nn)
 lgbm, crps_lgbm = get_lgbm(tr_x, tr_y, val_x, val_y, shape=val_x.shape[0])
 models_lgbm.append(lgbm)
 print("the %d fold crps(LGBM)is %f"%(( k_fold+1), crps_lgbm))
crps_csv_lgbm.append(crps_lgbm)
 feature_importance[k, k_fold, :] = lgbm.feature_importance()
oof_nn[k, val_inds, :] = nn.predict(val_x)
 oof_lgbm[k, val_inds, :] = lgbm.predict(val_x, num_iteration=lgbm.best_iteration)
Surprize_words = ['finally', 'suddenly', 'mystery', 'fortune', 'sweet', 'horror', 'chance', '']<categorify>
a = lgbm.predict(val_x, num_iteration=lgbm.best_iteration )
start = time.time()
tfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,3))
full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 end = time.time()
print("Time taken in tf-idf is {}.".format(end-start))
cols_to_drop = ['id', 'text']
train_X = train_df.drop(cols_to_drop+['author'], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
train_y = train_df['author'].map(author_mapping_dict )<find_best_model_class>
crps_oof_nn = []
crps_oof_lgbm = []
crps_oof_rf = []
for k in range(loop):
crps_oof_nn.append(crps_score(oof_nn[k,...], y))
crps_oof_lgbm.append(crps_score(oof_lgbm[k,...], y))
def runMNB(train_X, train_y, test_X, test_y, test_X2):
model = naive_bayes.MultinomialNB()
model.fit(train_X, train_y)
 pred_test_y = model.predict_proba(test_X)
 pred_test_y2 = model.predict_proba(test_X2)
 return pred_test_y, pred_test_y2, model
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]<prepare_x_and_y>
print("mean crps(NN)is %f"%np.mean(crps_csv_nn))
print("mean crps(LGBM)is %f"%np.mean(crps_csv_lgbm))
tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,3))
tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]<concatenate>
print("mean OOF crps(NN)is %f"%np.mean(crps_oof_nn))
print("mean OOF crps(LGBM)is %f"%np.mean(crps_oof_lgbm))
start = time.time()
n_comp = 400
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
end = time.time()
print("Time taken in tf-idf is {}.".format(end-start))<categorify>
feature_importances = pd.DataFrame(np.mean(feature_importance, axis=0 ).T, columns=[[f'fold_{fold_n}' for fold_n in range(fold)]])
 feature_importances['feature'] = feature_columns
feature_importances['average'] = feature_importances[[f'fold_{fold_n}' for fold_n in range(fold)]].mean(axis=1)
 feature_importances.sort_values(by=('average',), ascending=False ).head(10 )
start = time.time()
tfidf_vec = TfidfVectorizer(ngram_range=(1,5), analyzer='char')
 full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 n_comp = 40
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
end = time.time()
print("Time taken in tf-idf is {}.".format(end-start))<feature_engineering>
def weight_opt(oof_nn, oof_rf, y_true):
weight_nn = np.inf
best_crps = np.inf
for i in np.arange(0, 1.01, 0.05):
crps_blend = np.zeros(oof_nn.shape[0])
 for k in range(oof_nn.shape[0]):
crps_blend[k] = crps_score(i * oof_nn[k,...] +(1-i)* oof_rf[k,...], y_true)
 if np.mean(crps_blend)< best_crps:
best_crps = np.mean(crps_blend)
 weight_nn = round(i, 2)
 print(str(round(i, 2)) + ' : mean crps(Blend)is ', round(np.mean(crps_blend), 6))
print('-'*36)
 print('Best weight for NN: ', weight_nn)
 print('Best weight for LGBM: ', round(1-weight_nn, 2))
print('Best mean crps(Blend): ', round(best_crps, 6))
return weight_nn, round(1-weight_nn, 2 )
class Dictogram(dict):
def __init__(self, iterable=None):

 super(Dictogram, self ).__init__()
self.types = 0
self.tokens = 0
if iterable:
self.update(iterable)
 def update(self, iterable):

 for item in iterable:
if item in self:
self[item] += 1
self.tokens += 1
else:
self[item] = 1
self.types += 1
self.tokens += 1
def count(self, item):

 if item in self:
return self[item]
return 0
def return_random_word(self):
random_key = random.sample(self, 1)
 return random_key[0]
def return_weighted_random_word(self):
random_int = random.randint(0, self.tokens-1)
 index = 0
list_of_keys = self.keys()
for i in range(0, self.types):
index += self[list_of_keys[i]]
if(index > random_int):
return list_of_keys[i]<define_search_model>
weight_nn, weight_lgbm = weight_opt(oof_nn, oof_lgbm, y)
def make_higher_order_markov_model(order, data):
markov_model = dict()
for i in range(0, len(data)-order):
window = tuple(data[i: i+order])
 if window in markov_model:
markov_model[window].update([data[i+order]])
 else:
markov_model[window] = Dictogram([data[i+order]])
 return markov_model<feature_engineering>
def predict(x_te, models_nn, models_rf, weight_nn, weight_rf, iteration=False):
model_num_nn = len(models_nn)
 model_num_rf = len(models_rf)
 for k,m in enumerate(models_nn):
if k==0:
y_pred_nn = m.predict(x_te, batch_size=1024)
 if iteration:
y_pred_rf = models_rf[k].predict(x_te, num_iteration=models_rf[k].best_iteration)
 else:
y_pred_rf = models_rf[k].predict(x_te)
 else:
y_pred_nn += m.predict(x_te, batch_size=1024)
 if iteration:
y_pred_rf += models_rf[k].predict(x_te, num_iteration=models_rf[k].best_iteration)
 else:
y_pred_rf += models_rf[k].predict(x_te)
 y_pred_nn = y_pred_nn / model_num_nn
y_pred_rf = y_pred_rf / model_num_rf
return weight_nn * y_pred_nn + weight_rf * y_pred_rf
<define_search_model><EOS> 
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable_features = scaler.transform(basetable[feature_columns].values)
 scaled_basetable = pd.DataFrame(scaled_basetable_features, columns=feature_columns)
 y_pred = predict(scaled_basetable, models_nn, models_lgbm, weight_nn, weight_lgbm, iteration=True)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()