Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<concatenate>
%config InlineBackend.figure_format = 'svg'
%matplotlib inline
sns.set()
pd.options.display.max_colwidth = 160
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 features.shape<sort_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
nulls = np.sum(features.isnull())
 nullcols = nulls.loc[(nulls != 0)]
dtypes = features.dtypes
dtypes2 = dtypes.loc[(nulls != 0)]
info = pd.concat([nullcols, dtypes2], axis=1 ).sort_values(by=0, ascending=False)
 print(info)
 print("There are", len(nullcols), "columns with missing values" )<feature_engineering>
import spacy
import nltk
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0] )<set_options>
nlp = spacy.en.English()
pd.set_option('max_columns', None)
 features[features['PoolArea'] > 0 & features['PoolQC'].isnull() ]<feature_engineering>
spacy.en.STOP_WORDS.add("'s")
 for word in spacy.en.STOP_WORDS:
lexeme = nlp.vocab[word]
lexeme.is_stop = True
features.loc[2418, 'PoolQC'] = 'Fa'
features.loc[2501, 'PoolQC'] = 'Gd'
features.loc[2597, 'PoolQC'] = 'Fa'<set_options>
def replace_ents(doc):
prefix = 'ent__'
text = str(doc.doc)
 for ent in doc.ents:
text = text.replace(ent.orth_, prefix + ent.label_)
 return text
pd.set_option('max_columns', None)
 features[(features['GarageType'] == 'Detchd')& features['GarageYrBlt'].isnull() ]<feature_engineering>
def preprocess(df):
print('Started parsing...')
 doc = df.text.apply(nlp)
 print('Text parsed')
 df['n_char'] = df.text.apply(len)
 df['n_words'] = doc.apply(lambda x: len([t for t in x if not t.is_punct]))
df['n_punct'] = doc.apply(lambda x: len([t for t in x if t.is_punct]))
df['n_ents'] = doc.apply(lambda x: len(x.ents))
df['n_chunks'] = doc.apply(lambda x: len(list(x.noun_chunks)))
 df['n_unique_words'] = doc.apply(lambda x: len(set([t.lower_ for t in x if not t.is_punct])))
 df['n_stop_words'] = doc.apply(lambda x: len([t for t in x if t.is_stop]))
df['char_by_word'] = doc.apply(lambda x: np.mean([len(t.orth_)for t in x if not t.is_punct]))
print('Features created')
 df['text_ent_repl'] = doc.apply(replace_ents)
 print('Entities replaced')
 clean_and_lemmatize = lambda x: ' '.join([t.lemma_ for t in x if not t.is_punct and not t.is_stop])
 df['text_cleaned'] = doc.apply(clean_and_lemmatize)
 print('Text cleaned' )
features.loc[2124, 'GarageYrBlt'] = features['GarageYrBlt'].median()
features.loc[2574, 'GarageYrBlt'] = features['GarageYrBlt'].median()
features.loc[2124, 'GarageFinish'] = features['GarageFinish'].mode() [0]
features.loc[2574, 'GarageFinish'] = features['GarageFinish'].mode() [0]
features.loc[2574, 'GarageCars'] = features['GarageCars'].median()
features.loc[2124, 'GarageArea'] = features['GarageArea'].median()
features.loc[2574, 'GarageArea'] = features['GarageArea'].median()
features.loc[2124, 'GarageQual'] = features['GarageQual'].mode() [0]
features.loc[2574, 'GarageQual'] = features['GarageQual'].mode() [0]
features.loc[2124, 'GarageCond'] = features['GarageCond'].mode() [0]
features.loc[2574, 'GarageCond'] = features['GarageCond'].mode() [0]<filter>
eap = train.loc[train.author == 'EAP']
hpl = train.loc[train.author == 'HPL']
mws = train.loc[train.author == 'MWS']
eap_t = eap.text_cleaned
hpl_t = hpl.text_cleaned
mws_t = mws.text_cleaned
basement_columns = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF']
tempdf = features[basement_columns]
tempdfnulls = tempdf[tempdf.isnull().any(axis=1)]<filter>
fd = nltk.FreqDist([y for x in train.text_cleaned.str.split() for y in x])
 fd_eap = nltk.FreqDist([y for x in eap_t.str.split() for y in x])
 fd_hpl = nltk.FreqDist([y for x in hpl_t.str.split() for y in x])
 fd_mws = nltk.FreqDist([y for x in mws_t.str.split() for y in x] )
tempdfnulls[(tempdfnulls.isnull() ).sum(axis=1)< 5]<feature_engineering>
sns.set_style({'axes.grid' : False} )
features.loc[332, 'BsmtFinType2'] = 'ALQ'
features.loc[947, 'BsmtExposure'] = 'No'
features.loc[1485, 'BsmtExposure'] = 'No'
features.loc[2038, 'BsmtCond'] = 'TA'
features.loc[2183, 'BsmtCond'] = 'TA'
features.loc[2215, 'BsmtQual'] = 'Po'
features.loc[2216, 'BsmtQual'] = 'Fa'
features.loc[2346, 'BsmtExposure'] = 'No'
features.loc[2522, 'BsmtCond'] = 'Gd'<groupby>
WordCloud(min_font_size=8, width=1000, height=400, colormap='cubehelix_r' ).generate(' '.join(hpl_t)).to_image()
subclass_group = features.groupby('MSSubClass')
 Zoning_modes = subclass_group['MSZoning'].apply(lambda x : x.mode() [0])
 Zoning_modes<groupby>
WordCloud(min_font_size=8, width=1000, height=400, colormap='hot' ).generate(' '.join(mws_t)).to_image()
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))<sort_values>
sns.set()
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
nulls = np.sum(features.isnull())
 nullcols = nulls.loc[(nulls != 0)]
dtypes = features.dtypes
dtypes2 = dtypes.loc[(nulls != 0)]
info = pd.concat([nullcols, dtypes2], axis=1 ).sort_values(by=0, ascending=False)
 print(info)
 print("There are", len(nullcols), "columns with missing values" )<groupby>
def drop_outliers(s):
med = s.mean()
std = s.std()
return s[(med - 3*std <= s)&(s <= med + 3*std)]
neighborhood_group = features.groupby('Neighborhood')
 lot_medians = neighborhood_group['LotFrontage'].median()
lot_medians<groupby>
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<filter>
y = train.author
pd.set_option('max_columns', None)
 features[(features['GarageYrBlt'].isnull())& features['GarageArea'] > 0]<set_options>
vectorizer = CountVectorizer(
 token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words='english' 
)
 X = vectorizer.fit_transform(train.text )
pd.set_option('max_columns', None)
 features[(features['MasVnrArea'].isnull())] 
<sort_values>
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
nulls = np.sum(features.isnull())
 nullcols = nulls.loc[(nulls != 0)]
dtypes = features.dtypes
dtypes2 = dtypes.loc[(nulls != 0)]
info = pd.concat([nullcols, dtypes2], axis=1 ).sort_values(by=0, ascending=False)
 print(info)
 print("There are", len(nullcols), "columns with missing values" )<filter>
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
features[features['GarageYrBlt'] == 2207]<feature_engineering>
vectorizer = CountVectorizer(ngram_range=(1,2))
X = vectorizer.fit_transform(train.text_cleaned )
features.loc[2590, 'GarageYrBlt'] = 2007<data_type_conversions>
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
factors = ['MSSubClass']
for i in factors:
features.update(features[i].astype('str'))<define_variables>
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
objects3 = []
for i in features.columns:
if features[i].dtype == object:
objects3.append(i )<count_unique_values>
vectorizer = CountVectorizer(token_pattern=r'\w{1,}', stop_words='english', ngram_range=(1,2))
X = vectorizer.fit_transform(train.text_ent_repl )
print("Training Set incomplete cases")
 sums_features = features[objects3].apply(lambda x: len(np.unique(x)))
 sums_features.sort_values(ascending=False )<count_values>
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
print(features['Street'].value_counts())
 print('-----')
 print(features['Utilities'].value_counts())
 print('-----')
 print(features['CentralAir'].value_counts())
 print('-----')
 print(features['PavedDrive'].value_counts() )<drop_column>
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
features = features.drop(['Utilities', 'Street'], axis=1 )<feature_engineering>
vectorizer = TfidfVectorizer(
 token_pattern=r'\w{1,}', ngram_range=(1, 1),
use_idf=True, smooth_idf=True, sublinear_tf=True, 
)
 X = vectorizer.fit_transform(train.text_cleaned )
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5*features['HalfBath'])+
features['BsmtFullBath'] +(0.5*features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<categorify>
%%time
scores = cross_val_score(LogisticRegression() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
final_features = pd.get_dummies(features ).reset_index(drop=True)
 final_features.shape<define_variables>
%%time
scores = cross_val_score(MultinomialNB() , X, y, cv=10, n_jobs=-1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
outliers = list(test2[test2<1e-3].index)
 outliers<define_variables>
import xgboost as xgb
import lightgbm as lgb
outliers = [30, 88, 462, 631, 1322]<drop_column>
drop = ['id', 'text', 'text_cleaned', 'text_ent_repl']
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )<count_values>
X_meta = train.drop(drop + ['author'], axis=1 )
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 >99.94:
overfit.append(i )<train_model>
%%time
lgbc = lgb.LGBMClassifier(objective='multiclass', n_estimators=100)
 scores = cross_val_score(lgbc, X_meta, y, cv=4, n_jobs=1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
overfit = list(overfit)
 overfit<concatenate>
%%time
xgbc = xgb.XGBClassifier(objective='multi:softprob', n_estimators=200)
 scores = cross_val_score(xgbc, X_meta, y, cv=4, n_jobs=4, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
overfit.append('MSZoning_C(all)' )<train_model>
xgbc.fit(X_meta, y);
xgb.plot_importance(xgbc);
overfit<train_model>
from sklearn.model_selection import StratifiedKFold
from sklearn import metrics
X.drop(overfit,axis=1,inplace=True)
 testing_features.drop(overfit,axis=1,inplace=True )<compute_train_metric>
def add_prob_features(vectorizer, col, model, prefix, cv=5):
vectorizer.fit(train[col].append(test[col]))
X = vectorizer.transform(train[col])
 X_test = vectorizer.transform(test[col])
 cv_scores = []
pred_test = 0
pred_train = np.zeros([train.shape[0], 3])
 kf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=123)
 print('CV started')
 for train_index, dev_index in kf.split(X, y):
X_train, X_dev = X[train_index], X[dev_index]
y_train, y_dev = y[train_index], y[dev_index]
model.fit(X_train, y_train)
 pred_dev = model.predict_proba(X_dev)
 pred_test += model.predict_proba(X_test)
 pred_train[dev_index, :] = pred_dev
cv_scores.append(metrics.log_loss(y_dev, pred_dev))
print('.', end='')
 print('')
 print("Mean CV LogLoss: %.3f" %(np.mean(cv_scores)))
 pred_test /= cv
train[prefix+'eap'] = pred_train[:, 0]
train[prefix+'hpl'] = pred_train[:, 1]
train[prefix+'mws'] = pred_train[:, 2]
test[prefix+'eap'] = pred_test[:, 0]
test[prefix+'hpl'] = pred_test[:, 1]
test[prefix+'mws'] = pred_test[:, 2]
lm = LinearRegression()
kfolds = KFold(n_splits=10, shuffle=True, random_state=23)
 def cv_rmse(model):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv = kfolds))
return(rmse)
 def cv_rmsle(model):
rmsle = np.sqrt(np.log(-cross_val_score(model, X, y,
scoring = 'neg_mean_squared_error',
cv=kfolds)))
 return(rmsle )<train_on_grid>
vectorizer = CountVectorizer(
 token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words='english' 
)
 add_prob_features(vectorizer, 'text', MultinomialNB() , 'nb_ctv_', cv=40 )
benchmark_model = make_pipeline(RobustScaler() ,
lm ).fit(X=X, y=y)
 cv_rmse(benchmark_model ).mean()<create_dataframe>
vectorizer = TfidfVectorizer(
 token_pattern=r'\w{1,}', ngram_range=(1, 1),
use_idf=True, smooth_idf=True, sublinear_tf=True, 
)
 add_prob_features(vectorizer, 'text_cleaned', MultinomialNB() , 'nb_tfv_', cv=40 )
coeffs = pd.DataFrame(list(zip(X.columns, benchmark_model.steps[1][1].coef_)) , columns=['Predictors', 'Coefficients'])
 coeffs.sort_values(by='Coefficients', ascending=False )<train_on_grid>
vectorizer = TfidfVectorizer(
 ngram_range=(1, 5), analyzer='char' 
)
 add_prob_features(vectorizer, 'text', MultinomialNB() , 'nb_char_', cv=40 )
def ridge_selector(k):
ridge_model = make_pipeline(RobustScaler() ,
RidgeCV(alphas = [k],
cv=kfolds)).fit(X, y)
 ridge_rmse = cv_rmse(ridge_model ).mean()
return(ridge_rmse )<find_best_params>
vectorizer = CountVectorizer(
 token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words='english' 
)
 add_prob_features(vectorizer, 'text_ent_repl', MultinomialNB() , 'nb_ent_', cv=40 )
r_alphas = [.0001,.0003,.0005,.0007,.0009, 
.01, 0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 20, 30, 50, 60, 70, 80]
ridge_scores = []
for alpha in r_alphas:
score = ridge_selector(alpha)
 ridge_scores.append(score )<train_on_grid>
X = train.drop(drop + ['author'], axis=1 )
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
ridge_model2 = make_pipeline(RobustScaler() ,
RidgeCV(alphas = alphas_alt,
cv=kfolds)).fit(X, y)
 cv_rmse(ridge_model2 ).mean()<find_best_params>
%%time
lgbc = lgb.LGBMClassifier(objective='multiclass', n_estimators=150, num_leaves=10)
 scores = cross_val_score(lgbc, X, y, cv=4, n_jobs=1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
ridge_model2.steps[1][1].alpha_<choose_model_class>
%%time
xgbc = xgb.XGBClassifier(objective='multi:softprob', n_estimators=150)
 scores = cross_val_score(xgbc, X, y, cv=4, n_jobs=1, scoring='neg_log_loss')
 print('LogLoss: %.3f +- %.3f' %(-np.mean(scores), 2*np.std(scores)) )
alphas = [0.00005, 0.0001, 0.0003, 0.0005, 0.0007,
0.0009, 0.01]
alphas2 = [0.00005, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005,
0.0006, 0.0007, 0.0008]
lasso_model2 = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7,
alphas = alphas2,
random_state = 42)).fit(X, y )<find_best_params>
%%time
xgbc.fit(X, y);
lasso_model2.steps[1][1].alpha_<compute_train_metric>
def sub(est, name='sub.csv'):
sub = pd.DataFrame(est.predict_proba(test.drop(drop, axis=1)) , columns=['EAP', 'HPL', 'MWS'])
 sub.insert(0, 'id', test.id)
 sub.to_csv(name, index=False )
cv_rmse(lasso_model2 ).mean()<create_dataframe>
from sklearn import model_selection
coeffs = pd.DataFrame(list(zip(X.columns, lasso_model2.steps[1][1].coef_)) , columns=['Predictors', 'Coefficients'] )<sort_values>
clf = xgb.XGBClassifier(objective = 'multi:softprob', nthread=1)
 parameters = {
'n_estimators': [150],
'max_depth': [3],
'subsample': [0.65],
'colsample_bytree': [0.95],
'min_child_weight': [1],
}
clf = model_selection.GridSearchCV(clf, parameters, n_jobs=4, verbose=1, scoring='neg_log_loss', refit=True )
used_coeffs = coeffs[coeffs['Coefficients'] != 0].sort_values(by='Coefficients', ascending=False)
 print(used_coeffs.shape)
 print(used_coeffs )<train_model>
clf.fit(X, y);
overfit_test2 = []
for i in used_coeffs_values.columns:
counts2 = used_coeffs_values[i].value_counts()
zeros2 = counts2.iloc[0]
if zeros2 / len(used_coeffs_values)* 100 > 99.5:
overfit_test2.append(i)
 overfit_test2<train_on_grid>
-clf.best_score_
<normalization><EOS> 
sub(clf.best_estimator_ )