Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<set_options>
eng_stopwords = set(stopwords.words("english"))
warnings.filterwarnings("ignore")
 plt.style.use('ggplot')
<load_from_csv>
train_df = pd.read_csv('.. /input/spooky-author-identification/train.csv')
 test_df = pd.read_csv('.. /input/spooky-author-identification/test.csv')
 sample = pd.read_csv('.. /input/spooky-author-identification/sample_submission.csv' )
df_train=pd.read_csv('.. /input/train.csv', index_col='Id')
 df_test=pd.read_csv('.. /input/test.csv', index_col='Id')
 train_ID = df_train.index
test_ID = df_test.index 
<drop_column>
train_df['num_words'] = train_df['text'].apply(lambda x: len(str(x ).split()))
test_df['num_words'] = test_df['text'].apply(lambda x: len(str(x ).split()))
df_train = df_train.drop(( df_train[df_train['GrLivArea']>4000] ).index )<drop_column>
train_df['num_unique_words'] = train_df['text'].apply(lambda x: len(set(str(x ).split())))
 test_df['num_unique_words'] = test_df['text'].apply(lambda x: len(set(str(x ).split())) )
quantitative = [i for i in df_train.columns if df_train.dtypes[i] != 'object']
quantitative.remove('SalePrice')
 qualitative = [j for j in df_train.columns if df_train.dtypes[j] == 'object']
print('Quantitative:',quantitative)
 print('')
 print('Qualitative:',qualitative )<prepare_x_and_y>
train_df["num_chars"] = train_df["text"].apply(lambda x: len(str(x)))
 test_df["num_chars"] = test_df["text"].apply(lambda x: len(str(x)))
 train_df["num_stopwords"] = train_df["text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
test_df["num_stopwords"] = test_df["text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
train_df["num_punctuations"] =train_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test_df["num_punctuations"] =test_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train_df["num_words_upper"] = train_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
test_df["num_words_upper"] = test_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
train_df["num_words_title"] = train_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
test_df["num_words_title"] = test_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
merged_data = df_train.append(df_test, sort=False ).reset_index(drop=True)
 print("The size of the merged data is:", merged_data.shape)
 ntrain = df_train.shape[0]
ntest = df_test.shape[0]
y_train = df_train.SalePrice.values<data_type_conversions>
author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
train_y = train_df['author'].map(author_mapping_dict)
 train_id = train_df['id'].values
test_id = test_df['id'].values
merged_data["Alley"] = merged_data["Alley"].fillna("None")
 merged_data["Fence"] = merged_data["Fence"].fillna("None")
 merged_data["MiscFeature"] = merged_data["MiscFeature"].fillna("None")
 merged_data["PoolQC"] = merged_data["PoolQC"].fillna("None")
 merged_data["FireplaceQu"] = merged_data["FireplaceQu"].fillna("None")
 merged_data["LotFrontage"] = merged_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
merged_data[col] = merged_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
merged_data[col] = merged_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
merged_data[col] = merged_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
merged_data[col] = merged_data[col].fillna('None')
 merged_data["MasVnrType"] = merged_data["MasVnrType"].fillna("None")
 merged_data["MasVnrArea"] = merged_data["MasVnrArea"].fillna(0)
 merged_data['MSZoning'] = merged_data['MSZoning'].fillna(merged_data['MSZoning'].mode() [0])
 merged_data = merged_data.drop(['Utilities'], axis=1)
 merged_data["Functional"] = merged_data["Functional"].fillna("Typ")
 merged_data['Electrical'] = merged_data['Electrical'].fillna(merged_data['Electrical'].mode() [0])
 merged_data['KitchenQual'] = merged_data['KitchenQual'].fillna(merged_data['KitchenQual'].mode() [0])
 merged_data['Exterior1st'] = merged_data['Exterior1st'].fillna(merged_data['Exterior1st'].mode() [0])
 merged_data['Exterior2nd'] = merged_data['Exterior2nd'].fillna(merged_data['Exterior2nd'].mode() [0])
 merged_data['SaleType'] = merged_data['SaleType'].fillna(merged_data['SaleType'].mode() [0])
 merged_data['MSSubClass'] = merged_data['MSSubClass'].fillna("None" )<define_variables>
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1 )
merged_data_NA_VALUES =merged_data.isnull().sum()<feature_engineering>
def runLR(train_X,train_y,test_X,test_y=None,test_X2=None):
model = LogisticRegression()
model.fit(train_X,train_y)
 return model.predict_proba(test_X),model.predict_proba(test_X2),model
def runMNB(train_X, train_y, test_X, test_y, test_X2):
model = naive_bayes.MultinomialNB()
model.fit(train_X, train_y)
 pred_test_y = model.predict_proba(test_X)
 pred_test_y2 = model.predict_proba(test_X2)
 return pred_test_y, pred_test_y2, model
merged_data['TotalSF'] = merged_data['TotalBsmtSF'] + merged_data['1stFlrSF'] + merged_data['2ndFlrSF']<data_type_conversions>
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runLR(dev_X, dev_y, val_X, val_y, test_X)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores )
merged_data['MSSubClass'] = merged_data['MSSubClass'].apply(str)
 merged_data['OverallCond'] = merged_data['OverallCond'].astype(str)
 merged_data['YrSold'] = merged_data['YrSold'].astype(str)
 merged_data['MoSold'] = merged_data['MoSold'].astype(str )<categorify>
tfidf_vec = TfidfVectorizer(stop_words='english',ngram_range=(1,3))
full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() +test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist() )
columns = ['LandContour', 'MSZoning', 'Alley',
'MasVnrType', 'ExterQual', 'ExterCond',
'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2', 'HeatingQC',
'CentralAir', 'KitchenQual', 'FireplaceQu',
'GarageFinish', 'GarageQual', 'GarageCond',
'PavedDrive', 'PoolQC', 'MiscFeature']
for col in columns:
lbl = LabelEncoder()
lbl.fit(list(merged_data[col].values))
merged_data[col] = lbl.transform(list(merged_data[col].values))
columns_Qual=['LotShape', 'LotConfig', 'LandSlope', 'Neighborhood',
'Condition1', 'Condition2', 'BldgType', 'HouseStyle',
'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
'Foundation', 'Heating', 'Electrical', 'Functional',
'GarageType', 'GarageCond', 'Fence', 'SaleType',
'SaleCondition', 'Street']
temp = pd.get_dummies(merged_data[columns_Qual], drop_first=True)
 merged_data = merged_data.drop(columns_Qual, axis=1)
 merged_data = pd.concat([merged_data, temp], axis=1 )<split>
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores )
df_train = merged_data[merged_data['SalePrice'].notnull() ]
df_test = merged_data[merged_data['SalePrice'].isnull() ].drop('SalePrice', axis=1 )<prepare_x_and_y>
n_comp =20
svd_tf = TruncatedSVD(n_components=20,algorithm='arpack')
 svd_tf.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_tf.transform(train_tfidf))
test_svd = pd.DataFrame(svd_tf.transform(test_tfidf))
train_svd_cols = ['svd_word_'+str(i)for i in range(n_comp)]
test_svd_cols = ['svd_word_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df,train_svd],axis=1)
 test_df = pd.concat([test_df,test_svd],axis=1)
 train_df.head()
x_train = df_train.drop(['SalePrice'], axis=1)
 y_train = df_train['SalePrice']
x_test = df_test<normalization>
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_df):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runLR(dev_X, dev_y, val_X, val_y, test_X)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores )
scaler = preprocessing.RobustScaler() ;
x_train_scaled = pd.DataFrame(scaler.fit_transform(x_train), columns=x_train.columns)
 x_test_scaled = pd.DataFrame(scaler.transform(x_test), columns=x_test.columns )<choose_model_class>
tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,2))
tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist() )
KRR = KernelRidge(alpha=0.05, kernel='polynomial', degree=1, coef0=2.5 )<choose_model_class>
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 print("cv scores : ", cv_scores)
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]
lasso = linear_model.Lasso(alpha=0.001, max_iter=5000, random_state=42 )<choose_model_class>
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_df):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runLR(dev_X, dev_y, val_X, val_y, test_X)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores)
GBoost = ensemble.GradientBoostingRegressor(n_estimators=1000, learning_rate=0.05, max_depth=3, max_features='sqrt', loss='huber', random_state=42 )<train_model>
wv = ".. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt"
def loadWordVecs() :
embeddings_index = {}
f = open(wv)
 for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))
return embeddings_index
def sent2vec(embeddings_index,s):
words = str(s ).lower()
words = word_tokenize(words)
 words = [w for w in words if not w in stopwords.words('english')]
words = [w for w in words if w.isalpha() ]
M = []
for w in words:
try:
M.append(embeddings_index[w])
 except:
continue
M = np.array(M)
 if M.shape[0] != 0:
v = M.min(axis=0)
 w = M.max(axis=0)
 return np.concatenate(( v,w),axis=0)
 else:
return np.zeros(200)
 def doGlove(x_train,x_test):
embeddings_index = loadWordVecs()
xtrain_glove = [sent2vec(embeddings_index,x)for x in tqdm(x_train)]
xtest_glove = [sent2vec(embeddings_index,x)for x in tqdm(x_test)]
xtrain_glove = np.array(xtrain_glove)
 xtest_glove = np.array(xtest_glove)
 return xtrain_glove,xtest_glove,embeddings_index
glove_vecs_train,glove_vecs_test,embeddings_index = doGlove(train_df['text'],test_df['text'] )
class Averaging_the_models(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, cool_models, peso):
self.cool_models = cool_models
self.peso = peso
def fit(self, X, y):
self.cool_models_ = [clone(x)for x in self.cool_models]
for model in self.cool_models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([(model.predict(X)* peso)for model, peso in zip(self.cool_models_, self.peso)])
 return np.sum(predictions, axis=1 )<choose_model_class>
train_df[['sent_vec_'+str(i)for i in range(200)]] = pd.DataFrame(glove_vecs_train.tolist())
 test_df[['sent_vec_'+str(i)for i in range(200)]] = pd.DataFrame(glove_vecs_test.tolist())
 train_df.head()
regression = Averaging_the_models(cool_models=(KRR, lasso, GBoost), peso=[0.25, 0.25, 0.50] )<predict_on_test>
def runXGB(train_X, train_y, test_X, test_y=None, test_X2=None, seed_val=0, child=1, colsample=0.3):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 3
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = child
param['subsample'] = 0.8
param['colsample_bytree'] = colsample
param['seed'] = seed_val
num_rounds = 2000
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=50, verbose_eval=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest, ntree_limit = model.best_ntree_limit)
 if test_X2 is not None:
xgtest2 = xgb.DMatrix(test_X2)
 pred_test_y2 = model.predict(xgtest2, ntree_limit = model.best_ntree_limit)
 return pred_test_y, pred_test_y2, model
regression.fit(x_train_scaled, np.log1p(y_train))
result = np.expm1(regression.predict(x_test_scaled))<save_to_csv>
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_df):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0, colsample=0.7)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores)
subFDS = pd.DataFrame({
"Id": test_ID,
"SalePrice": result
})
 subFDS.to_csv("subFDS.csv", index=False )<compute_train_metric>
p = pred_full_test/5
result = pd.DataFrame()
result['id'] = test_id
result['EAP'] = p[:,0]
result['HPL'] = p[:,1]
result['MWS'] = p[:,2]
<load_from_csv><EOS> 
result.head()
result.to_csv("result.csv", index=False )