Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<feature_engineering>
warnings.filterwarnings('ignore')
 %config InlineBackend.figure_format = 'retina'
def find_title(name):
result = re.search(r'[A-Z][a-z]*[.]', name)
 if(result == None):
return ''
return result.group(0)
 dataset['Title'] = dataset.Name.apply(find_title)
 dataset.Title.unique()<feature_engineering>
PATH_TO_ALL_DATA = '.. /input/spooky-vw-tutorial/'
data_demo = pd.read_csv(os.path.join(PATH_TO_ALL_DATA, 'weights_heights.csv'))
def set_median_age_by_title(title):
title_age_mask = dataset.loc[(dataset['Title'] == title)&(dataset['Age'].isna())]
df = dataset.iloc[title_age_mask.index]
df.Age = dataset.Age.median()
dataset.iloc[title_age_mask.index] = df
for title in dataset.Title.unique() :
set_median_age_by_title(title )<drop_column>
df = pd.read_csv(os.path.join(PATH_TO_ALL_DATA, 'bank_train.csv'))
labels = pd.read_csv(os.path.join(PATH_TO_ALL_DATA,
'bank_train_target.csv'), header=None)
 df.head()
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
dataset.drop(['Name','Ticket'], axis=1, inplace=True)
 dataset.head()<categorify>
label_encoder = LabelEncoder()
fare_midean = dataset['Fare'].median()
values = {'Cabin':'Z','Embarked':'S','Fare':fare_midean}
dataset.fillna(value=values,inplace=True)
 dataset['Cabin1'] = dataset['Cabin'].str[0]
dataset.drop(['Cabin'], axis=1, inplace=True)
 dataset['Sex'] = LabelEncoder().fit_transform(dataset['Sex'].values)
 dataset['Cabin1'] = LabelEncoder().fit_transform(dataset['Cabin1'].values)
 dataset['Embarked'] = LabelEncoder().fit_transform(dataset['Embarked'].values)
 dataset['Title'] = LabelEncoder().fit_transform(dataset['Title'].values)
 dataset.head()<normalization>
mapped_education = pd.Series(label_encoder.fit_transform(df['education']))
mapped_education.value_counts().plot.barh()
print(dict(enumerate(label_encoder.classes_)) )
working_columns = ['Pclass','Sex','Age','Fare','Embarked','FamilySize','Cabin1','Title','SibSp','Parch']
columns_to_scale = ['Pclass','Sex','Age','Fare','Embarked','FamilySize','Cabin1','Title','SibSp','Parch']
def scale_dataset(dataset):
scalers = {}
for column in columns_to_scale:
scaler = StandardScaler()
dataset[column] = scaler.fit_transform(dataset[column].values.reshape(-1, 1))
scalers[column] = scaler
return dataset, scalers
dataset, scalers = scale_dataset(dataset )<predict_on_test>
df['education'] = mapped_education
df.head()
columns = [c for c in train_set.columns if c not in ["Survived", "PassengerId", "Pclass", "Sex", "Cabin1", "Embarked", 'FamilySize']]
clf = LocalOutlierFactor(n_neighbors=5, contamination=0.5)
 y_pred = clf.fit_predict(train_set[columns].values)
 X_scores = clf.negative_outlier_factor_
clf_df = pd.DataFrame(columns=['X_scores', 'y_pred'])
 clf_df.X_scores = X_scores
clf_df.y_pred = y_pred
outlier_index = clf_df.index[clf_df['y_pred'] == -1].tolist()
train_set.drop(outlier_index )<prepare_x_and_y>
categorical_columns = df.columns[df.dtypes == 'object'].union(['education'])
 for column in categorical_columns:
df[column] = label_encoder.fit_transform(df[column])
 df.head()
y_source = train_set['Survived'].values.astype(int)
 X_source = train_set[working_columns].values
X_test = test_set[working_columns].values
y_test = test_set['Survived'].values.astype(int )<split>
df.loc[1].job - df.loc[2].job
X_train, X_test, y_train, y_test = train_test_split(X_source, y_source, test_size = 0.2)
 print('X_train: {}'.format(X_train.shape))
print('y_train: {}'.format(y_train.shape))
print('X_test: {}'.format(X_test.shape))
print('y_test: {}'.format(y_test.shape))<choose_model_class>
def logistic_regression_accuracy_on(dataframe, labels):
features = dataframe.as_matrix()
train_features, test_features, train_labels, test_labels = \
train_test_split(features, labels)
 logit = LogisticRegression()
logit.fit(train_features, train_labels)
 return classification_report(test_labels, logit.predict(test_features))
print(logistic_regression_accuracy_on(df[categorical_columns], labels))
def create_model2(optimizer, units = 16, init = 'glorot_uniform', activation = 'relu', l2_init = 0.001, dropout=0.1):
model = Sequential()
model.add(Dense(units=units, input_dim=10, activation=activation, kernel_initializer=init, kernel_regularizer=l2(l2_init)))
 model.add(Dropout(dropout))
model.add(Dense(units=units, activation=activation, kernel_initializer=init, kernel_regularizer=l2(l2_init)))
 model.add(BatchNormalization())
 model.add(Dropout(dropout))
model.add(Dense(units=units, activation=activation, kernel_initializer=init))
model.add(Dense(1, activation='sigmoid',kernel_initializer=init))
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 return model<choose_model_class>
one_hot_example = pd.DataFrame([{i: 0 for i in range(10)}])
 one_hot_example.loc[0, 6] = 1
one_hot_example
callbacks = [EarlyStopping(monitor='val_loss', patience=100)]
adam = optimizers.Adam(lr=3e-4, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
 best_model = create_model2(units=16, optimizer=adam, init='glorot_uniform', activation='relu', l2_init=0.3)
 history = best_model.fit(X_train, y_train,
validation_data=[X_test, y_test],
batch_size=64,
epochs=1000,
callbacks=callbacks,
verbose=1 )<compute_test_metric>
onehot_encoder = OneHotEncoder(sparse=False )
scores = best_model.evaluate(X_train, y_train, verbose=0)
 print("%s: %.2f%%" %(best_model.metrics_names[1], scores[1]*100))
scores = best_model.evaluate(X_test, y_test, verbose=0)
 print("%s: %.2f%%" %(best_model.metrics_names[1], scores[1]*100))<save_to_csv>
encoded_categorical_columns = pd.DataFrame(onehot_encoder.fit_transform(df[categorical_columns]))
encoded_categorical_columns.head()
predictions = best_model.predict_classes(test_set[working_columns].values)
 submission = pd.DataFrame({
"PassengerId": test_set["PassengerId"],
"Survived": predictions.ravel()
})
 submission.to_csv('submission.csv', index=False )<set_options>
print(logistic_regression_accuracy_on(encoded_categorical_columns, labels))
df = pd.read_csv('.. /input/train.csv')
 %matplotlib inline
df.head()
df.isnull().sum()
df['Fare'] = df['Fare'].fillna(df['Fare'].median())
 df['Age'] = df['Age'].fillna(df['Age'].median())
 df['Embarked'] = df['Embarked'].fillna('S')
 df['Sex'] = df['Sex'].apply(lambda x: 1 if x == 'male' else 0)
 df['Embarked'] = df['Embarked'].map({'S':0, 'C':1, 'Q':2} ).astype(int)
 df = df.drop(['Cabin','Name','PassengerId','Ticket'],axis = 1)
 train_X = df.drop('Survived',axis = 1)
 train_y = df.Survived 
(train_X, test_X, train_y, test_y)= train_test_split(train_X, train_y, test_size = 0.3, random_state = 0)
 df.isnull().sum()<train_model>
hashing_example = pd.DataFrame([{i: 0.0 for i in range(hash_space)}])
 for s in('job=student', 'marital=single', 'day_of_week=mon'):
print(s, '->', hash(s)% hash_space)
 hashing_example.loc[0, hash(s)% hash_space] = 1
hashing_example
clf = DecisionTreeClassifier(random_state = 0)
 clf.fit(train_X, train_y)
<define_search_space>
assert hash('no')== hash('no')
 assert hash('housing=no')!= hash('loan=no' )
parameters = {
"max_depth":[2,3,None],
"n_estimators":[50,100,200,300,400,500],
"max_features":[1,3,7],
"min_samples_split":[2,3,10],
"min_samples_leaf":[1,3,10],
"bootstrap":[True,False],
"criterion":["gini","entropy"]
}
clf = GridSearchCV(
 estimator = RandomForestClassifier() ,
param_grid = parameters,
scoring = "accuracy",
n_jobs = -1,
cv = 5 
)
 clf.fit(train_X , train_y)
<import_modules>
%%capture
!git clone --recursive https://github.com/VowpalWabbit/vowpal_wabbit.git
!cd vowpal_wabbit/; make
!cd vowpal_wabbit/; make install
MLPClassifier?<set_options>
newsgroups = fetch_20newsgroups(PATH_TO_ALL_DATA )
parameters={
'solver':['lbfgs', 'sgd', 'adam'],
'activation':['identity', 'logistic', 'tanh', 'relu'],
'hidden_layer_sizes':[(100,),(100, 10),(100, 100, 10),(100, 100, 100, 10)],
'learning_rate':['constant', 'invscaling', 'adaptive'],
'early_stopping':[True],
}
clf = GridSearchCV(
 estimator = MLPClassifier() ,
param_grid = parameters,
n_jobs = -1,
cv = 5 
)
 clf.fit(train_X , train_y)
 print('clf.best_params_ : ', clf.best_params_)
 print('clf.best_estimator_ : ', clf.best_estimator_)
 print('clf.best_score_ : ', clf.best_score_)
<compute_test_metric>
def to_vw_format(document, label=None):
return str(label or '')+ ' |text ' + ' '.join(re.findall('\w{3,}', document.lower())) + '
'
to_vw_format(text, 1 if target == 'rec.autos' else -1 )
clf.score(train_X , train_y )<compute_train_metric>
all_documents = newsgroups['data']
all_targets = [1 if newsgroups['target_names'][target] == 'rec.autos'
else -1 for target in newsgroups['target']]
pred = clf.predict(test_X)
 fpr, tpr , thresholds = roc_curve(test_y,pred,pos_label = 1)
 auc(fpr,tpr)
 accuracy_score(pred,test_y )<feature_engineering>
train_documents, test_documents, train_labels, test_labels = \
train_test_split(all_documents, all_targets, random_state=7)
 with open(os.path.join('20news_train.vw'), 'w')as vw_train_data:
for text, target in zip(train_documents, train_labels):
vw_train_data.write(to_vw_format(text, target))
with open(os.path.join('20news_test.vw'), 'w')as vw_test_data:
for text in test_documents:
vw_test_data.write(to_vw_format(text))
fin = pd.read_csv('.. /input/test.csv')
 fin.head()
passsengerid = fin['PassengerId']
fin.isnull().sum()
fin['Fare'] = fin['Fare'].fillna(fin['Fare'].median())
 fin['Age'] = fin['Age'].fillna(fin['Age'].median())
 fin['Embarked'] = fin['Embarked'].fillna('S')
 fin['Sex'] = fin['Sex'].apply(lambda x: 1 if x == 'male' else 0)
 fin['Embarked'] = fin['Embarked'].map({'S':0 , 'C':1 , 'Q':2} ).astype(int)
 fin= fin.drop(['Cabin','Name','Ticket','PassengerId'],axis =1)
 predictions = clf.predict(fin)
 submission = pd.DataFrame({'PassengerId':passsengerid, 'Survived':predictions})
 submission.to_csv('submission.csv' , index = False)
<import_modules>
!vw -d 20news_train.vw --loss_function hinge -f 20news_model.vw
import numpy as np
import pandas as pd
import os
import seaborn as sns
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.ensemble import VotingClassifier
import torch
from torch import nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import math
from torch.utils.data import TensorDataset, DataLoader
import matplotlib.pyplot as plt
from tqdm import tqdm
import catboost
from sklearn.metrics import mean_squared_error
import scipy
from scipy import stats
from scipy.stats import norm<load_from_csv>
all_documents = newsgroups['data']
topic_encoder = LabelEncoder()
all_targets_mult = topic_encoder.fit_transform(newsgroups['target'])+ 1
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', sep=',' )<drop_column>
train_documents, test_documents, train_labels_mult, test_labels_mult = \
train_test_split(all_documents, all_targets_mult, random_state=7)
 with open(os.path.join('20news_train_mult.vw'), 'w')as vw_train_data:
for text, target in zip(train_documents, train_labels_mult):
vw_train_data.write(to_vw_format(text, target))
with open(os.path.join('20news_test_mult.vw'), 'w')as vw_test_data:
for text in test_documents:
vw_test_data.write(to_vw_format(text))
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )<train_model>
%%time
!vw --oaa 20 20news_train_mult.vw -f 20news_model_mult.vw --loss_function=hinge
target = 'SalePrice'
print(train.loc[:, target].isnull().any())
(mu, sigma)= norm.fit(train['SalePrice'])
 print(f'mu = {mu:.2f} and sigma = {sigma:.2f}' )<set_options>
with open('20news_test_predictions_mult.txt')as pred_file:
test_prediction_mult = [float(label)for label in pred_file.readlines() ]
sns.set_style('whitegrid' )<data_type_conversions>
accuracy_score(test_labels_mult, test_prediction_mult )
train['Alley'] = train['Alley'].fillna('No alley access')
 train['BsmtQual'] = train['BsmtQual'].fillna('No Basement')
 train['BsmtCond'] = train['BsmtCond'].fillna('No Basement')
 train['BsmtExposure'] = train['BsmtExposure'].fillna('No Basement')
 train['BsmtFinType1'] = train['BsmtFinType1'].fillna('No Basement')
 train['BsmtFinType2'] = train['BsmtFinType2'].fillna('No Basement')
 train['FireplaceQu'] = train['FireplaceQu'].fillna('No Fireplace')
 train['GarageType'] = train['GarageType'].fillna('No Garage')
 train['GarageFinish'] = train['GarageFinish'].fillna('No Garage')
 train['GarageQual'] = train['GarageQual'].fillna('No Garage')
 train['GarageCond'] = train['GarageCond'].fillna('No Garage')
 train['PoolQC'] = train['PoolQC'].fillna('No Pool')
 train['Fence'] = train['Fence'].fillna('No Fence')
 train['MiscFeature'] = train['MiscFeature'].fillna('None')
 test['Alley'] = test['Alley'].fillna('No alley access')
 test['BsmtQual'] = test['BsmtQual'].fillna('No Basement')
 test['BsmtCond'] = test['BsmtCond'].fillna('No Basement')
 test['BsmtExposure'] = test['BsmtExposure'].fillna('No Basement')
 test['BsmtFinType1'] = test['BsmtFinType1'].fillna('No Basement')
 test['BsmtFinType2'] = test['BsmtFinType2'].fillna('No Basement')
 test['FireplaceQu'] = test['FireplaceQu'].fillna('No Fireplace')
 test['GarageType'] = test['GarageType'].fillna('No Garage')
 test['GarageFinish'] = test['GarageFinish'].fillna('No Garage')
 test['GarageQual'] = test['GarageQual'].fillna('No Garage')
 test['GarageCond'] = test['GarageCond'].fillna('No Garage')
 test['PoolQC'] = test['PoolQC'].fillna('No Pool')
 test['Fence'] = test['Fence'].fillna('No Fence')
 test['MiscFeature'] = test['MiscFeature'].fillna('None' )<feature_engineering>
M = confusion_matrix(test_labels_mult, test_prediction_mult)
 for i in np.where(M[0,:] > 0)[0][1:]:
print(newsgroups['target_names'][i], M[0,i] )
train['Pool_presence'] = [0 if val=='No Pool' else 1 for val in train['PoolQC']]
test['Pool_presence'] = [0 if val=='No Pool' else 1 for val in test['PoolQC']]
train['CentralAir'] = [1 if x == train['CentralAir'].unique() [0] else 0 for x in train.loc[:,'CentralAir'].values]
test['CentralAir'] = [1 if x == test['CentralAir'].unique() [0] else 0 for x in test.loc[:,'CentralAir'].values]<import_modules>
with open(os.path.join(PATH_TO_ALL_DATA, 'reviews_train.pkl'), 'rb')as reviews_train_pkl:
reviews_train = pickle.load(reviews_train_pkl)
 text_train, y_train = reviews_train.data, reviews_train.target
from catboost.eval.catboost_evaluation import *
from catboost.utils import create_cd<feature_engineering>
print("Number of documents in training data: %d" % len(text_train))
print(np.bincount(y_train))
cols = list(train.columns)
 a, b = cols.index('SalePrice'), cols.index('Pool_presence')
 cols[b], cols[a] = cols[a], cols[b]
train = train[cols]
train.head()<save_to_csv>
with open(os.path.join(PATH_TO_ALL_DATA, 'reviews_test.pkl'), 'rb')as reviews_test_pkl:
reviews_test = pickle.load(reviews_test_pkl)
 text_test, y_test = reviews_test.data, reviews_train.target
print("Number of documents in test data: %d" % len(text_test))
print(np.bincount(y_test))
train.to_csv('train2.csv', index=False )<drop_column>
to_vw_format(str(text_train[1]), 1 if y_train[0] == 1 else -1 )
feature_names = dict(list(enumerate(train.keys())))
 cat_dict = {i:label for i,label in enumerate(train.columns)if label in cat_features+['Fence', 'MiscFeature', 'Alley', 'PoolQC']}
del feature_names[80] 
<init_hyperparams>
train_share = int(0.7 * len(text_train))
train, valid = text_train[:train_share], text_train[train_share:]
train_labels, valid_labels = y_train[:train_share], y_train[train_share:]
fold_size = int(round(len(train)/2, -2))
folds_count=10
description_file='./train.cd'
train2_file='./train2.csv'
learn_params={'iterations':505,
'task_type' : 'GPU',
'random_seed': 2,
'learning_rate' : 0.1,
'max_depth': 10,
'l2_leaf_reg': 9.8,
'loss_function': 'RMSE',
'max_ctr_complexity' : 2,
'logging_level': 'Silent',
'boosting_type': 'Plain',}
evaluator = CatboostEvaluation(train2_file,
fold_size,
folds_count,
delimiter=',',
column_description=description_file,
partition_random_seed=2,
has_header=True, 
)
 result = evaluator.eval_features(learn_config=learn_params,
eval_metrics=["RMSE"],
features_to_eval=range(1,80),
)<compute_train_metric>
with open('movie_reviews_train.vw', 'w')as vw_train_data:
for text, target in zip(train, train_labels):
vw_train_data.write(to_vw_format(str(text), 1 if target == 1 else -1))
with open('movie_reviews_valid.vw', 'w')as vw_train_data:
for text, target in zip(valid, valid_labels):
vw_train_data.write(to_vw_format(str(text), 1 if target == 1 else -1))
with open('movie_reviews_test.vw', 'w')as vw_test_data:
for text in text_test:
vw_test_data.write(to_vw_format(str(text)) )
logloss_result = result.get_metric_results("RMSE")
 logloss_result.get_baseline_comparison()<load_from_csv>
!vw -d movie_reviews_train.vw --loss_function hinge \
-f movie_reviews_model.vw --quiet
good_features = logloss_result.get_baseline_comparison() [logloss_result.get_baseline_comparison() ['Decision']=='GOOD'].index
good_features = [pd.read_csv(train2_file ).columns[int(str(feature)[-2:])] for i,feature in enumerate(good_features)]
len(good_features), good_features<drop_column>
with open('movie_valid_pred.txt')as pred_file:
valid_prediction = [float(label)
 for label in pred_file.readlines() ]
print("Accuracy: {}".format(round(accuracy_score(valid_labels,
[int(pred_prob > 0)for pred_prob in valid_prediction]), 3)))
 print("AUC: {}".format(round(roc_auc_score(valid_labels, valid_prediction), 3)) )
train = train[good_features+[target]]
test = test[good_features]<categorify>
with open('movie_test_pred.txt')as pred_file:
test_prediction = [float(label)
 for label in pred_file.readlines() ]
print("Accuracy: {}".format(round(accuracy_score(y_test,
[int(pred_prob > 0)for pred_prob in test_prediction]), 3)))
 print("AUC: {}".format(round(roc_auc_score(y_test, test_prediction), 3)) )
def empty_features(test, train, threshhold=0, verbose=False):
if verbose:
print(' column \t test \t\t train
', '*'*40)
 i=1
empty_list=[]
for col in test.columns:
percentage_train =(train[col].isnull().sum() /len(train)) *100
percentage_test =(test[col].isnull().sum() /len(test)) *100
if percentage_train and percentage_test:
if(( percentage_test>=threshhold)|(percentage_train>=threshhold)) :
empty_list.append(col)
 if verbose:
print(i,'{}{}% \t{}%'.format(col.ljust(15,' '), round(percentage_test, 3), round(percentage_train,3)))
 i+=1
return empty_list<drop_column>
!vw -d movie_reviews_train.vw \
--loss_function hinge --ngram 2 -f movie_reviews_model2.vw --quiet
empty_features_list=empty_features(test=test, train=train, verbose=True, threshhold=80)
 empty_features_list<count_unique_values>
with open('movie_valid_pred2.txt')as pred_file:
valid_prediction = [float(label)
 for label in pred_file.readlines() ]
print("Accuracy: {}".format(round(accuracy_score(valid_labels,
[int(pred_prob > 0)for pred_prob in valid_prediction]), 3)))
 print("AUC: {}".format(round(roc_auc_score(valid_labels, valid_prediction), 3)) )
only_1_value = [feature for feature in train.nunique().index if train.nunique() [feature]==1]
only_1_value<data_type_conversions>
with open('movie_test_pred2.txt')as pred_file:
test_prediction2 = [float(label)
 for label in pred_file.readlines() ]
print("Accuracy: {}".format(round(accuracy_score(y_test,
[int(pred_prob > 0)for pred_prob in test_prediction2]), 3)))
 print("AUC: {}".format(round(roc_auc_score(y_test, test_prediction2), 3)) )
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
drop_by_corr = [column for column in upper.columns if any(upper[column] > 0.875)]
print(drop_by_corr )<count_values>
vw_pred = np.loadtxt(os.path.join(PATH_TO_ALL_DATA, 'vw_test_pred.csv'))
test_labels = np.loadtxt(os.path.join(PATH_TO_ALL_DATA, 'stackoverflow_test_labels.txt'))
accuracy_score(test_labels, vw_pred )
def quasi_constant(df, threshold=0, verbose=True):
features_list = []
for feature in df:
table = df[feature].value_counts() / np.float(len(df))
first=1
for name in table.index:
if table[name]>=threshold:
if first:
if verbose:
print(f'for feature "{feature}":', 'value\t\tscore', sep='
')
 first=0
if verbose:
print(name, table[name], sep='\t', end='
'+'*'*50+'
')
 if not first:
features_list.append(feature)
 return features_list<feature_engineering>
train_texts = pd.read_csv('.. /input/spooky-author-identification/train.zip', index_col='id')
 test_texts = pd.read_csv('.. /input/spooky-author-identification/test.zip', index_col='id')
 sample_sub = pd.read_csv('.. /input/spooky-author-identification/sample_submission.zip',
index_col='id' )
quasi_constant_features = quasi_constant(train, 0.95)
 quasi_constant_features<define_variables>
author_code = {"EAP": 1, "MWS": 2,"HPL": 3}
bad_features = drop_by_corr+only_1_value+quasi_constant_features+empty_features_list
len(bad_features), bad_features<define_variables>
train_texts["author_code"] = train_texts["author"].map(author_code )
good_features = [feature for feature in good_features if feature not in bad_features]
len(good_features), good_features<drop_column>
train_texts_part, valid_texts = train_test_split(train_texts, test_size=0.3, random_state=17,
stratify=train_texts["author_code"], shuffle=True )
train = train[good_features+[target]]
test = test[good_features]<load_from_csv>
def to_vw_only_text(out_vw, df, is_train=True):
with open(out_vw, "w")as out:
for i in range(df.shape[0]):
if is_train:
target = df["author_code"].iloc[i]
else:
target = 1
text = df["text"].iloc[i].strip().replace('|', '' ).replace(':', '' ).lower()
words = re.findall("\w{3,}", text)
 new_text = " ".join(words)
 s = "{} |text {}
".format(target, new_text)
 out.write(s )
len(train.columns), len(pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', sep=',' ).columns )<choose_model_class>
to_vw_only_text("train_part_only_text.vw", train_texts_part )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2 )<prepare_x_and_y>
to_vw_only_text("valid_only_text.vw", valid_texts )
X,y = pipelining_preprocessor(train, filler = 'out_of_range', target='SalePrice', type='cat')
 y = np.log1p(y )<split>
to_vw_only_text("train_only_text.vw", train_texts )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, shuffle=True, random_state=2 )<define_variables>
to_vw_only_text("test_only_text.vw", test_texts, is_train=False )
cats = list(train.select_dtypes(include='object' ).columns )<choose_model_class>
!vw --oaa 3 train_part_only_text.vw -f model_only_text_part.vw -b 28 --random_seed 17 \
--loss_function logistic --ngram 2 --passes 10 -k -c
full_pool = catboost.Pool(X, label = y, cat_features = cats)
 train_pool = catboost.Pool(X_train, label = y_train, cat_features = cats)
 val_pool = catboost.Pool(X_val, label = y_val, cat_features = cats)
 test_pool = catboost.Pool(pipelining_preprocessor(test[X.columns], filler = 'out_of_range', type='cat'), cat_features = cats )<choose_model_class>
!vw -i model_only_text_part.vw -t -d valid_only_text.vw -p valid_pred1.txt \
--random_seed 17 -r valid_prob1.txt
cat_params = {
}
cat_model = catboost.CatBoostRegressor(loss_function='RMSE',
random_seed=2,
max_depth = 10,
learning_rate = 0.05,
random_strength=1,
max_ctr_complexity=1,
l2_leaf_reg=8,
grow_policy = 'Lossguide',
task_type='GPU', 
)
 grid_search_results = cat_model.grid_search(cat_params, full_pool,
partition_random_seed=2, cv = skf,
search_by_train_test_split=True,
plot=True)
 grid_search_results['params']<choose_model_class>
def evaluate_vw_prediction(path_to_vw_pred_probs, is_test=False, target=None, write_submission=False,
submission_file=None, test_index=test_texts.index, columns=['EAP', 'MWS', 'HPL']):
def sigmoid(z):
return 1 /(1 + np.exp(-z))
with open(path_to_vw_pred_probs)as pred_file:
pred_probs = np.array([[float(pair.split(':')[1])for pair in line.strip().split() ]
for line in pred_file.readlines() ])
 pred_probs = sigmoid(pred_probs)
 if target is not None and not is_test:
print(log_loss(target, pred_probs))
if write_submission and submission_file is not None:
subm_df = pd.DataFrame(pred_probs, columns=columns)
 subm_df.index = test_index
subm_df.to_csv(submission_file )
cat_model = catboost.CatBoostRegressor(loss_function='RMSE',
random_seed=2,
max_depth = 10,
learning_rate = 0.05,
random_strength=1,
max_ctr_complexity=1,
l2_leaf_reg=8,
grow_policy = 'Lossguide',
task_type='GPU',
)<train_model>
evaluate_vw_prediction('valid_prob1.txt',
target=valid_texts['author_code'] )
cat_model.fit(train_pool, eval_set=val_pool )<import_modules>
%%time
!vw --oaa 3 train_only_text.vw -f model_only_text.vw -b 28 --random_seed 17 \
--loss_function logistic --ngram 2 --passes 10 -k -c --quiet
import shap<create_dataframe>
evaluate_vw_prediction('test_prob1.txt',
is_test=True, write_submission=True,
submission_file='submission1_only_text.csv' )
vals= np.abs(shap_values ).mean(0)
 feature_importance_shap = pd.DataFrame(list(zip(X_train.columns, vals)) , columns=['feature','feature_importance_shap'])
 feature_importance_shap.sort_values(by=['feature_importance_shap'], ascending=False, inplace=True)
 feature_importance_shap.reset_index(inplace=True, drop=True)
 feature_importance_shap<save_to_csv>
max_words_in_text = train_texts['text'].apply(lambda text: len(re.findall("\w{3,}", text.strip())) ).max()
max_unique_words_in_text = train_texts['text'].apply(lambda text: len(set(re.findall("\w{3,}", text.strip())))).max()
max_aver_word_len_in_text = train_texts['text'].apply(lambda text:
sum([len(w)for w in re.findall("\w{3,}", text.strip())])/
len(re.findall("\w{3,}", text.strip())) ).max()
gb_pred = list(map(int, map(np.expm1, cat_model.predict(test_pool))))
gb_output = pd.DataFrame({'Id': pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')['Id'],
'SalePrice': gb_pred})
 gb_output.to_csv('submission_catboost.csv', index=False)
 gb_output<set_options>
max_words_in_text, max_unique_words_in_text, max_aver_word_len_in_text
%matplotlib inline
warnings.filterwarnings('ignore')
 sns.set(style='white', context='notebook', palette='deep')
 np.random.seed(2)
 DEBUG = False<import_modules>
def to_vw_text_and_some_features(out_vw, df, is_train=True):
with open(out_vw, "w")as out:
for i in range(df.shape[0]):
if is_train:
target = df["author_code"].iloc[i]
else:
target = 1
text = df["text"].iloc[i].strip().replace('|', '' ).replace(':', '' ).lower()
words = re.findall("\w{3,}", text)
 new_text = " ".join(words)
 num_words = round(len(words)/ max_words_in_text, 4)
 num_uniq_words = round(len(set(words)) / max_unique_words_in_text, 4)
 aver_word_len = round(sum([len(w)for w in words])/ len(words)/ max_aver_word_len_in_text, 4)
 features = [num_words, num_uniq_words, aver_word_len]
features_vw = ' '.join(['{}:{}'.format(i[0], i[1])for i in zip(range(len(features)) , features)])
 s = "{} |text {} |num {}
".format(target, new_text, features_vw)
 out.write(s)
print("The Python version is %s.%s.%s." % sys.version_info[:3] )<set_options>
to_vw_text_and_some_features("train_part_text_feat.vw", train_texts_part )
%pwd<load_from_csv>
to_vw_text_and_some_features("valid_text_feat.vw", valid_texts )
%time digit_train = pd.read_csv(".. /input/train.csv", header=0, sep=",")
 %time digit_test = pd.read_csv(".. /input/test.csv", header=0, sep="," )<prepare_x_and_y>
to_vw_text_and_some_features("train_text_feat.vw", train_texts )
features_train = digit_train.iloc[:,1:]
target_train = digit_train.iloc[:,0]
features_test = digit_test.iloc[:,0:]<categorify>
to_vw_text_and_some_features("test_text_feat.vw", test_texts, is_train=False )
features_train = features_train/255.0
features_test = features_test/255.0<find_best_params>
%%time
!vw --oaa 3 train_part_text_feat.vw -f model_text_feat_part.vw -b 28 --random_seed 17 \
--loss_function logistic --ngram 2 --passes 10 -k -c --quiet
start_time = time.perf_counter()
train_results = []
test_results = []
k_range = list(range(1,5))
k_scores = []
for k in k_range:
knn = KNeighborsClassifier(n_neighbors=k, n_jobs=-1, algorithm='ball_tree', leaf_size=40, weights='uniform')
 scores = cross_val_score(knn, features_train, target_train, cv=10, scoring='accuracy', n_jobs=-1)
 k_scores.append(scores.mean())
 if DEBUG:
print(k_scores)
 print(time.perf_counter() - start_time, "seconds" )<create_dataframe>
evaluate_vw_prediction('valid_prob2.txt',
target=valid_texts['author_code'] )
if DEBUG:
scores = pd.DataFrame(k_scores)
 print(scores )<define_variables>
%%time
!vw --oaa 3 train_text_feat.vw -f model_text_feat.vw -b 28 --random_seed 17 \
--loss_function logistic --ngram 2 --passes 10 -k -c --quiet
print("The optimal number of neighbors is %d." % optimal_k )<train_model>
evaluate_vw_prediction('test_prob2.txt',
is_test=True, write_submission=True,
submission_file='submission2_text_feat.csv' )
start_time = time.perf_counter()
clf_knn = KNeighborsClassifier(n_neighbors=3, n_jobs=-1, algorithm='ball_tree', leaf_size=40, weights='uniform')
 print(clf_knn)
 clf_knn = clf_knn.fit(features_train, target_train)
 target_predicted_knn = clf_knn.predict(features_test)
 print(time.perf_counter() - start_time, "seconds" )<compute_train_metric>
def validate_submission_local_and_lb_mix(local_score, public_lb_score, local_size=5874, public_lb_size=2517):
return 1./(local_size + public_lb_size)*(local_size * local_score +
public_lb_size * public_lb_score )
start_time = time.perf_counter()
scores_knn = cross_val_score(clf_knn, features_train, target_train, cv=10, scoring='accuracy', n_jobs=-1)
 print("Cross Validation Score for each K",scores_knn)
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores_knn.mean() , scores_knn.std() * 2))
print(time.perf_counter() - start_time, "seconds" )<feature_engineering>
validate_submission_local_and_lb_mix(local_score=.47951, public_lb_score=.43187 )
digit_test['Label'] = pd.Series(target_predicted_knn)
 digit_test['ImageId'] = pd.Series(range(1,28001))<save_to_csv>
validate_submission_local_and_lb_mix(local_score=.469,
public_lb_score=.43267 )