Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<define_variables>
%matplotlib inline
pd.set_option('display.max_colwidth', 240)
 if 'transformers' not in sys.modules:
!pip install transformers
if 'nltk' not in sys.modules:
!pip install nltk
nltk.download('stopwords')
 is_colab = 'google.colab' in sys.modules
if is_colab:
is_kaggle = 'kaggle' in os.getcwd()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 device
if torch.cuda.is_available() :
print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
cat_attr = ['Sex','Embarked','Survived','Pclass','SibSp','Parch']
cat_attr<feature_engineering>
if is_colab:
drive.mount('/content/drive')
 DIR_NAME = '/content/drive/My Drive/Colab Notebooks/Spooky/data/'
elif is_kaggle:
DIR_NAME = "/kaggle/input/spooky-author-identification/"
zips = os.listdir(DIR_NAME)
 for name in zips:
with zipfile.ZipFile(DIR_NAME + name, 'r')as zip_ref:
zip_ref.extractall(".")
 else:
fnames = [r'./train.csv',r'./test.csv', r'./sample_submission.csv']
url = 'https://drive.google.com/drive/folders/1tP8T8_-6Xy5BgQa3q2g7reFHBv4Uer_a?usp=sharing'
if not os.path.exists(fnames[0]):
for fname in fnames:
urllib.request.urlretrieve(url, fname)
 print(fname, 'exists:', os.path.exists(fname))
DIR_NAME = './'
Titanic_data['Age'] = round(( Titanic_data['Age']),0)
 Titanic_data['Fare'] = round(( Titanic_data['Fare']),0)
 Titanic_data_test['Age'] = round(( Titanic_data_test['Age']),0)
 Titanic_data_test['Fare'] = round(( Titanic_data_test['Fare']),0 )<data_type_conversions>
if is_colab:
train = pd.read_csv(DIR_NAME+'train.csv')
 test = pd.read_csv(DIR_NAME+'test.csv')
 sample = pd.read_csv(DIR_NAME+'sample_submission.csv')
 else:
train = pd.read_csv('./train.csv')
 test = pd.read_csv('./test.csv')
 sample = pd.read_csv('./sample_submission.csv')
 train.head(3 )
Titanic_data['Age'].fillna(( Titanic_data['Age'].median()), inplace=True)
 Titanic_data_test['Age'].fillna(( Titanic_data_test['Age'].median()), inplace=True)
 Titanic_data_test['Fare'].fillna(( Titanic_data_test['Fare'].median()), inplace=True )<data_type_conversions>
PRE_TRAINED_MODEL_NAME = 'distilbert-base-cased'
tokenizer = DistilBertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)
 bert_model = DistilBertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)
 bert_embeddings = bert_model.get_input_embeddings()
for i in cat_attr:
Titanic_data[i] = Titanic_data[i].astype('category')
<data_type_conversions>
sample_sentence = "Creative minds are uneven, and the best of fabrics have their dull spots."
tokens = tokenizer.tokenize(sample_sentence)
 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 encoded = tokenizer.encode(sample_sentence)
 print(tokens)
 print(token_ids)
 print(encoded )
for i in cat_attr:
if i not in ['Survived']:
Titanic_data_test[i] = Titanic_data_test[i].astype('category' )<data_type_conversions>
train['word_ids'] = train['text'].apply(lambda x: tokenizer.convert_tokens_to_ids(tokenizer.tokenize(x)))
 train['word_count'] = train['word_ids'].apply(lambda x: len(x))
eng_stopword_toekns = set([tokenizer.convert_tokens_to_ids(str(x)) for x in stopwords.words("english")])
 train['stopword_count'] = train['word_ids'].apply(lambda x: len([token for token in x if token in eng_stopword_toekns]))
train.head(1 )
for i in num_attr:
Titanic_data[i] = Titanic_data[i].astype('int64' )<data_type_conversions>
def count_words(df, author_key):
df = df[df['author']==author_key]
word_count = df['text'].str.split(expand=True ).stack().value_counts()
return word_count
for i in num_attr:
Titanic_data_test[i] = Titanic_data_test[i].astype('int64' )<count_missing_values>
word_count_EAP = count_words(train, 'EAP')
 word_count_HPL = count_words(train, 'HPL')
 word_count_MWS = count_words(train, 'MWS')
 print(f'word_count_EAP size: {len(word_count_EAP)}')
 print(f'word_count_HPL size: {len(word_count_HPL)}')
 print(f'word_count_MWS size: {len(word_count_MWS)}' )
Titanic_data_test.isna().sum()<count_missing_values>
english_stopwords = stopwords.words("english")+ list(string.punctuation)
 word_count_EAP = word_count_EAP.drop(labels=english_stopwords, errors='ignore')
 word_count_HPL = word_count_HPL.drop(labels=english_stopwords, errors='ignore')
 word_count_MWS = word_count_MWS.drop(labels=english_stopwords, errors='ignore' )
Titanic_data.isna().sum()<prepare_x_and_y>
PCA_ON_N_WORDS = 1000
COMMON_WORDS_POOL = 400
PLOT_M_WORDS = 100
common_words = pd.Series(list(set(word_count_EAP[:COMMON_WORDS_POOL].index)& set(word_count_HPL[:COMMON_WORDS_POOL].index)& set(word_count_MWS[:COMMON_WORDS_POOL].index)))
 word_count_EAP = word_count_EAP.drop(common_words)
 word_count_HPL = word_count_HPL.drop(common_words)
 word_count_MWS = word_count_MWS.drop(common_words )
y= Titanic_data["Survived"]
X= Titanic_data.drop('Survived',axis=1 )<split>
cols = ['count']
word_count_EAP_top_n_for_pca = pd.DataFrame(word_count_EAP[:PCA_ON_N_WORDS], columns = cols)
 word_count_EAP_top_n_for_plot = pd.DataFrame(word_count_EAP[:PLOT_M_WORDS], columns = cols)
 word_count_EAP_top_n_for_pca['author'] = 'EAP'
word_count_EAP_top_n_for_plot['author'] = 'EAP'
word_count_HPL_top_n_for_pca = pd.DataFrame(word_count_HPL[:PCA_ON_N_WORDS], columns = cols)
 word_count_HPL_top_n_for_plot = pd.DataFrame(word_count_HPL[:PLOT_M_WORDS], columns = cols)
 word_count_HPL_top_n_for_pca['author'] = 'HPL'
word_count_HPL_top_n_for_plot['author'] = 'HPL'
word_count_MWS_top_n_for_pca = pd.DataFrame(word_count_MWS[:PCA_ON_N_WORDS], columns = cols)
 word_count_MWS_top_n_for_plot = pd.DataFrame(word_count_MWS[:PLOT_M_WORDS], columns = cols)
 word_count_MWS_top_n_for_pca['author'] = 'MWS'
word_count_MWS_top_n_for_plot['author'] = 'MWS'
df_for_pca = pd.concat([word_count_EAP_top_n_for_pca, word_count_HPL_top_n_for_pca, word_count_MWS_top_n_for_pca])
 df_for_pca = df_for_pca.reset_index()
df_for_pca.columns = ['word', 'count', 'author']
df_for_plot = pd.concat([word_count_EAP_top_n_for_plot, word_count_HPL_top_n_for_plot, word_count_MWS_top_n_for_plot])
 df_for_plot = df_for_plot.reset_index()
df_for_plot.columns = ['word', 'count', 'author']
df_for_plot.head(2 )
X_train,X_test,y_train,y_test = train_test_split(X,y, test_size = 0.3 , random_state = 2, stratify=y)
 X_train.shape<categorify>
def word_to_index(word):
token_id = tokenizer.convert_tokens_to_ids(word)
 return token_id
print(word_to_index('Hello'))
def indices_to_vec(word_ids):
embeded_tokens = bert_embeddings(torch.Tensor(word_ids ).to(torch.long))
return embeded_tokens.detach().numpy()
print(indices_to_vec([word_to_index('Hello'), word_to_index('Jacob')] ).shape)
 def index_to_vec(word_id):
embeded_token = bert_embeddings(torch.Tensor([word_id] ).to(torch.long))
return embeded_token.detach().numpy()
vec = index_to_vec(word_to_index('Hello'))
print(vec.shape )
cat_attr_x = ['Sex','Embarked','Pclass','SibSp','Parch']
cat_attr_x
X_train_imputed_dummies = pd.get_dummies(columns=cat_attr_x, data=X_train,prefix=cat_attr_x, prefix_sep="_", drop_first=True)
 X_test_imputed_dummies = pd.get_dummies(columns=cat_attr_x, data=X_test,prefix=cat_attr_x, prefix_sep="_", drop_first=True)
 Titanic_data_test_dummies = pd.get_dummies(columns=cat_attr_x, data=Titanic_data_test,prefix=cat_attr_x, prefix_sep="_", drop_first=True)
 X_train_imputed_dummies.columns<categorify>
df_for_plot['word_id'] = df_for_plot['word'].apply(word_to_index)
 df_for_pca['word_id'] = df_for_pca['word'].apply(word_to_index)
 vectors = index_to_vec(df_for_pca['word_id'].to_numpy())
 pca = PCA(n_components=2)
 pca.fit(vectors.squeeze())
 df_for_pca['word_vec'] = df_for_pca['word_id'].apply(index_to_vec)
 vectors.shape
Titanic_data_test_dummies=Titanic_data_test_dummies.drop(['Parch_9'], axis=1 )<normalization>
def vec_to_2dim(word_vec):
xy = pca.transform(word_vec)
 return xy[0][0], xy[0][1]
x, y = vec_to_2dim(vec)
 print(x, y)
 def tuple_x(xy):
return tuple(xy)[0]
print(tuple_x(( 1,2)))
 def tuple_y(xy):
return tuple(xy)[1]
print(tuple_y(( 1,2)) )
scaler = StandardScaler()
scaler.fit(X_train[num_attr])
 X_train_imputed_dummies[num_attr]=scaler.transform(X_train_imputed_dummies[num_attr])
 X_test_imputed_dummies[num_attr]=scaler.transform(X_test_imputed_dummies[num_attr])
 Titanic_data_test_dummies[num_attr]=scaler.transform(Titanic_data_test_dummies[num_attr] )<train_model>
df_for_pca['word_xy'] = df_for_pca['word_vec'].apply(vec_to_2dim)
 df_for_pca['word_x'] = df_for_pca['word_xy'].apply(tuple_x)
 df_for_pca['word_y'] = df_for_pca['word_xy'].apply(tuple_y )
clf = tree.DecisionTreeClassifier(max_depth=7)
 clf = clf.fit(X_train_imputed_dummies,y_train )<save_to_csv>
merged_df = df_for_plot.join(df_for_pca, on='word_id',lsuffix='_l', rsuffix='_r')
 df_cleaned = merged_df[merged_df['word_x'] < 0.6 ]
df_cleaned.columns
dot_data = tree.export_graphviz(clf, out_file=None,
feature_names=X_train_imputed_dummies.columns,
class_names='target',
filled=True, rounded=True, special_characters=True)
 graph = graphviz.Source(dot_data)
 graph<predict_on_test>
stats.normaltest(train['word_count'] )
train_pred = clf.predict(X_train_imputed_dummies)
 print(accuracy_score(y_train,train_pred))
test_pred = clf.predict(X_test_imputed_dummies)
 print(accuracy_score(y_test,test_pred))<save_to_csv>
stats.normaltest(train['stopword_count'] )
Titanic_data_test_pred = clf.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred })
 submission_svc.to_csv("submission_5_DT.csv", index=False )<train_model>
def calculate_one_way_anova(column_name):
return stats.f_oneway(train[column_name][train['author'] == 'EAP'],
train[column_name][train['author'] == 'HPL'],
train[column_name][train['author'] == 'MWS'] )
BaggingClassifier = BaggingClassifier(max_features=8,max_samples=0.3)
 BaggingClassifier.fit(X_train_imputed_dummies, y_train)
 y_pred_train = BaggingClassifier.predict(X_train_imputed_dummies)
 y_pred = BaggingClassifier.predict(X_test_imputed_dummies)
 bagging_accy = round(accuracy_score(y_pred, y_test), 20)
 print(bagging_accy)
 bagging_accy_train = round(accuracy_score(y_pred_train, y_train), 20)
 print(bagging_accy_train )<save_to_csv>
one_way = calculate_one_way_anova('word_count')
 print(f'The F score when comaring all the authors: {one_way.statistic}, which reflect pvalue of: {one_way.pvalue}' )
Titanic_data_test_pred_Bg = BaggingClassifier.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred })
 submission_svc.to_csv("submission_1_Bagging.csv", index=False )<set_options>
t_EAP_HPL = stats.ttest_ind(train['word_count'][train['author'] == 'EAP'], train['word_count'][train['author'] == 'HPL'])
 t_MWS_HPL = stats.ttest_ind(train['word_count'][train['author'] == 'MWS'], train['word_count'][train['author'] == 'HPL'])
 t_MWS_EAP = stats.ttest_ind(train['word_count'][train['author'] == 'MWS'], train['word_count'][train['author'] == 'EAP'])
 print(f'The T score when comaring EAP and HPL: {t_EAP_HPL.statistic} which reflect pvalue of: {t_EAP_HPL.pvalue}')
 print(f'The T score when comaring MWS and HPL: {t_MWS_HPL.statistic} which reflect pvalue of: {t_MWS_HPL.pvalue}')
 print(f'The T score when comaring MWS and EAP: {t_MWS_EAP.statistic} which reflect pvalue of: {t_MWS_EAP.pvalue}' )
np.random.seed(0)
 lr = LogisticRegression(solver='lbfgs')
 gnb = GaussianNB()
svc = LinearSVC(C=1.0)
 rfc = RandomForestClassifier(n_estimators=100)
 plt.figure(figsize=(10, 10))
ax1 = plt.subplot2grid(( 3, 1),(0, 0), rowspan=2)
 ax2 = plt.subplot2grid(( 3, 1),(2, 0))
ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated")
 for clf, name in [(lr, 'Logistic'), 
(gnb, 'Naive Bayes'), 
(svc, 'Support Vector Classification'), 
(rfc, 'Random Forest')]:
clf.fit(X_train_imputed_dummies, y_train)
 if hasattr(clf, "predict_proba"):
prob_pos = clf.predict_proba(X_test_imputed_dummies)[:, 1]
else:
prob_pos = clf.decision_function(X_test_imputed_dummies)
 prob_pos = \ 
(prob_pos - prob_pos.min())/(prob_pos.max() - prob_pos.min())
 fraction_of_positives, mean_predicted_value = \
calibration_curve(y_test, prob_pos, n_bins=10)
 ax1.plot(mean_predicted_value, fraction_of_positives, "s-",
label="%s" %(name,))
ax2.hist(prob_pos, range=(0, 1), bins=10, label=name,
histtype="step", lw=2)
 ax1.set_ylabel("Fraction of positives")
 ax1.set_ylim([-0.05, 1.05])
 ax1.legend(loc="lower right")
 ax1.set_title('Calibration plots(reliability curve)')
 ax2.set_xlabel("Mean predicted value")
 ax2.set_ylabel("Count")
 ax2.legend(loc="upper center", ncol=2)
 plt.tight_layout()
plt.show()<predict_on_test>
one_way = calculate_one_way_anova('stopword_count')
 print(f'The F score when comaring all the authors: {one_way.statistic}, which reflect pvalue of: {one_way.pvalue}' )
classifier = RandomForestClassifier(n_estimators=10000,max_depth=7)
 clf = Pipeline(memory = './',steps=[('classifier', classifier)])
 clf.fit(X=X_train_imputed_dummies, y=y_train)
 y_pred = clf.predict(X_train_imputed_dummies)
 print(accuracy_score(y_train,y_pred))
y_pred = clf.predict(X_test_imputed_dummies)
 print("Test Accuracy = ",accuracy_score(y_test,y_pred))<save_to_csv>
t_EAP_HPL = stats.ttest_ind(train['stopword_count'][train['author'] == 'EAP'], train['stopword_count'][train['author'] == 'HPL'])
 t_MWS_HPL = stats.ttest_ind(train['stopword_count'][train['author'] == 'MWS'], train['stopword_count'][train['author'] == 'HPL'])
 t_MWS_EAP = stats.ttest_ind(train['stopword_count'][train['author'] == 'MWS'], train['stopword_count'][train['author'] == 'EAP'])
 print(f'The T score when comaring EAP and HPL: {t_EAP_HPL.statistic} which reflect pvalue of: {t_EAP_HPL.pvalue}')
 print(f'The T score when comaring MWS and HPL: {t_MWS_HPL.statistic} which reflect pvalue of: {t_MWS_HPL.pvalue}')
 print(f'The T score when comaring MWS and EAP: {t_MWS_EAP.statistic} which reflect pvalue of: {t_MWS_EAP.pvalue}' )
Titanic_data_test_pred = clf.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred })
 submission_svc.to_csv("submission_3_RF.csv", index=False )<predict_on_test>
def author_to_label(author):
labels = {'EAP': 0,'HPL': 1,'MWS': 2}
return labels[author]
def label_to_author(label):
authors = ['EAP','HPL','MWS']
return authors[int(label)]
XGB_model = XGBClassifier(n_estimators=10000, gamma=2,learning_rate=0.05,max_depth=8,subsample=0.8,reg_lambda= 2)
 %time XGB_model.fit(X_train_imputed_dummies, y_train)
 y_pred = XGB_model.predict(X_test_imputed_dummies)
 print(accuracy_score(y_test,y_pred))
y_pred_train = XGB_model.predict(X_train_imputed_dummies)
 print(accuracy_score(y_train,y_pred_train))<save_to_csv>
train['label'] = train['author'].apply(author_to_label)
 dummies = pd.get_dummies(train['author'])
 train = pd.concat([train,dummies], axis=1)
 train.head(1 )
Titanic_data_test_pred_xg = XGB_model.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred_xg })
 submission_svc.to_csv("submission_4_XG.csv", index=False )<categorify>
class TrainDataSet(data.Dataset):
def __init__(self, excerpts, labels, tokenizer, max_len):
self.excerpts = excerpts
self.labels = labels
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.excerpts)
 def __getitem__(self, item):
excerpt = str(self.excerpts[item])
 encoding = self.tokenizer.encode_plus(
 excerpt,
max_length = self.max_len,
add_special_tokens=True,
pad_to_max_length=True,
return_attention_mask=True,
return_token_type_ids=False,
return_tensors='pt' 
)
 return {
'excerpt_text': excerpt,
'input_ids': encoding['input_ids'].flatten() ,
'attention_mask': encoding['attention_mask'].flatten() ,
'targets': torch.tensor(self.labels[item], dtype=torch.long)
 }
encoding_dim = 9
actual_dim = X_train_imputed_dummies.shape[1]
input_img = Input(shape =(actual_dim,))
encoded = Dense(encoding_dim,activation='relu' )(input_img)
 decoded = Dense(actual_dim,activation='sigmoid' )(encoded)
 autoencoder = Model(input_img,decoded)
 print(autoencoder.summary() )<categorify>
class TestDataSet(data.Dataset):
def __init__(self, ids, excerpts, tokenizer, max_len):
self.ids = ids
self.excerpts = excerpts
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.excerpts)
 def __getitem__(self, item):
excerpt = str(self.excerpts[item])
 excerpt_id = str(self.ids[item])
 encoding = self.tokenizer.encode_plus(
 excerpt,
max_length = self.max_len,
add_special_tokens=True,
pad_to_max_length=True,
return_attention_mask=True,
return_token_type_ids=False,
return_tensors='pt' 
)
 return {
'excerpt_id': excerpt_id,
'excerpt_text': excerpt,
'input_ids': encoding['input_ids'].flatten() ,
'attention_mask': encoding['attention_mask'].flatten() ,
}
autoencoder.compile(optimizer='adam', loss='binary_crossentropy')
 autoencoder.fit(X_train_imputed_dummies, X_train_imputed_dummies, epochs=100, batch_size=80)
 encoder = Model(input_img,encoded)
 print(encoder.summary() )<predict_on_test>
def create_train_data_loader(df, tokenizer, max_len, batch_size):
excerpts = df['text'].to_numpy() ,
print(f'Excerpts size: {len(excerpts)}')
 labels = df['label'].to_numpy() ,
dataset = TrainDataSet(excerpts=excerpts[0], labels=labels[0], tokenizer=tokenizer, max_len=max_len)
 print(f'Dataset size: {len(dataset)}')
 return data.DataLoader(dataset, batch_size=batch_size, num_workers=4, shuffle=True)
 def create_test_data_loader(df, tokenizer, max_len, batch_size):
excerpts = df['text'].to_numpy() ,
ids = df['id'].to_numpy() ,
print(f'Excerpts size: {len(excerpts)}')
 dataset = TestDataSet(ids= ids[0], excerpts=excerpts[0], tokenizer=tokenizer, max_len=max_len)
 print(f'Dataset size: {len(dataset)}')
 return data.DataLoader(dataset, batch_size=batch_size, num_workers=4 )
X_train_nonLinear_features = encoder.predict(X_train_imputed_dummies)
 X_test_nonLinear_features = encoder.predict(X_test_imputed_dummies)
 titanic_nonLinear_features = encoder.predict(Titanic_data_test_dummies)
 X_train1=np.concatenate(( X_train_imputed_dummies, X_train_nonLinear_features), axis=1)
 X_test1=np.concatenate(( X_test_imputed_dummies, X_test_nonLinear_features), axis=1)
 Titatanic_data1=np.concatenate(( Titanic_data_test_dummies, titanic_nonLinear_features), axis=1 )<train_model>
train_set, val_set = train_test_split(train, test_size=0.2 )
clf = SVC()
clf.fit(X_train1,y_train)
 print("Train Accuracy on Support Vector MAchine is -----", clf.score(X_train1,y_train))
print("Train Accuracy on Support Vector MAchine is -----",clf.score(X_test1,y_test))<choose_model_class>
BATCH_SIZE = 16
MAX_LEN = 160
model = Sequential()
model.add(Dense(32, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(32, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(16, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(16, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(4, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(1, activation='sigmoid'))<choose_model_class>
train_data_loader = create_train_data_loader(train_set, tokenizer, max_len=MAX_LEN, batch_size=BATCH_SIZE)
 val_data_loader = create_train_data_loader(val_set, tokenizer, max_len=MAX_LEN, batch_size=BATCH_SIZE )
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()<train_model>
class DistilBertAuthorClassifier(nn.Module):
def __init__(self):
super(DistilBertAuthorClassifier, self ).__init__()
self.num_labels = 3
self.softmax = nn.Softmax(dim=1)
 self.distilbert = DistilBertModel.from_pretrained('distilbert-base-cased')
 self.classifier = nn.Linear(self.distilbert.config.dim, 3)
 self.dropout = nn.Dropout(0.3)
 nn.init.xavier_normal_(self.classifier.weight)
 def forward(self, input_ids, attention_mask):
distilbert_output = self.distilbert(input_ids=input_ids,
attention_mask=attention_mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0, :]
pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits
model.fit(X_train1, y_train, epochs=80, verbose=1,batch_size=90, validation_data=(X_test1, y_test))<compute_test_metric>
gc.collect()
model = DistilBertAuthorClassifier()
model = model.to(device )
model.evaluate(X_test1, y_test, )<import_modules>
input_ids = sample['input_ids'].to(device)
 attention_mask = sample['attention_mask'].to(device)
 print(input_ids.shape)
 print(attention_mask.shape)
 prob, pred = torch.max(model(input_ids=input_ids, attention_mask=attention_mask),dim=1)
 print(prob)
 print(pred)
 print(sample['targets'] )
import pickle
import warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost as xgb
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, KFold, GridSearchCV<load_from_csv>
model.distilbert.config
training_set = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_set = pd.read_csv("/kaggle/input/titanic/test.csv" )<set_options>
EPOCHES = 3
if torch.cuda.is_available() :
optimizer = transformers.AdamW(model.parameters() , lr=2e-5, correct_bias=False)
 total_steps = len(train_data_loader)* EPOCHES
scheduler = transformers.get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=total_steps 
)
 loss_fn = nn.CrossEntropyLoss().to(device )
sns.set(style="whitegrid")
 pd.options.display.max_rows = 100
pd.options.display.max_columns = 100
warnings.filterwarnings('ignore' )<train_on_grid>
def train_epoch(model, data_loader, loss_fn, optimizer, scheduler, n_examples):
model = model.train()
losses = []
correct_predictions = 0
for d in data_loader:
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 targets = d['targets'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 _, preds = torch.max(outputs, dim=1)
 loss = loss_fn(outputs, targets)
 correct_predictions += torch.sum(preds == targets)
 losses.append(loss.item())
 loss.backward()
optimizer.step()
scheduler.step()
optimizer.zero_grad()
return correct_predictions.double() / n_examples, np.mean(losses )
def my_xgb(data, target, params):
X = data.drop(target, axis=1)
 y = data[target]
scaler = StandardScaler()
scaler.fit(X)
 X_scaled = scaler.transform(X)
 xgb_clf = xgb.XGBClassifier()
kf = KFold(n_splits=10, random_state=42, shuffle=True)
 gridsearch = GridSearchCV(xgb_clf, param_grid=params, scoring="accuracy", cv=kf, return_train_score=True)
 gridsearch.fit(X_scaled, y)
 return gridsearch, scaler<feature_engineering>
def eval_model(model, data_loader, loss_fn, n_examples):
model = model.eval()
losses = []
correct_predictions = 0
with torch.no_grad() :
for d in data_loader:
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 targets = d['targets'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 _, preds = torch.max(outputs, dim=1)
 loss = loss_fn(outputs, targets)
 correct_predictions += torch.sum(preds == targets)
 losses.append(loss.item())
 return correct_predictions.double() / n_examples, np.mean(losses )
nanbyfeature = pd.DataFrame(training_set.isna().sum() ).sort_values(by=0, ascending=False)
 nanbyfeature["percent"] = np.round(nanbyfeature[0] / len(training_set)* 100,2)
 nanbyfeature<categorify>
def predict_authors(model, data_loader, submission_df):
model = model.eval()
with torch.no_grad() :
for d in data_loader:
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 print(outputs)
def name_title(title):

 for word in title.split(" ")[1:-1]:
if(word in ["Mme.", "Ms.", "Mrs."]):
return "woman"
elif(word in ["Mr."]):
return "man"
elif(word in ["Master."]):
return "boy"
elif(word in ["Miss.", "Mlle."]):
return "miss"
elif(word in ["Capt.", "Col.", "Major.", "Rev.", "Dr."]):
return "army"
elif(word in ["Jonkheer.", "Don.", "Sir.", "Countess.", "Dona.", "Lady."]):
return "gentry"
else:
return "other"
training_set["title"] = training_set["Name"].apply(name_title)
 cat_analysis(training_set, "title", "title", "total_people", rotation=90, figsize=(16,5))
fig, ax = plt.subplots(figsize=(15,5))
ax = sns.barplot(x="title", y="Survived", data=training_set);
plt.xticks(rotation=0)
 plt.title("Percentage of survived by title")
 plt.show()
training_set.drop("title", axis=1, inplace=True )<categorify>
if is_colab and torch.cuda.is_available() :
optimizer = transformers.AdamW(model.parameters() , lr=2e-5, correct_bias=False)
 total_steps = len(train_data_loader)* EPOCHES
scheduler = transformers.get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=total_steps 
)
 loss_fn = nn.CrossEntropyLoss().to(device )
train_prep = prepare_data(training_set)
 test_prep = prepare_data(test_set )<create_dataframe>
filename = 'finalized_model.pt'
if is_colab and torch.cuda.is_available() :
model = model.to(device)
 history = defaultdict(list)
 best_accuracy = 0
for epoch in range(EPOCHES):
print(f'Epoch {epoch + 1}/{EPOCHES}')
 print('-'*10)
 train_acc, train_loss = train_epoch(
 model,
train_data_loader,
loss_fn,
optimizer,
scheduler,
len(train_set)
)
 print(f'Train loss: {train_loss}, accuracy: {train_acc}')
 val_acc, val_loss = eval_model(
 model,
val_data_loader,
loss_fn,
len(val_set)
)
 print(f'Validation loss: {val_loss}, accuracy: {val_acc}')
 torch.save(model.state_dict() , DIR_NAME+filename )
data_wcg_name = train_prep.copy() [["Sex_female", "boy", "wcg_surname", "Survived"]]
data_wcg_name["wcg_surname_size"] = 1
mask = data_wcg_name["wcg_surname"] != "no_group"
data_wcg_name = data_wcg_name[mask]
columns = {'Survived': 'survived_number'}
data_wcg_name = data_wcg_name.groupby("wcg_surname" ).agg({'Survived':'sum', 'wcg_surname_size':'count'} ).reset_index().rename(columns=columns)
 data_wcg_name["wcg_name_all_died"] = data_wcg_name["survived_number"].apply(lambda x: 1 if x == 0 else 0)
 data_wcg_name["wcg_name_all_survived"] = data_wcg_name["survived_number"] == data_wcg_name["wcg_surname_size"]
data_wcg_name["wcg_name_all_survived"] = data_wcg_name["wcg_name_all_survived"] * 1
wcg_name_test = test_prep.copy() [["wcg_surname"]]
wcg_name_test["wcg_surname_size"] = 1
wcg_name_test["survived_number"] = 0
wcg_name_test["wcg_name_all_died"] = 0
wcg_name_test["wcg_name_all_survived"] = 0
mask = wcg_name_test["wcg_surname"] != "no_group"
wcg_name_test = wcg_name_test[mask]
data_wcg_name = pd.concat([data_wcg_name, wcg_name_test])
 data_wcg_name = data_wcg_name.groupby("wcg_surname" ).sum().reset_index()
mask = data_wcg_name["wcg_surname_size"] > 1
data_wcg_name = data_wcg_name[mask]
data_wcg_name.drop(["survived_number", "wcg_surname_size"], axis=1, inplace=True)
 train_prep = train_prep.merge(data_wcg_name, how="left", on="wcg_surname" )<create_dataframe>
if not is_colab:
model.load_state_dict(torch.load('/kaggle/input/spookydistilbert/finalized_model.pt', map_location=device))
model.eval()
data_wcg_ticket = train_prep.copy() [["Sex_female", "boy", "wcg_ticket", "Survived"]]
data_wcg_ticket["wcg_ticket_size"] = 1
mask = data_wcg_ticket["wcg_ticket"] != "no_group"
data_wcg_ticket = data_wcg_ticket[mask]
columns = {'Survived': 'survived_number'}
data_wcg_ticket = data_wcg_ticket.groupby("wcg_ticket" ).agg({'Survived':'sum', 'wcg_ticket_size':'count'} ).reset_index().rename(columns=columns)
 data_wcg_ticket["wcg_ticket_all_died"] = data_wcg_ticket["survived_number"].apply(lambda x: 1 if x == 0 else 0)
 data_wcg_ticket["wcg_ticket_all_survived"] = data_wcg_ticket["survived_number"] == data_wcg_ticket["wcg_ticket_size"]
data_wcg_ticket["wcg_ticket_all_survived"] = data_wcg_ticket["wcg_ticket_all_survived"] * 1
wcg_ticket_test = test_prep.copy() [["wcg_ticket"]]
wcg_ticket_test["wcg_ticket_size"] = 1
wcg_ticket_test["survived_number"] = 0
wcg_ticket_test["wcg_ticket_all_died"] = 0
wcg_ticket_test["wcg_ticket_all_survived"] = 0
mask = wcg_ticket_test["wcg_ticket"] != "no_group"
wcg_ticket_test = wcg_ticket_test[mask]
data_wcg_ticket = pd.concat([data_wcg_ticket, wcg_ticket_test])
 data_wcg_ticket = data_wcg_ticket.groupby("wcg_ticket" ).sum().reset_index()
mask = data_wcg_ticket["wcg_ticket_size"] > 1
data_wcg_ticket = data_wcg_ticket[mask]
data_wcg_ticket.drop(["survived_number", "wcg_ticket_size"], axis=1, inplace=True)
 train_prep = train_prep.merge(data_wcg_ticket, how="left", on="wcg_ticket" )<feature_engineering>
test_data_loader = create_test_data_loader(test, tokenizer, max_len=MAX_LEN, batch_size=BATCH_SIZE)
 sample = next(iter(test_data_loader))
input_ids = sample['input_ids'].to(device)
 attention_mask = sample['attention_mask'].to(device)
 print(input_ids.shape)
 print(attention_mask.shape)
 prob, pred = torch.max(model(input_ids=input_ids, attention_mask=attention_mask),dim=1)
 print(prob)
 print(pred )
for i, row in train_prep.iterrows() :
if(( train_prep.loc[i, "wcg_name_all_died"] == 1)or(train_prep.loc[i, "wcg_ticket_all_died"] == 1)) :
train_prep.loc[i, "all_died"] = 1
elif(( train_prep.loc[i, "wcg_name_all_died"] == 0)or(train_prep.loc[i, "wcg_ticket_all_died"] == 0)) :
train_prep.loc[i, "all_died"] = 0
else:
train_prep.loc[i, "all_died"] = np.nan
if(( train_prep.loc[i, "wcg_name_all_survived"] == 1)or(train_prep.loc[i, "wcg_ticket_all_survived"] == 1)) :
train_prep.loc[i, "all_survived"] = 1
elif(( train_prep.loc[i, "wcg_name_all_survived"] == 0)or(train_prep.loc[i, "wcg_ticket_all_survived"] == 0)) :
train_prep.loc[i, "all_survived"] = 0
else:
train_prep.loc[i, "all_survived"] = np.nan<drop_column>
def test_model(model, data_loader, results_df):
model = model.eval()
submission = []
with torch.no_grad() :
for d in data_loader:
excerpt_ids = d['excerpt_id'],
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 outputs = nn.functional.softmax(outputs,dim=1)
 for i, excerpt_id in enumerate(excerpt_ids[0]):
results_df.loc[results_df['id'] == excerpt_id, ['EAP','HPL','MWS']] = outputs[i].tolist()
return results_df
train_prep.drop(["wcg_surname", "wcg_ticket", "wcg_name_all_died", "wcg_name_all_survived", "wcg_ticket_all_died", "wcg_ticket_all_survived"], axis=1, inplace=True )<feature_engineering>
results_df = pd.read_csv('./sample_submission.csv')
 results_df.head()
nanbyfeature2 = pd.DataFrame(train_prep.isna().sum() ).sort_values(by=0, ascending=False)
 nanbyfeature2["percent"] = np.round(nanbyfeature2[0] / len(train_prep)* 100,2)
 nanbyfeature2<define_search_space>
results_df = test_model(model, test_data_loader, results_df)
 results_df.head()
params = {
'learning_rate': [0.01, 0.05, 0.1],
'subsample': [1],
'colsample_bylevel': [1],
'colsample_bynode': [1],
'colsample_bytree': [0.5],
'gamma': [0, 1, 2],
'max_delta_step': [0],
'max_depth': [2, 3],
'min_child_weight': [1.6],
'n_estimators': [100],
'random_state': [42],
'scale_pos_weight': [1],
'seed': [42],
'n_jobs': [-1],
'reg_lambda': [1, 2, 4, 16]
}<normalization>
results_df.to_csv("submission.csv", index=False )
gridsearch, scaler = my_xgb(train_prep, "Survived", params )<find_best_params>
class LM:
def __init__(self, n):
self.n_gram = n
def train(self, text):
self.n_counts = defaultdict(Counter)
 for i in range(0, len(text)- self.n_gram + 1):
t = text[i:i+self.n_gram-1]
n_char = text[i+self.n_gram-1]
self.n_counts[t][n_char] += 1
def generate(self, init_text, n):
text = init_text
while len(text)< n:
lookup_text = text[-self.n_gram+1:]
if lookup_text not in self.n_counts:
break
counter = self.n_counts[text[-self.n_gram+1:]]
keys = list(counter.keys())
 values = list(counter.values())
 probs = [v/sum(values)for v in values]
cummulative_probs = np.cumsum(probs)
 p = np.random.rand()
for i in range(len(cummulative_probs)) :
if p <= cummulative_probs[i]:
text += keys[i]
break
return text
display(gridsearch.best_params_ )<find_best_model_class>
def get_trained_lm(n, excerpts):
lm = LM(n)
 text = ' '.join(list(excerpts))
lm.train(text)
 return lm
X_training_error = train_prep.copy()
y_training_error_true = X_training_error.pop("Survived")
 X_training_error_scaled = scaler.transform(X_training_error)
 y_training_error_pred = gridsearch.predict(X_training_error_scaled)
 training_set["true"] = y_training_error_true
training_set["pred"] = y_training_error_pred
true_pos = training_set["true"] == 1
true_false = training_set["true"] == 0
pred_pos = training_set["pred"] == 1
pred_false = training_set["pred"] == 0<filter>
n=7
lm_EAP = get_trained_lm(n, train[train['author']=='EAP']['text'])
 lm_HPL = get_trained_lm(n, train[train['author']=='HPL']['text'])
 lm_MWS = get_trained_lm(n, train[train['author']=='MWS']['text'])
 print(f"lm_EAP size: {len(lm_EAP.n_counts)}, lm_HPL size: {len(lm_HPL.n_counts)}, lm_MWS size: {len(lm_MWS.n_counts)}" )
<compute_test_metric><EOS> 
seed_string = "Dark night"
l = 120
models = {'Poe':lm_EAP, 'Lovecraft': lm_HPL, 'Shelley': lm_MWS}
for model_name in models.keys() :
generated = models[model_name].generate(seed_string, l)
 print(f""{generated}...", The bot {model_name}, 2020
" )