Kaggle competition: Spooky Author Identification
train_df= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_df= pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train_df.head()<categorify>
df = pd.read_csv('/kaggle/input/spooky-author-identification/train.zip' )
def clean_data(df, text_col, new_col='cleaned_text'):

 tweets_data = df.copy()
tweets_data[new_col] = tweets_data[text_col]
tweets_data[new_col] = tweets_data[new_col].apply(lambda x : re.sub(
 '@[A-Za-z0-9]+', '', x))
tweets_data[new_col] = tweets_data[new_col].str.strip()
tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub('\
',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\W',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'https\s+|www.\s+',r'', str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'http\s+|www.\s+',r'', str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\s+[a-zA-Z]\s+',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\^[a-zA-Z]\s+',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\s+',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].str.lower()
tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\’", "'", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"won't", "will not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"can't", "can not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"don't", "do not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"dont", "do not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"n\’t", " not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"n't", " not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'re", " are", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'s", " is", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\’d", " would", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\d", " would", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'ll", " will", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'t", " not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'ve", " have", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'m", " am", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"
", "", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\r", "", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"[0-9]", "digit", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'", "", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r<string_transform>
df.drop(columns='id', inplace=True )
train_df = clean_data(train_df, "text", 'cleaned_text')
 stemming = PorterStemmer()
stemming.stem('runs')
 temp = train_df['cleaned_text'].apply(lambda sentence : [stemming.stem(x)for x in sentence.split(" ")])
 sentences = []
for i in temp:
sentences.append(" ".join(i))
sentences[0]
train_df['stemmed_text'] = sentences
del sentences, temp<string_transform>
df['text'] = df['text'].str.lower()
test_df = clean_data(test_df, "text", 'cleaned_text')
 stemming = PorterStemmer()
stemming.stem('runs')
 temp = test_df['cleaned_text'].apply(lambda sentence : [stemming.stem(x)for x in sentence.split(" ")])
 sentences = []
for i in temp:
sentences.append(" ".join(i))
sentences[0]
test_df['stemmed_text'] = sentences
del sentences, temp<concatenate>
df.author.value_counts()
tweets_data = pd.concat([train_df, test_df], axis=0, sort=False, ignore_index=True)
 tweets_data.head()<define_variables>
authors_dict = {'EAP' : 0,
'MWS' : 1,
'HPL' : 2}
path = Path('/kaggle/working/' )<create_dataframe>
df['author'] = df['author'].map(authors_dict )
data_lm =(TextList.from_df(tweets_data, path, cols = "cleaned_text" ).split_by_rand_pct(0.1 ).label_for_lm().databunch(bs=32))<define_variables>
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB
from sklearn.preprocessing import MaxAbsScaler
from sklearn.svm import SVC
from sklearn.pipeline import make_pipeline
from sklearn.metrics import log_loss, confusion_matrix, classification_report
from sklearn.model_selection import train_test_split
data_lm.show_batch()<load_pretrained>
X_train, X_test, y_train, y_test = train_test_split(df['text'], df['author'], test_size=0.30,
stratify=df['author'], random_state=1234)
 print(X_train.shape, X_test.shape, y_train.shape, y_test.shape )
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3 )<find_best_params>
vectorizer = TfidfVectorizer(ngram_range=(1,1), min_df=2, max_features=5000, lowercase=False, stop_words=None)
 scaler = MaxAbsScaler()
classifier = BernoulliNB(binarize=0, alpha=1 )
learn.lr_find()<train_model>
pl = make_pipeline(vectorizer, classifier )
learn.fit_one_cycle(5, 1e-2, moms=(0.8,0.7))<train_model>
pl.fit(X_train, y_train )
learn.fit_one_cycle(5, 1e-4, moms=(0.8,0.7))<categorify>
preds_prob = pl.predict_proba(X_test)
 preds_class = pl.predict(X_test )
learn.save_encoder('fine_tuned_enc' )<split>
nb_loss = log_loss(y_test, preds_prob)
 print(nb_loss )
X_train, X_val = train_test_split(train_df, test_size = 0.3 )<create_dataframe>
print(confusion_matrix(y_test, preds_class))
print('
')
 print(classification_report(y_test, preds_class))
data_classifier =(TextDataBunch.from_df('.', X_train, X_val, test_df, text_cols = "cleaned_text", label_cols = "target", vocab = data_lm.vocab))<define_variables>
!pip install tensorflow_text==2.3
data_classifier.show_batch()<load_pretrained>
import tensorflow
import tensorflow_hub as hub
import tensorflow_text as text
learn = text_classifier_learner(data_classifier, AWD_LSTM, drop_mult=0.7)
 learn.load_encoder('fine_tuned_enc' )<train_model>
URL_PREPROCESSOR = "http://tfhub.dev/tensorflow/albert_en_preprocess/2"
URL_ENCODER = "https://tfhub.dev/tensorflow/albert_en_base/2"
learn.fit_one_cycle(5, 5e-04, moms=(0.8,0.7))<train_model>
def build_classifier_model() :
text_input = tensorflow.keras.layers.Input(shape=() , dtype=tensorflow.string, name='text')
 preprocessing_layer = hub.KerasLayer(URL_PREPROCESSOR, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(URL_ENCODER, trainable=True, name='ALBERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tensorflow.keras.layers.Dropout(0.50 )(net)
 net = tensorflow.keras.layers.Dense(3, activation='softmax', name='classifier_b' )(net)
 return tensorflow.keras.Model(text_input, net )
learn.freeze_to(-2)
 learn.fit_one_cycle(5, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))<train_model>
classifier_model = build_classifier_model()
learn.freeze_to(-3)
 learn.fit_one_cycle(5, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))<train_model>
!pip install -q tf-models-official
learn.unfreeze()
learn.fit_one_cycle(5, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))<save_model>
from official.nlp import optimization
learn.save('final_learner' )<predict_on_test>
train_tf, test_tf = train_test_split(df, test_size=0.30, stratify=df['author'], random_state=1234)
 print(train_tf.shape, test_tf.shape )
def get_preds_as_nparray(ds_type)-> np.ndarray:

 preds = learn.get_preds(ds_type)[0].detach().cpu().numpy()
sampler = [i for i in data_classifier.dl(ds_type ).sampler]
reverse_sampler = np.argsort(sampler)
 return preds[reverse_sampler, :]<predict_on_test>
def df_to_dataset(dataframe, batch_size=32):
dataframe = dataframe.copy()
labels = dataframe.pop('author')
 ds = tensorflow.data.Dataset.from_tensor_slices(( dict(dataframe), labels))
ds = ds.batch(batch_size)
 return ds
test_preds = get_preds_as_nparray(DatasetType.Test)
 preds = []<prepare_output>
train_ds = df_to_dataset(train_tf, batch_size=32 )
for i in test_preds:
preds.append(np.argmax(i))<load_from_csv>
test_ds = df_to_dataset(test_tf, batch_size=32 )
sub = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sub.head(3 )<save_to_csv>
epochs = 5
steps_per_epoch = tensorflow.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * epochs
num_warmup_steps = int(0.1*num_train_steps)
 init_lr = 3e-5
optimizer = optimization.create_optimizer(init_lr=init_lr,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
optimizer_type='adamw' )
sub['target'] = preds
sub.to_csv('submission.csv', index=False)
 sub.head(3 )<import_modules>
classifier_model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] )
from tensorflow.python import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, Conv2D, Dropout, BatchNormalization, Activation, MaxPooling2D
from sklearn.model_selection import train_test_split<load_from_csv>
EPOCHS=3
with tensorflow.device('/device:GPU:0'):
history = classifier_model.fit(x=train_ds, validation_data=test_ds, batch_size=32, epochs=EPOCHS )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
classifier_model.evaluate(train_ds, batch_size=32 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
classifier_model.evaluate(test_ds, batch_size=32 )
img_rows, img_cols = 28, 28
num_classes = 10
train_x = train.values[:, 1:].astype('float32')
 train_y = keras.utils.to_categorical(train.label, num_classes)
 print(train_x.shape)
 print(train_y.shape)
 test_x = test.values.astype('float32')
 train_x /= 255
test_x /= 255
train_x = train_x.reshape(train_x.shape[0], img_rows, img_cols, 1)
 test_x = test_x.reshape(test_x.shape[0], img_rows, img_cols, 1)
 print(train_x.shape)
 print(train_y.shape )<choose_model_class>
tf_preds = classifier_model.predict(test_ds )
models = []
for idx in range(15):
model = Sequential()
model.add(Conv2D(32, kernel_size=3,
activation='relu',
input_shape=(img_rows, img_cols, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size=5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
 models.append(model )<categorify>
classifier_model.predict(X_test)[0:5]
def shift(arr, num, axis, fill_value=0.) :
result = np.empty_like(arr)
 if axis == 0:
if num > 0:
result[:num, :] = fill_value
result[num:, :] = arr[:-num, :]
elif num < 0:
result[num:, :] = fill_value
result[:num, :] = arr[-num:, :]
else:
result = arr
elif axis == 1:
if num > 0:
result[:, :num] = fill_value
result[:, num:] = arr[:, :-num]
elif num < 0:
result[:, num:] = fill_value
result[:, :num] = arr[:, -num:]
else:
result = arr
return result
def gen_img(arr):
shif = 4 *(1 if random.random() < 0.5 else -1)
 axis = random.randint(0, 1)
 return shift(arr, shif, axis)
 def gen_data(x, y):
new_x = np.empty(x.shape)
 new_y = np.empty(y.shape)
 for idx in range(x.shape[0]):
new_x[idx] = gen_img(x[idx])
 new_y[idx] = y[idx]
return new_x, new_y 
<train_model>
albert_loss = log_loss(y_test, tf_preds)
 print(albert_loss )
es = keras.callbacks.EarlyStopping(monitor='val_acc', mode='max', verbose=1, patience=50, restore_best_weights=True)
 lrs = keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 for i in range(0, 0):
extra_train_x, extra_train_y = gen_data(train_x, train_y)
 x = np.concatenate(( extra_train_x, train_x), axis=0)
 y = np.concatenate(( extra_train_y, train_y), axis=0)
 models[i].fit(x, y,
batch_size=128,
epochs=50,
validation_split=0.1,
callbacks=[es, lrs] )<split>
diff_loss = nb_loss - albert_loss
if albert_loss < nb_loss:
print("ALBERT improved the loss metric by {}".format(diff_loss))
else: print("loss metric was not improved by ALBERT compared to the base NB model" )
es = keras.callbacks.EarlyStopping(monitor='val_acc', mode='max', verbose=1, patience=50, restore_best_weights=True)
 lrs = keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 for i in range(0, 15):
datagen = keras.preprocessing.image.ImageDataGenerator(
 rotation_range=10,
zoom_range=0.10,
width_shift_range=0.10,
height_shift_range=0.10)
 gen_train_x, gen_val_x, gen_train_y, gen_val_y = train_test_split(train_x, train_y, test_size=0.1)
 models[i].fit_generator(datagen.flow(gen_train_x, gen_train_y, batch_size=64),
epochs=50,
steps_per_epoch=gen_train_x.shape[0]//64,
validation_data=(gen_val_x,gen_val_y),
callbacks=[lrs])
<predict_on_test>
tf_preds_class = np.argmax(tf_preds, axis=1 )
results = np.zeros(( test_x.shape[0], 10))
for i in range(15):
results = results + models[i].predict(test_x )<prepare_output>
print(confusion_matrix(y_test, tf_preds_class))
print('
')
 print(classification_report(y_test, tf_preds_class))
predictions = np.argmax(results, axis=1 )<prepare_output>
dep = pd.read_csv('/kaggle/input/spooky-author-identification/test.zip' )
submission = pd.DataFrame(data={'ImageId': pd.RangeIndex(start=1, stop=test_x.shape[0]+1), 'Label': predictions})
 submission = submission.set_index('ImageId' )<save_to_csv>
dep_preds = classifier_model.predict(dep['text'].values)
 dep_df = pd.DataFrame(data=dep_preds, columns=['EAP','MWS','HPL'] )
<install_modules><EOS> 
submit_df = pd.concat([dep,dep_df], axis='columns')
 submit_df.drop(columns='text', inplace=True)
 submit_df.to_csv('submit.csv', index=False, index_label=False )