Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_output>
%pip install --upgrade tensorflow-gpu
submission = pd.DataFrame({'id': test_id, 'type': predictions})
 submission.head()<save_to_csv>
pd.set_option("display.max_colwidth", None)
 set_config(display='diagram')
 def md(text: str):
display(Markdown(text))
def benchmark(y_test, y_hat):
print('_' * 80)
 print(f"f1 score: {metrics.f1_score(y_test, y_hat, average='macro'):.3f}")
 print("classification report:")
 print(metrics.classification_report(y_test, y_hat))
submission.to_csv('submission.csv', index=False )<save_to_csv>
folder = ".. /input/nlp-getting-started"
test = pd.read_csv(os.path.join(folder, "test.csv"), index_col="id")
 train = pd.read_csv(os.path.join(folder, "train.csv"), index_col="id")
 X = train["text"]
y = train["target"]
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42 )
submission.to_csv('submission.csv', index=False )<import_modules>
print(f"We are using `{len(X_train)}` rows for model training, and `{len(X_valid)}` rows for validation" )
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from datetime import datetime
from sklearn.neighbors import KNeighborsClassifier<load_from_csv>
vectorizers = {
"hashing": HashingVectorizer(stop_words="english", strip_accents="unicode", alternate_sign=False),
"tf-idf": TfidfVectorizer(stop_words="english", strip_accents="unicode", sublinear_tf=True),
}
pipe = Pipeline([ 
("vectorizer", None), 
("model", None)
 ] )
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv")
 sample = pd.read_csv('.. /input/sample_submission.csv')
<prepare_x_and_y>
feature_options = {
"vectorizer": [vectorizers["hashing"], vectorizers["tf-idf"]]
}
baseline_params = [
{
"model": [
ComplementNB() ,
],
**feature_options,
},
{
"model": [SGDClassifier() ],
"model__penalty": ["elasticnet", "l2"],
**feature_options,
},
{
"model": [LinearSVC() ],
"model__penalty": ["l1", "l2"],
**feature_options
}
]
grid = GridSearchCV(pipe, param_grid=baseline_params, cv=5, scoring="f1", n_jobs=-1, verbose=3)
 _ = grid.fit(X_train, y_train )
Y = train_data['type']
train_data = train_data.drop(['id'], axis=1)
 train_data = train_data.drop(['type'], axis=1)
 test_data = test_data.drop(['id'], axis=1)
 print(train_data.columns)
 print(test_data.columns )<count_values>
print(grid.best_params_)
 baseline = grid.best_estimator_
baseline
def exploreData(data, column):
return data[column].value_counts()
def imputation(data, column, value):
data.loc[data[column].isnull() , column] = value
def countMissing(data):
missing = data.columns[data.isnull().any() ].tolist()
return missing<categorify>
failed =(
 X_valid 
.to_frame("text")
.assign(
 y_hat=baseline.predict(X_valid),
y_true=y_valid 
)
)
 false_negatives = failed.pipe(lambda df: df[df["y_hat"].eq(0)& df["y_true"].eq(1)])
 false_positives = failed.pipe(lambda df: df[df["y_hat"].eq(1)& df["y_true"].eq(0)])
 false_negatives[:20]
cat_cols = ['color']
for col in cat_cols:
train_data = pd.concat(( train_data, pd.get_dummies(train_data[col], prefix=col)) , axis=1)
 del train_data[col]
test_data = pd.concat(( test_data, pd.get_dummies(test_data[col], prefix=col)) , axis=1)
 del test_data[col]<normalization>
del false_positives
del failed
train_data =(train_data - train_data.min())/(train_data.max() - train_data.min())
 test_data =(test_data - test_data.min())/(test_data.max() - test_data.min() )<count_values>
del false_negatives
print(Y.value_counts() )<define_variables>
def custom_tokenizer(nlp):

 tokenizer = nlp.tokenizer
special_cases = {
":)": [{"ORTH": ":)"}],
"&amp": [{"ORTH": "&"}]
}
for text, change in special_cases.items() :
tokenizer.add_special_case(text, change)
 infixes = nlp.Defaults.infixes +(r [\^\.] [\^\.] https?://
Pipeline step to merge hashtags together.
 Code based on examples here: https://spacy.io/usage/rule-based-matching
Y = [0 if y == 'Ghoul' else 1 if y == 'Goblin' else 2 for y in Y]<train_model>
nlp = create_nlp()
doc = nlp("^oo^ hello-world.:)http://t.co/ABC-DEF measures
pd.DataFrame({
"text": [token.text for token in doc],
"like_url": [token.like_url for token in doc],
"entity_type": [token.ent_type_ for token in doc],
"is_hashtag": [token._.is_hashtag for token in doc],
"is_ascii": [token.is_ascii for token in doc],
"is_punct": [token.is_punct for token in doc]
} )
rfe = RandomForestRegressor(n_estimators = 500)
 rfe.fit(train_data, Y)
 imp_score = rfe.feature_importances_
imp_score = pd.DataFrame({'feature': train_data.columns, 'score': imp_score})
 print(imp_score.sort_values(by = 'score', ascending = False))<find_best_params>
def is_valid_token(token: spacy.tokens.Token)-> bool:

 return(
 not token.like_url
and not token._.is_hashtag
and token.ent_type_ != "USERNAME"
and not token.is_punct
and token.text.strip() != ""
and token.is_ascii 
)
 def extract_text(
 tweet: spacy.tokens.Doc,
include_text: bool = True,
include_hashtags: bool = True,
include_handles: bool = False 
)-> str:
cleaned_tokens = []
for token in tweet:
if(token._.is_hashtag and include_hashtags):
cleaned_tokens.append(token.text.replace("
elif(token.ent_type_ == "USERNAME" and include_handles):
cleaned_tokens.append(token.text.replace("@", ""))
elif(is_valid_token(token)and include_text):
cleaned_tokens.append(token.text.lower())
 return " ".join(cleaned_tokens)
 def clean_tweets(tweets: List[str], **kwargs)-> List[str]:
nlp = create_nlp()
cleaned_tweets = []
for doc in nlp.pipe(tweets, n_process=1):
string = extract_text(doc, **kwargs)
 cleaned_tweets.append(string)
 return cleaned_tweets
neighbors = [1, 2, 5, 10]
param = []
scores = []
start = datetime.now()
print('start time: ', start)
 for n in neighbors:
knn = KNeighborsClassifier(n_neighbors = n)
 scores.append(cross_val_score(knn, train_data, Y, scoring="accuracy", cv = 10 ).mean())
 param.append(n)
 finish = datetime.now()
secs =(finish-start ).seconds
print('elapsed time in minutes: ', secs/60.0)
 scores = pd.DataFrame({'parameter': param, 'score': scores})
 print(scores.sort_values(by = 'score', ascending = False)) 
<predict_on_test>
cleaned_grid = clone(grid)
 new_pipe = Pipeline([ 
("clean", FunctionTransformer(clean_tweets)) , 
("vectorizer", None), 
("model", None)
 ])
 cleaning_param_grid = {
"clean__kw_args": [
{"include_text": text, "include_hashtags": hashtags, "include_handles": handles}
for text in [True]
for hashtags in [True]
for handles in [False]
]
}
old_param_grid = cleaned_grid.get_params() ["param_grid"]
cleaned_grid.set_params(
 estimator=new_pipe,
param_grid={**{key: [value] for key,value in grid.best_params_.items() }, **cleaning_param_grid} 
)
 cleaned_grid.get_params() ["param_grid"]
knn = KNeighborsClassifier(n_neighbors = 5)
 knn.fit(train_data, Y)
 res = knn.predict(test_data )<count_values>
cleaned_grid.fit(X_train, y_train )
type = ['Ghoul' if r == 0 else 'Goblin' if r == 1 else 'Ghost' for r in res]
type = pd.Series(type)
 print(type.value_counts() )<feature_engineering>
print(cleaned_grid.best_params_)
 cleaned_baseline = cleaned_grid.best_estimator_
sample = pd.read_csv('.. /input/sample_submission.csv')
 id = sample['id']
df = pd.DataFrame({'type': type})
 df['id'] = id
df = df[['id', 'type']]
df.columns
print(df.head(5))
df['type'].value_counts()<save_to_csv>
benchmark(y_valid, cleaned_baseline.predict(X_valid))
df.to_csv('knn_submission.csv', index=False)
 print(os.listdir('.'))<choose_model_class>
print(device_lib.list_local_devices() )
alphas = [0.01, 0.03, 0.1, 0.3, 1.0, 10]
regs = ["l1","l2"]
scores_1 = []
param = []
scores_2 = []
start = datetime.now()
print('start time: ', start)
 for alpha in alphas:
for reg in regs:
lm1 = OneVsOneClassifier(linear_model.LogisticRegression(C = alpha, multi_class = 'multinomial', solver = 'saga', penalty = reg))
lm2 = OneVsRestClassifier(linear_model.LogisticRegression(C = alpha, multi_class = 'multinomial', solver = 'saga', penalty = reg))
scores_1.append(cross_val_score(lm1, train_data, Y, scoring="accuracy", cv = 10 ).mean())
 scores_2.append(cross_val_score(lm2, train_data, Y, scoring="accuracy", cv = 10 ).mean())
 param.append([alpha, reg])
 finish = datetime.now()
secs =(finish-start ).seconds
print('elapsed time in minutes: ', secs/60.0)
 scores_1 = pd.DataFrame({'parameter': param, 'score': scores_1})
 scores_2 = pd.DataFrame({'parameter': param, 'score': scores_2})
 print('OneVsONe')
 print(scores_1.sort_values(by = 'score', ascending = False))
print('OneVsRest')
 print(scores_2.sort_values(by = 'score', ascending = False)) 
<train_model>
MAX_LENGTH = 64
PRETRAINED = "roberta-base"
def construct_bert(bert_layer, max_length: int = MAX_LENGTH, lr=5e-5, dropout=0.2):
attention_mask = tf.keras.Input(shape=(max_length,), dtype="int32", name="attention_mask")
 input_ids = tf.keras.Input(shape=(max_length,), dtype="int32", name="input_ids")
 output = bert_layer([input_ids, attention_mask])
 output = output[1]
output = tf.keras.layers.Dense(32,activation='relu' )(output)
 output = tf.keras.layers.Dropout(dropout )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs=[input_ids, attention_mask], outputs=output)
 model.compile(Adam(lr=lr), loss="binary_crossentropy", metrics=["accuracy"])
 return model
def bert_encode(tokenizer, texts: list, max_length: int = MAX_LENGTH):
encoded = tokenizer.batch_encode_plus(
 batch_text_or_text_pairs=texts,
max_length=max_length,
pad_to_max_length=True,
return_token_type_ids=False,
return_attention_mask=True,
return_tensors="tf" 
)
 return [encoded["input_ids"], encoded["attention_mask"]]
lm1 = OneVsOneClassifier(linear_model.LogisticRegression(C = 10, multi_class = 'multinomial', solver = 'saga', penalty = 'l1'))
lm1.fit(train_data, Y)
 res = lm1.predict(test_data)
 print(res )<count_values>
EPOCHS = 5
def objective(trial: optuna.Trial):
lr = trial.suggest_categorical("lr", [1e-5, 2e-5, 3e-5])
 batch_size = trial.suggest_categorical("batch_size", [16, 32])
 dropout_rate = trial.suggest_uniform('dropout_rate', 0.0, 1.0)
 clean = trial.suggest_categorical("clean_tweets", [True, False])
 monitor = "val_accuracy"
tokenizer = RobertaTokenizer.from_pretrained(PRETRAINED, do_lower_case=True)
 if clean:
include_hashtags = trial.suggest_categorical("include_hashtags", [True, False])
 include_handles = trial.suggest_categorical("include_handles", [True, False])
 tweets = clean_tweets(X_train, include_hashtags=include_hashtags, include_handles=include_handles)
 else:
tweets = X_train
encoded_tweets = bert_encode(tokenizer, tweets)
 model = construct_bert(TFRobertaModel.from_pretrained(PRETRAINED), lr=lr, dropout=dropout_rate)
 fit_params = {
"batch_size": batch_size,
"epochs": EPOCHS,
"callbacks": [
tf.keras.callbacks.EarlyStopping(
 monitor=monitor,
patience=1,
restore_best_weights=True 
),
TFKerasPruningCallback(trial, monitor)
 ],
"validation_split": 0.2
}
history = model.fit(encoded_tweets, y_train.to_numpy() , verbose=2, **fit_params)
 return max(history.history[monitor] )
type = ['Ghoul' if r == 0 else 'Goblin' if r == 1 else 'Ghost' for r in res]
type = pd.Series(type)
 print(type.value_counts() )<count_values>
if params["clean_tweets"]:
tweets = clean_tweets(X_train, include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets = X_train
tokenizer = RobertaTokenizer.from_pretrained(PRETRAINED, do_lower_case=True)
 encoded_tweets = bert_encode(tokenizer, tweets)
 model = construct_bert(TFRobertaModel.from_pretrained(PRETRAINED), lr=params["lr"], dropout=params["dropout_rate"])
 fit_params = {
"batch_size": params["batch_size"],
"epochs": 3,
"callbacks": [
tf.keras.callbacks.EarlyStopping(
 monitor="val_accuracy",
patience=0,
restore_best_weights=True 
)
 ],
"validation_split": 0.1
}
history = model.fit(encoded_tweets, y_train.to_numpy() , **fit_params )
df = pd.DataFrame({'type': type})
 df['id'] = id
df = df[['id', 'type']]
df.columns
print(df.head(5))
df['type'].value_counts()<save_to_csv>
if params["clean_tweets"]:
tweets_valid = clean_tweets(X_valid, include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets_valid = X_valid
encoded_tweets_valid = bert_encode(tokenizer, tweets_valid)
 preds = model.predict(encoded_tweets_valid )
df.to_csv('onevsone_submission.csv', index=False)
 print(os.listdir('.'))<train_model>
benchmark(y_valid, preds > 0.5 )
lm2 = OneVsRestClassifier(linear_model.LogisticRegression(C = 1.0, multi_class = 'multinomial', solver = 'saga', penalty = 'l1'))
lm2.fit(train_data, Y)
 res = lm2.predict(test_data)
 print(res )<count_values>
model = construct_bert(TFRobertaModel.from_pretrained(PRETRAINED), lr=params["lr"], dropout=params["dropout_rate"])
 if params["clean_tweets"]:
tweets_all = clean_tweets(X, include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets_all = X
encoded_tweets_all = bert_encode(tokenizer, tweets_all)
 history = model.fit(encoded_tweets_all, y.to_numpy() , **fit_params )
type = ['Ghoul' if r == 0 else 'Goblin' if r == 1 else 'Ghost' for r in res]
type = pd.Series(type)
 print(type.value_counts() )<count_values>
if params["clean_tweets"]:
tweets_test = clean_tweets(test["text"], include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets_test = test["text"]
encoded_tweets_test = bert_encode(tokenizer, tweets_test )
df = pd.DataFrame({'type': type})
 df['id'] = id
df = df[['id', 'type']]
df.columns
print(df.head(5))
df['type'].value_counts()<save_to_csv>
target = model.predict(encoded_tweets_test )
df.to_csv('onevsrest_submission.csv', index=False)
 print(os.listdir('.'))<set_options>
submission = pd.DataFrame({"target": target.reshape(-1 ).round().astype(int)}, index=test.index)
 submission.head()
<load_from_csv><EOS> 
submission.to_csv("submission.csv" )