Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
%%capture
!apt update && apt install -y openjdk-8-jdk
!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
!python -m pip install --upgrade pip
!pip install --upgrade language-check pycontractions commonregex ekphrasis
sample = pd.read_csv("sample_submission.csv" )<load_from_csv>
%%capture
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
test_data = pd.read_csv("test.csv" )<load_from_csv>
%%capture
EMBEDDINGS = "glove-twitter-200"
cont = Contractions(api_key=EMBEDDINGS)
 cont.load_models()
text_processor = TextPreProcessor(
 normalize=['url', 'email', 'percent', 'money', 'phone', 'user', 'time', 'url', 'date', 'number'],
annotate={},
fix_html=True,
segmenter="twitter",
corrector="twitter",
unpack_hashtags=True,
unpack_contractions=True,
spell_correct_elong=True,
tokenizer=SocialTokenizer(lowercase=False ).tokenize,
dicts=[emoticons],
fix_text=True,
)
train_data = pd.read_csv("train.csv" )<drop_column>
def decontraction(corpus: List[str])-> List[str]:

 return list(cont.expand_texts(corpus))
def clean(corpus: List[str])-> List[str]:

 return list(map(lambda text: " ".join(text_processor.pre_process_doc(text)) , corpus))
train =train_data.drop(["id",'color','type'], axis=1)
 train<drop_column>
%%capture
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 train["clean_text"] = decontraction(train["text"].tolist())
 train["clean_text"] = clean(train["clean_text"].tolist())
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 test["clean_text"] = decontraction(test["text"].tolist())
 test["clean_text"] = clean(test["clean_text"].tolist() )
test = test_data.drop(["id",'color'], axis=1)
 test<filter>
%%capture
model = Pipeline(
 [ 
('embedder', FeatureUnion(
 [ 
('char', TfidfVectorizer(sublinear_tf=True, lowercase=True, ngram_range=(1, 10), analyzer='char_wb')) , 
('word', TfidfVectorizer(sublinear_tf=True, lowercase=True, ngram_range=(1, 4), stop_words='english'))
])
), 
('classifier', VotingClassifier(
 [ 
('clf1', CalibratedClassifierCV(LinearSVC() , cv=5)) , 
('clf2', ComplementNB()), 
('clf3', SGDClassifier(alpha=1e-4, max_iter=50, penalty="elasticnet"))
])
)
 ]
)
categorical=train_data.columns[train_data.dtypes=='object']
categorical<categorify>
macro_f1s = []
for seed in [0, 1, 2, 42, 56]:
train_, eval_ = tts(train, test_size=0.05, stratify=train["target"], random_state=seed)
 model.fit(train_["clean_text"], train_["target"])
 macro_f1s.append(classification_report(eval_["target"], model.predict(eval_["clean_text"]), output_dict=True)["macro avg"]["f1-score"])
 print(f"Max: {np.max(macro_f1s)} Min: {np.min(macro_f1s)} Avg: {np.mean(macro_f1s)}" )
dummies=pd.get_dummies(train_data['color'])
 dummies.columns=str('color')+'_'+dummies.columns
train=pd.concat([train,dummies],axis=1)
 train<categorify>
model.fit(train["clean_text"], train["target"] )
dummies=pd.get_dummies(test_data['color'])
 dummies.columns=str('color')+'_'+dummies.columns
test=pd.concat([test,dummies],axis=1)
 test<categorify>
model_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 model_submission['target'] = model.predict(test["clean_text"])
 model_submission.to_csv('model_submission.csv', index=False )
<prepare_x_and_y><EOS> 
train.to_csv("clean_train.csv")
 test.to_csv("clean_test.csv" )