Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
import pandas as pd
import numpy as np
from sklearn.metrics import f1_score
lm_learn.save_encoder('fine_tuned' )<define_variables>
train=pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )
ys = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult',
'identity_hate']<create_dataframe>
train.target.value_counts()
blocks =(TextBlock.from_df('comment_text', seq_len=lm_dls.seq_len, vocab=lm_dls.vocab),
MultiCategoryBlock(encoded=True, vocab=ys))<load_pretrained>
nltk.download('punkt')
 nltk.download('stopwords')
 !pip install contractions
nltk.download('wordnet')
 !pip install pyspellchecker
dls = toxic_clas.dataloaders(df )<choose_model_class>
stop_words=nltk.corpus.stopwords.words('english')
 i=0
wnl=WordNetLemmatizer()
stemmer=SnowballStemmer('english')
 for doc in train.text:
doc=re.sub(r'https?://\S+|www\.\S+','',doc)
 doc=re.sub(r'<.*?>','',doc)
 doc=re.sub(r'[^a-zA-Z\s]','',doc,re.I|re.A)
 doc=' '.join([wnl.lemmatize(i)for i in doc.lower().split() ])
 doc=contractions.fix(doc)
 tokens=nltk.word_tokenize(doc)
 filtered=[token for token in tokens if token not in stop_words]
doc=' '.join(filtered)
 train.text[i]=doc
i+=1
i=0
for doc in test.text:
doc=re.sub(r'https?://\S+|www\.\S+','',doc)
 doc=re.sub(r'<.*?>','',doc)
 doc=re.sub(r'[^a-zA-Z\s]','',doc,re.I|re.A)
 doc=' '.join([wnl.lemmatize(i)for i in doc.lower().split() ])
 doc=contractions.fix(doc)
 tokens=nltk.word_tokenize(doc)
 filtered=[token for token in tokens if token not in stop_words]
doc=' '.join(filtered)
 test.text[i]=doc
i+=1
loss_func = BCEWithLogitsLossFlat(thresh=0.8)
 metrics = [partial(accuracy_multi, thresh=0.8)]<choose_model_class>
tfidf=TfidfVectorizer(ngram_range=(1,1),use_idf=True)
 mat=tfidf.fit_transform(train.text ).toarray()
train_df=pd.DataFrame(mat,columns=tfidf.get_feature_names())
 test_df=pd.DataFrame(tfidf.transform(test.text ).toarray() ,columns=tfidf.get_feature_names())
 train_df.head()
learn = text_classifier_learner(dls, AWD_LSTM, metrics=metrics, loss_func=loss_func )<find_best_params>
model=LogisticRegression()
model.fit(train_df,train.target)
 print(f1_score(model.predict(train_df),train.target))
pred=model.predict(test_df )
<categorify><EOS> 
pd.DataFrame({
'id':test.id,
'target':pred
} ).to_csv('submission.csv',index=False )