Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import string
import re
import nltk
from nltk.corpus import stopwords
from nltk import FreqDist, word_tokenize
from nltk.stem import WordNetLemmatizer,PorterStemmer
from sklearn import model_selection
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.linear_model import LogisticRegression
n=7
commits_df.loc[n, 'n_commit'] = 13
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.4
commits_df.loc[n, 'CV_logloss'] = 0.014625250378417162
commits_df.loc[n, 'LB_score'] = 0.01844<feature_engineering>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 train.head()
n=8
commits_df.loc[n, 'n_commit'] = 14
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.01
commits_df.loc[n, 'train_features'] = 1604
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014713482787703418
commits_df.loc[n, 'LB_score'] = 0.01849<feature_engineering>
train['target'].value_counts()
n=9
commits_df.loc[n, 'n_commit'] = 18
commits_df.loc[n, 'n_comp_GENES'] = 363
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014568689235607534
commits_df.loc[n, 'LB_score'] = 0.01841<feature_engineering>
temp = train.groupby('target' ).count() ['text'].reset_index()
temp['label'] = temp['target'].apply(lambda x : 'Disaster Tweet' if x==1 else 'Non Disaster Tweet')
 temp
n=10
commits_df.loc[n, 'n_commit'] = 19
commits_df.loc[n, 'n_comp_GENES'] = 550
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.91
commits_df.loc[n, 'train_features'] = 1218
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014577509066710863
commits_df.loc[n, 'LB_score'] = 0.01841<feature_engineering>
train.isnull().sum()
n=11
commits_df.loc[n, 'n_commit'] = 20
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1218
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014572358066092783
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
train.isnull().sum()
commits_df['QT_n_quantile_max'] = 100<feature_engineering>
test.isnull().sum()
n=12
commits_df.loc[n, 'n_commit'] = 21
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1223
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_max'] = 200
commits_df.loc[n, 'CV_logloss'] = 0.014585887029392697
commits_df.loc[n, 'LB_score'] = 0.01841<feature_engineering>
test.isnull().sum()
n=13
commits_df.loc[n, 'n_commit'] = 22
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1224
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_max'] = 500
commits_df.loc[n, 'CV_logloss'] = 0.014572447523411875
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
train.keyword.nunique() ,test.keyword.nunique()
n=14
commits_df.loc[n, 'n_commit'] = 23
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1212
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_max'] = 50
commits_df.loc[n, 'CV_logloss'] = 0.014581633680902033
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
train.location.nunique() ,test.location.nunique()
commits_df['QT_n_quantile_min'] = commits_df['QT_n_quantile_max']<feature_engineering>
train['location'].replace({'United States':'USA',
'New York':'USA',
"London":'UK',
"Los Angeles, CA":'USA',
"Washington, D.C.":'USA',
"California":'USA',
"Chicago, IL":'USA',
"Chicago":'USA',
"New York, NY":'USA',
"California, USA":'USA',
"FLorida":'USA',
"Nigeria":'Africa',
"Kenya":'Africa',
"Everywhere":'Worldwide',
"San Francisco":'USA',
"Florida":'USA',
"United Kingdom":'UK',
"Los Angeles":'USA',
"Toronto":'Canada',
"San Francisco, CA":'USA',
"NYC":'USA',
"Seattle":'USA',
"Earth":'Worldwide',
"Ireland":'UK',
"London, England":'UK',
"New York City":'USA',
"Texas":'USA',
"London, UK":'UK',
"Atlanta, GA":'USA',
"Mumbai":"India"},inplace=True)
 sns.barplot(y=train['location'].value_counts() [:5].index,x=train['location'].value_counts() [:5] )
n=15
commits_df.loc[n, 'n_commit'] = 24
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.84
commits_df.loc[n, 'train_features'] = 1215
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_min'] = 10
commits_df.loc[n, 'QT_n_quantile_max'] = 200
commits_df.loc[n, 'CV_logloss'] = 0.014578913453054567
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
for col in ['keyword', 'location']:
train[col] = train[col].fillna(f'no_{col}')
 for col in ['keyword', 'location']:
test[col] = test[col].fillna(f'no_{col}' )
n=15
commits_df.loc[n, 'n_commit'] = 24
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1223
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'QT_n_quantile_min'] = 50
commits_df.loc[n, 'QT_n_quantile_max'] = 1000
commits_df.loc[n, 'CV_logloss'] = 0.014576369890002664
commits_df.loc[n, 'LB_score'] = 0.01842<feature_engineering>
def clean_text(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 text = re.sub(r'<.*?>',' ' ,text)
 return text
train['text'] = train['text'].apply(lambda x: clean_text(x))
test['text'] = test['text'].apply(lambda x: clean_text(x))
train['text'].head()
commits_df['n_comp_total'] = commits_df['n_comp_GENES'] + commits_df['n_comp_CELLS']
commits_df['seed'] = 42<feature_engineering>
tokenizer = nltk.tokenize.RegexpTokenizer(r'\w+')
 train['text'] = train['text'].apply(lambda x: tokenizer.tokenize(x))
test['text'] = test['text'].apply(lambda x: tokenizer.tokenize(x))
train['text'].head()
commits_df['l_rate'] = 1e-3
commits_df.loc[11, 'l_rate'] = 5e-4<feature_engineering>
def remove_stopwords(text):

 words = [w for w in text if w not in stopwords.words('english')]
return words
train['text'] = train['text'].apply(lambda x : remove_stopwords(x))
test['text'] = test['text'].apply(lambda x : remove_stopwords(x))
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df = commits_df.sort_values(by=['LB_score', 'CV_logloss'], ascending = True ).reset_index(drop=True)
 commits_df['min'] = 0
commits_df.loc[0, 'min'] = 1
commits_df<sort_values>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )
commits_df.sort_values(by=['CV_logloss'], ascending = True )<load_from_csv>
def stemming(words):
ps=PorterStemmer()
return [ps.stem(word)for word in words]
train['text']=train['text'].apply(lambda x: stemming(x))
test['text']=test['text'].apply(lambda x: stemming(x))
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
def lemmatizing(words):
lemmatizer =WordNetLemmatizer()
return [lemmatizer.lemmatize(word)for word in words]
train['text']=train['text'].apply(lambda x: lemmatizing(x))
test['text']=test['text'].apply(lambda x: lemmatizing(x))
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<feature_engineering>
def final_text(words):
return ' '.join(words)
 train['text']=train['text'].apply(lambda x:final_text(x))
test['text']=test['text'].apply(lambda x:final_text(x))
train.head(10 )
<define_search_space>
count_vectorizer = CountVectorizer()
train_vectors = count_vectorizer.fit_transform(train['text'])
 test_vectors = count_vectorizer.transform(test["text"])
def calc_QT_par_kurt(QT_n_quantile_min=10, QT_n_quantile_max=200):
a = np.array([[1.53655,1], [30.4929,1]])
 b = np.array([QT_n_quantile_min, QT_n_quantile_max])
 return np.linalg.solve(a, b )<compute_train_metric>
tfidf = TfidfVectorizer(min_df=2, max_df=0.5, ngram_range=(1, 2))
train_vectors = tfidf.fit_transform(train['text'])
 test_vectors = tfidf.transform(test["text"] )
def n_quantile_for_kurt(kurt, calc_QT_par_kurt_transform):
return int(calc_QT_par_kurt_transform[0]*kurt + calc_QT_par_kurt_transform[1] )<feature_engineering>
clf = LogisticRegression(C=0.9,max_iter=1000,penalty='l2')
 scores = model_selection.cross_val_score(clf, train_vectors, train["target"], cv=7, scoring="f1")
 scores
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
clf.fit(train_vectors, train["target"] )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test_vectors)
 sample_submission.to_csv("submission.csv", index=False )