Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
!pip install -U cufflinks
!pip install lemminflect
!pip install twitter-text-python
!pip install folium
<feature_engineering>
init_notebook_mode(connected=False)
 cf.go_offline()
pd.options.display.max_rows = 200
pd.options.display.max_columns = 200
pd.options.display.max_colwidth = 200
plt.style.use('ggplot' )
df['peak'] = df[['hour', 'workingday']].apply(lambda x:(0, 1)[(x['workingday'] == 1 and(x['hour'] == 8 or 17 <= x['hour'] <= 18 or 12 <= x['hour'] <= 13)) or(x['workingday'] == 0 and 10 <= x['hour'] <= 19)], axis = 1 )<feature_engineering>
DATA_DIR = '/kaggle/input/nlp-getting-started/'
train = pd.read_csv(pjoin(DATA_DIR, 'train.csv'))
test = pd.read_csv(pjoin(DATA_DIR, 'test.csv'))
train['is_train'] = True
test['is_train'] = False
df = pd.concat(
 [train, test],
sort=False, ignore_index=True
).set_index('id' ).sort_index()
print(train.shape, test.shape, df.shape)
 df.head()
df['holiday'] = df[['month', 'day', 'holiday', 'year']].apply(lambda x:(x['holiday'], 1)[x['year'] == 2012 and x['month'] == 10 and(x['day'] in [30])], axis = 1)
 df['holiday'] = df[['month', 'day', 'holiday']].apply(lambda x:(x['holiday'], 1)[x['month'] == 12 and(x['day'] in [24, 26, 31])], axis = 1)
 df['workingday'] = df[['month', 'day', 'workingday']].apply(lambda x:(x['workingday'], 0)[x['month'] == 12 and x['day'] in [24, 31]], axis = 1)
<feature_engineering>
df.loc[df.is_train, 'target'].value_counts(normalize=True )
df['ideal'] = df[['temp', 'windspeed']].apply(lambda x:(0, 1)[x['temp'] > 27 and x['windspeed'] < 30], axis = 1)
 df['sticky'] = df[['humidity', 'workingday']].apply(lambda x:(0, 1)[x['workingday'] == 1 and x['humidity'] >= 60], axis = 1 )<predict_on_test>
def lemmatize(word, upos='VERB'):
try:
lemma = getLemma(word, upos=upos, lemmatize_oov=True)
 except AssertionError:
lemma = getLemma(word, upos='VERB', lemmatize_oov=True)
 if not lemma:
lemma = getLemma(word, upos='VERB', lemmatize_oov=True)
 return lemma[0].lower()
df.keyword = df.keyword.fillna('NaN' ).astype(str ).str.replace('(%20)+', '_')
 print(df.keyword.nunique())
 top_n = 10
df.groupby('keyword' ).agg(
 tweet_cnt=pd.NamedAgg(column='target', aggfunc='count'),
mean_target=pd.NamedAgg(column='target', aggfunc='mean'),
).sort_values(by=['tweet_cnt', 'mean_target'], ascending=[False, False] ).head(top_n )
def get_rmsle(y_pred, y_actual):
diff = np.log(y_pred + 1)- np.log(y_actual + 1)
 mean_error = np.square(diff ).mean()
return np.sqrt(mean_error)
 def get_data() :
data = df[df['_data'] == 'train'].copy()
return data
def custom_train_test_split(data, cutoff_day=15):
train = data[data['day'] <= cutoff_day]
test = data[data['day'] > cutoff_day]
return train, test
def prep_data(data, input_cols):
X = data[input_cols]
y_r = data['registered_log']
y_c = data['casual_log']
return X, y_r, y_c
def predict_on_validation_set(model, input_cols):
data = get_data()
train, test = custom_train_test_split(data)
 X_train, y_train_r, y_train_c = prep_data(train, input_cols)
 X_test, y_test_r, y_test_c = prep_data(test, input_cols)
 model_r = model.fit(X_train, y_train_r)
 y_pred_r = np.exp(model_r.predict(X_test)) - 1
model_c = model.fit(X_train, y_train_c)
 y_pred_c = np.exp(model_c.predict(X_test)) - 1
y_pred_comb = np.round(y_pred_r + y_pred_c)
 y_pred_comb[y_pred_comb < 0] = 0
y_test_comb = np.exp(y_test_r)+ np.exp(y_test_c)- 2
score = get_rmsle(y_pred_comb, y_test_comb)
 return(y_pred_comb, y_test_comb, score)
 df_test = df[df['_data'] == 'test'].copy()
def predict_on_test_set(model, x_cols):
df_train = df[df['_data'] == 'train'].copy()
X_train = df_train[x_cols]
y_train_cas = df_train['casual_log']
y_train_reg = df_train['registered_log']
X_test = df_test[x_cols]
casual_model = model.fit(X_train, y_train_cas)
 y_pred_cas = casual_model.predict(X_test)
 y_pred_cas = np.exp(y_pred_cas)- 1
registered_model = model.fit(X_train, y_train_reg)
 y_pred_reg = registered_model.predict(X_test)
 y_pred_reg = np.exp(y_pred_reg)- 1
return y_pred_cas + y_pred_reg<choose_model_class>
lemmatized = dict(
 zip(
 df.keyword.drop_duplicates() ,
df.keyword.drop_duplicates().str.split('[_ ]+')
.apply(
 lambda words: '_'.join(
 lemmatize(w, upos='VERB')for w in sorted(words)) 
)
)
)
 print(
 len(set(lemmatized.keys())) ,
len(set(lemmatized.values())) ,
1 - len(set(lemmatized.values())) / len(set(lemmatized.keys()))
)
params = {'n_estimators': 1000, 'max_depth': 15, 'random_state': 0, 'min_samples_split' : 5, 'n_jobs': -1}
rf_model = RandomForestRegressor(**params)
 rf_cols = [
'weather', 'temp', 'atemp', 'windspeed',
'workingday', 'season', 'holiday', 'sticky',
'hour', 'dow', 'woy', 'peak',
]
rf_p, rf_t, rf_score = predict_on_validation_set(rf_model, rf_cols)
 print(rf_score)
<choose_model_class>
merged = pd.DataFrame(
 data=lemmatized.items() ,
columns=['init', 'new']
).groupby('new')['init'].apply(list)
 merged[merged.apply(len)> 1]
params = {'n_estimators': 150, 'max_depth': 5, 'random_state': 0, 'min_samples_leaf' : 10, 'learning_rate': 0.1, 'subsample': 0.7, 'loss': 'ls'}
gbm_model = GradientBoostingRegressor(**params)
 gbm_cols = [
'weather', 'temp', 'atemp', 'humidity', 'windspeed',
'holiday', 'workingday', 'season',
'hour', 'dow', 'year', 'ideal'
] 
(gbm_p, gbm_t, gbm_score)= predict_on_validation_set(gbm_model, gbm_cols)
 print(gbm_score)
 y_p = np.round (.2*rf_p +.8*gbm_p)
 print(get_rmsle(y_p, rf_t)) 
<predict_on_test>
df['keyword_normalized'] = df.keyword.map(lemmatized)
 print(df.keyword.nunique())
 keywords_grouped = df.groupby('keyword_normalized' ).agg(
 tweet_cnt=pd.NamedAgg(column='target', aggfunc='count'),
mean_target=pd.NamedAgg(column='target', aggfunc='mean'),
).sort_values(by=['tweet_cnt', 'mean_target'], ascending=[False, False])
 keywords_grouped.head(top_n*2 )
rf_pred = predict_on_test_set(rf_model, rf_cols)
 gbm_pred = predict_on_test_set(gbm_model, gbm_cols)
<prepare_x_and_y>
wc_params = dict(
 max_font_size=42,
background_color=None,
mode='RGBA',
max_words=200,
width=300,
height=300,
collocations=False,
relative_scaling=0.6, 
)
 wordcloud_disaster = WordCloud(**wc_params ).generate(
 ' '.join(
 df.loc[
df['keyword_normalized'].replace({'nan': np.nan} ).isin(
 keywords_grouped[keywords_grouped['mean_target'] >= 0.5].index.tolist() 
),
'keyword_normalized'
].values.tolist() 
)
)
 wordcloud_neutral = WordCloud(**wc_params ).generate(
 ' '.join(
 df.loc[
df['keyword_normalized'].replace({'nan': np.nan} ).isin(
 keywords_grouped[keywords_grouped['mean_target'] <= 0.33].index.tolist() 
),
'keyword_normalized'
].values.tolist() 
)
)
 plt.figure(figsize=(7, 7))
plt.title('`Disastrous` keywords')
 plt.imshow(wordcloud_disaster, interpolation="bilinear")
 plt.axis("off")
 plt.show()
plt.figure(figsize=(7, 7))
plt.title('`Neutral` keywords')
 plt.imshow(wordcloud_neutral, interpolation="bilinear")
 plt.axis("off")
 plt.show()
y_pred = np.round (.20*rf_pred +.80*gbm_pred)
<save_to_csv>
tparser = ttp.Parser()
def extract_tweet_entities(tweet_text):

 return tparser.parse(tweet_text)
 tweet_entities = {
tweet_id: extract_tweet_entities(text)
 for(tweet_id, text)in df.reset_index() [['id', 'text']].values.tolist()
}
df_test['count'] = y_pred
final_df = df_test[['datetime', 'count']].copy()
final_df.to_csv('output5.csv', index=False )<load_from_csv>
hashtags_stats[ 
(hashtags_stats.tweet_cnt > 6)
 &(hashtags_stats.mean_target > 0.75)
 ].sort_values(by=['mean_target', 'tweet_cnt'], ascending=[False, False] )
train = pd.read_csv(".. /input/train.csv", parse_dates = ["datetime"])
 test = pd.read_csv(".. /input/test.csv", parse_dates = ["datetime"] )<feature_engineering>
hashtags_stats[ 
(hashtags_stats.tweet_cnt > 6)
 &(hashtags_stats.mean_target <= 0.25)
 ].sort_values(by=['mean_target', 'tweet_cnt'], ascending=[True, False] )
train["datetime"].dt.hour<count_values>
G_ht = nx.Graph()
G_ht.add_nodes_from(hashtags_stats.index.tolist())
 d = defaultdict(int)
 for taglist in df.hashtags:
for c in combinations(taglist, 2):
d[c] += 1
G_ht.add_edge(( *c))
for n1, n2, attrs in G_ht.edges(data=True):
attrs['weight'] = np.log1p(d[(n1, n2)])
 print(
 f'Init stats:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 weak_edges = []
for n1, n2, attrs in G_ht.edges(data=True):
if attrs['weight'] <= np.log1p(1):
weak_edges.append(( n1, n2))
G_ht.remove_edges_from(weak_edges)
 print(
 f'After weak edge removal:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 isolates = list(nx.isolates(G_ht))
G_ht.remove_nodes_from(isolates)
 print(
 f'After isolates removal:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 node_degrees = pd.DataFrame(
 [x for x in G_ht.degree() ],
columns=['node', 'degree']
).sort_values(by='degree', ascending=False ).reset_index()
G_ht.remove_node(node_degrees.iloc[0]['node'])
 print(
 f'After highest node degree removal:
Nodes: {G_ht.number_of_nodes() }
Edges: {G_ht.number_of_edges() }')
 cc_ht = sorted(list(nx.connected_components(G_ht)) ,
key=lambda x: len(x), reverse=True)
 min_size = 5
cc_ht = [c for c in cc_ht if len(c)> min_size]
print(f'Top connected Components: {len(cc_ht)}' )
train["datetime"].dt.hour.value_counts()<feature_engineering>
df['hashtag_cluster'] = [[] for i in range(len(df)) ]
for(ri, r)in tqdm(list(df.iterrows()), total=len(df)) :
for(i, c)in enumerate(cc_ht):
if set(r['hashtags'] ).intersection(c):
df.loc[ri, 'hashtag_cluster'].append(i)
 if not df.loc[ri, 'hashtag_cluster']:
df.loc[ri, 'hashtag_cluster'].append(-1)
 t = df.explode('hashtag_cluster')['hashtag_cluster']\ 
.reset_index().rename(columns={'index': 'id'})
 df_ht_clusters = pd.crosstab(t.id, t.hashtag_cluster)
 df_ht_clusters.columns = [f'ht_cluster__{c}' for c in df_ht_clusters.columns]
print(df.shape)
 df = df.drop(df.filter(regex='^ht_cluster' ).columns, axis=1)
 df = pd.concat([df, df_ht_clusters], axis=1)
 print(df.shape)
 df_ht_clusters.head()
train["year"] = train["datetime"].dt.year
train["hour"] = train["datetime"].dt.hour
train["dayofweek"] = train["datetime"].dt.dayofweek<feature_engineering>
geodata = pd.read_csv('.. /input/disaster-tweets-geodata/geodata.csv' ).set_index('id')
 geodata.head()
test["year"] = test["datetime"].dt.year
test["hour"] = test["datetime"].dt.hour
test["dayofweek"] = test["datetime"].dt.dayofweek<count_values>
print(df.shape)
 df = df.merge(geodata, left_index=True, right_index=True, how='left')
 df = df[~df.index.duplicated(keep='first')]
print(df.shape)
 df[['location', 'target'] + geodata.columns.tolist() ].sample(5, random_state=911 )
train["datetime"].dt.month.value_counts()<count_values>
EMOJI_REGEX = re.compile(
 "["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+",
flags=re.UNICODE 
)
 NON_WORD_PATTERN = r"[^A-Za-z0-9\.'!\?,\$\s]"
PUNCT_DEDUPLICATION_REGEX = re.compile(r'[\?\.\!]+(?=[\?\.\!])', re.I )
train["season"].value_counts()<count_values>
df['text_processed'] = df.text.str.replace(EMOJI_REGEX, ' EMOJI ')\ 
.str.replace(PUNCT_DEDUPLICATION_REGEX, '')\ 
.str.replace(URL_REGEX, ' URL ' ).str.replace(NON_WORD_PATTERN, '' ).str.replace('\s+', ' ')
 df['text_processed'].sample(10, random_state=911 ).values.tolist()
train["datetime"].dt.day.value_counts()<count_values>
def compute_doc2vec(words, w2v, weight_dict, emb_size):

 doc_vector = np.zeros(emb_size, dtype=np.float32)
 weights = 0
for w in words:
try:
doc_vector += w2v.wv[w]
weights += weight_dict.get(w, 0)
 except KeyError:
pass
doc_vector /= max(0.01, weights)
 return doc_vector / np.linalg.norm(doc_vector, ord=2)
 text_field = 'text_processed'
prepared_texts = df[text_field].str.lower().str.replace(
 '', ''
).str.extractall(
 '(?P<words>[A-Za-z]+)' ).groupby(level=0)['words'].apply(list ).reindex(df.index ).fillna('EMPTY')
 w2v_size = 16
w2v = Word2Vec(
 sentences=prepared_texts,
size=w2v_size,
window=32,
min_count=7,
iter=5,
negative=5,
sg=1,
batch_words=2**11, 
)
 word_dict = np.log1p(prepared_texts.explode().value_counts() ).to_dict()
w2v_df = pd.DataFrame(
 data=np.concatenate(
 prepared_texts.apply(
 lambda x: compute_doc2vec(
 x, w2v, weight_dict=word_dict, emb_size=w2v_size)
 ).values
).reshape(( len(df), w2v_size)) ,
columns=[f'w2v__{i}' for i in range(w2v_size)] 
)
 df = df.drop(df.filter(regex='^w2v__' ).columns, axis=1)
 print(df.shape)
 df = pd.concat([df, w2v_df], axis=1)
 print(df.shape )
test["datetime"].dt.day.value_counts()<prepare_x_and_y>
hashtags_sentences = df.explode('hashtags')['hashtags'].groupby(level=[0] ).apply(
 lambda x: ' '.join([re.sub(r'[^\x00-\x7f]', r'', s)for s in list(x.astype(str)) ])
)
 hashtags_sentences.sample(10, random_state=911 )
y_casual = np.log1p(train.casual)
 y_registered = np.log1p(train.registered )<drop_column>
tfidf = TfidfVectorizer(
 max_df=1.0,
min_df=5,
dtype=np.float32,
max_features=200, 
)
 tfidf.fit(hashtags_sentences)
 tag_df = pd.DataFrame(
 data=tfidf.transform(hashtags_sentences ).todense() ,
columns=[f'tag__{k}' for k, v in sorted(tfidf.vocabulary_.items() , key=lambda item: item[1])] 
)
 print(tag_df.shape)
 tag_df.head()
df = df.drop(df.filter(regex='^tag__' ).columns, axis=1)
 print(df.shape)
 df = pd.concat([df, tag_df], axis=1)
 print(df.shape )
train.drop(["datetime", "windspeed", "casual", "registered", "count"], 1)
 test.drop(["datetime", "windspeed"], 1 )<import_modules>
embedder = hub.load("https://tfhub.dev/google/universal-sentence-encoder-large/5")
 df_univ_embeddings = pd.DataFrame(
 data=embedder(df[text_field].values ).numpy() ,
columns=[f'tf_emb__{i}' for i in range(512)] 
)
 df_univ_embeddings.head()
from lightgbm import LGBMRegressor<choose_model_class>
df['text_len'] = df.text.apply(len ).astype(np.int16)
 df['text_word_cnt'] = df.text.str.split('\s+' ).map(len ).astype(np.int16)
 df['has_exclamations'] = df.text.str.contains(r'[?!]' ).astype(np.int8)
 df['has_uppercased'] = df.text.str.contains(r'[A-Z]{2,}' ).astype(np.int8)
 nlp_features = [
'has_exclamations',
'has_uppercased',
'text_len',
'text_word_cnt',
]
df[nlp_features + ['target']].corr()
model = LGBMRegressor()<feature_engineering>
features_num =(
 [
'hashtag_cnt',
'url_cnt',
'user_cnt',
'lat',
'lon'
]
+ tag_df.columns.tolist()
+ w2v_df.columns.tolist()
+ df_ht_clusters.columns.tolist()
+ df_univ_embeddings.columns.tolist() 
)
 features_cat = [
'country',
'city',
'keyword_normalized',
]
cat_cols = []
for col in features_cat:
df[f'{col}_code'] = df[col].astype('category' ).cat.codes
cat_cols.append(f'{col}_code')
 features = features_num + cat_cols
print(df[features].dtypes)
 df[features].head()
train = pd.read_csv(".. /input/train.csv", parse_dates = ["datetime"])
 test = pd.read_csv(".. /input/test.csv", parse_dates = ["datetime"])
 train["year"] = train["datetime"].dt.year
train["hour"] = train["datetime"].dt.hour
train["dayofweek"] = train["datetime"].dt.dayofweek
test["year"] = test["datetime"].dt.year
test["hour"] = test["datetime"].dt.hour
test["dayofweek"] = test["datetime"].dt.dayofweek
y_casual = np.log1p(train.casual)
 y_registered = np.log1p(train.registered)
 train.drop(["datetime", "windspeed", "casual", "registered", "count"], 1, inplace=True)
 test.drop(["datetime", "windspeed", ], 1, inplace=True )<train_model>
def f1_score_lgb(preds, dtrain):
labels = dtrain.get_label()
f_score = f1_score(
 np.round(preds),
labels,
average='macro' 
)
 return 'f1_score', f_score, True
skf = StratifiedKFold(n_splits=5, random_state=911, shuffle=True)
 lgb_params = {
'num_leaves': 63,
'learning_rate': 0.01,
'max_depth': -1,
'subsample': 0.8,
'colsample_bytree': 0.25,
'cat_l2': 10.,
'cat_smooth': 10.,
'min_data_per_group': 20,
'min_data_in_leaf': 20,
'reg_lambda': 0.5,
'boost_from_average': True,
'max_cat_threshold': 64,
'max_bin': 255,
'min_data_in_bin': 5,
}
sample_weight =(
 1 / df.loc[
df.is_train,
'target'
].map(df.loc[df.is_train, 'target'].value_counts(normalize=True)) 
)
 cv_res = lgb.cv(
 params=lgb_params,
train_set=lgb.Dataset(
 data=df.loc[df.is_train, features],
label=df.loc[df.is_train, 'target'],
categorical_feature=cat_cols,
weight=sample_weight 
),
folds=skf,
metrics=['binary_logloss'],
feval=f1_score_lgb,
verbose_eval=50,
early_stopping_rounds=200,
num_boost_round=1000,
)
model.fit(train, y_casual )<predict_on_test>
model = lgb.LGBMClassifier(
 **lgb_params,
n_esimators=int(len(cv_res['binary_logloss-mean'])*(skf.n_splits + 1)/skf.n_splits)
)
 model.fit(
 X=df.loc[df.is_train, features],
y=df.loc[df.is_train, 'target'],
sample_weight=sample_weight,
categorical_feature=cat_cols,
)
<train_model><EOS> 
y_pred = model.predict(df.loc[~df.is_train, features])
 pd.DataFrame({'target': y_pred, 'id': df[~df.is_train].index} ).astype(np.int32 ).to_csv(
 'submission.csv',
index=False,
encoding='utf-8'
)