Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from nltk.corpus import stopwords
import tensorflow as tf
import tensorflow_addons as tfa
from transformers import TFAutoModel, AutoTokenizer
scores_nn = dict()
for mol_type_index, mol_type in enumerate(mol_types):
print(mol_type, f'- run number {run_number}')
 try:
scores_nn = np.load(f'run_{run_number}_scores_nn.npy', allow_pickle=True ).item()
except:
scores_nn = dict()
train = pd.read_csv(train_and_test_with_feats_folder + '/train_' + mol_type + '.csv' ).fillna(0)
 test_full = pd.read_csv(train_and_test_with_feats_folder + '/test_' + mol_type + '.csv' ).fillna(0)
 print(f' Working with data of type {mol_type} and shape {train.shape}')
 df_oofs, sc_columns = get_oofs_and_preds('oof', mol_type)
 df_pred, sc_columns = get_oofs_and_preds('pred', mol_type)
 molecules_names = train['molecule_name'].unique()
type_scalar_contributions = scalar_coupling_contributions[scalar_coupling_contributions['molecule_name'].isin(molecules_names)]
type_scalar_contributions = type_scalar_contributions[type_scalar_contributions['type'] == mol_type]
for col in ['fc', 'sd','pso', 'dso']:
train[col] = type_scalar_contributions[col].values
del type_scalar_contributions, molecules_names
nn_mol_features = create_or_load_selected_features(preds_and_oofs_folder, train, scalar_coupling_contributions)
 for col in sc_columns:
nn_mol_features.append(col)
 train, index_folds = preprocess_train_data(mol_type, train, scalar_coupling_contributions,
k_folds=k_folds, val_data_ratio_if_no_kfolds = 0.2, verbose = True)
 train = pd.merge(train, df_oofs, on=['id'])
 test_full = pd.merge(test_full, df_pred, on=['id'])
 del df_oofs, df_pred
std_scaler = StandardScaler().fit(train[nn_mol_features])
 test_selected = std_scaler.transform(test_full[nn_mol_features].values)
 del test_full
oof_nn = pd.DataFrame(np.zeros(( len(train), 1)))
 pred_nn = pd.DataFrame(np.zeros(( len(test_selected), k_folds)) , columns = list(range(k_folds)))
 n_features = 32
cols_nn_features_for_lgb = np.array(['sc', 'fc', 'sd','pso', 'dso'] + [f'nn_feat_{k_fold}_{i}' for k_fold in range(k_folds)for i in range(1, n_features+1)] ).flatten()
nn_features_for_lgb_train = pd.DataFrame(np.zeros(( train.shape[0], n_features + 5)) , columns = cols_nn_features_for_lgb[:n_features + 5])
 nn_features_for_lgb_test= pd.DataFrame(np.zeros(( test_selected.shape[0], n_features*k_folds + 5)) , columns = cols_nn_features_for_lgb)
 if mol_type not in scores_nn:
scores_nn[mol_type] = dict()
gc.collect()
for k_fold,(trn_idx, val_idx)in enumerate(index_folds):
load_existing_model = True
nn_features_for_lgb_test, nn_features_for_lgb_train, pred_nn, oof_nn, scores_nn = run_nn(load_existing_model, k_fold, trn_idx, val_idx, train, nn_mol_features, k_folds,
preds_and_oofs_folder, nn_features_for_lgb_test, nn_features_for_lgb_train,
scores_nn, test_selected, mol_type, oof_nn, pred_nn, run_number)
 gc.collect()
nn_features_for_lgb_test.loc[:, ['sc', 'fc', 'sd','pso', 'dso']] /= k_folds
np.save(f'run_{run_number}_scores_nn', scores_nn)
 np.save(f'run_{run_number}_{mol_type}_oof_nn', oof_nn)
 pred_nn.to_pickle(f'run_{run_number}_{mol_type}_pred_nn')
 nn_features_for_lgb_train.to_pickle(f'run_{run_number}_{mol_type}_nn_features_for_lgb_train')
 nn_features_for_lgb_test.to_pickle(f'run_{run_number}_{mol_type}_nn_features_for_lgb_test')
 gc.collect() 
<save_to_csv>
tweet_tokenizer = TweetTokenizer()
def normalizeToken(token):
lowercased_token = token.lower()
if token.startswith("@"):
return "@USER"
elif lowercased_token.startswith("http")or lowercased_token.startswith("www"):
return "HTTPURL"
elif len(token)== 1:
return demojize(token)
 else:
if token == "’":
return "'"
elif token == "…":
return "..."
else:
return token
def normalizeTweet(tweet):
tokens = tweet_tokenizer.tokenize(tweet.replace("’", "'" ).replace("…", "..."))
normTweet = " ".join([normalizeToken(token)for token in tokens])
 normTweet = normTweet.replace("cannot ", "can not " ).replace("n't ", " n't " ).replace("n 't ", " n't " ).replace("ca n't", "can't" ).replace("ai n't", "ain't")
 normTweet = normTweet.replace("'m ", " 'm " ).replace("'re ", " 're " ).replace("'s ", " 's " ).replace("'ll ", " 'll " ).replace("'d ", " 'd " ).replace("'ve ", " 've ")
 normTweet = normTweet.replace(" p.m.", " p.m." ).replace(" p.m ", " p.m " ).replace(" a.m.", " a.m." ).replace(" a.m ", " a.m ")
 normTweet = re.sub(r",([0-9]{2,4}),([0-9]{2,4})", r",\1,\2", normTweet)
 normTweet = re.sub(r"([0-9]{1,3})/([0-9]{2,4})", r"\1/\2", normTweet)
 normTweet = re.sub(r"([0-9]{1,3})-([0-9]{2,4})", r"\1-\2", normTweet)
 return " ".join(normTweet.split() )
sub = pd.read_csv(f'{preds_and_oofs_folder}/final_model_submission.csv')
 sub.to_csv('final_sub.csv', index=False )<import_modules>
def load_train_set() :
df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")[["text", "target"]]
df["text"] = df["text"].apply(normalizeTweet)
 return df
def load_test_set() :
df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")[["id", "text"]]
df["text"] = df["text"].apply(normalizeTweet)
 return df
print(os.listdir('.. /input/')) 
<load_from_csv>
print(train['target'].value_counts())
 print()
print(train['target'].value_counts(normalize=True))
%%time
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
train = pd.read_csv('.. /input/pmp-oof/final_train_oof_pmp.csv')
 test = pd.read_csv('.. /input/pmp-oof/final_test_oof_pmp.csv')
 drop_features=['id','type','scalar_coupling_constant','molecule_name'
]
feats = [f for f in train.columns if f not in drop_features]
print('features:',feats)
 n_splits= 5
folds = GroupKFold(n_splits=n_splits)
 oof_preds = np.zeros(( train.shape[0]))
sub_preds = np.zeros(( test.shape[0]))
for t in train['type'].unique() :
train_t = train[train['type']==t].reset_index(drop=True)
 idx = train[train['type']==t].index
test_t = test[test['type']==t].reset_index(drop=True)
 idx_test = test[test['type']==t].index
cv_list = []
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_t,groups=train_t['molecule_name'])) :
train_x, train_y = train_t[feats].iloc[train_idx], train_t['scalar_coupling_constant'].iloc[train_idx]
valid_x, valid_y = train_t[feats].iloc[valid_idx], train_t['scalar_coupling_constant'].iloc[valid_idx]
valid_type = train_t['type'].iloc[valid_idx]
train_x = train_x.values
valid_x = valid_x.values
clf = BayesianRidge(
 n_iter=1000,
tol=0.1,
normalize=False)
 clf.fit(train_x, train_y)
 oof_preds[idx[valid_idx]] = clf.predict(valid_x)
 oof_cv = group_mean_log_mae(y_true=valid_y,
y_pred=oof_preds[idx[valid_idx]],
types=valid_type)
 cv_list.append(oof_cv)
 sub_preds[idx_test] += clf.predict(test_t[feats].values)/ folds.n_splits
print('type=' + str(t),cv_list)
 train['stacking'] = oof_preds
test['scalar_coupling_constant'] = sub_preds
for t in train['type'].unique() :
train_t = train[train['type']==t]
oof_type = group_mean_log_mae(y_true=train_t['scalar_coupling_constant'],
y_pred=train_t['stacking'],
types=train_t['type'])
 print(t,oof_type)
 oof_full = group_mean_log_mae(y_true=train['scalar_coupling_constant'],
y_pred=oof_preds,
types=train['type'])
 print('All type',oof_full)
 train[['id','stacking']].to_csv('train_stacking.csv',index=False)
 test[['id','scalar_coupling_constant']].to_csv('test_stacking.csv',index=False )<load_from_csv>
disaster_tweets = train[train['target']==1]['text']
non_disaster_tweets = train[train['target']==0]['text']
freq_dist_disaster_tweets= nltk.FreqDist([word for tweet in disaster_tweets for word in tweet.lower().split() if word not in stopwords.words("english")and len(word)> 2])
 freq_dist_non_disaster_tweets= nltk.FreqDist([word for tweet in non_disaster_tweets for word in tweet.lower().split() if word not in stopwords.words("english")and len(word)> 2])
 results = {
'word' : [],
'disaster_count' : [],
'non_disaster_count' : []
}
for word, value in freq_dist_disaster_tweets.most_common(200):
results['word'].append(word)
 results['disaster_count'].append(freq_dist_disaster_tweets[word])
 results['non_disaster_count'].append(freq_dist_non_disaster_tweets[word])
 df = pd.DataFrame(results)
 df.head(30 )
def get_median_from_files(files):
print(len(files))
outs = [pd.read_csv(f, index_col=0)for f in files]
concat_sub = pd.concat(outs, axis=1, sort=True)
 champ_median = concat_sub.median(axis=1 ).values
return champ_median
test = pd.read_csv(f".. /input/champs-scalar-coupling/test.csv")
 TARGET = 'scalar_coupling_constant'
test['conservative'] = pd.read_csv('.. /input/champs-ensemble-conservative/ensemble_sub_conservative.csv')[TARGET]
test['n1'] = get_median_from_files(
 [
'.. /input/champ-preds/gnn_median_2302.csv',
'.. /input/champ-preds/gnn_median_2301.csv',
'.. /input/champ-preds/gnn_median_adjusted_1JHC_2296.csv',
] 
)
 gnn_2312 = pd.read_csv('.. /input/champ-preds/gnn_median_65_68_2312.csv')[TARGET]
lastgnn = pd.read_csv('.. /input/champ-preds/gnn_median_69_73.csv')[TARGET]
test['n1'] = lastgnn * 0.5 + gnn_2312 * 0.3 + test['n1'] *0.2
test['n2'] = pd.read_csv('.. /input/champ-preds/gnn0_median_2068.csv')[TARGET]
test['lgb_a'] = get_median_from_files(
 [
'.. /input/champ-preds/submission_type_2100.csv',
'.. /input/champ-preds/submission_type_2085.csv',
] 
)
 test['lgb_m'] = get_median_from_files(
 [
'.. /input/champ-preds/lgb_type_full_f286_10.csv',
'.. /input/champ-preds/lgb_type_full_f262_10.csv',
] 
)
 test['nnet'] = get_median_from_files(
 [
'.. /input/nnpvals-seed-20/nnet_sub_s11.csv',
'.. /input/champ-preds/nnet_sub.csv',
'.. /input/nn-seed-10/nnet_sub.csv',
'.. /input/nn-seed-11/nnet_sub.csv',
'.. /input/nnet-c-seed-10/lgb_type_cv-1.7126_mae0.23572_fd5_10.csv',
'.. /input/nnet-c-seed-11/lgb_type_cv-1.70994_mae0.23497_fd5_11.csv',
'.. /input/nnet-c-seed-12/lgb_type_cv-1.71029_mae0.23523_fd5_12.csv',
'.. /input/nnet-b-seed-10/lgb_type_cv-1.72296_mae0.24019_bags-1_f120_fd5_10.csv',
'.. /input/nnet-b-seed-11/lgb_type_cv-1.70647_mae0.2376_bags-1_f120_fd5_11.csv',
'.. /input/nnet-b-seed-12/lgb_type_cv-1.69833_mae0.24106_bags-1_f120_fd5_12.csv',
'.. /input/nnet-try-seed-11/lgb_type_cv-1.64944_mae0.23965_bags-1_f120_fd5_11.csv',
] 
)
 test['nnet_cont'] = get_median_from_files(
 [
'.. /input/nncont-seed-23-p/nnetCont_sub_only_predict.csv',
'.. /input/nncont-seed-22/nnetCont_sub-1.8213.csv',
] 
)
 nn_contof = pd.read_csv('.. /input/nncont-seed-26/nnetCont_sub_-2.6677.csv')[TARGET]
test['nnet_cont'] = nn_contof * 0.6 + test['nnet_cont'] * 0.4
test['final_mpnn'] = pd.read_csv('.. /input/champ-preds/final_mpnn.csv')[TARGET]
test['mpnn'] = pd.read_csv('.. /input/champ-preds/mpnn_5_fold_pseudo_1449.csv')[TARGET]
test['lb'] = pd.read_csv('.. /input/chemistry-of-best-models-1-895/stack_median.csv')[TARGET]
test['stack15'] = pd.read_csv('.. /input/15th-stacking/test_stacking.csv')[TARGET]
test.head(10 )<feature_engineering>
MAX_LENGTH = 50
short_tweets = sum(np.array(tweets_length)<= MAX_LENGTH)
 long_tweets = sum(np.array(tweets_length)> MAX_LENGTH)
 print("{} reviews with LEN > {}({:.2f} % of total data)".format(
 long_tweets,
MAX_LENGTH,
100 * long_tweets / len(train)
))
%matplotlib inline
test['nnet_ens'] = test['nnet_cont'] * 0.6 + test['nnet'] * 0.4
test['lgb_ens'] = test['lgb_a'] * 0.8 + test['lgb_m'] * 0.2
test['final_preds'] =(
 test['n1']*0.5 +
test['n2']*0.1 +
test['lgb_ens']*0.15 +
test['nnet_ens']*0.1 +
test['lb']*0.1 +
test['final_mpnn'] * 0.03 + test['mpnn'] *0.02 
)
 test.head(20)
 fig, ax = plt.subplots(1, 1, figsize=(12, 8))
cols = [col for col in test.columns if col not in ['id', 'molecule_name', 'atom_index_0', 'atom_index_1']]
fig = sns.heatmap(test[cols].corr() , cmap='viridis', square=True)
 test[cols].corr()<save_to_csv>
def encode_tweets(tokenizer, tweets, max_len):
nb_tweets = len(tweets)
 tokens = np.ones(( nb_tweets,max_len),dtype='int32')
 masks = np.zeros(( nb_tweets,max_len),dtype='int32')
 segs = np.zeros(( nb_tweets,max_len),dtype='int32')
 for k in range(nb_tweets):
tweet = tweets[k]
enc = tokenizer.encode(tweet)
 if len(enc)<max_len-2:
tokens[k,:len(enc)+2] = [0] + enc + [2]
masks[k,:len(enc)+2] = 1
else:
tokens[k,:max_len] = [0] + enc[:max_len-2] + [2]
masks[k,:max_len] = 1
return tokens,masks,segs
def build_model(max_len):
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 bertweet = TFAutoModel.from_pretrained("vinai/bertweet-base")
 x,_ = bertweet(ids,attention_mask=att,token_type_ids=tok)
 out=tf.keras.layers.Dense(1,activation='sigmoid' )(x[:,0,:])
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model
model = build_model(MAX_LENGTH)
 model.summary()
submission = pd.DataFrame()
submission['id'] = test.id
submission['scalar_coupling_constant'] = test['final_preds'] * 0.1 + test['stack15'] * 0.9
submission.to_csv('ensemble_sub.csv', index=False )<set_options>
train_tokens, train_masks, train_segs = encode_tweets(tokenizer,train["text"].to_list() , MAX_LENGTH)
 train_labels = train["target"]
print(pd.__version__)
 SEED = 26
LR = 1e-4<set_options>
es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', patience=3, restore_best_weights=True, verbose=1)
 train_labels = train['target']
train_history = model.fit(
 [train_tokens,train_masks,train_segs], train_labels,
validation_split=0.2,
epochs=5,
batch_size=16,
verbose = 1,
callbacks = [es]
)
<define_variables><EOS> 
test_tokens, test_masks, test_segs = encode_tweets(tokenizer,test["text"].to_list() , MAX_LENGTH)
 test["target"] = model.predict([test_tokens, test_masks, test_segs] ).round().astype(int)
 submission = test[["id", "target"]]
submission.to_csv("submission.csv",index=False )