Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_x_and_y>
warnings.filterwarnings('ignore')
data = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 data=data[data.weight!=0]
NAN_VALUE = -9999
features = [c for c in data.columns if 'feature' in c]
target = 'resp'
X = data.loc[:, features].fillna(NAN_VALUE)
 Y =(data.loc[:, target] > 0 ).astype(int)
 del data
gc.collect()
model = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.7,
missing=NAN_VALUE,
random_state=2020,
tree_method='gpu_hist',
nthread=multiprocessing.cpu_count() 
)
 model.fit(X, Y)
 print('Finished training model')
 del X, Y
gc.collect()
env = janestreet.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
X_test = test_df.loc[:, features]
X_test = X_test.fillna(NAN_VALUE)
 sample_prediction_df.action = model.predict(X_test)
 env.predict(sample_prediction_df )<predict_on_test>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()
def concat_respx_pred(X,models):
y_multi_pre = np.stack([ models[f'{respx}'].predict_proba(X)[:,1] for respx in models if respx != 'lr'] ).T
return y_multi_pre
def model_predict_slow(X,models):
y_multi_pre = concat_respx_pred(X,models)
 y = models['lr'].predict(y_multi_pre)
 return y.astype(int)
 def model_predict_fast(X,models):
y = models['resp'].predict(X)
 return y.astype(int )<load_pretrained>
sentences = ['The weather is sunny', 'The weather is partly sunny and partly cloudy.']
with open('.. /input/models/lgb_model_ensemble_0208_01.pkl', 'rb')as f:
models = pickle.load(f)
<predict_on_test>
vectorizer = CountVectorizer()
vectorizer.fit(sentences)
 vectorizer.vocabulary_
env = janestreet.make_env()
iter_test = env.iter_test()
model_predict = model_predict_slow
features = [f'feature_{x}' for x in range(130)]
for(test_df, sample_prediction_df)in tqdm(iter_test):
if test_df['weight'].values[0] < 0.001:
y_preds = 0
else:
X_test = test_df.loc[:, features].fillna(NANFILL ).values
y_preds = model_predict(X_test,models)
 sample_prediction_df.action = y_preds
env.predict(sample_prediction_df )<load_from_csv>
vectorizer.transform(sentences ).toarray()
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<define_variables>
stopwords = stopwords.words('english')
 count_vectorizer = CountVectorizer(stop_words = stopwords)
 count_vectorizer.fit(train['text'])
 train_vectors = count_vectorizer.transform(train['text'])
 test_vectors = count_vectorizer.transform(test['text'])
 train_vectors.shape
features1 = [f'feature_{i}' for i in range(71)]
features2 = [f'feature_{i}' for i in range(71, 130)]<prepare_x_and_y>
count_vectorizer = CountVectorizer(stop_words = stopwords, min_df=2 ,max_df=0.8)
 count_vectorizer.fit(train['text'])
 train_vectors = count_vectorizer.transform(train['text'])
 test_vectors = count_vectorizer.transform(test['text'] )
X_train = [train.loc[:, features1].values,
train.loc[:, features2].values]<define_variables>
def custom_preprocessor(text):

 text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub("\\W"," ",text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
assert len(features)== len(features1)+ len(features2 )<choose_model_class>
count_vectorizer = CountVectorizer(list(train['text']),preprocessor=custom_preprocessor)
 train_vectors = count_vectorizer.fit_transform(list(train['text']))
test_vectors = count_vectorizer.transform(list(test['text']))
def create_resnet(n_features, n_features_2, n_labels, learning_rate=1e-03, label_smoothing=1e-02):
input_1 = tf.keras.layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = tf.keras.layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation="elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "relu"),
],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(128, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, input_2], outputs = output)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer=opt,
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=['AUC'])
 return model
tf.keras.backend.clear_session()
clf = create_resnet(len(features1), len(features2), len(resp_cols), 1e-03, 1e-02)
 clf.summary()<train_model>
count_vectorizer = CountVectorizer(list(train['text']),preprocessor=custom_preprocessor,ngram_range=(1,2))
train_vectors = count_vectorizer.fit_transform(list(train['text']))
test_vectors = count_vectorizer.transform(list(test['text']))
list(count_vectorizer.vocabulary_)[:10]
clf.fit(X_train, y_train, epochs=200, batch_size=4096 * 16)
 clf.save('resnet.h5')
 models = [clf]
th = 0.501
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model([x_tt[:, :len(features1)], x_tt[:, len(features1):]], training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
count_vectorizer = CountVectorizer(list(train['text']),preprocessor=custom_preprocessor,ngram_range=(2,2),
analyzer='char_wb')
 train_vectors = count_vectorizer.fit_transform(list(train['text']))
test_vectors = count_vectorizer.transform(list(test['text']))
print(list(count_vectorizer.vocabulary_)[:20] )
import random
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from tqdm import tqdm
from random import choices<define_variables>
count_vectorizer = CountVectorizer(token_pattern=r'\w{1,}',
ngram_range=(1, 2), stop_words = stopwords,preprocessor=custom_preprocessor)
 count_vectorizer.fit(train['text'])
 train_vectors = count_vectorizer.transform(train['text'])
 test_vectors = count_vectorizer.transform(test['text'] )
FOLDS = 5
SEED = 1111
TRAINING = True
USE_FINETUNE = True
FILE_NAME = 'notebookec7152784a'
print('FOLDS : {}
SEED : {}
TRAINING : {}
USE_FINETUNE : {}
FILE_NAME : {}'.format(FOLDS, SEED, TRAINING,
USE_FINETUNE, FILE_NAME))<feature_engineering>
clf = LogisticRegression(C=1.0)
 scores = model_selection.cross_val_score(clf, train_vectors, train["target"], cv=5, scoring="f1")
 scores
def check_random(SEED=SEED):
os.environ['TF_DETERMINISTIC_OPS'] = '1'
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED)
 check_random()<load_from_csv>
clf.fit(train_vectors, train["target"] )
print('load train start')
 try:
train = pd.read_parquet('train.parquet')
 except:
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv', index_col='ts_id')
 train.to_parquet('train.parquet')
 print('load train end' )<prepare_x_and_y>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test_vectors)
 sample_submission.to_csv("submission.csv", index=False )
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<choose_model_class>
tfidf = TfidfVectorizer(analyzer='word',token_pattern=r'\w{1,}',max_features=5000)
 train_tfidf = tfidf.fit_transform(train['text'])
 test_tfidf = tfidf.transform(test["text"] )
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
inp_2 = tf.keras.backend.cos(inp)
 x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x_2 = tf.keras.layers.BatchNormalization()(inp_2)
 x_2 = tf.keras.layers.Dropout(dropout_rates[0] )(x_2)
 for i in range(len(hidden_units)) :
x_2 = tf.keras.layers.Dense(hidden_units[i] )(x_2)
 x_2 = tf.keras.layers.BatchNormalization()(x_2)
 x_2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_2)
 x_2 = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x_2)
 x = tf.keras.layers.Concatenate(axis=-1 )([x, x_2])
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model<init_hyperparams>
tfidf = TfidfVectorizer(analyzer='word',ngram_range=(2,3),token_pattern=r'\w{1,}',max_features=5000)
 train_tfidf = tfidf.fit_transform(train['text'])
 test_tfidf = tfidf.transform(test["text"] )
batch_size = 5000
hidden_units = [150, 150, 150]
dropout_rates = [0.20, 0.20, 0.20, 0.20]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate
)<train_model>
tfidf = TfidfVectorizer(analyzer='char',ngram_range=(2,3),token_pattern=r'\w{1,}',max_features=5000)
 train_tfidf = tfidf.fit_transform(train['text'])
 test_tfidf = tfidf.transform(test["text"])
<train_model>
tfidf_vectorizer = TfidfVectorizer(min_df=3, max_features=None,analyzer='word',token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,
stop_words = stopwords)
 train_tfidf = tfidf.fit_transform(train['text'])
 test_tfidf = tfidf.transform(test["text"])
check_random()
clf.fit(X_train, y_train, epochs=200, batch_size=5000 )<find_best_params>
clf = LogisticRegression(C=1.0)
 scores = model_selection.cross_val_score(clf, train_tfidf, train["target"], cv=5, scoring="f1")
 scores
models = []
models.append(clf)
 th = 0.5000
print(th )<feature_engineering>
clf.fit(train_tfidf, train["target"] )
print('predict')
 f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<define_search_space>
hash_vectorizer = HashingVectorizer(n_features=10000,norm=None,alternate_sign=False)
 hash_vectorizer.fit(train['text'])
epochs = 40
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
th = 0.502<compute_test_metric>
train_vectors = hash_vectorizer.transform(train['text'])
 test_vectors = hash_vectorizer.transform(test['text'] )
@njit(fastmath = True)
 def utility_score_numba(date, weight, resp, action):
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / len(Pi))
u = min(max(t, 0), 6)* np.sum(Pi)
 return u<import_modules>
clf = LogisticRegression(C=1.0)
 scores = model_selection.cross_val_score(clf, train_vectors, train["target"], cv=5, scoring="f1")
 scores
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pandas as pd
from tqdm import tqdm
from random import choices<define_variables>
clf.fit(train_vectors, train["target"] )