Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<init_hyperparams>
random_state_split = 42
Dropout_num = 0
learning_rate = 6e-6
valid = 0.15
epochs_num = 3
batch_size_num = 16
target_corrected = False
target_big_corrected = False
<categorify>
plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
warnings.simplefilter('ignore' )
def run_k_fold(NFOLDS, seed, pretrain=False):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed, pretrain)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
tweet= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
def run_k_fold_none_scored(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_non_scored_cols)))
 predictions = np.zeros(( len(test), len(target_non_scored_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training_non_scored(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
SEED = [1903, 1881]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 run_k_fold_none_scored(NFOLDS, SEED[0])
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed, pretrain=True)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
Real_len = tweet[tweet['target'] == 1].shape[0]
Not_len = tweet[tweet['target'] == 0].shape[0]
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
tweet['length'] = tweet['text'].apply(length )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<import_modules>
def create_corpus(target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
<define_variables>
def create_corpus_df(tweet, target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
FOLDS = 5
EPOCHS = 80
BATCH_SIZE = 128
LR = 0.001
VERBOSE = 2
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )<categorify>
corpus=create_corpus(0)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)[:10]
def mapping_and_filter(train, train_targets, test):
cp_type = {'trt_cp': 0, 'ctl_vehicle': 1}
cp_dose = {'D1': 0, 'D2': 1}
for df in [train, test]:
df['cp_type'] = df['cp_type'].map(cp_type)
 df['cp_dose'] = df['cp_dose'].map(cp_dose)
 train_targets = train_targets[train['cp_type'] == 0].reset_index(drop = True)
 train = train[train['cp_type'] == 0].reset_index(drop = True)
 train_targets.drop(['sig_id'], inplace = True, axis = 1)
 return train, train_targets, test
def scaling(train, test):
features = train.columns[2:]
scaler = StandardScaler()
train[features] = scaler.fit_transform(train[features])
 test[features] = scaler.transform(test[features])
 return train, test, features
def mean_log_loss(y_true, y_pred):
metrics = []
for target in range(len(train_targets.columns)) :
metrics.append(log_loss(y_true[:, target], y_pred[:, target]))
return np.mean(metrics)
 def create_model(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.5 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.5 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam()
opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = 'binary_crossentropy')
 return model
def train_and_evaluate(train, train_targets, test, features, perm_imp = False):
models = []
trn_indices, val_indices = [], []
oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
print(f'Training fold {fold + 1}')
 K.clear_session()
model = create_model(len(features))
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss',
mode = 'min',
patience = 10,
restore_best_weights = True,
verbose = 1)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
mode = 'min',
factor = 0.3,
patience = 3)
 x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
model.fit(x_train, y_train,
validation_data =(x_val, y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr],
verbose = VERBOSE)
 oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
models.append(model)
 trn_indices.append(trn_ind)
 val_indices.append(val_ind)
 print('-'*50)
 print('
')
 oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 if perm_imp:
return models, trn_indices, val_indices, oof_score
else:
return test_pred
def permutation_importance(train, train_targets, test, features):
models, trn_indices, val_indices, oof_score = train_and_evaluate(train, train_targets, test, features, perm_imp = True)
 scores = np.zeros(len(features))
for num, feature in enumerate(tqdm(features)) :
train_ = train.copy()
train_[feature] = train_[feature].sample(frac = 1.0 ).reset_index(drop = True)
 oof_pred = np.zeros(( train_.shape[0], 206))
for model, trn_ind, val_ind in zip(models, trn_indices, val_indices):
oof_pred[val_ind] = model.predict(train_[features].values[val_ind])
 score = mean_log_loss(train_targets.values, oof_pred)
 scores[num] = oof_score - score
perm_dataframe = pd.DataFrame({'features': features, 'score': scores})
 perm_dataframe = perm_dataframe[perm_dataframe['score'] < 0]
new_features = list(perm_dataframe['features'])
 print(f'We have select {len(new_features)} features')
 return new_features<load_from_csv>
plt.figure(figsize=(16,5))
counter=Counter(corpus)
 most=counter.most_common()
x=[]
y=[]
for word,count in most[:40]:
if(word not in stop):
x.append(word)
 y.append(count )
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, train_targets, test = mapping_and_filter(train, train_targets, test)
 train, test, features = scaling(train, test)
 new_features = permutation_importance(train, train_targets, test, features )<save_to_csv>
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n]
test_preds = train_and_evaluate(train, train_targets, test, new_features, perm_imp = False)
 sample_submission.loc[:, train_targets.columns] = test_preds
sample_submission.loc[test['cp_type'] == 1, train_targets.columns] = 0
sample_submission.to_csv('submission.csv', index = False)
 sample_submission.head()<load_from_csv>
df=pd.concat([tweet,test])
 df.shape
df_feature = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 df_label = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 features = df_feature.columns
features = [_ for _ in features if _ != 'sig_id']
labels = df_label.columns
labels = [_ for _ in labels if _ != 'sig_id']<categorify>
example="New competition launched :https://www.kaggle.com/c/nlp-getting-started"
def preprocess(df=df_feature):
df['cp_type'] = df['cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df['cp_dose'] = df['cp_dose'].map({'D1': 0, 'D2': 1})
 df['cp_time'] = df['cp_time'] / 24
return df
df_feature = preprocess()<drop_column>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 remove_URL(example )
def MoA_dirctionary_IDs() :

 df_moa = df_label[df_label[labels].sum(axis=1)>= 1].reset_index(drop=True)
 moa_dict = {}
for label in labels:
try:
ls = df_moa[df_moa[label] == 1]['sig_id'].values
index = df_moa[df_moa[label] == 1].index
moa_dict[label] = ls
df_moa = df_moa.drop(index, axis=0)
 except:
print(f'An error occurred at label ==> {label}')
 return moa_dict<feature_engineering>
df['text']=df['text'].apply(lambda x : remove_URL(x))
multi_label = pd.DataFrame(df_label[labels].sum(axis=1))
multi_label['sig_id'] = df_label['sig_id']
ids_no_label = multi_label[multi_label[0] == 0]['sig_id'].values
train_dict = MoA_dirctionary_IDs()
train_dict['no MoA'] = ids_no_label
for i,(k, v)in enumerate(train_dict.items()):
if i <= 4:
print(f'{k}: {v[0]}')
 c = 0
for k, v in train_dict.items() :
c += len(v)
 print(c )<choose_model_class>
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 print(remove_html(example))
class TrainSchema:
def __init__(self, features, labels, train_dict, k=5):
self.features = features
self.labels = labels
self.train_dict = train_dict
self.k = k
self.models = {'model_'+str(i): self.create_model(i)for i in range(self.k)}
self.total_params = self.models['model_0'].count_params() * self.k
print(f'Total parameters: {self.total_params}')
 def create_model(self, i):
template = 'model_'+str(i)+'_'
inp_dim = len(self.features)
 out_dim = len(self.labels)
 inp = layers.Input(shape=(inp_dim,), name=template+'inp')
 x = layers.Reshape(( 1, 1, inp_dim))(inp)
 x = layers.BatchNormalization()(x)
 x = tfa.layers.WeightNormalization(layers.Conv2D(256, 1, 1))(x)
 x = layers.BatchNormalization()(x)
 x = layers.LeakyReLU()(x)
 x = layers.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(layers.Conv2D(256, 1, 1))(x)
 x = layers.BatchNormalization()(x)
 x = layers.LeakyReLU()(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.Flatten()(x)
 x = tfa.layers.WeightNormalization(layers.Dense(200))(x)
 x = layers.BatchNormalization()(x)
 x = layers.LeakyReLU()(x)
 x = layers.Dropout(0.2 )(x)
 x = tfa.layers.WeightNormalization(layers.Dense(200))(x)
 x = layers.BatchNormalization()(x)
 x = layers.LeakyReLU()(x)
 x = layers.Dropout(0.2 )(x)
 out = tfa.layers.WeightNormalization(layers.Dense(206, activation='sigmoid', name=template+'out'))(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10), loss='binary_crossentropy')
 return model
def create_dataset(self, df_feature, df_target, i, batch_size=256):
def kfold_data_trim() :
train_ids = []
valid_ids = []
for moa, ids in self.train_dict.items() :
IDs = ids.copy()
IDs = IDs.tolist()
if len(ids)> self.k:
chunk = len(ids)// self.k
if i == self.k-1:
valid_ids.extend(IDs[::-1][chunk*i:chunk*(i+1)])
 for ID in IDs[::-1][chunk*i:chunk*(i+1)]:
IDs.remove(ID)
 else:
valid_ids.extend(IDs[chunk*i:chunk*(i+1)])
 for ID in IDs[chunk*i:chunk*(i+1)]:
IDs.remove(ID)
 train_ids.extend(IDs)
 else:
train_ids.extend(ids)
 return train_ids, valid_ids
train_ids, valid_ids = kfold_data_trim()
def train_valid_df() :
df = pd.merge(df_feature, df_target, on='sig_id')
 df = df.set_index('sig_id')
 df_train = df.loc[train_ids]
df_valid = df.loc[valid_ids]
return df_train, df_valid
df_train, df_valid = train_valid_df()
print(f'train samples {df_train.shape[0]}, valid samples {df_valid.shape[0]}')
 x_trn = df_train[self.features].to_numpy()
y_trn = df_train[self.labels].to_numpy()
x_val = df_valid[self.features].to_numpy()
y_val = df_valid[self.labels].to_numpy()
dataset_valid = tf.data.Dataset.from_tensor_slices(( x_val, y_val)).batch(batch_size)
 dataset_train = tf.data.Dataset.from_tensor_slices(( x_trn, y_trn)).shuffle(y_trn.shape[0] ).batch(batch_size)
 return dataset_train, dataset_valid
def train(self, df_feature, df_target, path_int=0, epochs=100, patience=10):
self.ckpt_path = '/tmp/checkpoint_'+str(path_int)+'/'
loss = []
val_loss = []
es = tf.keras.callbacks.EarlyStopping(patience=patience)
 rlr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_delta=0.0001, cooldown=0, min_lr=0)
 for i in range(self.k):
start = time.time()
ckpt = tf.keras.callbacks\ 
.ModelCheckpoint(self.ckpt_path+'model_'+str(i), monitor='val_loss',
verbose=0, save_best_only=True, save_weights_only=True)
 dataset_train, dataset_valid = self.create_dataset(df_feature, df_target, i)
 print('
Model_'+str(i), ' Starts to Train...')
 h = self.models['model_'+str(i)].fit(dataset_train, epochs=epochs,
validation_data=dataset_valid,
verbose=0, callbacks=[es, rlr, ckpt])
 loss.append(h.history['loss'][-1])
 val_loss.append(h.history['val_loss'][-1])
 end = time.time() - start
print('train time: {:.2f} mins.'.format(end/60))
for i in range(len(loss)) :
print('Model_{}: loss {:.5f}, val_loss {:.5f}'.format(str(i), loss[i], val_loss[i]))
def load_weights(self):
for i in range(self.k):
self.models['model_'+str(i)].load_weights(self.ckpt_path+'model_'+str(i))
def evaluate_models(self, df_feature, df_target):
print('
Evaluating')
 loss = []
val_loss = []
for i in range(self.k):
dataset_train, dataset_valid = self.create_dataset(df_feature, df_target, i)
 rel1 = self.models['model_'+str(i)].evaluate(dataset_train, verbose=0)
 rel2 = self.models['model_'+str(i)].evaluate(dataset_valid, verbose=0)
 loss.append(rel1)
 val_loss.append(rel2)
 for i in range(len(loss)) :
print('Model_{}: loss {:.5f}, val_loss {:.5f}'.format(str(i), loss[i], val_loss[i]))
model_losses = {'model_'+str(i): _/sum(val_loss)for i, _ in enumerate(val_loss)}
model_names = [k for k, v in sorted(model_losses.items() , key=lambda item: item[1], reverse=False)]
model_desc = {k: v for k, v in sorted(model_losses.items() , key=lambda item: item[1], reverse=True)}
self.amount_to_say = {}
for i,(k, v)in enumerate(model_desc.items()):
self.amount_to_say[model_names[i]] = v<train_model>
df['text']=df['text'].apply(lambda x : remove_html(x))
train_schema = TrainSchema(features, labels, train_dict, k=5)
 train_schema.train(df_feature, df_label)
 train_schema.load_weights()
train_schema.evaluate_models(df_feature, df_label )<merge>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 remove_emoji("Omg another Earthquake 😔😔" )
df_merge = pd.merge(df_feature, df_label, on='sig_id')
 df_pos = df_merge[df_merge[labels].sum(axis=1)>= 1]
for k, model in train_schema.models.items() :
model.evaluate(df_pos[features].to_numpy() , df_pos[labels].to_numpy() )<load_from_csv>
df['text']=df['text'].apply(lambda x: remove_emoji(x))
df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 df_test = preprocess(df_test)
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 label_cols = submission.columns.tolist() [1:]<predict_on_test>
def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 example="I am a
print(remove_punct(example))
pred = np.zeros(( df_test.shape[0], len(labels)))
 for i in range(len(train_schema.models)) :
pred += train_schema.models['model_'+str(i)].predict([df_test[features].to_numpy() ])
 pred = pred / len(train_schema.models )<save_to_csv>
df['text']=df['text'].apply(lambda x : remove_punct(x))
for i, c in enumerate(labels):
submission[c] = pred[:, i]
submission.loc[df_test['cp_type']==1, labels] = 0
submission.to_csv('submission.csv', index=False )<set_options>
corpus_new1=create_corpus_df(df,1)
 len(corpus_new1 )
warnings.filterwarnings("ignore")
 sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 np.random.seed(42)
 tf.random.set_seed(42)
 sns.set_style('white')
 sns.set(font_scale=1.2)
<load_from_csv>
corpus_new0=create_corpus_df(df,0)
 len(corpus_new0 )
df_train = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 display(df_train.head(3))
print('train data size', df_train.shape)
 df_target_ns = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 display(df_target_ns.head(3))
print('train target nonscored size', df_target_ns.shape)
 df_target_s = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 display(df_target_s.head(3))
print('train target scored size', df_target_s.shape)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 display(df_test.head(3))
print('test data size', df_test.shape)
 df_sample = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 display(df_sample.head(3))
print('sample submission size', df_sample.shape)
<define_variables>
def cv(data):
count_vectorizer = CountVectorizer()
emb = count_vectorizer.fit_transform(data)
 return emb, count_vectorizer
list_corpus = df["text"].tolist()
list_labels = df["target"].tolist()
X_train, X_test, y_train, y_test = train_test_split(list_corpus, list_labels, test_size=0.2,
random_state=random_state_split)
 X_train_counts, count_vectorizer = cv(X_train)
 X_test_counts = count_vectorizer.transform(X_test )
g_features = [cols for cols in df_train.columns if cols.startswith('g-')]<define_variables>
def tfidf(data):
tfidf_vectorizer = TfidfVectorizer()
train = tfidf_vectorizer.fit_transform(data)
 return train, tfidf_vectorizer
X_train_tfidf, tfidf_vectorizer = tfidf(X_train)
 X_test_tfidf = tfidf_vectorizer.transform(X_test )
c_features = [cols for cols in df_train.columns if cols.startswith('c-')]<filter>
def create_corpus_new(df):
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)]
corpus.append(words)
 return corpus
ind_tr = df_train[df_train['cp_type']=='ctl_vehicle'].index
ind_te = df_test[df_test['cp_type']=='ctl_vehicle'].index<categorify>
corpus=create_corpus_new(df )
transformer = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution="normal")
 def preprocess(df):
df['cp_time'] = df['cp_time'].map({24:1, 48:2, 72:3})
 df['cp_dose'] = df['cp_dose'].map({'D1':0, 'D2':1})
 g_features = [cols for cols in df.columns if cols.startswith('g-')]
c_features = [cols for cols in df.columns if cols.startswith('c-')]
for col in(g_features + c_features):
vec_len = len(df[col].values)
 raw_vec = df[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 df[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
return df
X = preprocess(df_train)
 X_test = preprocess(df_test)
 display(X.head(5))
print('Train data size', X.shape)
 display(X_test.head(3))
print('Test data size', X_test.shape)
 y = df_target_s.drop('sig_id', axis=1)
 display(y.head(3))
print('target size', y.shape)
 y0 = df_target_ns.drop('sig_id', axis=1 )<concatenate>
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word = values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()
g_features = [cols for cols in X.columns if cols.startswith('g-')]
n_comp = 0.95
data = pd.concat([pd.DataFrame(X[g_features]), pd.DataFrame(X_test[g_features])])
 data2 =(PCA(0.95, random_state=42 ).fit_transform(data[g_features]))
train2 = data2[:X.shape[0]]
test2 = data2[-X_test.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_g-{i}' for i in range(data2.shape[1])])
 test2 = pd.DataFrame(test2, columns=[f'pca_g-{i}' for i in range(data2.shape[1])])
 X = pd.concat(( X, train2), axis=1)
 X_test = pd.concat(( X_test, test2), axis=1)
 c_features = [cols for cols in X.columns if cols.startswith('c-')]
n_comp = 0.95
data = pd.concat([pd.DataFrame(X[c_features]), pd.DataFrame(X_test[c_features])])
 data2 =(PCA(0.95, random_state=42 ).fit_transform(data[c_features]))
train2 = data2[:X.shape[0]]
test2 = data2[-X_test.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_c-{i}' for i in range(data2.shape[1])])
 test2 = pd.DataFrame(test2, columns=[f'pca_c-{i}' for i in range(data2.shape[1])])
 X = pd.concat(( X, train2), axis=1)
 X_test = pd.concat(( X_test, test2), axis=1)
 display(X.head(2))
display(X_test.head(2))<feature_engineering>
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
X,X_test=fe_stats(X,X_test)
 display(X.head(2))
print(X.shape)
 display(X_test.head(2))
print(X_test.shape )<concatenate>
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 239):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
X ,X_test=fe_cluster(X,X_test)
 display(X.head(2))
print(X.shape)
 display(X_test.head(2))
print(X_test.shape )<create_dataframe>
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i < num_words:
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
var_thresh = VarianceThreshold(0.8)
 data = X.append(X_test)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : X.shape[0]]
test_features_transformed = data_transformed[-X_test.shape[0] : ]
X = pd.DataFrame(X[['sig_id','cp_type', 'cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 X = pd.concat([X, pd.DataFrame(train_features_transformed)], axis=1)
 X_test = pd.DataFrame(X_test[['sig_id','cp_type', 'cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 X_test = pd.concat([X_test, pd.DataFrame(test_features_transformed)], axis=1)
 display(X.head(2))
print(X.shape)
 display(X_test.head(2))
print(X_test.shape )<prepare_x_and_y>
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=3e-4)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'] )
y0 = y0[X['cp_type'] == 'trt_cp'].reset_index(drop = True)
 y = y[X['cp_type'] == 'trt_cp'].reset_index(drop = True)
 X = X[X['cp_type'] == 'trt_cp'].reset_index(drop = True)
 X.drop(['cp_type','sig_id'], axis=1, inplace=True)
 X_test.drop(['cp_type','sig_id'], axis=1, inplace=True)
 print('New data shape', X.shape )<compute_test_metric>
train=tweet_pad[:tweet.shape[0]]
test=tweet_pad[tweet.shape[0]:]
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -K.mean(y_true*K.log(y_pred)+(1-y_true)*K.log(1-y_pred))
def create_model(num_cols, hid_layers, activations, dropout_rate, lr, num_cols_y):
inp1 = tf.keras.layers.Input(shape =(num_cols,))
x1 = tf.keras.layers.BatchNormalization()(inp1)
 for i, units in enumerate(hid_layers):
x1 = tf.keras.layers.Dense(units, activation=activations[i] )(x1)
 x1 = tf.keras.layers.Dropout(dropout_rate[i] )(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Dense(num_cols_y,activation='sigmoid' )(x1)
 model = tf.keras.models.Model(inputs= inp1, outputs= x1)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=lr),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 return model 
<define_variables>
X_train,X_test,y_train,y_test=train_test_split(train,tweet['target'].values,test_size=0.2)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_test.shape )
hid_layers = [[[512, 768, 896],[384, 640, 1024],[768,768,896],[512, 384, 1024],
[512, 640, 640], [640, 896, 1024], [256,640,896],[512,512,768],
[512, 384, 896],[512,768,768]],
[[512, 768, 896],[384, 640, 1024],[768,768,896],[512, 384, 1024],
[512, 640, 640], [640, 896, 1024], [256,640,896],[512,512,768],
[512, 384, 896],[512,768,768]],
[[1152, 640, 3072],[896, 1024, 3584],[1920,1024,3712],[896, 1024, 3456],
[1408, 896, 3456], [1408, 768, 3456], [2176,640,3840],[1664,1280,2688],
[1792, 768, 2432],[1280,1664,4096]],
[[896, 1792, 3712],[2048, 1024, 1664],[1664,1408,1920],[896, 1408, 3072],
[1152, 1152, 3072], [2816, 3072, 3328], [2304,2432,2176],[3968,4096,2816],
[1920, 1536, 3072],[128,1920,1664]],
[[896, 1152, 1408],[1920, 768, 2176],[2048,2048,1792],[2304, 1664, 512],
[768, 384, 512], [640, 1664, 512], [1664,1920,2688],[2432,1664,1536],
[640, 896, 2432],[1536,2176,2176]]]
dropout_rate = [[[0.65,0.35,0.35],[0.65,0.35,0.45],[0.7,0.4,0.4],[0.65,0.35,0.45],
[0.65,0.35,0.45],[0.7,0.3,0.45],[0.7,0.35,0.4],[0.7,0.4,0.4],
[0.7, 0.3, 0.4],[0.65, 0.3, 0.4]],
[[0.65,0.35,0.35],[0.65,0.35,0.45],[0.7,0.4,0.4],[0.65,0.35,0.45],
[0.65,0.35,0.45],[0.7,0.3,0.45],[0.7,0.35,0.4],[0.7,0.4,0.4],
[0.7, 0.3, 0.4],[0.65, 0.3, 0.4]],
[[0.7,0.55,0.7],[0.7,0.4,0.6],[0.7,0.5,0.55],[0.7,0.55,0.6],
[0.7,0.5,0.65],[0.7,0.5,0.65],[0.7,0.55,0.7],[0.7,0.5,0.65],
[0.7, 0.35, 0.6],[0.7, 0.5, 0.7]],
[[0.7,0.4,0.7],[0.7,0.7,0.7],[0.7,0.7,0.7],[0.7,0.25,0.7],
[0.7,0.4,0.6],[0.7,0.6,0.7],[0.7,0.5,0.7],[0.7,0.7,0.7],
[0.7, 0.45, 0.7],[0.7, 0.3, 0.6]],
[[0.7,0.7,0.45],[0.7,0.7,0.6],[0.7,0.7,0.4],[0.7,0.7,0.55],
[0.65,0.45,0.4],[0.7,0.35,0.5],[0.7,0.7,0.45],[0.7,0.6,0.6],
[0.7, 0.7, 0.25],[0.7, 0.7, 0.25]]]
activations = [[['elu', 'swish', 'selu'], ['selu','swish','selu'], ['selu','swish','selu'],['selu','swish','elu'],
['selu','swish','elu'],['elu','swish','selu'],
['selu','swish','elu'],['selu','elu','selu'],['selu','swish','selu'],
['selu','swish','elu']],
[['elu', 'swish', 'selu'], ['selu','swish','selu'], ['selu','swish','selu'],['selu','swish','elu'],
['selu','swish','elu'],['elu','swish','selu'],
['selu','swish','elu'],['selu','elu','selu'],['selu','swish','selu'],
['selu','swish','elu']],
[['selu', 'relu', 'swish'], ['selu','relu','swish'], ['selu','relu','swish'],['selu','relu','swish'],
['selu','relu','swish'],['selu','relu','swish'],['selu','relu','swish'],['selu','relu','swish'],
['selu','elu','swish'],['selu','relu','swish']],
[['selu', 'elu', 'swish'], ['elu','swish','relu'], ['elu','swish','selu'],['selu','elu','swish'],
['selu','elu','swish'],['selu','elu','swish'],['selu','elu','swish'],['selu','selu','swish'],
['selu','relu','swish'],['elu','elu','swish']],
[['selu', 'swish', 'selu'], ['selu','swish','selu'], ['elu','swish','selu'],['selu','swish','selu'],
['selu','relu','relu'],['selu','relu','relu'],['selu','swish','elu'],['selu','swish','relu'],
['elu','swish','selu'],['elu','swish','swish']]]
lr = 5e-4
feats = np.arange(0,X.shape[1],1)
 inp_size = int(np.ceil(1* len(feats)))
 res = y.copy()
df_sample.loc[:, y.columns] = 0
res.loc[:, y.columns] = 0 
<choose_model_class>
history=model.fit(X_train,y_train,batch_size=4,epochs=10,validation_data=(X_test,y_test),verbose=2 )
def callbacks() :
rlr = ReduceLROnPlateau(monitor = 'val_logloss', factor = 0.2, patience = 3, verbose = 0,
min_delta = 1e-4, min_lr = 1e-6, mode = 'min')
 ckp = ModelCheckpoint("model.h5", monitor = 'val_logloss', verbose = 0,
save_best_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_logloss', min_delta = 1e-5, patience = 10, mode = 'min',
baseline = None, restore_best_weights = True, verbose = 0)
 return rlr, ckp, es<compute_test_metric>
train_pred_GloVe = model.predict(train)
 train_pred_GloVe_int = train_pred_GloVe.round().astype('int' )
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in y.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics )<categorify>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
test_preds = []
res_preds = []
np.random.seed(seed=42)
 n_split = 5
n_top = 10
n_round = 1
for seed in range(n_round):
split_cols = np.random.choice(feats, inp_size, replace=False)
 res.loc[:, y.columns] = 0
df_sample.loc[:, y.columns] = 0
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits = n_split, random_state = seed, shuffle = True ).split(X, y)) :
start_time = time()
x_tr = X.astype('float64' ).values[tr][:, split_cols]
x_val = X.astype('float64' ).values[te][:, split_cols]
y0_tr, y0_val = y0.astype(float ).values[tr], y0.astype(float ).values[te]
y_tr, y_val = y.astype(float ).values[tr], y.astype(float ).values[te]
x_tt = X_test.astype('float64' ).values[:, split_cols]
for num in range(n_top):
model = create_model(inp_size, hid_layers[n][num], activations[n][num], dropout_rate[n][num], lr, y0.shape[1])
 model.fit(x_tr, y0_tr,validation_data=(x_val, y0_val), epochs = 150, batch_size = 128,
callbacks = callbacks() , verbose = 0)
 model.load_weights("model.h5")
 model2 = create_model(inp_size, hid_layers[n][num], activations[n][num], dropout_rate[n][num], lr, y.shape[1])
 for i in range(len(model2.layers)-1):
model2.layers[i].set_weights(model.layers[i].get_weights())
 model2.fit(x_tr, y_tr,validation_data=(x_val, y_val),
epochs = 150, batch_size = 128,
callbacks = callbacks() , verbose = 0)
 model2.load_weights('model.h5')
 df_sample.loc[:, y.columns] += model2.predict(x_tt, batch_size = 128)/(n_split*n_top)
 res.loc[te, y.columns] += model2.predict(x_val, batch_size = 128)/(n_top)
 oof = log_loss_metric(y.loc[te,y.columns], res.loc[te, y.columns])
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}], Seed {seed}, Fold {n}:', oof)
 K.clear_session()
del model2
x = gc.collect()
df_sample.loc[ind_te, y.columns] = 0
test_preds.append(df_sample.copy())
 res_preds.append(res.copy() )<compute_test_metric>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
aa = [1.0]
res2= res.copy()
res2.loc[:, y.columns] = 0
for i in range(n_round):
res2.loc[:, y.columns] += aa[i] * res_preds[i].loc[:, y.columns]
print(log_loss_metric(y, res2)) 
<feature_engineering>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
df_sample.loc[:, y.columns] = 0
for i in range(n_round):
df_sample.loc[:, y.columns] += aa[i] * test_preds[i].loc[:, y.columns]
df_sample.loc[ind_te, y.columns] = 0<save_to_csv>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
display(df_sample.head())
 df_sample.to_csv('submission.csv', index=False )<prepare_x_and_y>
def clean_tweets(tweet):

 tweet = ''.join([x for x in tweet if x in string.printable])
 tweet = re.sub(r"http\S+", "", tweet)
 return tweet
y_true = y
y_preds = res2<compute_test_metric>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )
losses = []
for i in range(y.shape[1]):
losses.append(log_loss(y.iloc[:,i], res2.iloc[:,i]))<train_model>
def remove_punctuations(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
max_loss_ind= np.argmax(losses)
 max_loss = np.max(losses)
 print("Max loss is", max_loss,'For index', max_loss_ind,'which is',y.iloc[:,max_loss_ind].name )<import_modules>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook 
<load_from_csv>
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train<load_from_csv>
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
<drop_column>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )<define_variables>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]<categorify>
if target_corrected:
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train.loc[train['id'].isin(ids_with_target_error),'target'] = 0
train[train['id'].isin(ids_with_target_error)]
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 train["cp_time"]=train["cp_time"].map({24: 1, 48: 2, 72: 3})
 train["cp_dose"] = train["cp_dose"].map({"D1": 1, "D2": 2})
 test["cp_time"]= test["cp_time"].map({24: 1, 48: 2, 72: 3})
 test["cp_dose"]=test["cp_dose"].map({"D1": 1, "D2": 2})
 train<feature_engineering>
if target_big_corrected:
train["text"] = train["text"].apply(lambda x: clean_tweets(x))
test["text"] = test["text"].apply(lambda x: clean_tweets(x))
train["text"] = train["text"].apply(lambda x: remove_emoji(x))
test["text"] = test["text"].apply(lambda x: remove_emoji(x))
train["text"] = train["text"].apply(lambda x: remove_punctuations(x))
test["text"] = test["text"].apply(lambda x: remove_punctuations(x))
train["text"] = train["text"].apply(lambda x: convert_abbrev_in_text(x))
test["text"] = test["text"].apply(lambda x: convert_abbrev_in_text(x))
for col in train.iloc[:, 3:].columns:
percent = train[col].quantile([0.01, 0.99] ).values
train[col] = np.clip(train[col], percent[0], percent[1])
 for col in test.iloc[:, 3:].columns:
percent = test[col].quantile([0.01, 0.99] ).values
test[col] = np.clip(test[col], percent[0], percent[1] )<train_model>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
g_pca = PCA(n_components=0.99)
 c_pca = PCA(n_components=0.99)
 train_test_g_concat = pd.concat([train[gcols], test[gcols]], axis=0)
 train_test_c_concat = pd.concat([train[ccols], test[ccols]], axis=0)
 g_pca.fit(train_test_g_concat)
 c_pca.fit(train_test_c_concat)
 train_gtrans = pd.DataFrame(g_pca.transform(train[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=train.index)
 test_gtrans = pd.DataFrame(g_pca.transform(test[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=test.index)
 train_ctrans = pd.DataFrame(c_pca.transform(train[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=train.index)
 test_ctrans = pd.DataFrame(c_pca.transform(test[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=test.index)
 g_pca.n_components_, c_pca.n_components_<concatenate>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
train = pd.concat([train_gtrans, train_ctrans, train[cpcols]], axis=1)
 test = pd.concat([test_gtrans, test_ctrans, test[cpcols]], axis=1)
 train<drop_column>
checkpoint = ModelCheckpoint('model_BERT.h5', monitor='val_loss', save_best_only=True)
 train_history = model_BERT.fit(
 train_input, train_labels,
validation_split = valid,
epochs = epochs_num,
callbacks=[checkpoint],
batch_size = batch_size_num
)
train_target = train_target.drop("sig_id", axis=1)
 train_target<choose_model_class>
model_BERT.load_weights('model_BERT.h5')
 test_pred_BERT = model_BERT.predict(test_input)
 test_pred_BERT_int = test_pred_BERT.round().astype('int' )
def create_shallow_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(128, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=adam)
 return model
def create_mid_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_adamw)
 return model
def create_deep_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="selu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="elu")) ,
L.BatchNormalization() ,
L.Dropout(0.1),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_radam)
 return model<train_on_grid>
train_pred_BERT = model_BERT.predict(train_input)
 train_pred_BERT_int = train_pred_BERT.round().astype('int' )
predictions = []
kf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 tf.random.set_seed(42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train, train_target)) :
model1 = create_shallow_model()
model2 = create_mid_model()
model3 = create_deep_model()
history1 = model1.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model1_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 1, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history1.history["loss"]), min(history1.history["val_loss"])))
 history2 = model2.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model2_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 2, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history2.history["loss"]), min(history2.history["val_loss"])))
 history3 = model3.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model3_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 3, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history3.history["loss"]), min(history3.history["val_loss"])))
 model1.load_weights("model1_fold" + str(fold_id)+ ".h5")
 model2.load_weights("model2_fold" + str(fold_id)+ ".h5")
 model3.load_weights("model3_fold" + str(fold_id)+ ".h5")
 pred1 = model1.predict(test)
 pred2 = model2.predict(test)
 pred3 = model3.predict(test)
 predictions.append(np.average([pred1, pred2, pred3], weights=[0.15, 0.7, 0.15], axis=0))<prepare_output>
pred = pd.DataFrame(test_pred_BERT, columns=['preds'])
 pred.plot.hist()
pred = np.mean(predictions, axis=0)
 pred = np.clip(pred, 0.001, 0.999)
 pred.shape<load_from_csv>
submission['target'] = test_pred_BERT_int
submission.head(10 )
<save_to_csv><EOS> 
submission.to_csv("submission_BERT.csv", index=False, header=True )