Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
!pip install inflect
warnings.filterwarnings('ignore')
<feature_engineering>
!pip install contractions
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
seed_everything(seed=42 )<load_from_csv>
%matplotlib inline
warnings.filterwarnings('ignore')
 logging.basicConfig(level=logging.INFO )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<normalization>
def text_preprocessing_platform(df, text_col):
def denoise_text(text):
soup = BeautifulSoup(text, "html.parser")
 text = soup.get_text()
text = contractions.fix(text)
 return text
def remove_non_ascii(words):

 new_words = []
for word in words:
new_word = unicodedata.normalize('NFKD', word ).encode('ascii', 'ignore' ).decode('utf-8', 'ignore')
 new_words.append(new_word)
 return new_words
def to_lowercase(words):

 new_words = []
for word in words:
new_word = word.lower()
new_words.append(new_word)
 return new_words
def remove_punctuation(words):

 new_words = []
for word in words:
new_word = re.sub(r'[^\w\s]', '', word)
 if new_word != '':
new_words.append(new_word)
 return new_words
def replace_numbers(words):

 p = inflect.engine()
new_words = []
for word in words:
if word.isdigit() :
new_word = p.number_to_words(word)
 new_words.append(new_word)
 else:
new_words.append(word)
 return new_words
def remove_stopwords(words):

 new_words = []
for word in words:
if word not in stopwords.words('english'):
new_words.append(word)
 return new_words
def stem_words(words):

 stemmer = LancasterStemmer()
stems = []
for word in words:
stem = stemmer.stem(word)
 stems.append(stem)
 return stems
def lemmatize_verbs(words):

 lemmatizer = WordNetLemmatizer()
lemmas = []
for word in words:
lemma = lemmatizer.lemmatize(word, pos='v')
 lemmas.append(lemma)
 return lemmas
def normalize_text(words):
words = remove_non_ascii(words)
 words = to_lowercase(words)
 words = remove_punctuation(words)
 words = replace_numbers(words)
 words = remove_stopwords(words)
 words = lemmatize_verbs(words)
 return words
def tokenize(text):
return nltk.word_tokenize(text)
 def text_prepare(text):
text = denoise_text(text)
 text = ' '.join([x for x in normalize_text(tokenize(text)) ])
 return text
df[text_col] = [text_prepare(x)for x in df[text_col]]
return df
for col in(GENES + CELLS):
normalize = Normalizer()
vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 train_features[col] = normalize.fit_transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = normalize.fit_transform(test_features[col].values.reshape(vec_len_test,1)).reshape(1, vec_len_test)[0] 
<concatenate>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 before = train.head(10)
 print("Before Text Preprocessing")
 display(before[['text']])
 after = text_preprocessing_platform(before, 'text')
 print("After Text Preprocessing")
 display(after[['text']] )
n_comp = 100
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
def loadData_Tokenizer(X_train, X_test,MAX_NB_WORDS=75000,MAX_SEQUENCE_LENGTH=500):
np.random.seed(7)
 text = np.concatenate(( X_train, X_test), axis=0)
 text = np.array(text)
 tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(text)
 sequences = tokenizer.texts_to_sequences(text)
 word_index = tokenizer.word_index
text = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
 print('Found %s unique tokens.' % len(word_index))
indices = np.arange(text.shape[0])
 text = text[indices]
print(text.shape)
 X_train = text[0:len(X_train), ]
X_test = text[len(X_train):, ]
embeddings_index = {}
f = open("/kaggle/input/glove6b50dtxt/glove.6B.50d.txt", encoding='utf-8')
 for line in f:
try:
values = line.split()
word = values[0]
try:
coefs = np.asarray(values[1:], dtype='float32')
 except:
pass
embeddings_index[word] = coefs
except UnicodeDecodeError:
pass
f.close()
print('Total %s word vectors.' % len(embeddings_index))
return(X_train, X_test, word_index,embeddings_index, tokenizer )
n_comp = 20
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
def Build_Model_RNN_Text(word_index, embeddings_index, nclasses, MAX_SEQUENCE_LENGTH=500, EMBEDDING_DIM=50, dropout=0.5):
model = Sequential()
hidden_layer = 2
lstm_node = 32
embedding_matrix = np.random.random(( len(word_index)+ 1, EMBEDDING_DIM))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
if len(embedding_matrix[i])!= len(embedding_vector):
print("could not broadcast input array from shape", str(len(embedding_matrix[i])) ,
"into shape", str(len(embedding_vector)) , " Please make sure your"
" EMBEDDING_DIM is equal to embedding_vector file ,GloVe,")
 exit(1)
 embedding_matrix[i] = embedding_vector
model.add(Embedding(len(word_index)+ 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True))
print(lstm_node)
 for i in range(0,hidden_layer):
model.add(Bidirectional(LSTM(lstm_node,return_sequences=True, recurrent_dropout=0.5)))
 model.add(Dropout(dropout))
model.add(Bidirectional(LSTM(lstm_node, recurrent_dropout=0.5)))
 model.add(Dropout(dropout))
model.add(Dense(256, activation='relu'))
model.add(Dense(nclasses, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
var_thresh = VarianceThreshold(threshold=0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features<merge>
def get_eval_report(labels, preds):
mcc = matthews_corrcoef(labels, preds)
 tn, fp, fn, tp = confusion_matrix(labels, preds ).ravel()
precision =(tp)/(tp+fp)
 recall =(tp)/(tp+fn)
 f1 =(2*(precision*recall)) /(precision+recall)
 return {
"mcc": mcc,
"tp": tp,
"tn": tn,
"fp": fp,
"fn": fn,
"pricision" : precision,
"recall" : recall,
"F1" : f1,
"accuracy":(tp+tn)/(tp+tn+fp+fn)
 }
def compute_metrics(labels, preds):
assert len(preds)== len(labels)
 return get_eval_report(labels, preds)
 def plot_graphs(history, string):
plt.plot(history.history[string])
 plt.plot(history.history['val_'+string], '')
 plt.xlabel("Epochs")
 plt.ylabel(string)
 plt.legend([string, 'val_'+string])
 plt.show()
def class_balance(df, target):
cls = df[target].value_counts()
cls.plot(kind='bar')
 plt.show()
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<categorify>
preprocess = True
text = 'text'
target = 'target'
train['cp_time'] = train['cp_time'].map({24:0, 48:0.5, 72:1})
 train['cp_dose'] = train['cp_dose'].map({'D1':0,'D2':1})
 test['cp_time'] = test['cp_time'].map({24:0, 48:0.5, 72:1})
 test['cp_dose'] = test['cp_dose'].map({'D1':0,'D2':1} )<drop_column>
traindf = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 testdf = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
target_cols<split>
if preprocess:
traindf = text_preprocessing_platform(traindf, text)
 testdf = text_preprocessing_platform(testdf, text)
 train_final = traindf
test_final = testdf
print("Train DataFrame")
 display(train_final.head(3))
print("Leadeboard DataFrame")
 display(test_final.head(3))
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<define_variables>
df = train_final
test = test_final
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
X = df[text]
y = df[target]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
 print("Generating Glove...")
 X_train_Glove,X_test_Glove, word_index,embeddings_index, tokenizer = loadData_Tokenizer(X_train,X_test )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024<prepare_x_and_y>
with warnings.catch_warnings() :
print("Building Model...")
 model_RNN = Build_Model_RNN_Text(word_index,embeddings_index, 2)
 model_RNN.summary()
print("
Starting Training...
 ")
 history = model_RNN.fit(X_train_Glove, y_train,
validation_data=(X_test_Glove, y_test),
epochs=5,
batch_size=128,
verbose=1)
 warnings.simplefilter("ignore" )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.05, div_factor=1.5e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
test.head()
ids = test['id']
SEED = [0,1,2,3,4,5]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
test_input = test[text]
test_input = np.array(test_input)
 test_sequences = tokenizer.texts_to_sequences(test_input)
 test_padded = pad_sequences(test_sequences, maxlen=500)
 indices = np.arange(test_padded.shape[0])
 test_padded = test_padded[indices]
unlabbeled_glove = test_padded
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
test_predictions = model_RNN.predict_classes(unlabbeled_glove )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<install_modules>
output = pd.DataFrame({'id':ids,'target':test_predictions} )
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<feature_engineering>
output.set_index('id' )
os.environ["CUDA_LAUNCH_BLOCKING"] = "1" 
<set_options>
output.to_csv('/kaggle/working/glove6b50d_bi_lstm_p-2LAYERS.csv', index=False )