Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<compute_train_metric>
!pip install -U catalyst transformers
def blended_preds(y_pred,ws):
return(y_pred*ws[:,np.newaxis,np.newaxis] ).sum(axis=0)
 def blended_oof(ws):
y_vals = blended_preds(y_val,ws)
 loss = logloss(tf.constant(labels_train,dtype=tf.float32),tf.constant(y_vals,dtype=tf.float32))
return loss.numpy() +(( 1-np.sum(ws)) **2 )<find_best_params>
import os
import warnings
import logging
from typing import Mapping, List
from pprint import pprint
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from transformers import AutoConfig, AutoModel, AutoTokenizer
from catalyst.dl import SupervisedRunner
from catalyst.dl.callbacks import AccuracyCallback, F1ScoreCallback, OptimizerCallback
from catalyst.dl.callbacks import CheckpointCallback, InferCallback
from catalyst.utils import set_global_seed, prepare_cudnn
opt = dual_annealing(blended_oof,bounds=(( 0,1),(0,1),(0,1)))
 ws = opt.x
print('The weights are ' + str(ws))
print('Sum of weights is ' + str(ws.sum()))
print('OOF score is ' + str(opt.fun))<merge>
MODEL_NAME = 'distilbert-base-uncased'
LOG_DIR = "./logdir"
NUM_EPOCHS = 5
BATCH_SIZE = 128
MAX_SEQ_LENGTH = 64
LEARN_RATE = 5e-5
ACCUM_STEPS = 1
SEED = 42
y_val = blended_preds(y_val,ws)
 y_pred = blended_preds(y_pred,ws )<save_to_csv>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
p_min = 0.001
p_max = 0.999
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<import_modules>
train['text'] = train['text'].apply(lambda x: x.lower())
 train['len'] = train['text'].apply(lambda x: len(x.split()))
import numpy as np
import pandas as pd<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(train, train.target.values, random_state = 42, test_size = 0.2, stratify=train.target.values )
df1 = pd.read_csv('/kaggle/input/best-subs-moa/submission_over_over.csv')
 df2 = pd.read_csv('.. /input/notebook31e74ea284/submission.csv')
 df3=pd.read_csv('.. /input/moa-ensemble/submission.csv' )<sort_values>
class TextClassificationDataset(Dataset):

 def __init__(self,
texts: List[str],
labels: List[str] = None,
label_dict: Mapping[str, int] = None,
max_seq_length: int = 512,
model_name: str = 'distilbert-base-uncased'):

 self.texts = texts
self.labels = labels
self.label_dict = label_dict
self.max_seq_length = max_seq_length
if self.label_dict is None and labels is not None:
self.label_dict = dict(zip(sorted(set(labels)) ,
range(len(set(labels)))))
 self.tokenizer = AutoTokenizer.from_pretrained(model_name)
 logging.getLogger(
 "transformers.tokenization_utils" ).setLevel(logging.FATAL)
 self.sep_vid = self.tokenizer.vocab["[SEP]"]
self.cls_vid = self.tokenizer.vocab["[CLS]"]
self.pad_vid = self.tokenizer.vocab["[PAD]"]
def __len__(self):

 return len(self.texts)
 def __getitem__(self, index)-> Mapping[str, torch.Tensor]:

 x = self.texts[index]
x_encoded = self.tokenizer.encode(
 x,
add_special_tokens=True,
max_length=self.max_seq_length,
return_tensors="pt",
).squeeze(0)
 true_seq_length = x_encoded.size(0)
 pad_size = self.max_seq_length - true_seq_length
pad_ids = torch.Tensor([self.pad_vid] * pad_size ).long()
x_tensor = torch.cat(( x_encoded, pad_ids))
mask = torch.ones_like(x_encoded, dtype=torch.int8)
 mask_pad = torch.zeros_like(pad_ids, dtype=torch.int8)
 mask = torch.cat(( mask, mask_pad))
output_dict = {
"features": x_tensor,
'attention_mask': mask
}
if self.labels is not None:
y = self.labels[index]
y_encoded = torch.Tensor(
 [self.label_dict.get(y, -1)]
).long().squeeze(0)
 output_dict["targets"] = y_encoded
return output_dict
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
train_dataset = TextClassificationDataset(
 texts=X_train['text'].values.tolist() ,
labels=y_train.tolist() ,
label_dict=None,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 valid_dataset = TextClassificationDataset(
 texts=X_test['text'].values.tolist() ,
labels=y_test.tolist() ,
label_dict=train_dataset.label_dict,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 test_dataset = TextClassificationDataset(
 texts=test['text'].values.tolist() ,
labels=None,
label_dict=None,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME
)
df1 = order_sub(df1)
 df2 = order_sub(df2)
 df3 = order_sub(df3 )<prepare_output>
NUM_CLASSES = len(train_dataset.label_dict )
BLEND=df1.copy()
BLEND.iloc[:,1:] = df1.iloc[:,1:]*0.25 +df2.iloc[:,1:]*0.3 +df3.iloc[:,1:]*0.45<define_variables>
train_val_loaders = {
"train": DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True),
"valid": DataLoader(dataset=valid_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL )<load_from_csv>
class DistilBertForSequenceClassification(nn.Module):

 def __init__(self, pretrained_model_name: str, num_classes: int = None):

 super().__init__()
config = AutoConfig.from_pretrained(
 pretrained_model_name, num_labels=num_classes)
 self.distilbert = AutoModel.from_pretrained(pretrained_model_name,
config=config)
 self.pre_classifier = nn.Linear(config.dim, config.dim)
 self.classifier = nn.Linear(config.dim, num_classes)
 self.dropout = nn.Dropout(config.seq_classif_dropout)
 def forward(self, features, attention_mask=None, head_mask=None):

 assert attention_mask is not None, "attention mask is none"
distilbert_output = self.distilbert(input_ids=features,
attention_mask=attention_mask,
head_mask=head_mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0]
pooled_output = self.pre_classifier(pooled_output)
 pooled_output = nn.ReLU()(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<import_modules>
model = DistilBertForSequenceClassification(pretrained_model_name=MODEL_NAME,
num_classes=NUM_CLASSES )
import os
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns
import math
from datetime import datetime<define_variables>
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=LEARN_RATE)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer )
FILEPATH = os.path.join(".. ", "input")
 TRAINPATH = os.path.join(FILEPATH, "train.csv")
 TESTPATH = os.path.join(FILEPATH, "test.csv" )<load_from_csv>
os.environ['CUDA_VISIBLE_DEVICES'] = "0"
set_global_seed(SEED)
 prepare_cudnn(deterministic=True )
df = pd.read_csv(TRAINPATH, index_col=0)
 df.head()<load_from_csv>
%%time
runner = SupervisedRunner(
 input_key=(
 "features",
"attention_mask" 
)
)
 runner.train(
 model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=train_val_loaders,
callbacks=[
AccuracyCallback(num_classes=NUM_CLASSES),
OptimizerCallback(accumulation_steps=ACCUM_STEPS)
 ],
fp16=None,
logdir=LOG_DIR,
num_epochs=NUM_EPOCHS,
verbose=True
)
df_test = pd.read_csv(TESTPATH)
 df_test.head()<filter>
torch.cuda.empty_cache()
df = df[(df['trip_duration'] > 60)&(df['trip_duration'] < 3600 * 6)]<data_type_conversions>
test_loaders = {
"test": DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }
df['store_and_fwd_flag'] = df['store_and_fwd_flag'].astype('category' ).cat.codes
df_test['store_and_fwd_flag'] = df_test['store_and_fwd_flag'].astype('category' ).cat.codes<import_modules>
runner.infer(
 model=model,
loaders=test_loaders,
callbacks=[
CheckpointCallback(
 resume=f"{LOG_DIR}/checkpoints/best.pth" 
),
InferCallback() ,
],
verbose=True
)
from sklearn.decomposition import PCA<categorify>
predicted_probs = runner.callbacks[0].predictions['logits']
coords = np.vstack(( df[['pickup_latitude', 'pickup_longitude']].values,
df[['dropoff_latitude', 'dropoff_longitude']].values,
df_test[['pickup_latitude', 'pickup_longitude']].values,
df_test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df_test['pickup_pca0'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 0]
df_test['pickup_pca1'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 1]
df_test['dropoff_pca0'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df_test['dropoff_pca1'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]<data_type_conversions>
submission['target'] = predicted_probs.argmax(axis=1)
 submission.head()
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )<feature_engineering>
submission.to_csv('submission.csv', index=False )
<feature_engineering><EOS> 
submission.to_csv('submission.csv', index=False )