Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
! java -version
SEED = [0, 1, 2]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
! apt remove -y openjdk-11-jre-headless
! apt install -y openjdk-8-jdk openjdk-8-jre
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<save_to_csv>
! java -version
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<import_modules>
! pip install pyspark==2.4.6
! pip install spark-nlp==2.5.2
! pip install pycontractions
import numpy as np
import pandas as pd
import os 
<load_from_csv>
%matplotlib inline
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape<load_from_csv>
from pyspark.sql import SQLContext, SparkSession
from pyspark.sql.types import *
import pyspark.sql.functions as sqlF
from pyspark import SparkContext, SparkConf
import sparknlp
from sparknlp.base import *
from sparknlp.annotator import *
from sparknlp.common import *
from sparknlp.embeddings import *
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )<load_from_csv>
spark = sparknlp.start(gpu=True)
 sqlContext = SQLContext(sparkContext=spark.sparkContext,
sparkSession=spark )
def make_filename(idx):
return glob.glob('.. /input/moa-1082020/' + str(idx)+ '.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx))
return temp 
<load_from_csv>
train = pd.read_csv(".. /input/real-or-not-data-cleaned/train.csv" )
predictions = []
predictions.append(pd.read_csv('.. /input/blendings/submissionfeats.csv'))
predictions.append(pd.read_csv('.. /input/blendings/submissionorj.csv'))<prepare_x_and_y>
train_spark = spark.createDataFrame(train[["id","text","target","clean_text"]] )
y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape<save_to_csv>
document_assembler = DocumentAssembler() \ 
.setInputCol("clean_text")\ 
.setOutputCol("document")
 encoder = UniversalSentenceEncoder.pretrained() \ 
.setInputCols(["document"])\ 
.setOutputCol("embeddings")
 clf = ClassifierDLApproach() \ 
.setInputCols(["embeddings"])\ 
.setOutputCol("prediction")\ 
.setLabelColumn("target")\ 
.setMaxEpochs(30)\ 
.setBatchSize(32)
 pipeline = Pipeline(
 stages = [
document_assembler,
encoder,
clf
] )
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
<set_options>
training, valid = train_spark.select("clean_text","target" ).randomSplit([0.7,0.3], seed=41)
 model = pipeline.fit(training)
 pred = model.transform(valid)
 pandpred = pred.select("target","prediction.result" ).toPandas()
pandpred.result = pandpred.result.apply(lambda x: x[0])
 print(classification_report(pandpred.target, pandpred.result.astype(int)) )
warnings.filterwarnings('ignore' )<import_modules>
test_p = pd.read_csv(".. /input/real-or-not-data-cleaned/test.csv")
 test_p = test_p[["id","text", "clean_text"]]
test = spark.createDataFrame(test_p)
from sklearn.preprocessing import QuantileTransformer<import_modules>
final = model.transform(test)
 final = final.select("id","prediction.result" ).toPandas()
final.result = final.result.apply(lambda x: x[0] ).astype(int)
 final.id = final.id.astype(int )
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from tqdm.notebook import tqdm
from time import time<import_modules>
sub = final[["id","result"]]
sub.columns = ["id","target"]
sub.to_csv("submission_use.csv", index=None )
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from tensorflow.keras import layers,regularizers,Sequential,backend,callbacks,optimizers,metrics,losses<load_from_csv>
spark.stop()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
for name in dir() :
if not name.startswith('_'):
del globals() [name]
gc.collect()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<normalization>
train = pd.read_csv(".. /input/real-or-not-data-cleaned/train.csv")
 test = pd.read_csv(".. /input/real-or-not-data-cleaned/test.csv" )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
def tokenize_map(sentence, labs='None'):

 global labels
input_ids = []
attention_masks = []
for text in sentence:
encoded_dict = tokenizer.encode_plus(
 text,
add_special_tokens = True,
truncation='longest_first',
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 if labs != 'None':
labels = torch.tensor(labels)
 return input_ids, attention_masks, labels
else:
return input_ids, attention_masks
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
train_sentences = train.clean_text.values
labels = train.target.values
test_sentences = test.clean_text.values
input_ids, attention_masks, labels = tokenize_map(train_sentences, labels)
 test_input_ids, test_attention_masks= tokenize_map(test_sentences )
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape 
<merge>
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(0.75 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size] )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = 32 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = 32
)
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
model = BertForSequenceClassification.from_pretrained(
 'bert-large-uncased',
num_labels = 2,
output_attentions = False,
output_hidden_states = False 
)
 model.to(device )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<load_from_csv>
gc.collect()
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
ss_rf = ss_svc.copy()
ss = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']<compute_test_metric>
EPOCHS = 4
optimizer = AdamW(model.parameters() ,
lr = 6e-6,
eps = 1e-8 
)
 total_steps = len(train_dataloader)* EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<compute_test_metric>
prediction_data = TensorDataset(test_input_ids, test_attention_masks)
 prediction_dataloader = DataLoader(prediction_data,
sampler=SequentialSampler(prediction_data),
batch_size=64 )
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<choose_model_class>
model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits )
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.0015),metrics=logloss 
)
 return model<choose_model_class>
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['target'] = flat_predictions
<define_variables><EOS> 
submission.to_csv('submission_bert.csv', index=False, header=True )