Kaggle competition: Natural Language Processing with Disaster Tweets
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<load_from_csv>
DATA_PATH = '/kaggle/input/nlp-getting-started/'
SEED = 42
DROPOUT = 0.5
EPOCHS = 3
LEARN_RATE = 1e-4
SPLIT = 0.2
BATCH_SIZE = 32
data_path = ".. /input/lish-moa/"
train = pd.read_csv(data_path+'train_features.csv')
 train.drop(columns=["sig_id"], inplace=True)
 train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 train_targets_scored.drop(columns=["sig_id"], inplace=True)
 test = pd.read_csv(data_path+'test_features.csv')
 test.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 remove_vehicle = False
if remove_vehicle:
kept_index = train['cp_type']=='trt_cp'
train = train.loc[kept_index].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[kept_index].reset_index(drop=True)
 train["cp_type"] =(train["cp_type"]=="trt_cp")+ 0
train["cp_dose"] =(train["cp_dose"]=="D1")+ 0
test["cp_type"] =(test["cp_type"]=="trt_cp")+ 0
test["cp_dose"] =(test["cp_dose"]=="D1")+ 0
X_test = test.values<import_modules>
train = pd.read_csv(os.path.join(DATA_PATH, 'train.csv'))
test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))
sample_submission = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))
ds = ['train', 'test', 'sample submission']
print("Training set has {} rows and {} columns.".format(train.shape[0], train.shape[1]))
print("Test set has {} rows and {} columns.".format(test.shape[0], test.shape[1]))
print()
print(train.columns)
 print(test.columns )
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score<choose_model_class>
print('Train Keyword Distribution:
')
 print(train.keyword.value_counts())
 print('
', '-' * 50, '
')
 print('Test Keyword Distribution:
')
 print(test.keyword.value_counts() )
MAX_EPOCH=200
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10,
)<import_modules>
print('Count NaN:')
 print(train.isnull().sum() , '
')
 print('Percentage NaN:')
 print(train.isnull().sum() / len(train))
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )<split>
print('Count NaN:')
 print(test.isnull().sum() , '
')
 print('Percentage NaN:')
 print(test.isnull().sum() / len(test))
scores_auc_all= []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, random_state=0, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, train_targets_scored)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train.values[train_idx, :], train_targets_scored.values[train_idx, :]
X_val, y_val = train.values[val_idx, :], train_targets_scored.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train.loc[train['id'].isin(ids_with_target_error),'target'] = 0
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )<save_to_csv>
train.loc[train['keyword'].notnull() , 'text'] = train['keyword'] + ' ' + train['text']
test.loc[test['keyword'].notnull() , 'text'] = test['keyword'] + ' ' + test['text']
train[train['keyword'].notnull() ].head()
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==0, submission.columns[1:]] = 0
submission.to_csv('submission.csv', index=None )<load_from_csv>
train = train.drop(['id', 'keyword', 'location'], axis=1)
 test = test.drop(['keyword', 'location'], axis=1)
 train.head()
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape<load_from_csv>
stop_words = set(stopwords.words('english'))
tknzr = TweetTokenizer(strip_handles=True)
 corpus = []
def clean_text(text):

 text = re.sub(r"\x89Û_", "", text)
 text = re.sub(r"\x89ÛÒ", "", text)
 text = re.sub(r"\x89ÛÓ", "", text)
 text = re.sub(r"\x89ÛÏWhen", "When", text)
 text = re.sub(r"\x89ÛÏ", "", text)
 text = re.sub(r"China\x89Ûªs", "China's", text)
 text = re.sub(r"let\x89Ûªs", "let's", text)
 text = re.sub(r"\x89Û÷", "", text)
 text = re.sub(r"\x89Ûª", "", text)
 text = re.sub(r"\x89Û\x9d", "", text)
 text = re.sub(r"å_", "", text)
 text = re.sub(r"\x89Û¢", "", text)
 text = re.sub(r"\x89Û¢åÊ", "", text)
 text = re.sub(r"fromåÊwounds", "from wounds", text)
 text = re.sub(r"åÊ", "", text)
 text = re.sub(r"åÈ", "", text)
 text = re.sub(r"JapÌ_n", "Japan", text)
 text = re.sub(r"Ì©", "e", text)
 text = re.sub(r"å¨", "", text)
 text = re.sub(r"SuruÌ¤", "Suruc", text)
 text = re.sub(r"åÇ", "", text)
 text = re.sub(r"å£3million", "3 million", text)
 text = re.sub(r"åÀ", "", text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 text = emoji_pattern.sub(r'', text)

text = re.sub(r'[0-9]', '', text)
 for p in string.punctuation.replace('!', ''):
text = text.replace(p, '')
 text = re.sub(r'http\S+', '', text)
 text = tknzr.tokenize(text)
 text = [w.lower() for w in text if not w in stop_words]
corpus.append(text)
 text = ' '.join(text)
 return text
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )<load_from_csv>
%%time
train['text'] = train['text'].apply(lambda s: clean_text(s))
test['text'] = test['text'].apply(lambda s: clean_text(s))
train.sample(10 )
kernels = pd.read_csv('/kaggle/input/best-kernels/kernels.csv', index_col='id' )<load_from_csv>
texts = train['text'].to_numpy()
word_freq = {}
for text in texts:
for word in text.split() :
word_freq[word] = word_freq.get(word, 0)+ 1
def make_filename(idx):
return glob.glob('/kaggle/input/best-kernels/' + str(idx)+ '__submission.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx))
return temp
predictions = [read_predictions(idx)for idx in kernels.index]
len(predictions )<prepare_x_and_y>
MAX_SEQUENCE_LENGTH = 40
tokenizer = Tokenizer()
tokenizer.fit_on_texts(texts)
 sequences = tokenizer.texts_to_sequences(texts)
 word_index = tokenizer.word_index
num_words = len(word_index)+ 1
print('Found %s unique tokens.' %(num_words - 1))
data = pad_sequences(
 sequences,
maxlen=MAX_SEQUENCE_LENGTH,
padding='post',
truncating='post' 
)
 labels = train['target'].to_numpy()
print('Shape of data tensor:', data.shape)
 print('Shape of label tensor:', labels.shape )
y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape<define_variables>
x_train = data
y_train = labels
scores = 1.- kernels['score']
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]<prepare_output>
vectorizer = CountVectorizer()
x_train_vectorized = vectorizer.fit_transform(train['text'])
 print(vectorizer.get_feature_names() [2500:2600] )
weighted_y_pred = pd.DataFrame()
weighted_y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column] * weights[i])
 weighted_y_pred[column] = np.sum(column_data, axis=0)
 weighted_y_pred.shape<prepare_output>
y_train_NB = np.array(train['target'] )
y_pred = weighted_y_pred<save_to_csv>
model_NB = BernoulliNB(alpha=1.0)
 model_NB.fit(x_train_vectorized, y_train_NB)
 x_test_NB = vectorizer.transform(test['text'] )
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T<set_options>
embeddings_df = pd.read_pickle('/kaggle/input/fasttext-twitter-derived-embeddings/twitter_derived_embeddings')
 embeddings_df.head()
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 sys.path.append('.. /input/rank-gauss')
 simplefilter(action='ignore', category=FutureWarning)
 sns.set()<define_variables>
EMBEDDING_DIM = 400
fasttext_embedding_idx = {}
for idx, row in embeddings_df.iterrows() :
word = row[0]
embeddings = np.asarray(row[1], 'float32')
 fasttext_embedding_idx[word] = embeddings
fasttext_embedding_idx['earthquake'][:20]
data_path = '.. /input/lish-moa/'
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.7
decompo = 'PCA'
ncompo_genes = 80
ncompo_cells = 10
encoding = 'dummy'
base_seed = 2020<set_options>
iv = 0
oov = 0
embedding_idx = fasttext_embedding_idx
embedding_matrix = np.zeros(( num_words, EMBEDDING_DIM))
for word, i in word_index.items() :
embedding_vector = embedding_idx.get(word)
 if embedding_vector is not None:
iv += 1
embedding_matrix[i] = embedding_vector
else: oov += 1
print('%i tokens in vocab, %i tokens out of vocab' %(iv, oov))
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False<train_model>
model_NN = tf.keras.Sequential()
embedding_layer = Embedding(
 num_words,
EMBEDDING_DIM,
weights = [embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False
)
seed_everything(base_seed )<load_from_csv>
model_NN = tf.keras.Sequential([
tf.keras.layers.Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32'),
embedding_layer,
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)) ,
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dropout(DROPOUT),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 loss = tf.keras.losses.BinaryCrossentropy(
 from_logits=False,
name='binary_crossentropy' 
)
 optimizer = tf.keras.optimizers.Adam(LEARN_RATE)
 model_NN.compile(
 loss=loss,
optimizer=optimizer,
metrics=['accuracy']
)
test = pd.read_csv(data_path + 'test_features.csv')
 train = pd.read_csv(data_path + 'train_features.csv')
 targets = pd.read_csv(data_path + 'train_targets_scored.csv')
 if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True )<concatenate>
model_NN.fit(x_train, y_train, epochs=EPOCHS, validation_split=SPLIT, batch_size=BATCH_SIZE )
data_all = pd.concat([train, test], ignore_index=True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]<normalization>
test_tokenizer = Tokenizer()
test_texts = test['text'].to_numpy()
test_tokenizer.fit_on_texts(texts)
 test_sequences = test_tokenizer.texts_to_sequences(test_texts)
 test_word_index = test_tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)
 test_embedding_matrix = np.zeros(( len(test_word_index)+ 1, EMBEDDING_DIM))
for word, i in test_word_index.items() :
embedding_vector = embedding_idx.get(word)
 if embedding_vector is not None:
test_embedding_matrix[i] = embedding_vector
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == 'boxcox':
print('boxcox')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == 'norm':
print('norm')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis=0)
 elif scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass<categorify>
raw_lstm_preds = model_NN.predict(test_data)
 lstm_preds = raw_lstm_preds.round().astype(int )
if decompo == 'PCA':
print('PCA')
 GENES = [col for col in data_all.columns if col.startswith('g-')]
CELLS = [col for col in data_all.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis=1)
 else:
pass<categorify>
train = pd.read_csv(os.path.join(DATA_PATH, 'train.csv'))
test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))
if encoding == 'lb':
print('Label Encoding')
 for feat in ['cp_time', 'cp_dose']:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == 'dummy':
print('One-hot')
 data_all = pd.get_dummies(data_all, columns=['cp_time', 'cp_dose'] )<feature_engineering>
import tensorflow_hub as hub
GENES = [col for col in data_all.columns if col.startswith('g-')]
CELLS = [col for col in data_all.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
data_all['g_'+stats] = getattr(data_all[GENES], stats )(axis=1)
 data_all['c_'+stats] = getattr(data_all[CELLS], stats )(axis=1)
 data_all['gc_'+stats] = getattr(data_all[GENES+CELLS], stats )(axis=1)
<load_pretrained>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
with open('data_all.pickle', 'wb')as f:
pickle.dump(data_all, f )<load_pretrained>
%%time
max_seq_length = 180
bert_url = "https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/1"
input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32)
 input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32)
 segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32)
 bert_inputs = [input_word_ids, input_mask, segment_ids]
bert_layer = hub.KerasLayer(bert_url, trainable=True)
 pooled_output, sequence_output = bert_layer(bert_inputs )
with open('data_all.pickle', 'rb')as f:
data_all = pickle.load(f )<concatenate>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
features_todrop = ['sig_id', 'cp_type']
data_all.drop(features_todrop, axis=1, inplace=True)
 try:
targets.drop('sig_id', axis=1, inplace=True)
 except:
pass
train_df = data_all[:train.shape[0]]
train_df.reset_index(drop=True, inplace=True)
 train_df = pd.concat([train_df, targets], axis=1)
 test_df = data_all[train_df.shape[0]:]
test_df.reset_index(drop=True, inplace=True )<init_hyperparams>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
device =('cuda' if torch.cuda.is_available() else 'cpu')
 hsize = 1024
dropratio = 0.2
batchsize = 128
lr = 0.001
wd = 1e-5
smoothing = 0.001
p_min = smoothing
p_max = 1 - smoothing
nepoch = 20
earlystop = True
earlystop_step = 10
lr_scheduler = 'OneCycleLR'
pct_start = 0.1
div_factor = 1e3
factor=0.5
patience=3
nseed = 5
nfold = 7
eval_strategy = 'kfold'<normalization>
x_train_bert = bert_encode(train['text'].values, tokenizer, max_len=max_seq_length)
 x_test_bert = bert_encode(test['text'].values, tokenizer, max_len=max_seq_length)
 y_train_bert = train['target'].values
class Model(nn.Module):
def __init__(self, n_features, n_targets, hidden_size=512, dropratio=0.2):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(n_features)
 self.dropout1 = nn.Dropout(dropratio)
 self.dense1 = nn.utils.weight_norm(nn.Linear(n_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropratio)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropratio)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, n_targets))
self.relu = nn.ReLU()
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = self.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = self.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<normalization>
dense = tf.keras.layers.Dense(64, activation='relu' )(sequence_output[:, 0, :])
 pred = tf.keras.layers.Dense(1, activation='sigmoid' )(dense)
 model_NN_bert = tf.keras.models.Model(inputs=bert_inputs, outputs=pred)
 optimizer = tf.keras.optimizers.Adam(LEARN_RATE)
 model_NN_bert.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 model_NN_bert.summary()
class SmoothCrossEntropyLoss(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets, n_classes, smoothing=0.0):
assert 0 <= smoothing <= 1
with torch.no_grad() :
targets = targets *(1 - smoothing)+ torch.ones_like(targets ).to(device)* smoothing / n_classes
return targets
def forward(self, inputs, targets):
targets = SmoothCrossEntropyLoss()._smooth(targets, inputs.shape[1], self.smoothing)
 if self.weight is not None:
inputs = inputs * self.weight.unsqueeze(0)
 loss = F.binary_cross_entropy_with_logits(inputs, targets)
 return loss<train_on_grid>
model_NN_bert.fit(
 x_train_bert,
y_train_bert,
validation_split=SPLIT,
epochs=EPOCHS,
batch_size=BATCH_SIZE
)
def running_train(X_train, Y_train, X_val, Y_val, dataloader, i_fold=None, seed=None):
model = Model(n_features=X_train.shape[1], n_targets=Y_train.shape[1], hidden_size=hsize, dropratio=dropratio ).to(device)
 criterion = SmoothCrossEntropyLoss(smoothing=smoothing)
 metric = lambda inputs, targets : F.binary_cross_entropy(( torch.clamp(torch.sigmoid(inputs), p_min, p_max)) , targets)
 optimizer = torch.optim.Adam(params=model.parameters() , lr=lr, weight_decay=wd)
 if lr_scheduler == 'OneCycleLR' or lr_scheduler == 'both':
scheduler1 = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=pct_start, div_factor=div_factor,
max_lr=1e-2, epochs=nepoch, steps_per_epoch=len(dataloader))
if lr_scheduler == 'ReduceLROnPlateau' or lr_scheduler == 'both':
scheduler2 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=0.5, patience=3)
 min_valmetric = np.inf
step = 0
for epoch in range(nepoch):
train_loss = 0
train_metric = 0
for i,(X, Y)in enumerate(dataloader):
model.train()
predictions = model(X.to(device=device))
loss = criterion(predictions, Y.to(device=device))
optimizer.zero_grad()
loss.backward()
optimizer.step()
if lr_scheduler == 'OneCycleLR' or lr_scheduler == 'both':
scheduler1.step()
train_loss += loss.item()
train_metric += metric(predictions, Y.to(device=device))
train_loss /= len(dataloader)
 train_metric /= len(dataloader)
 model.eval()
predictions = model(X_val.to(device=device))
val_loss = criterion(predictions, Y_val.to(device=device))
val_metric = metric(predictions, Y_val.to(device=device))
if lr_scheduler == 'ReduceLROnPlateau' or lr_scheduler == 'both':
scheduler2.step(val_metric)
 print('Epoch {}/{}, Train Loss={:5f}, Train Metric={:.5f}, Val Loss={:.5f}, Val Metric={:.5f}'.format(
 epoch + 1, nepoch, train_loss, train_metric, val_loss, val_metric))
if val_metric.item() < min_valmetric:
min_valmetric = val_metric.item()
model_name = 'model_{}_{}.pth'.format(i_fold + 1, seed)if eval_strategy == 'kfold' else 'model_single.pth'
torch.save(model.state_dict() , model_name)
 elif earlystop:
step += 1
if step > earlystop_step:
break<categorify>
bert_preds = model_NN_bert.predict(x_test_bert )
def train_kfold_model(train_df):
X_train_val = torch.from_numpy(train_df.iloc[:, :-targets.shape[1]].values ).to(dtype=torch.float32)
 Y_train_val = torch.from_numpy(train_df.iloc[:, -targets.shape[1]:].values ).to(dtype=torch.float32)
 X_test = torch.from_numpy(test_df.values ).to(dtype=torch.float32)
 oof = torch.zeros(X_train_val.shape[0], Y_train_val.shape[1])
 prediction_test = torch.zeros(X_test.shape[0], Y_train_val.shape[1])
 for i_seed in range(nseed):
seed = random.randint(0, base_seed)
 seed_everything(seed)
 print('Seed: {}, {}/{}'.format(seed, i_seed + 1, nseed))
mskf = MultilabelStratifiedKFold(n_splits=nfold, random_state=seed, shuffle=True)
 for i_fold,(train_idx, val_idx)in enumerate(mskf.split(X_train_val, Y_train_val)) :
print("
X_train, Y_train = X_train_val[train_idx], Y_train_val[train_idx]
X_val, Y_val = X_train_val[val_idx], Y_train_val[val_idx]
dataset = TensorDataset(X_train, Y_train)
 dataloader = DataLoader(dataset, batch_size=batchsize, shuffle=True)
 running_train(X_train, Y_train, X_val, Y_val, dataloader, i_fold=i_fold, seed=seed)
 print('predict on oof...', end='')
 model = Model(n_features=X_train.shape[1], n_targets=Y_train.shape[1], hidden_size=hsize, dropratio=dropratio ).to(device)
 model.load_state_dict(torch.load('model_{}_{}.pth'.format(i_fold + 1, seed)))
 model.eval()
oof[val_idx] += torch.clamp(torch.sigmoid(model(X_val.to(device=device)).detach().cpu()), p_min, p_max)/ nseed
print(' done.')
 print('predict on test...', end='')
 prediction_test += torch.clamp(torch.sigmoid(model(X_test.to(device=device)).detach().cpu()), p_min, p_max)/(nfold * nseed)
 print(' done.
 ')
 cv_score = F.binary_cross_entropy(oof, Y_train_val)
 print('{} folds cv_score: {:.5f}'.format(nfold, cv_score))
return oof, pd.DataFrame(prediction_test.numpy() , columns=targets.columns )<split>
nb_preds_prob = model_NB.predict_proba(x_test_NB)[:, 1]
def train_single_model(train_df):
train_df, val_df = train_test_split(train_df, test_size=0.2, shuffle=True, random_state=base_seed)
 X_train = train_df.iloc[:, :-targets.shape[1]].values
Y_train = train_df.iloc[:, -targets.shape[1]:].values
X_val = val_df.iloc[:, :-targets.shape[1]].values
Y_val = val_df.iloc[:, -targets.shape[1]:].values
X_test = test_df.values
X_train = torch.from_numpy(X_train ).to(dtype=torch.float32)
 Y_train = torch.from_numpy(Y_train ).to(dtype=torch.float32)
 X_val = torch.from_numpy(X_val ).to(dtype=torch.float32)
 Y_val = torch.from_numpy(Y_val ).to(dtype=torch.float32)
 X_test = torch.from_numpy(X_test.astype(np.float32))
dataset = TensorDataset(X_train, Y_train)
 dataloader = DataLoader(dataset, batch_size=batchsize, shuffle=True)
 running_train(X_train, Y_train, X_val, Y_val, dataloader)
 print('predict on test...', end='')
 model = Model(n_features=X_train.shape[1], n_targets=Y_train.shape[1], hidden_size=hsize, dropratio=dropratio ).to(device)
 model.load_state_dict(torch.load('model_single.pth'))
model.eval()
predictions = torch.clamp(torch.sigmoid(model(X_test.to(device)).detach().cpu()), p_min, p_max)
 print(' done.
 ')
 return pd.DataFrame(predictions.numpy() , columns=targets.columns )<categorify>
nb_preds = model_NB.predict(x_test_NB)
 sample_submission['target'] = nb_preds
sample_submission.head()
if eval_strategy == 'kfold':
oof, test_pred = train_kfold_model(train_df)
 elif eval_strategy == 'single':
test_pred = train_single_model(train_df)
 else:
print('eval_strategy should be "kfold" or "single"' )<save_to_csv>
np.allclose(nb_preds, nb_preds_prob.round().astype(int))
test = pd.read_csv(data_path + 'test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 test_pred['sig_id'] = sig_id
sub = pd.merge(test[['sig_id']], test_pred, on='sig_id', how='left')
 sub.fillna(0, inplace=True)
 sub.to_csv('submission.csv', index=False )<set_options>
sample_submission.to_csv('submission_NB.csv', index=False )
warnings.filterwarnings('ignore' )<load_from_csv>
sample_submission['target'] = lstm_preds
sample_submission.head()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
sample_submission.to_csv('submission_NN.csv', index=False )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<set_options>
sample_submission['target'] = bert_preds.round().astype(int)
 sample_submission.head()
def seed_everything(seed=321):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=321 )<concatenate>
sample_submission.to_csv('submission_NN_bert.csv', index=False )
n_comp = 29
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=321 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=321 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
bert_preds = bert_preds[:, 0]
bert_preds[:5]
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features<merge>
ensemble_preds =.5*nb_preds_prob +.5*bert_preds
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
<drop_column>
sample_submission['target'] = ensemble_preds.round().astype(int)
 sample_submission.head()
<split><EOS> 
sample_submission.to_csv('submission_ensemble.csv', index=False )