Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<predict_on_test>
import numpy as np
import pandas as pd
import os
y_pred = model.predict(x_test, batch_size=1024, verbose=1)
<save_to_csv>
! pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False)
 model.save_weights('best.hdf5')
<import_modules>
!pip install.. /input/iterative-stratification/iterative_stratification-0.1.6-py3-none-any.whl
start_real = datetime.now()
np.random.seed(123 )<prepare_x_and_y>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
os.environ['JOBLIB_START_METHOD'] = 'forkserver'
INPUT_PATH = r'.. /input'
class ItemSelector(BaseEstimator, TransformerMixin):
def __init__(self, field, start_time=time.time()):
self.field = field
self.start_time = start_time
def fit(self, x, y=None):
return self
def transform(self, dataframe):
print(f'[{time.time() -self.start_time}] select {self.field}')
 dt = dataframe[self.field].dtype
if is_categorical_dtype(dt):
return dataframe[self.field].cat.codes[:, None]
elif is_numeric_dtype(dt):
return dataframe[self.field][:, None]
else:
return dataframe[self.field]
class DropColumnsByDf(BaseEstimator, TransformerMixin):
def __init__(self, min_df=1, max_df=1.0):
self.min_df = min_df
self.max_df = max_df
def fit(self, X, y=None):
m = X.tocsc()
self.nnz_cols =(( m != 0 ).sum(axis=0)>= self.min_df ).A1
if self.max_df < 1.0:
max_df = m.shape[0] * self.max_df
self.nnz_cols = self.nnz_cols &(( m != 0 ).sum(axis=0)<= max_df ).A1
return self
def transform(self, X, y=None):
m = X.tocsc()
return m[:, self.nnz_cols]
def get_rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))
 def split_sub(text):
return re.sub('[^0-9'a-zA-Z]+', ' ', text)
 def preprocess_pandas(train, test, start_time=time.time()):
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
print('Train shape : ', train.shape)
 nrow_train = train.shape[0]
y_train = train[list_classes]
merge: pd.DataFrame = pd.concat([train, test])
 del train
del test
gc.collect()
merge['comment_text'] = merge['comment_text'] \ 
.fillna('')\ 
.str.lower() \ 
.apply(lambda x: split_sub(x))
print(f'[{time.time() - start_time}] Missing filled.')
 return merge, y_train, nrow_train
def intersect_drop_columns(train: csr_matrix, valid: csr_matrix, min_df=0):
t = train.tocsc()
v = valid.tocsc()
nnz_train =(( t != 0 ).sum(axis=0)>= min_df ).A1
nnz_valid =(( v != 0 ).sum(axis=0)>= min_df ).A1
nnz_cols = nnz_train & nnz_valid
res = t[:, nnz_cols], v[:, nnz_cols]
return res
def wordCount(text):
try:
if text == 'No description yet':
return 0
else:
text = text.lower()
words = [w for w in text.split(" ")]
return len(words)
 except:
return 0
mp.set_start_method('forkserver', True)
 start_time = time.time()
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 print(f'[{time.time() - start_time}] Finished to load data')
 print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 train['char_len'] = train['comment_text'].apply(lambda x: len(x))
test['char_len'] = test['comment_text'].apply(lambda x: len(x))
train['desc_len'] = train['comment_text'].apply(lambda x: wordCount(x))
test['desc_len'] = test['comment_text'].apply(lambda x: wordCount(x))
repl = {
"&lt;3": " good ",
":d": " good ",
":dd": " good ",
":p": " good ",
"8)": " good ",
":-)": " good ",
":)": " good ",
";)": " good ",
"(-:": " good ",
"(:": " good ",
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " bad ",
":(": " bad ",
":s": " bad ",
":-s": " bad ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
print(f'[{time.time() - start_time}] Crap Removed.')
 trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
stem = SnowballStemmer('english')
 def token_stemmer(text):
tokens = nltk.word_tokenize(text)
 tokens = [stem.stem(item)for item in tokens]
return ' '.join(tokens)
 train['comment_text'] = train['comment_text'].apply(lambda x: token_stemmer(x))
test['comment_text'] = test['comment_text'].apply(lambda x: token_stemmer(x))
print(f'[{time.time() - start_time}] Stemming Completed.')
 abuses = pd.read_csv('.. /input/abusive-words3/abusive_words.csv', header=None)
 def has_abuse(text):
abuse_dict={} 
(abuses[0] ).apply(lambda x: abuse_dict.update({str(x): text.count(str(x)) }))
return abuse_dict
train['abuse_dict'] = train['new_comment_text'].apply(lambda x: has_abuse(x))
test['abuse_dict'] = test['new_comment_text'].apply(lambda x: has_abuse(x))
print(f'[{time.time() - start_time}] Abuses Dict Completed.')
 train['num_abuses'] = train['abuse_dict'].apply(lambda x: sum(x.values()))
test['num_abuses'] = test['abuse_dict'].apply(lambda x: sum(x.values()))
print(f'[{time.time() - start_time}] Abuses Completed.')
 merge, y_train, nrow_train = preprocess_pandas(train, test, start_time)
 meta_params = {
'desc_ngram':(1, 3),
'desc_max_f': 150000,
'desc_max_df': 0.5,
'desc_min_df': 10}
stopwords = frozenset(['the', 'a', 'an', 'is', 'it', 'this', ])
 vectorizer = FeatureUnion([ 
('comment_text_2', Pipeline([ 
('select', ItemSelector('comment_text', start_time=start_time)) , 
('transform', TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
stop_words='english',
token_pattern=r'\w{1,}',
ngram_range=(1, 1),
max_features=15000 
)) , 
('drop_cols', DropColumnsByDf(min_df=2))
])) , 
('comment_text_3', Pipeline([ 
('select', ItemSelector('comment_text', start_time=start_time)) , 
('transform', TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 6),
max_features=20000 
)) , 
('drop_cols', DropColumnsByDf(min_df=2))
])) , 
('abuse_dict', Pipeline([ 
('select', ItemSelector('abuse_dict', start_time=start_time)) , 
('ohe', DictVectorizer(sparse=False))
])) , 
('num_abuses', Pipeline([ 
('select', ItemSelector('num_abuses', start_time=start_time)) , 
('ohe', OneHotEncoder())
 ]))
], n_jobs=1)
 sparse_merge = vectorizer.fit_transform(merge)
 print(f'[{time.time() - start_time}] Merge vectorized')
 print(sparse_merge.shape)
 tfidf_transformer = TfidfTransformer()
X = tfidf_transformer.fit_transform(sparse_merge)
 print(f'[{time.time() - start_time}] TF/IDF completed')
 X_train = X[:nrow_train]
print(X_train.shape)
 X_test = X[nrow_train:]
del merge
del sparse_merge
del vectorizer
del tfidf_transformer
gc.collect()
X_train, X_test = intersect_drop_columns(X_train, X_test, min_df=1)
 print(f'[{time.time() - start_time}] Drop only in train or test cols: {X_train.shape[1]}')
 gc.collect()
losses = []
predictions = {'id': test['id']}
for class_name in list_classes:
train_target = train[class_name]
classifier = LogisticRegression(solver='sag')
 cv_loss = np.mean(cross_val_score(classifier, X_train, train_target, cv=3, scoring='roc_auc'))
losses.append(cv_loss)
 print('CV score for class {} is {}'.format(class_name, cv_loss))
classifier.fit(X_train, train_target)
 predictions[class_name] = classifier.predict_proba(X_test)[:, 1]
print(f'[{time.time() - start_time}] Predict Ridge completed.')
 submission = pd.DataFrame.from_dict(predictions)
 submission.to_csv("submission_ridge.csv", index=False)
 print("Evaluating the model on validation data...")
<set_options>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'<compute_test_metric>
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 remove_vehicle = True
if remove_vehicle:
train_features = train.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 else:
train_features = train
EMBEDDING_FILE = '.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))<load_from_csv>
col_features = list(train_features.columns)[1:]
print(train_features[col_features[0]].value_counts())
 print(test_features[col_features[0]].value_counts())
 print(train_features[col_features[1]].value_counts())
 print(test_features[col_features[1]].value_counts())
 print(train_features[col_features[2]].value_counts())
 print(test_features[col_features[2]].value_counts())
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv' )<prepare_x_and_y>
print(test_features[col_features].isna().sum().values.sum())
 print(np.isinf(test_features[col_features[3:]].values ).sum())
 print(train_features[col_features].isna().values.sum())
 print(np.isinf(train_features[col_features[3:]].values ).sum() )
X_train = train["comment_text"].fillna("fillna" ).values
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].fillna("fillna" ).values 
<define_variables>
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 print(ratios)
max_features = 30000
maxlen = 120
embed_size = 300 
<string_transform>
columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored)
 print(ratios_nonscored)
 print(len(columns_nonscored), len(ratios_nonscored))
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen)
<feature_engineering>
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features.columns)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32)
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector.copy()<compute_train_metric>
class MOADataset(Dataset):
def __init__(self, x_cats, x_nums, y=None, y2=None):
self.cats = x_cats
self.nums = x_nums
self.y = y
self.y2 = y2
def __len__(self):
return len(self.cats)
 def __getitem__(self, index):
x1 = torch.as_tensor(self.cats[index], dtype=torch.long)
 x2 = torch.as_tensor(self.nums[index], dtype=torch.float)
 if self.y is not None:
label = torch.as_tensor(self.y[index], dtype=torch.float)
 if self.y2 is not None:
label2 = torch.as_tensor(self.y2[index], dtype=torch.float)
 return x1, x2, label, label2
return x1, x2, label
return x1, x2
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))<choose_model_class>
class MOA_MLP(nn.Module):
def __init__(self, num_cats=[2,3,2] , cats_emb_size=[2,2,2], num_numericals=872, hidden_size_numericals=2048,
num_class=206, aux=None):
super().__init__()
self.cat_emb1 = nn.Embedding(num_cats[0], cats_emb_size[0], padding_idx=0)
 self.cat_emb2 = nn.Embedding(num_cats[1], cats_emb_size[1], padding_idx=0)
 self.norms = nn.BatchNorm1d(sum(cats_emb_size)+num_numericals)
 self.dropout = nn.Dropout(0.2)
 self.proj = nn.utils.weight_norm(nn.Linear(sum(cats_emb_size)+ num_numericals, hidden_size_numericals))
self.norm_proj = nn.BatchNorm1d(hidden_size_numericals)
 self.dropout2 = nn.Dropout(0.5)
 hd_1 = hidden_size_numericals//2
hd_2 = hd_1//2
self.extractor = nn.Sequential(nn.utils.weight_norm(nn.Linear(hidden_size_numericals, hd_1)) ,
nn.PReLU() ,
nn.BatchNorm1d(hd_1),
nn.Dropout(0.5), 
)
 self.cls = nn.utils.weight_norm(nn.Linear(hd_1, num_class))
self.cls_aux=None
if aux is not None:
self.cls_aux = nn.utils.weight_norm(nn.Linear(hd_1, aux))
def forward(self, x_cat, x_num):
cat_features = torch.cat([self.cat_emb1(x_cat[:,0]), self.cat_emb2(x_cat[:,1])], dim=1)
 all_features = torch.cat([cat_features, x_num], dim=1)
 all_features = self.norms(all_features)
 all_features = self.dropout(all_features)
 proj_features = self.proj(all_features)
 proj_features = self.norm_proj(F.relu(proj_features))
proj_features = self.dropout2(proj_features)
 features_reduced = self.extractor(proj_features)
 outputs = self.cls(features_reduced)
 if self.cls_aux is not None:
outputs2 = self.cls_aux(features_reduced)
 return outputs, outputs2
return outputs
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(80, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()<split>
def train_one_epoch(model, dataloader, cfg, optimizer, loss_fn, loss_fn_aux=None, accumulation=1, with_aux_class=False, verbose=True):
model.train()
scaler = GradScaler()
optimizer.zero_grad()
N = 0.
 total_loss = 0.
 t=tqdm(dataloader, disable=~verbose)
 for i, batch in enumerate(t):
x1 = batch[0]
x2 = batch[1]
labels = batch[2]
x1 = x1.to(cfg.device)
 x2 = x2.to(cfg.device)
 labels = labels.to(cfg.device)
 if with_aux_class:
labels2 = batch[3]
labels2 = labels2.to(cfg.device)
 with autocast(cfg.use_apex):
if with_aux_class:
outputs, outputs2 = model(x1, x2)
 loss1 = loss_fn(outputs, labels ).mean(0 ).mean()
loss2 = loss_fn_aux(outputs2, labels2 ).mean(0 ).mean()
loss = loss1 + 0.5*loss2
else:
outputs = model(x1, x2)
 loss = loss_fn(outputs, labels ).mean(0 ).mean()
N += len(x1)
 total_loss +=(loss.item() * len(x1))
if cfg.use_apex:
loss = loss/accumulation
scaler.scale(loss ).backward()
else:
loss = loss/accumulation
loss.backward()
if(i+1)%accumulation == 0 or i-1 == len(dataloader):
if cfg.use_apex:
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
else:
optimizer.step()
optimizer.zero_grad()
t.set_description("Loss : {0}".format(total_loss/N))
t.refresh()
def evals(model, dataloader, cfg, loss_fn, loss_fn_aux=None, verbose=True):
model.eval()
N = 0.
 total_loss = 0.
 y_preds = []
y_targets = []
t=tqdm(dataloader, disable=~verbose)
 with torch.no_grad() :
for i, batch in enumerate(t):
x1 = batch[0]
x2 = batch[1]
labels = batch[2]
x1 = x1.to(cfg.device)
 x2 = x2.to(cfg.device)
 labels = labels.to(cfg.device)
 if cfg.with_aux_class:
labels2 = batch[3]
labels2 = labels2.to(cfg.device)
 with autocast(cfg.use_apex):
if cfg.with_aux_class:
outputs, outputs2 = model(x1, x2)
 loss1 = loss_fn(outputs, labels ).mean(0 ).mean()
loss2 = loss_fn_aux(outputs2, labels2 ).mean(0 ).mean()
loss = loss1 + 0.5*loss2
else:
outputs = model(x1, x2)
 loss = loss_fn(outputs, labels ).mean(0 ).mean()
N += len(x1)
 total_loss +=(loss.item() * len(x1))
t.set_description("Loss : {0}".format(total_loss/N))
t.refresh()
y_preds.append(torch.sigmoid(outputs ).detach().cpu().numpy())
 y_targets.append(labels.detach().cpu().numpy())
 y_preds = np.concatenate(y_preds, axis=0)
 y_targets = np.concatenate(y_targets, axis=0)
 score = log_loss_multi(y_targets, y_preds)
 return y_preds, y_targets, score
def inference_fn(model, dataloader, cfg,verbose=True):
model.eval()
N = 0.
 y_preds = []
with torch.no_grad() :
for i, batch in enumerate(tqdm(dataloader, disable=~verbose)) :
x1 = batch[0]
x2 = batch[1]
x1 = x1.to(cfg.device)
 x2 = x2.to(cfg.device)
 with autocast(cfg.use_apex):
if cfg.with_aux_class:
outputs, outputs2 = model(x1, x2)
 else:
outputs = model(x1, x2)
 y_preds.append(torch.sigmoid(outputs ).detach().cpu().numpy())
 y_preds = np.concatenate(y_preds, axis=0)
 return y_preds
batch_size = 32
epochs = 2
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.85, random_state=233)
 RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1 )<train_model>
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=2 )<save_to_csv>
np.log(0)*0
y_pred = model.predict(x_test, batch_size=1024,verbose=99)
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )<load_from_csv>
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ' )<define_variables>
def train_fold(fold, model, tr_dataloader, val_dataloader, cfg, optimizer, reducer, loss_fn, loss_fn_aux=None, accumulation=1, with_aux_class=False):
best_score = np.inf
best_preds = None
best_targets = None
for e in range(cfg.EPOCHS):
train_one_epoch(model, tr_dataloader, cfg, optimizer, loss_fn, loss_fn_aux=loss_fn_aux, accumulation=accumulation, with_aux_class=with_aux_class, verbose=cfg.verbose)
 preds, targets, score = evals(model, val_dataloader, cfg, loss_fn, loss_fn_aux=loss_fn_aux, with_aux_class=with_aux_class, verbose=cfg.verbose)
 reducer.step(score)
 if score < best_score:
print("
best_score = score
best_preds = preds
best_targets= targets
torch.save(model.state_dict() , cfg.save_name + f"_{fold}.pth")
 print("
return best_preds, best_targets, score
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
del trate, tete
train.drop(["new_comment_text"], axis=1, inplace=True)
 test.drop(["new_comment_text"], axis=1, inplace=True )<concatenate>
def inference_fold(folds, model, test_loader,cfg):
preds = []
for fold in range(folds):
name = cfg.save_name + f"_{fold}.pth"
model.load_state_dict(torch.load(name))
p = inference_fn(model, test_loader, cfg, cfg.verbose,cfg.with_aux_class)
 preds.append(p)
 return preds
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )<string_transform>
def check_targets(targets):
for i in range(targets.shape[1]):
if len(np.unique(targets[:,i])) != 2:
return False
return True
def check_labels(A):
w = np.zeros(A.shape[1])
 for i in range(A.shape[1]):
if len(np.unique(A[:,i])) == 2:
w[i] = 1
return w.reshape(1, -1)
 def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()<define_variables>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
cont_patterns = [ 
(b'US', b'United States'), 
(b'IT', b'Information Technology'), 
(b'(W|w)on't', b'will not'), 
(b'(C|c)an't', b'can not'), 
(b'(I|i)'m', b'i am'), 
(b'(A|a)in't', b'is not'), 
(b'(\w+)'ll', b'\g<1> will'), 
(b'(\w+)n't', b'\g<1> not'), 
(b'(\w+)'ve', b'\g<1> have'), 
(b'(\w+)'s', b'\g<1> is'), 
(b'(\w+)'re', b'\g<1> are'), 
(b'(\w+)'d', b'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def prepare_for_char_n_gram(text):

 in r""
clean = bytes(text.lower() , encoding="utf-8")
 and \t
clean = clean.replace(b"
", b" ")
 clean = clean.replace(b"\t", b" ")
 clean = clean.replace(b"\b", b" ")
 clean = clean.replace(b"\r", b" ")
 for(pattern, repl)in patterns:
clean = re.sub(pattern, repl, clean)
 exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))
 clean = b" ".join([exclude.sub(b'', token)for token in clean.split() ])
 clean = re.sub(b"\d+", b" ", clean)
 clean = re.sub(b'\s+', b' ', clean)
 clean = re.sub(b'\s+$', b'', clean)
 clean = re.sub(b" ", b"
clean = b"
return str(clean, 'utf-8')
 def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))<feature_engineering>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.aux_class = targets2_tr.shape[1]
self.use_apex=False
self.verbose=False
self.batch_size = 128
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 10
self.num_cats=[3+1,2+1] if remove_vehicle else [2+1,3+1,2+1]
self.cats_emb_size=[1]* cat_tr.shape[1]
self.num_numericals= numerical_tr.shape[1]
self.hidden_size_numericals=2048
self.num_ensembling = 1
self.strategy = "KFOLD"
self.save_name = ".. /input/moannweights/MOAMLP-aux-raw/MOA_mlp-KFOLD10_"
cfg = Config()
cfg.with_aux_class = True if cfg.aux_class is not None else False
print(cfg.num_class, cfg.aux_class,cfg.with_aux_class )
def get_indicators_and_clean_comments(df):

 ", x))
df["raw_word_len"] = df["comment_text"].apply(lambda x: len(x.split()))
df["raw_char_len"] = df["comment_text"].apply(lambda x: len(x))
df["nb_upper"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[A-Z]", x))
df["nb_fk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ff]\S{2}[Kk]", x))
df["nb_sk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ss]\S{2}[Kk]", x))
df["nb_dk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[dD]ick", x))
df["nb_you"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\W[Yy]ou\W", x))
df["nb_mother"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wmother\W", x))
df["nb_ng"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wnigger\W", x))
df["start_with_columns"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"^\:+", x))
df["has_timestamp"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\d{2}|:\d{2}", x))
df["has_date_long"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{2}:\d{2}, \d{1,2} \w+ \d{4}", x))
df["has_date_short"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{1,2} \w+ \d{4}", x))
df["has_mail"] = df["comment_text"].apply(
 lambda x: count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', x)
)
 df["has_emphasize_equal"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\={2}.+\={2}", x))
df["has_emphasize_quotes"] = df["comment_text"].apply(lambda x: count_regexp_occ(r""{4}\S+"{4}", x))
df["clean_comment"] = df["comment_text"].apply(lambda x: prepare_for_char_n_gram(x))
df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
df["clean_chars"] = df["clean_comment"].apply(lambda x: len(set(x)))
 df["clean_chars_ratio"] = df["clean_comment"].apply(lambda x: len(set(x)))/ df["clean_comment"].apply(
 lambda x: 1 + min(99, len(x)) )<feature_engineering>
loss_fn = nn.BCEWithLogitsLoss(reduction="none")
 loss_fn_aux = nn.BCEWithLogitsLoss(reduction="none" )
num_features = [f_ for f_ in train.columns
if f_ not in ["comment_text", "clean_comment", "id", "remaining_chars", 'has_ip_address'] + class_names]
for f in num_features:
all_cut = pd.cut(pd.concat([train[f], test[f]], axis=0), bins=20, labels=False, retbins=False)
 train[f] = all_cut.values[:train.shape[0]]
test[f] = all_cut.values[train.shape[0]:]
train_num_features = train[num_features].values
test_num_features = test[num_features].values<concatenate>
test_dataset = MOADataset(cat_test, numerical_test)
 test_dataloader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False, num_workers=2)
train_text = train['clean_comment'].fillna("")
 test_text = test['clean_comment'].fillna("")
 all_text = pd.concat([train_text, test_text] )<choose_model_class>
model = MOA_MLP(num_cats=cfg.num_cats , cats_emb_size=cfg.cats_emb_size, num_numericals=cfg.num_numericals, hidden_size_numericals=cfg.hidden_size_numericals,
num_class=cfg.num_class, aux=cfg.aux_class)
 print(model )
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
tokenizer=lambda x: regex.findall(r'[^\p{P}\W]+', x),
analyzer='word',
token_pattern=None,
min_df=5,
ngram_range=(1, 2),
max_features=60000)
<train_model>
if cfg.strategy == "KFOLD":
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
preds_test = []
masks = []
for seed in range(cfg.num_ensembling):
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=42+seed, shuffle=True)
 p = []
temp_mask = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 model = MOA_MLP(num_cats=cfg.num_cats , cats_emb_size=cfg.cats_emb_size, num_numericals=cfg.num_numericals, hidden_size_numericals=cfg.hidden_size_numericals,
num_class=cfg.num_class, aux=cfg.aux_class ).to(cfg.device)
 if cfg.with_aux_class:
val_dataset = MOADataset(cat_tr[val_idx], numerical_tr[val_idx], y=targets_tr[val_idx], y2=targets2_tr[val_idx])
 else:
val_dataset = MOADataset(cat_tr[val_idx], numerical_tr[val_idx], y=targets_tr[val_idx])
 val_dataloader = DataLoader(val_dataset, batch_size=cfg.batch_size, shuffle=False, num_workers=2)
 temp_mask.append(check_labels(targets_tr[train_idx]))
name = cfg.save_name + f"_{j}_{seed}.pth"
model.load_state_dict(torch.load(name, map_location=cfg.device))
preds , targets, score = evals(model, val_dataloader, cfg, loss_fn, loss_fn_aux=loss_fn_aux, verbose=True)
 p.append(inference_fn(model, test_dataloader, cfg,verbose=False))
oof_preds.append(preds)
 oof_targets.append(targets)
 scores.append(score)
 scores_auc.append(auc_multi(targets,preds))
preds_test.append(np.array(p))
masks.append(np.stack(temp_mask))
word_vectorizer.fit(all_text )<feature_engineering>
if cfg.strategy == "KFOLD":
oof_preds = np.concatenate(oof_preds ).reshape(cfg.num_ensembling, -1, 206)
 oof_targets = np.concatenate(oof_targets ).reshape(cfg.num_ensembling, -1, 206)
 scores_auc = np.array(scores_auc ).reshape(cfg.num_ensembling, -1)
 for i in range(oof_preds.shape[0]):
print("CV score : ", log_loss_multi(oof_targets[i,:], oof_preds[i,:]))
print("auc mean : ", sum(scores_auc[i,:])/len(scores_auc[i,:]))
preds_test = np.stack(preds_test)
 masks = np.stack(masks )
train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text )<concatenate>
preds_test2 = preds_test*masks
preds_test2 = preds_test2.sum(1 ).sum(0)/masks.sum(1 ).sum(0 ).sum(0 )
train_features = hstack([train_word_features, train_links_n, train_num_features] ).tocsr()
test_features = hstack([test_word_features, test_links_n, test_num_features] ).tocsr()<split>
submission[columns] = preds_test2
submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0
X_train, X_test, y_train, y_test = train_test_split(train_features, train['toxic'], test_size=0.3, random_state=42 )<train_model>
submission.to_csv("submission.csv", index=False )
lgb_train = lgb.Dataset(X_train , y_train)
 lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective':'binary',
'metric': {'auc'},
'nthread': -1,
'feature_fraction': 0.4,
'num_leaves': 50,
'verbose': 0
}
print('Start training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=200,
valid_sets=lgb_eval,
early_stopping_rounds=20 )<train_model>
submission.to_csv("submission.csv", index=False )
<create_dataframe><EOS>