Kaggle competition: Toxic Comment Classification Challenge
def predict_torch(test_df):
test_df.drop(columns=['weight', 'date'], inplace=True)
 test_df.reset_index(drop=True, inplace=True)
 test_df = preprocess_pipe.transform(test_df)
 test_df = torch.tensor(test_df ).float().view(-1, 130 ).to(DEVICE)
 predictions = []
for i in range(test_df.shape[0]):
pred_p = model(test_df ).logits.detach().cpu().numpy().reshape(( -1, 5))
predictions.append(pred_p)
 predictions = np.vstack(predictions)
 predictions = np.median(predictions, axis=1)
 return(predictions >= 0.5 ).astype(int )<define_variables>
!unzip.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip
!unzip.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip
!unzip.. /input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip
!unzip.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip
env = janestreet.make_env()
env_iter = env.iter_test()<predict_on_test>
TRAIN = './train.csv'
TEST = './test.csv'
TEST_LABEL = './test_labels.csv'
SAMPLE = './sample_submission.csv'
EPOCHS = 2
MAX_TOKEN_COUNT = 128
BATCH_SIZE = 32
for test_df, pred_df in env_iter:
if test_df["weight"].item() > 0:
pred_df.action = predict_torch(test_df)
 else:
pred_df.action = 0
env.predict(pred_df )<train_model>
%matplotlib inline
%config InlineBackend.figure_format='retina'
RANDOM_SEED = 42
sns.set(style='whitegrid', palette='muted', font_scale=1.2)
 HAPPY_COLORS_PALETTE = ["
sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))
rcParams['figure.figsize'] = 12, 8
pl.seed_everything(RANDOM_SEED )
print('Done !' )<load_from_csv>
df = pd.read_csv(TRAIN)
 test_df = pd.read_csv(TEST)
 test_label = pd.read_csv(TEST_LABEL)
 sample_sub = pd.read_csv(SAMPLE)
 df.describe()
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
batch_size = 5000
hidden_units = [150, 150, 150]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.fit(X_train, y_train, epochs=200, batch_size=5000)
 models = []
models.append(clf)
 th = 0.5000
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
train_df, val_df = train_test_split(df, test_size=0.05)
 train_df.shape, val_df.shape, test_df.shape
LABEL_COLUMNS = df.columns.tolist() [2:]
df[LABEL_COLUMNS].sum().sort_values().plot(kind="barh");
sns.set_context("talk")
 style.use('fivethirtyeight')
 pd.options.display.max_columns = None
warnings.filterwarnings('ignore' )<set_options>
class ToxicCommentsDataset(Dataset):

 def __init__(
 self,
data: pd.DataFrame,
tokenizer: BertTokenizer,
max_token_len: int = 128,
test= False 
):
self.data = data
self.tokenizer = tokenizer
self.max_token_len = max_token_len
self.test = test
def __len__(self):
return len(self.data)
 def __getitem__(self, index: int):
data_row = self.data.iloc[index]
_id = data_row['id']
comment_text = data_row.comment_text
if not self.test:
labels = data_row[LABEL_COLUMNS]
encoding = self.tokenizer.encode_plus(
 comment_text,
max_length=self.max_token_len,
padding="max_length",
truncation=True,
add_special_tokens=True,
return_token_type_ids=False,
return_attention_mask=True,
return_tensors='pt', 
)
 if not self.test:
return dict(
 _id = _id,
comment_text=comment_text,
input_ids = encoding["input_ids"].flatten() ,
attention_mask=encoding["attention_mask"].flatten() ,
labels=torch.FloatTensor(labels)
)
 else:
return dict(
 _id = _id,
comment_text=comment_text,
input_ids = encoding["input_ids"].flatten() ,
attention_mask=encoding["attention_mask"].flatten() 
)
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
MIXED_PRECISION = False
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
<define_variables>
BERT_MODEL_NAME = 'bert-base-cased'
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_NAME )
DEBUG = False
SEED = 2021
INPUT_DIR = '.. /input/janestreet-save-as-feather/'
TRADING_THRESHOLD = 0.50
DATE_BEGIN = 86
CV_STRATEGY = 'PurgedGroupTimeSeriesSplit'
FOLDS = 5
THRESHOLD = 0.5
STAGE = 3
if STAGE == 2:
WEIGHT_DIR = '.. /input/janestreet-mlp-pretrain-stage1-weight-0/'
elif STAGE == 3:
WEIGHT_DIR = '.. /input/janestreet-mlp-train-stage2-weight-0/'<import_modules>
train_dataset = ToxicCommentsDataset(
 train_df,
tokenizer,
max_token_len=MAX_TOKEN_COUNT 
)
 val_dataset = ToxicCommentsDataset(
 val_df,
tokenizer,
max_token_len=MAX_TOKEN_COUNT 
)
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<split>
test_dataset = ToxicCommentsDataset(
 test_df,
tokenizer,
max_token_len=MAX_TOKEN_COUNT,
test=True 
)
class GroupKFold(object):

 def __init__(self, n_splits=4, shuffle=True, random_state=42):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_n_splits(self, X=None, y=None, group=None):
return self.n_splits
def split(self, X, y, group):
kf = model_selection.KFold(n_splits=self.n_splits, shuffle=self.shuffle, random_state=self.random_state)
 unique_ids = X[group].unique()
for fold,(tr_group_idx, va_group_idx)in enumerate(kf.split(unique_ids)) :
tr_group, va_group = unique_ids[tr_group_idx], unique_ids[va_group_idx]
train_idx = np.where(X[group].isin(tr_group)) [0]
val_idx = np.where(X[group].isin(va_group)) [0]
yield train_idx, val_idx
class StratifiedGroupKFold(object):

 def __init__(self, n_splits=4, shuffle=True, random_state=42):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_n_splits(self, X=None, y=None, group=None):
return self.n_splits
def split(self, X, y, group):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
groups = X[group].values
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(self.n_splits)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(self.random_state ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(self.n_splits):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(self.n_splits):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_idx = [i for i, g in enumerate(groups)if g in train_groups]
test_idx = [i for i, g in enumerate(groups)if g in test_groups]
yield train_idx, test_idx 
<load_from_disk>
train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 val_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)
 test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False )
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather' )<train_model>
gc.collect()
original_size = train.shape[0]
train = train.query(f'date >= {DATE_BEGIN}')
 if STAGE == 1:
train = train.query('weight == 0' ).reset_index(drop=True)
 elif STAGE == 2:
train = train.query('weight > 0' ).reset_index(drop=True)
 elif STAGE == 3:
train = train.query('weight > 0' ).reset_index(drop=True)
 print('inference')
 print('Train size reduced from {:,} to {:,}.'.format(original_size, train.shape[0]))<data_type_conversions>
class ToxicCommentTagger(nn.Module):
def __init__(self, n_classes: int, n_training_steps=None, n_warmup_steps=None):
super().__init__()
self.bert = BertModel.from_pretrained(BERT_MODEL_NAME, return_dict=True)
 self.classifier = nn.Linear(self.bert.config.hidden_size, n_classes)
 self.n_training_steps = n_training_steps
self.n_warmup_steps = n_warmup_steps
self.criterion = nn.BCELoss()
def forward(self, input_ids, attention_mask, labels=None):
output = self.bert(input_ids, attention_mask=attention_mask)
 output = self.classifier(output.pooler_output)
 output = torch.sigmoid(output)
 loss = 0
if labels is not None:
loss = self.criterion(output, labels)
 return loss, output
train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']<prepare_x_and_y>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 bert_model = ToxicCommentTagger(len(LABEL_COLUMNS)).to(device )
if DEBUG:
train = train.sample(10000, random_state=SEED)
 X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<define_variables>
model = bert_model
def seed_everything(seed : int)-> NoReturn :
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )<choose_model_class>
N_EPOCHS = EPOCHS
steps_per_epoch=len(train_df)// BATCH_SIZE
total_training_steps = steps_per_epoch * N_EPOCHS
warmup_steps = total_training_steps // 5
warmup_steps, total_training_steps
nn_params = {
'hidden_unit': 256,
'num_layers': 3,
'gaussnoise': 0.01,
'dropout': 0.2,
'learning_rate': 1e-03,
'label_smoothing': 1e-02,
'batch_size': 1024,
'epochs': 196
}
def create_model(nn_params, input_dim, output_dim):
inputs = tf.keras.layers.Input(input_dim)
 x = tf.keras.layers.BatchNormalization()(inputs)
 for i in range(nn_params['num_layers']):
x = tf.keras.layers.Dense(nn_params['hidden_unit'] //(2 ** i), activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.GaussianNoise(nn_params['gaussnoise'] )(x)
 x = tf.keras.layers.Dropout(nn_params['dropout'] )(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs, outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=nn_params['learning_rate'])
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=nn_params['label_smoothing'])
 model.compile(optimizer=opt,
loss=loss,
metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
model = create_model(nn_params, 130, 5)
 model.summary()<load_pretrained>
optimizer = AdamW(model.parameters() , lr=2e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=warmup_steps,
num_training_steps=total_training_steps
)
%%time
if STAGE == 3:
models = []
for fold in tqdm(range(FOLDS)) :
tf.keras.backend.clear_session()
model = create_model(nn_params, X.shape[-1], y.shape[-1])
 model.load_weights(WEIGHT_DIR + f'model_{SEED}_{fold}.hdf5')
 models.append(model)
 else:
if CV_STRATEGY == 'PurgedGroupTimeSeriesSplit':
gkf = PurgedGroupTimeSeriesSplit(n_splits=FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))
elif CV_STRATEGY == "GroupKFold":
cv = GroupKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)
 splits = cv.split(train, train['resp'].values.astype(int), 'date')
 elif CV_STRATEGY == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)
 splits = cv.split(train, train['resp'].values.astype(int), 'date')
 models = []
for fold,(train_indices, test_indices)in tqdm(enumerate(splits)) :
X_train, X_test = X[train_indices], X[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
tf.keras.backend.clear_session()
model = create_model(nn_params, X.shape[-1], y.shape[-1])
 if STAGE == 2:
model.load_weights(WEIGHT_DIR + f'model_{SEED}_{fold}.hdf5')
 er = tf.keras.callbacks.EarlyStopping(patience=8, restore_best_weights=True, monitor='val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, verbose=1, mode='min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=f'./model_{SEED}_{fold}.hdf5',
save_weights_only=True, verbose=0, monitor='val_loss', save_best_only=True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
model.fit(X_train, y_train, validation_data=(X_test,y_test),
epochs=nn_params['epochs'], batch_size=nn_params['batch_size'], callbacks=nn_callbacks)
 models.append(model )<feature_engineering>
def train() :
model.train()
total_loss, total_accuracy = 0, 0
avg_loss = 0
total_preds=[]
for step,batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
print(' Batch {:>5,} of {:>5,}.'.format(step, len(train_dataloader)))
 input_ids = batch["input_ids"].to(device)
 attention_mask = batch["attention_mask"].to(device)
 labels = batch["labels"].to(device)
 model.zero_grad()
loss, outputs = model(input_ids, attention_mask, labels)
 total_loss = total_loss + loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
outputs=outputs.detach().cpu().numpy()
total_preds.append(outputs)
 avg_loss = total_loss / len(train_dataloader)
 print(f"{step}: {avg_loss}")
 total_preds = np.concatenate(total_preds, axis=0)
 return avg_loss, total_preds
if STAGE == 3:
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= THRESHOLD, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df)
 else:
print('Training done!' )<import_modules>
import cv2
import pandas as pd
import numpy as np
from skimage.morphology import closing, square
from skimage.segmentation import clear_border
from skimage.measure import label, regionprops
from skimage.filters import threshold_otsu
from skimage import data
from skimage.color import label2rgb
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt<define_variables>
def evaluate() :
print("
Evaluating...")
 model.eval()
total_loss, total_accuracy = 0, 0
total_preds = []
total_labels = []
for step,batch in enumerate(val_dataloader):
if step % 50 == 0 and not step == 0:
print(' Batch {:>5,} of {:>5,}.'.format(step, len(val_dataloader)))
 input_ids = batch["input_ids"].to(device)
 attention_mask = batch["attention_mask"].to(device)
 labels = batch["labels"].to(device)
 with torch.no_grad() :
loss, outputs = model(input_ids, attention_mask, labels)
 total_loss = total_loss + loss.item()
outputs = outputs.detach().cpu().numpy()
labels = labels.detach().cpu().numpy()
total_preds.append(outputs)
 total_labels.append(labels)
 avg_loss = total_loss / len(val_dataloader)
 print(f"{step}: {avg_loss}")
 total_preds = np.concatenate(total_preds, axis=0)
 total_labels = np.concatenate(total_labels, axis=0)
 true = np.array(total_labels)
 pred = np.array(total_preds>0.5)
 for i, name in enumerate(LABEL_COLUMNS):
try:
print(f"{name} roc_auc {roc_auc_score(true[:, i], pred[:, i])}")
 except Exception as e:
print(e)
 pass
print(f"Evaluate loss {total_loss / len(val_dataloader)}")
 return avg_loss, total_preds, total_labels
image_path = ".. /input/stage2_test_final/0a8dfb6763ec049b57454e6b5605f872cdf41cd13951fb2a2f31a6e3c8667712/images/0a8dfb6763ec049b57454e6b5605f872cdf41cd13951fb2a2f31a6e3c8667712.png"<train_on_grid>
%%time
best_valid_loss = float('inf')
 train_losses=[]
valid_losses=[]
EPOCHS = 2
for epoch in range(EPOCHS):
print('
Epoch {:} / {:}'.format(epoch + 1, EPOCHS))
train_loss, _ = train()
valid_loss, _, _ = evaluate()
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict() , 'saved_weights.pt')
 train_losses.append(train_loss)
 valid_losses.append(valid_loss)
 print(f'
Training Loss: {train_loss:.3f}')
 print(f'Validation Loss: {valid_loss:.3f}' )
image = cv2.imread(image_path)
 image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY )<set_options>
def test() :
print("
Testing...")
 model.eval()
total_loss, total_accuracy = 0, 0
total_preds = []
_ids = []
for step,batch in enumerate(test_dataloader):
if step % 50 == 0 and not step == 0:
print(' Batch {:>5,} of {:>5,}.'.format(step, len(test_dataloader)))
 _id = batch["_id"]
input_ids = batch["input_ids"].to(device)
 attention_mask = batch["attention_mask"].to(device)
 with torch.no_grad() :
loss, outputs = model(input_ids, attention_mask)
 total_loss = total_loss + loss
outputs = outputs
_ids.append(_id)
 total_preds.append(outputs)
 avg_loss = total_loss / len(test_dataloader)
 _ids = np.concatenate(_ids, axis=0)
 total_preds = torch.cat(total_preds, axis=0)
 results = dict(id=_ids,
predictions = total_preds 
)
 return avg_loss, total_preds, results
thresh = threshold_otsu(image)
 if thresh:
bw = closing(image > thresh, square(3))
else:
ret, bw = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY )<categorify>
def evaluate_roc(probs, y_true):

 preds = probs
fpr, tpr, threshold = roc_curve(y_true, preds)
 roc_auc = auc(fpr, tpr)
 print(f'AUC: {roc_auc:.4f}')
 y_pred = np.where(preds >= 0.5, 1, 0)
 accuracy = accuracy_score(y_true, y_pred)
 print(f'Accuracy: {accuracy*100:.2f}%')
 plt.title('Receiver Operating Characteristic')
 plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
 plt.legend(loc = 'lower right')
 plt.plot([0, 1], [0, 1],'r--')
 plt.xlim([0, 1])
 plt.ylim([0, 1])
 plt.ylabel('True Positive Rate')
 plt.xlabel('False Positive Rate')
 plt.show()
def rle_encoding(x):

 dots = np.where(x.T.flatten() ==1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b+1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths<import_modules>
avg_loss, total_preds, total_labels = evaluate()
import glob<save_to_csv>
for i, name in enumerate(LABEL_COLUMNS):
print(f"label: {name}")
 evaluate_roc(total_preds[:,i]>0.5, total_labels[:,i] )
df.to_csv("submission.csv", index=False )<import_modules>
avg_test_loss, total_test_preds, sub = test()
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<train_model>
D = pd.DataFrame()
D['id'] = sub['id']
D
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<load_from_disk>
D[LABEL_COLUMNS] =(sub['predictions'].cpu().numpy())
 D
data_path = ".. /input/"
train_file = data_path + "train.json"
test_file = data_path + "test.json"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape )<define_variables>
D.to_csv("submission.csv", index=False )