Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<string_transform>
!pip install -q.. /input/tensorflow-determinism
!pip install -q.. /input/huggingfacetokenizers/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl
!pip uninstall --yes pytorch-transformers
!pip install -q.. /input/huggingface-transformers-master
review_to_words(train['review'][0] )<string_transform>
os.environ['TF_DETERMINISTIC_OPS'] = '1'
gc.enable()
np.set_printoptions(suppress=True)
 print('Tensorflow version', tf.__version__)
 print('PyTorch version', torch.__version__)
 print('Transformers version',
transformers.__version__ )
num_reviews = train.shape[0]
train_corpus = []
for i in tqdm_notebook(range(num_reviews)) :
train_corpus.append(review_to_words(train['review'][i]))<string_transform>
gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
tf.config.experimental.set_visible_devices(gpus[0], 'GPU')
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPU")
 except RuntimeError as e:
print(e)
len(train_corpus[-1].split() )<choose_model_class>
rand_seed = 20201120
n_splits = 5
BERT_PATH = ".. /input/"
dataset_folder = Path(".. /input/google-quest-challenge")
 MODEL_PATH_list = [
".. /input/tf-roberta-base-exp-v7/",
".. /input/bert-base-uncased-exp-v4/",
".. /input/tf-bert-base-cased-exp-v4/",
".. /input/tf-roberta-base-exp-v4/",
".. /input/xlnet-base-cased-exp-v7/",
]
pretrained_model_metadata = [ 
("tf-roberta-base", True, 56, True), 
("bert-base-uncased", True, 56, False), 
("tf-bert-base-cased", True, 56, False), 
("tf-roberta-base", True, 56, False), 
("xlnet-base-cased", True, 48, True),
]
model_filename_prefix_list = [
"tf-roberta-base_exp_cate_embed",
"bert-base-uncased_exp_split_dense",
"tf-bert-base-cased_exp_split_dense",
"tf-roberta-base_exp_split_dense",
"xlnet-base-cased_exp_cate_embed",
]
MAX_SEQUENCE_LENGTH = 512
max_title_length = 100
vectorizer = CountVectorizer(analyzer = 'word', tokenizer = None,
preprocessor = None, stop_words = None,
max_features = 8000 )<train_model>
for i, p in enumerate(MODEL_PATH_list):
prefix = model_filename_prefix_list[i]
for f in os.listdir(p):
if f != "dataset-metadata.json":
print(p+f)
 assert prefix in f
vectorizer.fit(train_corpus )<train_model>
df_train = pd.read_csv(dataset_folder / 'train.csv')
 df_test = pd.read_csv(dataset_folder / 'test.csv')
 df_sub = pd.read_csv(dataset_folder / 'sample_submission.csv')
 print('Train shape:', df_train.shape)
 print('Test shape:', df_test.shape )
X_train = vectorizer.transform(train_corpus ).toarray()
y_train = train.sentiment.values.reshape(-1,1)
 print(X_train.shape, y_train.shape )<count_missing_values>
def extract_netloc(x):
tokens = x.split(".")
 if len(tokens)> 3:
print(x)
 return ".".join(tokens[:2])
 else:
return tokens[0]
df_train['netloc'] = df_train['host'].apply(lambda x: x.split(".")[0])
 df_test['netloc'] = df_test['host'].apply(lambda x: x.split(".")[0] )
print(len(train_corpus[0]))
print(np.sum(X_train, axis = 1))<split>
def set_all_seeds(rand_seed):
np.random.seed(rand_seed)
 random.seed(rand_seed)
 os.environ['PYTHONHASHSEED'] = str(rand_seed)
 tf.random.set_seed(rand_seed)
 torch.manual_seed(rand_seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
X_train, X_val, y_train, y_val = train_test_split(X_train , y_train.reshape(-1,),
test_size = 0.2,
stratify = y_train.reshape(-1,),
random_state = 42)
 print("Training Data = ", X_train.shape, y_train.shape)
 print("Validation Date = ", X_val.shape, y_val.shape )<load_from_csv>
set_all_seeds(rand_seed )
test = pd.read_csv(".. /input/testData.tsv", delimiter = "\t",
header = 0, quoting = 3)
 print(test.shape)
 test.head()<string_transform>
def _convert_to_transformer_inputs(title, question, answer, tokenizer,
max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1,
str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
def remove_html_special_symbols(x):
html_entities = [ 
("&quot;",
num_test_reviews = test.shape[0]
test_corpus = []
for i in tqdm_notebook(range(num_test_reviews)) :
test_corpus.append(review_to_words(test['review'][i]))
print(len(test))<train_model>
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [
np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)
 ]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
clfrNB = naive_bayes.MultinomialNB()
clfrNB.fit(X_train, y_train.reshape(-1,))
y_pred = clfrNB.predict(X_train)
 cm_train = confusion_matrix(y_train, y_pred)
 y_pred = clfrNB.predict(X_val)
 cm_val = confusion_matrix(y_val, y_pred)
 print("Training Accuracy = ", clfrNB.score(X_train, y_train))
print("Validation Accuracy", clfrNB.score(X_val, y_val))
print(cm_train)
 print(cm_val )<predict_on_test>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def compute_spearmanr(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.mean(rhos )
y_test = clfrNB.predict(X_test)
 print(y_test.shape)
 y_test<load_from_csv>
class SpearmanMonitorCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f" Fold {self.fold+1} Validation Score: {rho_val:.6f}" )
sample = pd.read_csv(".. /input/sampleSubmission.csv", header = 0)
 sample.head()<save_to_csv>
def create_model(pretrained_model_name):
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 pretrained_model = model_class.from_pretrained(BERT_PATH +
f"{pretrained_model_name}")
 q_embedding = pretrained_model(q_id,
attention_mask=q_mask,
token_type_ids=q_atn)[0]
a_embedding = pretrained_model(a_id,
attention_mask=a_mask,
token_type_ids=a_atn)[0]
q = q_embedding[:, 0, :]
a = a_embedding[:, 0, :]
q = tf.keras.layers.Dense(256, activation='relu' )(q)
 a = tf.keras.layers.Dense(256, activation='relu' )(a)
 q = tf.keras.layers.Dense(21, activation='sigmoid' )(q)
 a = tf.keras.layers.Dense(9, activation='sigmoid' )(a)
 x = tf.keras.layers.Concatenate()([q, a])
 model = tf.keras.models.Model(inputs=[
q_id,
q_mask,
q_atn,
a_id,
a_mask,
a_atn,
],
outputs=x)
 return model, pretrained_model
sub = pd.DataFrame()
sub['id'] = sample['id'].values
sub['sentiment'] = y_test
sub.to_csv("naiveBayesPred.csv", index = False )<train_model>
def create_model_cate_embed(pretrained_model_name, embed_info):
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 pretrained_model = model_class.from_pretrained(BERT_PATH +
f"{pretrained_model_name}")
 if "xlnet" in pretrained_model_name:
q_embedding = pretrained_model(q_id,
attention_mask=q_mask,
token_type_ids=q_atn)[0]
a_embedding = pretrained_model(a_id,
attention_mask=a_mask,
token_type_ids=a_atn)[0]
else:
q_embedding, q_pooler_output = pretrained_model(q_id,
attention_mask=q_mask,
token_type_ids=q_atn)
 a_embedding, a_pooler_output = pretrained_model(a_id,
attention_mask=a_mask,
token_type_ids=a_atn)
 q = q_embedding[:, 0, :]
a = a_embedding[:, 0, :]
host_input = tf.keras.Input(shape=(1,), name="host_input")
 netloc_input = tf.keras.Input(shape=(1,), name="netloc_input")
 cate_input = tf.keras.Input(shape=(1,), name="category_input")
 host_embed_info = embed_info["host"]
host_embed = tf.keras.layers.Embedding(input_dim=host_embed_info[0],
output_dim=host_embed_info[1],
input_length=(1,))(host_input)
 netloc_embed_info = embed_info["netloc"]
netloc_embed = tf.keras.layers.Embedding(input_dim=netloc_embed_info[0],
output_dim=netloc_embed_info[1],
input_length=(1,))(netloc_input)
 cate_embed_info = embed_info["category"]
cate_embed = tf.keras.layers.Embedding(input_dim=cate_embed_info[0],
output_dim=cate_embed_info[1],
input_length=(1,))(cate_input)
 host_embed = tf.keras.layers.Reshape(
 target_shape=(host_embed_info[1],))(host_embed)
 netloc_embed = tf.keras.layers.Reshape(
 target_shape=(netloc_embed_info[1],))(netloc_embed)
 cate_embed = tf.keras.layers.Reshape(
 target_shape=(cate_embed_info[1],))(cate_embed)
 embed_concat = tf.keras.layers.Concatenate()(
 [host_embed, netloc_embed, cate_embed])
 embed_concat = tf.keras.layers.Dense(128, activation='relu' )(embed_concat)
 q_concat = tf.keras.layers.Concatenate()([q, embed_concat])
 q_concat = tf.keras.layers.Dense(256, activation='relu' )(q_concat)
 a_concat = tf.keras.layers.Concatenate()([a, embed_concat])
 a_concat = tf.keras.layers.Dense(256, activation='relu' )(a_concat)
 q_concat = tf.keras.layers.Dense(21, activation='sigmoid' )(q_concat)
 a_concat = tf.keras.layers.Dense(9, activation='sigmoid' )(a_concat)
 x = tf.keras.layers.Concatenate()([q_concat, a_concat])
 model = tf.keras.models.Model(inputs=[
q_id, q_mask, q_atn, a_id, a_mask, a_atn, host_input, netloc_input,
cate_input
],
outputs=x)
 return model, pretrained_model
clfrRMC = RandomForestClassifier(n_estimators= 500, criterion = "gini",
max_features = "auto", max_depth = 8,
min_samples_split = 2, random_state = 42)
 clfrRMC.fit(X_train, y_train.reshape(-1,))
y_pred = clfrRMC.predict(X_train)
 cm_train = confusion_matrix(y_train, y_pred)
 y_pred = clfrRMC.predict(X_val)
 cm_val = confusion_matrix(y_val, y_pred)
 print("Training Accuracy = ", clfrRMC.score(X_train, y_train))
print("Validation Accuracy = ", clfrRMC.score(X_val, y_val))
print(cm_train)
 print(cm_val )<predict_on_test>
set_all_seeds(rand_seed)
 gkf = GroupKFold(n_splits=n_splits ).split(X=df_train.question_body,
groups=df_train.question_body)
 gkf = list(gkf)
 len(gkf )
y_test = clfrRMC.predict(X_test)
 print(y_test.shape)
 print(y_test )<save_to_csv>
outputs = compute_output_arrays(df_train, output_categories )
sub = pd.DataFrame()
sub['id'] = sample['id']
sub['sentiment'] = y_test
sub.to_csv("randomForestPred.csv", index = False)
 print(sub.shape)
 sub.head()<train_model>
def optimize_ranks(preds, unique_labels):
new_preds = np.zeros(preds.shape)
 for i in range(preds.shape[1]):
interpolate_bins = np.digitize(preds[:, i],
bins=unique_labels,
right=False)
 if len(np.unique(interpolate_bins)) == 1:
new_preds[:, i] = preds[:, i]
else:
new_preds[:, i] = unique_labels[interpolate_bins]
return new_preds
clfrLR = LogisticRegression(penalty = 'l2', C = 0.005)
 clfrLR.fit(X_train, y_train)
 y_pred = clfrLR.predict(X_train)
 cm_train = confusion_matrix(y_train, y_pred)
 y_pred = clfrLR.predict(X_val)
 cm_val = confusion_matrix(y_val, y_pred)
 print("Training Accuracy = ", clfrLR.score(X_train, y_train))
print("Validation Accuracy = ", clfrLR.score(X_val, y_val))
print(cm_train)
 print(cm_val )<predict_on_test>
y_labels = df_train[output_categories].copy()
y_labels = y_labels.values.flatten()
unique_labels = np.array(sorted(np.unique(y_labels)))
 unique_labels
y_test = clfrLR.predict(X_test)
 print(y_test.shape)
 print(y_test )<save_to_csv>
denominator = 60
q = np.arange(0, 101, 100 / denominator)
 exp_labels = np.percentile(unique_labels, q)
 exp_labels
sub = pd.DataFrame()
sub['id'] = sample['id']
sub['sentiment'] = y_test
sub.to_csv("LogisticPrediction.csv", index = False)
 print(sub.shape)
 sub.head()<import_modules>
infer_start_time = time.time()
all_test_preds = []
all_val_preds = []
all_val_scores = []
all_magic_val_scores = []
gc.collect()
for k, MODEL_PATH in enumerate(MODEL_PATH_list):
pretrained_model_name, is_tf, infer_batch_size, cate_embed_mode = pretrained_model_metadata[k]
model_filename_prefix = model_filename_prefix_list[k]
print(f"Generating prediction from fine-tuned {pretrained_model_name}...... ")
 preprocessing_start = time.time()
if is_tf:
model_class = TFAutoModel
tokenizer_class = AutoTokenizer
else:
model_class = AutoModel
tokenizer_class = AutoTokenizer
tokenizer = tokenizer_class.from_pretrained(BERT_PATH +
f"{pretrained_model_name}")
 inputs = compute_input_arrays(df_train, input_categories, tokenizer,
MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories,
tokenizer, MAX_SEQUENCE_LENGTH)
 print(f"Time spent on preprocessing: {(time.time() -preprocessing_start)/60:,.2f} minutes")
 test_preds = []
val_preds = []
val_scores = []
magic_val_scores = []
for i,(train_idx, valid_idx)in enumerate(gkf):
set_all_seeds(rand_seed)
 print(f"Using {pretrained_model_name} Fold {i+1} {pretrained_model_name} for inference...... ")
 fold_start = time.time()
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_outputs = outputs[valid_idx]
if cate_embed_mode:
embed_info = {}
category_features = {}
def extract_category_ids(train, val, test, c, info):
le = LabelEncoder()
le.fit(train[c])
 val[c] = val[c].map(lambda s: '<unknown>'
if s not in le.classes_ else s)
 test[c] = test[c].map(lambda s: '<unknown>'
if s not in le.classes_ else s)
 le_classes = le.classes_.tolist()
bisect.insort_left(le_classes, '<unknown>')
 le.classes_ = le_classes
train[c + "_label"] = le.transform(train[c])
 val[c + "_label"] = le.transform(val[c])
 test[c + "_label"] = le.transform(test[c])
 no_of_unique_cat = train[c + "_label"].nunique()
embedding_size = min(np.ceil(( no_of_unique_cat)/ 2), 50)
 embedding_size = int(embedding_size)
 vocab_size = no_of_unique_cat + 1
info[c] =(vocab_size, embedding_size)
 print(f"Extracted(vocab_size, embedding_size)for {c}:({vocab_size}, {embedding_size})")
 return val[c + "_label"], test[c + "_label"]
host_val, host_test = extract_category_ids(df_train.iloc[train_idx, :].copy() ,
df_train.iloc[valid_idx, :].copy() ,
df_test.copy() ,
"host",
embed_info)
 netloc_val, netloc_test = extract_category_ids(df_train.iloc[train_idx, :].copy() ,
df_train.iloc[valid_idx, :].copy() ,
df_test.copy() ,
"netloc",
embed_info)
 cate_val, cate_test = extract_category_ids(df_train.iloc[train_idx, :].copy() ,
df_train.iloc[valid_idx, :].copy() ,
df_test.copy() ,
"category",
embed_info)
 valid_inputs.append(host_val)
 valid_inputs.append(netloc_val)
 valid_inputs.append(cate_val)
 submit_inputs = [np.copy(x)for x in test_inputs]
submit_inputs.append(host_test)
 submit_inputs.append(netloc_test)
 submit_inputs.append(cate_test)
 print("Cleaning session...")
 K.clear_session()
print("Loading pretrained model and weights...")
 if cate_embed_mode:
model, pretrained_model = create_model_cate_embed(pretrained_model_name, embed_info)
 else:
model, pretrained_model = create_model(pretrained_model_name)
 model_filename = f"{model_filename_prefix}_fold{i+1}.h5"
model.load_weights(MODEL_PATH + model_filename)
 fold_val_preds = model.predict(valid_inputs,
batch_size=infer_batch_size)
 rho_val = compute_spearmanr(valid_outputs, fold_val_preds)
 print(f"Fold {i+1} Validation Score: {rho_val:.6f}")
 val_preds.append(fold_val_preds)
 val_scores.append(rho_val)
 val_magic_preds = optimize_ranks(fold_val_preds, exp_labels)
 magic_rho_val = compute_spearmanr(valid_outputs, val_magic_preds)
 print(f"Fold {i+1} Magic Validation Score: {magic_rho_val:.6f}")
 magic_val_scores.append(magic_rho_val)
 if cate_embed_mode:
test_preds.append(model.predict(submit_inputs,
batch_size=infer_batch_size))
else:
test_preds.append(model.predict(test_inputs,
batch_size=infer_batch_size))
print(f"Time spent on fold {i+1} inference: {(time.time() -fold_start)/60:,.2f} minutes")
 del model, pretrained_model, valid_inputs, valid_outputs, fold_val_preds, rho_val
gc.collect()
all_test_preds.append(test_preds)
 all_val_preds.append(val_preds)
 all_val_scores.append(val_scores)
 all_magic_val_scores.append(magic_val_scores)
 del tokenizer, inputs, test_inputs
gc.collect()
print(f"Time spent on ensemble inference: {(time.time() -infer_start_time)/60:,.2f} minutes" )
sys.path.append('.. /input/timm-pytorch-image-models/pytorch-image-models-master')
 warnings.filterwarnings("ignore" )<define_variables>
print(f"Mean Validation Score: {np.mean(all_val_scores):.6f}")
 print(f"Mean Magic Validation Score: {np.mean(all_magic_val_scores):.6f}" )
DATA_PATH = '.. /input/cassava-leaf-disease-classification/'
TRAIN_DIR = DATA_PATH + 'train_images/'
TEST_DIR = DATA_PATH + 'test_images/'
MODEL_PATH = '.. /input/cassavanet-baseline-models/'
N_TTA = 8
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
MODEL_LIST = [0,1,2,3,4,5]
IMG_MEAN = [0.485, 0.456, 0.406]
IMG_STD = [0.229, 0.224, 0.225]<set_options>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
SEED = 1234
seed_everything(SEED)
 DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<choose_model_class>
def val_ensemble_preds(all_val_preds, weights):
oof_preds = np.zeros(outputs.shape)
 for i, model_preds in enumerate(all_val_preds):
for j,(train_idx, valid_idx)in enumerate(gkf):
tmp = np.vstack(model_preds[j])
 oof_preds[valid_idx] += tmp * weights[i]
oof_preds /= np.sum(weights)
 return oof_preds
class CassavaNet(nn.Module):
def __init__(self, model_name=None, pretrained=False):
super().__init__()
self.model_name = model_name
if model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':
self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=pretrained)
 else:
self.model = timm.create_model(model_name, pretrained=pretrained)
 if 'efficientnet' in model_name:
self.n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(self.n_features, N_CLASSES)
 elif model_name == 'vit_large_patch16_384' or model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':
self.n_features = self.model.head.in_features
self.model.head = nn.Linear(self.n_features, N_CLASSES)
 elif 'resnext' in model_name:
self.n_features = self.model.fc.in_features
self.model.fc = nn.Linear(self.n_features, N_CLASSES)
 def forward(self, x):
return self.model(x)
 def freeze(self):
for param in self.model.parameters() :
param.requires_grad = False
if 'efficientnet' in self.model_name:
for param in self.model.classifier.parameters() :
param.requires_grad = True
elif self.model_name == 'vit_large_patch16_384' or 'deit_base_patch16_224':
for param in self.model.head.parameters() :
param.requires_grad = True
elif 'resnext' in self.model_name:
for param in self.model.fc.parameters() :
param.requires_grad = True
def unfreeze(self):
for param in self.model.parameters() :
param.requires_grad = True<categorify>
with open('ensemble-models-v4-v7.pickle', 'wb')as handle:
pickle.dump(all_val_preds, handle, protocol=pickle.HIGHEST_PROTOCOL )
class GetData(Dataset):
def __init__(self, Dir, FNames, labels,Type):
self.dir = Dir
self.fnames = FNames
self.lbs = labels
self.type = Type
def __len__(self):
return len(self.fnames)
 def __getitem__(self, index):
x = imread(os.path.join(self.dir, self.fnames[index]))
if "train" in self.type:
aug_data = train_transforms(image = x)
 return aug_data['image'], self.lbs[index]
elif "valid" in self.type:
aug_data = valid_transforms(image = x)
 return aug_data['image'], self.lbs[index]
elif "tr-tst" in self.type:
return x, self.lbs[index]
elif "test" in self.type:
return x, self.fnames[index]<set_options>
weights = [1.0, 1.0, 1.0, 1.0, 1.0]
oof_preds = val_ensemble_preds(all_val_preds, weights)
 magic_preds = optimize_ranks(oof_preds, exp_labels)
 blend_score = compute_spearmanr(outputs, magic_preds)
 print(weights, blend_score)
 weights = [2.0, 1.0, 1.0, 1.0, 2.0]
oof_preds = val_ensemble_preds(all_val_preds, weights)
 magic_preds = optimize_ranks(oof_preds, exp_labels)
 blend_score = compute_spearmanr(outputs, magic_preds)
 print(weights, blend_score)
 weights = [2.0, 1.0, 1.0, 1.0, 1.5]
oof_preds = val_ensemble_preds(all_val_preds, weights)
 magic_preds = optimize_ranks(oof_preds, exp_labels)
 blend_score = compute_spearmanr(outputs, magic_preds)
 print(weights, blend_score)
 weights = [1.5, 1.0, 1.0, 1.0, 1.5]
oof_preds = val_ensemble_preds(all_val_preds, weights)
 magic_preds = optimize_ranks(oof_preds, exp_labels)
 blend_score = compute_spearmanr(outputs, magic_preds)
 print(weights, blend_score )
Aug_Norm = A.Normalize(mean=IMG_MEAN, std=IMG_STD, max_pixel_value=255.0, p=1.0)
 test_aug = Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.ShiftScaleRotate(p = 1.0),
A.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=1.0),
A.RandomCrop(height= HEIGHT, width = WIDTH,always_apply=True, p=1.0),
Aug_Norm,
ToTensorV2(p=1.0)
 ], p=1.)<load_pretrained>
submit_preds = [np.average(x, axis=0)for x in all_test_preds]
models = []
count = 0
for model_fpath in os.listdir(MODEL_PATH):
if count in MODEL_LIST:
print("Model Loaded:",model_fpath)
 model_name_split = model_fpath.split('_f')[0]
model = CassavaNet(model_name_split,pretrained = False)
 info = torch.load(MODEL_PATH + model_fpath,map_location = torch.device(DEVICE))
model.load_state_dict(info)
 models.append(model)
 count+=1<prepare_output>
submit_preds = np.average(submit_preds, weights = [2.0, 1.0, 1.0, 1.0, 1.5],
axis=0 )
submission = pd.DataFrame()
list_files = os.listdir(TEST_DIR)
 submission['image_id'] = pd.Series(list_files)
 submission['label'] = 0
submission.head()<categorify>
submit_preds = optimize_ranks(submit_preds, exp_labels )
<save_to_csv><EOS> 
df_sub.iloc[:, 1:] = submit_preds
df_sub.to_csv('submission.csv', index=False )