Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<import_modules>
tqdm.pandas()
from tensorflow.keras.applications import ResNet50, Xception, DenseNet201
from tensorflow.keras import Sequential
from keras.layers import Dense, GlobalAveragePooling2D<choose_model_class>
NROWS = None
N_EMBED = 300
MAX_FEATURES = 95000
MAX_LEN = 72
N_SPLITS = 5
N_BATCH = 1021
N_EPOCH = 8
SEED = 1031
def build_xception_network() :
with strategy.scope() :
network = Xception(input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),weights='imagenet',include_top=False)
 network.trainable = True
model = Sequential([network, GlobalAveragePooling2D() , Dense(len(CLASSES), activation = 'softmax', dtype = 'float32')])
 opt = 'Adam'
model.compile(optimizer = opt, loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
def build_resnet_network() :
with strategy.scope() :
network = ResNet50(input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),weights='imagenet',include_top=False)
 network.trainable = True
model = Sequential([network, GlobalAveragePooling2D() , Dense(len(CLASSES), activation = 'softmax', dtype = 'float32')])
 opt = 'Adam'
model.compile(optimizer = opt, loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
def build_densenet_network() :
with strategy.scope() :
network = DenseNet201(input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),weights='imagenet',include_top=False)
 network.trainable = True
model = Sequential([network, GlobalAveragePooling2D() , Dense(len(CLASSES), activation = 'softmax', dtype = 'float32')])
 opt = 'Adam'
model.compile(optimizer = opt, loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model<save_to_csv>
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
print('Computing predictions.... ')
 test_images_ds = test_dataset.map(lambda image, idnum: image)
 test_ids_ds = test_dataset.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 probabilities = 0.25*model_resnet.predict(test_images_ds)+0.25*model_xception.predict(test_images_ds)+0.5*model_densenet.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print('Compiling submission file...')
 np.savetxt('submission.csv',np.rec.fromarrays([test_ids, predictions]),fmt=['%s', '%d'],delimiter=',',header='id,label',comments='' )<compute_train_metric>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
cmdataset = get_validation_dataset(load_dataset(VALIDATION_FILENAMES, labeled = True, ordered = True))
images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities = model_xception.predict(images_ds)
 cm_predictions = np.argmax(cm_probabilities, axis=-1)
 cmat_1 = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score_1 = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision_1 = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall_1 = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 cmat_1 =(cmat_1.T / cmat_1.sum(axis=1)).T
display_confusion_matrix(cmat_1, score_1, precision_1, recall_1)
 cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities = model_resnet.predict(images_ds)
 cm_predictions = np.argmax(cm_probabilities, axis=-1)
 cmat_2 = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score_2 = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision_2 = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall_2 = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 cmat_2 =(cmat_2.T / cmat_2.sum(axis=1)).T
display_confusion_matrix(cmat_2, score_2, precision_2, recall_2)
 cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities = model_densenet.predict(images_ds)
 cm_predictions = np.argmax(cm_probabilities, axis=-1)
 cmat_3 = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score_3 = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision_3 = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall_3 = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 cmat_3 =(cmat_3.T / cmat_3.sum(axis=1)).T
display_confusion_matrix(cmat_3, score_3, precision_3, recall_3)
 cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities = 0.25*model_resnet.predict(images_ds)+0.25*model_xception.predict(images_ds)+0.5*model_densenet.predict(images_ds)
 cm_predictions = np.argmax(cm_probabilities, axis=-1)
 cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 cmat =(cmat.T / cmat.sum(axis=1)).T
display_confusion_matrix(cmat, score, precision, recall)
<import_modules>
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
print('Tensorflow version ' + tf.__version__)
 !pip install -q efficientnet 
<set_options>
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv", nrows=NROWS)
 test_df = pd.read_csv(".. /input/test.csv", nrows=NROWS)
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 for df in [train_df, test_df]:
df["question_text"] = df["question_text"].apply(lambda x: x.lower())
 df["question_text"] = df["question_text"].apply(lambda x: clean_text(x))
df["question_text"] = df["question_text"].apply(lambda x: clean_numbers(x))
df["question_text"] = df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=MAX_LEN)
 test_X = pad_sequences(test_X, maxlen=MAX_LEN)
 train_y = train_df['target'].values
test_id = test_df["qid"].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index, test_id
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
N_EMBED = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, N_EMBED))
for word, i in word_index.items() :
if i >= MAX_FEATURES: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, N_EMBED))
for word, i in word_index.items() :
if i >= MAX_FEATURES: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
N_EMBED = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index)+ 1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, N_EMBED))
for word, i in word_index.items() :
if i >= MAX_FEATURES: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
AUTO = tf.data.experimental.AUTOTUNE
IMAGE_SIZE = [224, 224]
EPOCHS = 30
FOLDS = 3
SEED = 777
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<set_options>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
MIXED_PRECISION = False
XLA_ACCELERATE = False
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )<categorify>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset,do_aug=True):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<normalization>
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 61
self.embedding = nn.Embedding(MAX_FEATURES, N_EMBED)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.11)
 self.lstm = nn.LSTM(N_EMBED, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, MAX_LEN)
 self.gru_attention = Attention(hidden_size * 2, MAX_LEN)
 self.linear = nn.Linear(hidden_size * 8, 17)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.11)
 self.out = nn.Linear(17, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<create_dataframe>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_dataset_all = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES})['TRAINING_FILENAMES']), labeled = True)
 test_dataset_all = load_dataset(list(pd.DataFrame({'TEST_FILENAMES': TEST_FILENAMES})['TEST_FILENAMES']), labeled = True, ordered=True)
 train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES_ONLY': TRAINING_FILENAMES_ONLY})['TRAINING_FILENAMES_ONLY']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'VAL_FILENAMES_ONLY': VAL_FILENAMES_ONLY})['VAL_FILENAMES_ONLY']), labeled=True, ordered=True)
 train_all = get_training_dataset(train_dataset_all)
 test = get_test_dataset(test_dataset_all)
 test_images_ds = test.map(lambda image, idnum: image)
 train = get_training_dataset(train_dataset)
 val = get_validation_dataset(val_dataset)
 def Xception_model() :
with strategy.scope() :
rnet = Xception(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def VGG16_model() :
with strategy.scope() :
rnet = VGG16(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def DenseNet201_model() :
with strategy.scope() :
rnet = DenseNet201(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def InceptionV3_model() :
with strategy.scope() :
rnet = InceptionV3(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def EfficientNetB7_model() :
with strategy.scope() :
rnet = EfficientNetB7(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def ResNet152V2_model() :
with strategy.scope() :
rnet = ResNet152V2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def MobileNetV2_model() :
with strategy.scope() :
rnet = MobileNetV2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def InceptionResNetV2_model() :
with strategy.scope() :
rnet = InceptionResNetV2(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
models = {'Xception' : Xception_model,
'VGG16' : VGG16_model,
'DenseNet201' : DenseNet201_model,
'InceptionV3' : InceptionV3_model,
'EfficientNetB7' : EfficientNetB7_model,
'MobileNetV2' : MobileNetV2_model,
}
historys = {}
predictions = {}
predictions_val = {}
predictions_prob = {}
MODELS_NUMBER = 6<prepare_output>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
df = pd.DataFrame(predictions)
 pred = []
for i in range(0, 7382):
if df.loc[i,:].unique().shape[0] < MODELS_NUMBER :
pred.append(stats.mode(df.loc[i,:].values)[0][0])
 else :
pred.append(df.loc[i,'Xception'])
 df = pd.DataFrame(predictions_val)
 pred_val = []
for i in range(0, 3712):
if df.loc[i,:].unique().shape[0] < MODELS_NUMBER :
pred_val.append(stats.mode(df.loc[i,:].values)[0][0])
 else :
pred_val.append(df.loc[i,'Xception'] )<create_dataframe>
start_time = time.time()
train_X, test_X, train_y, word_index, test_id = load_and_prec()
glove_embeddings = load_glove(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 paragram_embeddings = load_para(word_index)
 total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
embedding_matrix = glove_embeddings * 0.33 + fasttext_embeddings * 0.34 + paragram_embeddings * 0.33
print(np.shape(embedding_matrix))
del glove_embeddings, fasttext_embeddings, paragram_embeddings
gc.collect()
avg_prob = predictions_prob['Xception'] + predictions_prob['VGG16'] + predictions_prob['DenseNet201'] + predictions_prob['InceptionV3'] + predictions_prob['EfficientNetB7']
pred_avg = pd.DataFrame(np.argmax(avg_prob, axis=-1))<save_to_csv>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED ).split(train_X, train_y))
test_ids_ds = test.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission_vote.csv', np.rec.fromarrays([test_ids, pred]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, np.argmax(avg_prob, axis=-1)]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=N_BATCH, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr, step_size=step_size, mode='exp_range', gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=N_BATCH, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=N_BATCH, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(N_EPOCH):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * N_BATCH:(i+1)* N_BATCH] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, N_EPOCH, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * N_BATCH:(i+1)* N_BATCH] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )
!pip install -U -t /kaggle/working/ git+https://github.com/Kaggle/learntools.git
binder.bind(globals())
 step_1.check()<import_modules>
output = pd.DataFrame(data={"qid": test_id, "prediction": test_preds > 0.33})
 output.to_csv("submission.csv", index=False )