Kaggle competition: Natural Language Processing with Disaster Tweets
submission_df.label = submission_df.filename.map(fname2pred )<feature_engineering>
test_set = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 train_set = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv" )
submission_df['label'] = np.clip(submission_df['label'], 0.01, 0.99 )<save_to_csv>
import seaborn as sns
from matplotlib import pyplot as plt
submission_df.to_csv("submission.csv",index=False )<install_modules>
print("train set")
 print(train_set.count())
 print("test set")
 print(test_set.count() )
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
!pip install torchtext==0.2.3<define_variables>
!pip install emoji
MODEL_NAME = 'Densenet201'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = resnet50
num_workers = 8<feature_engineering>
def strip_emoji(text: str)-> str:

 return re.sub(emoji.get_emoji_regexp() , r"", text )
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)] 
<feature_engineering>
train_set["text"] = train_set.text.str.strip().str.replace("
", "")
 train_set["text"] = train_set.text.str.strip().str.replace("\r", "")
 train_set["text"] = train_set.text.apply(strip_emoji)
 train_set["text"] = train_set.text.str.strip().str.replace("
train_set["text"] = train_set.text.str.lower()
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<prepare_x_and_y>
test_set["text"] = test_set.text.str.strip().str.replace("
", "")
 test_set["text"] = test_set.text.str.strip().str.replace("\r", "")
 test_set["text"] = test_set.text.apply(strip_emoji)
 test_set["text"] = test_set.text.str.strip().str.replace("
test_set["text"] = test_set.text.str.lower()
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels )<categorify>
!pip install torch>=1.6.0 transformers==3.3.1
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md<load_pretrained>
import random
from typing import Sequence
from transformers import BertForMaskedLM, BertTokenizer
import torch
image_size = 224
batch_size = 48
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam<init_hyperparams>
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
 text = train_set.text[0]
tokenized_text = tokenizer.tokenize(text)
 print(tokenized_text )
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]<train_model>
index = 4
tokenized_text.insert(index, "[MASK]")
 print("Text Tokens")
 print(tokenized_text)
 tokenized_text_ids = torch.LongTensor(tokenizer.encode(tokenized_text, max_length=512, truncation=True))
print("Text Tokens ids")
 print(tokenized_text_ids )
base_lr = 5e-4
fc_lr = 5e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn.fit(lrs, n_cycle=3, cycle_len=1, cycle_mult=2)
 learn.save('weights' )<define_variables>
model = BertForMaskedLM.from_pretrained("bert-base-uncased")
 model.eval()
with torch.no_grad() :
predictions = model(tokenized_text_ids.unsqueeze(0)) [0]
predicted_word = tokenizer.convert_ids_to_tokens([torch.argmax(predictions[0, index+1] ).item() ])[0]
print(predicted_word )
image_size = 448
batch_size = 24
md = get_data(image_size, batch_size)
 learn.set_data(md )<train_model>
tokenized_text[index] = predicted_word
augmented_text = " ".join(tokenized_text)
 print(augmented_text )
base_lr = 1e-5
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs, n_cycle=6, cycle_len=1)
 learn.save('weights_v2' )<prepare_output>
class WordAugmentation:

 _mask_token: str = "[MASK]"
def __init__(self,
model_path: str="bert-base-uncased",
device: str="cuda"):

 self._tokenizer = BertTokenizer.from_pretrained(model_path)
 self._device = device
self._model = BertForMaskedLM.from_pretrained(model_path ).to(self._device)
 def _predict(self,
inputs_ids: torch.Tensor,
mask_index: int)-> str:

 self._model.eval()
inputs_ids = inputs_ids.to(self._device ).unsqueeze(0)
 with torch.no_grad() :
predictions = self._model(inputs_ids)[0]
return self._tokenizer.convert_ids_to_tokens([torch.argmax(predictions[0, mask_index] ).item() ])[0]
def apply(self,
text: str,
n_word=1,
action='insert')-> str:

 if isinstance(text, str)and text != "":
text = self._tokenizer.tokenize(text)
 masks_index = []
for i in range(n_word):
random_index = random.choice([idx for idx in range(0, len(text)) if idx not in masks_index])
 masks_index.insert(i, random_index)
 if action == "insert":
text.insert(masks_index[i], self._mask_token)
 elif action == "replace":
text[masks_index[i]] = self._mask_token
else:
raise ValueError(f"action {action} not recognized.Please use: insert or replace")
 text_ids = torch.LongTensor(self._tokenizer.encode(text, max_length=512, truncation=True))
predicted_word = self._predict(text_ids, random_index+1)
 if(random_index != 0)and(random_index != len(text)- 1):
previous_word = text[random_index - 1]
next_word = text[random_index + 1]
elif(random_index == 0)and(len(text)> 1):
previous_word = ""
next_word = text[random_index + 1]
elif(random_index == len(text)- 1)and(len(text)> 1):
previous_word = text[random_index - 1]
next_word = ""
else:
previous_word = ""
next_word = ""
if predicted_word in [previous_word, next_word, " "]:
del text[random_index]
else:
text[random_index] = predicted_word
text = " ".join(text)
 else:
text = ""
return text
best_th = 0.38
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1)
 np.save('preds_dn201.npy', preds_t )<save_to_csv>
augmenter = WordAugmentation(device='cuda' )
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('sub_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()<import_modules>
tqdm.tqdm.pandas()
new_sentences_replace = train_set.progress_apply(lambda x: augmenter.apply(x["text"], n_word=max(int(len(x["text"])*0.05), 1), action='replace'), axis=1 )
!pip install lap
Lambda, MaxPooling2D, Reshape 
<load_from_csv>
new_train_set_replace = pd.DataFrame({"text": new_sentences_replace.to_numpy() , "target": train_set.target.to_numpy() } )
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv"
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit<train_model>
augmented_train_set = pd.concat([train_set[["text", "target"]], new_train_set_replace], ignore_index=True )
if isfile(P2SIZE):
print("P2SIZE exists.")
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 else:
p2size = {}
for p in tqdm(join):
size = pil_image.open(expand_path(p)).size
p2size[p] = size<compute_test_metric>
augmented_train_set.drop_duplicates(subset="text", inplace=True )
def match(h1, h2):
for p1 in h2ps[h1]:
for p2 in h2ps[h2]:
i1 = pil_image.open(expand_path(p1))
i2 = pil_image.open(expand_path(p2))
if i1.mode != i2.mode or i1.size != i2.size: return False
a1 = np.array(i1)
 a1 = a1 - a1.mean()
a1 = a1 / sqrt(( a1 ** 2 ).mean())
 a2 = np.array(i2)
 a2 = a2 - a2.mean()
a2 = a2 / sqrt(( a2 ** 2 ).mean())
 a =(( a1 - a2)** 2 ).mean()
if a > 0.1: return False
return True
if isfile(P2H):
print("P2H exists.")
 with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 else:
p2h = {}
for p in tqdm(join):
img = pil_image.open(expand_path(p))
h = phash(img)
 p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 hs = list(h2ps.keys())
 h2h = {}
for i, h1 in enumerate(tqdm(hs)) :
for h2 in hs[:i]:
if h1 - h2 <= 6 and match(h1, h2):
s1 = str(h1)
 s2 = str(h2)
 if s1 < s2: s1, s2 = s2, s1
h2h[s1] = s2
for p, h in p2h.items() :
h = str(h)
 if h in h2h: h = h2h[h]
p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p )<categorify>
!pip install fastai==2.0.16
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5]<set_options>
accuracy, Perplexity, F1Score
p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.05<normalization>
train_target_0, validate_target_0 = np.split(augmented_train_set[augmented_train_set.target == 0].sample(frac=1,
random_state=1),
[int (.85 * len(augmented_train_set[augmented_train_set.target == 0])) ])
 train_target_1, validate_target_1 = np.split(augmented_train_set[augmented_train_set.target == 1].sample(frac=1,
random_state=1),
[int (.85 * len(augmented_train_set[augmented_train_set.target == 1])) ])
 train = pd.concat([train_target_0, train_target_1])
 validate = pd.concat([validate_target_0, validate_target_1])
 train["is_valid"] = 0
validate["is_valid"] = 1
train = pd.concat([train, validate])
 seed=42
data_lm = TextDataLoaders.from_df(train,
path="fastai_files/",
is_lm=True,
valid_col="is_valid",
text_col="text",
seed=seed,
bs=32)
 data_class = TextDataLoaders.from_df(train,
path="fastai_files/",
is_lm=False,
valid_col="is_valid",
label_col="target",
text_col="text",
seed=seed,
bs=32)
 learn = language_model_learner(data_lm, arch=AWD_LSTM, pretrained=True, drop_mult=0.5, metrics=[accuracy, Perplexity() ])
 learn.fit_one_cycle(1, 1e-2)
 learn.unfreeze()
learn.fit_one_cycle(10, 1e-3)
 learn.save_encoder('processed_ft_enc')
 learn = text_classifier_learner(data_class, arch=AWD_LSTM, pretrained=True, drop_mult=0.5, metrics=[accuracy, F1Score() ])
 learn.load_encoder('processed_ft_enc')
 learn.fit_one_cycle(1, 1e-2)
 learn.freeze_to(-2)
 learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))
learn.freeze_to(-3)
 learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))
learn.unfreeze()
learn.fit_one_cycle(3, slice(1e-3/(2.6**4),1e-3))
def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):

 rotation = np.deg2rad(rotation)
 shear = np.deg2rad(shear)
 rotation_matrix = np.array(
 [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])
 shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])
 zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])
 return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))<prepare_x_and_y>
test_set["target"] = test_set.text.apply(lambda x : learn.predict(x)[0])
 test_set[["id", "target"]].to_csv("submission.csv", index=False )