Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_pretrained>
import numpy as np
import pandas as pd
from pathlib import Path
from fastai.text.all import *
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
datadir = Path.cwd().parent / 'input' / 'nlp-getting-started'
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform, n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
<normalization>
df_train = pd.read_csv(datadir/'train.csv')
 df_test = pd.read_csv(datadir/'test.csv')
 df_submission = pd.read_csv(datadir/'sample_submission.csv' )
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
dls_lm = TextDataLoaders.from_df(df_train, text_col='text', is_lm=True )
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y 
<define_search_model>
dls_lm.show_batch(max_n=2 )
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.0002
lr_d = 0.0002
beta1 = 0.5
epochs = 10000
netG = Generator(100, 32, 3 ).to(device)
 netD = Discriminator(3, 48 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005)
 nz = 100
fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.7
fake_label = 0.0
batch_size = train_loader.batch_size
step = 0
for epoch in tqdm(range(epochs)) :
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end -start)> 25000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 200 == 0:
show_generated_img()
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 100
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, 100, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<import_modules>
learn_lm = language_model_learner(
 dls_lm, AWD_LSTM, drop_mult=0.3,
metrics=[accuracy, Perplexity() ] ).to_fp16()
begin = time.time()
sns.set_style('white')
 warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter(action='ignore', category=UserWarning)
 warnings.simplefilter(action='ignore', category=RuntimeWarning )<define_variables>
learn_lm.fit_one_cycle(1, 2e-2 )
KERNEL_START_TIME = time.time()<define_variables>
learn_lm.unfreeze()
learn_lm.fit_one_cycle(5,2e-3 )
DATA_PATH = '.. /input/all-dogs/all-dogs/'
IMAGE_FOLDER = '.. /input/all-dogs'
IMAGES = [DATA_PATH + p for p in os.listdir(DATA_PATH)]
ANNOTATION_PATH = '.. /input/annotation/Annotation'
print('Number of doge images :', len(IMAGES))<set_options>
learn_lm.save_encoder('finetuned' )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True 
<define_variables>
dls = TextDataLoaders.from_df(df_train, text_col='text', label_col='target' )
seed = 2019
seed_everything(seed )<normalization>
dls.show_batch(max_n=2 )
class Rescale() :
def __init__(self):
self.a = 2
self.b = -1
def __call__(self, tensor):
return tensor.mul(self.a ).add(self.b)
 def __repr__(self):
return self.__class__.__name__ + '(x{}, +{})'.format(self.a, self.b )<normalization>
learn = text_classifier_learner(dls, AWD_LSTM, drop_mult=0.5, metrics=accuracy )
def get_transforms(size):
base_transforms = transforms.Compose([transforms.Resize(size)])
 additional_transforms = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomChoice([transforms.CenterCrop(size), transforms.RandomCrop(size)]),
transforms.RandomApply([transforms.ColorJitter(brightness=0.2, contrast=(0.9, 1.2), saturation=0.3, hue=0.01)], p=0.5),
transforms.ToTensor() ,
Rescale() ,
])
 return base_transforms, additional_transforms<categorify>
learn.load_encoder('finetuned' )
class DogeDataset(Dataset):
def __init__(self, folder, base_transforms, additional_transforms):
self.folder = folder
self.classes = [dirname[10:] for dirname in os.listdir(ANNOTATION_PATH)]
self.base_transforms = base_transforms
self.additional_transforms = additional_transforms
self.imgs, self.labels = self.load_subfolders_images(folder)
 le = LabelEncoder().fit(self.classes)
 self.y = torch.from_numpy(le.transform(self.labels)).long()
self.classes = le.inverse_transform(range(len(self.classes)))
 def __getitem__(self, index):
return self.additional_transforms(self.imgs[index]), self.y[index]
def __len__(self):
return len(self.imgs)
 @staticmethod
def is_valid_file(x):
img_extensions =('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 return has_file_allowed_extension(x, img_extensions)
 @staticmethod
def get_bbox(o):
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 return xmin, ymin, xmax, ymax
@staticmethod
def larger_bbox(bbox, ximg, yimg, a=10):
xmin, ymin, xmax, ymax = bbox
xmin = max(xmin - a, 0)
 ymin = max(ymin - a, 0)
 xmax = min(xmax + a, ximg)
 ymax = min(ymax + a, yimg)
 return(xmin, ymin, xmax, ymax)
 def load_subfolders_images(self, root):
imgs = []
paths = []
labels = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames):
path = os.path.join(root, fname)
 if self.is_valid_file(path):
paths.append(path)
 for path in paths:
img = default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir(ANNOTATION_PATH)if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join(ANNOTATION_PATH, annotation_dirname, annotation_basename)
 label = annotation_dirname[10:]
tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bbox = self.get_bbox(o)
 bbox = self.larger_bbox(bbox, img.size[0], img.size[1])
 object_img = self.base_transforms(img.crop(bbox))
imgs.append(object_img)
 labels.append(label)
 return imgs, labels<create_dataframe>
lr_min,lr_steep = learn.lr_find()
print(f"Minimum/10: {lr_min:.2e}, steepest point: {lr_steep:.2e}" )
%%time
base_transforms, additional_transforms = get_transforms(64)
 dataset = DogeDataset(DATA_PATH, base_transforms, additional_transforms )<define_variables>
learn.fine_tune(5, 6e-3 )
nb_classes = len(dataset.classes)
 print(f'Number of classes : {nb_classes}' )<define_variables>
text_list = list(df_test['text'].values )
nb_dogs = len(dataset)
 print(f'Number of dogs : {nb_dogs}' )<define_variables>
%%capture
prediction = [learn.predict(i)[0] for i in text_list]
WEIGHTS_PATH = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'<init_hyperparams>
df_submission['target'] = prediction
df_submission['target'] = df_submission['target'].astype(int )
<load_pretrained><EOS> 
df_submission.to_csv('submission.csv', index=False )