Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<save_to_csv>
from transformers import BertTokenizer, BertModel
from transformers import XLNetModel, XLNetTokenizer
from sklearn import preprocessing
from sklearn.model_selection import GroupKFold
import re
import pandas as pd
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<install_modules>
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv(".. /input/google-quest-challenge/test.csv")
 train = pd.read_csv(".. /input/google-quest-challenge/train.csv" )
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 1.6 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers > /dev/null
!pip install pandarallel > /dev/null<set_options>
test_v3 = pd.read_csv(".. /input/google-quest-qa-add-on/test_v3.csv")
 train_v3 = pd.read_csv(".. /input/google-quest-qa-add-on/train_v3.csv" )
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=2, progress_bar=True )<define_variables>
l_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
SEED = 42
MAX_LENGTH = 224
BACKBONE_PATH = '.. /input/multitpu-inference'
CHECKPOINT_PATH = '.. /input/multitpu-inference/checkpoint-xlm-roberta.bin'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<string_transform>
columns = ['question_title', 'question_cleaned', 'answer_cleaned', 'category',
'question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()<categorify>
def txt_re(content):
res_list = []
for txt in content:
if pd.isnull(txt):
res_list.append('code')
 else:
txt = txt.strip()
txt = re.sub('https?.*$', '', txt)
 txt = re.sub('https?.*\s', '', txt)
 txt = re.sub('
+', '', txt)
 txt = re.sub('\r+', '', txt)
 txt = re.sub('&gt;', '>', txt)
 txt = re.sub('&lt;', '<', txt)
 txt = re.sub('&amp;', '&', txt)
 txt = re.sub('&quot;', '"', txt)
 res_list.append(txt)
 return res_list
class DatasetRetriever(Dataset):
def __init__(self, df):
self.comment_texts = df['comment_text'].values
self.ids = df['id'].values
self.tokenizer = XLMRobertaTokenizer.from_pretrained(BACKBONE_PATH)
 def get_tokens(self, text):
encoded = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LENGTH,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def __len__(self):
return self.ids.shape[0]
def __getitem__(self, idx):
text = self.comment_texts[idx]
tokens, attention_mask = self.get_tokens(text)
 tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 return self.ids[idx], tokens, attention_mask<load_from_csv>
content = []
labels = []
for col in zip(train_v3[columns].values.tolist()):
cont = col[0][:3]
cont = txt_re(cont)
 label = col[0][4:]
content.append(cont)
 labels.append(label )
%%time
df_test = pd.read_csv(f'.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 df_test['comment_text'] = df_test.parallel_apply(lambda x: clean_text(x['content'], x['lang']), axis=1)
 df_test = df_test.drop(columns=['content'])
 df_test.head()<feature_engineering>
test_content = []
for col in zip(test_v3[['question_title', 'question_body', 'answer']].values.tolist()):
cont = col[0]
cont = txt_re(cont)
 test_content.append(cont )
test_dataset = DatasetRetriever(df_test )<normalization>
tokenizer = BertTokenizer.from_pretrained('.. /input/bert-base-uncased-huggingface' )
class ToxicSimpleNNModel(nn.Module):
def __init__(self, backbone):
super(ToxicSimpleNNModel, self ).__init__()
self.backbone = backbone
self.dropout = nn.Dropout(0.3)
 self.linear = nn.Linear(
 in_features=self.backbone.pooler.dense.out_features*2,
out_features=2, 
)
 def forward(self, input_ids, attention_masks):
bs, seq_length = input_ids.shape
seq_x, _ = self.backbone(input_ids=input_ids, attention_mask=attention_masks)
 apool = torch.mean(seq_x, 1)
 mpool, _ = torch.max(seq_x, 1)
 x = torch.cat(( apool, mpool), 1)
 x = self.dropout(x)
 return self.linear(x)
 backbone = XLMRobertaModel(XLMRobertaConfig.from_pretrained(BACKBONE_PATH))<load_pretrained>
q_inputs = []
q_input_masks = []
q_segment_masks = []
a_inputs = []
a_input_masks = []
a_segment_masks = []
for cont in content:
q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(q_input['input_ids'])== len(q_input['token_type_ids'])== len(q_input['attention_mask'])
 q_inputs.append(q_input['input_ids'])
 q_segment_masks.append(q_input['token_type_ids'])
 q_input_masks.append(q_input['attention_mask'])
 a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(a_input['input_ids'])== len(a_input['token_type_ids'])== len(a_input['attention_mask'])
 a_inputs.append(a_input['input_ids'])
 a_segment_masks.append(a_input['token_type_ids'])
 a_input_masks.append(a_input['attention_mask'] )
net = ToxicSimpleNNModel(backbone=backbone)
 checkpoint = torch.load(CHECKPOINT_PATH, map_location=torch.device('cpu'))
net.load_state_dict(checkpoint);
checkpoint = None
del checkpoint<choose_model_class>
q_title_len = []
q_body_len = []
a_len = []
for col in zip(train_v3[['question_title', 'question_body', 'answer']].values.tolist()):
q_title_len.append(len(tokenizer.tokenize(col[0][0])))
 q_body_len.append(len(tokenizer.tokenize(col[0][1])))
 a_len.append(len(tokenizer.tokenize(col[0][2])) )
def _mp_fn(rank, flags):
device = xm.xla_device()
model = net.to(device)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=1 
)
 fitter = MultiCoreTPUPredictor(model=model, device=device)
 fitter.run_inference(test_loader )<load_from_csv>
t_q_inputs = []
t_q_input_masks = []
t_q_segment_masks = []
t_a_inputs = []
t_a_input_masks = []
t_a_segment_masks = []
for cont in test_content:
t_q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_q_input['input_ids'])== len(t_q_input['token_type_ids'])== len(t_q_input['attention_mask'])
 t_q_inputs.append(t_q_input['input_ids'])
 t_q_segment_masks.append(t_q_input['token_type_ids'])
 t_q_input_masks.append(t_q_input['attention_mask'])
 t_a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_a_input['input_ids'])== len(t_a_input['token_type_ids'])== len(t_a_input['attention_mask'])
 t_a_inputs.append(t_a_input['input_ids'])
 t_a_segment_masks.append(t_a_input['token_type_ids'])
 t_a_input_masks.append(t_a_input['attention_mask'] )
ensemble = pd.read_csv('.. /input/multitpu-inference/submission-ensemble.csv', index_col='id')
 ensemble['toxic'].hist(bins=100);<categorify>
t_q_title_len = []
t_q_body_len = []
t_a_len = []
for col in zip(test_v3[['question_title', 'question_body', 'answer']].values.tolist()):
t_q_title_len.append(len(tokenizer.tokenize(col[0][0])))
 t_q_body_len.append(len(tokenizer.tokenize(col[0][1])))
 t_a_len.append(len(tokenizer.tokenize(col[0][2])) )
def scale_min_max_submission(submission):
min_, max_ = submission['toxic'].min() , submission['toxic'].max()
submission['toxic'] =(submission['toxic'] - min_)/(max_ - min_)
 return submission<save_to_csv>
train_text = [[q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask]\
for q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask in \
zip(q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks)]
submission.to_csv('submission.csv' )<set_options>
train_num = [[q_t, q_b, a] for q_t, q_b, a in zip(q_title_len, q_body_len, a_len)]
%matplotlib inline
def get_features(path):
try:
st = []
img = Image.open(path)
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
im_stats_ = ImageStat.Stat(img)
 st += im_stats_.sum
st += im_stats_.mean
st += im_stats_.rms
st += im_stats_.var
st += im_stats_.stddev
img = np.array(img)
 m, s = cv2.meanStdDev(img)
 st += list(m)
 st += list(s)
 st += [cv2.Laplacian(img, cv2.CV_64F ).var() ]
st += [cv2.Sobel(img,cv2.CV_64F,1,0,ksize=5 ).var() ]
st += [cv2.Sobel(img,cv2.CV_64F,0,1,ksize=5 ).var() ]
img = img[:,:,:3]
st += [scipy.stats.kurtosis(img[:,:,0].ravel())]
st += [scipy.stats.kurtosis(img[:,:,1].ravel())]
st += [scipy.stats.kurtosis(img[:,:,2].ravel())]
st += [scipy.stats.skew(img[:,:,0].ravel())]
st += [scipy.stats.skew(img[:,:,1].ravel())]
st += [scipy.stats.skew(img[:,:,2].ravel())]
except:
print(path)
 return [path, st]
def normalize_img(paths):
imf_d = {}
p = Pool(cpu_count())
 ret = p.map(get_features, paths)
 for i in range(len(ret)) :
imf_d[ret[i][0]] = ret[i][1]
ret = []
fdata = [imf_d[f] for f in paths]
return pd.DataFrame(fdata)
 dog_bytes = pd.DataFrame(glob.glob('.. /input/all-dogs/all-dogs/**'), columns=['Path'])
 dog_bytes = pd.concat(( dog_bytes, normalize_img(dog_bytes.Path.values)) , axis=1 )<feature_engineering>
scaler = StandardScaler()
train_num = scaler.fit_transform(train_num ).tolist()
labels = pd.DataFrame(glob.glob('.. /input/annotation/Annotation/**/**'), columns=['Path'])
 labels['Labels'] = labels['Path'].map(lambda x: x.split('/')[4].split('-')[1])
 labels['FileName'] = labels['Path'].map(lambda x: x.split('/')[-1] + '.jpg')
 labels = {f:l for f,l in labels[['FileName', 'Labels']].values}
dog_bytes['FileName'] = dog_bytes['Path'].map(lambda x: x.split('/')[-1])
 dog_bytes['Labels'] = dog_bytes['FileName'].map(labels)
 dog_bytes.head()<find_best_params>
test_text = [[t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask] \
for t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask in \
zip(t_q_inputs, t_q_input_masks, t_q_segment_masks, t_a_inputs, t_a_input_masks, t_a_segment_masks)]
dog_bytes['Group'] = cluster.KMeans(n_clusters=400, random_state=4 ).fit_predict(dog_bytes[list(range(30)) ])
 dog_bytes['Group'].value_counts() [:5]<categorify>
test_num = [[q_t, q_b, a] for q_t, q_b, a in zip(t_q_title_len, t_q_body_len, t_a_len)]
def sim_img(path):
img = Image.open(path ).convert('RGB')
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
return img
samples = []
for i in range(400):
g = dog_bytes[dog_bytes['Group'] == i]
p = g['Path'].values
for i in range(0,len(p)-2, 2):
samples.append([p[i],p[i+1]])
 if len(samples)> 11_000: break
for i in range(0,len(samples)-1):
samples.append([samples[i][0],samples[i+1][0]])
 if len(samples)> 11_000: break
print(len(samples))<import_modules>
test_num = scaler.transform(test_num ).tolist()
import gzip, pickle
import os
import numpy as np
import pandas as pd
import random
import shutil
import numpy as np
import os
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
from scipy import linalg
import pathlib
import urllib
import warnings
from PIL import Image
from tqdm import tqdm_notebook as tqdm<define_variables>
data = {'txt': train_text, 'num': train_num, 'label': labels}
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<categorify>
t_data = {'txt': test_text, 'num': test_num}
class dog_dataset(Dataset):
def __init__(self, train_y, train_X, zeros, device):
self.train_y = torch.Tensor(train_y ).to(device)
 self.train_X = torch.Tensor(train_X ).to(device)
 self.zeros = torch.Tensor(zeros ).to(device)
 def __len__(self):
return len(train_y)
 def __getitem__(self, idx):
return self.train_y[idx], self.train_X[idx], self.zeros[idx]<prepare_x_and_y>
torch.save(data, './t7/data.t7' )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 bs = 256
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
data_set = dog_dataset(train_y, train_X, zeros, device)
 data_loader = DataLoader(data_set, bs )<normalization>
torch.save(t_data, './t7/test.t7' )
class Discriminator(nn.Module):
def __init__(self):
super(Discriminator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 self.conv1 = nn.Conv2d(1, 1,(2, 1), bias=False)
 def forward(self, imgs, imgnames):
x = self.fc1(imgnames)
 x = torch.sigmoid(x)
 x = torch.cat(( imgs, x), dim=1 ).view(-1, 1, 2, 12288)
 x = self.conv1(x)
 return x.view(-1, 12288 )<choose_model_class>
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertTokenizer, BertModel
from scipy.stats import spearmanr
lr = 0.005
epochs = 360
netD = Discriminator().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr)
 criteria = nn.BCELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.0],
[1.0]]]] ).to(device))
for param in netD.conv1.parameters() :
param.requires_grad = False<compute_test_metric>
class Model(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/bert-base-uncased-huggingface')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output = nn.Linear(768 * 2 + 3, 30)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks, q_a_len):
q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze(1 ).squeeze(1)
 a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze(1 ).squeeze(1)
 x = torch.cat(( q_x, a_x, q_a_len), -1)
 x = torch.sigmoid(self.output(x))
return x
for k in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netD.zero_grad()
y_pred = netD(Zeros, X)
 loss = criteria(y_pred, y)
 loss.backward()
optimizerD.step()
if(k + 1)% 2 == 0:
print(f"Epoch: {k+1}/{epochs} | Loss: {loss}" )<normalization>
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset, TensorDataset
from sklearn.model_selection import train_test_split
from transformers import AdamW, get_linear_schedule_with_warmup
class Generator(nn.Module):
def __init__(self):
super(Generator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 def forward(self, imgnames):
x = self.fc1(imgnames)
 return x, imgnames.view(-1, 10000 )<data_type_conversions>
data = torch.load('./t7/data.t7' )
def show_image() :
plt.figure(figsize=(15,3))
for j in range(5):
with torch.no_grad() :
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 inp = torch.Tensor(xx.reshape(( -1,10000)) ).to(device)
 img = netG(inp)[0].reshape(( -1,64,64,3)).to("cpu" ).clone().detach().numpy()
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
def show_d_images(imgs, title):
with torch.no_grad() :
imgs = imgs.reshape(( -1,64,64,3))
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = imgs[j].detach().cpu().numpy()
img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.title(title)
 plt.imshow(img)
 plt.show()<choose_model_class>
test_data = torch.load('./t7/test.t7' )
lr = 0.01
beta1 = 0.5
netG = Generator().to(device)
 optimizerG = optim.Adam(netG.parameters() , lr=lr)
 criterion = nn.MSELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.],
[1.]]]] ).to(device))
for param in netD.parameters() :
param.requires_grad = False<compute_train_metric>
test_set = TensorDataset(torch.LongTensor(np.array(test_data['txt'])) ,
torch.FloatTensor(np.array(test_data['num'])) )
epochs = 50
for epoch in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netG.zero_grad()
fake, seed = netG(X)
 y_pred = netD(Zeros, seed)
 errG = criterion(fake, y_pred)
 errG.backward()
optimizerG.step()
if(epoch+1)% 5 == 0:
print(f"Epoch: {epoch+1}: G_Loss: {errG}")
 show_d_images(fake, "Generator output" )<set_options>
test_loader = DataLoader(
 test_set,
batch_size=1,
shuffle=False )
class DogGenerator() :
index = 0
t = [
transforms.RandomCrop(( 48, 48), padding=None, pad_if_needed=True, fill=0, padding_mode='symmetric'),
transforms.Scale(( 64,64))
]
tfms = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.ColorJitter(brightness=(1,1.3), contrast=(1,1.3), saturation=0, hue=0)
 ])
 def getDog(self,seed):
xx = torch.Tensor(np.zeros(( 10000)) ).to(device)
 xx[self.index] = 0.999999
xx[np.random.randint(10000)] = 0.000001
img = netG(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3)).detach().cpu().numpy() * 255
self.index =(self.index+1)%10000
return self.tfms(Image.fromarray(img.astype('uint8')) )<import_modules>
criterion = nn.BCELoss()
import os
import numpy as np
import pandas as pd
import random
from tqdm import tqdm
import shutil<choose_model_class>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance<define_variables>
gkf = GroupKFold(n_splits=5 ).split(X=train.question_body, groups=train.question_body)
 final_predicts = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 1, 2]:
model = Model()
model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=2e-5)
 train_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[train_idx]),\
torch.FloatTensor(np.array(data['num'])[train_idx]), \
torch.FloatTensor(np.array(data['label'])[train_idx]))
dev_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[valid_idx]),\
torch.FloatTensor(np.array(data['num'])[valid_idx]), \
torch.FloatTensor(np.array(data['label'])[valid_idx]))
train_loader = DataLoader(
 train_set,
batch_size=6,
shuffle=True, drop_last=True)
 dev_loader = DataLoader(
 dev_set,
batch_size=min(len(dev_set), 1),
shuffle=False)
 for epoch_idx in range(3):
for batch_idx,(model_in, model_num_in, labels)in enumerate(train_loader):
model.train()
optimizer.zero_grad()
model_in = model_in.cuda()
model_num_in = model_num_in.cuda()
labels = labels.cuda()
model_in = Variable(model_in, requires_grad=False)
 model_num_in = Variable(model_num_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], \
model_in[:, 4], model_in[:, 5], model_num_in)
 labels = Variable(labels, requires_grad=False)
 labels = labels.transpose(0, 1)
 scores = scores.transpose(0, 1)
 losses = [criterion(score, label)for score, label in zip(scores, labels)]
loss = sum(losses)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
print("train epoch: {} loss: {}".format(epoch_idx, loss.item() /30))
torch.cuda.empty_cache()
model.eval()
pre_list = []
tru_list = []
with torch.no_grad() :
for model_in, model_num_in, labels in dev_loader:
model_in = model_in.cuda()
model_num_in = model_num_in.cuda()
model_in = Variable(model_in, requires_grad=False)
 model_num_in = Variable(model_num_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], \
model_in[:, 3], model_in[:, 4], model_in[:, 5], model_num_in)
 pre_list.append(scores)
 tru_list.append(labels)
 dev_predicts = [pre.squeeze(0 ).cpu().numpy().tolist() for pre in pre_list]
truthes = [t.squeeze(0 ).numpy().tolist() for t in tru_list]
dev_rho = compute_spearmanr_ignore_nan(dev_predicts, truthes)
 print("dev score: ", dev_rho)
 torch.cuda.empty_cache()
model.eval()
test_predicts = []
with torch.no_grad() :
for model_in, model_num_in in test_loader:
model_in = model_in.cuda()
model_num_in = model_num_in.cuda()
model_in = Variable(model_in, requires_grad=False)
 model_num_in = Variable(model_num_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], \
model_in[:, 3], model_in[:, 4], model_in[:, 5], model_num_in)
 test_predicts.append(scores.reshape(scores.shape[-1]))
final_predicts.append(test_predicts)
 torch.cuda.empty_cache()
dogs = os.listdir('.. /input/generative-dog-images/all-dogs/all-dogs/')
 print("Total images = ", len(dogs))
sub = random.sample(dogs, 10000)
 print("Sample images = ", len(sub))<normalization>
pres = np.average(final_predicts, axis=0 )
path = '/kaggle/working/images/'
def resize() :
for item in tqdm(os.listdir(path)) :
im = Image.open(path+item)
 imResize = im.resize(( 64,64), Image.ANTIALIAS)
 imResize.save(path+item, 'PNG', quality=100)
 resize()<compute_train_metric>
test_output = [[p.item() for p in pre] for pre in pres]
user_images_unzipped_path = 'images'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public = calculate_kid_given_paths(images_path, 'Inception', public_path)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ", fid_value_public /(distance_public + fid_epsilon))<set_options>
output_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
!rm -r images<define_variables>
output_values = np.transpose(test_output ).tolist()
ComputeLB = True
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<import_modules>
output_dict = {k: v for k, v in zip(output_cols, output_values)}
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam<choose_model_class>
output_dict['qa_id'] = sample_submission['qa_id'].values.tolist()
def define_discriminator(in_shape=(28,28,1)) :
	model = Sequential()
	
	model.add(Conv2D(128,(3,3), strides=(2,2), padding='same', input_shape=in_shape))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Conv2D(128,(3,3), strides=(2,2), padding='same'))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Flatten())
 	model.add(Dropout(0.4))
	model.add(Dense(1, activation='sigmoid'))
	
	opt = Adam(lr=0.0002, beta_1=0.5)
 	model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
 	return model<categorify>
output = pd.DataFrame.from_dict(output_dict )
def define_discriminator(in_shape=(28,28,1), n_classes=10):
	
	in_label = Input(shape=(1,))
	
	li = Embedding(n_classes, 50 )(in_label)
 	
	n_nodes = in_shape[0] * in_shape[1]
	li = Dense(n_nodes )(li)
 	
	li = Reshape(( in_shape[0], in_shape[1], 1))(li)
 	
	in_image = Input(shape=in_shape)
 	
	merge = Concatenate()([in_image, li])
 	
	fe = Conv2D(128,(3,3), strides=(2,2), padding='same' )(merge)
 	fe = LeakyReLU(alpha=0.2 )(fe)
 	
	fe = Conv2D(128,(3,3), strides=(2,2), padding='same' )(fe)
 	fe = LeakyReLU(alpha=0.2 )(fe)
 	
	fe = Flatten()(fe)
 	
	fe = Dropout(0.4 )(fe)
 	
	out_layer = Dense(1, activation='sigmoid' )(fe)
 	
	model = Model([in_image, in_label], out_layer)
 	
	opt = Adam(lr=0.0002, beta_1=0.5)
 	model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
 	return model<define_search_model>
order = ['qa_id', 'question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
dog = Input(( 12288,))
dogName = Input(( 10000,))
x = Dense(12288, activation='sigmoid' )(dogName)
 x = Reshape(( 2,12288,1))(concatenate([dog,x]))
x = Conv2D(1,(2,1),use_bias=False,name='conv' )(x)
 discriminated = Flatten()(x)
 discriminator = Model([dog,dogName], discriminated)
 discriminator.get_layer('conv' ).trainable = False
discriminator.get_layer('conv' ).set_weights([np.array([[[[-1.0 ]]],[[[1.0]]]])])
 discriminator.compile(optimizer='adam', loss='binary_crossentropy')
<train_model>
output = output[order]
<choose_model_class><EOS> 
output.to_csv('submission.csv', index=False )