Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<categorify>
from transformers import BertTokenizer, BertModel
from transformers import XLNetModel, XLNetTokenizer
from sklearn import preprocessing
from sklearn.model_selection import GroupKFold
import re
import pandas as pd
def define_generator(latent_dim, n_classes=120):
	
	in_label = Input(shape=(1,))
	
	li = Embedding(n_classes, 50 )(in_label)
 	
	n_nodes = 7 * 7
	li = Dense(n_nodes )(li)
 	
	li = Reshape(( 7, 7, 1))(li)
 	
	in_lat = Input(shape=(latent_dim,))
	
	n_nodes = 128 * 7 * 7
	gen = Dense(n_nodes )(in_lat)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	gen = Reshape(( 7, 7, 128))(gen)
 	
	merge = Concatenate()([gen, li])
 	
	gen = Conv2DTranspose(128,(4,4), strides=(2,2), padding='same' )(merge)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	
	gen = Conv2DTranspose(128,(4,4), strides=(2,2), padding='same' )(gen)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	
	out_layer = Conv2D(1,(7,7), activation='tanh', padding='same' )(gen)
 	
	model = Model([in_lat, in_label], out_layer)
 	return model<choose_model_class>
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv('.. /input/google-quest-qa-add-on/test_v3.csv')
 train = pd.read_csv(".. /input/google-quest-qa-add-on/train_v3.csv" )
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
<train_model>
l_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 5.
 for k in range(50):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = 1, batch_size=256, callbacks=[annealer], verbose=0)
 if(k<10)|(k%5==4):
print('Epoch',(k+1)*10,'/500 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5
if k<10:
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()<categorify>
columns = ['question_title', 'question_cleaned', 'answer_cleaned', 'category',
'question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.999
xx[np.random.randint(10000)] = 0.001
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))<choose_model_class>
train[train.isnull().T.any() ]
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2<load_pretrained>
def txt_re(content):
res_list = []
for txt in content:
if pd.isnull(txt):
res_list.append('code')
 else:
txt = txt.strip()
txt = re.sub('https?.*$', '', txt)
 txt = re.sub('https?.*\s', '', txt)
 txt = re.sub('
+', '', txt)
 txt = re.sub('\r+', '', txt)
 res_list.append(txt)
 return res_list
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<define_variables>
content = []
labels = []
for col in zip(train[columns].values.tolist()):
cont = col[0][:4]
cont = txt_re(cont)
 label = col[0][4:]
content.append(cont)
 labels.append(label )
ComputeLB = True
DogsOnly = True
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
x = np.random.choice(np.arange(20579),10000)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]
h = img.size[1]
sz = np.min(( w,h))
a=0; b=0
if w<h: b =(h-sz)//2
else: a =(w-sz)//2
img = img.crop(( 0+a, 0+b, sz+a, sz+b))
img = img.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 if idxIn%1000==0: print(idxIn)
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<import_modules>
test_content = []
for col in zip(test[['question_title', 'question_body', 'answer', 'category']].values.tolist()):
cont = col[0][:4]
cont = txt_re(cont)
 test_content.append(cont )
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate, UpSampling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam<train_model>
tokenizer = BertTokenizer.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-tokenizer' )
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 10, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/30 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1<drop_column>
q_inputs = []
q_input_masks = []
q_segment_masks = []
a_inputs = []
a_input_masks = []
a_segment_masks = []
for cont in content:
q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(q_input['input_ids'])== len(q_input['token_type_ids'])== len(q_input['attention_mask'])
 q_inputs.append(q_input['input_ids'])
 q_segment_masks.append(q_input['token_type_ids'])
 q_input_masks.append(q_input['attention_mask'])
 a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(a_input['input_ids'])== len(a_input['token_type_ids'])== len(a_input['attention_mask'])
 a_inputs.append(a_input['input_ids'])
 a_segment_masks.append(a_input['token_type_ids'])
 a_input_masks.append(a_input['attention_mask'] )
del train_X, train_y, imagesIn<define_search_model>
t_q_inputs = []
t_q_input_masks = []
t_q_segment_masks = []
t_a_inputs = []
t_a_input_masks = []
t_a_segment_masks = []
for cont in test_content:
t_q_input = tokenizer.encode_plus(cont[0], text_pair=cont[1], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_q_input['input_ids'])== len(t_q_input['token_type_ids'])== len(t_q_input['attention_mask'])
 t_q_inputs.append(t_q_input['input_ids'])
 t_q_segment_masks.append(t_q_input['token_type_ids'])
 t_q_input_masks.append(t_q_input['attention_mask'])
 t_a_input = tokenizer.encode_plus(cont[0], text_pair=cont[2], add_special_tokens=True,max_length=512, pad_to_max_length='right')
 assert len(t_a_input['input_ids'])== len(t_a_input['token_type_ids'])== len(t_a_input['attention_mask'])
 t_a_inputs.append(t_a_input['input_ids'])
 t_a_segment_masks.append(t_a_input['token_type_ids'])
 t_a_input_masks.append(t_a_input['attention_mask'] )
BadMemory = True
if BadMemory:
seed = Input(( 10000,))
x = Dense(2048, activation='elu' )(seed)
 x = Reshape(( 8,8,32))(x)
 x = Conv2D(128,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(64,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(32,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(3,(3, 3), activation='linear', padding='same' )(x)
 generated = Flatten()(x)
 else:
seed = Input(( 10000,))
generated = Dense(12288, activation='linear' )(seed)
 generator = Model(seed, [generated,Reshape(( 10000,))(seed)])
 generator.summary()<choose_model_class>
text = [[q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask]\
for q_input, q_input_mask, q_segment_mask, a_input, a_input_mask, a_segment_mask in \
zip(q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks)]
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 gan.summary()<train_model>
data = {'txt': text, 'label': labels}
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
ep = 1; it = 9
if BadMemory: lr = 0.005
else: lr = 5.
 for k in range(it):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = ep, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1),'/'+str(it)+' - loss =',h.history['loss'][-1])
 plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
if BadMemory:
ep *= 2
if ep>=32: lr = 0.001
if ep>256: ep = 256
else:
if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5<categorify>
t_text = [[t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask] \
for t_q_input, t_q_input_mask, t_q_segment_mask, t_a_input, t_a_input_mask, t_a_segment_mask in \
zip(t_q_inputs, t_q_input_masks, t_q_segment_masks, t_a_inputs, t_a_input_masks, t_a_segment_masks)]
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.70
xx[np.random.randint(10000)] = 0.30
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))<choose_model_class>
t_data = {'txt': t_text}
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2<load_pretrained>
torch.save(data, './t7/data.t7' )
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<feature_engineering>
torch.save(t_data, './t7/test.t7' )
def get_features(path):
try:
st = []
img = Image.open(path)
 im_stats_ = ImageStat.Stat(img)
 st += im_stats_.sum
st += im_stats_.mean
st += im_stats_.rms
st += im_stats_.var
st += im_stats_.stddev
except:
print(path)
 return [path, st]
def normalize_img(paths):
imf_d = {}
p = Pool(cpu_count())
 ret = p.map(get_features, paths)
 for i in range(len(ret)) :
imf_d[ret[i][0]] = ret[i][1]
ret = []
fdata = [imf_d[f] for f in paths]
return pd.DataFrame(fdata)
 dog_bytes = pd.DataFrame(glob.glob('.. /input/all-dogs/all-dogs/**'), columns=['Path'])
 dog_bytes = pd.concat(( dog_bytes, normalize_img(dog_bytes.Path.values)) , axis=1)
 dog_bytes.head()<predict_on_test>
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertTokenizer, BertModel
from scipy.stats import spearmanr
dog_bytes['Group'] = cluster.KMeans(n_clusters=100, random_state=3, n_jobs=-1 ).fit_predict(dog_bytes[list(range(15)) ])
 dog_bytes['Group'].value_counts() [:5]<import_modules>
class Model(nn.Module):
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained('.. /input/huggingfacetransformermodels/model_classes/bert/bert-base-uncased-pytorch-model')
 self.dropout = nn.Dropout(0.2)
 self.pool = nn.AvgPool2d(( 512, 1))
self.output = nn.Linear(768 * 2, 30)
 def forward(self, q_inputs, q_input_masks, q_segment_masks, a_inputs, a_input_masks, a_segment_masks):
q_outputs = self.bert(q_inputs, attention_mask=q_input_masks, token_type_ids=q_segment_masks)
 q_x = q_outputs[0]
q_x = self.dropout(q_x)
 q_x = q_x.unsqueeze(1)
 q_x = self.pool(q_x)
 q_x = q_x.squeeze()
a_outputs = self.bert(a_inputs, attention_mask=a_input_masks, token_type_ids=a_segment_masks)
 a_x = a_outputs[0]
a_x = self.dropout(a_x)
 a_x = a_x.unsqueeze(1)
 a_x = self.pool(a_x)
 a_x = a_x.squeeze()
x = torch.cat(( q_x, a_x), -1)
 x = torch.sigmoid(self.output(x))
return x
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier<set_options>
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset, TensorDataset
from sklearn.model_selection import train_test_split
from transformers import AdamW, get_linear_schedule_with_warmup
%matplotlib inline
plt.rcParams['figure.figsize'] = [20.0, 7.0]
plt.rcParams.update({'font.size': 22,})
 sns.set_palette('viridis')
 sns.set_style('white')
 sns.set_context('talk', font_scale=0.8 )<load_from_csv>
data = torch.load('./t7/data.t7' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('Train Shape: ', train.shape)
 print('Test Shape: ', test.shape)
 train.head()<normalization>
test_data = torch.load('./t7/test.t7' )
X_train_df = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train_df)
 X_test = scaler.transform(X_test )<compute_train_metric>
test_set = TensorDataset(torch.LongTensor(np.array(test_data['txt'])) )
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr,
X_train,
y_train,
cv=5,
scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, X_train, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )<count_missing_values>
test_loader = DataLoader(
 test_set,
batch_size=1,
shuffle=False )
train.isnull().any().any()<normalization>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )
sel = feature_selection.VarianceThreshold()
train_variance = sel.fit_transform(train)
 train_variance.shape<define_variables>
gkf = GroupKFold(n_splits=5 ).split(X=train.question_body, groups=train.question_body)
 final_predicts = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 1, 2]:
model = Model()
model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=2e-5)
 train_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[train_idx]),\
torch.FloatTensor(np.array(data['label'])[train_idx]))
dev_set = TensorDataset(torch.LongTensor(np.array(data['txt'])[valid_idx]),\
torch.FloatTensor(np.array(data['label'])[valid_idx]))
train_loader = DataLoader(
 train_set,
batch_size=6,
shuffle=True, drop_last=True)
 dev_loader = DataLoader(
 dev_set,
batch_size=min(len(dev_set), 1),
shuffle=False)
 for epoch_idx in range(3):
for batch_idx,(model_in, labels)in enumerate(train_loader):
model.train()
optimizer.zero_grad()
model_in = model_in.cuda()
labels = labels.cuda()
model_in = Variable(model_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], model_in[:, 4], model_in[:, 5])
 labels = Variable(labels, requires_grad=False)
 labels = labels.transpose(0, 1)
 scores = scores.transpose(0, 1)
 losses = [criterion(score, label)for score, label in zip(scores, labels)]
loss = sum(losses)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
print("train epoch: {} loss: {}".format(epoch_idx, loss.item() /30))
torch.cuda.empty_cache()
model.eval()
pre_list = []
tru_list = []
with torch.no_grad() :
for model_in, labels in dev_loader:
model_in = model_in.cuda()
model_in = Variable(model_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], model_in[:, 4], model_in[:, 5])
 pre_list.append(scores)
 tru_list.append(labels)
 dev_predicts = [[p.item() for p in pre] for pre in pre_list]
truthes = [t.squeeze(0 ).numpy().tolist() for t in tru_list]
dev_rho = compute_spearmanr_ignore_nan(dev_predicts, truthes)
 print("dev score: ", dev_rho)
 torch.cuda.empty_cache()
model.eval()
test_predicts = []
with torch.no_grad() :
for model_in in test_loader:
model_in = model_in[0].cuda()
model_in = Variable(model_in, requires_grad=False)
 scores = model(model_in[:, 0], model_in[:, 1], model_in[:, 2], model_in[:, 3], model_in[:, 4], model_in[:, 5])
 test_predicts.append(scores)
 final_predicts.append(test_predicts)
 torch.cuda.empty_cache()
to_drop = [column for column in matrix.columns if any(matrix[column] > 0.50)]
print('Columns to drop: ' ,(len(to_drop)) )<train_model>
pres = np.average(final_predicts, axis=0 )
k_best = feature_selection.SelectKBest(score_func=feature_selection.f_classif, k=100)
 fit = k_best.fit(X_train, y_train)
 univariate_features = fit.transform(X_train )<compute_train_metric>
test_output = [[p.item() for p in pre] for pre in pres]
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, univariate_features, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, univariate_features, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )<train_model>
output_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
rfe = feature_selection.RFE(lr, n_features_to_select=100)
 fit = rfe.fit(X_train, y_train)
 recursive_features = fit.transform(X_train )<compute_train_metric>
output_values = np.transpose(test_output ).tolist()
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=10)
 lr_scores = cross_val_score(lr, recursive_features, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, recursive_features, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )<train_model>
output_dict = {k: v for k, v in zip(output_cols, output_values)}
select_model = feature_selection.SelectFromModel(lr)
 fit = select_model.fit(X_train, y_train)
 model_features = fit.transform(X_train )<compute_train_metric>
output_dict['qa_id'] = sample_submission['qa_id'].values.tolist()
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, model_features, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, model_features, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )<create_dataframe>
output = pd.DataFrame.from_dict(output_dict )
pca = PCA(0.90)
 principal_components = pca.fit_transform(X_train)
 principal_df = pd.DataFrame(data = principal_components)
 principal_df.shape<compute_train_metric>
order = ['qa_id', 'question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, principal_df, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, principal_df, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )<create_dataframe>
output = output[order]
pca = PCA(0.75)
 principal_components = pca.fit_transform(X_train)
 principal_df = pd.DataFrame(data = principal_components)
 principal_df.shape<compute_train_metric>
output.to_csv('submission.csv', index=False )
<train_on_grid><EOS> 
test_v3 = pd.read_csv(".. /input/google-quest-qa-add-on/test_v3.csv")
 train_v3 = pd.read_csv(".. /input/google-quest-qa-add-on/train_v3.csv" )