Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<predict_on_test>
!pip install.. /input/sacremoses/sacremoses-master/ > /dev/null
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
!pip install ".. /input/kerasswa/keras-swa-0.1.2" > /dev/null
submission_df_xception = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_xception.to_csv("submission_xception.csv", index=False )<create_dataframe>
sys.path.insert(0, ".. /input/transformers/transformers-master/")
 gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e)
 def save_obj(obj, name):
with open(name + '.pkl', 'wb')as f:
pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
 def load_obj(name):
with open(name + '.pkl', 'rb')as f:
return pickle.load(f)
submission_df = pd.DataFrame({"filename": test_videos})
 submission_df["label"] = 0.51*submission_df_resnext["label"] + 0.5*submission_df_xception["label"]<save_to_csv>
nltk.data.path.append(".. /input/nltk-data/nltk_data")
submission_df.to_csv("submission.csv", index=False )<set_options>
INPUT_PATH=".. /input/"
train = pd.read_csv(INPUT_PATH+'google-quest-challenge/train.csv')
 test = pd.read_csv(INPUT_PATH+'google-quest-challenge/test.csv')
 submission = pd.read_csv(INPUT_PATH+'google-quest-challenge/sample_submission.csv' )
%matplotlib inline 
<define_variables>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = ['question_title','question_body','answer']
package_path = '.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'
sys.path.append(package_path)
<set_options>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
return df
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
train = clean_data(train, input_columns)
 test = clean_data(test, input_columns )
seed_everything(0 )<set_options>
def constructLabeledSentences(data):
sentences=[]
for index, row in data.iteritems() :
sentences.append(TaggedDocument(utils.to_unicode(row ).split() , ['Text' + '_%s' % str(index)]))
return sentences
def textClean(text):
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = text.lower().split()
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 return(text)
 def cleanup(text):
text = textClean(text)
 text= text.translate(str.maketrans("","", string.punctuation))
return text
train_question_body_sentences = constructLabeledSentences(train['question_body'])
 train_question_title_sentences = constructLabeledSentences(train['question_title'])
 train_answer_sentences = constructLabeledSentences(train['answer'])
 test_question_body_sentences = constructLabeledSentences(test['question_body'])
 test_question_title_sentences = constructLabeledSentences(test['question_title'])
 test_answer_sentences = constructLabeledSentences(test['answer'] )
weight_path = 'efficientnet_b0_epoch_15_loss_0.158.pth'
trained_weights_path = os.path.join('.. /input/deepfake-detection-model-weight', weight_path)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 torch.backends.cudnn.benchmark=True<define_variables>
all_sentences = train_question_body_sentences + \
train_answer_sentences + \
test_question_body_sentences + \
test_answer_sentences
Text_INPUT_DIM=128
text_model = Doc2Vec(min_count=1, window=5, vector_size=Text_INPUT_DIM, sample=1e-4, negative=5, workers=4, epochs=5,seed=1)
 text_model.build_vocab(all_sentences)
 text_model.train(all_sentences, total_examples=text_model.corpus_count, epochs=text_model.iter )
test_dir = '.. /input/deepfake-detection-challenge/test_videos'
os.listdir(test_dir)[:5]<normalization>
def normalize_sentence(tokens):
lemmatizer = WordNetLemmatizer()
lemmatized_sentence = []
for word, tag in pos_tag(tokens):
if tag.startswith('NN')or tag.startswith('PRP'):
pos = 'n'
elif tag.startswith('VB'):
pos = 'v'
else:
continue
pos = 'a'
lemmatized_sentence.append(lemmatizer.lemmatize(word, pos ).lower())
 return lemmatized_sentence
class ImageTransform:
def __init__(self, size, mean, std):
self.data_transform = transforms.Compose([
transforms.Resize(( size, size), interpolation=Image.BILINEAR),
transforms.ToTensor() ,
transforms.Normalize(mean, std)
 ])
 def __call__(self, img):
return self.data_transform(img )<categorify>
def normalize_vectorize(df, columns: list):
for col in columns:
print(col)
 df[col+'_norm'] = df[col].apply(lambda x: ' '.join(set(normalize_sentence(word_tokenize(x)))))
 df[col+'_vec'] = df[col].apply(lambda x: text_model.infer_vector([x]))
return df
train = normalize_vectorize(train, input_columns)
 test = normalize_vectorize(test, input_columns)
class DeepfakeDataset(Dataset):
def __init__(self, file_list, device, detector, transform, img_num=20, frame_window=10):
self.file_list = file_list
self.device = device
self.detector = detector
self.transform = transform
self.img_num = img_num
self.frame_window = frame_window
def __len__(self):
return len(self.file_list)
 def __getitem__(self, idx):
mov_path = self.file_list[idx]
img_list = []
try:
all_image = get_img_from_mov(mov_path, self.img_num, self.frame_window)
 except:
return [], mov_path.split('/')[-1]
for image in all_image:
try:
_image = image[np.newaxis, :, :, :]
boxes, probs = self.detector.detect(_image, landmarks=False)
 x = int(boxes[0][0][0])
 y = int(boxes[0][0][1])
 z = int(boxes[0][0][2])
 w = int(boxes[0][0][3])
 image = image[y-15:w+15, x-15:z+15]
image = Image.fromarray(image)
 image = self.transform(image)
 img_list.append(image)
 except:
img_list.append(None)
 img_list = [c for c in img_list if c is not None]
return img_list, mov_path.split('/')[-1]<load_pretrained>
try:
pbe = load_obj(".. /input/questembeddings/precomputed_bert_embeddings")
 train_question_body_dense = pbe['train_question_body_dense']
train_answer_dense = pbe['train_answer_dense']
train_question_title_dense = pbe['train_question_title_dense']
test_question_body_dense = pbe['test_question_body_dense']
test_answer_dense = pbe['test_answer_dense']
test_question_title_dense = pbe['test_question_title_dense']
except:
print("Load failed, build embedding")
 def sigmoid(x):
return 1 /(1 + math.exp(-x))
def chunks(l, n):

 for i in range(0, len(l), n):
yield l[i:i + n]
def fetch_vectors(string_list, batch_size=64):
DEVICE = torch.device("cuda")
 tokenizer = transformers.DistilBertTokenizer.from_pretrained(".. /input/distilbertbaseuncased/")
 model = transformers.DistilBertModel.from_pretrained(".. /input/distilbertbaseuncased/")
 model.to(DEVICE)
 fin_features = []
for data in tqdm_notebook(chunks(string_list, batch_size)) :
tokenized = []
for x in data:
x = " ".join(x.strip().split() [:300])
 tok = tokenizer.encode(x, add_special_tokens=True)
 tokenized.append(tok[:512])
 max_len = 512
padded = np.array([i + [0] *(max_len - len(i)) for i in tokenized], dtype='int64')
 attention_mask = np.where(padded != 0, 1, 0)
 input_ids = torch.tensor(padded ).to(DEVICE)
 attention_mask = torch.tensor(attention_mask ).to(DEVICE)
 with torch.no_grad() :
last_hidden_states = model(input_ids, attention_mask=attention_mask)
 features = last_hidden_states[0][:, 0, :].cpu().numpy()
fin_features.append(features)
 fin_features = np.vstack(fin_features)
 return fin_features
train_question_body_dense = fetch_vectors(train.question_body.values)
 train_answer_dense = fetch_vectors(train.answer.values)
 train_question_title_dense = fetch_vectors(train.question_title.values)
 test_question_body_dense = fetch_vectors(test.question_body.values)
 test_answer_dense = fetch_vectors(test.answer.values)
 test_question_title_dense = fetch_vectors(test.question_title.values)
 precomputed_bert_embeddings = {
'train_question_body_dense': train_question_body_dense,
'train_answer_dense': train_answer_dense,
'train_question_title_dense': train_question_title_dense,
'test_question_body_dense': test_question_body_dense,
'test_answer_dense': test_answer_dense,
'test_question_title_dense': test_question_title_dense,
}
model = EfficientNet.from_name('efficientnet-b0')
 model._fc = nn.Linear(in_features=model._fc.in_features, out_features=1)
 model.load_state_dict(torch.load(trained_weights_path, map_location=torch.device(device)) )<define_variables>
tfidf = TfidfVectorizer(ngram_range=(1, 3))
tsvd = TruncatedSVD(n_components = 128, n_iter=5)
 tfquestion_title = tfidf.fit_transform(train["question_title"].values)
 tfquestion_title_test = tfidf.transform(test["question_title"].values)
 tfquestion_title = tsvd.fit_transform(tfquestion_title)
 tfquestion_title_test = tsvd.transform(tfquestion_title_test)
 tfquestion_body = tfidf.fit_transform(train["question_body"].values)
 tfquestion_body_test = tfidf.transform(test["question_body"].values)
 tfquestion_body = tsvd.fit_transform(tfquestion_body)
 tfquestion_body_test = tsvd.transform(tfquestion_body_test)
 tfanswer = tfidf.fit_transform(train["answer"].values)
 tfanswer_test = tfidf.transform(test["answer"].values)
 tfanswer = tsvd.fit_transform(tfanswer)
 tfanswer_test = tsvd.transform(tfanswer_test )
test_file = [os.path.join(test_dir, path)for path in os.listdir(test_dir)]<predict_on_test>
torch.cuda.empty_cache()
def predict_dfdc(dataset, model):
torch.cuda.empty_cache()
pred_list = []
path_list = []
model = model.to(device)
 model.eval()
with torch.no_grad() :
for i in tqdm(range(len(dataset))):
pred = 0
imgs, mov_path = dataset.__getitem__(i)
 if len(imgs)== 0:
pred_list.append(0.5)
 path_list.append(mov_path)
 continue
for i in range(len(imgs)) :
img = imgs[i]
output = model(img.unsqueeze(0 ).to(device))
pred += torch.sigmoid(output ).item() / len(imgs)
 pred_list.append(pred)
 path_list.append(mov_path)
 torch.cuda.empty_cache()
return path_list, pred_list<create_dataframe>
try:
embeddings_train = load_obj(".. /input/questembeddings/use_embeddings_train")
 embeddings_test = load_obj(".. /input/questembeddings/use_embeddings_test")
 except:
print("Load failed, build embedding")
 try:
module_url = INPUT_PATH+'universalsentenceencoderlarge4/'
embed = hub.load(module_url)
 def UniversalEmbedding(x):
results = embed(tf.squeeze(tf.cast(x, tf.string)))["outputs"]
return keras.backend.concatenate([results])
 except:
module_url = INPUT_PATH+'universalsentenceencoderlarge3/'
embed = hub.Module(module_url)
 def UniversalEmbedding(x):
results = embed(tf.squeeze(tf.cast(x, tf.string)) , signature="default", as_dict=True)["default"]
return keras.backend.concatenate([results])
 embeddings_train = {}
embeddings_test = {}
for text in input_columns:
print(text)
 train_text = train[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
test_text = test[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_train_emb = []
curr_test_emb = []
batch_size = 4
ind = 0
while ind*batch_size < len(train_text):
curr_train_emb.append(embed(train_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
ind = 0
while ind*batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
embeddings_train[text + '_embedding'] = np.vstack(curr_train_emb)
 embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
K.clear_session()
gc.collect()
res = pd.DataFrame({
'filename': path_list,
'label': pred_list,
})
 res.sort_values(by='filename', ascending=True, inplace=True )<save_to_csv>
find = re.compile(r"^[^.]*")
 train['netloc'] = train['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test['netloc'] = test['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 features_lrg = ['category', 'netloc', 'question_user_name','answer_user_name','host']
features_sml = ['category', 'netloc', 'host']
features_train = []
features_test = []
for feature in features_sml:
merged = pd.concat([train[feature], test[feature]] ).to_numpy().reshape(-1, 1 ).squeeze()
encoded_size = len(max(merged, key=len))
print(feature,encoded_size)
 ctr = 0
column = []
for val in train[feature].values:
ctr +=1
output = np.zeros(encoded_size)
 output[:len(val)] = [ord(i)/128.for i in val]
column.append(output)
 features_train.append(column)
 column = []
for val in test[feature].values:
output = np.zeros(encoded_size)
 output[:len(val)] = [ord(i)/128.for i in val]
column.append(output)
 features_test.append(column)
 features_train_lrg = []
features_test_lrg = []
for feature in features_lrg:
merged = pd.concat([train[feature], test[feature]] ).to_numpy().reshape(-1, 1 ).squeeze()
encoded_size = len(max(merged, key=len))
print(feature,encoded_size)
 ctr = 0
column = []
for val in train[feature].values:
ctr +=1
output = np.zeros(encoded_size)
 output[:len(val)] = [ord(i)/128.for i in val]
column.append(output)
 features_train.append(column)
 column = []
for val in test[feature].values:
output = np.zeros(encoded_size)
 output[:len(val)] = [ord(i)/128.for i in val]
column.append(output)
 features_test.append(column)
 l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 abs_dist = lambda x, y: np.abs(x-y ).sum(axis=1)
 sum_dist = lambda x, y:(x+y ).sum(axis=1)
 dist_features_train = np.array([
l2_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
l2_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
l2_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding']),
l2_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.answer_vec.values])) ,
l2_dist(np.array([x for x in train.question_title_vec.values]), np.array([x for x in train.answer_vec.values])) ,
l2_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.question_title_vec.values])) ,
cos_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
cos_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
cos_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding']),
cos_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.answer_vec.values])) ,
cos_dist(np.array([x for x in train.question_title_vec.values]), np.array([x for x in train.answer_vec.values])) ,
cos_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.question_title_vec.values])) ,
abs_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
abs_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
abs_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding']),
abs_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.answer_vec.values])) ,
abs_dist(np.array([x for x in train.question_title_vec.values]), np.array([x for x in train.answer_vec.values])) ,
abs_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.question_title_vec.values])) ,
sum_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
sum_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
sum_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding']),
sum_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.answer_vec.values])) ,
sum_dist(np.array([x for x in train.question_title_vec.values]), np.array([x for x in train.answer_vec.values])) ,
sum_dist(np.array([x for x in train.question_body_vec.values]), np.array([x for x in train.question_title_vec.values])) ,
l2_dist(train_question_body_dense, train_answer_dense),
cos_dist(train_question_body_dense, train_answer_dense),
abs_dist(train_question_body_dense, train_answer_dense),
sum_dist(train_question_body_dense, train_answer_dense),
l2_dist(train_question_body_dense, train_question_title_dense),
cos_dist(train_question_body_dense, train_question_title_dense),
abs_dist(train_question_body_dense, train_question_title_dense),
sum_dist(train_question_body_dense, train_question_title_dense),
l2_dist(train_answer_dense, train_question_title_dense),
cos_dist(train_answer_dense, train_question_title_dense),
abs_dist(train_answer_dense, train_question_title_dense),
sum_dist(train_answer_dense, train_question_title_dense),
] ).T
dist_features_test = np.array([
l2_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
l2_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.answer_vec.values])) ,
l2_dist(np.array([x for x in test.question_title_vec.values]), np.array([x for x in test.answer_vec.values])) ,
l2_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.question_title_vec.values])) ,
cos_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
cos_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.answer_vec.values])) ,
cos_dist(np.array([x for x in test.question_title_vec.values]), np.array([x for x in test.answer_vec.values])) ,
cos_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.question_title_vec.values])) ,
abs_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
abs_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
abs_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
abs_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.answer_vec.values])) ,
abs_dist(np.array([x for x in test.question_title_vec.values]), np.array([x for x in test.answer_vec.values])) ,
abs_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.question_title_vec.values])) ,
sum_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
sum_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
sum_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
sum_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.answer_vec.values])) ,
sum_dist(np.array([x for x in test.question_title_vec.values]), np.array([x for x in test.answer_vec.values])) ,
sum_dist(np.array([x for x in test.question_body_vec.values]), np.array([x for x in test.question_title_vec.values])) ,
l2_dist(test_question_body_dense, test_answer_dense),
cos_dist(test_question_body_dense, test_answer_dense),
abs_dist(test_question_body_dense, test_answer_dense),
sum_dist(test_question_body_dense, test_answer_dense),
l2_dist(test_question_body_dense, test_question_title_dense),
cos_dist(test_question_body_dense, test_question_title_dense),
abs_dist(test_question_body_dense, test_question_title_dense),
sum_dist(test_question_body_dense, test_question_title_dense),
l2_dist(test_answer_dense, test_question_title_dense),
cos_dist(test_answer_dense, test_question_title_dense),
abs_dist(test_answer_dense, test_question_title_dense),
sum_dist(test_answer_dense, test_question_title_dense),
] ).T
res.to_csv('submission.csv', index=False )<import_modules>
possible_features_train = [
[item for k, item in embeddings_train.items() ],
features_train,
features_train_lrg,
[ dist_features_train ],
[ [x for x in train.question_body_vec.values] ],
[ [x for x in train.question_title_vec.values] ],
[ [x for x in train.answer_vec.values] ],
[ train_question_body_dense ],
[ train_answer_dense ],
[ tfquestion_title ],
[ tfquestion_body ],
[ tfanswer ]
]
possible_features_test = [
[item for k, item in embeddings_test.items() ],
features_test,
features_test_lrg,
[ dist_features_test ],
[ [x for x in test.question_body_vec.values] ],
[ [x for x in test.question_title_vec.values] ],
[ [x for x in test.answer_vec.values] ],
[ test_question_body_dense ],
[ test_answer_dense ],
[ tfquestion_title_test ],
[ tfquestion_body_test ],
[ tfanswer_test ]
]
def get_train_test(split=0.8):
total_len = len(possible_features_train)
 r_idx = random.sample(range(total_len), int(total_len * split))
train = [ train_question_title_dense ]
test = [ test_question_title_dense ]
for i in r_idx:
train += possible_features_train[i]
test += possible_features_test[i]
return np.hstack(train),np.hstack(test)
 X_train,X_test = get_train_test()
y_train = train[targets].values
import numpy as np
import pandas as pd
import io
import bson
import matplotlib.pyplot as plt
from skimage.data import imread
from tqdm import tqdm_notebook<load_from_csv>
def bce(t,p):
return binary_crossentropy(t,p)
 def custom_loss(true,pred):
bce = binary_crossentropy(true,pred)
 return bce + logcosh(true,pred)
 def swish(x):
return K.sigmoid(x)* x
def relu1(x):
return keras.activations.relu(x, alpha=0.0, max_value=1., threshold=0.0)
 def create_model1(X_train):
input1 = Input(shape=(X_train.shape[1],))
x = Dense(64, activation='elu',kernel_initializer='lecun_normal' )(input1)
 x = Dense(128, activation='elu',kernel_initializer='lecun_normal' )(x)
 x = Dropout(0.4 )(x)
 output = Dense(len(targets),activation='sigmoid',name='output' )(x)
 model = Model(inputs=input1, outputs=output)
 return model
def create_model2(X_train):
input1 = Input(shape=(X_train.shape[1],))
x = Dense(512, activation='elu',kernel_initializer='lecun_normal' )(input1)
 x = Dense(256, activation='elu',
kernel_initializer='lecun_normal',
)(x)
 output = Dense(len(targets),activation='sigmoid',name='output' )(x)
 model = Model(inputs=input1, outputs=output)
 return model
def create_model3(X_train):
input1 = Input(shape=(X_train.shape[1],))
x = Dense(200, activation='selu',kernel_initializer='lecun_normal' )(input1)
 x = Dense(512, activation='selu',
kernel_initializer='lecun_normal',
)(x)
 x = Dropout(0.4 )(x)
 output = Dense(len(targets),activation='sigmoid',name='output' )(x)
 model = Model(inputs=input1, outputs=output)
 return model
def create_model4(X_train):
input1 = Input(shape=(X_train.shape[1],))
x = Dense(512, activation='elu',
kernel_initializer='lecun_normal',
)(input1)
 x = Dropout(0.2 )(x)
 output = Dense(len(targets),activation='sigmoid',name='output' )(x)
 model = Model(inputs=input1, outputs=output)
 return model
def create_model5(X_train):
input1 = Input(shape=(X_train.shape[1],))
x = Dense(4096, activation='selu',kernel_initializer='lecun_normal' )(input1)
 x = Dense(512, activation='selu',
kernel_initializer='lecun_normal',
kernel_regularizer=keras.regularizers.l2(0.01)
 )(x)
 output = Dense(len(targets),activation='sigmoid',name='output' )(x)
 model = Model(inputs=input1, outputs=output)
 return model
def create_model7(X_train):
input1 = Input(shape=(X_train.shape[1],))
x = Dense(4096, activation='selu',kernel_initializer='lecun_normal' )(input1)
 x = Dense(512, activation='selu',
kernel_initializer='lecun_normal',
kernel_regularizer=keras.regularizers.l2(0.01)
 )(x)
 x = Dropout(0.2 )(x)
 output = Dense(len(targets),activation='sigmoid',name='output' )(x)
 model = Model(inputs=input1, outputs=output)
 return model
def create_model(X_train):
model = random.choice([create_model1, create_model2,create_model3, create_model4,create_model7] )(X_train)
 model.summary()
return model
categories = pd.read_csv('.. /input/category_names.csv', index_col='category_id' )<create_dataframe>
print(gc.collect() )
df_dict = {
'category': prod_category,
'num_imgs': prod_num_imgs
}
df = pd.DataFrame(df_dict, index=prod_id)
 del df_dict<count_values>
def pearson_metric(y_true, y_pred):
y_true = K.clip(y_true, K.epsilon() , 1)
 y_pred = K.clip(y_pred, K.epsilon() , 1)
 y_true -= K.mean(y_true)
 y_pred -= K.mean(y_pred)
 y_true = K.l2_normalize(y_true, axis=-1)
 y_pred = K.l2_normalize(y_pred, axis=-1)
 pearson_correlation = K.sum(y_true * y_pred, axis=-1)
 return 1-pearson_correlation
cat_counts = df.category.value_counts().to_frame()
cat_counts = cat_counts / cat_counts["category"].sum()
print(cat_counts.head())
 print(pd.unique(df.category))<sort_values>
error_pred_y = None
error_y = None
class SpearmanRhoCallback(Callback):
def __init__(self, training_data, validation_data, patience, model_name, reload=False):
global noise
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.patience = patience
self.value = -1
self.bad_epochs = 0
self.model_name = model_name
self.reload = reload
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
y_pred_val = self.model.predict(self.x_val)
 noise = np.random.normal(0, 1e-7, y_pred_val.shape[0])
 rho_val = np.mean([spearmanr(self.y_val[:, ind] + noise, y_pred_val[:, ind] + noise ).correlation for ind in range(y_pred_val.shape[1])])
 print('\r val_spearman-rho: %s' %(str(round(rho_val, 4))), end=100*' '+'
')
 if rho_val >= self.value:
self.model.save_weights(self.model_name)
 self.value = rho_val
else:
self.model.load_weights(self.model_name)
 return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
global error_pred_y, error_y
y_pred_val = self.model.predict(self.x_val)
 noise = np.random.normal(0, 1e-7, y_pred_val.shape[0])
 rho_val = np.mean([spearmanr(self.y_val[:, ind] + noise, y_pred_val[:, ind] + noise ).correlation for ind in range(y_pred_val.shape[1])])
 if np.isnan(rho_val):
print("Error")
 error_pred_y = y_pred_val
error_y = self.y_val
print(y_pred_val)
 print(self.x_val)
 print(self.y_val)
 raise "bogus error"
if rho_val >= self.value:
self.model.save_weights(self.model_name)
 self.value = rho_val
else:
self.bad_epochs += 1
print('\r val_spearman-rho: %s' %(str(round(rho_val, 4))), end=100*' '+'
')
 ')
 if self.reload:
print(' reload best: %s' %(str(round(self.value, 4))))
self.model.load_weights(self.model_name)
 return rho_val
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
cat_counts.sort_values(by="category",inplace=True)
 bot_5_categories = cat_counts.head()
top_5_categories = cat_counts.tail()
print(bot_5_categories)
 print(top_5_categories )<save_to_csv>
all_predictions = []
model_idx =0
def run_model() :
global y_train,all_predictions, model_idx
X_train,X_test = get_train_test()
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1,
patience=7, min_lr=1e-6, verbose=1)
 early_stop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=15,
mode='auto')
 kf = KFold(n_splits=5, random_state=random.randint(0,1000), shuffle=True)
 for ind,(tr, val)in enumerate(kf.split(X_train)) :
X_tr = X_train[tr]
y_tr = y_train[tr]
X_vl = X_train[val]
y_vl = y_train[val]
model_idx+=1
model = create_model(X_train)
 optimizer = Adam(lr=5e-4,clipnorm=1.4)
 model.compile(optimizer=optimizer, loss=custom_loss, metrics=[bce,logcosh])
 model.fit(
 X_tr, y_tr,
epochs=100, batch_size=64,
validation_data=(X_vl, y_vl),
verbose=True,
callbacks=[SpearmanRhoCallback(training_data=(X_tr, y_tr),
validation_data=(X_vl, y_vl),
patience=10,
model_name=f'best_model_batch{model_idx}.h5',
reload=True),
reduce_lr,
early_stop
] 
)
 optimizer = SGD(lr=0.01,clipnorm=1.1)
 model.compile(optimizer=optimizer, loss=custom_loss, metrics=[bce,logcosh])
 history = model.fit(
 X_tr, y_tr,
epochs=50, batch_size=64,
validation_data=(X_vl, y_vl),
verbose=True,
callbacks=[SpearmanRhoCallback(training_data=(X_tr, y_tr),
validation_data=(X_vl, y_vl),
patience=10,
model_name=f'best_model_batch_sgd{model_idx}.h5'),
reduce_lr,
early_stop
] 
)
 if min(history.history['val_bce'])< 0.375:
all_predictions.append(model.predict(X_test))
optimizer = Adam(lr=2e-4,clipnorm=1.9)
 model.compile(optimizer=optimizer, loss=bce)
 model.fit(
 X_train, y_train,
epochs=30, batch_size=128,
validation_data=(X_train, y_train),
verbose=True,
callbacks=[SpearmanRhoCallback(training_data=(X_train, y_train),
validation_data=(X_train, y_train),
patience=10,
model_name=f'best_model_batchoverfit{model_idx}.h5',
reload=False),
reduce_lr,
early_stop
] 
)
 all_predictions.append(model.predict(X_test))
samp_sub_df = pd.read_csv(".. /input/sample_submission.csv")
 samp_sub_df["category_id"] = np.random.choice(cat_counts.index,size=len(samp_sub_df),p=cat_counts["category"].values)
 samp_sub_df.to_csv("baised_rand_submission.csv", index=False )<import_modules>
all_predictions = []
while len(all_predictions)< 20:
run_model()
!pip install lap
Lambda, MaxPooling2D, Reshape
%matplotlib inline<load_from_csv>
K.clear_session()
gc.collect()
df = pd.read_csv('.. /input/humpback-whale-identification/train.csv')
 df.head()<load_from_csv>
test_preds = np.array([np.array([rankdata(c)for c in p.T] ).T for p in all_predictions] ).mean(axis=0)
 max_val = test_preds.max() + 1
test_preds = test_preds/max_val + 1e-12
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv"
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit<load_pretrained>
submission = pd.read_csv(INPUT_PATH+'google-quest-challenge/sample_submission.csv')
 submission[targets] = test_preds
submission.head(20 )
def expand_path(p):
if isfile(TRAIN + p):
return TRAIN + p
if isfile(TEST + p):
return TEST + p
return p
if isfile(P2SIZE):
print("P2SIZE exists.")
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 else:
p2size = {}
for p in tqdm(join):
size = pil_image.open(expand_path(p)).size
p2size[p] = size<compute_test_metric>
submission.to_csv("submission.csv", index = False)