Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<train_model>
import numpy as np
import pandas as pd
import os
import sys
sc = []
for i in range(10, 510, 5):
top_corr_cols = list(cor.abs().sort_values().tail(i ).reset_index() ['index'].values)
 X_train_poly1 = X_train_poly[:, top_corr_cols]
X_test_poly1 = X_test_poly[:, top_corr_cols]
oof_lr_poly, prediction_lr_poly, scores = train_model(X_train_poly1, X_test_poly1, y_train, params=None, model_type='sklearn', model=model)
 sc.append(scores )<split>
import glob
from os.path import join as path_join
from scipy.stats import spearmanr, rankdata
top_corr_cols = list(cor.abs().sort_values().tail(300 ).reset_index() ['index'].values)
 X_train_poly1 = X_train_poly[:, top_corr_cols]
X_test_poly1 = X_test_poly[:, top_corr_cols]<train_model>
os.system('pip install.. /input/sacremoses/sacremoses-master/ > /dev/null')
 os.system('pip install.. /input/transformers/transformers-master/ > /dev/null' )
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_poly, prediction_lr_poly, scores = train_model(X_train_poly1, X_test_poly1, y_train, params=None, model_type='sklearn', model=model )<save_to_csv>
package_path = '.. /input/radam-pytorch/RAdam'
sys.path.append(package_path )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_poly
submission.head()<normalization>
folder = '.. /input/google-quest-challenge/'
pretrained_bert = ".. /input/pretrainedbertpytorch/pretrained-bert-pytorch/bert-base-uncased/"
model_weight_path1 = '.. /input/quest-003-55-bert-07-3-training-1/'
model_weight_path2 = '.. /input/quest-003-55-bert-07-3-training-2/'
model_weight_path3 = '.. /input/quest-003-55-bert-07-3-training-3/'
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 X_train['300'] = X_train.std(1)
 X_test['300'] = X_test.std(1)
 scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_1, prediction_lr_1, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model )<normalization>
warnings.simplefilter('ignore' )
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 X_train['300'] = X_train.std(1)
 X_test['300'] = X_test.std(1)
 scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_1, prediction_lr_1_repeated, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_1_repeated
submission.to_csv('repeated_fold_features.csv', index=False)
 submission.head()<drop_column>
import torch
from torch.utils.data import TensorDataset, DataLoader, Dataset
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.utils.data.sampler import SubsetRandomSampler
from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR
from torch.nn.utils.weight_norm import weight_norm
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 main_cols = X_train.columns.tolist()<train_on_grid>
print("transformers:", transformers.__version__ )
neigh = NearestNeighbors(5, n_jobs=-1)
 neigh.fit(X_train)
 dists, _ = neigh.kneighbors(X_train, n_neighbors=5)
 mean_dist = dists.mean(axis=1)
 max_dist = dists.max(axis=1)
 min_dist = dists.min(axis=1)
 X_train['300'] = X_train.std(1)
 X_train = np.hstack(( X_train, mean_dist.reshape(-1, 1), max_dist.reshape(-1, 1), min_dist.reshape(-1, 1)))
 test_dists, _ = neigh.kneighbors(X_test, n_neighbors=5)
 test_mean_dist = test_dists.mean(axis=1)
 test_max_dist = test_dists.max(axis=1)
 test_min_dist = test_dists.min(axis=1)
 X_test['300'] = X_test.std(1)
 X_test = np.hstack(( X_test, test_mean_dist.reshape(-1, 1), test_max_dist.reshape(-1, 1), test_min_dist.reshape(-1, 1)) )<save_to_csv>
SEED = 12345
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_2, prediction_lr_2, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model)
 submission['target'] = prediction_lr_2
submission.to_csv('nn_features.csv', index=False )<save_to_csv>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] =(prediction_lr_1 + prediction_lr_2)/ 2
submission.head()<train_model>
def read_pickle_from_file(pickle_file):
with open(pickle_file,'rb')as f:
x = pickle.load(f)
 return x
def write_pickle_to_file(pickle_file, x):
with open(pickle_file, 'wb')as f:
pickle.dump(x, f, pickle.HIGHEST_PROTOCOL )
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_1, prediction_lr_1, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )<categorify>
train_df = pd.read_csv(f"{folder}train.csv")
 train_df.shape
scores_dict = {'f_classif': [], 'mutual_info_classif': []}
for i in range(5, 100, 5):
s1 = SelectPercentile(f_classif, percentile=i)
 X_train1 = s1.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s1.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['f_classif'].append(np.mean(scores))
s2 = SelectPercentile(mutual_info_classif, percentile=i)
 X_train1 = s2.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s2.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['mutual_info_classif'].append(np.mean(scores))<categorify>
test_df = pd.read_csv(f"{folder}test.csv")
 test_df.shape
scores_dict = {'f_classif': [], 'mutual_info_classif': []}
for i in range(10, 301, 10):
s1 = SelectKBest(f_classif, k=i)
 X_train1 = s1.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s1.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['f_classif'].append(np.mean(scores))
s2 = SelectKBest(mutual_info_classif, k=i)
 X_train1 = s2.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s2.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['mutual_info_classif'].append(np.mean(scores))<categorify>
target_table = [
['question_asker_intent_understanding', 'question'],
['question_body_critical', 'question'],
['question_conversational', 'question'],
['question_expect_short_answer', 'question'],
['question_fact_seeking', 'question'],
['question_has_commonly_accepted_answer', 'question'],
['question_interestingness_others', 'question'],
['question_interestingness_self', 'question'],
['question_multi_intent', 'question'],
['question_not_really_a_question', 'question'],
['question_opinion_seeking', 'question'],
['question_type_choice', 'question'],
['question_type_compare', 'question'],
['question_type_consequence', 'question'],
['question_type_definition', 'question'],
['question_type_entity', 'question'],
['question_type_instructions', 'question'],
['question_type_procedure', 'question'],
['question_type_reason_explanation', 'question'],
['question_type_spelling', 'question'],
['question_well_written', 'question'],
['answer_helpful', 'answer'],
['answer_level_of_information', 'answer'],
['answer_plausible', 'answer'],
['answer_relevance', 'answer'],
['answer_satisfaction', 'answer'],
['answer_type_instructions', 'answer'],
['answer_type_procedure', 'answer'],
['answer_type_reason_explanation', 'answer'],
['answer_well_written', 'answer'],
]
input_columns = [
'question_title',
'question_body',
'answer'
]
selector = SelectKBest(f_classif, k=60)
 X_trainK = selector.fit_transform(X_train, y_train.values.astype(int))
X_testK = selector.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_trainK, X_testK, y_train, params=None, model_type='sklearn', model=model )<save_to_csv>
target_question_columns = []
target_answer_columns = []
for table in target_table:
if table[1] == 'question':
target_question_columns.append(table[0])
 elif table[1] == 'answer':
target_answer_columns.append(table[0])
 target_columns = target_question_columns + target_answer_columns
print('question:', len(target_question_columns))
print('answer:', len(target_answer_columns))
print('total:', len(target_columns))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_1
submission.head()<train_model>
for df in [train_df, test_df]:
df['domcom'] = df['url'].apply(lambda s: s.split('://')[1].split('/')[0].split('.'))
df['dom_cnt'] = df['domcom'].apply(lambda s: len(s))
df['domcom'] = df['domcom'].apply(lambda s: s + ['none', 'none'])
 for ii in range(0,4):
df['url_'+str(ii)] = df['domcom'].apply(lambda s: s[ii])
 df.drop('domcom', axis = 1, inplace = True )
scores_list = []
for i in range(10, 301, 5):
s = RFE(model, i, step=1)
 X_train1 = s.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_list.append(np.mean(scores))<train_model>
train_feature = pd.DataFrame()
test_feature = pd.DataFrame()
selector = RFE(model, 20, step=1)
 X_trainK = selector.fit_transform(X_train, y_train.values.astype(int))
X_testK = selector.transform(X_test)
 oof_lr_1, prediction_lr_rfe_20, scores = train_model(X_trainK, X_testK, y_train, params=None, model_type='sklearn', model=model )<save_to_csv>
eng_stopwords = set(stopwords.words("english"))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_rfe_20
submission.to_csv('rfe_20.csv', index=False)
 submission.head()<categorify>
for df, feature in [[train_df, train_feature], [test_df, test_feature]]:
for column in input_columns:
feature[column+'_total_length'] = df[column].apply(len)
 feature[column+'_capitals'] = df[column].apply(lambda comment: sum(1 for c in comment if c.isupper()))
feature[column+'_caps_vs_length'] = feature.apply(lambda row: float(row[column+'_capitals'])/float(row[column+'_total_length']),axis=1)
 feature[column+'_num_exclamation_marks'] = df[column].apply(lambda comment: comment.count('!'))
feature[column+'_num_question_marks'] = df[column].apply(lambda comment: comment.count('?'))
feature[column+'_num_punctuation'] = df[column].apply(lambda comment: sum(comment.count(w)for w in '.,;:'))
feature[column+'_num_symbols'] = df[column].apply(lambda comment: sum(comment.count(w)for w in '*&$%'))
feature[column+"_num_chars"] = df[column].apply(lambda x: len(str(x)))
 feature[column+'_num_words'] = df[column].apply(lambda comment: len(comment.split()))
feature[column+'_num_unique_words'] = df[column].apply(lambda comment: len(set(w for w in comment.split())))
 feature[column+'_words_vs_unique'] = feature[column+'_num_unique_words'] / feature[column+'_num_words']
feature[column+'_num_smilies'] = df[column].apply(lambda comment: sum(comment.count(w)for w in(':-)', ':)', ';-)', ';)')))
 feature[column+"_num_stopwords"] = df[column].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
feature[column+"_num_punctuations"] =df[column].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
feature[column+"_num_words_upper"] = df[column].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
selector = SelectKBest(f_classif, k=15)
 X_trainK = selector.fit_transform(X_train, y_train.values.astype(int))
X_testK = selector.transform(X_test )<train_model>
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer)
 ans_user_and_category=train_df[train_df[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape
oof_glm, prediction_glm, scores = train_model(X_trainK, X_testK, y_train, params=None, model_type='glm' )<save_to_csv>
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words])
 def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same
def add_external_features(df, feature):
feature['question_vs_answer_length'] = feature['question_body_num_words']/feature['answer_num_words']
feature['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in df[['answer_user_name', 'category']].values:
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
feature['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
feature = pd.concat([ feature, pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])], axis=1)
 return feature
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_glm
submission.to_csv('glm.csv', index=False)
 submission.head()<feature_engineering>
train_feature = add_external_features(train_df, train_feature)
 test_feature = add_external_features(test_df, test_feature )
eli5_weights = eli5.formatters.as_dataframe.explain_weights_df(model)
 eli5_weights['weight'] = eli5_weights['weight'].abs()
eli5_weights = eli5_weights.sort_values('weight', ascending=False)
 eli5_weights<normalization>
stop_words = nltk.corpus.stopwords.words('english')
 symbol = ["'", '"', ':', ';', '.', ',', '-', '!', '?', "'s", ")", "(", "...", '``', "''", "/", "$", "%", "*", "&", "{", "}", "[", "]"]
def get_prevalent(texts, top_count=15):
tokenized_sents = [nltk.word_tokenize(i)for i in texts]
tokenized_sents = [flatten for inner in tokenized_sents for flatten in inner]
fdist = nltk.FreqDist(w.lower() for w in tokenized_sents if w.lower() not in stop_words + symbol)
 comments = fdist.most_common(top_count)
 return [word[0] for word in comments]
for column in input_columns:
words = get_prevalent(train_df[column])
 print(column, words)
 for word in words:
for df, feature in [[train_df, train_feature], [test_df, test_feature]]:
feature[column+'_num'+word] = df[column].apply(lambda comment: comment.count(word))
train['mean'] = train.mean(1)
 train['std'] = train.std(1)
 test['mean'] = test.mean(1)
 test['std'] = test.std(1)
 scores_dict = {'simple': [], 'with_std': [], 'with_mean': []}
for i in range(1, eli5_weights.shape[0] + 1):
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['simple'].append(np.mean(scores))
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['with_mean'].append(np.mean(scores))
X_train = train[top_features + ['std']]
X_test = test[top_features + ['std']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['with_std'].append(np.mean(scores))<normalization>
find = re.compile(r"^[^.]*")
 train_df['netloc'] = train_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test_df['netloc'] = test_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 count_columns = ['question_title', 'question_user_name', 'answer_user_name', 'category', 'netloc']
for col in count_columns:
value = train_df[col].value_counts()
train_feature[col+'_count'] = train_df[col].map(value)
 test_feature[col+'_count'] = test_df[col].map(value ).fillna(1 )
train['mean'] = train.mean(1)
 train['std'] = train.std(1)
 test['mean'] = test.mean(1)
 test['std'] = test.std(1)
 scores_dict = {'simple': [], 'with_std': [], 'with_mean': []}
for i in range(1, eli5_weights.shape[0] + 1):
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 scores_dict['simple'].append(np.mean(scores))
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 scores_dict['with_mean'].append(np.mean(scores))
X_train = train[top_features + ['std']]
X_test = test[top_features + ['std']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 scores_dict['with_std'].append(np.mean(scores))<save_to_csv>
for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:8]
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr
submission.to_csv('submission_top8.csv', index=False)
 submission.head()<save_to_csv>
scaler = MinMaxScaler()
scaler.fit(train_feature)
 train_feature = pd.DataFrame(scaler.transform(train_feature), columns=train_feature.columns)
 test_feature = pd.DataFrame(scaler.transform(test_feature), columns=test_feature.columns)
 del scaler
gc.collect()
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:10]
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr
submission.to_csv('submission_top10.csv', index=False)
 submission.head()<set_options>
def convert_label(df, result_df, column, count=10):
labels = [(df[column].values>=(rate/count)).astype(np.int)for rate in range(1, count+1)]
columns = ['{}_{}'.format(column, i)for i in range(count)]
labels = np.array(labels ).T
label_df = pd.DataFrame(labels, columns=columns)
 result_df = pd.concat(( result_df, label_df), axis=1)
 return result_df
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
<load_from_csv>
label_convert_count = 12
train_feature2 = pd.DataFrame()
test_feature2 = pd.DataFrame()
for column in train_feature.columns:
train_feature2 = convert_label(train_feature, train_feature2, column, label_convert_count)
 test_feature2 = convert_label(test_feature, test_feature2, column, label_convert_count)
 train_feature = train_feature2.copy()
test_feature = test_feature2.copy()
del train_feature2, test_feature2
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
features = ['question_title', 'question_user_name', 'answer_user_name']
limits = [6, 8, 8]
for col, limit in zip(features, limits):
value = train_df[col].value_counts()
train_df['item_count'] = train_df[col].map(value)
 train_df['item_value'] = train_df[col].copy()
train_df.loc[train_df.item_count < limit, 'item_value'] = "___
test_df['item_count'] = test_df[col].map(value ).fillna(1)
 test_df['item_value'] = test_df[col].copy()
test_df.loc[test_df.item_count < limit, 'item_value'] = "___
lb = LabelBinarizer()
lb.fit(train_df['item_value'])
 encode_train = lb.transform(train_df['item_value'])
 encode_test = lb.transform(test_df['item_value'])
 columns = ['LabelBinarizer_{}'.format(i)for i in range(encode_train.shape[1])]
print("{}: {}".format(col, len(train_df['item_value'].value_counts())))
 encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del lb
train_df.drop(['item_count', 'item_count'], axis=1, inplace=True)
 test_df.drop(['item_count', 'item_count'], axis=1, inplace=True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
features = ['url_0', 'category']
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(train_df[features])
 encode_train = enc.transform(train_df[features] ).toarray()
encode_test = enc.transform(test_df[features] ).toarray()
columns = ['encode_{}'.format(i)for i in range(encode_train.shape[1])]
encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del encode_train, encode_test, enc
target_column = 'target'
column_to_drop = 'id'
train_set, validate_set = train_test_split(train, test_size = 0.2, random_state = 1)
 x_train = train_set.drop([target_column, column_to_drop], axis = 1 ).copy()
y_train = train_set[target_column].copy()
x_validate = validate_set.drop([target_column, column_to_drop], axis = 1 ).copy()
y_validate = validate_set[target_column].copy()
x_total = train.drop([target_column, column_to_drop], axis = 1 ).copy()
y_total = train[target_column].copy()
x_test = test.drop(column_to_drop, axis = 1 ).copy()<train_model>
for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))
en = ElasticNet(alpha = 0.085, l1_ratio = 0.5)
 en.fit(x_train, y_train )<compute_train_metric>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
y_validate_p = en.predict(x_validate)
 roc_auc_score(y_validate, y_validate_p )<train_model>
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df
train_df = clean_data(train_df, input_columns)
 test_df = clean_data(test_df, input_columns)
 del tokenizer
en = ElasticNet(alpha = 0.085, l1_ratio = 0.5)
 en.fit(x_total, y_total )<save_to_csv>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, num_features, num_labels, pretrained_bert):
super(NeuralNet, self ).__init__()
self.bert = transformers.BertModel.from_pretrained(pretrained_bert)
 self.num_features = num_features
self.num_labels = num_labels
self.encoded_dropout = SpatialDropout(0.2)
 self.pooled_dropout = nn.Dropout(0.2)
 self.feature_linear = nn.Sequential(
 nn.Linear(self.num_features, self.num_features),
nn.ReLU(inplace=True),
nn.Dropout(0.2), 
)
 dense_hidden_units = self.bert.config.hidden_size * 3 + self.num_features
self.linear1 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.linear2 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.classifier = nn.Sequential(
 nn.Linear(dense_hidden_units, num_labels), 
)
 def forward(self, ids, masks, segments, feature):
feature_output = self.feature_linear(feature)
 sequence_output, pooled_output = self.bert(input_ids=ids, attention_mask=masks, token_type_ids=segments)
 sequence_output = self.encoded_dropout(sequence_output)
 pooled_output = self.pooled_dropout(pooled_output)
 avg_pool = torch.mean(sequence_output, 1)
 max_pool, _ = torch.max(sequence_output, 1)
 h_conc = torch.cat(( avg_pool, max_pool, pooled_output, feature_output), 1)
 h_conc_linear = F.relu(self.linear1(h_conc))
hidden = h_conc + h_conc_linear
h_conc_linear = F.relu(self.linear2(hidden))
hidden = hidden + h_conc_linear
return self.classifier(hidden )
submission = pd.DataFrame(test.id)
 y_test_p = pd.Series(en.predict(x_test))
submission['target'] = y_test_p
submission.to_csv("submission.csv", index = False )<save_to_csv>
def compute_input_title_question(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 if(len(title)+len(question)+3)> max_sequence_length:
if len(title)> 30:
title = title[:30]
question_len = max_sequence_length - len(title)- 3
question = question[:question_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = cls_ids + title_ids + sep_ids + question_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)+2)+ [1]*(len(question_ids)+1)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
train = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 y_train = train['target']
X_train = train.drop('target', axis=1)
 test = pd.read_csv('.. /input/test.csv')
 X_test = test.drop('id', axis = 1)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 clf = LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear' ).fit(X_train, y_train)
 submission['target'] = clf.predict_proba(X_test)[:,1]
submission.to_csv('submission.csv', index=False )<load_from_csv>
def compute_input_arrays(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
t_max_len = 35
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 answer = tokenizer.tokenize(instance.answer)
 if(len(title)+len(question)+len(answer)+4)> max_sequence_length:
if len(title)> t_max_len:
title = title[:t_max_len]
question_len =(max_sequence_length - len(title)- 4)// 2
question = question[:question_len]
answer_len = max_sequence_length - len(title)- len(question)- 4
answer = answer[:answer_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 answer_ids = tokenizer.convert_tokens_to_ids(answer)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = cls_ids + title_ids + sep_ids + question_ids + sep_ids + answer_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)+len(question_ids)+3)+ [1]*(len(answer_ids)+1)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv' )<prepare_x_and_y>
test_pred_datas = {}
test_pred_weights = {}
for column in target_columns:
test_pred_datas[column] = np.zeros(len(test_df))
test_pred_weights[column] = 0.0
def add_test_pred_data(prediction, columns, weight):
for column_idx, column in enumerate(columns):
test_pred_datas[column] += weight * prediction[:, column_idx]
test_pred_weights[column] += weight
train_X = train.drop(['id','target'], axis=1 ).as_matrix()
train_y = train['target'].values
test_X = test.drop(['id'], axis=1 ).as_matrix()<save_to_csv>
validation_datas = {}
validation_counts = {}
for column in target_columns:
validation_datas[column] = np.zeros(len(train_df))
validation_counts[column] = np.zeros(len(train_df))
def add_validation_data(prediction, columns, idx):
for column_idx, column in enumerate(columns):
validation_datas[column][idx] += prediction[:, column_idx]
validation_counts[column][idx] += 1.0
clf = KNeighborsClassifier(n_neighbors=5)
 clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_KNeighborsClassifier.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
def mean_spearmanr_correlation_score(y_true, y_pred):
num_labels = y_pred.shape[1]
score = np.nanmean([ spearmanr(y_pred[:, idx], y_true[:, idx] ).correlation for idx in range(num_labels)])
 return score
clf = DecisionTreeClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_DecisionTreeClassifier.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, features, labels = None):
self.inputs = inputs
self.features = features
if labels is not None:
self.labels = labels
else:
self.labels = None
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
input_features = self.features[idx]
if self.labels is not None:
input_labels = self.labels[idx]
return input_ids, input_masks, input_segments, input_features, input_labels
return input_ids, input_masks, input_segments, input_features
def __len__(self):
return len(self.inputs[0] )
clf = RandomForestClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_RandomForestClassifier.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
def model_test_validation(label_columns, train_inputs, test_inputs, x_train, x_test, weight_files):
if len(kfold_split)!= len(weight_files):
return
batch_size = 6
num_features = x_test.shape[1]
num_labels = len(label_columns)
 test_dataset = QuestDataset(test_inputs, x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 for k,(train_idx, valid_idx)in enumerate(kfold_split):
fname = weight_files[k]
print(k+1, fname)
 x_train_valid = x_train[valid_idx]
train_inputs_valid = [x[valid_idx] for x in train_inputs]
valid_dataset = QuestDataset(train_inputs_valid, x_train_valid, None)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = NeuralNet(num_features, num_labels, pretrained_bert)
 model.to('cuda:0')
 model.load_state_dict(torch.load(fname))
model.eval()
valid_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in valid_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 add_validation_data(valid_preds_fold, label_columns, valid_idx)
 test_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in test_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, label_columns, 1.0)
 del model, valid_dataset, valid_loader
torch.cuda.empty_cache()
gc.collect()
del test_dataset, test_loader
clf = AdaBoostClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_AdaBoostClassifier.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
tokenizer = transformers.BertTokenizer.from_pretrained(pretrained_bert )
clf = GaussianNB()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_GaussianNB.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
train_inputs = compute_input_title_question(train_df, tokenizer, max_sequence_length=512)
 test_inputs = compute_input_title_question(test_df, tokenizer, max_sequence_length=512)
 x_train = train_feature.values
x_test = test_feature.values
print(x_train.shape)
 print(x_test.shape)
 print(len(target_question_columns))
weight_files = [
model_weight_path1 + 'best_weights_003_55_7_3_question_1.pth',
model_weight_path1 + 'best_weights_003_55_7_3_question_2.pth',
model_weight_path2 + 'best_weights_003_55_7_3_question_3.pth',
model_weight_path2 + 'best_weights_003_55_7_3_question_4.pth',
model_weight_path3 + 'best_weights_003_55_7_3_question_5.pth',
model_weight_path3 + 'best_weights_003_55_7_3_question_6.pth',
]
model_test_validation(target_question_columns, train_inputs, test_inputs, x_train, x_test, weight_files)
 del x_train, x_test
del train_inputs, test_inputs
clf = xgb.XGBClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_XGBClassifier.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
train_inputs = compute_input_arrays(train_df, tokenizer, max_sequence_length=512)
 test_inputs = compute_input_arrays(test_df, tokenizer, max_sequence_length=512)
 x_train = train_feature.values
x_test = test_feature.values
print(x_train.shape)
 print(x_test.shape)
 print(len(target_answer_columns))
weight_files = [
model_weight_path1 + 'best_weights_003_55_7_3_answer_1.pth',
model_weight_path1 + 'best_weights_003_55_7_3_answer_2.pth',
model_weight_path2 + 'best_weights_003_55_7_3_answer_3.pth',
model_weight_path2 + 'best_weights_003_55_7_3_answer_4.pth',
model_weight_path3 + 'best_weights_003_55_7_3_answer_5.pth',
model_weight_path3 + 'best_weights_003_55_7_3_answer_6.pth',
]
model_test_validation(target_answer_columns, train_inputs, test_inputs, x_train, x_test, weight_files)
 del x_train, x_test
del train_inputs, test_inputs
clf = LogisticRegression(class_weight='balanced', penalty='l1', C=1.0, solver='liblinear')
 clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_LogisticRegression.csv', index=False)
 test = test.drop('target', axis=1 )<compute_train_metric>
del train_feature, test_feature
del target_question_columns, target_answer_columns
for model_type in [KNeighborsClassifier, DecisionTreeClassifier, RandomForestClassifier,
AdaBoostClassifier, xgb.XGBClassifier, LogisticRegression]:
clf = model_type()
kfold = KFold(n_splits=5, shuffle=True)
 cv = ShuffleSplit(n_splits=100, test_size=0.3, random_state=0)
 scores = cross_val_score(clf, train_X, train_y, cv=cv, scoring='roc_auc')
 print("Print {} Accuracy: {}(+/- {})".format(model_type.__name__, scores.mean() , scores.std() * 2))<train_on_grid>
validationT = pd.DataFrame()
for column in target_columns:
preds = validation_datas[column]
count = validation_counts[column]
count = np.where(count < 0.5, 1.0, count)
 validationT[column] = preds / count
clf = LogisticRegression(class_weight='balanced', solver='liblinear')
 penalty = ['l1', 'l2']
C = uniform(loc=0, scale=4)
 hyperparameters = dict(C=C, penalty=penalty)
 rand_cv = RandomizedSearchCV(clf, hyperparameters, random_state=1, n_iter=100, cv=5, verbose=0, n_jobs=-1, scoring='roc_auc')
 best_model = rand_cv.fit(train_X, train_y)
 print('Best Penalty:', best_model.best_estimator_.get_params() ['penalty'])
 print('Best C:', best_model.best_estimator_.get_params() ['C'] )<prepare_output>
mean_spearmanr_correlation_score(validationT.values, train_df[target_columns].values )
cv_results = pd.DataFrame()
cv_results['params'] = best_model.cv_results_['params']
cv_results['mean_test_score'] = best_model.cv_results_['mean_test_score']
cv_results['std_test_score'] = best_model.cv_results_['std_test_score']
cv_results['rank_test_score'] = best_model.cv_results_['rank_test_score']
cv_results = pd.concat([cv_results.drop('params', axis=1), pd.DataFrame(cv_results['params'].tolist())], axis=1)
 cv_results.sort_values('rank_test_score' ).head()<train_on_grid>
test_predsT = pd.read_csv(f"{folder}sample_submission.csv")
 for column in target_columns:
preds = test_pred_datas[column]
weight = test_pred_weights[column]
test_predsT[column] = preds / weight
C = [0.001, 0.01, 0.02, 0.05, 0.1, 0.12, 0.13, 0.15, 0.16, 0.17, 0.178, 0.179, 0.175, 0.2, 0.3]
hyperparameters = dict(C=C)
 clf = LogisticRegression(solver='liblinear', class_weight='balanced', penalty='l1')
 rand_cv = RandomizedSearchCV(clf, hyperparameters, random_state=1, n_iter=100, cv=5, verbose=0, n_jobs=-1, scoring='roc_auc')
 best_model = rand_cv.fit(train_X, train_y)
 print('Best Penalty:', best_model.best_estimator_.get_params() ['penalty'])
 print('Best C:', best_model.best_estimator_.get_params() ['C'])
 print('Best Score: {}'.format(best_model.best_score_))<save_to_csv>
del target_columns, input_columns
del test_pred_datas, test_pred_weights
del validation_datas, validation_counts
test['target'] = best_model.predict(test_X)
 test[['id','target']].to_csv('submission_LogisticRegression_randomCV.csv', index=False)
 test = test.drop('target', axis=1 )<save_to_csv>
torch.cuda.empty_cache()
gc.collect()
test['target'] = best_model.predict_proba(test_X)[:,1]
test[['id','target']].to_csv('submission_LogisticRegression_randomCV_proba.csv', index=False)
 test = test.drop('target', axis=1 )<train_on_grid>
folder = '.. /input/google-quest-challenge/'
pretrained_bert = ".. /input/pretrainedbertpytorch/pretrained-bert-pytorch/gpt2/"
model_weight_path1 = '.. /input/photostage1/best_weights_006-55-gpt2-01_question_1.pth'
model_weight_path2 = '.. /input/photostage2/best_weights_006-55-gpt2-01_question_2.pth'
model_weight_path3 = '.. /input/photostage3/best_weights_006-55-gpt2-01_question_3.pth'
model_weight_path4 = '.. /input/photostage4/best_weights_006-55-gpt2-01_question_4.pth'
model_weight_path5 = '.. /input/photostage5/best_weights_006-55-gpt2-01_question_5.pth'
model_weight_path6 = '.. /input/photostage6/best_weights_006-55-gpt2-01_question_6.pth'
model_weight_path7 = '.. /input/photostagea1/best_weights_006-55-gpt2-01_answer_1.pth'
model_weight_path8 = '.. /input/photostagea2/best_weights_006-55-gpt2-01_answer_2.pth'
model_weight_path9 = '.. /input/photostagea3/best_weights_006-55-gpt2-01_answer_3.pth'
model_weight_path10 = '.. /input/photostagea4/best_weights_006-55-gpt2-01_answer_4.pth'
model_weight_path11 = '.. /input/photostagea5/best_weights_006-55-gpt2-01_answer_5.pth'
model_weight_path12 = '.. /input/photostagea6/best_weights_006-55-gpt2-01_answer_6.pth'
clf = xgb.XGBClassifier(learning_rate=0.02, n_estimators=600, objective='binary:logistic')
 params = {
'min_child_weight': [1, 5, 10],
'gamma': [0.5, 1, 1.5, 2, 5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'max_depth': [3, 4, 5]
}
rand_cv = RandomizedSearchCV(clf, params, random_state=1, n_iter=20, cv=5, verbose=0, n_jobs=-1, scoring='roc_auc')
 best_model = rand_cv.fit(train_X, train_y )<find_best_score>
BertTokenizer, BertModel, BertForSequenceClassification, BertConfig,
WEIGHTS_NAME, CONFIG_NAME, AdamW, get_linear_schedule_with_warmup,
get_cosine_schedule_with_warmup, GPT2Config,GPT2Model,
GPT2LMHeadModel,
GPT2Tokenizer, 
)
best_model.best_score_<save_to_csv>
print("transformers:", transformers.__version__ )
test['target'] = best_model.predict(test_X)
 test[['id','target']].to_csv('submission_XGBClassifier_randomCV.csv', index=False)
 test = test.drop('target', axis=1)
 test['target'] = best_model.predict_proba(test_X)[:,1]
test[['id','target']].to_csv('submission_XGBClassifier_randomCV_proba.csv', index=False)
 test = test.drop('target', axis=1 )<train_model>
SEED = 12345
start_time_all = time.time()
clf = LogisticRegression(class_weight='balanced', penalty='l1', C=0.01, solver='liblinear' ).fit(train_X, train_y )<define_variables>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
<define_variables>
def read_pickle_from_file(pickle_file):
with open(pickle_file,'rb')as f:
x = pickle.load(f)
 return x
def write_pickle_to_file(pickle_file, x):
with open(pickle_file, 'wb')as f:
pickle.dump(x, f, pickle.HIGHEST_PROTOCOL)
 def time_to_str(t, mode='min'):
if mode=='min':
t = int(t)/60
hr = t//60
min = t%60
return '%2d hr %02d min'%(hr,min)
 elif mode=='sec':
t = int(t)
 min = t//60
sec = t%60
return '%2d min %02d sec'%(min,sec)
 else:
raise NotImplementedError
NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8500000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]<categorify>
train_df = pd.read_csv(f"{folder}train.csv")
 train_df.shape
test_df = pd.read_csv(f"{folder}test.csv")
 test_df.shape
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
<feature_engineering>
target_table = [
['question_asker_intent_understanding', 'question'],
['question_body_critical', 'question'],
['question_conversational', 'question'],
['question_expect_short_answer', 'question'],
['question_fact_seeking', 'question'],
['question_has_commonly_accepted_answer', 'question'],
['question_interestingness_others', 'question'],
['question_interestingness_self', 'question'],
['question_multi_intent', 'question'],
['question_not_really_a_question', 'question'],
['question_opinion_seeking', 'question'],
['question_type_choice', 'question'],
['question_type_compare', 'question'],
['question_type_consequence', 'question'],
['question_type_definition', 'question'],
['question_type_entity', 'question'],
['question_type_instructions', 'question'],
['question_type_procedure', 'question'],
['question_type_reason_explanation', 'question'],
['question_type_spelling', 'question'],
['question_well_written', 'question'],
['answer_helpful', 'answer'],
['answer_level_of_information', 'answer'],
['answer_plausible', 'answer'],
['answer_relevance', 'answer'],
['answer_satisfaction', 'answer'],
['answer_type_instructions', 'answer'],
['answer_type_procedure', 'answer'],
['answer_type_reason_explanation', 'answer'],
['answer_well_written', 'answer'],
]
input_columns = [
'question_title',
'question_body',
'answer'
]
def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors 
<choose_model_class>
target_question_columns = []
target_answer_columns = []
for table in target_table:
if table[1] == 'question':
target_question_columns.append(table[0])
 elif table[1] == 'answer':
target_answer_columns.append(table[0])
 target_columns = target_question_columns + target_answer_columns
print('question:', len(target_question_columns))
print('answer:', len(target_answer_columns))
print('total:', len(target_columns))
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores 
<load_from_csv>
for df in [train_df, test_df]:
df['domcom'] = df['url'].apply(lambda s: s.split('://')[1].split('/')[0].split('.'))
df['dom_cnt'] = df['domcom'].apply(lambda s: len(s))
df['domcom'] = df['domcom'].apply(lambda s: s + ['none', 'none'])
 for ii in range(0,4):
df['url_'+str(ii)] = df['domcom'].apply(lambda s: s[ii])
 df.drop('domcom', axis = 1, inplace = True)
 df.question_body = df.question_body.apply(html.unescape)
 df.answer = df.answer.apply(html.unescape)
 df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&gt",">"))
df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&lt","<"))
df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&amp","&"))
df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&quot;",
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions 
<save_to_csv>
train_feature = pd.DataFrame()
test_feature = pd.DataFrame()
eng_stopwords = set(stopwords.words("english"))
for df, feature in [[train_df, train_feature], [test_df, test_feature]]:
for column in input_columns:
feature[column+'_total_length'] = df[column].apply(len)
 feature[column+'_capitals'] = df[column].apply(lambda comment: sum(1 for c in comment if c.isupper()))
feature[column+'_caps_vs_length'] = feature.apply(lambda row: float(row[column+'_capitals'])/float(row[column+'_total_length']),axis=1)
 feature[column+'_num_exclamation_marks'] = df[column].apply(lambda comment: comment.count('!'))
feature[column+'_num_question_marks'] = df[column].apply(lambda comment: comment.count('?'))
feature[column+'_num_punctuation'] = df[column].apply(lambda comment: sum(comment.count(w)for w in '.,;:'))
feature[column+'_num_symbols'] = df[column].apply(lambda comment: sum(comment.count(w)for w in '*&$%'))
feature[column+"_num_chars"] = df[column].apply(lambda x: len(str(x)))
 feature[column+'_num_words'] = df[column].apply(lambda comment: len(comment.split()))
feature[column+'_num_unique_words'] = df[column].apply(lambda comment: len(set(w for w in comment.split())))
 feature[column+'_words_vs_unique'] = feature[column+'_num_unique_words'] / feature[column+'_num_words']
feature[column+'_num_smilies'] = df[column].apply(lambda comment: sum(comment.count(w)for w in(':-)', ':)', ';-)', ';)')))
 feature[column+'_num_doubt'] = df[column].apply(lambda comment: comment.count('not sure'))
feature[column+'_num_think'] = df[column].apply(lambda comment: sum(comment.count(w)for w in ['thinking','think','thought']))
feature[column+"_num_stopwords"] = df[column].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
feature[column+"_num_punctuations"] =df[column].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
feature[column+"_num_words_upper"] = df[column].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer)
 ans_user_and_category=train_df[train_df[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
if(score < 0.5):
label = ""
score = ""
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 else:
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print('Copying sample submission.')
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 if __name__ == '__main__':
main()<set_options>
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words])
 def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same
def add_external_features(df, feature):
feature['question_vs_answer_length'] = feature['question_body_num_words']/feature['answer_num_words']
feature['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in df[['answer_user_name', 'category']].values:
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
feature['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
feature = pd.concat([ feature, pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])], axis=1)
 return feature
import time
import copy
import csv
import operator
import os
import pathlib
import shutil
import numpy as np
import PIL
import pydegensac
from scipy import spatial
import tensorflow as tf
import warnings 
<define_variables>
train_feature = add_external_features(train_df, train_feature)
 test_feature = add_external_features(test_df, test_feature )
INPUT_DIR = '.. /input/'
DATASET_DIR = INPUT_DIR + 'landmark-recognition-2020/'
TEST_DIR = DATASET_DIR + 'test/'
TRAIN_DIR = DATASET_DIR + 'train/'
TRAIN_LABELMAP_PATH = DATASET_DIR + 'train.csv'
DEBUG = False
DEBUG_SIZE = 500
PUBLIC_TRAIN_SIZE = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 5
TOP_K = 3
MAX_INLIER_SCORE = 30
MAX_REPROJECTION_ERROR = 6.0
MAX_RANSAC_ITERATIONS = 5_000
HOMOGRAPHY_CONFIDENCE = 0.96
DELG_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_IMAGE_SCALES = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
LOCAL_FEATURE_NUM = tf.constant(1000)
 LOG_FREQ = 500
QUICK_SUBMIT = True<load_from_csv>
stop_words = nltk.corpus.stopwords.words('english')
 symbol = ["'", '"', ':', ';', '.', ',', '-', '!', '?', "'s", ")", "(", "...", '``', "''", "/", "$", "%", "*", "&", "{", "}", "[", "]"]
def get_prevalent(texts, top_count=15):
tokenized_sents = [nltk.word_tokenize(i)for i in texts]
tokenized_sents = [flatten for inner in tokenized_sents for flatten in inner]
fdist = nltk.FreqDist(w.lower() for w in tokenized_sents if w.lower() not in stop_words + symbol)
 comments = fdist.most_common(top_count)
 return [word[0] for word in comments]
for column in input_columns:
words = get_prevalent(train_df[column])
 print(column, words)
 for word in words:
for df, feature in [[train_df, train_feature], [test_df, test_feature]]:
feature[column+'_num'+word] = df[column].apply(lambda comment: comment.count(word))
def image_path(subset, image_id):

 return os.path.join(DATASET_DIR, subset, image_id[0], image_id[1], image_id[2],
'{}.jpg'.format(image_id))
def load_image(image_path):

 return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 labelmap = None
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}<load_pretrained>
find = re.compile(r"^[^.]*")
 train_df['netloc'] = train_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test_df['netloc'] = test_df['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 count_columns = ['question_title', 'question_user_name', 'answer_user_name', 'category', 'netloc']
for col in count_columns:
value = train_df[col].value_counts()
train_feature[col+'_count'] = train_df[col].map(value)
 test_feature[col+'_count'] = test_df[col].map(value ).fillna(1)
 for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))
scaler = MinMaxScaler()
scaler.fit(train_feature)
 train_feature = pd.DataFrame(scaler.transform(train_feature), columns=train_feature.columns)
 test_feature = pd.DataFrame(scaler.transform(test_feature), columns=test_feature.columns)
 del scaler
gc.collect()
delg_model = tf.saved_model.load(DELG_MODEL_DIR)
 global_feature_fn = delg_model.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 local_feature_fn = delg_model.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'] )<feature_engineering>
def convert_label(df, result_df, column, count=10):
labels = [(df[column].values>=(rate/count)).astype(np.int)for rate in range(1, count+1)]
columns = ['{}_{}'.format(column, i)for i in range(count)]
labels = np.array(labels ).T
label_df = pd.DataFrame(labels, columns=columns)
 result_df = pd.concat(( result_df, label_df), axis=1)
 return result_df
def convert_label_origin(df, result_df, column, count=10):
columns = ['{}_{}'.format(column, i)for i in range(count)]
labels = df[columns].values
values = []
for i in range(len(labels)) :
value = 0.0
for j in range(count):
if labels[i][j] > 0.5:
value =(j+1)/ count
values.append(value)
 label_df = pd.DataFrame(values, columns=[column])
 result_df = pd.concat(( result_df, label_df), axis=1)
 return result_df
def global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
if DEBUG:
image_paths = image_paths[:DEBUG_SIZE]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
start = time.time()
total = 0
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = image_path.name.split('.')[0]
image_tensor = load_image(image_path)
 features = global_feature_fn(image_tensor,
DELG_IMAGE_SCALES, DELG_SCORE_THRESHOLD)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0, name='final_l2_normalization' ).numpy()
x = i + 1
if x % LOG_FREQ == 0 or x == num_embeddings:
end = time.time()
used = end - start
total += used
avg = total / x
remain = avg *(num_embeddings-x)/ 60.0
start = end
print('[{}/{}], [{:.2f}s/{:.2f}mins] used, {:.2f}mins remain'.format(x,
num_embeddings, used, total/60, remain))
print('Global features extracted, {:.2f}mins used'.format(total/60))
return ids, embeddings
def local_features(image_path):

 image_tensor = load_image(image_path)
 features = local_feature_fn(image_tensor, DELG_IMAGE_SCALES,
DELG_SCORE_THRESHOLD, LOCAL_FEATURE_NUM)
 keypoints = tf.divide(
 tf.add(tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors<train_on_grid>
label_convert_count = 12
train_feature2 = pd.DataFrame()
test_feature2 = pd.DataFrame()
for column in train_feature.columns:
train_feature2 = convert_label(train_feature, train_feature2, column, label_convert_count)
 test_feature2 = convert_label(test_feature, test_feature2, column, label_convert_count)
 train_feature = train_feature2.copy()
test_feature = test_feature2.copy()
del train_feature2, test_feature2
def match_keypoints(test_keypoints, test_descriptors, train_keypoints,
train_descriptors, max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def inliers_num(test_keypoints, test_descriptors,
train_keypoints, train_descriptors):

 test_match_kp, train_match_kp = match_keypoints(
 test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 if test_match_kp.shape[0] <= 4:
return 0
try:
H, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def score(num_inliers, global_score):

 local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore(test_image_id, labels_scores):

 test_image_path = image_path('test', test_image_id)
 test_keypoints, test_descriptors = local_features(test_image_path)
 for i in range(len(labels_scores)) :
train_image_id, label, global_score = labels_scores[i]
train_image_path = image_path('train', train_image_id)
 train_keypoints, train_descriptors = local_features(
 train_image_path)
 num_inliers = inliers_num(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 s = score(num_inliers, global_score)
 labels_scores[i] =(train_image_id, label, s)
 labels_scores.sort(key=lambda x: x[2], reverse=True)
 return labels_scores<predict_on_test>
features = ['question_title', 'question_user_name', 'answer_user_name']
limits = [6, 8, 8]
for col, limit in zip(features, limits):
value = train_df[col].value_counts()
train_df['item_count'] = train_df[col].map(value)
 train_df['item_value'] = train_df[col].copy()
train_df.loc[train_df.item_count < limit, 'item_value'] = "limit_abcdefg123456789"
test_df['item_count'] = test_df[col].map(value ).fillna(1)
 test_df['item_value'] = test_df[col].copy()
test_df.loc[test_df.item_count < limit, 'item_value'] = "limit_abcdefg123456789"
lb = LabelBinarizer()
lb.fit(train_df['item_value'])
 encode_train = lb.transform(train_df['item_value'])
 encode_test = lb.transform(test_df['item_value'])
 columns = ['LabelBinarizer_{}'.format(i)for i in range(encode_train.shape[1])]
print("{}: {}".format(col, len(train_df['item_value'].value_counts())))
 encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del lb
train_df.drop(['item_count', 'item_count'], axis=1, inplace=True)
 test_df.drop(['item_count', 'item_count'], axis=1, inplace=True )
def predict_map(test_ids, labels_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
aggregate_scores = {}
for _, label, score in labels_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[test_id] = {'score': score, 'class': label}
return prediction_map
def predict(test_ids, test_embeddings, train_ids, train_embeddings):

 labels_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 labels_scores[test_index] = [ 
(train_id, labelmap[train_id], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
pre_verify_preds = predict_map(test_ids, labels_scores)
 for test_index, test_id in enumerate(test_ids):
labels_scores[test_index] = rescore(
 test_id, labels_scores[test_index])
 post_verify_preds = predict_map(
 test_ids, labels_scores)
 return post_verify_preds
def save(predictions):

 with open('submission.csv', 'w')as f:
csv_writer = csv.DictWriter(f, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': '{} {:.8f}'.format(label, score)} )<load_from_csv>
features = ['url_0', 'category']
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(train_df[features])
 encode_train = enc.transform(train_df[features] ).toarray()
encode_test = enc.transform(test_df[features] ).toarray()
columns = ['encode_{}'.format(i)for i in range(encode_train.shape[1])]
encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del encode_train, encode_test, enc
print(train_feature.shape)
 print(test_feature.shape)
 print('time: {}'.format(time_to_str(( time.time() -start_time_all),'min')) )
def run() :
start = time.time()
training_size = len(labelmap.keys())
 print('Found {} training images.'.format(training_size))
if QUICK_SUBMIT and training_size == PUBLIC_TRAIN_SIZE:
print('Copying sample submission...')
 shutil.copyfile(DATASET_DIR + 'sample_submission.csv', 'submission.csv')
 return
print('Extracting global features on testing set...')
 test_ids, test_embeddings = global_features(TEST_DIR)
 print('
Extracting global features on training set...')
 train_ids, train_embeddings = global_features(TRAIN_DIR)
 print('
Predicting...')
 preds = predict(test_ids, test_embeddings,
train_ids, train_embeddings)
 print('Saving result to csv...')
 save(preds)
 end = time.time()
print('All done!({:.2f}mins used)'.format(( end-start)/60))<install_modules>
for col in train_feature.columns:
train_mean = np.nanmean(train_feature[col].values)
 train_feature[col].fillna(train_mean, inplace=True)
 test_feature[col].fillna(train_mean, inplace=True)
 print("train: nan=", np.sum(np.sum(pd.isnull(train_feature))))
print("test : nan=", np.sum(np.sum(pd.isnull(test_feature))))
!pip install.. /input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/glrec2020/efficientnet-1.1.0-py3-none-any.whl<import_modules>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
import operator
import gc
import pathlib
import shutil
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import backend as K
from scipy import spatial
import cv2
import efficientnet.tfkeras as efn
import math
import copy
import csv
import os
import numpy as np
import PIL
import pydegensac<define_search_space>
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df
train_df = clean_data(train_df, input_columns)
 test_df = clean_data(test_df, input_columns)
 del tokenizer
NUMBER_OF_CLASSES = 81313
IMAGE_SIZE = [384, 384]
LR = 0.0001<load_pretrained>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet3(nn.Module):
def __init__(self, num_features, num_labels, pretrained_bert):
super(NeuralNet3, self ).__init__()
self.config = transformers.GPT2Config.from_pretrained(pretrained_bert, output_hidden_states=False)
 self.gptmodel = transformers.GPT2Model(self.config)
 self.gpt = self.gptmodel.from_pretrained(pretrained_bert,config =self.config)
 self.num_features = num_features
self.num_labels = num_labels
self.encoded_dropout = SpatialDropout(0.2)
 self.pooled_dropout = nn.Dropout(0.2)
 self.feature_linear = nn.Sequential(
 nn.Linear(self.num_features, self.num_features),
nn.ReLU(inplace=True),
nn.Dropout(0.2), 
)

dense_hidden_units = self.gpt.config.hidden_size * 3 + self.num_features
self.linear1 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.linear2 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.classifier = nn.Sequential(
 nn.Linear(dense_hidden_units, num_labels), 
)
 def forward(self, ids, masks, segments, feature):
feature_output = self.feature_linear(feature)
 outputs = self.gpt(input_ids=ids, attention_mask=masks, token_type_ids=segments)
 last_hidden_state,present = self.gpt(input_ids=ids, attention_mask=masks, token_type_ids=segments)
 last_hidden_state = self.encoded_dropout(last_hidden_state)
 avg_pool = torch.mean(last_hidden_state, 1)
 max_pool, _ = torch.max(last_hidden_state, 1)
 h_conc = torch.cat(( avg_pool, max_pool, last_hidden_state[:, -1, :], feature_output), 1)
 h_conc_linear = F.relu(self.linear1(h_conc))
hidden = h_conc + h_conc_linear
h_conc_linear = F.relu(self.linear2(hidden))
hidden = hidden + h_conc_linear
return self.classifier(hidden)
MODEL1 = get_model_B6()
MODEL2 = get_model_B7()
MODEL1.load_weights('.. /input/effb6-512-ep18/effb6model512-18.h5')
 MODEL1 = tf.keras.models.Model(inputs = MODEL1.input[0], outputs = MODEL1.layers[-4].output)
 MODEL2.load_weights('.. /input/effb7-512-ep12/effb7model512-12.h5')
 MODEL2 = tf.keras.models.Model(inputs = MODEL2.input[0], outputs = MODEL2.layers[-4].output )<define_variables>
def compute_input_title_question(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 if(len(title)+len(question)+3)> max_sequence_length:
if len(title)> 30:
title = title[:30]
question_len = max_sequence_length - len(title)- 3
question = question[:question_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = title_ids + question_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
len_q = len(question_ids)
 len_qf = int(len(question_ids)/2)
 len_ql = len_q-len_qf
segments = [0]*(len(title_ids)) + [1]*(len(question_ids)) + padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
NUM_TO_RERANK = 3
NUM_PUBLIC_TEST_IMAGES = 10345<define_variables>
def compute_input_title_answer(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 answer = tokenizer.tokenize(instance.answer)
 if(len(title)+len(answer)+3)> max_sequence_length:
if len(title)> 30:
title = title[:30]
answer_len = max_sequence_length - len(title)- 3
answer = answer[:answer_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 answer_ids = tokenizer.convert_tokens_to_ids(answer)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = title_ids + answer_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)) + [1]*(len(answer_ids)) + padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv' )<define_variables>
def compute_input_arrays(df, tokenizer, max_sequence_length=512):
input_ids, input_masks, input_segments = [], [], []
t_max_len = 35
for _, instance in df.iterrows() :
title = tokenizer.tokenize(instance.question_title)
 question = tokenizer.tokenize(instance.question_body)
 answer = tokenizer.tokenize(instance.answer)
 if(len(title)+len(question)+len(answer)+4)> max_sequence_length:
if len(title)> t_max_len:
title = title[:t_max_len]
question_len =(max_sequence_length - len(title)- 4)// 2
question = question[:question_len]
answer_len = max_sequence_length - len(title)- len(question)- 4
answer = answer[:answer_len]
title_ids = tokenizer.convert_tokens_to_ids(title)
 question_ids = tokenizer.convert_tokens_to_ids(question)
 answer_ids = tokenizer.convert_tokens_to_ids(answer)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 token_ids = cls_ids + title_ids + sep_ids + question_ids + sep_ids + answer_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
segments = [0]*(len(title_ids)+len(question_ids)+3)+ [1]*(len(answer_ids)+1)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 25
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 100000
HOMOGRAPHY_CONFIDENCE = 0.99<define_variables>
from sklearn.model_selection import GroupKFold
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 2])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(375.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 3072
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'] )<categorify>
test_pred_datas = {}
test_pred_weights = {}
for column in target_columns:
test_pred_datas[column] = np.zeros(len(test_df))
test_pred_weights[column] = 0.0
def add_test_pred_data(prediction, columns, weight):
for column_idx, column in enumerate(columns):
test_pred_datas[column] += weight * prediction[:, column_idx]
test_pred_weights[column] += weight
validation_datas = {}
validation_counts = {}
for column in target_columns:
validation_datas[column] = np.zeros(len(train_df))
validation_counts[column] = np.zeros(len(train_df))
def add_validation_data(prediction, columns, idx):
for column_idx, column in enumerate(columns):
validation_datas[column][idx] += prediction[:, column_idx]
validation_counts[column][idx] += 1.0
def mean_spearmanr_correlation_score(y_true, y_pred):
num_labels = y_pred.shape[1]
score = np.nanmean([ spearmanr(y_pred[:, idx], y_true[:, idx] ).correlation for idx in range(num_labels)])
 return score
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, features, labels = None):
self.inputs = inputs
self.features = features
if labels is not None:
self.labels = labels
else:
self.labels = None
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
input_features = self.features[idx]
if self.labels is not None:
input_labels = self.labels[idx]
return input_ids, input_masks, input_segments, input_features, input_labels
return input_ids, input_masks, input_segments, input_features
def __len__(self):
return len(self.inputs[0] )
def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')) )<import_modules>
def model_test(test_inputs, x_test, label_columns, weight_files):
batch_size = 6
test_dataset = QuestDataset(test_inputs, x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 num_features = x_test.shape[1]
num_labels = len(label_columns)
 for fname in weight_files:
model = NeuralNet3(num_features, num_labels, pretrained_bert)
 model.cuda()
model.load_state_dict(torch.load(fname))
model.eval()
test_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in test_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, label_columns, 1.0)
 del model
torch.cuda.empty_cache()
gc.collect()
del test_dataset, test_loader
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 34.5
MAX_REPROJECTION_ERROR = 6.5
MAX_RANSAC_ITERATIONS = 8000000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<install_modules>
def model_validation(train_inputs, x_train, label_columns, weight_files):
if len(kfold_split)!= len(weight_files):
return
batch_size = 6
num_features = x_test.shape[1]
num_labels = len(label_columns)
 for k,(train_idx, valid_idx)in enumerate(kfold_split):
x_train_valid = x_train[valid_idx]
train_inputs_valid = [x[valid_idx] for x in train_inputs]
valid_dataset = QuestDataset(train_inputs_valid, x_train_valid, None)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = NeuralNet3(num_features, num_labels, pretrained_bert)
 model.cuda()
fname = weight_files[k]
model.load_state_dict(torch.load(fname))
model.eval()
valid_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in valid_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 add_validation_data(valid_preds_fold, label_columns, valid_idx)
 del model, valid_dataset, valid_loader
torch.cuda.empty_cache()
gc.collect()
!pip install.. /input/efficientnetpytorch/EfficientNet-PyTorch-master<import_modules>
def model_test_validation(label_columns, train_inputs, test_inputs, x_train, x_test, weight_files):
if len(kfold_split)!= len(weight_files):
return
batch_size = 6
num_features = x_test.shape[1]
num_labels = len(label_columns)
 test_dataset = QuestDataset(test_inputs, x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 for k,(train_idx, valid_idx)in enumerate(kfold_split):
fname = weight_files[k]
print(k+1, fname)
 x_train_valid = x_train[valid_idx]
train_inputs_valid = [x[valid_idx] for x in train_inputs]
valid_dataset = QuestDataset(train_inputs_valid, x_train_valid, None)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = NeuralNet3(num_features, num_labels, pretrained_bert)
 model.cuda()
model.load_state_dict(torch.load(fname))
model.eval()
valid_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in valid_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 add_validation_data(valid_preds_fold, label_columns, valid_idx)
 test_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in test_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, label_columns, 1.0)
 del model, valid_dataset, valid_loader
torch.cuda.empty_cache()
gc.collect()
del test_dataset, test_loader
import math
import heapq
import torch
import torch.nn as nn
from torch.cuda import amp
from efficientnet_pytorch import EfficientNet
from torch.nn import Parameter
import torch.nn.functional as F
from collections import OrderedDict
import pandas as pd
import numpy as np
from tqdm import tqdm
from sklearn.metrics.pairwise import cosine_similarity
import albumentations as A
from albumentations.pytorch import ToTensorV2
import os
import cv2
import glob
from shutil import copyfile
import multiprocessing
from torch.utils.data import Dataset, DataLoader
import time
import copy
import gc
import operator
import pathlib
import PIL
import pydegensac
from scipy import spatial
import tensorflow as tf<define_variables>
tokenizer = transformers.GPT2Tokenizer.from_pretrained(pretrained_bert )
DEBUG = False
train_data_dir = '/kaggle/input/landmark-recognition-2020/train/'
test_data_dir = '/kaggle/input/landmark-recognition-2020/test/'
train_csv_path = '/kaggle/input/landmark-recognition-2020/train.csv'
image_size = 640
embeddings_size = 512
batch_size = 30
num_workers = multiprocessing.cpu_count()
n_chunks = 200 if not DEBUG else 20
n_top_global = 5
n_top_local = 5
calculate_local = True
reranking = True
train_encodings_from_file = False
N_rerank_check = 10 if DEBUG else 5000
N_rerank_select = 5 if DEBUG else 300
reranking_th = 25
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<define_variables>
print('time: {}'.format(time_to_str(( time.time() -start_time_all), 'min')) )
def generate_test_df(data_dir, img_ext='.jpg'):
files = [f.split('/')[-1].replace(img_ext,'')for f in glob.glob(data_dir + f'**/*{img_ext}',recursive=True)]
return pd.DataFrame(files, columns=['id'])
 def my_collate(batch):
all_images = []
all_names = []
total_n_transforms = len(batch[0][0])
 for i in range(total_n_transforms):
curr_imgs_list = []
curr_name_list = []
for b in batch:
curr_imgs_list.append(b[0][i])
 curr_name_list.append(b[1][i])
 curr_imgs_list = torch.stack(curr_imgs_list)
 all_images.append(curr_imgs_list)
 all_names.append(curr_name_list)
 return [all_images, all_names]
class GLRDataset(Dataset):
def __init__(self, data_dir, transform, df_path=None, mapper=None, img_ext='.jpg'):
super().__init__()
self.df = pd.read_csv(df_path)if df_path is not None else generate_test_df(data_dir,img_ext=img_ext)
 self.data_dir = data_dir
self.transform = transform
self.img_ext = img_ext
self.n_classes = self.df['landmark_id'].nunique() if 'landmark_id' in self.df else None
self.n_samples = len(self.df)
 self.mapper = mapper
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name = self.df['id'].iloc[index]
f1,f2,f3 = img_name[:3]
img_path = os.path.join(self.data_dir, f1, f2, f3, img_name+self.img_ext)
 images = []
img_names = []
image = cv2.imread(img_path)
 for trans in self.transform:
augmented = trans(image=image)
 images.append(augmented['image'])
 img_names.append(img_name)
 return images, img_names
def get_dataloader(data_dir, transform=None, batch_size=8, df_path=None,
mapper=None, num_workers=2,
pin_memory=True, drop_last=True, collate_fn=None):
dataset = GLRDataset(data_dir=data_dir, df_path=df_path, mapper=mapper, transform=transform)
 sampler = None
data_loader = DataLoader(dataset=dataset,
batch_size=batch_size,
sampler=sampler,
pin_memory=pin_memory,
drop_last=drop_last,
collate_fn=collate_fn,
num_workers=num_workers)
 return data_loader, dataset
def load_ckp(checkpoint_fpath, model, optimizer=None, margin=None, remove_module=False):
checkpoint = torch.load(checkpoint_fpath)
 pretrained_dict = checkpoint['model']
model_state_dict = model.state_dict()
if remove_module:
new_state_dict = OrderedDict()
for k, v in pretrained_dict.items() :
name = k[7:]
new_state_dict[name] = v
pretrained_dict = new_state_dict
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_state_dict}
model_state_dict.update(pretrained_dict)
 model.load_state_dict(pretrained_dict)
 try:
optimizer.load_state_dict(checkpoint['optimizer'])
 except:
print('Cannot load optimizer params')
 if margin is not None and 'margin' in checkpoint:
margin.load_state_dict(checkpoint['margin'])
 epoch = checkpoint['epoch'] if 'epoch' in checkpoint else 0
idx_to_class = checkpoint['idx_to_class'] if 'idx_to_class' in checkpoint else {}
return model, optimizer, margin, epoch, idx_to_class
class EmbeddigsNet(nn.Module):
def __init__(self, model_name, embeddings_size):
super(EmbeddigsNet, self ).__init__()
self.effnet = EfficientNet.from_name(model_name)
 for param in self.effnet.parameters() :
param.requires_grad = True
num_ftrs = self.effnet._fc.in_features
self.effnet._fc = nn.Linear(num_ftrs, embeddings_size)
 def forward(self, x):
x = self.effnet(x)
 return x
def get_model(model_name, embeddings_size):
model = EmbeddigsNet(model_name, embeddings_size)
 return model
def calculate_top_n(sim_matrix,best_top_n_vals,
best_top_n_idxs,
curr_zero_idx=0,
n=10):
n_rows, n_cols = sim_matrix.shape
total_matrix_vals = sim_matrix
total_matrix_idxs = np.tile(np.arange(n_rows ).reshape(n_rows,1),(1,n_cols)).astype(int)+ curr_zero_idx
if curr_zero_idx>0:
total_matrix_vals = np.vstack(( total_matrix_vals, best_top_n_vals))
total_matrix_idxs = np.vstack(( total_matrix_idxs, best_top_n_idxs))
res = np.argpartition(total_matrix_vals, -n, axis=0)[-n:]
res_vals = np.take_along_axis(total_matrix_vals, res, axis=0)
 res_idxs = np.take_along_axis(total_matrix_idxs, res, axis=0)
 del res, total_matrix_idxs, total_matrix_vals
return res_vals, res_idxs
def cosine_similarity_chunks(X, Y, n_chunks=5, top_n=5):
ch_sz = X.shape[0]//n_chunks
best_top_n_vals = None
best_top_n_idxs = None
num_chunks = 2 if DEBUG else n_chunks
for i in tqdm(range(num_chunks)) :
chunk = X[i*ch_sz:,:] if i==n_chunks-1 else X[i*ch_sz:(i+1)*ch_sz,:]
cosine_sim_matrix_i = cosine_similarity(chunk, Y)
 best_top_n_vals, best_top_n_idxs = calculate_top_n(cosine_sim_matrix_i,
best_top_n_vals,
best_top_n_idxs,
curr_zero_idx=(i*ch_sz),
n=top_n)
 print(f'Best similar vals shape {best_top_n_vals.shape}')
 print(f'Best similar idxs shape {best_top_n_idxs.shape}')
 return best_top_n_vals, best_top_n_idxs
def get_embeddings(model, test_dataloader, device, l2_norm=True):
generated_embeddings = {}
model.eval()
tqdm_test = tqdm(test_loader, total=int(len(test_loader)))
 with torch.no_grad() :
for batch_index, [data, images_ids] in enumerate(tqdm_test):
if DEBUG and batch_index>10:
break
logits = None
for dat in data:
dat = dat.to(device)
 logits_i = model(dat)
 logits_i = logits_i.cpu().numpy()
if logits is None:
logits = logits_i/len(data)
 else:
logits += logits_i/len(data)
 image_ids = images_ids[0]
for logits_i, image_id in zip(logits, image_ids):
generated_embeddings[image_id] = logits_i / np.linalg.norm(logits_i)if l2_norm else logits_i
return generated_embeddings
def get_max_conf_pred(curr_n_pred, curr_n_conf):
dict_conf_list = {}
dict_conf_sum = {}
for p, c in zip(curr_n_pred, curr_n_conf):
if p in dict_conf_list:
dict_conf_list[p].append(c)
 else:
dict_conf_list[p] = [c]
dict_conf_sum = { k : sum(v)for k, v in dict_conf_list.items() }
pred_i = max(dict_conf_sum, key=dict_conf_sum.get)
 conf_i = sum(dict_conf_list[pred_i])
 return pred_i, conf_i
def generate_submission_df(ids, predictions, confidences, th=0):
ids, predictions, confidences = list(ids), list(predictions), list(confidences)
 pred_conf_strings = [f'{int(p)} {c}' if c>=th else '' for p,c in zip(predictions, confidences)]
df = pd.DataFrame(list(zip(ids, pred_conf_strings)) , columns =['id', 'landmarks'])
 return df 
<normalization>
train_inputs = compute_input_title_question(train_df, tokenizer, max_sequence_length=512)
 test_inputs = compute_input_title_question(test_df, tokenizer, max_sequence_length=512)
 x_train = train_feature.values
x_test = test_feature.values
print(x_train.shape)
 print(x_test.shape)
 print(len(target_question_columns))
weight_files = [
model_weight_path1 ,
model_weight_path2 ,
model_weight_path3 ,
model_weight_path4 ,
model_weight_path5,
model_weight_path6,
]
model_test_validation(target_question_columns, train_inputs, test_inputs, x_train, x_test, weight_files)
 del x_train, x_test
del train_inputs, test_inputs
transform_no_aug = A.Compose([
A.Resize(image_size, image_size),
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2()
])
 transform_resize_d = A.Compose([
A.Resize(int(0.75*image_size), int(0.75*image_size)) ,
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2()
])
 transform_resize_u = A.Compose([
A.Resize(int(1.25*image_size), int(1.25*image_size)) ,
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2()
])
 transform_resize_u2 = A.Compose([
A.Resize(int(2*image_size), int(2*image_size)) ,
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2()
])
 transform_flip_lr = A.Compose([
A.Resize(image_size, image_size),
A.HorizontalFlip(always_apply=True, p=1),
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2()
] )<define_variables>
train_inputs = compute_input_arrays(train_df, tokenizer, max_sequence_length=512)
 test_inputs = compute_input_arrays(test_df, tokenizer, max_sequence_length=512)
 x_train = train_feature.values
x_test = test_feature.values
print(x_train.shape)
 print(x_test.shape)
 print(len(target_answer_columns))
weight_files = [
model_weight_path7,
model_weight_path8,
model_weight_path9,
model_weight_path10,
model_weight_path11,
model_weight_path12,
]
model_test_validation(target_answer_columns, train_inputs, test_inputs, x_train, x_test, weight_files)
 del x_train, x_test
del train_inputs, test_inputs
model_names = ['efficientnet-b3']
weights_paths = ['/kaggle/input/e3-weights/last.pt']
transforms_all = [transform_no_aug, transform_resize_u]
max_n_samples = 10000000 if DEBUG else 1000000<load_from_csv>
del train_feature, test_feature
del target_question_columns, target_answer_columns
df_train = pd.read_csv(train_csv_path)
 n_files = len(df_train)
 print(f'Number of training samples {n_files}' )<load_from_csv>
for column in target_columns:
print(np.sum(validation_counts[column]), np.min(validation_counts[column]), np.max(validation_counts[column]), column)
 validationTG = pd.DataFrame()
for column in target_columns:
preds = validation_datas[column]
count = validation_counts[column]
count = np.where(count < 0.5, 1.0, count)
 validationTG[column] = preds / count
mean_spearmanr_correlation_score(validationTG.values, train_df[target_columns].values)
 validationTG.head()
if n_files>max_n_samples:
copyfile('/kaggle/input/landmark-recognition-2020/sample_submission.csv', 'submission.csv')
 else:
models = []
for model_name, weights_path in zip(model_names, weights_paths):
print(model_name)
 model = get_model(model_name, embeddings_size=embeddings_size)
 model = model.to(device)
 model, _, _, _, _ = load_ckp(weights_path, model, remove_module=True)
 models.append(model)
 embeddings_all = []
train_embeddings_final = {}
test_embeddings_final = {}
embeddings_transform = []
if train_encodings_from_file:
data_dirs = [test_data_dir]
df = pd.read_csv('/kaggle/input/landmark-recognition-2020/train.csv')
 encd = np.load('/kaggle/input/encodings/encodings.npy')
 df_nms = pd.read_csv('/kaggle/input/encodings/names.csv')
 cl_nms = df_nms['id'].tolist()
tr_nms = df['id'].tolist()
enc_dict = {}
for v_idx ,(cl_nm, enc)in tqdm(enumerate(zip(cl_nms, encd)) , total=len(cl_nms)) :
if DEBUG:
if v_idx>100:
break
if cl_nm in tr_nms:
enc_dict[cl_nm] = enc
embeddings_transform.append(enc_dict)
 del tr_nms, cl_nms, df_nms, encd, df, enc_dict
else:
data_dirs = [train_data_dir, test_data_dir]
for data_dir in data_dirs:
test_loader, test_data = get_dataloader(data_dir=data_dir,
transform=transforms_all ,
batch_size=batch_size,
num_workers=num_workers,
collate_fn=my_collate,
drop_last=False)
 embeddings = {}
for model in models:
embeddings_i = get_embeddings(model, test_loader, device)
 if not embeddings:
embeddings = embeddings_i
else:
for k, v in embeddings_i.items() :
embeddings[k] = np.concatenate([embeddings[k], v])
 embeddings_transform.append(embeddings)
 train_embeddings, test_embeddings = embeddings_transform
df_train['encodings'] = df_train['id'].map(train_embeddings)
 df_train = df_train.dropna()
df_test = pd.DataFrame(list(test_embeddings.items()), columns=['id', 'encodings'])
 encodings_train = np.array(df_train['encodings'].tolist() , dtype=np.float32)
 encodings_valid = np.array(df_test['encodings'].tolist() , dtype=np.float32)
 print(encodings_train.shape )<define_variables>
mean_spearmanr_correlation_score(validationTG.values, train_df[target_columns].values )
if calculate_local:
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 6.0
MAX_RANSAC_ITERATIONS = 5_000_000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = ['input_image:0', 'input_scales:0', 'input_abs_thres:0']
LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],['boxes:0', 'features:0'])
 def get_image_path(subset, name):
return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],f'{name}.jpg')
 def load_image_tensor(image_path):
return tf.convert_to_tensor(np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count])
 train_matching_keypoints = np.array([train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints, train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum() )<split>
test_predsTG = pd.read_csv(f"{folder}sample_submission.csv")
 for column in target_columns:
preds = test_pred_datas[column]
weight = test_pred_weights[column]
test_predsTG[column] = preds / weight
test_predsTG.head()
if n_files<=max_n_samples:
best_top_n_vals, best_top_n_idxs = cosine_similarity_chunks(encodings_train, encodings_valid, n_chunks=n_chunks, top_n=n_top_global)
 p = np.argsort(-best_top_n_vals, axis=0)
 best_top_n_vals = np.take_along_axis(best_top_n_vals, p, axis=0)
 best_top_n_idxs = np.take_along_axis(best_top_n_idxs, p, axis=0)
 predictions_n = np.zeros_like(best_top_n_idxs)
 confidences_n = best_top_n_vals
filenames_n = []
for row_idx, row in enumerate(best_top_n_idxs):
df_prediction = df_train.iloc[row, :]
pred = np.array(df_prediction['landmark_id'].tolist())
 fnames = np.array(df_prediction['id'].tolist() , dtype=str)
 predictions_n[row_idx, :] = pred
filenames_n.append(fnames)
 filenames_n = np.array(filenames_n)
 test_images_names = df_test['id'].tolist()
predictions_final = np.zeros(confidences_n.shape[1])
 confidences_final = np.zeros(confidences_n.shape[1])
 if calculate_local:
test_kp_ds = []
for test_img_idx, test_img_name in enumerate(tqdm(test_images_names)) :
test_img_path = get_image_path('test', test_img_name)
 test_keypoints, test_descriptors = extract_local_features(test_img_path)
 curr_n_conf = []
if DEBUG and test_img_idx>20:
break
for train_img_idx, train_img_name in enumerate(filenames_n[:n_top_local, test_img_idx]):
train_img_path = get_image_path('train', train_img_name)
 train_keypoints, train_descriptors = extract_local_features(train_img_path)
 n_matching = get_num_inliers(test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 curr_n_conf.append(n_matching)
 curr_n_conf = np.array(curr_n_conf)
 curr_n_pred = predictions_n[:, test_img_idx]
pred_i, conf_i = get_max_conf_pred(curr_n_pred, curr_n_conf)
 if reranking:
test_kp_ds.append(( test_img_name, test_keypoints, test_descriptors, pred_i, conf_i))
predictions_final[test_img_idx] = pred_i
confidences_final[test_img_idx] = conf_i
if reranking:
print('Start reranking')
 reranking_eps = 0.01
N_rerank_check = min(N_rerank_check, len(test_kp_ds))
test_kp_ds.sort(key=lambda data: data[4], reverse=True)
 test_kp_ds_to_rerank = test_kp_ds[:N_rerank_check]
reranked_vals = []
reranked_vals_names = []
for r_i in tqdm(range(N_rerank_select)) :
im_i, tk_i, td_i, pr_i, c_i = test_kp_ds_to_rerank[r_i]
if im_i in reranked_vals_names:
continue
curr_rerank = []
reranked_vals.append(( im_i, pr_i, c_i))
reranked_vals_names.append(im_i)
 for r_j in range(r_i+1, N_rerank_check):
im_j, tk_j, td_j, pr_j, c_j = test_kp_ds_to_rerank[r_j]
if pr_i != pr_j:
continue
if im_j in reranked_vals_names:
continue
n_matching = get_num_inliers(tk_i, td_i, tk_j, td_j)
 if n_matching < reranking_th:
continue
curr_rerank.append(( im_j, pr_j, n_matching))
curr_rerank.sort(key=lambda data: data[2], reverse=True)
 for j_j, curr_val in enumerate(curr_rerank):
im_j_j, pr_j_j, c_j_j = curr_val
new_score = c_i -(reranking_eps*j_j)
 reranked_vals_names.append(im_j_j)
 reranked_vals.append(( im_j_j, pr_j_j, new_score))
for t_kp_ds in test_kp_ds:
im_i, tk_i, td_i, pr_i, c_i = t_kp_ds
if im_i in reranked_vals_names:
continue
reranked_vals.append(( im_i, pr_i, c_i))
test_images_names, predictions_final, confidences_final = zip(*reranked_vals)
 print('Reranking completed')
 else:
for j in range(confidences_n.shape[1]):
curr_n_pred = predictions_n[:, j]
curr_n_conf = confidences_n[:, j]
pred_i, conf_i = get_max_conf_pred(curr_n_pred, curr_n_conf)
 predictions_final[j] = pred_i
confidences_final[j] = conf_i
sub_csv = generate_submission_df(test_images_names, predictions_final, confidences_final)
 sub_csv.to_csv('submission.csv', index=False)
 print(len(sub_csv))
sub_csv.head()<install_modules>
del target_columns, input_columns
del test_pred_datas, test_pred_weights
del validation_datas, validation_counts
torch.cuda.empty_cache()
gc.collect()
!pip install.. /input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/glrec2020/efficientnet-1.1.0-py3-none-any.whl<import_modules>
folder = '.. /input/google-quest-challenge/'
pretrained_distilbert_base_uncased = ".. /input/pretrainedbertpytorch/pretrained-bert-pytorch/distilbert-base-uncased/"
universal_sentence_encoder_path = ".. /input/universalsentenceencoderlarge4/"
import operator
import gc
import pathlib
import shutil
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import backend as K
from scipy import spatial
import cv2
import efficientnet.tfkeras as efn
import math<define_search_space>
warnings.simplefilter('ignore' )
NUMBER_OF_CLASSES = 81313
IMAGE_SIZE = [384, 384]
LR = 0.0001<define_variables>
import torch
from torch.utils.data import TensorDataset, DataLoader, Dataset
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.utils.data.sampler import SubsetRandomSampler
from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR
from torch.nn.utils.weight_norm import weight_norm
from scipy.stats import spearmanr
NUM_EMBEDDING_DIMENSIONS = 1024
DATASET_DIR = '.. /input/landmark-image-train/train_encoded.csv'
TEST_IMAGE_DIR = '.. /input/landmark-recognition-2020/test'
TRAIN_IMAGE_DIR = '.. /input/landmark-recognition-2020/train'<load_pretrained>
import tensorflow as tf
import tensorflow_hub as hub
MODEL1 = get_model_B6()
MODEL2 = get_model_B7()
MODEL1.load_weights('.. /input/effb6-512-ep18/effb6model512-18.h5')
 MODEL1 = tf.keras.models.Model(inputs = MODEL1.input[0], outputs = MODEL1.layers[-4].output)
 MODEL2.load_weights('.. /input/effb7-512-ep12/effb7model512-12.h5')
 MODEL2 = tf.keras.models.Model(inputs = MODEL2.input[0], outputs = MODEL2.layers[-4].output )<define_variables>
print("transformers:", transformers.__version__ )
NUM_TO_RERANK = 3
NUM_PUBLIC_TEST_IMAGES = 10345<load_from_csv>
from radam import RAdam
final = inference_and_save_submission_csv(DATASET_DIR, TEST_IMAGE_DIR, TRAIN_IMAGE_DIR )<set_options>
SEED = 12345
plt.rcdefaults()
plt.style.use('dark_background')
 warnings.filterwarnings("ignore" )<define_variables>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
INPUT_PATH = os.path.join('.. ', 'input')
 DATASET_PATH = os.path.join(INPUT_PATH, 'landmark-recognition-2020')
 TRAIN_IMAGE_PATH = os.path.join(DATASET_PATH, 'train')
 TEST_IMAGE_PATH = os.path.join(DATASET_PATH, 'test')
 TRAIN_CSV_PATH = os.path.join(DATASET_PATH, 'train.csv')
 SUBMISSION_CSV_PATH = os.path.join(DATASET_PATH, 'sample_submission.csv' )<load_from_csv>
def read_pickle_from_file(pickle_file):
with open(pickle_file,'rb')as f:
x = pickle.load(f)
 return x
def write_pickle_to_file(pickle_file, x):
with open(pickle_file, 'wb')as f:
pickle.dump(x, f, pickle.HIGHEST_PROTOCOL )
train = pd.read_csv(TRAIN_CSV_PATH)
 print("training dataset has {} rows and {} columns".format(train.shape[0],train.shape[1]))
submission = pd.read_csv(SUBMISSION_CSV_PATH)
 print("submission dataset has {} rows and {} columns
".format(submission.shape[0],submission.shape[1]))<define_variables>
train_df = pd.read_csv(f"{folder}train.csv")
 train_df.shape
print(Fore.BLUE + f"{'---'*20}
Mapping for Training Data
{'---'*20}")
 data_label_dict = {'image': [], 'target': []}
for i in tqdm(range(train.shape[0])) :
data_label_dict['image'].append(
 TRAIN_IMAGE_PATH + '/' +
train['id'][i][0] + '/' +
train['id'][i][1]+ '/' +
train['id'][i][2]+ '/' +
train['id'][i] + ".jpg")
 data_label_dict['target'].append(
 train['landmark_id'][i])
 train_pathlabel = pd.DataFrame(data_label_dict)
 print(train_pathlabel.head())
 print(Fore.BLUE + f"{'---'*20}
Mapping for Test Data
{'---'*20}",Style.RESET_ALL)
 data_label_dict = {'image': []}
for i in tqdm(range(submission.shape[0])) :
data_label_dict['image'].append(
 TEST_IMAGE_PATH + '/' +
submission['id'][i][0] + '/' +
submission['id'][i][1]+ '/' +
submission['id'][i][2]+ '/' +
submission['id'][i] + ".jpg")
 test_pathlabel = pd.DataFrame(data_label_dict)
 print(test_pathlabel.head() )<count_unique_values>
test_df = pd.read_csv(f"{folder}test.csv")
 test_df.shape
print("There are", train.landmark_id.nunique() , "landmarks in the training dataset" )<count_values>
target_columns = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = [
'question_title',
'question_body',
'answer'
]
train.landmark_id.value_counts()<import_modules>
for df in [train_df, test_df]:
df['domcom'] = df['url'].apply(lambda s: s.split('://')[1].split('/')[0].split('.'))
df['dom_cnt'] = df['domcom'].apply(lambda s: len(s))
df['domcom'] = df['domcom'].apply(lambda s: s + ['none', 'none'])
 for ii in range(0,4):
df['url_'+str(ii)] = df['domcom'].apply(lambda s: s[ii])
 df.drop('domcom', axis = 1, inplace = True )
import copy
import csv
import gc
import operator
import os
import pathlib
import shutil
import numpy as np
import PIL
import pydegensac
from scipy import spatial
import tensorflow as tf<define_variables>
train_feature = pd.DataFrame()
test_feature = pd.DataFrame()
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv' )<define_variables>
features = ['url_0', 'category']
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(train_df[features])
 encode_train = enc.transform(train_df[features] ).toarray()
encode_test = enc.transform(test_df[features] ).toarray()
columns = ['encode_{}'.format(i)for i in range(encode_train.shape[1])]
encode_train = pd.DataFrame(encode_train, columns=columns)
 train_feature = pd.concat(( train_feature, encode_train), axis=1)
 encode_test = pd.DataFrame(encode_test, columns=columns)
 test_feature = pd.concat(( test_feature, encode_test), axis=1)
 del encode_train, encode_test, enc
NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1<define_variables>
class QuestBertDataset(torch.utils.data.Dataset):
def __init__(self, inputs, labels = None):
self.inputs = inputs
if labels is not None:
self.labels = labels
else:
self.labels = None
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
if self.labels is not None:
input_labels = self.labels[idx]
return input_ids, input_masks, input_labels
return input_ids, input_masks
def __len__(self):
return len(self.inputs[0] )
NUM_TO_RERANK = 3
TOP_K = 3 
<init_hyperparams>
def compute_input_text(texts, tokenizer, max_sequence_length=512):
input_ids = []
input_masks = []
for text in texts:
text = tokenizer.tokenize(text)
 text_ids = tokenizer.convert_tokens_to_ids(text)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 if(len(text_ids)+2)> max_sequence_length:
text_ids = text_ids[:max_sequence_length-2]
token_ids = cls_ids + text_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long()
]
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8500000
HOMOGRAPHY_CONFIDENCE = 0.99<load_pretrained>
def get_bert_feature(tokenizer, model, df, column):
print(column)
 inputs = compute_input_text(df[column].values, tokenizer)
 dataset = QuestBertDataset(inputs, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size=6, shuffle=False, drop_last=False)
 preds_fold = []
model.eval()
with torch.no_grad() :
for ids, masks in tqdm(loader):
ids = ids.cuda()
masks = masks.cuda()
outputs = model(input_ids=ids, attention_mask=masks)
 x = outputs[0][:, 0, :]
preds_fold.extend(x.cpu().data.numpy().tolist())
 preds_fold = np.array(preds_fold)
 return preds_fold
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]<feature_engineering>
tokenizer = transformers.DistilBertTokenizer.from_pretrained(pretrained_distilbert_base_uncased)
 model = transformers.DistilBertModel.from_pretrained(pretrained_distilbert_base_uncased)
 model.cuda()
print('train:')
 train_question_title_dense = get_bert_feature(tokenizer, model, train_df, 'question_title')
 train_question_body_dense = get_bert_feature(tokenizer, model, train_df, 'question_body')
 train_answer_dense = get_bert_feature(tokenizer, model, train_df, 'answer')
 print('test:')
 test_question_title_dense = get_bert_feature(tokenizer, model, test_df, 'question_title')
 test_question_body_dense = get_bert_feature(tokenizer, model, test_df, 'question_body')
 test_answer_dense = get_bert_feature(tokenizer, model, test_df, 'answer')
 del tokenizer, model
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'] )<categorify>
embed = hub.load(universal_sentence_encoder_path)
 embeddings_train = {}
embeddings_test = {}
for text in input_columns:
print(text)
 train_text = train_df[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
test_text = test_df[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_train_emb = []
curr_test_emb = []
batch_size = 4
ind = 0
while ind*batch_size < len(train_text):
curr_train_emb.append(embed(train_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
ind = 0
while ind*batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
embeddings_train[text + '_embedding'] = np.vstack(curr_train_emb)
 embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
gc.collect()
def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'} )<load_from_csv>
embeddings_train_df = pd.DataFrame()
embeddings_test_df = pd.DataFrame()
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 abs_dist = lambda x, y: np.abs(x-y ).sum(axis=1)
 sum_dist = lambda x, y:(x+y ).sum(axis=1)
 dist_columns = [
['question_title_embedding', 'answer_embedding'],
['question_body_embedding', 'answer_embedding'],
['question_body_embedding', 'question_title_embedding'],
]
for i, columns in enumerate(dist_columns):
embeddings_train_df[f'l2_dist_embedding_{i}'] = l2_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_train_df[f'cos_dist_embedding_{i}'] = cos_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_train_df[f'abs_dist_embedding_{i}'] = abs_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_train_df[f'l2_dist_embedding_{i}'] = sum_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_test_df[f'l2_dist_embedding_{i}'] = l2_dist(embeddings_test[columns[0]], embeddings_test[columns[1]])
 embeddings_test_df[f'cos_dist_embedding_{i}'] = cos_dist(embeddings_test[columns[0]], embeddings_test[columns[1]])
 embeddings_test_df[f'abs_dist_embedding_{i}'] = abs_dist(embeddings_test[columns[0]], embeddings_test[columns[1]])
 embeddings_test_df[f'l2_dist_embedding_{i}'] = sum_dist(embeddings_test[columns[0]], embeddings_test[columns[1]] )
def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<import_modules>
x_train = np.hstack([item for k, item in embeddings_train.items() ] + [train_question_title_dense, train_question_body_dense, train_answer_dense, embeddings_train_df.values, train_feature.values])
 x_test = np.hstack([item for k, item in embeddings_test.items() ] + [test_question_title_dense, test_question_body_dense, test_answer_dense, embeddings_test_df.values, test_feature.values])
 y_train = train_df[target_columns].values
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 5
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 7000000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<define_variables>
del train_feature, test_feature
del embeddings_train, embeddings_test
del embeddings_train_df, embeddings_test_df
del train_question_title_dense, train_question_body_dense, train_answer_dense
del test_question_title_dense, test_question_body_dense, test_answer_dense
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8000000
HOMOGRAPHY_CONFIDENCE = 0.99
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()<import_modules>
def mean_spearmanr_correlation_score(y_true, y_pred):
num_labels = y_pred.shape[1]
return np.nanmean([ spearmanr(y_pred[:, idx], y_true[:, idx] ).correlation for idx in range(num_labels)] )
import os
import pprint as pp
import math as ma
import numpy as np
import pandas as pd
from plotnine import *
from plotnine.data import *
from sklearn import base
from sklearn import tree
from sklearn import naive_bayes as nb
from sklearn import svm as svm
from sklearn import neighbors as nei
from sklearn import preprocessing as pre
from sklearn import feature_extraction as fe
from sklearn import pipeline as pi
from sklearn import model_selection as ms
from sklearn import ensemble as ens
from sklearn.metrics import make_scorer
import xgboost as xgb
import bayes_opt as bo
import category_encoders as ce<define_variables>
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, x, y):
self.x = x
self.y = y
def __len__(self):
return len(self.x)
 def __getitem__(self, idx):
if self.y is not None:
return self.x[idx], self.y[idx]
else:
return self.x[idx]
verbose = 10
seed = round(ma.pi * 10**4 )<create_dataframe>
class NeuralNet(nn.Module):
def __init__(self, num_features, num_labels):
super(NeuralNet, self ).__init__()
self.num_features = num_features
self.num_labels = num_labels
self.classifier = nn.Sequential(
 nn.Linear(self.num_features, self.num_features),
nn.Dropout(0.2),
nn.Linear(self.num_features, self.num_features),
nn.Linear(self.num_features, self.num_labels), 
)
 def forward(self, features):
return self.classifier(features )
def cols_types_to_df(df, cols):
return pd.DataFrame(df[cols].dtypes.values.reshape(1, -1), columns=cols, index=['dtype'] )<prepare_x_and_y>
test_pred_datas = {}
test_pred_weights = {}
def init_test_pred_data() :
for column in target_columns:
test_pred_datas[column] = np.zeros(len(test_df))
test_pred_weights[column] = 0.0
def add_test_pred_data(prediction, columns, weight):
for column_idx, column in enumerate(columns):
test_pred_datas[column] += weight * prediction[:, column_idx]
test_pred_weights[column] += weight
ID_COL = 'id'
TARGET_COL = 'target'
SPECIAL_COL_NAMES = [ID_COL, TARGET_COL]
train = pd.read_csv('.. /input/train.csv' ).drop(ID_COL, axis=1)
 headers = train.columns
X = train.drop(TARGET_COL, axis=1)
 y = train[TARGET_COL]
train.head(10 )<count_values>
validation_datas = {}
validation_counts = {}
def init_validation_data() :
for column in target_columns:
validation_datas[column] = np.zeros(len(train_df))
validation_counts[column] = np.zeros(len(train_df))
def add_validation_data(prediction, columns, idx):
for column_idx, column in enumerate(columns):
validation_datas[column][idx] += prediction[:, column_idx]
validation_counts[column][idx] += 1.0
target_counts = train[TARGET_COL].value_counts()
target_ratio = target_counts[0]/target_counts[1]
print("Proporção entre classes 0 e 1: %s" % target_ratio )<drop_column>
init_test_pred_data()
init_validation_data()
def drop_cols_func(X):

 return X.drop(columns=CALC_COL_NAMES)
 drop_cols = pre.FunctionTransformer(func=drop_cols_func, validate=False)
 drop_cols.transform(X ).head()<categorify>
n_splits = 5
cv = GroupKFold(n_splits=n_splits)
 kfold_split = list(cv.split(X=train_df.question_body, groups=train_df.question_body))
one_hot = ce.OneHotEncoder(cols=CAT_COL_NAMES, drop_invariant=True, handle_unknown='impute')
 one_hot.fit_transform(X ).head()<compute_test_metric>
n_epochs = 50
patience = 5
scores = []
batch_size = 32
num_features = x_train.shape[1]
num_labels = y_train.shape[1]
for k,(train_idx, valid_idx)in enumerate(kfold_split):
print("k:", k+1)
 x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
train_dataset = QuestDataset(x_train_train, y_train_train)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)
 x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
valid_dataset = QuestDataset(x_train_valid, y_train_valid)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = NeuralNet(num_features, num_labels)
 model.cuda()
best_weights = copy.deepcopy(model.state_dict())
 loss_fn = nn.BCEWithLogitsLoss()
optimizer = RAdam(model.parameters() , lr=2e-5)
 schduler = CyclicScheduler2(min_lr=2e-6, max_lr=2e-5, period=20, warm_start=0, max_decay=0.9)
 min_loss = np.inf
counter = 0
for epoch in range(n_epochs):
lr = schduler(epoch)
 adjust_learning_rate(optimizer, lr)
 model.train()
train_loss = []
for features, labels in train_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 loss = loss_fn(y_pred, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss.append(loss.item())
 avg_train_loss = np.mean(train_loss)
 model.eval()
val_loss = []
with torch.no_grad() :
for features, labels in valid_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 loss = loss_fn(y_pred, labels)
 val_loss.append(loss.item())
 avg_val_loss = np.mean(val_loss)
 if avg_val_loss < min_loss:
min_loss = avg_val_loss
counter = 0
best_weights = copy.deepcopy(model.state_dict())
 else:
counter += 1
if counter >= patience and epoch > 12:
break
model.load_state_dict(best_weights)
 model.eval()
valid_preds_fold = []
valid_true_fold = []
with torch.no_grad() :
for features, labels in valid_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_true_fold.extend(labels.cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 valid_true_fold = np.array(valid_true_fold)
 add_validation_data(valid_preds_fold, target_columns, valid_idx)
 score = mean_spearmanr_correlation_score(valid_preds_fold, valid_true_fold)
 print('Score:', score)
 scores.append(score)
 test_preds_fold = []
test_dataset = QuestDataset(x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 with torch.no_grad() :
for features in test_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, target_columns, 1.0)
 del model, optimizer, loss_fn, best_weights
del train_dataset, train_loader
del valid_dataset, valid_loader
del test_dataset, test_loader
torch.cuda.empty_cache()
gc.collect()
print('================')
 print('Mean score:', np.mean(scores))
def gini(actual, pred):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(actual, pred):
return gini(actual, pred[:,1])/ gini(actual, actual)
 gini_normalized_scorer = make_scorer(score_func=gini_normalized, greater_is_better=True, needs_proba=True)
 def cv_metric(estimator, X, y, fit_params=None):
pred = ms.cross_val_predict(estimator, X, y, cv=ms.StratifiedKFold(n_splits=3, random_state=seed), verbose=verbose, fit_params=fit_params, method='predict_proba')
 return gini_normalized(y, pred)
 def xgb_metric(pred, dtrain):
print('xgb_gini_normalized')
 return 'gini', gini_normalized(dtrain.get_labels() , pred)
 xgb_fit_params = {
'clf__eval_metric': xgb_metric
}<choose_model_class>
validation0 = pd.DataFrame()
for column in target_columns:
preds = validation_datas[column]
count = validation_counts[column]
count = np.where(count < 0.5, 1.0, count)
 validation0[column] = preds / count
xgb_params = {
'objective': 'binary:logistic'
}
def pipeline() :
steps = [('drop_cols', drop_cols), 
('one_hot', one_hot), 
('clf', xgb.XGBClassifier(random_state=seed)) ]
return pi.Pipeline(steps)
 pipe = pipeline()<split>
mean_spearmanr_correlation_score(train_df[target_columns].values, validation0.values )
X_train, X_test, y_train, y_test = ms.train_test_split(X, y, test_size=0.33, random_state=seed )<train_on_grid>
test_preds0 = pd.read_csv(f"{folder}sample_submission.csv")
 for column in target_columns:
output = test_pred_datas[column] / test_pred_weights[column]
test_preds0[column] = output
<load_from_csv>
LSTM_UNITS = 512
DENSE_HIDDEN_UNITS = 6 * LSTM_UNITS
class MODEL_v001(nn.Module):
def __init__(self, num_features, num_labels):
super().__init__()
self.lstm1 = nn.LSTM(num_features , LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True,dropout=0.2)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linearnorm = nn.LayerNorm(DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linearnorm2 = nn.LayerNorm(DENSE_HIDDEN_UNITS)
 self.linear_sub_out = nn.Linear(DENSE_HIDDEN_UNITS, num_labels)
 def forward(self, x, lengths=None):
h_lstm1, _ = self.lstm1(x)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool1 = torch.mean(h_lstm1, 1)
 avg_pool2 = torch.mean(h_lstm2, 1)
 max_pool2, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( avg_pool1, max_pool2, avg_pool2), 1)
 h_conc_linear1 = self.linearnorm(F.relu(self.linear1(h_conc)))
 h_conc_linear2 = self.linearnorm2(F.relu(self.linear2(h_conc)))
 hidden = h_conc + h_conc_linear1 + h_conc_linear2
out = self.linear_sub_out(hidden)
 return out
if os.path.isfile('cache.csv'):
best_configs = pd.read_csv('cache.csv' ).sort_values(by='target', ascending=False)
 best_configs = best_configs.head(5 ).drop(columns='target' ).to_dict(orient='list')
 pp.pprint(best_configs)
 else:
best_configs = {'colsample_bylevel': [1.0, 1.0, 1.0, 1.0, 0.6210536972328008],
'colsample_bytree': [0.1639149309112309, 0.4719120160563106, 0.2738881448393379, 0.9597000000000002, 0.4998605743150732],
'gamma': [9.311242926501441, 9.106399343640415, 9.84350195229832, 1.3079, 8.495594735754976],
'learning_rate': [0.1, 0.1, 0.1, 0.1, 0.3],
'max_delta_step': [0.0, 0.0, 0.0, 0.0, 9.937144736135796],
'max_depth': [3.280267137915809, 3.744769687598659, 3.079866163690399, 3.8448, 3.0243950675489177],
'min_child_weight': [1.0, 1.0, 1.0, 1.0, 1.0],
'n_estimators': [496.3697387347585, 496.70662978009966, 362.50656654507236, 499.9892, 350.3584899983292],
'reg_alpha': [9.79619399515934, 0.7119072322272911, 0.05797462113553141, 8.128199999999998, 4.196426179286616],
'reg_lambda': [1.0, 1.0, 1.0, 1.0, 9.729243825652572],
'scale_pos_weight': [2.091440738145689, 1.7334510787353241, 1.327362597470453, 2.6812, 1.072600468578576],
'subsample': [0.7142488578466437, 0.9885285182085388, 0.7352809336635373, 0.9199, 0.8611579899201934]} 
<init_hyperparams>
x_train = np.expand_dims(x_train, 1)
 x_test = np.expand_dims(x_test, 1 )
def to_xgb_params(**kwargs):
params = {}
params['clf__max_depth'] = int(kwargs['max_depth'])
 params['clf__learning_rate'] = max(kwargs['learning_rate'], 0)
 params['clf__n_estimators'] = int(kwargs['n_estimators'])
 params['clf__min_child_weight'] = int(kwargs['min_child_weight'])
 params['clf__max_delta_step'] = int(kwargs['max_delta_step'])
 params['clf__subsample'] = max(min(kwargs['subsample'], 1), 0)
 params['clf__colsample_bytree'] = max(min(kwargs['colsample_bytree'], 1), 0)
 params['clf__colsample_bylevel'] = max(min(kwargs['colsample_bylevel'], 1), 0)
 params['clf__gamma'] = max(kwargs['gamma'], 0)
 params['clf__reg_alpha'] = max(kwargs['reg_alpha'], 0)
 params['clf__scale_pos_weight'] = max(kwargs['scale_pos_weight'], 0.001)
 return params
def xgb_cv(**kwargs):
xgb = base.clone(pipe)
 xgb.set_params(**to_xgb_params(**kwargs))
return cv_metric(xgb, X_train, y_train, fit_params=xgb_fit_params)
 bo_opt = bo.BayesianOptimization(xgb_cv, {'max_depth':(3, 15),
'learning_rate':(0.1, 0.0001),
'n_estimators':(100, 500),
'min_child_weight':(1, 1),
'max_delta_step':(0, 10),
'subsample':(0.5, 1.0),
'colsample_bytree':(0.1, 1.0),
'colsample_bylevel':(0.1, 1.0),
'gamma':(0.0, 10.0),
'reg_alpha':(0.0, 10.0),
'reg_lambda':(1.0, 10.0),
'scale_pos_weight':(1.0, target_ratio),
},
random_state=seed)

best_config = {'colsample_bylevel': 0.45879276636748534,
'colsample_bytree': 0.7824778832374746,
'gamma': 5.489188804299045,
'learning_rate': 0.1,
'max_delta_step': 1.5254539296007386,
'max_depth': 3.4967941454908598,
'min_child_weight': 1.0,
'n_estimators': 461.216046669665,
'reg_alpha': 8.531590560075347,
'reg_lambda': 9.89766553793669,
'scale_pos_weight': 1.2379042177844632,
'subsample': 0.798095844721928}
opt_params = to_xgb_params(**best_config)
 opt_pipe = base.clone(pipe)
 opt_pipe.set_params(**opt_params )<train_model>
init_test_pred_data()
init_validation_data()
pipe.fit(X_train, y_train, **xgb_fit_params)
 print('Configuração padrão: {}'.format(gini_normalized(y_test, pipe.predict_proba(X_test))))
opt_pipe.fit(X_train, y_train, **xgb_fit_params)
 print('Configuração otimizada: {}'.format(gini_normalized(y_test, opt_pipe.predict_proba(X_test))))<train_model>
n_splits = 5
cv = GroupKFold(n_splits=n_splits)
 kfold_split = list(cv.split(X=train_df.question_body, groups=train_df.question_body))
class FitParamsWrapper(base.BaseEstimator, base.ClassifierMixin):
def __init__(self, estimator, fit_params):
self.estimator = estimator
self.fit_params = fit_params
def fit(self, X, y=None):
self.estimator.fit(X, y, **self.fit_params)
 return self
def predict(self, X, y=None):
return self.estimator.predict(X)
 def predict_proba(self, X, y=None):
return self.estimator.predict_proba(X)
 def score(self, X, y=None):
return self.estimator.score(X, y )<save_to_csv>
n_epochs = 8
patience = 4
scores = []
batch_size = 8
num_features = x_train.shape[2]
num_labels = y_train.shape[1]
for k,(train_idx, valid_idx)in enumerate(kfold_split):
print("k:", k+1)
 x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
train_dataset = torch.utils.data.TensorDataset(
 torch.from_numpy(x_train[train_idx]),
torch.from_numpy(y_train[train_idx])
)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)
 valid_dataset = torch.utils.data.TensorDataset(
 torch.from_numpy(x_train[valid_idx]),
torch.from_numpy(y_train[valid_idx])
)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = MODEL_v001(num_features, num_labels)
 model.cuda()
best_weights = copy.deepcopy(model.state_dict())
 loss_fn = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters())
 min_loss = np.inf
counter = 0
for epoch in range(n_epochs):
model.train()
train_loss = []
for features, labels in train_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 loss = loss_fn(y_pred, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss.append(loss.item())
 avg_train_loss = np.mean(train_loss)
 model.eval()
val_loss = []
with torch.no_grad() :
for features, labels in valid_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 loss = loss_fn(y_pred, labels)
 val_loss.append(loss.item())
 avg_val_loss = np.mean(val_loss)
 if avg_val_loss < min_loss:
min_loss = avg_val_loss
counter = 0
best_weights = copy.deepcopy(model.state_dict())
 else:
counter += 1
if counter >= patience:
break
model.load_state_dict(best_weights)
 model.eval()
valid_preds_fold = []
valid_true_fold = []
with torch.no_grad() :
for features, labels in valid_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_true_fold.extend(labels.cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 valid_true_fold = np.array(valid_true_fold)
 add_validation_data(valid_preds_fold, target_columns, valid_idx)
 score = mean_spearmanr_correlation_score(valid_preds_fold, valid_true_fold)
 print('Score:', score)
 scores.append(score)
 test_preds_fold = []
test_dataset = QuestDataset(x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 with torch.no_grad() :
for features in test_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, target_columns, 1.0)
 del model, optimizer, loss_fn, best_weights
del train_dataset, train_loader
del valid_dataset, valid_loader
del test_dataset, test_loader
torch.cuda.empty_cache()
gc.collect()
print('================')
 print('Mean score:', np.mean(scores))
opt_pipe.fit(X, y, **xgb_fit_params)
 test = pd.read_csv('.. /input/test.csv')
 X_test = test[X.columns]
targets = opt_pipe.predict_proba(X_test)
 targets = pd.DataFrame({'id': test.id, 'target': targets[:,1]})
 targets.to_csv('submission.csv', index=False )<import_modules>
validationLSTM = pd.DataFrame()
for column in target_columns:
preds = validation_datas[column]
count = validation_counts[column]
count = np.where(count < 0.5, 1.0, count)
 validationLSTM[column] = preds / count
import numpy as np
import pandas as pd
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.metrics import log_loss
from sklearn.model_selection import StratifiedKFold
import os
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgb<feature_engineering>
mean_spearmanr_correlation_score(validationLSTM.values, train_df[target_columns].values )
def get_features_from_mjd(df):
df['mjd_detected'] = np.NaN
df.loc[df.detected == 1, 'mjd_detected'] = df.loc[df.detected == 1, 'mjd']
gr_mjd = df.groupby('object_id' ).mjd_detected
df['mjd_diff'] = gr_mjd.transform('max')- gr_mjd.transform('min')
 return df
def passbandSplit(df):
df["flux_0"] = df[df.passband==0].flux
df["flux_1"] = df[df.passband==1].flux
df["flux_2"] = df[df.passband==2].flux
df["flux_3"] = df[df.passband==3].flux
df["flux_4"] = df[df.passband==4].flux
df["flux_5"] = df[df.passband==5].flux
df["abs_flux_1"] = np.abs(df["flux_1"])
 df["abs_flux_2"] = np.abs(df["flux_2"])
 df["abs_flux_3"] = np.abs(df["flux_3"])
 df["abs_flux_4"] = np.abs(df["flux_4"])
 df["abs_flux_5"] = np.abs(df["flux_5"])
 df["flux_0_err"] = df[df.passband==0].flux_err
df["flux_1_err"] = df[df.passband==1].flux_err
df["flux_4_err"] = df[df.passband==4].flux_err
df["flux_5_err"] = df[df.passband==5].flux_err
df['flux_ratio_sq_0'] = np.power(df['flux_0'] / df['flux_0_err'], 2.0)
 df['flux_ratio_sq_1'] = np.power(df['flux_1'] / df['flux_1_err'], 2.0)
 df['flux_ratio_sq_4'] = np.power(df['flux_4'] / df['flux_4_err'], 2.0)
 df['flux_ratio_sq_5'] = np.power(df['flux_5'] / df['flux_5_err'], 2.0)
 df['flux_by_flux_ratio_sq_0'] = df['flux_0'] * df['flux_ratio_sq_0']
df['flux_by_flux_ratio_sq_1'] = df['flux_1'] * df['flux_ratio_sq_1']
df['flux_by_flux_ratio_sq_5'] = df['flux_5'] * df['flux_ratio_sq_5']
return df
def sabun_henkaritsu_cumsum(df):
gr_df = df.groupby(['object_id','passband'])
 gr_flux = gr_df['flux']
df["flux_henkaritsu"] = gr_flux.pct_change()
df["flux_sabun_diff"] = gr_flux.transform('max')- gr_flux.transform('min')
 gr_mag = gr_df['magnitude']
df["mag_sabun_diff"] = gr_mag.transform('max')- gr_mag.transform('min')
 df["mag_sabun"] = gr_mag.diff()
gr_fl_ratio_sq = gr_df['flux_ratio_sq']
df["fl_ratio_sabun"] = gr_fl_ratio_sq.diff()
df["flux_detected"] = df[df.detected==1].flux
df["dtd_fl_by_mjd_dif"] = df["flux_detected"] / df['mjd_diff']
df["dtd_magnitude"] = df[df.detected==1].magnitude
df["dtd_magnitude_1"] = df[df.detected==1].magnitude_1
df["dtd_magnitude_2"] = df[df.detected==1].magnitude_2
df["dtd_magnitude_3"] = df[df.detected==1].magnitude_3
gr_ob = df.groupby('object_id')
 df['dtdmag_diff'] = gr_ob.dtd_magnitude.transform('max')- gr_ob.dtd_magnitude.transform('min')
 df['dtdmag_diff_by_mjd'] = df['dtdmag_diff'] / df['mjd_diff']
df['dtdmag_1_diff'] = gr_ob.dtd_magnitude_1.transform('max')- gr_ob.dtd_magnitude_1.transform('min')
 df['dtdmag_2_diff'] = gr_ob.dtd_magnitude_2.transform('max')- gr_ob.dtd_magnitude_2.transform('min')
 df['dtdmag_3_diff'] = gr_ob.dtd_magnitude_3.transform('max')- gr_ob.dtd_magnitude_3.transform('min')
 df['dtdmag_1_diff_by_mjd'] = df['dtdmag_1_diff'] / df['mjd_diff']
df['dtdmag_2_diff_by_mjd'] = df['dtdmag_2_diff'] / df['mjd_diff']
df['dtdmag_3_diff_by_mjd'] = df['dtdmag_3_diff'] / df['mjd_diff']
del df['dtdmag_1_diff'], df['dtdmag_2_diff'], df['dtdmag_3_diff']
df["dtdmag_pct"] = gr_ob.dtd_magnitude.pct_change()
return df<feature_engineering>
test_predsLSTM = pd.read_csv(f"{folder}sample_submission.csv")
 for column in target_columns:
output = test_pred_datas[column] / test_pred_weights[column]
test_predsLSTM[column] = output
def get_flux_decays(df):
df['flux_1_err_diff_per_mjd'] =(df['flux_1_err_max'] - df['flux_1_err_min'])/ df['mjd_diff_mean']
del df['flux_1_err_max'], df['flux_1_err_min']
df["flux_5_4_max"] =(df["flux_5_max"] - df["flux_4_max"])
 df["flux_4_3_max"] =(df["flux_4_max"] - df["flux_3_max"])
 df["flux_3_2_max"] =(df["flux_3_max"] - df["flux_2_max"])
 df["flux_2_1_max"] =(df["flux_2_max"] - df["flux_1_max"])
 df["flux_1_0_max"] =(df["flux_1_max"] - df["flux_0_max"])
 df["flux_5_0_mean"] =(df["flux_5_mean"] - df["flux_0_mean"])
 df["flux_3_0_mean"] =(df["flux_3_mean"] - df["flux_0_mean"])
 df["flux_4_3_std"] =(df["flux_4_std"] - df["flux_3_std"])
 df["flux_3_2_std"] =(df["flux_3_std"] - df["flux_2_std"])
 df["flux_2_1_std"] =(df["flux_2_std"] - df["flux_1_std"])
 df["flux_5_0_median"] =(df["flux_5_median"] - df["flux_0_median"])
 return df<load_from_csv>
del train_df, test_df
del x_train, x_test, y_train
del validation_datas, validation_counts
del test_pred_datas, test_pred_weights
train_set = pd.read_csv('.. /input/training_set.csv', dtype={"object_id": "object"})
 train_set_meta = pd.read_csv('.. /input/training_set_metadata.csv', dtype={"object_id": "object"})
 test_set_meta = pd.read_csv('.. /input/test_set_metadata.csv' )<feature_engineering>
data_dir = '.. /input/google-quest-challenge/'
train = pd.read_csv(path_join(data_dir, 'train.csv'))
test = pd.read_csv(path_join(data_dir, 'test.csv'))
sample = pd.read_csv(path_join(data_dir, 'sample_submission.csv'))
print(train.shape, test.shape)
 train.head()
gc.enable()
train_set = get_features_from_mjd(train_set)
 train_set = passbandSplit(train_set)
 train_set['flux_ratio_sq'] = np.power(train_set['flux'] / train_set['flux_err'], 2.0)
 train_set['flux_by_flux_ratio_sq'] = train_set['flux'] * train_set['flux_ratio_sq']
train_set["magnitude"] = -2.5 * np.log(train_set["flux"] ).fillna(0)
 train_set["magnitude_0"] = -2.5 * np.log(train_set["flux_0"] ).fillna(0)
 train_set["magnitude_1"] = -2.5 * np.log(train_set["flux_1"] ).fillna(0)
 train_set["magnitude_2"] = -2.5 * np.log(train_set["flux_2"] ).fillna(0)
 train_set["magnitude_3"] = -2.5 * np.log(train_set["flux_3"] ).fillna(0)
 train_set = sabun_henkaritsu_cumsum(train_set)
 aggs = {
"dtdmag_pct": ['min', 'max', 'mean', 'median', 'std','skew', "sum"],
'dtdmag_1_diff_by_mjd': ['min'],
'dtdmag_2_diff_by_mjd': ['min'],
'dtdmag_3_diff_by_mjd': ['min'],
'dtdmag_diff': ['min', "sum"],
'dtdmag_diff_by_mjd': ['min', 'max', 'mean', 'median', "sum"],
"dtd_magnitude": ['max', 'mean', 'median', 'std','skew'],
"dtd_magnitude_1": ['mean', 'skew'],
"dtd_magnitude_2": ['mean', 'std','skew'],
"dtd_magnitude_3": ['std','skew'],
"dtd_fl_by_mjd_dif": ['max', 'std'],
"flux_detected": ['min', 'median', 'std','skew'],
'magnitude': ['min', 'max', 'mean', 'std','skew'],
'flux': ['min', 'max', 'mean', 'median', 'std','skew'],
'flux_err': ['min', 'std'],
'detected': ['mean'],
'flux_ratio_sq':['sum','skew'],
'flux_by_flux_ratio_sq':['sum','skew'],
'mjd_diff': ['mean'],
'mjd_detected': ['std'],
'flux_0': ['min', 'max', 'mean', 'median', 'std','skew'],
'flux_1': ['min', 'max', 'mean', 'median', 'std','skew'],
'flux_2': ['min', 'max', 'mean', 'median', 'std','skew'],
'flux_3': ['min', 'max', 'mean', 'median', 'std','skew'],
'flux_4': ['min', 'max', 'mean', 'median', 'std','skew'],
'flux_5': ['min', 'max', 'mean', 'median', 'std','skew'],
'abs_flux_1': ['median', 'skew'],
'abs_flux_2': ['median', 'skew'],
'abs_flux_3': ['skew'],
'abs_flux_4': ['skew'],
'abs_flux_5': ['mean', 'median'],
'magnitude_0': ['sum'],
'magnitude_1': ['min', 'max', 'mean'],
'magnitude_2': ['min', 'max', 'mean'],
'magnitude_3': ['min', 'max'],
'flux_ratio_sq_0':['sum'],
'flux_ratio_sq_4':['skew'],
'flux_ratio_sq_5':['sum'],
'flux_by_flux_ratio_sq_0':['sum'],
'flux_by_flux_ratio_sq_1':['skew'],
'flux_by_flux_ratio_sq_5':['skew'],
'flux_1_err': ['min', 'max'],
"flux_henkaritsu" : ['min', 'median'],
"flux_sabun_diff": ['skew'],
"mag_sabun_diff": ['std'],
"mag_sabun": ['min', 'std', "sum"],
"fl_ratio_sabun": ['min', 'skew'],
}
agg_train = train_set.groupby('object_id' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train.columns = new_columns
agg_train['magnitude_diff_by_mjd'] =(agg_train['magnitude_max'] - agg_train['magnitude_min'])/ agg_train['mjd_diff_mean']
agg_train['magnitude_dif2'] =(agg_train['magnitude_max'] - agg_train['magnitude_min'])/ agg_train['magnitude_mean']
agg_train['flux_w_mean'] = agg_train['flux_by_flux_ratio_sq_sum'] / agg_train['flux_ratio_sq_sum']
agg_train['flux_dif3'] =(agg_train['flux_max'] - agg_train['flux_min'])/ agg_train['flux_w_mean']
agg_train["flux_dif3_expo2"] = np.power(agg_train['flux_dif3'], 2)
 agg_train["flux_0_median_expo2"] = np.power(agg_train['flux_0_median'], 2)
 agg_train["flux_1_median_expo2"] = np.power(agg_train['flux_1_median'], 2)
 agg_train["flux_5_median_expo2"] = np.power(agg_train['flux_5_median'], 2)
 agg_train['magnitude_1_diff_by_mjd'] =(agg_train['magnitude_1_max'] - agg_train['magnitude_1_min'])/ agg_train['mjd_diff_mean']
agg_train['magnitude_2_diff_by_mjd'] =(agg_train['magnitude_2_max'] - agg_train['magnitude_2_min'])/ agg_train['mjd_diff_mean']
agg_train['magnitude_3_diff_by_mjd'] =(agg_train['magnitude_3_max'] - agg_train['magnitude_3_min'])/ agg_train['mjd_diff_mean']
del agg_train['magnitude_1_max'], agg_train['magnitude_1_min']
del agg_train['magnitude_2_max'], agg_train['magnitude_2_min']
del agg_train['magnitude_3_max'], agg_train['magnitude_3_min']
agg_train =get_flux_decays(agg_train)
 del train_set
del agg_train['flux_min'], agg_train['flux_mean'], agg_train['flux_std'], agg_train['flux_median']
del agg_train["magnitude_min"], agg_train["magnitude_max"]
print(gc.collect())
 agg_train.head()<merge>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
full_train = agg_train.reset_index().merge(
 right=train_set_meta, how='outer', on='object_id')
 if 'target' in full_train:
y = full_train['target']
del full_train['target']
classes = sorted(y.unique())
 class_weight = {
c: 1 for c in classes
}
for c in [64, 15]:
class_weight[c] = 2
print('Unique classes : ', classes)
 del full_train["ra"], full_train["decl"], full_train["gal_l"], full_train["gal_b"], full_train["ddf"]
print(full_train.shape)
 unique_y = np.unique(y)
 class_map = dict()
for i,val in enumerate(unique_y):
class_map[val] = i
y_map = np.zeros(( y.shape[0],))
y_map = np.array([class_map[val] for val in y])
 y_categorical = to_categorical(y_map )<feature_engineering>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )
def get_luminosity_features(df):
df["sqrd_rederr"] = np.power(df['hostgal_photoz_err'], 2.0)
 df["red_max_err"] = df["flux_max"] * df["sqrd_rederr"]
df["sqrd_rederr_0_max"] = df["flux_0_max"] * df["sqrd_rederr"]
df["sqrd_rederr_1_max"] = df["flux_1_max"] * df["sqrd_rederr"]
df["sqrd_rederr_2_max"] = df["flux_2_max"] * df["sqrd_rederr"]
df["sqrd_rederr_3_max"] = df["flux_3_max"] * df["sqrd_rederr"]
df["sqrd_rederr_4_max"] = df["flux_4_max"] * df["sqrd_rederr"]
df["sqrd_rederr_5_max"] = df["flux_5_max"] * df["sqrd_rederr"]
df["rederr_1_0_max_diff"] = df["sqrd_rederr_1_max"] - df["sqrd_rederr_0_max"]
df["rederr_2_1_max_diff"] = df["sqrd_rederr_2_max"] - df["sqrd_rederr_1_max"]
df["rederr_3_2_max_diff"] = df["sqrd_rederr_3_max"] - df["sqrd_rederr_2_max"]
df["rederr_4_3_max_diff"] = df["sqrd_rederr_4_max"] - df["sqrd_rederr_3_max"]
df["rederr_5_4_max_diff"] = df["sqrd_rederr_5_max"] - df["sqrd_rederr_4_max"]
df["rederr_4_3_max_by"] = df["sqrd_rederr_4_max"] / df["rederr_4_3_max_diff"]
df["rederr_3_2_max_by"] = df["sqrd_rederr_3_max"] / df["rederr_3_2_max_diff"]
df["rederr_2_1_max_by"] = df["sqrd_rederr_2_max"] / df["rederr_2_1_max_diff"]
df["rederr_1_0_max_by"] = df["sqrd_rederr_1_max"] / df["rederr_1_0_max_diff"]
df["rederr__0_max_diff"] = df["red_max_err"] - df["sqrd_rederr_0_max"]
df["rederr__1_max_diff"] = df["red_max_err"] - df["sqrd_rederr_1_max"]
df["rederr__2_max_diff"] = df["red_max_err"] - df["sqrd_rederr_2_max"]
del df["sqrd_rederr"]
del df["sqrd_rederr_3_max"], df["sqrd_rederr_1_max"], df["sqrd_rederr_2_max"],
del df["sqrd_rederr_4_max"], df["sqrd_rederr_5_max"], df["sqrd_rederr_0_max"]
del df["rederr_1_0_max_diff"], df["rederr_4_3_max_diff"], df["rederr_3_2_max_diff"], df["rederr_2_1_max_diff"],
df["sqrd_redshift"] = np.power(df['hostgal_photoz'], 2.0)
 df["sqrd_red_2_median"] = df["flux_2_median"] * df["sqrd_redshift"]
df["sqrd_red_4_mean"] = df["flux_4_mean"] * df["sqrd_redshift"]
df["sqrd_red_5_mean"] = df["flux_5_mean"] * df["sqrd_redshift"]
df["sqrd_red_0_max"] = df["flux_0_max"] * df["sqrd_redshift"]
df["sqrd_red_1_max"] = df["flux_1_max"] * df["sqrd_redshift"]
df["sqrd_red_2_max"] = df["flux_2_max"] * df["sqrd_redshift"]
df["sqrd_red_3_max"] = df["flux_3_max"] * df["sqrd_redshift"]
df["sqrd_red_4_max"] = df["flux_4_max"] * df["sqrd_redshift"]
df["sqrd_red_5_max"] = df["flux_5_max"] * df["sqrd_redshift"]
df["abs_lumino_1_median"] = df["abs_flux_1_median"] * 4 * 3.14 ** df['distmod']
df["abs_lumino_1_skew"] = df["abs_flux_1_skew"] * 4 * 3.14 ** df['distmod']
df["abs_lumino_4_skew"] = df["abs_flux_4_skew"] * 4 * 3.14 ** df['distmod']
df["dtd_red_min"] = df["flux_detected_min"] * df["sqrd_redshift"]
df["dtd_red_median"] = df["flux_detected_median"] * df["sqrd_redshift"]
df["dtd_red_std"] = df["flux_detected_std"] * df["sqrd_redshift"]
df["dtd_red_skew"] = df["flux_detected_skew"] * df["sqrd_redshift"]
df["dtd_red_2_median"] = df["sqrd_red_2_median"] - df["dtd_red_median"]
df["abs_dtd_red_1_skew"] = df["abs_lumino_1_skew"] / df["dtd_red_skew"]
df["abs_dtd_red_4_skew"] = df["abs_lumino_4_skew"] / df["dtd_red_skew"]
df["red_max"] = df["flux_max"] * df["sqrd_redshift"]
df["red__0_max_diff"] = df["red_max"] - df["sqrd_red_0_max"]
df["red__1_max_diff"] = df["red_max"] - df["sqrd_red_1_max"]
df["red__2_max_diff"] = df["red_max"] - df["sqrd_red_2_max"]
del df["flux_max"], df["red_max"], df["sqrd_redshift"]
df["red_2_1_max_diff"] = df["sqrd_red_2_max"] - df["sqrd_red_1_max"]
df["red_3_2_max_diff"] = df["sqrd_red_3_max"] - df["sqrd_red_2_max"]
df["red_5_4_max_diff"] = df["sqrd_red_5_max"] - df["sqrd_red_4_max"]
del df["sqrd_red_3_max"]
del df["flux_0_min"], df["flux_1_min"], df["flux_2_min"], df["flux_3_min"], df["flux_4_min"], df["flux_5_min"]
del df["mwebv"]
del df["flux_1_median"], df["flux_2_median"], df["flux_3_median"], df["flux_4_median"], df["flux_5_median"]
del df["flux_3_mean"], df["flux_4_max"]
del df['flux_ratio_sq_sum'], df['flux_by_flux_ratio_sq_sum']
del df["flux_0_median"], df["flux_0_mean"], df["flux_0_std"]
del df["flux_1_mean"], df["flux_1_max"],
del df["flux_2_max"], df["flux_2_mean"], df["flux_2_std"],
del df["flux_3_max"], df["flux_3_std"],
del df["flux_4_std"], df["flux_4_mean"],
del df["flux_5_max"], df["flux_5_mean"], df["flux_5_skew"]
del df["flux_w_mean"],
del df["magnitude_mean"],
return df
full_train = get_luminosity_features(full_train)
 del full_train['object_id'], full_train['distmod'], full_train['hostgal_specz']
train_mean = full_train.mean(axis=0)
 full_train.fillna(train_mean, inplace=True)
 print(full_train.shape)
 full_train.head()<compute_train_metric>
class OptimizedRounder(object):
def __init__(self,correlation):
self.correlation = correlation
self.coef_ = 0
self.score = 0
def _kappa_loss(self, coef, X, y):
a= X.copy()
b=y.copy()
X_p = pd.cut(a, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0,1,2])
 a[X_p == 0] = 0
a[X_p == 2] = 1
if spearmanr(a, b ).correlation < self.correlation:
self.score = spearmanr(a, b ).correlation
return - spearmanr(a, b ).correlation +(self.correlation - spearmanr(a, b ).correlation + 1)**10
else:
self.score = spearmanr(a, b ).correlation
return - spearmanr(a, b ).correlation
def fit(self, X, y,coef_ini):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = coef_ini
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def coefficients(self):
return self.coef_['x']
def score_fc(self):
return self.score
def lgb_multi_weighted_logloss(y_true, y_preds):
classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
if len(np.unique(y_true)) > 14:
classes.append(99)
 class_weight[99] = 2
y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return 'wloss', loss, False
def multi_weighted_logloss(y_true, y_preds):
classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
if len(np.unique(y_true)) > 14:
classes.append(99)
 class_weight[99] = 2
y_p = y_preds
y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss
lgb_params = {
'device': 'cpu',
'objective': 'multiclass',
'num_class': 14,
'boosting_type': 'gbdt',
'n_jobs': -1,
'max_depth': 7,
'n_estimators': 500,
'subsample_freq': 2,
'subsample_for_bin': 5000,
'min_data_per_group': 100,
'max_cat_to_onehot': 4,
'cat_l2': 1.0,
'cat_smooth': 59.5,
'max_cat_threshold': 32,
'metric_freq': 10,
'verbosity': -1,
'metric': 'multi_logloss',
'xgboost_dart_mode': False,
'uniform_drop': False,
'colsample_bytree': 0.5,
'drop_rate': 0.173,
'learning_rate': 0.0267,
'max_drop': 5,
'min_child_samples': 10,
'min_child_weight': 100.0,
'min_split_gain': 0.1,
'num_leaves': 7,
'reg_alpha': 0.1,
'reg_lambda': 0.00023,
'skip_drop': 0.44,
'subsample': 0.75
}
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1111)
 clfs = []
importances = pd.DataFrame()
oof_preds = np.zeros(( len(full_train), len(classes)))
 for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
clf = lgb.LGBMClassifier(**lgb_params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgb_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=50,
sample_weight=trn_y.map(weights)
)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print(multi_weighted_logloss(val_y, clf.predict_proba(val_x, num_iteration=clf.best_iteration_)))
 imp_df = pd.DataFrame()
imp_df['feature'] = full_train.columns
imp_df['gain'] = clf.feature_importances_
imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 clfs.append(clf)
 print('MULTI WEIGHTED LOG LOSS : %.5f ' % multi_weighted_logloss(y_true=y, y_preds=oof_preds))
mean_gain = importances[['gain', 'feature']].groupby('feature' ).mean()
importances['mean_gain'] = importances['feature'].map(mean_gain['gain'])
 plt.figure(figsize=(8, 24))
sns.barplot(x='gain', y='feature', data=importances.sort_values('mean_gain', ascending=False))
plt.tight_layout()<load_from_csv>
def optimize_preds(y_df, x_df, preds_df):
for title in targets:
y1 = np.asarray(y_df[title].copy())
 X1 = np.asarray(x_df[title].copy())
 x_original = np.asarray(x_df[title].copy())
 preds_original = np.asarray(preds_df[title].copy())
 correlation = spearmanr(y1, X1 ).correlation
print(title)
 print(correlation)
 coefficients_target = [0,0]
correlation = spearmanr(y1, X1 ).correlation
okcor = spearmanr(y1, X1 ).correlation
maxi = correlation
liste = [[0.1,0.9],[0.2,0.9],[0.3,0.9],[0.3,0.8],[0.2,0.7],[0.3,0.7],[0.2,0.6],[0.5,0.9],[0.3,0.6],[0.4,0.8],[0.7,0.9],[0.8,0.9]]
for L in liste:
optR = OptimizedRounder(correlation)
 optR.fit(X1,y1,L)
 coefficients = optR.coefficients()
if optR.score_fc() > maxi:
maxi = optR.score_fc()
coefficients_target = coefficients
if maxi != spearmanr(y1, X1 ).correlation:
oof = X1.copy()
oof[oof > coefficients_target[1]] = 1
oof[oof <= coefficients_target[0]] = 0
X1 = np.asarray(preds_df[title].copy())
 oof_test = X1.copy()
oof_test[oof_test > coefficients_target[1]] = 1
oof_test[oof_test <= coefficients_target[0]] = 0
score = spearmanr(y1, oof ).correlation
if score - okcor > 0.001:
print("difference validation score = {}".format(score - okcor))
x_df[title] = oof
dist = Counter(x_df[title])
 if 0 in list(dist.keys()):
dist[0] /= len(x_df)
 if 1 in list(dist.keys()):
dist[1] /= len(x_df)
 acum = 0
bound = {}
if 0 in list(dist.keys()):
acum = dist[0]
bound[0] = np.percentile(preds_df[title], acum * 100)
 if 1 in list(dist.keys()):
acum = 1 - dist[1]
bound[1] = np.percentile(preds_df[title], acum * 100)
 def classify(x):
if 0 in list(dist.keys()):
if 1 in list(dist.keys()):
if x <= bound[0]:
return 0
elif x <= bound[1]:
return x
else:
return 1
else:
if x <= bound[0]:
return 0
else:
return x
else:
if 1 in list(dist.keys()):
if x <= bound[1]:
return x
else:
return 1
else:
return x
final_pred = np.array(list(map(classify, preds_df[title])))
 if len(np.unique(oof_test)) != 1:
print(coefficients_target)
 preds_df[title] = final_pred
cnf_matrix = confusion_matrix(y_map, np.argmax(oof_preds,axis=-1))
np.set_printoptions(precision=2)
 sample_sub = pd.read_csv('.. /input/sample_submission.csv')
 class_names = list(sample_sub.columns[1:-1])
 del sample_sub;gc.collect()
plt.figure(figsize=(12,12))
foo = plot_confusion_matrix(cnf_matrix, classes=class_names,normalize=True,
title='Confusion matrix' )<feature_engineering>
validationN=pd.read_csv('.. /input/nirjharbert003/validation_preds_df_003.csv')
 test_predsN=pd.read_csv('.. /input/nirjharbert003/submission_003.csv')
 print("Nirjhar model : ")
 compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validationN[targets].copy()))
start = time.time()
chunks = 5000000
for i_c, df in enumerate(pd.read_csv('.. /input/test_set.csv', chunksize=chunks, iterator=True)) :
df = get_features_from_mjd(df)
 df = passbandSplit(df)
 df['flux_ratio_sq'] = np.power(df['flux'] / df['flux_err'], 2.0)
 df['flux_by_flux_ratio_sq'] = df['flux'] * df['flux_ratio_sq']
df["magnitude"] = -2.5 * np.log(df["flux"] ).fillna(0)
 df["magnitude_0"] = -2.5 * np.log(df["flux_0"] ).fillna(0)
 df["magnitude_1"] = -2.5 * np.log(df["flux_1"] ).fillna(0)
 df["magnitude_2"] = -2.5 * np.log(df["flux_2"] ).fillna(0)
 df["magnitude_3"] = -2.5 * np.log(df["flux_3"] ).fillna(0)
 df = sabun_henkaritsu_cumsum(df)
 agg_test = df.groupby('object_id' ).agg(aggs)
 agg_test.columns = new_columns
agg_test['magnitude_diff_by_mjd'] =(agg_test['magnitude_max'] - agg_test['magnitude_min'])/ agg_test['mjd_diff_mean']
agg_test['magnitude_dif2'] =(agg_test['magnitude_max'] - agg_test['magnitude_min'])/ agg_test['magnitude_mean']
agg_test['flux_w_mean'] = agg_test['flux_by_flux_ratio_sq_sum'] / agg_test['flux_ratio_sq_sum']
agg_test['flux_dif3'] =(agg_test['flux_max'] - agg_test['flux_min'])/ agg_test['flux_w_mean']
agg_test["flux_dif3_expo2"] = np.power(agg_test['flux_dif3'], 2)
 agg_test["flux_0_median_expo2"] = np.power(agg_test['flux_0_median'], 2)
 agg_test["flux_1_median_expo2"] = np.power(agg_test['flux_1_median'], 2)
 agg_test["flux_5_median_expo2"] = np.power(agg_test['flux_5_median'], 2)
 agg_test['magnitude_1_diff_by_mjd'] =(agg_test['magnitude_1_max'] - agg_test['magnitude_1_min'])/ agg_test['mjd_diff_mean']
agg_test['magnitude_2_diff_by_mjd'] =(agg_test['magnitude_2_max'] - agg_test['magnitude_2_min'])/ agg_test['mjd_diff_mean']
agg_test['magnitude_3_diff_by_mjd'] =(agg_test['magnitude_3_max'] - agg_test['magnitude_3_min'])/ agg_test['mjd_diff_mean']
del agg_test['magnitude_1_max'], agg_test['magnitude_1_min']
del agg_test['magnitude_2_max'], agg_test['magnitude_2_min']
del agg_test['magnitude_3_max'], agg_test['magnitude_3_min']
agg_test =get_flux_decays(agg_test)
 del agg_test['flux_min'], agg_test['flux_mean'], agg_test['flux_std'], agg_test['flux_median']
del agg_test["magnitude_min"], agg_test["magnitude_max"]
full_test = agg_test.reset_index().merge(
 right=test_set_meta, how='left', on='object_id')
 del full_test["ra"], full_test["decl"], full_test["gal_l"], full_test["gal_b"], full_test["ddf"]
full_test = get_luminosity_features(full_test)
 full_test = full_test.fillna(train_mean)
 preds = None
for clf in clfs:
if preds is None:
preds = clf.predict_proba(full_test[full_train.columns])/ folds.n_splits
else:
preds += clf.predict_proba(full_test[full_train.columns])/ folds.n_splits
preds_99 = np.ones(preds.shape[0])
 for i in range(preds.shape[1]):
preds_99 *=(1 - preds[:, i])
 preds_df = pd.DataFrame(preds, columns=['class_' + str(s)for s in clfs[0].classes_])
 preds_df['object_id'] = full_test['object_id']
preds_df['class_99'] = 0.14 * preds_99 / np.mean(preds_99)
 y = pd.DataFrame()
y['mymean'] = preds_df[feats].mean(axis=1)
 y['mymedian'] = preds_df[feats].median(axis=1)
 y['mymax'] = preds_df[feats].max(axis=1)
 preds_df['class_99'] = GenUnknown(y)
 if i_c == 0:
preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)
 else:
preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)
 del agg_test, full_test, preds_df, preds
gc.collect()
if(i_c + 1)% 10 == 0:
print('%15d done in %5.1f' %(chunks *(i_c + 1),(time.time() - start)/ 60))<save_to_csv>
validationN1=pd.read_csv('.. /input/dataset-nirjhar/validation_preds_df_002.csv')
 test_predsN1=pd.read_csv('.. /input/dataset-nirjhar/submission_002.csv')
 print("Nirjhar model : ")
 compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validationN1[targets].copy()))
z = pd.read_csv('predictions.csv')
 print(z.groupby('object_id' ).size().max())
 print(( z.groupby('object_id' ).size() > 1 ).sum())
 z = z.groupby('object_id' ).mean()
z.to_csv('great_trust_cv_predictions.csv', index=True )<compute_test_metric>
data_table = []
data_table.append({'Name': 'QUEST-003-55', 'valid': copy.deepcopy(validationT), 'test_preds': copy.deepcopy(test_predsT)})
 data_table.append({'Name': 'QUEST-006-03-GPT2', 'valid': copy.deepcopy(validationTG), 'test_preds': copy.deepcopy(test_predsTG)})
 data_table.append({'Name': 'QUEST-005-07-distilbert', 'valid': copy.deepcopy(validation0), 'test_preds': copy.deepcopy(test_preds0)})
 data_table.append({'Name': 'QUEST-005-07-lstm', 'valid': copy.deepcopy(validationLSTM), 'test_preds': copy.deepcopy(test_predsLSTM)})
 data_table.append({'Name': 'QUEST-Bert-Nirjhar', 'valid': copy.deepcopy(validationN), 'test_preds': copy.deepcopy(test_predsN)})
 data_table.append({'Name': 'QUEST-Bert-Nirjhar1', 'valid': copy.deepcopy(validationN1), 'test_preds': copy.deepcopy(test_predsN1)} )