Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
%matplotlib inline
train = pd.read_csv('.. /input/nyc-taxi-trip-duration/train.csv')
 test = pd.read_csv('.. /input/nyc-taxi-trip-duration/test.csv' )<data_type_conversions>
from xgboost import XGBRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.model_selection import GridSearchCV
from sklearn.feature_selection import RFECV
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, make_scorer
from mlxtend.preprocessing import minmax_scaling
coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 sample_ind = np.random.permutation(len(coords)) [:500000]
kmeans = MiniBatchKMeans(n_clusters=100, batch_size=10000 ).fit(coords[sample_ind])
 def toDateTime(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['pickup_weekday'] = df['pickup_datetime'].dt.weekday_name
df['pickup_day'] = df['pickup_datetime'].dt.day
df['pickup_month'] = df['pickup_datetime'].dt.month.astype('object')
 df['pickup_hour'] = df['pickup_datetime'].dt.hour
df['pickup_minute'] = df['pickup_datetime'].dt.minute
df['pickup_dt'] =(df['pickup_datetime'] - df['pickup_datetime'].min() ).map(
 lambda x: x.total_seconds())
 df.drop('pickup_datetime', axis = 1, inplace = True)
 return df
def haversine_np(lon1, lat1, lon2, lat2):
lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat/2.0)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon/2.0)**2
c = 2 * np.arcsin(np.sqrt(a))
km = 6367 * c
return km
def dummy_manhattan_distance(lat1, lng1, lat2, lng2):
a = haversine_np(lat1, lng1, lat1, lng2)
 b = haversine_np(lat1, lng1, lat2, lng1)
 return a + b
def bearing_array(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
def locationFeatures(df):
df['up_town'] = np.sign(df['pickup_longitude'] - df['dropoff_longitude'])
 df['est_side'] = np.sign(df['pickup_latitude'] - df['dropoff_latitude'])
 df['haversine_distance'] = haversine_np(
 df['pickup_longitude'], df['pickup_latitude'],
df['dropoff_longitude'], df['dropoff_latitude'] 
)
 df['log_haversine_distance'] = np.log1p(df['haversine_distance'])
 df['distance_dummy_manhattan'] = dummy_manhattan_distance(
 df['pickup_latitude'], df['pickup_longitude'],
df['dropoff_latitude'], df['dropoff_longitude'] 
)
 df['log_distance_dummy_manhattan'] = np.log1p(df['distance_dummy_manhattan'])
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df.loc[:, 'pca_manhattan'] =(np.abs(df['dropoff_pca1'] - df['pickup_pca1'])+
np.abs(df['dropoff_pca0'] - df['pickup_pca0']))
df.loc[:, 'pickup_cluster'] = kmeans.predict(df[['pickup_latitude', 'pickup_longitude']] ).astype('object')
 df.loc[:, 'dropoff_cluster'] = kmeans.predict(df[['dropoff_latitude', 'dropoff_longitude']] ).astype('object')
 df.drop(['pickup_longitude', 'dropoff_longitude'], axis = 1, inplace = True)
 df.drop(['pickup_latitude', 'dropoff_latitude'], axis = 1, inplace = True)
 return df
def featureCreate(df):
print('Date time features')
 df = toDateTime(df)
 print('Location Features')
 df = locationFeatures(df)
 return df<feature_engineering>
warnings.filterwarnings("ignore" )
train = featureCreate(train)
 test = featureCreate(test)
 train['trip_duration'] = np.log1p(train['trip_duration'] )<define_variables>
def show_uniqs(cols):
for col in cols:
print(col)
 show_uniq(col)
 print('=======================================' )
q1 = np.percentile(train['trip_duration'], 25)
 q3 = np.percentile(train['trip_duration'], 75)
 iqr = q3 - q1
train = train[ train['trip_duration'] <= q3 + 3.0*iqr]
train = train[ q1 - 3.0*iqr <= train['trip_duration']]<drop_column>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv' )
labels = train.pop('trip_duration')
 train.drop(["id", "dropoff_datetime"], axis=1, inplace = True)
 sub = pd.DataFrame(columns = ['id', 'trip_duration'])
 sub['id'] = test.pop('id' )<categorify>
count_empty_columns_train = train[empty_columns_train].isnull().sum(axis = 0)
 count_empty_columns_train
train = pd.get_dummies(train)
 test = pd.get_dummies(test )<normalization>
for c, name_col in zip(count_empty_columns_train,count_empty_columns_train.index):
if c > 584:
print(name_col )
mmScale = MinMaxScaler()
n = train.shape[1]
train = mmScale.fit_transform(train )<choose_model_class>
test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
model = Sequential()
model.add(Dense(n, activation='relu', input_shape=(n,)))
 model.add(Dense(int(0.5*n), activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mse', optimizer='rmsprop', metrics=['mae'] )<train_model>
count_empty_columns_test = test[empty_columns_test].isnull().sum(axis = 0)
 count_empty_columns_test
print('On to the next one')
 model.fit(train, labels.values, epochs = 3)
 print('Finished' )<predict_on_test>
for col in count_empty_columns_test.index:
if col not in count_empty_columns_train:
print(col )
train_pred = model.predict(train)
 test_pred = model.predict(mmScale.transform(test))<save_to_csv>
garage_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Garage"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
garage_colums.append(col)
 garage_colums = np.array(garage_colums)
 garage_colums
sub['trip_duration'] = np.expm1(test_pred)
 sub.to_csv('submission111.csv', index=False )<install_modules>
for col in garage_num_cols:
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )
!pip install.. /input/apex01cp37cp37mlinux-x86-64whl/apex-0.1-cp37-cp37m-linux_x86_64.whl
!pip install --upgrade.. /input/tokenizers0011cp37/tokenizers-0.0.11-cp37-cp37m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl<import_modules>
show_uniqs(garage_cat_cols )
from transformers import BertConfig, BertTokenizer, RobertaConfig, RobertaTokenizer 
<predict_on_test>
basement_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Bsmt"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
basement_colums.append(col)
 basement_colums = np.array(basement_colums)
 basement_colums
def bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len):
class TFQADataset(Dataset):
def __init__(self, id_list):
self.id_list=id_list
def __len__(self):
return len(self.id_list)
 def __getitem__(self, index):
return self.id_list[index]
class Collator(object):
def __init__(self, data_dict, new_token_dict, tokenizer, max_seq_len=384, max_question_len=64):
self.data_dict = data_dict
self.new_token_dict = new_token_dict
self.tokenizer = tokenizer
self.max_seq_len = max_seq_len
self.max_question_len = max_question_len
def _get_input_ids(self, doc_id, candidate_index):
data = self.data_dict[doc_id]
question_tokens = self.tokenizer.tokenize(data['question_text'])[:self.max_question_len]
doc_words = data['document_text'].split()
max_answer_tokens = self.max_seq_len-len(question_tokens)-3
candidate = data['long_answer_candidates'][candidate_index]
candidate_start = candidate['start_token']
candidate_end = candidate['end_token']
candidate_words = doc_words[candidate_start:candidate_end]
for i, word in enumerate(candidate_words):
if re.match(r'<.+>', word):
if word in self.new_token_dict:
candidate_words[i] = self.new_token_dict[word]
else:
candidate_words[i] = '<'
words_to_tokens_index = []
candidate_tokens = []
for i, word in enumerate(candidate_words):
words_to_tokens_index.append(len(candidate_tokens))
tokens = self.tokenizer.tokenize(word)
 if len(candidate_tokens)+len(tokens)> max_answer_tokens:
break
for token in tokens:
candidate_tokens.append(token)
 input_tokens = ['[CLS]'] + question_tokens + ['[SEP]'] + candidate_tokens + ['[SEP]']
input_ids = self.tokenizer.convert_tokens_to_ids(input_tokens)
 return input_ids, words_to_tokens_index, len(input_ids), len(question_tokens)+2
def __call__(self, batch_ids):
batch_size = len(batch_ids)
 batch_input_ids_temp = []
batch_seq_len = []
batch_offset = []
batch_words_to_tokens_index = []
for i,(doc_id, candidate_index)in enumerate(batch_ids):
input_ids, words_to_tokens_index, seq_len, offset = self._get_input_ids(doc_id, candidate_index)
 batch_input_ids_temp.append(input_ids)
 batch_seq_len.append(seq_len)
 batch_offset.append(offset)
 batch_words_to_tokens_index.append(words_to_tokens_index)
 batch_max_seq_len = max(batch_seq_len)
 batch_input_ids = np.zeros(( batch_size, batch_max_seq_len), dtype=np.int64)
 batch_token_type_ids = np.ones(( batch_size, batch_max_seq_len), dtype=np.int64)
 for i in range(batch_size):
input_ids = batch_input_ids_temp[i]
batch_input_ids[i, :len(input_ids)] = input_ids
batch_token_type_ids[i, :len(input_ids)] = [0 if k<=input_ids.index(102)else 1 for k in range(len(input_ids)) ]
batch_attention_mask = batch_input_ids > 0
return torch.from_numpy(batch_input_ids), torch.from_numpy(batch_attention_mask), torch.from_numpy(batch_token_type_ids), batch_words_to_tokens_index, batch_offset, batch_max_seq_len
class BertForQuestionAnswering(BertPreTrainedModel):
def __init__(self, config):
super(BertForQuestionAnswering, self ).__init__(config)
 self.bert = BertModel(config)
 self.qa_outputs = nn.Linear(config.hidden_size, 2)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.init_weights()
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 sequence_output = outputs[0]
pooled_output = outputs[1]
qa_logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = qa_logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 pooled_output = self.dropout(pooled_output)
 classifier_logits = self.classifier(pooled_output)
 return start_logits, end_logits, classifier_logits
max_seq_len = 360
max_question_len = 64
batch_size = int(384/4)
 model_path = '.. /models/models/bert/'
config = BertConfig.from_pretrained(model_path)
 config.num_labels = 5
config.vocab_size = 30531
tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = BertForQuestionAnswering.from_pretrained(model_dir, config=config)
 new_token_dict = {
'<P>':'qw1',
'<Table>':'qw2',
'<Tr>':'qw3',
'<Ul>':'qw4',
'<Ol>':'qw5',
'<Fl>':'qw6',
'<Li>':'qw7',
'<Dd>':'qw8',
'<Dt>':'qw9',
}
new_token_list = [
'qw1',
'qw2',
'qw3',
'qw4',
'qw5',
'qw6',
'qw7',
'qw8',
'qw9',
]
num_added_toks = tokenizer.add_tokens(new_token_list)
 model.resize_token_embeddings(len(tokenizer))
model.cuda()
optimizer = optim.Adam(model.parameters() , lr=1e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)
 if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
 test_datagen = TFQADataset(id_list=id_candidate_list_sorted)
 test_collate = Collator(data_dict=data_dict,
new_token_dict=new_token_dict,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
max_question_len=max_question_len)
 test_generator = DataLoader(dataset=test_datagen,
collate_fn=test_collate,
batch_size=batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
 model.eval()
list_offset = []
list_words_to_tokens_index = []
test_prob1 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob2 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob3 = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 for j,(batch_input_ids, batch_attention_mask, batch_token_type_ids, batch_words_to_tokens_index, batch_offset, batch_max_seq_len)in tqdm(enumerate(test_generator)) :
with torch.no_grad() :
start = j*batch_size
end = start+batch_size
if j == len(test_generator)-1:
end = len(test_generator.dataset)
 batch_input_ids = batch_input_ids.cuda()
batch_attention_mask = batch_attention_mask.cuda()
batch_token_type_ids = batch_token_type_ids.cuda()
logits1, logits2, logits3 = model(batch_input_ids, batch_attention_mask, batch_token_type_ids)
 test_prob1[start:end, :batch_max_seq_len] += F.softmax(logits1,dim=1 ).cpu().data.numpy()
test_prob2[start:end, :batch_max_seq_len] += F.softmax(logits2,dim=1 ).cpu().data.numpy()
test_prob3[start:end] += F.softmax(logits3,dim=1 ).cpu().data.numpy()
list_words_to_tokens_index += batch_words_to_tokens_index
list_offset += batch_offset
test_word_prob1 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 test_word_prob2 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 for i in range(len(id_candidate_list_sorted)) :
for j in range(len(list_words_to_tokens_index[i])) :
test_word_prob1[i,j] = test_prob1[i, list_words_to_tokens_index[i][j]+list_offset[i]]
test_word_prob2[i,j] = test_prob2[i, list_words_to_tokens_index[i][j]+list_offset[i]]
return test_word_prob1, test_word_prob2, test_prob3 
<categorify>
for col in basement_num_cols:
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )
start_time = time.time()
data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted = reduce1(n_candidate=10, th_candidate=0.2)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
id_candidate_list_sorted = reduce2(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, n_candidate=4, th_candidate=0.35)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
word_len = 360
start_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 end_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 start_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 end_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 class_prob = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 print("before bert predicting")
 weight_albert = 0.7
model_dir = '.. /models/models/bert/'
test_prob1, test_prob2, test_prob3 = bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob +=(1-weight_albert)*test_prob1
end_prob +=(1-weight_albert)*test_prob2
class_prob +=(1-weight_albert)*test_prob3
print("before albert predicting")
 model_dir = '.. /models/models/albert/'
test_prob1, test_prob2, test_prob3 = albert_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob += weight_albert*test_prob1
end_prob += weight_albert*test_prob2
class_prob += weight_albert*test_prob3
start_label = np.argmax(start_prob, axis=1)
 end_label = np.argmax(end_prob, axis=1)
 temp_dict = {}
for doc_id in id_list:
temp_dict[doc_id] = {
'long_answer': {'start_token': -1, 'end_token': -1},
'long_answer_score': -1.0,
'short_answers': [{'start_token': -1, 'end_token': -1}],
'short_answers_score': -1.0,
'yes_no_answer': 'NONE'
}
for i,(doc_id, candidate_index)in tqdm(enumerate(id_candidate_list_sorted)) :
long_answer_score = 1.0 - class_prob[i,0]
if long_answer_score > temp_dict[doc_id]['long_answer_score']:
temp_dict[doc_id]['long_answer_score'] = long_answer_score
temp_dict[doc_id]['long_answer']['start_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
temp_dict[doc_id]['long_answer']['end_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['end_token']
short_answer_score = 1.0 - class_prob[i,0] - class_prob[i,1]
temp_dict[doc_id]['short_answers_score'] = short_answer_score
temp_dict[doc_id]['short_answers'][0]['start_token'] = -1
temp_dict[doc_id]['short_answers'][0]['end_token'] = -1
temp_dict[doc_id]['yes_no_answer'] = 'NONE'
if max([class_prob[i,3], class_prob[i,4]])> class_prob[i,2]:
if class_prob[i,3] > class_prob[i,4]:
temp_dict[doc_id]['yes_no_answer'] = 'NO'
else:
temp_dict[doc_id]['yes_no_answer'] = 'YES'
else:
short_start_word = int(start_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
short_end_word = int(end_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
if short_end_word > short_start_word:
temp_dict[doc_id]['short_answers'][0]['start_token'] = short_start_word
temp_dict[doc_id]['short_answers'][0]['end_token'] = short_end_word
final_dict = {}
final_dict['predictions'] = []
for doc_id in id_list:
prediction_dict = {
'example_id': doc_id,
'long_answer': {'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['long_answer']['start_token'], 'end_token': temp_dict[doc_id]['long_answer']['end_token']},
'long_answer_score': temp_dict[doc_id]['long_answer_score'],
'short_answers': [{'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['short_answers'][0]['start_token'], 'end_token': temp_dict[doc_id]['short_answers'][0]['end_token']}],
'short_answers_score': temp_dict[doc_id]['short_answers_score'],
'yes_no_answer': temp_dict[doc_id]['yes_no_answer']
}
final_dict['predictions'].append(prediction_dict)
 print("--- %s seconds ---" %(time.time() - start_time))<define_variables>
masvnr_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"MasVnr"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
masvnr_colums.append(col)
 masvnr_colums = np.array(masvnr_colums)
 masvnr_colums
path='/kaggle/input/models/models'
os.chdir(path)
 answers_df = pd.DataFrame.from_dict(final_dict)
 long_best_threshold = 0.7
short_best_threshold = 0.7
def df_long_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['long_answer_score'] > long_best_threshold:
index['start'] = df["predictions"][i]['long_answer']['start_token']
index['end'] = df["predictions"][i]['long_answer']['end_token']
index['score'] = df["predictions"][i]['long_answer_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_short_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['short_answers_score'] > short_best_threshold and df["predictions"][i]['short_answers'][0]['start_token'] != -1:
index['start'] = df["predictions"][i]['short_answers'][0]['start_token']
index['end'] = df["predictions"][i]['short_answers'][0]['end_token']
index['score'] = df["predictions"][i]['short_answers_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_example_id(df):
return df['example_id']
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':' + str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
<feature_engineering>
all(train.loc[train['MasVnrArea'].isnull() ].index == train.loc[train['MasVnrType'].isnull() ].index )
answers_df['long_indexes_and_scores'] = df_long_index_score(answers_df)
 answers_df['short_indexes_and_scores'] = df_short_index_score(answers_df)
 answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(
 create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(
 create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df = answers_df.drop(
 ['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 sample_submission = pd.read_csv(
 ".. /.. /.. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains(
 "_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains(
 "_short"), "PredictionString"] = short_prediction_strings 
<set_options>
train['MasVnrArea'] = train['MasVnrArea'].fillna(0)
 test['MasVnrArea'] = test['MasVnrArea'].fillna(0 )
torch.cuda.empty_cache()<drop_column>
train['MasVnrType'] = train['MasVnrType'].fillna('None')
 test['MasVnrType'] = test['MasVnrType'].fillna('None' )
del BertTokenizer, BertModel, BertPreTrainedModel, BertConfig, AlbertTokenizer, AlbertModel, AlbertPreTrainedModel, AlbertConfig<install_modules>
pool_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Pool"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
pool_colums.append(col)
 pool_colums = np.array(pool_colums)
 pool_colums
<import_modules>
train['PoolQC'] = train['PoolQC'].fillna('NA')
 test['PoolQC'] = test['PoolQC'].fillna('NA' )
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',<save_to_csv>
train['Alley'] = train['Alley'].fillna('NA')
 test['Alley'] = test['Alley'].fillna('NA' )
path = '/kaggle/working'
os.chdir(path)
 sample_submission.to_csv('submission2.csv', index=False )<load_from_csv>
fireplace_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Fireplace"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
fireplace_colums.append(col)
 fireplace_colums = np.array(fireplace_colums)
 fireplace_colums
submission_no2 = pd.read_csv("submission1.csv")
 count = 0
for i in range(len(submission_no2)) :
if ':' not in str(submission_no2["PredictionString"][i])and ':' in str(sample_submission["PredictionString"][i]):
count += 1
submission_no2["PredictionString"][i] = sample_submission["PredictionString"][i]
print('count:', count )<drop_column>
train['FireplaceQu'] = train['FireplaceQu'].fillna('NA')
 test['FireplaceQu'] = test['FireplaceQu'].fillna('NA' )
path = 'submission1.csv'
if os.path.exists(path):
os.remove(path)
 path = 'submission2.csv'
if os.path.exists(path):
os.remove(path )<save_to_csv>
train['Fence'] = train['Fence'].fillna('NA')
 test['Fence'] = test['Fence'].fillna('NA' )
submission_no2.to_csv('submission.csv', index=False)
<install_modules>
misc_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Misc"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
misc_colums.append(col)
 misc_colums = np.array(misc_colums)
 misc_colums
!pip install --upgrade.. /input/tokenizers-repo/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl<import_modules>
train['MiscFeature'] = train['MiscFeature'].fillna('NA')
 test['MiscFeature'] = test['MiscFeature'].fillna('NA' )
<define_variables>
train = train.drop(train.loc[train['Electrical'].isnull() ].index )
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0<define_variables>
for i in test['Neighborhood'].unique() :
if test.MSZoning[test['Neighborhood'] == i].isnull().sum() > 0:
test.loc[test['Neighborhood'] == i,'MSZoning'] = \
test.loc[test['Neighborhood'] == i,'MSZoning'].fillna(test.loc[test['Neighborhood'] == i,'MSZoning'].mode() [0] )
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',<drop_column>
train['LotFrontage'].fillna(train['LotFrontage'].median() , inplace=True)
 test['LotFrontage'].fillna(test['LotFrontage'].median() , inplace=True )
def find_closing_tag(tokens, opening_tag):
closing_tag = f'</{opening_tag[1: -1]}>'
index, stack = -1, []
for token_index, token in enumerate(tokens):
if token == opening_tag:
stack.insert(0, opening_tag)
 elif token == closing_tag:
stack.pop()
if len(stack)== 0:
index = token_index
break
return index<load_pretrained>
y_train = train['SalePrice']
train = train.drop('SalePrice',axis=1)
 train = train.drop('Id',axis=1 )
def read_candidates(candidate_files, do_cache=True):
assert isinstance(candidate_files,(tuple, list)) , candidate_files
for fn in candidate_files:
assert os.path.exists(fn), f'Missing file {fn}'
cache_fn = 'candidates.pkl'
candidates = {}
if not os.path.exists(cache_fn):
for fn in candidate_files:
with open(fn)as f:
for line in tqdm(f):
entry = json.loads(line)
 example_id = str(entry['example_id'])
 cnds = entry.pop('long_answer_candidates')
 cnds = [LongAnswerCandidate(c['start_token'], c['end_token'],
c['top_level'])for c in cnds]
candidates[example_id] = cnds
if do_cache:
with open(cache_fn, 'wb')as f:
pickle.dump(candidates, f)
 else:
print(f'Loading from cache: {cache_fn}')
 with open(cache_fn, 'rb')as f:
candidates = pickle.load(f)
 return candidates<string_transform>
test_id = test['Id']
test = test.drop('Id', axis=1 )
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" or ord(c)== 0x202F:
return True
return False<load_pretrained>
df = train.append(test )
def read_nq_examples(input_file_or_data, is_training):

 if isinstance(input_file_or_data, str):
with open(input_file_or_data, "r", encoding='utf-8')as f:
input_data = json.load(f)["data"]
else:
input_data = input_file_or_data
for entry_index, entry in enumerate(tqdm(input_data, total=len(input_data))):
assert len(entry["paragraphs"])== 1
paragraph = entry["paragraphs"][0]
paragraph_text = paragraph["context"]
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in paragraph_text:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 assert len(paragraph["qas"])== 1
qa = paragraph["qas"][0]
start_position = None
end_position = None
long_position = None
orig_answer_text = None
short_is_impossible = False
long_is_impossible = False
if is_training:
short_is_impossible = qa["short_is_impossible"]
short_answers = qa["short_answers"]
if len(short_answers)>= 2:
short_answers = sorted(short_answers, key=lambda sa: sa["answer_start"])
 short_answers = short_answers[0: 1]
if not short_is_impossible:
answer = short_answers[0]
orig_answer_text = answer["text"]
answer_offset = answer["answer_start"]
answer_length = len(orig_answer_text)
 start_position = char_to_word_offset[answer_offset]
end_position = char_to_word_offset[
answer_offset + answer_length - 1]
actual_text = " ".join(doc_tokens[start_position:
end_position + 1])
 cleaned_answer_text = " ".join(
 whitespace_tokenize(orig_answer_text))
if actual_text.find(cleaned_answer_text)== -1:
logger.warning(
 "Could not find answer: '%s' vs.'%s'",
actual_text, cleaned_answer_text)
 continue
else:
start_position = -1
end_position = -1
orig_answer_text = ""
long_is_impossible = qa["long_is_impossible"]
long_answers = qa["long_answers"]
if(len(long_answers)!= 1)and not long_is_impossible:
raise ValueError(f"For training, each question"
f" should have exactly 1 long answer.")
 if not long_is_impossible:
long_answer = long_answers[0]
long_answer_offset = long_answer["answer_start"]
long_position = char_to_word_offset[long_answer_offset]
else:
long_position = -1
if not short_is_impossible and not long_is_impossible:
assert long_position <= start_position
if not short_is_impossible and long_is_impossible:
assert False, f'Invalid pair short, long pair'
example = NQExample(
 qas_id=qa["id"],
question_text=qa["question"],
doc_tokens=doc_tokens,
orig_answer_text=orig_answer_text,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible,
crop_start=qa["crop_start"])
 yield example<define_variables>
df['GarageCond'] = df['GarageCond'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['GarageQual'] = df['GarageQual'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['BsmtCond'] = df['BsmtCond'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['BsmtExposure'] = df['BsmtExposure'].map({'NA':0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4})
 df['BsmtFinType1'] = df['BsmtFinType1'].map({'NA':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})
 df['BsmtFinType2'] = df['BsmtFinType2'].map({'NA':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})
 df['BsmtQual'] = df['BsmtQual'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['PoolQC'] = df['PoolQC'].map({'NA':0, 'Fa':1, 'TA':2, 'Gd':3, 'Ex':4})
 df['Alley'] = df['Alley'].map({'NA':0, 'Grvl':1, 'Pave':2})
 df['FireplaceQu'] = df['FireplaceQu'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['ExterCond'] = df['ExterCond'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['ExterQual'] = df['ExterQual'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['KitchenQual'] = df['KitchenQual'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['LandSlope'] = df['LandSlope'].map({'Sev':1, 'Mod':2, 'Gtl':3})
 df['PavedDrive'] = df['PavedDrive'].map({'N':1, 'P':2, 'Y':3})
 df['Functional'] = df['Functional'].map({'Sal':1, 'Sev':2, 'Maj2':3, 'Maj1':4, 'Mod':5, 'Min2':6, 'Min1':7, 'Typ':8})
 df['HeatingQC'] = df['HeatingQC'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['Street'] = df['Street'].map({'Grvl':1, 'Pave':2})
 df['Utilities'] = df['Utilities'].map({'ELO':1, 'NoSeWa':2, 'NoSewr':3, 'AllPub':4})
 df=df.drop('MoSold',axis=1)
 df['MSSubClass'] = df['MSSubClass'].map({20:'class1', 30:'class2', 40:'class3', 45:'class4',
50:'class5', 60:'class6', 70:'class7', 75:'class8',
80:'class9', 85:'class10', 90:'class11', 120:'class12',
150:'class13', 160:'class14', 180:'class15', 190:'class16'} )
DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
 def get_spans(doc_stride, max_tokens_for_doc, max_len):
doc_spans = []
start_offset = 0
while start_offset < max_len:
length = max_len - start_offset
if length > max_tokens_for_doc:
length = max_tokens_for_doc
doc_spans.append(DocSpan(start=start_offset, length=length))
if start_offset + length == max_len:
break
start_offset += min(length, doc_stride)
 return doc_spans<categorify>
df = df.reset_index()
df = df.drop('index',axis = 1 )
def convert_examples_to_crops(examples_gen, tokenizer, max_seq_length,
doc_stride, max_query_length, is_training,
cls_token='[CLS]', sep_token='[SEP]', pad_id=0,
sequence_a_segment_id=0,
sequence_b_segment_id=1,
cls_token_segment_id=0,
pad_token_segment_id=0,
mask_padding_with_zero=True,
p_keep_impossible=None,
sep_token_extra=False):

 assert p_keep_impossible is not None, '`p_keep_impossible` is required'
unique_id = 1000000000
num_short_pos, num_short_neg = 0, 0
num_long_pos, num_long_neg = 0, 0
sub_token_cache = {}
crops = []
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info('Converting %s: short_pos %s short_neg %s'
' long_pos %s long_neg %s',
example_index, num_short_pos, num_short_neg,
num_long_pos, num_long_neg)
 query_tokens = tokenizer.tokenize(example.question_text)
 if len(query_tokens)> max_query_length:
query_tokens = query_tokens[0:max_query_length]
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for i, token in enumerate(example.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = sub_token_cache.get(token)
 if sub_tokens is None:
sub_tokens = tokenizer.tokenize(token)
 sub_token_cache[token] = sub_tokens
tok_to_orig_index.extend([i for _ in range(len(sub_tokens)) ])
 all_doc_tokens.extend(sub_tokens)
 tok_start_position = None
tok_end_position = None
if is_training and example.short_is_impossible:
tok_start_position = -1
tok_end_position = -1
if is_training and not example.short_is_impossible:
tok_start_position = orig_to_tok_index[example.start_position]
if example.end_position < len(example.doc_tokens)- 1:
tok_end_position = orig_to_tok_index[
example.end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
tok_long_position = None
if is_training and example.long_is_impossible:
tok_long_position = -1
if is_training and not example.long_is_impossible:
tok_long_position = orig_to_tok_index[example.long_position]
special_tokens_count = 3
if sep_token_extra:
special_tokens_count += 1
max_tokens_for_doc = max_seq_length - len(query_tokens)- special_tokens_count
assert max_tokens_for_doc > 0
doc_spans = get_spans(doc_stride, max_tokens_for_doc, len(all_doc_tokens))
for doc_span_index, doc_span in enumerate(doc_spans):
tokens = []
token_to_orig_map = UNMAPPED * np.ones(( max_seq_length,), dtype=np.int32)
 token_is_max_context = np.zeros(( max_seq_length,), dtype=np.bool)
 token_type_ids = []
short_is_impossible = example.short_is_impossible
start_position = None
end_position = None
special_tokens_offset = special_tokens_count - 1
doc_offset = len(query_tokens)+ special_tokens_offset
if is_training and not short_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_start_position >= doc_start and tok_end_position <= doc_end):
start_position = 0
end_position = 0
short_is_impossible = True
else:
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
long_is_impossible = example.long_is_impossible
long_position = None
if is_training and not long_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_long_position >= doc_start and tok_long_position <= doc_end):
long_position = 0
long_is_impossible = True
else:
long_position = tok_long_position - doc_start + doc_offset
if long_is_impossible:
if np.random.rand() > p_keep_impossible:
continue
tokens.append(cls_token)
 token_type_ids.append(cls_token_segment_id)
 tokens += query_tokens
token_type_ids += [sequence_a_segment_id] * len(query_tokens)
 tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 if sep_token_extra:
tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[
split_token_index] + example.crop_start
token_is_max_context[len(tokens)] = check_is_max_context(doc_spans,
doc_span_index, split_token_index)
 tokens.append(all_doc_tokens[split_token_index])
 token_type_ids.append(sequence_b_segment_id)
 paragraph_len = doc_span.length
tokens.append(sep_token)
 token_type_ids.append(sequence_b_segment_id)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
 while len(input_ids)< max_seq_length:
input_ids.append(pad_id)
 attention_mask.append(0 if mask_padding_with_zero else 1)
 token_type_ids.append(pad_token_segment_id)
 input_ids = np.array(input_ids, dtype=np.int32)
 attention_mask = np.array(attention_mask, dtype=np.bool)
 token_type_ids = np.array(token_type_ids, dtype=np.uint8)
 if is_training and short_is_impossible:
start_position = CLS_INDEX
end_position = CLS_INDEX
if is_training and long_is_impossible:
long_position = CLS_INDEX
if example_index in(0, 10):
if doc_span_index in(0, 5):
logger.info("*** Example ***")
 logger.info("unique_id: %s" %(unique_id))
logger.info("example_index: %s" %(example_index))
logger.info("doc_span_index: %s" %(doc_span_index))
logger.info("tokens: %s" % " ".join(tokens))
logger.info("input_ids: %s" % input_ids)
 logger.info("attention_mask: %s" % np.uint8(attention_mask))
logger.info("token_type_ids: %s" % token_type_ids)
 if is_training and short_is_impossible:
logger.info("short impossible example")
 if is_training and long_is_impossible:
logger.info("long impossible example")
 if is_training and not short_is_impossible:
answer_text = " ".join(tokens[start_position: end_position + 1])
 logger.info("start_position: %d" %(start_position))
logger.info("end_position: %d" %(end_position))
logger.info("answer: %s" %(answer_text))
if short_is_impossible:
num_short_neg += 1
else:
num_short_pos += 1
if long_is_impossible:
num_long_neg += 1
else:
num_long_pos += 1
crop = Crop(
 unique_id=unique_id,
example_index=example_index,
doc_span_index=doc_span_index,
tokens=tokens,
token_to_orig_map=token_to_orig_map,
token_is_max_context=token_is_max_context,
input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
paragraph_len=paragraph_len,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible)
 crops.append(crop)
 unique_id += 1
return crops<define_variables>
drop_id = df[df['LotArea'] > 100000].index
PrelimPrediction = collections.namedtuple("PrelimPrediction",
["crop_index", "start_index", "end_index", "start_logit", "end_logit"])
 NbestPrediction = collections.namedtuple("NbestPrediction", [
"text", "start_logit", "end_logit",
"start_index", "end_index",
"orig_doc_start", "orig_doc_end", "crop_index"] )<string_transform>
drop_id = drop_id[drop_id < 1459]
def clean_text(tok_text):
tok_text = tok_text.replace("
tok_text = tok_text.replace("
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
 return tok_text<split>
df['MasVnrArea'][df[df['MasVnrArea'] > 1500].index] = df['MasVnrArea'].mean()
df['Utilities'][df[df['Utilities']==2].index] = df['Utilities'].mean()
def get_nbest(prelim_predictions, crops, example, n_best_size):
seen, nbest = set() , []
for pred in prelim_predictions:
if len(nbest)>= n_best_size:
break
crop = crops[pred.crop_index]
orig_doc_start, orig_doc_end = -1, -1
if pred.start_index > 0:
if pred.end_index == -1:
tok_tokens = crop.tokens[pred.start_index: pred.start_index + 11]
else:
tok_tokens = crop.tokens[pred.start_index: pred.end_index + 1]
tok_text = " ".join(tok_tokens)
 tok_text = clean_text(tok_text)
 orig_doc_start = int(crop.token_to_orig_map[pred.start_index])
 if pred.end_index == -1:
orig_doc_end = orig_doc_start + 10
else:
orig_doc_end = int(crop.token_to_orig_map[pred.end_index])
 final_text = tok_text
if final_text in seen:
continue
else:
final_text = ""
seen.add(final_text)
 nbest.append(NbestPrediction(
 text=final_text,
start_logit=pred.start_logit, end_logit=pred.end_logit,
start_index=pred.start_index, end_index=pred.end_index,
orig_doc_start=orig_doc_start, orig_doc_end=orig_doc_end,
crop_index=pred.crop_index))
if len(nbest)in(0, 1):
nbest.insert(0, NbestPrediction(text="empty",
start_logit=0.0, end_logit=0.0,
start_index=-1, end_index=-1,
orig_doc_start=-1, orig_doc_end=-1,
crop_index=UNMAPPED))
assert len(nbest)>= 1
return nbest<save_to_csv>
df = df.drop(drop_id )
def write_predictions(examples_gen, all_crops, all_results, n_best_size,
max_answer_length, output_prediction_file,
output_nbest_file, output_null_log_odds_file, verbose_logging,
short_null_score_diff, long_null_score_diff):

 logger.info("Writing predictions to: %s" % output_prediction_file)
 logger.info("Writing nbest to: %s" % output_nbest_file)
 example_index_to_crops = collections.defaultdict(list)
 for crop in all_crops:
example_index_to_crops[crop.example_index].append(crop)
 unique_id_to_result = {result.unique_id: result for result in all_results}
all_predictions = collections.OrderedDict()
all_nbest_json = collections.OrderedDict()
scores_diff_json = collections.OrderedDict()
short_num_empty, long_num_empty = 0, 0
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info(f'[{example_index}]: {short_num_empty} short and {long_num_empty} long empty')
 crops = example_index_to_crops[example_index]
short_prelim_predictions, long_prelim_predictions = [], []
for crop_index, crop in enumerate(crops):
assert crop.unique_id in unique_id_to_result, f"{crop.unique_id}"
result = unique_id_to_result[crop.unique_id]
start_indexes = np.argpartition(result.start_logits, -n_best_size)[-n_best_size:]
start_indexes = [int(x)for x in start_indexes]
end_indexes = np.argpartition(result.end_logits, -n_best_size)[-n_best_size:]
end_indexes = [int(x)for x in end_indexes]
for start_index in start_indexes:
if start_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[start_index] == UNMAPPED:
continue
if not crop.token_is_max_context[start_index]:
continue
for end_index in end_indexes:
if end_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[end_index] == UNMAPPED:
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
short_prelim_predictions.append(PrelimPrediction(
 crop_index=crop_index,
start_index=start_index,
end_index=end_index,
start_logit=result.start_logits[start_index],
end_logit=result.end_logits[end_index]))
long_indexes = np.argpartition(result.long_logits, -n_best_size)[-n_best_size:].tolist()
for long_index in long_indexes:
if long_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[long_index] == UNMAPPED:
continue
if not crop.token_is_max_context[long_index]:
continue
long_prelim_predictions.append(PrelimPrediction(
 crop_index=crop_index,
start_index=long_index, end_index=-1,
start_logit=result.long_logits[long_index],
end_logit=result.long_logits[long_index]))
short_prelim_predictions = sorted(short_prelim_predictions,
key=lambda x: x.start_logit + x.end_logit, reverse=True)
 short_nbest = get_nbest(short_prelim_predictions, crops,
example, n_best_size)
 short_best_non_null = None
for entry in short_nbest:
if short_best_non_null is None:
if entry.text != "":
short_best_non_null = entry
long_prelim_predictions = sorted(long_prelim_predictions,
key=lambda x: x.start_logit, reverse=True)
 long_nbest = get_nbest(long_prelim_predictions, crops,
example, n_best_size)
 long_best_non_null = None
for entry in long_nbest:
if long_best_non_null is None:
if entry.text != "":
long_best_non_null = entry
nbest_json = {'short': [], 'long': []}
for kk, entries in [('short', short_nbest),('long', long_nbest)]:
for i, entry in enumerate(entries):
output = {}
output["text"] = entry.text
output["start_logit"] = entry.start_logit
output["end_logit"] = entry.end_logit
output["start_index"] = entry.start_index
output["end_index"] = entry.end_index
output["orig_doc_start"] = entry.orig_doc_start
output["orig_doc_end"] = entry.orig_doc_end
nbest_json[kk].append(output)
 assert len(nbest_json['short'])>= 1
assert len(nbest_json['long'])>= 1
try:
crop_unique_id = crops[short_best_non_null.crop_index].unique_id
start_score_null = unique_id_to_result[crop_unique_id].start_logits[CLS_INDEX]
end_score_null = unique_id_to_result[crop_unique_id].end_logits[CLS_INDEX]
short_score_null = start_score_null + end_score_null
short_score_diff = short_score_null -(short_best_non_null.start_logit +
short_best_non_null.end_logit)
 if short_score_diff > short_null_score_diff:
final_pred =("", -1, -1)
 short_num_empty += 1
else:
final_pred =(short_best_non_null.text, short_best_non_null.orig_doc_start,
short_best_non_null.orig_doc_end)
 except Exception as e:
print(e)
 final_pred =("", -1, -1)
 short_num_empty += 1
try:
long_score_null = unique_id_to_result[crops[
long_best_non_null.crop_index].unique_id].long_logits[CLS_INDEX]
long_score_diff = long_score_null - long_best_non_null.start_logit
scores_diff_json[example.qas_id] = {'short_score_diff': short_score_diff,
'long_score_diff': long_score_diff}
if long_score_diff > long_null_score_diff:
final_pred +=("", -1)
 long_num_empty += 1
else:
final_pred +=(long_best_non_null.text, long_best_non_null.orig_doc_start)
 except Exception as e:
print(e)
 final_pred +=("", -1)
 long_num_empty += 1
all_predictions[example.qas_id] = final_pred
all_nbest_json[example.qas_id] = nbest_json
if output_prediction_file is not None:
with open(output_prediction_file, "w")as writer:
writer.write(json.dumps(all_predictions, indent=2))
if output_nbest_file is not None:
with open(output_nbest_file, "w")as writer:
writer.write(json.dumps(all_nbest_json, indent=2))
if output_null_log_odds_file is not None:
with open(output_null_log_odds_file, "w")as writer:
writer.write(json.dumps(scores_diff_json, indent=2))
logger.info(f'{short_num_empty} short and {long_num_empty} long empty of'
f' {example_index}')
 return all_predictions<categorify>
y_train = y_train.drop(drop_id )
def convert_preds_to_df(preds, candidates):
num_found_long, num_searched_long = 0, 0
df = {'example_id': [], 'PredictionString': []}
for example_id, pred in preds.items() :
short_text, start_token, end_token, long_text, long_token = pred
df['example_id'].append(example_id + '_short')
 short_answer = ''
if start_token != -1:
short_answer = f'{start_token}:{end_token + 1}'
df['PredictionString'].append(short_answer)
 long_answer = ''
found_long = False
min_dist = 1_000_000
if long_token != -1:
num_searched_long += 1
for candidate in candidates[example_id]:
cstart, cend = candidate.start_token, candidate.end_token
dist = abs(cstart - long_token)
 if dist < min_dist:
min_dist = dist
if long_token == cstart:
long_answer = f'{cstart}:{cend}'
found_long = True
break
if found_long:
num_found_long += 1
else:
logger.info(f"Not found: {min_dist}")
 df['example_id'].append(example_id + '_long')
 df['PredictionString'].append(long_answer)
 df = pd.DataFrame(df)
 print(f'Found {num_found_long} of {num_searched_long}(total {len(preds)})')
 return df<choose_model_class>
dummy_drop = []
for i in cat_columns:
dummy_drop += [ i+'_'+str(df[i].unique() [-1])]
df = pd.get_dummies(df,columns=cat_columns)
 df = df.drop(dummy_drop,axis=1 )
class TFBertForNaturalQuestionAnswering(TFBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.num_labels = config.num_labels
self.bert = TFBertMainLayer(config, name='bert')
 self.initializer = get_initializer(config.initializer_range)
 self.qa_outputs = L.Dense(config.num_labels,
kernel_initializer=self.initializer, name='qa_outputs')
 self.long_outputs = L.Dense(1, kernel_initializer=self.initializer,
name='long_outputs')
 def call(self, inputs, **kwargs):
outputs = self.bert(inputs, **kwargs)
 sequence_output = outputs[0]
logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, -1)
 end_logits = tf.squeeze(end_logits, -1)
 long_logits = tf.squeeze(self.long_outputs(sequence_output), -1)
 return start_logits, end_logits, long_logits
class TFRobertaForNaturalQuestionAnswering(TFRobertaPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.num_labels = config.num_labels
self.roberta = TFRobertaMainLayer(config, name='roberta')
 self.initializer = get_initializer(config.initializer_range)
 self.qa_outputs = L.Dense(config.num_labels,
kernel_initializer=self.initializer, name='qa_outputs')
 self.long_outputs = L.Dense(1, kernel_initializer=self.initializer,
name='long_outputs')
 def call(self, inputs, **kwargs):
outputs = self.roberta(inputs, **kwargs)
 sequence_output = outputs[0]
logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, -1)
 end_logits = tf.squeeze(end_logits, -1)
 long_logits = tf.squeeze(self.long_outputs(sequence_output), -1)
 return start_logits, end_logits, long_logits<string_transform>
X_train = df[:-1459].drop(['index'], axis=1)
 X_test = df[-1459:].drop(['index'], axis=1)
 scaler = StandardScaler()
X_train[num_columns]= scaler.fit_transform(X_train[num_columns])
 X_test[num_columns]= scaler.transform(X_test[num_columns])
 X_train.shape, X_test.shape
def enumerate_tags(text_split):

 special_tokens = ['<P>', '<Table>']
special_token_counts = [0 for _ in range(len(special_tokens)) ]
for index, token in enumerate(text_split):
for special_token_index, special_token in enumerate(special_tokens):
if token == special_token:
cnt = special_token_counts[special_token_index]
if cnt <= 10:
text_split[index] = f'<{special_token[1: -1]}{cnt}>'
special_token_counts[special_token_index] = cnt + 1
return text_split<define_variables>
y_train_log = np.log(y_train )
ALL_MODELS = sum(( tuple(conf.pretrained_config_archive_map.keys())
 for conf in(BertConfig,)) ,())
 MODEL_CLASSES = {
'bert':(BertConfig, TFBertForNaturalQuestionAnswering, BertTokenizer),
'roberta':(RobertaConfig, TFRobertaForNaturalQuestionAnswering, RobertaTokenizer),
}
RawResult = namedtuple("RawResult", ["unique_id", "start_logits", "end_logits",
"long_logits"])
 def set_seed(args):
random.seed(args.seed)
 np.random.seed(args.seed)
 tf.random.set_seed(args.seed )<define_variables>
xgb = XGBRegressor()
xgb.fit(X_train, y_train)
 imp_feature = pd.DataFrame(xgb.feature_importances_ ,columns = ['Importance'],index = X_train.columns)
 imp_feature = imp_feature.sort_values(['Importance'], ascending = False)
 print(imp_feature )
def submit(args, model, tokenizer):
csv_fn = 'submission.csv'
eval_dataset, crops, entries = load_and_cache_crops(args, tokenizer, evaluate=True)
 args.eval_batch_size = args.per_tpu_eval_batch_size
eval_dataset_length = len(eval_dataset[0])
 padded_length = math.ceil(eval_dataset_length / args.eval_batch_size)* args.eval_batch_size
num_pad = padded_length - eval_dataset[0].shape[0]
for ti, t in enumerate(eval_dataset):
pad_tensor = tf.expand_dims(tf.zeros_like(t[0]), 0)
 pad_tensor = tf.repeat(pad_tensor, num_pad, 0)
 eval_dataset[ti] = tf.concat([t, pad_tensor], 0)
 eval_ds = tf.data.Dataset.from_tensor_slices({
'input_ids': tf.constant(eval_dataset[0]),
'attention_mask': tf.constant(eval_dataset[1]),
'token_type_ids': tf.constant(eval_dataset[2]),
'example_index': tf.range(padded_length, dtype=tf.int32)
 })
 eval_ds = eval_ds.batch(batch_size=args.eval_batch_size, drop_remainder=True)
 print("***** Running evaluation *****")
 print(" Num examples = %d", eval_dataset_length)
 print(" Batch size = %d", args.eval_batch_size)
 @tf.function
def predict_step(batch):
outputs = model(batch, training=False)
 return outputs
all_results = []
tic = time.time()
for batch_ind, batch in tqdm(enumerate(eval_ds), total=padded_length // args.per_tpu_eval_batch_size):
example_indexes = batch['example_index']
outputs = predict_step(batch)
 batched_start_logits = outputs[0].numpy()
batched_end_logits = outputs[1].numpy()
batched_long_logits = outputs[2].numpy()
for i, example_index in enumerate(example_indexes):
if example_index >= eval_dataset_length:
continue
eval_crop = crops[example_index]
unique_id = int(eval_crop.unique_id)
 start_logits = batched_start_logits[i].tolist()
end_logits = batched_end_logits[i].tolist()
long_logits = batched_long_logits[i].tolist()
result = RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits,
long_logits=long_logits)
 all_results.append(result)
 eval_time = time.time() - tic
print(" Evaluation done in total %f secs(%f sec per example)",
eval_time, eval_time / padded_length)
 examples_gen = read_nq_examples(entries, is_training=False)
 preds = write_predictions(examples_gen, crops, all_results, args.n_best_size,
args.max_answer_length,
None, None, None,
args.verbose_logging,
args.short_null_score_diff_threshold, args.long_null_score_diff_threshold)
 del crops, all_results
gc.collect()
candidates = read_candidates(['.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'], do_cache=False)
 sub = convert_preds_to_df(preds, candidates ).sort_values('example_id')
 sub.to_csv(csv_fn, index=False, columns=['example_id', 'PredictionString'])
 print(f'***** Wrote submission to {csv_fn} *****')
 result = {}
return result<load_pretrained>
ans = {}
for i in range(1, 222):
imp_col = imp_feature.iloc[:i].index
ridge = KernelRidge(alpha = 0.5263157894736842, coef0 = 3.5, degree = 2, kernel ='polynomial')
 ridge = ridge.fit(X_train[imp_col], y_train_log)
 ans[i] = np.sqrt(mean_squared_error(y_train_log,ridge.predict(X_train[imp_col])) )
def get_convert_args() :
convert_args = argparse.Namespace()
convert_args.fn = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
convert_args.version = 'v0.0.1'
convert_args.prefix = 'nq'
convert_args.num_samples = 1_000_000
convert_args.val_ids = None
convert_args.do_enumerate = False
convert_args.do_not_dump = True
convert_args.num_max_tokens = 400_000
return convert_args<load_pretrained>
minimum = ans[1]
ind_min = 1
for ind in range(1,len(ans.values())) :
if ans[ind] < minimum:
minimum = ans[ind]
ind_min = ind
def load_and_cache_crops(args, tokenizer, evaluate=False):
do_cache = False
cached_crops_fn = 'cached_{}_{}.pkl'.format('test', str(args.max_seq_length))
if os.path.exists(cached_crops_fn)and do_cache:
print("Loading crops from cached file %s", cached_crops_fn)
 with open(cached_crops_fn, "rb")as f:
crops = pickle.load(f)
 else:
entries = convert_nq_to_squad(get_convert_args())
 examples_gen = read_nq_examples(entries, is_training=not evaluate)
 crops = convert_examples_to_crops(examples_gen=examples_gen,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=not evaluate,
cls_token_segment_id=0,
pad_token_segment_id=0,
p_keep_impossible=args.p_keep_impossible if not evaluate else 1.0)
 if do_cache:
with open(cached_crops_fn, "wb")as f:
pickle.dump(crops, f)
 all_input_ids = tf.stack([c.input_ids for c in crops], 0)
 all_attention_mask = tf.stack([c.attention_mask for c in crops], 0)
 all_token_type_ids = tf.stack([c.token_type_ids for c in crops], 0)
 all_attention_mask = tf.cast(all_attention_mask, tf.int32)
 if evaluate:
dataset = [all_input_ids, all_attention_mask, all_token_type_ids]
else:
all_start_positions = tf.convert_to_tensor([f.start_position for f in crops], dtype=tf.int32)
 all_end_positions = tf.convert_to_tensor([f.end_position for f in crops], dtype=tf.int32)
 all_long_positions = tf.convert_to_tensor([f.long_position for f in crops], dtype=tf.int32)
 dataset = [all_input_ids, all_attention_mask, all_token_type_ids,
all_start_positions, all_end_positions, all_long_positions]
return dataset, crops, entries<install_modules>
imp_col = imp_feature.iloc[:ind_min+1].index
!python --version<set_options>
def neg_rmse(y_true, y_pred):
return -1.0*np.sqrt(mean_squared_error(y_true,y_pred))
neg_rmse = make_scorer(neg_rmse )
!nvidia-smi<install_modules>
model = KernelRidge(alpha = 0.6842105263157894, coef0 = 3.5, degree = 2, kernel = 'polynomial')
 model.fit(X_train[imp_col], y_train_log)
 print("RMSE of the whole training set: {}".format(np.sqrt(mean_squared_error(y_train_log,model.predict(X_train[imp_col])))) )
%%time
!pip install /kaggle/input/tf-115-dependencies/tensorboard-1.15.0-py3-none-any.whl
!pip install /kaggle/input/tf-115-dependencies/tensorflow_estimator-1.15.1-py2.py3-none-any.whl
!pip install /kaggle/input/tf-115/tensorflow_gpu-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl
!pip install /kaggle/input/tf-115-dependencies/bert_tensorflow-1.0.1-py2.py3-none-any.whl<import_modules>
y_pred = np.exp(model.predict(X_test[imp_col]))
tf.__version__<set_options>
def save_ans(ans, pasanger_id, name_alg):
submission = pd.DataFrame({'Id':pasanger_id,'SalePrice':ans})
 print(submission.shape)
 filename = r'./{}.csv'.format(name_alg)
 submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )
<find_best_params><EOS> 
save_ans(y_pred, test_id,'submission' )