Kaggle competition: Costa Rican Household Poverty Level Prediction
train['pickup_date']= train['pickup_datetime'].dt.date
train['pickup_day']=train['pickup_datetime'].apply(lambda x:x.day)
 train['pickup_hour']=train['pickup_datetime'].apply(lambda x:x.hour)
 train['pickup_day_of_week']=train['pickup_datetime'].apply(lambda x:calendar.day_name[x.weekday() ])
 train['dropoff_date']= train['dropoff_datetime'].dt.date
train['dropoff_day']=train['dropoff_datetime'].apply(lambda x:x.day)
 train['dropoff_hour']=train['dropoff_datetime'].apply(lambda x:x.hour)
 train['dropoff_day_of_week']=train['dropoff_datetime'].apply(lambda x:calendar.day_name[x.weekday() ] )<feature_engineering>
import torch
train['pickup_latitude_round3']=train['pickup_latitude'].apply(lambda x:round(x,3))
train['pickup_longitude_round3']=train['pickup_longitude'].apply(lambda x:round(x,3))
train['dropoff_latitude_round3']=train['dropoff_latitude'].apply(lambda x:round(x,3))
train['dropoff_longitude_round3']=train['dropoff_longitude'].apply(lambda x:round(x,3))
train.head() 
<feature_engineering>
torch.cuda.is_available()
def calculateDistance(row):
R=6373.0
pickup_lat=radians(row['pickup_latitude'])
 pickup_lon=radians(row['pickup_longitude'])
 dropoff_lat=radians(row['dropoff_latitude'])
 dropoff_lon=radians(row['dropoff_longitude'])
 dlon = dropoff_lon - pickup_lon
dlat = dropoff_lat - pickup_lat
a = sin(dlat / 2)**2 + cos(pickup_lat)* cos(dropoff_lat)* sin(dlon / 2)**2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
distance = R * c
return distance 
<feature_engineering>
from fastai.structured import *
from fastai.column_data import *
train['trip_distance']=train.apply(lambda row:calculateDistance(row),axis=1)
 train.head()<feature_engineering>
np.set_printoptions(threshold=50, edgeitems=20)
 PATH='./'
train['trip_duration_in_hour']=train['trip_duration'].apply(lambda x:x/3600)
 train.head()<filter>
from IPython.display import HTML, display
outlier_trip_duration=train.loc[train['trip_duration_in_hour']>24]
outlier_trip_duration<filter>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
outlier_trip_duration=train.loc[train['trip_duration_in_hour']>24]
outlier_trip_duration<groupby>
train.loc[train['idhogar'] == '2b58d945f'][['Id', 'idhogar', 'v2a1', 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']]
pickup=train.groupby(['pickup_latitude_round3','pickup_longitude_round3'])['id'].count().reset_index().rename(columns={'id':'Num_Trips'} )<groupby>
def show_null(df):
df_null = df.isnull().sum()
print(df_null[df_null>0])
 def show_inf(df):
df_inf = df.isinf().sum()
print(df_inf[df_inf>0])
 def fill_null_value(df):
df['v2a1'].fillna(0, inplace=True)
 df['v18q1'].fillna(0, inplace=True)
 df['rez_esc'].fillna(-1, inplace=True)
 df['meaneduc'].fillna(df['escolari'], inplace=True)
 df['SQBmeaned'].fillna(df['meaneduc']**2, inplace=True)
 return df
def replace_yes_no(df, column):
mapping = {'yes': 1, 'no': 0}
df[column] = [mapping[item] if item in mapping else float(item)for item in df[column]]
return df
def calculate_dependency(df):
dep = []
for index, row in df.iterrows() :
if row['hogar_adul'] == row['hogar_mayor']:
if row['hogar_adul'] > 0:
dep.append(( row['hogar_nin'] + row['hogar_mayor'])* 2 / row['hogar_adul'])
 else:
dep.append(4)
 else:
dep.append(( row['hogar_nin'] + row['hogar_mayor'])/(row['hogar_adul'] - row['hogar_mayor']))
df['dependency'] = dep
return df
def pick_features(df):
return df[['v2a1', 'hacdor', 'hacapo', 'tipovivi2', 'computer', 'total_dis', 'tipovivi4', 'lugar5', 'lugar3', 'female', 'no_toilet_or_energy', 'toilet_and_refrig', 'wall_cond', 'floor_cond', 'roof_cond', 'home_cond', 'SQB_home_cond', 'v14a', 'refrig', 'v18q1', 'pisonotiene', 'cielorazo', 'abastaguadentro', 'abastaguafuera', 'abastaguano', 'noelec', 'sanitario1', 'energcocinar1', 'energcocinar4', 'elimbasu4', 'elimbasu5', 'dependency', 'overcrowding', 'SQBovercrowding', 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4']]
def add_total_disable(df):
print('add_total_disable')
 df['total_dis'] = df['dis']
for index, row in df[df.dis == 1].iterrows() :
others = df[df.idhogar == row['idhogar']]
for index_2, row in others.iterrows() :
if index != index_2:
df.loc[index_2, 'total_dis'] += 1
df['avg_dis_adult'] = df['dis'] / df['total_adults']
df['avg_dis'] = df['dis'] / df['tamhog']
return df
def add_total_adults(df):
print('add_total_adults')
 df['is_adult'] = df.apply(lambda x: int(bool(int(x['age']/18))), axis=1)
 def get_total_adults(x):
if x['parentesco1'] == 1:
value = df.loc[df.idhogar == x['idhogar']]['is_adult'].sum()
if value == 0:
return 0.01
else:
return value
else:
return 0
df['total_adults'] = df.apply(get_total_adults, axis=1)
 return df
def add_avg_devices(df):
df['avg_qmobilephone_adult'] = df['qmobilephone']/df['total_adults']
df['avg_qmobilephone'] = df['qmobilephone']/df['tamhog']
df['avg_computer_adult'] = df['computer'] / df['total_adults']
df['avg_computer'] = df['computer'] / df['tamhog']
df['avg_television_room'] = df['television'] / df['rooms']
df['avg_television_adult'] = df['television'] / df['total_adults']
df['avg_television'] = df['television'] / df['tamhog']
df['avg_tablet'] = df['v18q1'] / df['tamhog']
df['avg_tablet_adult'] = df['v18q1'] / df['total_adults']
return df
def add_education_score(df):
print('add_education_score')
 df['edu_level'] = df['instlevel9']*(2**8)+ df['instlevel8']*(2**7)+ df['instlevel7']*(2**6)+ df['instlevel6']*(2**5)+ df['instlevel5']*(2**4)+ df['instlevel4']*(2**3)+ df['instlevel3']*(2**2)+ df['instlevel2']*(2**1)+ df['instlevel1']
def get_total_adults_edu(x):
if x['parentesco1'] == 1:
adults = df.loc[(df.idhogar == x['idhogar'])&(df.is_adult == 1)]
if len(adults)> 0:
return adults['edu_level'].sum() / len(adults)
 else:
return 0
else:
return 0
df['adults_edu'] = df.apply(get_total_adults_edu, axis=1)
 return df
def add_no_toilet_or_energy(df):
df['no_toilet_or_energy'] = df['energcocinar1'] + df['sanitario1']
df['toilet_and_refrig'] = df['v14a'] + df['refrig']
df['wall_cond'] = df.apply(lambda row: row['epared1'] + 2*row['epared2'] + 4*row['epared3'], axis=1)
 df['floor_cond'] = df.apply(lambda row: row['etecho1'] + 2*row['etecho2'] + 4*row['etecho3'], axis=1)
 df['roof_cond'] = df.apply(lambda row: row['eviv1'] + 2*row['eviv2'] + 4*row['eviv3'], axis=1)
 df['home_cond'] = df.apply(lambda row: row['wall_cond'] + row['floor_cond'] + row['roof_cond'], axis=1)
 df['SQB_home_cond'] = df.apply(lambda row: row['home_cond']**2, axis=1)
 return df
drop=train.groupby(['dropoff_latitude_round3','dropoff_longitude_round3'])['id'].count().reset_index().rename(columns={'id':'Num_Trips'} )<groupby>
train = fill_null_value(train)
 train = replace_yes_no(train, 'edjefe')
 train = replace_yes_no(train, 'edjefa')
 train = calculate_dependency(train)
 train = add_total_adults(train)
 train = add_total_disable(train)
 train = add_no_toilet_or_energy(train)
 train = add_education_score(train)
 test = fill_null_value(test)
 test = replace_yes_no(test, 'edjefe')
 test = replace_yes_no(test, 'edjefa')
 test = calculate_dependency(test)
 test = add_total_adults(test)
 test = add_total_disable(test)
 test = add_no_toilet_or_energy(test)
 test = add_education_score(test)
pickup=train.groupby(['pickup_latitude_round3','pickup_longitude_round3'])['trip_duration'].mean().reset_index().rename(columns={'trip_duration':'Avg_Trip_duration'} )<groupby>
avg_duration_day=train.groupby(['pickup_day_of_week'])['trip_duration'].mean().reset_index().rename(columns={'trip_duration':'avg_trip_duration'} )<compute_test_metric>
def calculateBearing(lat1,lng1,lat2,lng2):
R = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))<feature_engineering>
train_head = train.loc[train.parentesco1 == 1]
test_head = test.loc[test.parentesco1 == 1]
train['bearing']=train.apply(lambda row:calculateBearing(row['pickup_latitude_round3'],row['pickup_longitude_round3'],row['dropoff_latitude_round3'],row['dropoff_longitude_round3']),axis=1 )<count_values>
def get_overfit_training(ptrain):
traind = ptrain
ntrain = ptrain[ptrain.Target == 1]
ntrain['age'] += 1
traind = traind.append(ntrain)
 ntrain['age'] += 1
traind = traind.append(ntrain)
 ntrain['age'] += 1
traind = traind.append(ntrain)
 ntrain['age'] -= 4
traind = traind.append(ntrain)
 ntrain['age'] -= 1
traind = traind.append(ntrain)
 ntrain['age'] -= 1
traind = traind.append(ntrain)
 ntrain['age'] -= 1
traind = traind.append(ntrain)
 n2train = ptrain[ptrain.Target == 2]
n2train['age'] += 1
traind = traind.append(n2train)
 n2train['age'] += 1
traind = traind.append(n2train)
 n2train['age'] -=3
traind = traind.append(n2train)
 n3train = ptrain[ptrain.Target == 3]
n3train['age'] += 1
traind = traind.append(n3train)
 n2train['age'] += 1
traind = traind.append(n3train)
 n2train['age'] += 1
traind = traind.append(n3train)
 n2train['age'] -= 4
traind = traind.append(n3train)
 return traind
ptrain_head = get_overfit_training(train_head); ptrain_head.shape
train['store_and_fwd_flag'].value_counts()<predict_on_test>
ptest_head = test_head.copy()
ptest_head['Target'] = np.random.randint(1, high=5, size=ptest_head.shape[0] )
coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
kmeans = KMeans(n_clusters=8, random_state=0 ).fit(coords)
 train.loc[:, 'pickup_neighbourhood'] = kmeans.predict(train[['pickup_latitude', 'pickup_longitude']])
 train.loc[:, 'dropoff_neighbourhood'] = kmeans.predict(train[['dropoff_latitude', 'dropoff_longitude']])
 city_long_border =(-74.03, -73.75)
 city_lat_border =(40.63, 40.85)
 fig, ax = plt.subplots(ncols=1, sharex=True, sharey=True)
 ax.scatter(train['pickup_longitude'], train['pickup_latitude'],
c=train['pickup_neighbourhood'], label='train', alpha=0.1)
 fig.suptitle('Pickup Neighbourhood')
 ax.set_ylabel('latitude')
 ax.set_xlabel('longitude')
 plt.ylim(city_lat_border)
 plt.xlim(city_long_border )<compute_test_metric>
ptrain_target = ptrain_head['Target']
ptrain_head.drop('Target', axis=1, inplace=True)
 ptrain_head['Target'] = ptrain_target
a =(ptrain_head.columns == ptest_head.columns)
 a[a == False]
print("Average speed is",np.mean(train['avg_speed_kph']),"kph" )<feature_engineering>
ptrain_head.set_index('Id', inplace=True)
 ptest_head.set_index('Id', inplace=True )
test['pickup_datetime']=pd.to_datetime(test['pickup_datetime'],format='%Y-%m-%d %H:%M:%S')
 test['pickup_date']= test['pickup_datetime'].dt.date
test['pickup_day']=test['pickup_datetime'].apply(lambda x:x.day)
 test['pickup_hour']=test['pickup_datetime'].apply(lambda x:x.hour)
 test['pickup_day_of_week']=test['pickup_datetime'].apply(lambda x:calendar.day_name[x.weekday() ])
 test['pickup_latitude_round3']=test['pickup_latitude'].apply(lambda x:round(x,3))
test['pickup_longitude_round3']=test['pickup_longitude'].apply(lambda x:round(x,3))
test['dropoff_latitude_round3']=test['dropoff_latitude'].apply(lambda x:round(x,3))
test['dropoff_longitude_round3']=test['dropoff_longitude'].apply(lambda x:round(x,3))
test['trip_distance']=test.apply(lambda row:calculateDistance(row),axis=1)
 test['bearing']=test.apply(lambda row:calculateBearing(row['pickup_latitude_round3'],row['pickup_longitude_round3'],row['dropoff_latitude_round3'],row['dropoff_longitude_round3']),axis=1)
 test.loc[:, 'pickup_neighbourhood'] = kmeans.predict(test[['pickup_latitude', 'pickup_longitude']])
 test.loc[:, 'dropoff_neighbourhood'] = kmeans.predict(test[['dropoff_latitude', 'dropoff_longitude']] )<drop_column>
alld = ptrain_head.append(ptest_head )
drop_cols=['avg_speed_kph','trip_duration_in_hour','dropoff_date','dropoff_day','dropoff_hour','dropoff_day_of_week','dropoff_datetime','pickup_latitude','pickup_longitude','dropoff_latitude','dropoff_longitude']
training=train.drop(drop_cols,axis=1)
 testing=test.drop(['pickup_latitude','pickup_longitude','dropoff_latitude','dropoff_longitude'],axis=1)
<drop_column>
alld.loc[set(ptrain_head.index.tolist())].shape == ptrain_head.shape
training['log_trip_duration']=training['trip_duration'].apply(lambda x:np.log(x))
training.drop(['trip_duration'],axis=1,inplace=True )<categorify>
pd.options.mode.chained_assignment = None
def encodeDays(day_of_week):
day_dict={'Sunday':0,'Monday':1,'Tuesday':2,'Wednesday':3,'Thursday':4,'Friday':5,'Saturday':6}
return day_dict[day_of_week]<feature_engineering>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000 )
training['pickup_day_of_week']=training['pickup_day_of_week'].apply(lambda x:encodeDays(x))
testing['pickup_day_of_week']=testing['pickup_day_of_week'].apply(lambda x:encodeDays(x)) 
<save_to_csv>
cats = ['hacdor', 'rooms', 'hacapo', 'v14a', 'refrig', 'r4h1',
'r4h2', 'r4m1', 'r4m2', 'tamhog', 'tamviv', 'escolari',
'hhsize', 'pared', 'piso', 'techo', 'cielorazo', 'abastagua', 'electricity', 'sanitario',
'energcocinar', 'elimbasu', 'epared', 'etecho', 'eviv', 'dis', 'male_or_famale', 'estadocivil',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total', 'instlevel', 'bedrooms', 'tipovivi',
'computer', 'television', 'mobilephone', 'lugar', 'area', 'is_adult']
training.to_csv("input_training.csv",index=False)
 testing.to_csv("input_testing.csv",index=False)
 del training
del testing
del train
del test<categorify>
to_drop = ['v18q', 'r4h3', 'r4m3', 'r4t1', 'r4t2', 'r4t3', 'parentesco', 'idhogar', 'SQB', 'agesq', ]
def LabelEncoding(train_df,test_df,max_levels=2):
for col in train_df:
if train_df[col].dtype == 'object':
if len(list(train_df[col].unique())) <= max_levels:
le = preprocessing.LabelEncoder()
le.fit(train_df[col])
 train_df[col]=le.transform(train_df[col])
 test_df[col]=le.transform(test_df[col])
 return [train_df,test_df]
def readInputAndEncode(input_path,train_file,test_file,target_column):
training=pd.read_csv(input_path+train_file)
 testing=pd.read_csv(input_path+test_file)
 training,testing=LabelEncoding(training,testing)
 train_cols=training.columns.tolist()
test_cols=testing.columns.tolist()
col_in_train_not_test=set(train_cols)-set(test_cols)
 for col in col_in_train_not_test:
if col!=target_column:
testing[col]=0
col_in_test_not_train=set(test_cols)-set(train_cols)
 for col in col_in_test_not_train:
training[col]=0
print("Training Data Shape after Processing ",training.shape)
 print("Testing Data Shape after Processing ",testing.shape)
 return [training,testing]<drop_column>
to_merge = ['pared', 'piso', 'techo', 'abastagua',
{'electricity': ['public', 'planpri', 'noelec', 'coopele']},
'sanitario', 'energcocinar', 'elimbasu', 'epared', 'etecho',
'eviv', {'male_or_famale': ['male', 'female']},
'estadocivil', 'instlevel', 'tipovivi', 'lugar', 'area']
train,test=readInputAndEncode("",'input_training.csv','input_testing.csv','log_trip_duration')
 train.drop(['pickup_date'],axis=1,inplace=True)
 test.drop(['pickup_date'],axis=1,inplace=True)
 train.drop(['pickup_datetime'],axis=1,inplace=True)
 test.drop(['pickup_datetime'],axis=1,inplace=True)
 test_id=test['id']
train.drop(['id'],axis=1,inplace=True)
 test.drop(['id'],axis=1,inplace=True )<split>
def merge_columns(df, to_merge):
for c in to_merge:
if isinstance(c, str):
new_col = df[[ col for col in df if col.startswith(c)]].astype(str ).apply(lambda x: x.str.cat() , axis=1)
 df = df.drop([ col for col in df if col.startswith(c)], axis=1)
 df[c] = new_col
elif isinstance(c, dict):
new_col_name = list(c.keys())[0]
new_col = df[c[new_col_name]].astype(str ).apply(lambda x: x.str.cat() , axis=1)
 df = df.drop(c[new_col_name], axis=1)
 df[new_col_name] = new_col
return df
def GetFeaturesAndSplit(train,test,target,imputing_strategy='median',split=0.25,imputation=True):
labels=np.array(train[target])
 training=train.drop(target, axis = 1)
 training = np.array(training)
 testing=np.array(test)
 if imputation==True:
imputer=Imputer(strategy=imputing_strategy,missing_values=np.nan)
 imputer.fit(training)
 training=imputer.transform(training)
 testing=imputer.transform(testing)
 train_features, validation_features, train_labels, validation_labels = train_test_split(training, labels, test_size = split, random_state = 42)
 return [train_features,validation_features,train_labels,validation_labels,testing]<split>
def drop_columns(df, to_drop):
for c in to_drop:
df = df.drop([ col for col in df if col.startswith(c)], axis=1)
 return df
train_features,validation_features,train_labels,validation_labels,testing=GetFeaturesAndSplit(train,test,'log_trip_duration',imputation=False )<train_model>
alld = drop_columns(alld, to_drop )
lm=linear_model.LinearRegression()
lm.fit(train_features,train_labels )<predict_on_test>
def macro_f1(y_pred, targ):
target = targ.cpu().numpy().argmax(axis=1)
 y_predict = y_pred.cpu().numpy().argmax(axis=1)
 return f1_score(target, y_predict, average='macro' )
valid_pred=lm.predict(validation_features )<compute_test_metric>
rmse=mean_squared_error(validation_labels,valid_pred)
 print("Root Mean Squared Error for Linear Regression(log scale)",rmse )<save_to_csv>
test_pred=lm.predict(testing)
 submit=pd.DataFrame()
submit['id']=test_id
submit['trip_duration']=np.exp(test_pred)
 submit.to_csv("submission_linear_regression_baseline.csv",index=False)
 del submit<choose_model_class>
for v in cats: alld[v] = alld[v].astype('category' ).cat.as_ordered()
rf = RandomForestRegressor(n_estimators = 100, random_state = 42 )<train_model>
alld['Target'] = alld['Target'].astype('int' )
rf.fit(train_features,train_labels )<compute_train_metric>
alld.drop('rez_esc', axis=1, inplace=True )
valid_pred_rf=rf.predict(validation_features)
 rmse=mean_squared_error(validation_labels,valid_pred_rf)
 print("Root Mean Squared Error for Random Forest",rmse)
<save_to_csv>
DataFrameSummary(alld ).summary()
test_pred=rf.predict(testing)
 submit=pd.DataFrame()
submit['id']=test_id
submit['trip_duration']=np.exp(test_pred)
 submit.to_csv("submission_random_forest_baseline.csv",index=False )<define_variables>
contin_vars = [c for c in alld.columns if c not in cats + ['Target', 'Id']]
use_wta = True
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000
model_path = '.. /input/tpu-2020-01-22/'
for k in ['on_kaggle_server','nq_test_file','public_dataset','private_dataset']:
print(k,globals() [k],sep=': ' )<load_pretrained>
for v in contin_vars:
alld[v] = alld[v].fillna(0 ).astype('float32' )
model = tf.saved_model.load(model_path )<save_to_csv>
_, _, nas, mapper = proc_df(alld, 'Target', do_scale=True )
to_pkl.jsonl_to_pkl(source=nq_test_file,output='features.pkl',
vocab=model_path +'assets/vocab-nq.txt',
max_contexts=-1,lower_case=True )<data_type_conversions>
traindf = alld.loc[set(ptrain_head.index.tolist())]
traindf.shape[0] == ptrain_head.shape[0]
pkl_to_tfrecords._convert(source='features.pkl',output='all.tfrecords',meta_data='meta_data',shuffle=False,shuffle_size=0,yield_segment_variant='nolabels' )<categorify>
testdf = alld.loc[set(ptest_head.index.tolist())]
testdf.shape[0] == ptest_head.shape[0]
def input_fn(input_file_pattern,seq_length=512,batch_size=4):
def mk_labels(ex):
qlen = ex.pop('question_len')
 dlen = ex.pop('data_len')
 input_mask = tf.sequence_mask(dlen,seq_length,dtype=tf.int32)
 ex['input_mask'] = input_mask
ex['segment_ids'] = tf.minimum(input_mask,1-tf.sequence_mask(qlen,seq_length,dtype=tf.int32))
return ex
name_to_features = {
'input_ids' : tf.io.FixedLenFeature([seq_length], tf.int64),
'question_len': tf.io.FixedLenFeature([], tf.int64),
'data_len' : tf.io.FixedLenFeature([], tf.int64),
}
name_to_features['unique_id'] = tf.io.FixedLenFeature([2], tf.int64)
 def decode(record):
ex = tf.io.parse_single_example(record, name_to_features)
 for k,v in ex.items() :
if k!='unique_id':
ex[k] = tf.cast(v,tf.int32)
 return ex
input_files = tf.io.gfile.glob(input_file_pattern)
 d = tf.data.TFRecordDataset(input_files)
 d = d.map(decode)
 d = d.batch(batch_size,drop_remainder=False)
 d = d.prefetch(128)
 return d 
<categorify>
df_test, _, nas, mapper = proc_df(testdf, 'Target', do_scale=True, mapper=mapper )
def output_fn() :
def _output_fn(unique_id,model_output,n_keep=100):
pos_logits,ans_logits,long_mask,short_mask,cross = model_output
long_span_logits = pos_logits
mask = tf.cast(tf.expand_dims(long_mask,-1),long_span_logits.dtype)
 long_span_logits = long_span_logits-10000*mask
long_p = tf.nn.softmax(long_span_logits,axis=1)
 short_span_logits = pos_logits
short_span_logits -= 10000*tf.cast(tf.expand_dims(short_mask,-1),short_span_logits.dtype)
 start_logits,end_logits = short_span_logits[:,:,0],short_span_logits[:,:,1]
batch_size,seq_length = short_span_logits.shape[0],short_span_logits.shape[1]
seq = tf.range(seq_length)
 i_leq_j_mask = tf.cast(tf.expand_dims(seq,1)>tf.expand_dims(seq,0),short_span_logits.dtype)
 i_leq_j_mask = tf.expand_dims(i_leq_j_mask,0)
 logits = tf.expand_dims(start_logits,2)+tf.expand_dims(end_logits,1)+cross
logits -= 10000*i_leq_j_mask
logits = tf.reshape(logits, [batch_size,seq_length*seq_length])
 short_p = tf.nn.softmax(logits)
 indices = tf.argsort(short_p,axis=1,direction='DESCENDING')[:,:n_keep]
short_p = tf.gather(short_p,indices,batch_dims=1)
 return dict(unique_id = unique_id,
ans_logits= ans_logits,
long_p = long_p,
short_p = short_p,
short_p_indices = indices)
 return _output_fn 
<categorify>
df, y, nas, mapper = proc_df(traindf, 'Target', do_scale=True, mapper=mapper )
d = input_fn('all.tfrecords',batch_size=64)
 if public_dataset:
d = d.take(3)
 if not on_kaggle_server:
d = tqdm.notebook.tqdm(d)
 results = []
output = output_fn()
for b in d:
unique_id = b.pop('unique_id' ).numpy()
b = [b['data_len'],b['input_ids'],b['question_len']]
out_dict = output(unique_id,model(b,training=False))
for k,v in out_dict.items() :
if isinstance(v,tf.Tensor):
out_dict[k] = v.numpy()
results.append(out_dict)
 raw_results = postprocess.read_rawresult(results)
<statistical_test>
DataFrameSummary(df ).summary()
examples = postprocess.compute_examples(raw_results,records )<load_from_csv>
samp_size = df.shape[0]
train_ratio = 0.8
train_size = int(samp_size * train_ratio)
 train_size
index = pd.read_csv('.. /input/tensorflow2-question-answering/sample_submission.csv' ).example_id
submission = postprocess.create_submission_df(predictions,index=index,
long_threshold=0.94 if use_wta else 0.77,
short_threshold=0.94 if use_wta else 0.77 ,
yes_no_threshold=0.6 )<save_to_csv>
val_idx = random.sample(range(0, df.shape[0]), df.shape[0] - train_size )
submission.to_csv('submission.csv' )<import_modules>
val_idx = sorted(val_idx)
 len(val_idx )
import sys
import json
from tqdm.notebook import tqdm
import numpy as np
import pandas as pd
import time
from contextlib import contextmanager<define_variables>
yl = np.zeros(( y.size, 4))
PATH_TO_DATA = '.. /input/tensorflow2-question-answering/'
PATH_TO_CUSTOM_BERT = '.. /input/bert-wwm-063065-checkpoint/'
PATH_TO_CUSTOM_BERT_WEIGHTS = PATH_TO_CUSTOM_BERT + \
'20200109_bert_joint_wwm_output/20200109_bert_joint_wwm_output/'
CKPT_NAME = PATH_TO_CUSTOM_BERT_WEIGHTS + '20200109_bert_joint_wwm_output_model.ckpt-15458'
PATH_TO_TF_WHEELS = PATH_TO_CUSTOM_BERT + \
'tensorflow_gpu_1_13_1_with_deps_whl/tensorflow_gpu_1_13_1_with_deps_whl/'
OUT_PREDICT_JSON = 'simplified-nq-test-pred.json'<define_variables>
yindex = np.array([t -1 for t in y]); yindex
LRU_CACHE_SIZE = 30000
MAX_SEQ_LEN = 512
DOC_STRIDE = 192
MAX_QUERY_LEN = 24
LONG_ANS_THRES = 4.4524
SHORT_ANS_THRES = 7.7251
LONG_ANS_THRES_FINAL = 2.1165
SHORT_ANS_THRES_FINAL = 7.6657<feature_engineering>
yl[np.arange(y.size), yindex] = 1; yl
def process_test_set_for_bert_joint() :
with open(PATH_TO_DATA + 'simplified-nq-test.jsonl')as f_in, \
open('simplified-nq-test_with_doc_tokens.jsonl', 'w')as f_out:
for i, line in tqdm(enumerate(f_in)) :
json_line = json.loads(line)
 json_line["document_tokens"] = []
for token in json_line['document_text'].split(' '):
json_line["document_tokens"].append({"token":token,
"start_byte": -1,
"end_byte": -1,
"html_token": '<' in token})
 json_line['annotations'] = []
json_line["document_title"] = \
json_line["document_tokens"][0]["token"]
f_out.write(json.dumps(json_line)+ '
')
 !gzip simplified-nq-test_with_doc_tokens.jsonl<install_modules>
md = ColumnarModelData.from_data_frame(PATH, val_idx, df, yl, cat_flds=cats, bs=64, is_reg=False, is_multi=True, test_df=df_test )
def setup_tensorflow_1_13_1() :
!pip install --no-deps $PATH_TO_TF_WHEELS/*.whl > /dev/null 2>&1
!cp -r $PATH_TO_CUSTOM_BERT/bert-tensorflow-1.0.1/.
 !cd bert-tensorflow-1.0.1/bert-tensorflow-1.0.1/; python setup.py install > /dev/null 2>&1<define_variables>
cat_sz = [(c, len(alld[c].cat.categories)+1)for c in cats]
def run_bert_inference() :
!python $PATH_TO_CUSTOM_BERT/run_nq.py --logtostderr \
--bert_config_file=$PATH_TO_CUSTOM_BERT/bert_config.json \
--vocab_file=$PATH_TO_CUSTOM_BERT/vocab-nq.txt \
--tokenizer_cache_size=$LRU_CACHE_SIZE \
--max_seq_length=$MAX_SEQ_LEN \
--doc_stride=$DOC_STRIDE \
--max_query_length=$MAX_QUERY_LEN \
--init_checkpoint=$CKPT_NAME \
--predict_file=simplified-nq-test_with_doc_tokens.jsonl.gz \
--do_predict \
--output_dir=bert_model_output \
--output_prediction_file=simplified-nq-test-pred.json \
> /dev/null 2>&1
!rm -rf bert-tensorflow-1.0.1/
!rm simplified-nq-test_with_doc_tokens.jsonl.gz
!rm -rf bert_model_output<define_variables>
emb_szs = [(c, min(50,(c+1)//2)) for _,c in cat_sz]
def postprocess_predictions(pred_json_path=OUT_PREDICT_JSON,
long_thres=LONG_ANS_THRES,
short_thres=SHORT_ANS_THRES):
empty_answer = {'candidate_index': -1,
'end_byte': -1,
'end_token': -1,
'start_byte': -1,
'start_token': -1}
with open(pred_json_path)as f:
pred_json = json.load(f)
 pred_json_processed = {'predictions': []}
for i, entry in enumerate(pred_json['predictions']):
entry_copy = entry.copy()
ans_type = entry['answer_type']
if entry['long_answer_score'] < long_thres:
entry_copy['long_answer'] = empty_answer
if entry['short_answers_score'] < short_thres:
entry_copy['short_answers'] = []
if ans_type== 0:
entry_copy['long_answer'] = empty_answer
entry_copy['short_answers'] = [empty_answer]
elif ans_type == 1:
entry_copy['yes_no_answer'] = "YES"
entry_copy['short_answers'] = [empty_answer]
elif ans_type== 2:
entry_copy['yes_no_answer'] = "NO"
entry_copy['short_answers'] = [empty_answer]
elif ans_type == 3:
entry_copy['yes_no_answer'] = "NONE"
elif ans_type == 4:
entry_copy['yes_no_answer'] = "NONE"
entry_copy['short_answers'] = [empty_answer]
pred_json_processed['predictions'].append(entry_copy)
 return pred_json_processed<define_variables>
DataFrameSummary(df )
def form_submission_file(pred_json,
long_thres=LONG_ANS_THRES,
short_thres=SHORT_ANS_THRES):
example_ids, preds = [], []
for entry in pred_json['predictions']:
example_ids.append(str(entry['example_id'])+ '_long')
 example_ids.append(str(entry['example_id'])+ '_short')
 score = entry['long_answer_score']
if score >= long_thres:
long_pred = '{}:{}'.format(entry['long_answer']['start_token'],
entry['long_answer']['end_token'])
 if long_pred == '-1:-1':
long_pred = ""
else:
long_pred = ""
if entry['yes_no_answer'] != "NONE":
short_pred = entry['yes_no_answer']
elif score >= short_thres:
if entry['short_answers']:
short_pred = '{}:{}'.format(entry['short_answers'][0]['start_token'],
entry['short_answers'][0]['end_token'])
 else:
short_pred = ""
if short_pred == '-1:-1':
short_pred = ""
else:
short_pred = ""
preds.extend([long_pred, short_pred])
 sub_df = pd.DataFrame({'example_id':example_ids,
'PredictionString': preds})\ 
.sort_values(by='example_id' ).reset_index(drop=True)
 return sub_df<load_pretrained>
m = md.get_learner(emb_szs, len(df.columns)-len(cats),
0.3, 4, [500,250], [0.1,0.05])
 lr = 1e-3
with timer('ALL'):
with timer('Processing test set'):
process_test_set_for_bert_joint()
with timer('Setting up packages'):
setup_tensorflow_1_13_1()
with timer('Running inference'):
run_bert_inference()
with timer('Forming final submission file'):
pred_json_processed = postprocess_predictions(OUT_PREDICT_JSON,
long_thres=LONG_ANS_THRES,
short_thres=SHORT_ANS_THRES 
)
 sub_df = form_submission_file(pred_json_processed,
long_thres=LONG_ANS_THRES_FINAL,
short_thres=SHORT_ANS_THRES_FINAL 
)
 sub_df.to_csv("submission.csv", index=False )<define_variables>
m.fit(lr, 3, metrics=[macro_f1], cycle_len=3 )
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
m.fit(lr, 2, metrics=[macro_f1], cycle_len=3 )
if True:
importlib.reload(bert_utils )<load_from_disk>
m.fit(lr, 1, metrics=[macro_f1], cycle_len=3 )
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4)) 
<choose_model_class>
pred_test=m.predict(is_test=True )
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<define_variables>
predicts = pred_test.argmax(axis=1); predicts
small_config = config.copy()
small_config['vocab_size']=16
small_config['hidden_size']=64
small_config['max_position_embeddings'] = 32
small_config['num_hidden_layers'] = 4
small_config['num_attention_heads'] = 4
small_config['intermediate_size'] = 256
small_config<categorify>
predicts = predicts + 1; predicts
model= mk_model(config )<feature_engineering>
df_test['Target'] = predicts
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))<categorify>
df_test['Id'] = df_test.index
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int32)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
<categorify>
final = df_test[['Id', 'Target']]
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )<predict_on_test>
test['Target'] = np.random.randint(1, high=5, size=test.shape[0])
 ftest = test[['Id', 'Target']]
result=model.predict_generator(ds,verbose=1 )<load_pretrained>
dtest = ftest.set_index('Id')
 dtest['Id'] = dtest.index; dtest
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )<create_dataframe>
dtest.loc[final.index.tolist() ]['Target'] = final['Target']
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx"] )<import_modules>
final.loc['ID_21471f283']['Target']
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None<prepare_output>
dtest['Target'] = dtest.apply(lambda x: x['Target'] if x['Id'] not in final['Id'] else final.loc[x['Id']]['Target'], axis=1 )
<find_best_params><EOS> 
dtest.to_csv('submission_dl.csv', index=False )