Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<randomize_order>
Dropout_model = 0.25
FVC_weight = 0.15
Confidence_weight = 0.15
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df<categorify>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle(AUX_MODELS+'test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]<init_hyperparams>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(45 )
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'tweedie_variance_power': 1.1,
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 2**11-1,
'min_data_in_leaf': 2**12-1,
'feature_fraction': 0.5,
'max_bin': 100,
'n_estimators': 1400,
'boost_from_average': False,
'verbose': 1,
} 
<define_variables>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
SEED = 42
seed_everything(SEED)
 lgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1913
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/m5-simple-fe/grid_part_1.pkl'
PRICE = '.. /input/m5-simple-fe/grid_part_2.pkl'
CALENDAR = '.. /input/m5-simple-fe/grid_part_3.pkl'
LAGS = '.. /input/m5-lags-features/lags_df_28.pkl'
MEAN_ENC = '.. /input/m5-custom-features/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/m5groupkfold/'
STORES_IDS = ['CA_1']
CV_FOLDS = [0,1,2]
SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )<init_hyperparams>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
for store_id in STORES_IDS:
print('Train', store_id)
 grid_df, features_columns = get_data_by_store(store_id)
 train_mask = grid_df['d']<=END_TRAIN
preds_mask = grid_df['d']>(END_TRAIN-100)
 folds = GroupKFold(n_splits=3)
 grid_df['groups'] = grid_df['tm_w'].astype(str)+ '_' + grid_df['tm_y'].astype(str)
 split_groups = grid_df[train_mask]['groups']
X,y = grid_df[train_mask][features_columns], grid_df[train_mask][TARGET]
grid_df = grid_df[preds_mask].reset_index(drop=True)
 keep_cols = [col for col in list(grid_df)if '_tmp_' not in col]
grid_df = grid_df[keep_cols]
grid_df.to_pickle('test_'+store_id+'.pkl')
 del grid_df
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_)
 print(len(trn_idx),len(val_idx))
tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
v_X, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 seed_everything(SEED)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 100 
)
 model_name = 'lgb_model_'+STORES_IDS[0]+'_'+str(fold_)+'.bin'
pickle.dump(estimator, open(model_name, 'wb'))
del train_data, valid_data, estimator
gc.collect()
MODEL_FEATURES = features_columns<define_variables>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
for fold_ in CV_FOLDS:
print("FOLD:", fold_)
 all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 for store_id in STORES_IDS:
model_path = 'lgb_model_'+store_id+'_'+str(fold_)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb'))
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
all_preds = all_preds.reset_index(drop=True)
 print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
all_preds.to_csv('all_preds_'+str(fold_)+'.csv',index=False)
 del temp_df, all_preds<load_from_csv>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
all_preds_0=pd.read_csv(AUX_MODELS+'all_preds_0'+'.csv')
 all_preds_1=pd.read_csv(AUX_MODELS+'all_preds_1'+'.csv')
 all_preds_2=pd.read_csv(AUX_MODELS+'all_preds_2'+'.csv' )<create_dataframe>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
final_all_preds = pd.DataFrame()
final_all_preds['id'] = all_preds_1['id']
for item in all_preds_1:
if item!='id':
final_all_preds[item]=(all_preds_0[item]*(1/3)) +(all_preds_1[item]*(1/3)) +(all_preds_2[item]*(1/3))
final_all_preds<save_to_csv>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(final_all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('submission'+'.csv', index=False )<import_modules>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.95 )
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb<define_variables>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }<set_options>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
pd.options.display.max_columns = 50<define_variables>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
h = 28
max_lags = 70
tr_last = 1913
fday = datetime(2016,4, 25)
 fday<load_from_csv>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )<define_variables>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
FIRST_DAY = 800<correct_missing_values>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
train_data = lgb.Dataset(X_train, label = y_train, categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_inds = np.random.choice(len(X_train), 1000000)
 fake_valid_data = lgb.Dataset(X_train.iloc[fake_valid_inds], label = y_train.iloc[fake_valid_inds],categorical_feature=cat_feats,
free_raw_data=False )<init_hyperparams>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.85,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 2500,
}<train_model>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data],early_stopping_rounds=200, verbose_eval=100 )<save_model>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
m_lgb.save_model("model.lgb" )<count_unique_values>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()<set_options>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
%load_ext Cython<set_options>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
modules =
os.environ['DATADIR'] = '/kaggle/input'<load_from_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
def load_qiqc(n_rows=None):
train_df = pd.read_csv(f'{os.environ["DATADIR"]}/train.csv', nrows=n_rows)
 submit_df = pd.read_csv(f'{os.environ["DATADIR"]}/test.csv', nrows=n_rows)
 n_labels = {
0:(train_df.target == 0 ).sum() ,
1:(train_df.target == 1 ).sum() ,
}
train_df['target'] = train_df.target.astype('f')
 train_df['weights'] = train_df.target.apply(lambda t: 1 / n_labels[t])
 return train_df, submit_df
def build_datasets(train_df, submit_df, holdout=False, seed=0):
submit_dataset = QIQCDataset(submit_df)
 if holdout:
splitter = sklearn.model_selection.StratifiedShuffleSplit(
 n_splits=1, test_size=0.1, random_state=seed)
 train_indices, test_indices = list(splitter.split(
 train_df, train_df.target)) [0]
train_indices.sort() , test_indices.sort()
train_dataset = QIQCDataset(
 train_df.iloc[train_indices].reset_index(drop=True))
test_dataset = QIQCDataset(
 train_df.iloc[test_indices].reset_index(drop=True))
else:
train_dataset = QIQCDataset(train_df)
 test_dataset = QIQCDataset(train_df.head(0))
return train_dataset, test_dataset, submit_dataset
class QIQCDataset(object):
def __init__(self, df):
self.df = df
@property
def tokens(self):
return self.df.tokens.values
@tokens.setter
def tokens(self, tokens):
self.df['tokens'] = tokens
@property
def positives(self):
return self.df[self.df.target == 1]
@property
def negatives(self):
return self.df[self.df.target == 0]
def build(self, device):
self._X = self.tids
self.X = torch.Tensor(self._X ).type(torch.long ).to(device)
 if 'target' in self.df:
self._t = self.df.target[:, None]
self._W = self.df.weights
self.t = torch.Tensor(self._t ).type(torch.float ).to(device)
 self.W = torch.Tensor(self._W ).type(torch.float ).to(device)
 if hasattr(self, '_X2'):
self.X2 = torch.Tensor(self._X2 ).type(torch.float ).to(device)
 else:
self._X2 = np.zeros(( self._X.shape[0], 1), 'f')
 self.X2 = torch.Tensor(self._X2 ).type(torch.float ).to(device)
 def build_labeled_dataset(self, indices):
return torch.utils.data.TensorDataset(
 self.X[indices], self.X2[indices],
self.t[indices], self.W[indices] )<define_variables>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
NORMALIZER_REGISTRY = {}
TOKENIZER_REGISTRY = {}
WORD_EMBEDDING_FEATURIZER_REGISTRY = {}
WORD_EXTRA_FEATURIZER_REGISTRY = {}
SENTENCE_EXTRA_FEATURIZER_REGISTRY = {}
ENCODER_REGISTRY = {}
AGGREGATOR_REGISTRY = {}
ATTENTION_REGISTRY = {}
def register_preprocessor(name):
def register_cls(cls):
NORMALIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_tokenizer(name):
def register_cls(cls):
TOKENIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_word_embedding_features(name):
def register_cls(cls):
WORD_EMBEDDING_FEATURIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_word_extra_features(name):
def register_cls(cls):
WORD_EXTRA_FEATURIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_sentence_extra_features(name):
def register_cls(cls):
SENTENCE_EXTRA_FEATURIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_encoder(name):
def register_cls(cls):
ENCODER_REGISTRY[name] = cls
return cls
return register_cls
def register_aggregator(name):
def register_cls(cls):
AGGREGATOR_REGISTRY[name] = cls
return cls
return register_cls
def register_attention(name):
def register_cls(cls):
ATTENTION_REGISTRY[name] = cls
return cls
return register_cls 
<count_values>
%%time
cnt = 0
EPOCHS = 1000
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
class WordVocab(object):
def __init__(self, mincount=1):
self.counter = Counter()
self.n_documents = 0
self._counters = {}
self._n_documents = defaultdict(int)
 self.mincount = mincount
def __len__(self):
return len(self.token2id)
 def add_documents(self, documents, name):
self._counters[name] = Counter()
for document in documents:
bow = dict.fromkeys(document, 1)
 self._counters[name].update(bow)
 self.counter.update(bow)
 self.n_documents += 1
self._n_documents[name] += 1
def build(self):
counter = dict(self.counter.most_common())
 self.word_freq = {
**{'<PAD>': 0},
**counter,
}
self.token2id = {
**{'<PAD>': 0},
**{word: i + 1 for i, word in enumerate(counter)}
}
self.lfq = np.array(list(self.word_freq.values())) < self.mincount
self.hfq = ~self.lfq 
<categorify>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
%%cython
c
cdef class StringReplacer:
cpdef public dict rule
cpdef list keys
cpdef list values
cpdef int n_rules
def __init__(self, dict rule):
self.rule = rule
self.keys = list(rule.keys())
 self.values = list(rule.values())
 self.n_rules = len(rule)
 def __call__(self, str x):
cdef int i
for i in range(self.n_rules):
if self.keys[i] in x:
x = x.replace(self.keys[i], self.values[i])
 return x
def __getstate__(self):
return(self.rule, self.keys, self.values, self.n_rules)
 def __setstate__(self, state):
self.rule, self.keys, self.values, self.n_rules = state
cdef class RegExpReplacer:
cdef dict rule
cdef list keys
cdef list values
cdef regexp
cdef int n_rules
def __init__(self, dict rule):
self.rule = rule
self.keys = list(rule.keys())
 self.values = list(rule.values())
 self.regexp = re.compile('(%s)' % '|'.join(self.keys))
self.n_rules = len(rule)
 @property
def rule(self):
return self.rule
def __call__(self, str x):
def replace(match):
x = match.group(0)
 if x in self.rule:
return self.rule[x]
else:
for i in range(self.n_rules):
x = re.sub(self.keys[i], self.values[i], x)
 return x
return self.regexp.sub(replace, x)
 cpdef str cylower(str x):
return x.lower()
Cache = {}
is_alphabet = re.compile(r'[a-zA-Z]')
 cpdef str unidecode_weak(str string):

 cdef list retval = []
cdef int i = 0
cdef int n = len(string)
 cdef str char
for i in range(n):
char = string[i]
codepoint = ord(char)
 if codepoint < 0x80:
retval.append(char)
 continue
if codepoint > 0xeffff:
continue
section = codepoint >> 8
position = codepoint % 256
try:
table = Cache[section]
except KeyError:
try:
mod = __import__('unidecode.x%03x'%(section), [], [], ['data'])
 except ImportError:
Cache[section] = None
continue
Cache[section] = table = mod.data
if table and len(table)> position:
if table[position] == '[?]' or is_alphabet.match(table[position]):
retval.append(' ' + char + ' ')
 else:
retval.append(table[position])
 return ''.join(retval)
<drop_column>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
class PunctSpacer(StringReplacer):
def __init__(self, edge_only=False):
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
if edge_only:
rule = {
**dict([(f' {p}', f' {p} ')for p in puncts]),
**dict([(f'{p} ', f' {p} ')for p in puncts]),
}
else:
rule = dict([(p, f' {p} ')for p in puncts])
 super().__init__(rule)
 class NumberReplacer(RegExpReplacer):
def __init__(self, with_underscore=False):
prefix, suffix = '', ''
if with_underscore:
prefix += ' __'
suffix = '__ '
rule = {
'[0-9]{5,}': f'{prefix}
'[0-9]{4}': f'{prefix}
'[0-9]{3}': f'{prefix}
'[0-9]{2}': f'{prefix}
}
super().__init__(rule)
 class KerasFilterReplacer(StringReplacer):
def __init__(self):
filters = '!"
'
rule = dict([(f, ' ')for f in filters])
 super().__init__(rule)
 class MisspellReplacer(StringReplacer):
def __init__(self):
rule = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd'y": "how do you",
"how'd": "how did",
"how'll": "how will",
"how's": "how is",
"i'd've": "i would have",
"i'd": "i would",
"i'll've": "i will have",
"i'll": "i will",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd've": "it would have",
"it'd": "it would",
"it'll've": "it will have",
"it'll": "it will",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't've": "might not have",
"mightn't": "might not",
"must've": "must have",
"mustn't've": "must not have",
"mustn't": "must not",
"needn't've": "need not have",
"needn't": "need not",
"o'clock": "of the clock",
"oughtn't've": "ought not have",
"oughtn't": "ought not",
"shan't've": "shall not have",
"shan't": "shall not",
"sha'n't": "shall not",
"she'd've": "she would have",
"she'd": "she would",
"she'll've": "she will have",
"she'll": "she will",
"she's": "she is",
"should've": "should have",
"shouldn't've": "should not have",
"shouldn't": "should not",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd've": "that would have",
"that'd": "that would",
"that's": "that is",
"there'd've": "there would have",
"there'd": "there would",
"there's": "there is",
"here's": "here is",
"they'd've": "they would have",
"they'd": "they would",
"they'll've": "they will have",
"they'll": "they will",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd've": "we would have",
"we'd": "we would",
"we'll've": "we will have",
"we'll": "we will",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll've": "what will have",
"what'll": "what will",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll've": "who will have",
"who'll": "who will",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't've": "will not have",
"won't": "will not",
"would've": "would have",
"wouldn't've": "would not have",
"wouldn't": "would not",
"y'all'd've": "you all would have",
"y'all'd": "you all would",
"y'all're": "you all are",
"y'all've": "you all have",
"y'all": "you all",
"you'd've": "you would have",
"you'd": "you would",
"you'll've": "you will have",
"you'll": "you will",
"you're": "you are",
"you've": "you have",
"colour": "color",
"centre": "center",
"favourite": "favorite",
"travelling": "traveling",
"counselling": "counseling",
"theatre": "theater",
"cancelled": "canceled",
"labour": "labor",
"organisation": "organization",
"wwii": "world war 2",
"citicise": "criticize",
"youtu ": "youtube ",
"qoura": "quora",
"sallary": "salary",
"whta": "what",
"narcisist": "narcissist",
"howdo": "how do",
"whatare": "what are",
"howcan": "how can",
"howmuch": "how much",
"howmany": "how many",
"whydo": "why do",
"doi": "do i",
"thebest": "the best",
"howdoes": "how does",
"mastrubation": "masturbation",
"mastrubate": "masturbate",
"mastrubating": "masturbating",
"pennis": "penis",
"etherium": "ethereum",
"narcissit": "narcissist",
"bigdata": "big data",
"2k17": "2017",
"2k18": "2018",
"qouta": "quota",
"exboyfriend": "ex boyfriend",
"airhostess": "air hostess",
"whst": "what",
"watsapp": "whatsapp",
"demonitisation": "demonetization",
"demonitization": "demonetization",
"demonetisation": "demonetization",
}
super().__init__(rule)
 register_preprocessor('lower' )(cylower)
 register_preprocessor('punct' )(PunctSpacer())
 register_preprocessor('unidecode' )(unidecode)
 register_preprocessor('unidecode_weak' )(unidecode_weak)
 register_preprocessor('number' )(NumberReplacer())
 register_preprocessor('number+underscore' )(
 NumberReplacer(with_underscore=True))
register_preprocessor('misspell' )(MisspellReplacer())
 register_preprocessor('keras' )(KerasFilterReplacer() )<load_pretrained>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
def load_pretrained_vectors(names, token2id, test=False):
assert isinstance(names, list)
 with Pool(processes=len(names)) as pool:
f = partial(load_pretrained_vector, token2id=token2id, test=test)
 vectors = pool.map(f, names)
 return dict([(n, v)for n, v in zip(names, vectors)])
 def load_pretrained_vector(name, token2id, test=False):
loader = dict(
 gnews=GNewsPretrainedVector,
wnews=WNewsPretrainedVector,
paragram=ParagramPretrainedVector,
glove=GlovePretrainedVector, 
)
 return loader[name].load(token2id, test)
 class BasePretrainedVector(object):
@classmethod
def load(cls, token2id, test=False, limit=None):
embed_shape =(len(token2id), 300)
 freqs = np.zeros(( len(token2id)) , dtype='f')
 if test:
np.random.seed(0)
 vectors = np.random.normal(0, 1, embed_shape)
 vectors[0] = 0
vectors[len(token2id)// 2:] = 0
else:
vectors = np.zeros(embed_shape, dtype='f')
 path = f'{os.environ["DATADIR"]}/{cls.path}'
for i, o in enumerate(
 open(path, encoding="utf8", errors='ignore')) :
token, *vector = o.split(' ')
 token = str.lower(token)
 if token not in token2id or len(o)<= 100:
continue
if limit is not None and i > limit:
break
freqs[token2id[token]] += 1
vectors[token2id[token]] += np.array(vector, 'f')
 vectors[freqs != 0] /= freqs[freqs != 0][:, None]
vec = KeyedVectors(300)
 vec.add(list(token2id.keys()), vectors, replace=True)
 return vec
class GNewsPretrainedVector(object):
name = 'GoogleNews-vectors-negative300'
path = f'embeddings/{name}/{name}.bin'
@classmethod
def load(cls, tokens, limit=None):
raise NotImplementedError
path = f'{os.environ["DATADIR"]}/{cls.path}'
return KeyedVectors.load_word2vec_format(
 path, binary=True, limit=limit)
 class WNewsPretrainedVector(BasePretrainedVector):
name = 'wiki-news-300d-1M'
path = f'embeddings/{name}/{name}.vec'
class ParagramPretrainedVector(BasePretrainedVector):
name = 'paragram_300_sl999'
path = f'embeddings/{name}/{name}.txt'
class GlovePretrainedVector(BasePretrainedVector):
name = 'glove.840B.300d'
path = f'embeddings/{name}/{name}.txt'<load_pretrained>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
@register_word_embedding_features('pretrained')
 class PretrainedVectorFeaturizer(object):
def __init__(self, config, vocab):
self.config = config
self.vocab = vocab
@classmethod
def add_args(self, parser):
pass
def __call__(self, features, datasets):
return features
class Any2VecFeaturizer(object):
def __init__(self, config, vocab):
self.config = config
self.vocab = vocab
def build_fillvalue(self, mode, initialW):
n_embed = initialW.shape[1]
n_fill = initialW[self.vocab.unk].shape
assert mode in {'zeros', 'mean', 'noise'}
if mode == 'zeros':
return np.zeros(n_embed, 'f')
 elif mode == 'mean':
return initialW.mean(axis=0)
 elif mode == 'noise':
mean, std = initialW.mean() , initialW.std()
return np.random.normal(mean, std,(n_fill, n_embed))
def __call__(self, features, datasets):
tokens = np.concatenate([d.tokens for d in datasets])
 model = self.build_model()
model.build_vocab_from_freq(self.vocab.word_freq)
 initialW = features.copy()
initialW[self.vocab.unk] = self.build_fillvalue(
 self.config.finetune_word2vec_init_unk, initialW)
 idxmap = np.array(
 [self.vocab.token2id[w] for w in model.wv.index2entity])
 model = self.initialize(model, initialW, idxmap)
 model.train(tokens, total_examples=len(tokens), epochs=model.epochs)
 finetunedW = np.zeros(( initialW.shape), 'f')
 for i, word in enumerate(self.vocab.token2id):
if word in model.wv:
finetunedW[i] = model.wv.get_vector(word)
 return finetunedW
@register_word_embedding_features('word2vec')
 class Word2VecFeaturizer(Any2VecFeaturizer):
@classmethod
def add_args(self, parser):
parser.add_argument('--finetune-word2vec-init-unk', type=str,
choices=['zeros', 'mean', 'noise'])
 parser.add_argument('--finetune-word2vec-mincount', type=int)
 parser.add_argument('--finetune-word2vec-workers', type=int)
 parser.add_argument('--finetune-word2vec-iter', type=int)
 parser.add_argument('--finetune-word2vec-size', type=int)
 parser.add_argument('--finetune-word2vec-window', type=int, default=5)
 parser.add_argument('--finetune-word2vec-sorted-vocab', type=int,
default=0)
 parser.add_argument('--finetune-word2vec-sg', type=int, choices=[0, 1])
 def build_model(self):
model = Word2Vec(
 min_count=self.config.finetune_word2vec_mincount,
workers=self.config.finetune_word2vec_workers,
iter=self.config.finetune_word2vec_iter,
size=self.config.finetune_word2vec_size,
window=self.config.finetune_word2vec_window,
sg=self.config.finetune_word2vec_sg, 
)
 return model
def initialize(self, model, initialW, idxmap):
model.wv.vectors[:] = initialW[idxmap]
model.trainables.syn1neg[:] = initialW[idxmap]
return model
@register_word_embedding_features('fasttext')
 class FastTextFeaturizer(Any2VecFeaturizer):
@classmethod
def add_args(self, parser):
parser.add_argument('--finetune-fasttext-init-unk', type=str,
choices=['zeros', 'mean', 'noise'])
 parser.add_argument('--finetune-fasttext-mincount', type=int)
 parser.add_argument('--finetune-fasttext-workers', type=int)
 parser.add_argument('--finetune-fasttext-iter', type=int)
 parser.add_argument('--finetune-fasttext-size', type=int)
 parser.add_argument('--finetune-fasttext-sg', type=int, choices=[0, 1])
 parser.add_argument('--finetune-fasttext-min_n', type=int)
 parser.add_argument('--finetune-fasttext-max_n', type=int)
 def build_model(self):
model = FastText(
 min_count=self.config.finetune_fasttext_mincount,
workers=self.config.finetune_fasttext_workers,
iter=self.config.finetune_fasttext_iter,
size=self.config.finetune_fasttext_size,
sg=self.config.finetune_fasttext_sg,
min_n=self.config.finetune_fasttext_min_n,
max_n=self.config.finetune_fasttext_max_n, 
)
 return model
def initialize(self, model, initialW, idxmap):
model.wv.vectors[:] = initialW[idxmap]
model.wv.vectors_vocab[:] = initialW[idxmap]
model.trainables.syn1neg[:] = initialW[idxmap]
return model<feature_engineering>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
@register_word_extra_features('idf')
 class IDFWordFeaturizer(object):
def __call__(self, vocab):
dfs = np.array(list(vocab.word_freq.values()))
dfs[0] = vocab.n_documents
features = np.log(vocab.n_documents / dfs)
 features = features[:, None]
return features
@register_word_extra_features('unk')
 class UnkWordFeaturizer(object):
def __call__(self, vocab):
features = vocab.unk.astype('f')
 features[0] = 0
features = features[:, None]
return features<feature_engineering>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
@register_sentence_extra_features('char')
 class CharacterStatisticsFeaturizer(object):
n_dims = 3
def __call__(self, sentence):
feature = {}
feature['n_chars'] = len(sentence)
 feature['n_caps'] = sum(1 for char in sentence if char.isupper())
 feature['caps_rate'] = feature['n_caps'] / feature['n_chars']
features = np.array(list(feature.values()))
return features
@register_sentence_extra_features('word')
 class WordStatisticsFeaturizer(object):
n_dims = 3
def __call__(self, sentence):
feature = {}
tokens = sentence.split()
feature['n_words'] = len(tokens)
 feature['unique_words'] = len(set(tokens))
feature['unique_rate'] = feature['unique_words'] / feature['n_words']
features = np.array(list(feature.values()))
return features<load_pretrained>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
register_tokenizer('space' )(cysplit)
 register_tokenizer('word_tokenize' )(nltk.word_tokenize )<normalization>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
<string_transform><EOS> 
df.to_csv('submission.csv', index=False )