Kaggle competition: Avito Demand Prediction Challenge
stats = []
for col in train.columns:
stats.append(( col,
train[col].nunique() ,
train[col].value_counts().index[0],
train[col].value_counts().values[0],
train[col].isnull().sum() * 100 / train.shape[0],
train[col].value_counts(normalize=True, dropna=False ).values[0] * 100,
train[col].dtype))
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique values', 'Most frequent item', 'Freuquence of most frequent item', 'Percentage of missing values', 'Percentage of values in the biggest category', 'Type'])
 stats_df.sort_values('Percentage of missing values', ascending=False )<groupby>
h2o.init(max_mem_size = "10G" )
data = train.groupby("Patient" ).first().reset_index(drop=True)
 data.head()<feature_engineering>
print("
Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 testdex = testing.index
y = training.deal_probability.copy()
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))
print("Feature Engineering")
 df["price"] = np.log(df["price"]+0.001)
 df["price"].fillna(-999,inplace=True)
 df["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 df["Weekday"] = df['activation_date'].dt.weekday
df["Weekd of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day
df.drop(["activation_date","image"],axis=1,inplace=True)
 print("
Encode Variables")
 categorical = ["user_id","region","city","parent_category_name","category_name","item_seq_number","user_type","image_top_1"]
messy_categorical = ["param_1","param_2","param_3","title","description"]
print("Encoding :",categorical + messy_categorical)
 lbl = preprocessing.LabelEncoder()
for col in categorical + messy_categorical:
df[col] = lbl.fit_transform(df[col].astype(str))
X = df.loc[traindex,:].copy()
print("Training Set shape",X.shape)
 test = df.loc[testdex,:].copy()
print("Submission Set Shape: {} Rows, {} Columns".format(*test.shape))
del df
gc.collect()
%%time
def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(528, 528, 1), model_class=None):
inp = L.Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = L.GlobalAveragePooling2D()(x)
 inp2 = L.Input(shape=(4,))
x2 = L.GaussianNoise(0.2 )(inp2)
 x = L.Concatenate()([x, x2])
 x = L.Dropout(0.32 )(x)
 x = L.Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ["b6"]
models = [build_model(model_class=m, shape=(528, 528, 1)) for m in model_classes]
print('Number of models: ' + str(len(models)) )<compute_test_metric>
test.drop("deal_probability",axis=1, inplace=True )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip * sq2)
 return np.mean(metric )<split>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size=0.8)
 subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test, W, FVC = {}, {}, {}, {}, {}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, "FVC"] = fvc
sub.loc[sub.Patient_Week == k, "Confidence"] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
htrain = h2o.H2OFrame(X)
 htest = h2o.H2OFrame(test )
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
x =htrain.columns
y ='deal_probability'
x.remove(y)
 aml = H2OAutoML(max_runtime_secs = 18000)
 aml.train(x=x, y =y, training_frame=htrain)
 print('Generate predictions...')
 htrain.drop(['deal_probability'])
sub[["Patient_Week", "FVC", "Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
aml.leader
sub = pd.read_csv(ROOT + "sample_submission.csv")
 sub.head()<merge>
preds = aml.leader.predict(htest)
 preds = preds.as_data_frame()
<count_unique_values><EOS> 
lgsub = pd.DataFrame(preds.predict.values,columns=["deal_probability"],index=testdex)
 lgsub['deal_probability'].clip(0.0, 1.0, inplace=True)
 lgsub.to_csv("lgsub.csv",index=True,header=True)
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))