Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<data_type_conversions>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
df_Train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_Test.rename(columns={'Country_Region':'Country'}, inplace=True)
 EMPTY_VAL = "EMPTY_VAL"
df_Train.rename(columns={'Province_State':'State'}, inplace=True)
 df_Train['State'].fillna(EMPTY_VAL, inplace=True)
 df_Train['State'] = df_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 df_Test.rename(columns={'Province_State':'State'}, inplace=True)
 df_Test['State'].fillna(EMPTY_VAL, inplace=True)
 df_Test['State'] = df_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 df_Test['Date'] = pd.to_datetime(df_Test['Date'], infer_datetime_format=True)
<prepare_x_and_y>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
MIN_TEST_DATE = df_Test.Date.min()
df_train = df_Train.copy()
y1_Train = df_train.iloc[:, -2]
y2_Train = df_train.iloc[:, -1]<feature_engineering>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
def extractDate(df, colName = 'Date'):

 assert colName in df.columns
df = df.assign(
 day = df.loc[:, colName].dt.day,
month = df.loc[:, colName].dt.month,
dayofyear = df.loc[:, colName].dt.dayofyear)
 return df<data_type_conversions>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
def createNewDataset(df):

 df_New = df.copy()
df_New = extractDate(df_New)
 df_New.drop(columns=['Date'], axis=1, inplace=True)
 return df_New<create_dataframe>
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
X_Train = createNewDataset(df_train)
 X_Test = createNewDataset(df_Test )<compute_test_metric>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
def RMSLE(pred, actual):
return np.sqrt(np.mean(np.power(( np.log(pred + 1)- np.log(actual + 1)) , 2)) )<set_options>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
filterwarnings('ignore' )<predict_on_test>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
def modelPredict(model, X_Test):
return model.predict(X_Test )<train_model>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
def modelFit(model, X_Train, y_Train):
model.fit(X_Train, y_Train )<categorify>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 df_Test_Out = pd.DataFrame()
LEncoder = LabelEncoder()
pfC = PolynomialFeatures(degree = 3)
 pfF = PolynomialFeatures(degree = 3)
 modelC = LinearRegression()
modelF = LinearRegression()
countries = X_Train.Country.unique().tolist()
for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique().tolist()
for state in states:
categoricalFeatures = ['Country', 'State']
df_Test_Updated = pd.DataFrame()
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), :]
yC_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases'].to_frame()
yF_Train_CS = X_Train_CS.loc[:, 'Fatalities'].to_frame()
X_Train_CS.drop(columns=['Id', 'ConfirmedCases', 'Fatalities', 'day', 'month'], axis=1, inplace=True)
 X_Train_CS.loc[:, 'Country'] = LEncoder.fit_transform(X_Train_CS.loc[:, 'Country'])
 X_Train_CS.loc[:, 'State'] = LEncoder.fit_transform(X_Train_CS.loc[:, 'State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), :]
df_Test_Updated = df_Test_Updated.append(X_Test_CS)
 X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS.drop(columns=['ForecastId', 'day', 'month'], axis=1, inplace=True)
 X_Test_CS.loc[:, 'Country'] = LEncoder.fit_transform(X_Test_CS.loc[:, 'Country'])
 X_Test_CS.loc[:, 'State'] = LEncoder.fit_transform(X_Test_CS.loc[:, 'State'])
 scTrain = StandardScaler()
X_Train_CS_SS = scTrain.fit_transform(X_Train_CS)
 X_Test_CS_SS = scTrain.transform(X_Test_CS)
 scC = StandardScaler()
yC_Train_CS_SS = scC.fit_transform(yC_Train_CS)
 scF = StandardScaler()
yF_Train_CS_SS = scF.fit_transform(yF_Train_CS)
 modelC.fit(pfC.fit_transform(X_Train_CS_SS), yC_Train_CS_SS)
 yC_Pred = scC.inverse_transform(modelC.predict(pfC.fit_transform(X_Test_CS_SS)))
 modelF.fit(pfF.fit_transform(X_Train_CS_SS), yF_Train_CS_SS)
 yF_Pred = scF.inverse_transform(modelF.predict(pfF.fit_transform(X_Test_CS_SS)))
 yC_Pred[yC_Pred < 0] = yC_Pred[yC_Pred < 0] * -1
yF_Pred[yF_Pred < 0] = yF_Pred[yF_Pred < 0] * -1
df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': yC_Pred.flatten() , 'Fatalities': yF_Pred.flatten() })
 df_out = pd.concat([df_out, df], axis=0)
 df_Test_Updated['ConfirmedCases'] = yC_Pred
df_Test_Updated['Fatalities'] = yF_Pred
df_Test_Out = df_Test_Out.append(df_Test_Updated)
<data_type_conversions>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
df_out.ForecastId = df_out.ForecastId.astype('int' )<save_to_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
df_out.to_csv('submission.csv', index=False )<filter>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
df_Test_Out.loc[df_Test_Out.Country == 'India', :]<data_type_conversions>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
df_Test_Out['Date'] = df_Test_Out.apply(lambda x : '2020-' + str(x.month)+ '-' + str(x.day), axis=1)
 df_Test_Out['Date'] = pd.to_datetime(df_Test_Out['Date'], infer_datetime_format=True )<set_options>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
warnings.filterwarnings("ignore" )<import_modules>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
from math import sqrt
import numba as nb
import scipy.sparse as sp
from scipy.optimize import fmin_l_bfgs_b, fmin_ncg
from numpy.lib.stride_tricks import as_strided
from sklearn.utils.extmath import safe_sparse_dot
from sklearn.utils import check_consistent_length, check_array
from sklearn.utils import check_random_state
from sklearn.base import BaseEstimator
from datetime import datetime
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from functools import reduce 
<import_modules>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
from numpy.random import normal
from matplotlib import pyplot as plt
import os
import copy
from sklearn.cluster import KMeans<import_modules>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
from tsfresh import extract_relevant_features
from tsfresh import select_features
from tsfresh.utilities.dataframe_functions import impute
from tsfresh.feature_extraction.extraction import extract_features<compute_test_metric>
sub['FVC1'] = 0.996*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
def trcg(Ax, r, x, n_iterations=1000, tr_delta=0, rtol=1e-5, atol=1e-8, args=() , verbose=False):
if n_iterations > 0:
n_iterations = min(n_iterations, len(x))
p, iteration = r.copy() , 0
tr_delta_sq = tr_delta ** 2
rtr, rtr_old = np.dot(r, r), 1.0
cg_tol = sqrt(rtr)* rtol + atol
region_breached = False
while(iteration < n_iterations)and(sqrt(rtr)> cg_tol):
Ap = Ax(p, *args)
 iteration += 1
if verbose:
print(
 % 
(iteration, np.linalg.norm(Ap), np.linalg.norm(p),
np.linalg.norm(r), np.linalg.norm(x), rtr / rtr_old))
alpha = rtr / np.dot(p, Ap)
 x += alpha * p
r -= alpha * Ap
if tr_delta_sq > 0:
xTx = np.dot(x, x)
 if xTx > tr_delta_sq:
xTp = np.dot(x, p)
 if xTp > 0:
p_nrm = np.linalg.norm(p)
 q = xTp / p_nrm
eta =(q - sqrt(max(q * q + tr_delta_sq - xTx, 0)))/ p_nrm
r += eta * Ap
x -= eta * p
else:
pass
region_breached = True
break
rtr, rtr_old = np.dot(r, r), rtr
p *= rtr / rtr_old
p += r
return iteration, region_breached
def tron(func, x, n_iterations=1000, rtol=1e-3, atol=1e-5, args=() ,
verbose=False):
eta0, eta1, eta2 = 1e-4, 0.25, 0.75
sigma1, sigma2, sigma3 = 0.25, 0.5, 4.0
f_valp_, f_grad_, f_hess_ = func
iteration, cg_iter = 0, 0
fval = f_valp_(x, *args)
 grad = f_grad_(x, *args)
 grad_norm = np.linalg.norm(grad)
 delta, grad_norm_tol = grad_norm, grad_norm * rtol + atol
while iteration < n_iterations and grad_norm > grad_norm_tol:
r, z = -grad, np.zeros_like(x)
 cg_iter, region_breached = trcg(
 f_hess_, r, z, tr_delta=delta, args=args,
n_iterations=20, rtol=1e-1, atol=0.0)
 z_norm = np.linalg.norm(z)
 if iteration == 0:
delta = min(delta, z_norm)
 linear = np.dot(z, grad)
 approxred = -0.5 *(linear - np.dot(z, r))
fnew = f_valp_(x + z, *args)
 actualred = fval - fnew
if linear + actualred < 0:
alpha = max(sigma1, 0.5 * linear /(linear + actualred))
else:
alpha = sigma3
if actualred < eta0 * approxred:
delta = min(max(alpha, sigma1)* z_norm, sigma2 * delta)
 elif actualred < eta1 * approxred:
delta = max(sigma1 * delta, min(alpha * z_norm, sigma2 * delta))
elif actualred < eta2 * approxred:
delta = max(sigma1 * delta, min(alpha * z_norm, sigma3 * delta))
else:
if region_breached:
delta = sigma3 * delta
else:
delta = max(delta, min(alpha * z_norm, sigma3 * delta))
if verbose:
print(
 % 
(iteration, actualred, approxred,
delta, fval, z_norm, grad_norm, cg_iter))
if actualred > eta0 * approxred:
x += z
fval, grad = fnew, f_grad_(x, *args)
 grad_norm = np.linalg.norm(grad)
 iteration += 1
if fval < -1e32:
if verbose:
print("WARNING: f < -1.0e+32")
 break
if abs(actualred)<= 0 and approxred <= 0:
if verbose:
print("WARNING: actred and prered <= 0")
 break
if abs(actualred)<= 1e-12 * abs(fval)and \
abs(approxred)<= 1e-12 * abs(fval):
if verbose:
print("WARNING: actred and prered too small")
 break
if delta <= rtol *(z_norm + atol):
if verbose:
print("WARNING: degenerate trust region")
 break
return cg_iter<statistical_test>
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
@nb.njit("float64[:,::1](float64[:,::1], float64[:,::1])",
fastmath=True, cache=False, error_model="numpy")
 def ar_resid(Z, phi):

 n_components, n_order = phi.shape
resid = Z[n_order:].copy()
for k in range(n_order):
resid -= Z[k:k - n_order] * phi[:, k]
return resid
@nb.njit("float64[:,::1](float64[:,::1], float64[:,::1], float64[:,::1])",
fastmath=True, cache=False, error_model="numpy")
 def ar_hess_vect(V, Z, phi):

 n_components, n_order = phi.shape
resid = ar_resid(V, phi)
 hess_v = np.zeros_like(V)
 hess_v[n_order:] = resid
for k in range(n_order):
hess_v[k:k - n_order] -= resid * phi[:, k]
return hess_v
@nb.njit("float64[:,::1](float64[:,::1], float64[:,::1])",
fastmath=True, cache=False, error_model="numpy")
 def ar_grad(Z, phi):

 return ar_hess_vect(Z, Z, phi)
 def precompute_graph_reg(adj):

 resid, deg = adj.astype(float), adj.getnnz(axis=1)
 resid.data /= deg[adj.nonzero() [0]]
return sp.diags(( deg > 0 ).astype(float)) - resid
def graph_resid(F, adj):

 return safe_sparse_dot(adj, F.T ).T
def graph_grad(F, adj):

 return safe_sparse_dot(adj.T, graph_resid(F, adj ).T ).T
def graph_hess_vect(V, F, adj):

 return graph_grad(V, adj)
 def l2_loss_valj(Y, Z, F):
if sp.issparse(Y):
R = csr_gemm(1, Z, F, -1, Y.copy())
 return sp.linalg.norm(R, ord="fro")** 2
return np.linalg.norm(Y - np.dot(Z, F), ord="fro")** 2
def f_step_tron_valj(f, Y, Z, C_F, eta_F, adj):

(n_samples, n_targets), n_components = Y.shape, Z.shape[1]
F = f.reshape(n_components, n_targets)
 objective = l2_loss_valj(Y, Z, F)
 if sp.issparse(Y):
coef = C_F * Y.nnz /(n_components * n_targets)
 else:
coef = C_F * n_samples / n_components
if C_F > 0:
if eta_F < 1:
reg_f_l2 = np.linalg.norm(F, ord="fro")** 2
else:
reg_f_l2, eta_F = 0., 1.
 if sp.issparse(adj)and(eta_F > 0):
reg_f_graph = np.linalg.norm(graph_resid(F, adj), ord="fro")** 2
else:
reg_f_graph, eta_F = 0., 0.
 reg_f = reg_f_l2 *(1 - eta_F)+ reg_f_graph * eta_F
objective += reg_f * coef
return 0.5 * objective
def f_step_tron_grad(f, Y, Z, C_F, eta_F, adj):

(n_samples, n_targets), n_components = Y.shape, Z.shape[1]
F = f.reshape(n_components, n_targets)
 if sp.issparse(Y):
coef = C_F * Y.nnz /(n_components * n_targets)
 grad = safe_sparse_dot(Z.T, csr_gemm(1, Z, F, -1, Y.copy()))
grad +=(1 - eta_F)* coef * F
else:
coef = C_F * n_samples / n_components
ZTY, ZTZ = np.dot(Z.T, Y), np.dot(Z.T, Z)
 if C_F > 0 and eta_F < 1:
ZTZ.flat[::n_components + 1] +=(1 - eta_F)* coef
grad = np.dot(ZTZ, F)- ZTY
if C_F > 0 and sp.issparse(adj)and eta_F > 0:
grad += graph_grad(F, adj)*(eta_F * coef)
 return grad.reshape(-1)
 def f_step_tron_hess(v, Y, Z, C_F, eta_F, adj):

(n_samples, n_targets), n_components = Y.shape, Z.shape[1]
V = v.reshape(n_components, n_targets)
 if sp.issparse(Y):
coef = C_F * Y.nnz /(n_components * n_targets)
 hess_v = safe_sparse_dot(Z.T, csr_gemm(1, Z, V, 0, Y.copy()))
hess_v +=(1 - eta_F)* coef * V
else:
coef = C_F * n_samples / n_components
ZTZ = np.dot(Z.T, Z)
 if C_F > 0 and eta_F < 1:
ZTZ.flat[::n_components + 1] +=(1 - eta_F)* coef
hess_v = np.dot(ZTZ, V)
 if C_F > 0 and sp.issparse(adj)and eta_F > 0:
hess_v += graph_grad(V, adj)*(eta_F * coef)
 return hess_v.reshape(-1)
 def f_step_tron(F, Y, Z, C_F, eta_F, adj, rtol=5e-2, atol=1e-4, verbose=False,
**kwargs):

 f_call = f_step_tron_valj, f_step_tron_grad, f_step_tron_hess
tron(f_call, F.ravel() , n_iterations=5, rtol=rtol, atol=atol,
args=(Y, Z, C_F, eta_F, adj), verbose=verbose)
 return F
def f_step_ncg_hess_(F, v, Y, Z, C_F, eta_F, adj):

 return f_step_tron_hess(v, Y, Z, C_F, eta_F, adj)
 def f_step_ncg(F, Y, Z, C_F, eta_F, adj, **kwargs):

 FF = fmin_ncg(f=f_step_tron_valj, x0=F.ravel() , disp=False,
fprime=f_step_tron_grad, fhess_p=f_step_ncg_hess_,
args=(Y, Z, C_F, eta_F, adj))
return FF.reshape(F.shape)
 def f_step_lbfgs(F, Y, Z, C_F, eta_F, adj, **kwargs):

 FF, f, d = fmin_l_bfgs_b(func=f_step_tron_valj, x0=F.ravel() , iprint=0,
fprime=f_step_tron_grad,
args=(Y, Z, C_F, eta_F, adj))
return FF.reshape(F.shape)
 def f_step_prox_func(F, Y, Z, C_F, eta_F, adj):

 return f_step_tron_valj(F.ravel() , Y, Z, C_F, eta_F, adj)
 def f_step_prox_grad(F, Y, Z, C_F, eta_F, adj):

 return f_step_tron_grad(F.ravel() , Y, Z, C_F, eta_F, adj ).reshape(F.shape)
 def f_step_prox(F, Y, Z, C_F, eta_F, adj, lip=1e-2, n_iter=25, alpha=1.0,
**kwargs):
gamma_u, gamma_d = 2, 1.1
grad = f_step_prox_grad(F, Y, Z, C_F, eta_F, adj)
 grad_F = np.dot(grad.flat, F.flat)
 f0, lip0 = f_step_prox_func(F, Y, Z, C_F, eta_F, adj), lip
for _ in range(n_iter):
F_new = np.maximum(F - grad / lip, 0.)
delta = f_step_prox_func(F_new, Y, Z, C_F, eta_F, adj)- f0
linear = np.dot(grad.flat, F_new.flat)- grad_F
quad = np.linalg.norm(F_new - F, ord="fro")** 2
if delta <= linear + lip * quad / 2:
break
lip *= gamma_u
lip = lip / gamma_d
return F_new, lip
def f_step(F, Y, Z, C_F, eta_F, adj, kind="fgm", **kwargs):

 lip = np.inf
if kind == "fgm":
F, lip = f_step_prox(F, Y, Z, C_F, eta_F, adj, **kwargs)
 elif kind == "tron":
F = f_step_tron(F, Y, Z, C_F, eta_F, adj, **kwargs)
 elif kind == "ncg":
F = f_step_ncg(F, Y, Z, C_F, eta_F, adj, **kwargs)
 elif kind == "lbfgs":
F = f_step_lbfgs(F, Y, Z, C_F, eta_F, adj, **kwargs)
 else:
raise ValueError(f)
 return F, lip
def phi_step(phi, Z, C_Z, C_phi, eta_Z, nugget=1e-8): 
(n_components, n_order), n_samples = phi.shape, Z.shape[0]
if n_order < 1 or n_components < 1:
return np.empty(( n_components, n_order))
if not(C_Z > 0 and eta_Z > 0):
return np.zeros_like(phi)
 shape = Z.shape[1:] +(Z.shape[0] - n_order, n_order + 1)
 strides = Z.strides[1:] + Z.strides[:1] + Z.strides[:1]
Z_view = as_strided(Z, shape=shape, strides=strides)
 y, Z_lagged = Z_view[..., -1], Z_view[..., :-1]
U, s, Vh = np.linalg.svd(Z_lagged, full_matrices=False)
 if C_phi > 0:
gain =(C_Z * eta_Z * n_order)* s
gain /= gain * s + C_phi *(n_samples - n_order)
 else:
large = s > nugget * np.max(s, axis=-1, keepdims=True)
 gain = np.divide(1, s, where=large, out=s)
 gain[~large] = 0
return np.einsum("ijk,ij,isj,is->ik", Vh, gain, U, y)
 def z_step_tron_valh(z, Y, F, phi, C_Z, eta_Z):

 n_samples, n_targets = Y.shape
n_components, n_order = phi.shape
Z = z.reshape(n_samples, n_components)
 objective = l2_loss_valj(Y, Z, F)
 if sp.issparse(Y):
coef = C_Z * Y.nnz /(n_samples * n_components)
 else:
coef = C_Z * n_targets / n_components
if C_Z > 0:
if eta_Z < 1:
reg_z_l2 = np.linalg.norm(Z, ord="fro")** 2
else:
reg_z_l2, eta_Z = 0., 1.
 if eta_Z > 0 and n_samples > n_order:
reg_z_ar_j = np.linalg.norm(ar_resid(Z, phi), ord=2, axis=0)** 2
reg_z_ar = np.sum(reg_z_ar_j)* n_samples /(n_samples - n_order)
 else:
reg_z_ar, eta_Z = 0., 0.
 reg_z = reg_z_l2 *(1 - eta_Z)+ reg_z_ar * eta_Z
objective += reg_z * coef
return 0.5 * objective
def z_step_tron_grad(z, Y, F, phi, C_Z, eta_Z):

 n_samples, n_targets = Y.shape
n_components, n_order = phi.shape
Z = z.reshape(n_samples, n_components)
 if sp.issparse(Y):
coef = C_Z * Y.nnz /(n_samples * n_components)
 grad = safe_sparse_dot(csr_gemm(1, Z, F, -1, Y.copy()), F.T)
 grad +=(1 - eta_Z)* coef * Z
else:
coef = C_Z * n_targets / n_components
YFT, FFT = np.dot(Y, F.T), np.dot(F, F.T)
 if C_Z > 0 and eta_Z < 1:
FFT.flat[::n_components + 1] +=(1 - eta_Z)* coef
grad = np.dot(Z, FFT)- YFT
if C_Z > 0 and eta_Z > 0:
ratio = n_samples /(n_samples - n_order)
 grad += ar_grad(Z, phi)*(ratio * eta_Z * coef)
 return grad.reshape(-1)
 def z_step_tron_hess(v, Y, F, phi, C_Z, eta_Z):

 n_samples, n_targets = Y.shape
n_components, n_order = phi.shape
V = v.reshape(n_samples, n_components)
 if sp.issparse(Y):
coef = C_Z * Y.nnz /(n_samples * n_components)
 hess_v = safe_sparse_dot(csr_gemm(1, V, F, 0, Y.copy()), F.T)
 hess_v +=(1 - eta_Z)* coef * V
else:
coef = C_Z * n_targets / n_components
FFT = np.dot(F, F.T)
 if C_Z > 0 and eta_Z < 1:
FFT.flat[::n_components + 1] +=(1 - eta_Z)* coef
hess_v = np.dot(V, FFT)
 if C_Z > 0 and eta_Z > 0:
ratio = n_samples /(n_samples - n_order)
 hess_v += ar_grad(V, phi)* ratio * eta_Z * coef
return hess_v.reshape(-1)
 def z_step_tron(Z, Y, F, phi, C_Z, eta_Z, rtol=5e-2, atol=1e-4, verbose=False):

 f_call = z_step_tron_valh, z_step_tron_grad, z_step_tron_hess
tron(f_call, Z.ravel() , n_iterations=5, rtol=rtol, atol=atol,
args=(Y, F, phi, C_Z, eta_Z), verbose=verbose)
 return Z
def z_step_ncg_hess_(Z, v, Y, F, phi, C_Z, eta_Z):

 return z_step_tron_hess(v, Y, F, phi, C_Z, eta_Z)
 def z_step_ncg(Z, Y, F, phi, C_Z, eta_Z, **kwargs):

 ZZ = fmin_ncg(f=z_step_tron_valh, x0=Z.ravel() , disp=False,
fprime=z_step_tron_grad, fhess_p=z_step_ncg_hess_,
args=(Y, F, phi, C_Z, eta_Z))
return ZZ.reshape(Z.shape)
 def z_step_lbfgs(Z, Y, F, phi, C_Z, eta_Z, **kwargs):

 ZZ, f, d = fmin_l_bfgs_b(func=z_step_tron_valh, x0=Z.ravel() , iprint=0,
fprime=z_step_tron_grad,
args=(Y, F, phi, C_Z, eta_Z))
return ZZ.reshape(Z.shape)
 def z_step(Z, Y, F, phi, C_Z, eta_Z, kind="tron", **kwargs):

 if kind == "tron":
Z = z_step_tron(Z, Y, F, phi, C_Z, eta_Z, **kwargs)
 elif kind == "ncg":
Z = z_step_ncg(Z, Y, F, phi, C_Z, eta_Z, **kwargs)
 elif kind == "lbfgs":
Z = z_step_lbfgs(Z, Y, F, phi, C_Z, eta_Z, **kwargs)
 else:
raise ValueError(f)
 return Z<compute_test_metric>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
@nb.njit("(float64, float64[:,::1], float64[:,::1], "
"float64, int32[::1], int32[::1], float64[::1])",
fastmath=True, error_model="numpy", parallel=False, cache=False)
 def _csr_gemm(alpha, X, D, beta, Sp, Sj, Sx):
if abs(beta)> 0:
for i in nb.prange(len(X)) :
for j in range(Sp[i], Sp[i+1]):
dot = np.dot(X[i], D[:, Sj[j]])
 Sx[j] = beta * Sx[j] + alpha * dot
else:
for i in nb.prange(len(X)) :
for j in range(Sp[i], Sp[i+1]):
Sx[j] = alpha * np.dot(X[i], D[:, Sj[j]])
 def csr_gemm(alpha, X, D, beta, Y):
_csr_gemm(alpha, X, D, beta, Y.indptr, Y.indices, Y.data)
 return Y
def csr_column_means(X):
f_sums = np.bincount(X.indices, weights=X.data, minlength=X.shape[1])
 n_nnz = np.maximum(np.bincount(X.indices, minlength=X.shape[1]), 1.)
return(f_sums / n_nnz)[np.newaxis]<compute_test_metric>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<statistical_test><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )