Kaggle competition: Competição DSA de Machine Learning
<SOS> metric: cateagorizationaccuracy Kaggle data source: competicao-dsa-machine-learning-jan-2019<compute_train_metric>
%matplotlib inline
warnings.filterwarnings('ignore')
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
d = np.diff(temp[temp > threshold])[-num_days:]
w = np.arange(len(d)) +1
w = w**5
w = w / np.sum(w)
 gr_base.append(np.clip(np.average(d, weights=w), 0, params['growth rate max']))
d2 = np.diff(d)
 w = np.arange(len(d2)) +1
w = w**10
w = w / np.sum(w)
 gr_base_factor.append(np.clip(np.average(d2, weights=w), -0.5, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(np.log1p(i))
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 0,
"last N days": 15,
"growth rate default": 0.10,
"growth rate max": 0.2,
"growth rate factor max": -0.1,
"growth rate factor": -0.3,
"growth rate factor factor": 0.01,
}
x = train_p_c
preds_c = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
<feature_engineering>
teste_base = pd.read_csv('.. /input/dataset_teste.csv')
 teste = teste_base.copy()
teste.info()
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1] 
<normalization>
featuresTeste = [
"Postal Code",
"Latitude",
"Longitude",
"DOF Gross Floor Area",
"Year Built",
"Number of Buildings - Self-reported",
"Occupancy",
"Site EUI(kBtu/ft²)",
"Property GFA - Self-Reported(ft²)",
"Source EUI(kBtu/ft²)",
"Community Board",
"Council District",
"Census Tract",
"Weather Normalized Site EUI(kBtu/ft²)",
"Weather Normalized Site Electricity Intensity(kWh/ft²)",
"Weather Normalized Source EUI(kBtu/ft²)",
"Weather Normalized Site Natural Gas Use(therms)",
"Weather Normalized Site Electricity(kWh)",
"Water Use(All Water Sources )(kgal)",
"Water Intensity(All Water Sources )(gal/ft²)",
"Total GHG Emissions(Metric Tons CO2e)",
"Direct GHG Emissions(Metric Tons CO2e)",
"Indirect GHG Emissions(Metric Tons CO2e)",
"Electricity Use - Grid Purchase(kBtu)",
"Natural Gas Use(kBtu)",
"Manhattan", "Queens", "Brooklyn", "Staten Island"]
featuresTreino = featuresTeste + ["ENERGY STAR Score"]
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.03, 0.5)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 5
d = np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize]
delta = np.average(d, axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 7,
"absolute growth": 0.02,
"fatality_rate_lower": 0.02,
"fatality_rate_upper": 0.3,
}
preds_f_1 = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f_1 = np.log(1+preds_f_1 )<set_options>
def setCity(df):
lista = df["Borough"].value_counts()
for item in lista.index:
df[item] = df["Borough"] == item
df[item] = df[item].astype(int)
 return df
class ZDatasetF(Dataset):
def __init__(self, X_c, X_f=None, hist_len=10):
self.X_c = X_c
self.X_f = X_f
self.hist_len = hist_len
self.is_test = X_f is None
def __len__(self):
return self.X_c.shape[1]
def __getitem__(self, idx):
if self.is_test:
return {'x_c':self.X_c[:, idx-self.hist_len:idx]}
else:
return {'x_c':self.X_c[:, idx-self.hist_len:idx],
'x_f':self.X_f[:, idx-1],
'y':np.log(1+self.X_f[:, idx])}
class PrLayer2(nn.Module):
def __init__(self, in_features1, in_features2):
super(PrLayer2, self ).__init__()
self.weight0 = Parameter(torch.Tensor(1, 1, in_features2))
self.weight1 = Parameter(torch.Tensor(1, in_features1, in_features2))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight0, a=math.sqrt(5))
init.kaiming_uniform_(self.weight1, a=math.sqrt(5))
def forward(self, input):
return input * torch.sigmoid(self.weight0 + self.weight1)
 class ZModelF(nn.Module):
def __init__(self, hist_len):
super(ZModelF, self ).__init__()
self.l_conv = PrLayer2(len(X_c),hist_len-1)
 def forward(self, x_c, x_f):
x = x_c[:,:,1:] - x_c[:,:,:-1]
res = torch.sum(self.l_conv(x), 2)
 return {'preds': torch.log(1 + x_f + res)}
class DummySampler(torch.utils.data.sampler.Sampler):
def __init__(self, idx):
self.idx = idx
def __iter__(self):
return iter(self.idx)
 def __len__(self):
return len(self.idx)
 def _smooth_l1_loss(target):
t = torch.abs(target)
 t = torch.where(t < 1, 0.5 * t ** 2, t - 0.5)
 return torch.mean(t)
 n_epochs = 5000
lr = 0.18
bag_size = 4
device = 'cpu'
hist_len = 14
loss_func = torch.nn.MSELoss()
reg_loss_func = _smooth_l1_loss
reg_factor = 0.035
train_dataset = ZDatasetF(np.exp(X_c)-1, X_f, hist_len=hist_len)
 test_dataset = ZDatasetF(np.exp(preds_c)-1, hist_len=hist_len)
 trn_idx = np.arange(hist_len+1, len(train_dataset))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(trn_idx)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=len(trn_idx), sampler=train_sampler, num_workers=0, pin_memory=True)
 test_idx = np.arange(TRAIN_N, len(test_dataset))
test_sampler = DummySampler(test_idx)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, sampler=test_sampler, num_workers=0, pin_memory=True)
 gradient_accumulation = 1
preds_f = 0
for m_i in range(bag_size):
model_f = ZModelF(hist_len=hist_len ).to(device)
 optimizer_f = torch.optim.Adam(model_f.parameters() , lr=lr)
 model_f.train()
start_time = time.time()
for epoch in range(n_epochs):
s = time.time()
avg_train_loss = 0
optimizer_f.zero_grad()
for idx, data in enumerate(train_loader):
X1 = data['x_c'].to(device ).float()
X2 = data['x_f'].to(device ).float()
y = data['y'].to(device ).float()
preds = model_f(X1, X2)['preds'].float()
cond = X2 > np.log(10)
 preds = preds[cond]
y = y[cond]
loss = loss_func(preds, y)
 loss += reg_factor * reg_loss_func(model_f.l_conv.weight1)
 avg_train_loss += loss / len(train_loader)
 loss.backward()
if(idx+1)% gradient_accumulation == 0 or idx == len(train_loader)- 1:
optimizer_f.step()
optimizer_f.zero_grad()
if False:
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f_delta = np.log(1 + preds_f_delta)
 val_len = train_p_c.values.shape[1] - TRAIN_N
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , \
preds_f_delta[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{epoch:2} train_loss {avg_train_loss:<8.4f} val_loss {m2:8.5f} {time.time() -s:<2.2f}")
 model_f.train()
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f += preds_f_delta / bag_size
preds_f_2 = np.log(1 + preds_f)
 print("Done")
<compute_test_metric>
def setPostalCode(df):
df["Postal Code"] = df["Postal Code"].str.replace("-", "")
 df["Postal Code"] = df["Postal Code"].astype(int)
 return df
preds_f = np.average([preds_f_1, preds_f_2], axis=0, weights=[1,2] )<compute_test_metric>
def setMean(df, features):
df = df.replace('Not Available',np.nan, regex=True)
 for item in features:
if df[item].dtype == "object":
df[item] = df[item].astype(float)
 for item in features:
df[item] = df[item].fillna(df[item].mean())
 return df
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<define_variables>
def setGeneralData(df, features):
df["Number of Buildings - Self-reported"][df["Number of Buildings - Self-reported"] > 30 ] = 30
df["Number of Buildings - Self-reported"][df["Number of Buildings - Self-reported"] <= 0] = 1
df["Occupancy"][df["Occupancy"] <= 0] = 1
df["Site EUI(kBtu/ft²)"][df["Site EUI(kBtu/ft²)"] <= 0] = 1
df["Property GFA - Self-Reported(ft²)"][df["Property GFA - Self-Reported(ft²)"] >= 2500000] = 2500000
df["Source EUI(kBtu/ft²)"][df["Source EUI(kBtu/ft²)"] < 1] = 1
df["Year Built"][df["Year Built"] < 1800] = 1800
df["Year Built"][df["Year Built"] > 2015] = 2015
df = df.round(2)
 return df
EU_COUNTRIES = ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia',
'Netherlands', 'Denmark', 'Poland', 'Estonia', 'Portugal', 'Finland', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia',
'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
EUROPE_OTHER = ['Albania', 'Andorra', 'Bosnia and Herzegovina', 'Liechtenstein', 'Monaco', 'Montenegro', 'North Macedonia',
'Norway', 'San Marino', 'Serbia', 'Switzerland', 'Turkey', 'United Kingdom']
AFRICA = ['Algeria', 'Burkina Faso', 'Cameroon', 'Congo(Kinshasa)', "Cote d'Ivoire", 'Egypt', 'Ghana', 'Kenya', 'Madagascar',
'Morocco', 'Nigeria', 'Rwanda', 'Senegal', 'South Africa', 'Togo', 'Tunisia', 'Uganda', 'Zambia']
NORTH_AMERICA = ['US', 'Canada', 'Mexico']
SOUTH_AMERICA = ['Argentina', 'Bolivia', 'Brazil', 'Chile', 'Colombia', 'Ecuador', 'Paraguay', 'Peru', 'Uruguay', 'Venezuela']
MIDDLE_EAST = ['Afghanistan', 'Bahrain', 'Iran', 'Iraq', 'Israel', 'Jordan', 'Kuwait', 'Lebanon', 'Oman', 'Qatar', 'Saudi Arabia', 'United Arab Emirates']
ASIA = ['Bangladesh', 'Brunei', 'Cambodia', 'India', 'Indonesia', 'Japan', 'Kazakhstan', 'Korea, South', 'Kyrgyzstan', 'Malaysia',
'Pakistan', 'Singapore', 'Sri Lanka', 'Taiwan*', 'Thailand', 'Uzbekistan', 'Vietnam']<merge>
treino = setPostalCode(treino)
 teste = setPostalCode(teste)
 treino = setCity(treino)
 teste = setCity(teste)
 treino = setMean(treino, featuresTreino)
 teste = setMean(teste, featuresTeste)
 treino = setGeneralData(treino, featuresTreino)
 teste = setGeneralData(teste, featuresTeste)
 print(treino.shape)
 print(teste.shape)
 treino = treino.filter(items=featuresTreino)
 teste = teste.filter(items=featuresTeste)
 print(treino.shape)
 print(teste.shape)
 showCorr(treino )
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test_orig.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(treino.drop(columns=['ENERGY STAR Score']), pd.DataFrame(treino["ENERGY STAR Score"]))
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )<count_unique_values>
finalModel = xgb.XGBClassifier(max_depth=10, learning_rate=0.1, n_estimators=1000, n_jobs=50)
 finalModel
test.days.nunique()<sort_values>
finalModel.fit(X_train, y_train, eval_metric='mae' )
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country_Region']}, {rec['Province_State']}")
<compute_test_metric>
y_pred = finalModel.predict(X_test)
 print("Mean squared error: %.2f" % mean_squared_error(y_test, y_pred))
print('Variance score: %.2f' % r2_score(y_test, y_pred))
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
<load_from_csv>
envio_final = pd.DataFrame(teste_base["Property Id"])
 envio_final['score'] = finalModel.predict(teste ).round()
envio_final['score'] = envio_final["score"].astype(int)
 sb.countplot(x='score',data=envio_final)
 envio_final.describe().T
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
envio_final.to_csv('final.csv', index=False )