Kaggle competition: New York City Taxi Fare Prediction
<load_pretrained><EOS> 
import pandas as pd
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<define_search_space>
import pandas as pd
seed = 42
test_size = 0.2
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 workspace = "./unet"
backbone = [1, 1, 1, 1]
encoder_channels = np.array([64, 128, 256, 512, 1024])*2
decoder_channels = np.array([512, 256, 128, 64])*2
fold = 0
time_step = 4000
time_step_test = 10000
stride = 2
batch_size = 8
TRAIN = True
PREDICT = True
ENSEMBLE = False
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 print(device )<load_from_csv>
melbourne_model = DecisionTreeRegressor(random_state=1)
 melbourne_model.fit(X,y)
 print("Making predictions for the following 5 houses:")
 print(X.head())
 print("The predictions are")
 print(melbourne_model.predict(X.head()))
df_train = pd.read_csv(".. /input/liverpool-ion-switching/train.csv")
 df_test = pd.read_csv(".. /input/liverpool-ion-switching/test.csv" )<categorify>
yy = iowa_data.SalePrice
iowa_features = ["LotArea", "YearBuilt", "1stFlrSF", "2ndFlrSF","FullBath", "BedroomAbvGr", "TotRmsAbvGrd"]
XX = iowa_data[iowa_features]
print(XX.describe())
 print(XX.head())
 iowa_model = DecisionTreeRegressor(random_state=1)
 iowa_model.fit(XX,yy)
 print("Making predictions for the following 5 houses:")
 print(XX.head())
 print("The predictions are")
 print(iowa_model.predict(XX.head()))
def do_identity(image, label):
return image, label
def do_flip(image, label):
image = np.ascontiguousarray(np.flip(image, axis=1))
label = np.ascontiguousarray(np.flip(label))
return image, label
def train_augment(image, label):
for op in np.random.choice([
lambda image, label: do_identity(image, label),
lambda image, label: do_flip(image, label),
], 1):
image, label = op(image, label)
 return image, label
def valid_augment(image, label):
return image, label<data_type_conversions>
predicted_melbourne_prices = melbourne_model.predict(X)
 print(mean_absolute_error(y, predicted_melbourne_prices))
class IonDataset(Dataset):
def __init__(self, data, labels=None, type='train', transform=None):
self.data = data
self.labels = labels
self.type = type
self.transform = transform
def __getitem__(self, i):
signal = self.data[i].astype(np.float32)
 if self.type == 'train':
label = self.labels[i].astype(np.int64)
 if self.transform is not None:
signal, label = self.transform(signal, label)
 return signal, label
else:
return signal
def __len__(self):
return len(self.data )<define_search_model>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state = 0)
 melbourne_model = DecisionTreeRegressor()
melbourne_model.fit(train_X, train_y)
 val_predictions = melbourne_model.predict(val_X)
 print(mean_absolute_error(val_y, val_predictions))
class SEModule(nn.Module):
def __init__(self, in_channels, reduction=4):
super().__init__()
self.conv1 = nn.Conv1d(in_channels, in_channels//reduction, kernel_size=1, padding=0)
 self.conv2 = nn.Conv1d(in_channels//reduction, in_channels, kernel_size=1, padding=0)
 def forward(self, x):
s = F.adaptive_avg_pool1d(x, 1)
 s = self.conv1(s)
 s = F.relu(s, inplace=True)
 s = self.conv2(s)
 x = x + torch.sigmoid(s)
 return x
class ConvBR1d(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=1, padding=0, dilation=1, stride=1, groups=1, is_activation=True):
super().__init__()
self.conv = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation, stride=stride, groups=groups, bias=False)
 self.bn = nn.BatchNorm1d(out_channels)
 self.is_activation = is_activation
if is_activation:
self.relu = nn.ReLU(inplace=True)
 def forward(self, x):
x = self.bn(self.conv(x))
if self.is_activation:
x = self.relu(x)
 return x
class SENextBottleneck(nn.Module):
def __init__(self, in_channels, out_channels, stride=1, groups=32, reduction=16, pool=None, is_shortcut=False):
super().__init__()
mid_channels = out_channels // 2
self.conv1 = ConvBR1d(in_channels, mid_channels, 1, 0, 1,)
 self.conv2 = ConvBR1d(mid_channels, mid_channels, 3, 1, 1, groups=groups)
 self.conv3 = ConvBR1d(mid_channels, out_channels, 1, 0, 1, is_activation=False)
 self.se = SEModule(out_channels, reduction)
 self.stride = stride
self.is_shortcut = is_shortcut
if is_shortcut:
self.shortcut = ConvBR1d(in_channels, out_channels, 1, 0, 1, is_activation=False)
 if stride > 1:
if pool == 'max':
self.pool = nn.MaxPool1d(stride, stride)
 elif pool == 'avg':
self.pool = nn.AvgPool1d(stride, stride)
 def forward(self, x):
s = self.conv1(x)
 s = self.conv2(s)
 if self.stride > 1:
s = self.pool(s)
 s = self.conv3(s)
 s = self.se(s)
 if self.is_shortcut:
if self.stride > 1:
x = F.avg_pool1d(x, self.stride, self.stride)
 x = self.shortcut(x)
 x = x + s
x = F.relu(x, inplace=True)
 return x
class Encoder(nn.Module):
def __init__(self, num_features=1):
super().__init__()
self.block0 = nn.Sequential(
 ConvBR1d(num_features, encoder_channels[0], kernel_size=5, stride=1, padding=2),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1), 
)
 self.block1 = nn.Sequential(
 SENextBottleneck(encoder_channels[0], encoder_channels[1], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[1], encoder_channels[1], stride=1, is_shortcut=False)for i in range(backbone[0])] 
)
 self.block2 = nn.Sequential(
 SENextBottleneck(encoder_channels[1], encoder_channels[2], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[2], encoder_channels[2], stride=1, is_shortcut=False)for i in range(backbone[1])] 
)
 self.block3 = nn.Sequential(
 SENextBottleneck(encoder_channels[2], encoder_channels[3], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[3], encoder_channels[3], stride=1, is_shortcut=False)for i in range(backbone[2])] 
)
 self.block4 = nn.Sequential(
 SENextBottleneck(encoder_channels[3], encoder_channels[4], stride=stride, is_shortcut=True, pool='avg'),
*[SENextBottleneck(encoder_channels[4], encoder_channels[4], stride=1, is_shortcut=False)for i in range(backbone[3])] 
)
 def forward(self, x):
x0 = self.block0(x)
 x1 = self.block1(x0)
 x2 = self.block2(x1)
 x3 = self.block3(x2)
 x4 = self.block4(x3)
 return [x0, x1, x2, x3, x4]
class SCSEModule(nn.Module):
def __init__(self, in_channels, reduction=16):
super().__init__()
self.cSE = nn.Sequential(
 nn.AdaptiveAvgPool1d(1),
nn.Conv1d(in_channels, in_channels // reduction, 1),
nn.ReLU(inplace=True),
nn.Conv1d(in_channels // reduction, in_channels, 1),
nn.Sigmoid() , 
)
 self.sSE = nn.Sequential(nn.Conv1d(in_channels, 1, 1), nn.Sigmoid())
 def forward(self, x):
return x * self.cSE(x)+ x * self.sSE(x)
 class DecoderBlock(nn.Module):
def __init__(self, in_channels, skip_channels, out_channels):
super().__init__()
self.conv1 = ConvBR1d(in_channels + skip_channels, out_channels, kernel_size=3, padding=1)
 self.conv2 = ConvBR1d(out_channels, out_channels, kernel_size=3, padding=1)
 def forward(self, x, skip=None):
x = F.interpolate(x, scale_factor=stride, mode="linear", align_corners=True)
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.conv1(x)
 x = self.conv2(x)
 return x
class Decoder(nn.Module):
def __init__(self):
super().__init__()
self.block4 = DecoderBlock(encoder_channels[-1], encoder_channels[-2], decoder_channels[0])
 self.block3 = DecoderBlock(decoder_channels[0], encoder_channels[-3], decoder_channels[1])
 self.block2 = DecoderBlock(decoder_channels[1], encoder_channels[-4], decoder_channels[2])
 self.block1 = DecoderBlock(decoder_channels[2], encoder_channels[-5], decoder_channels[3])
 def forward(self, xs):
x = self.block4(xs[4], xs[3])
 x = self.block3(x, xs[2])
 x = self.block2(x, xs[1])
 x = self.block1(x, xs[0])
 return x
class Unet(nn.Module):
def __init__(self, num_features=1, num_classes=11):
super().__init__()
self.num_features = num_features
self.num_classes = num_classes
self.encoder = Encoder(num_features=num_features)
 self.decoder = Decoder()
self.segmentation_head = nn.Conv1d(decoder_channels[-1], num_classes, kernel_size=1, padding=0, stride=1)
 def forward(self, x):
features = self.encoder(x)
 x = self.decoder(features)
 x = self.segmentation_head(x)
 return x 
<init_hyperparams>
train_X, val_X, train_y, val_y = train_test_split(XX, yy, random_state = 0)
 iowa_model = DecisionTreeRegressor()
iowa_model.fit(train_X, train_y)
 val_predictions = iowa_model.predict(val_X)
 print(mean_absolute_error(val_y, val_predictions))
class ClassificationMeter:

 def __init__(self, nCls, eps=1e-5, names=None):
self.nCls = nCls
self.names = names
self.eps = eps
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 self._measure = None
def clear(self):
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 def prepare_inputs(self, outputs, truths):

 if torch.is_tensor(outputs):
outputs = outputs.detach().cpu().numpy()
if torch.is_tensor(truths):
truths = truths.detach().cpu().numpy()
return outputs, truths
def update(self, preds, truths):
preds, truths = self.prepare_inputs(preds, truths)
 self.N += np.prod(truths.shape)
 for Cls in range(self.nCls):
true_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths == Cls))
true_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths != Cls))
false_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths != Cls))
false_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths == Cls))
self.table[Cls] += [true_positive, true_negative, false_positive, false_negative]
def measure(self):
return self._measure
def better(self, A, B):
return A > B
def write(self, writer, global_step, prefix=""):
writer.add_scalar(os.path.join(prefix, "Accuracy"), self.measure() , global_step)
 def report(self, each_class=False):
precisions = []
recalls = []
f1s = []
for Cls in range(self.nCls):
recall = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,3] + self.eps)
 precision = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,2] + self.eps)
 f1 = 2 * precision * recall /(precision + recall + self.eps)
 precisions.append(precision)
 recalls.append(recall)
 f1s.append(f1)
 total_TP = np.sum(self.table[:, 0])
 accuracy = total_TP/self.N
accuracy_mean_class = np.mean(precisions)
 macro_f1 = np.mean(f1s)
 self._measure = macro_f1
text = f"Macro F1 = {macro_f1:.4f}
"
text += f"\tOverall Accuracy = {accuracy:.4f}({total_TP}/{self.N})
 "
text += f"\tMean-class Accuracy = {accuracy_mean_class:.4f}
"
if each_class:
for Cls in range(self.nCls):
text += f"\tClass {str(Cls)+'('+self.names[Cls]+')' if self.names is not None else Cls}: precision = {precisions[Cls]:.3f} recall = {recalls[Cls]:.3f}
"
return text<feature_engineering>
def get_mae(max_leaf_nodes, train_X, val_X, train_y, val_y):
model = DecisionTreeRegressor(max_leaf_nodes=max_leaf_nodes, random_state=0)
 model.fit(train_X, train_y)
 preds_val = model.predict(val_X)
 mae = mean_absolute_error(val_y, preds_val)
 return mae
melbourne_file_path = '.. /input/melbourne-housing-snapshot/melb_data.csv'
melbourne_data = pd.read_csv(melbourne_file_path)
 filtered_melbourne_data = melbourne_data.dropna(axis=0)
 y = filtered_melbourne_data.Price
melbourne_features = ['Rooms', 'Bathroom', 'Landsize', 'BuildingArea', 'YearBuilt', 'Lattitude', 'Longtitude']
X = filtered_melbourne_data[melbourne_features]
train_X, val_X, train_y, val_y = train_test_split(X, y,random_state = 0 )
def five_type_remove_drift(train):
a=500000; b=600000
train.loc[train.index[a:b],'signal'] = train.signal[a:b].values - 3*(train.time.values[a:b] - 50)/10.
 def f(x,low,high,mid):
return -(( -low+high)/625)*(x-mid)**2+high -low
batch = 7; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,-1.817,3.186,325)
 batch = 8; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,-0.094,4.936,375)
 batch = 9; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,1.715,6.689,425)
 batch = 10; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,3.361,8.45,475)
 return train<prepare_x_and_y>
for max_leaf_nodes in [5, 50, 500, 5000]:
my_mae = get_mae(max_leaf_nodes, train_X, val_X, train_y, val_y)
 print("Max leaf nodes: %d \t\t Mean Absolute Error: %d" %(max_leaf_nodes, my_mae))
if TRAIN:
df_train = five_type_remove_drift(df_train)
 batch = 1; a = 500000*(batch-1); b = 500000*batch
batch = 2; c = 500000*(batch-1); d = 500000*batch
X_train_1s = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1s = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 3; a = 500000*(batch-1); b = 500000*batch
batch = 7; c = 500000*(batch-1); d = 500000*batch
X_train_1f = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1f = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 4; a = 500000*(batch-1); b = 500000*batch
batch = 8; c = 500000*(batch-1); d = 500000*batch
X_train_3 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_3 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 6; a = 500000*(batch-1); b = 500000*batch
batch = 9; c = 500000*(batch-1); d = 500000*batch
X_train_5 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_5 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 5; a = 500000*(batch-1); b = 500000*batch
batch = 10; c = 500000*(batch-1); d = 500000*batch
X_train_10 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_10 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
for X, y, num_classes, model_name, max_epoch in zip(
 [X_train_1s, X_train_1f, X_train_3, X_train_5, X_train_10],
[y_train_1s, y_train_1f, y_train_3, y_train_5, y_train_10],
np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'],
[30, 30, 60, 60, 120], 
):
X = X.reshape(-1, 1, time_step)
 y = y.reshape(-1, time_step)
 idx = np.arange(X.shape[0])
 kf = model_selection.KFold(5, shuffle=True, random_state=seed)
 train_idx, val_idx = list(kf.split(idx)) [fold]
print("model_name:", model_name)
 print("train dataset shape:", X[train_idx].shape, y[train_idx].shape)
 train_dataset = IonDataset(X[train_idx], y[train_idx], 'train', train_augment)
 valid_dataset = IonDataset(X[val_idx], y[val_idx], 'train', valid_augment)
 loaders = {
'train': DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True),
'valid': DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True),
}
model = Unet(num_classes=num_classes)
 loss_function = nn.CrossEntropyLoss()
optimizer = optim.AdamW(model.parameters() , lr=0.001, weight_decay=1e-5)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10, factor=0.1, min_lr=1e-8)
 metrics = [ClassificationMeter(num_classes),]
trainer = Trainer(device, workspace, model, optimizer, scheduler, loss_function, loaders, metrics,
model_name=model_name,
report_step_interval=-1, 
)
 trainer.train(max_epoch)
<feature_engineering>
melbourne_forest_model = RandomForestRegressor(random_state=1)
 melbourne_forest_model.fit(train_X, train_y)
 melbourne_forest_preds = melbourne_forest_model.predict(val_X)
 print(mean_absolute_error(val_y, melbourne_forest_preds))
<import_modules><EOS> 
iowa_file_path = '.. /input/house-prices-advanced-regression-techniques/train.csv'
home_data = pd.read_csv(iowa_file_path)
 y = home_data.SalePrice
features = ['LotArea', 'YearBuilt', '1stFlrSF', '2ndFlrSF', 'FullBath', 'BedroomAbvGr', 'TotRmsAbvGrd', 'OverallQual', 'OverallCond']
X = home_data[features]
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 iowa_model = DecisionTreeRegressor(random_state=1)
 iowa_model.fit(train_X, train_y)
 val_predictions = iowa_model.predict(val_X)
 val_mae = mean_absolute_error(val_predictions, val_y)
 print("Validation MAE when not specifying max_leaf_nodes: {:,.0f}".format(val_mae))
iowa_model = DecisionTreeRegressor(max_leaf_nodes=100, random_state=1)
 iowa_model.fit(train_X, train_y)
 val_predictions = iowa_model.predict(val_X)
 val_mae = mean_absolute_error(val_predictions, val_y)
 print("Validation MAE for best value of max_leaf_nodes: {:,.0f}".format(val_mae))
rf_model = RandomForestRegressor(random_state=1)
 rf_model.fit(train_X, train_y)
 rf_val_predictions = rf_model.predict(val_X)
 rf_val_mae = mean_absolute_error(rf_val_predictions, val_y)
 print("Validation MAE for Random Forest Model: {:,.0f}".format(rf_val_mae))
rf_model_on_full_data = RandomForestRegressor(random_state=1)
 rf_model_on_full_data.fit(X,y)
 test_data_path = '.. /input/house-prices-advanced-regression-techniques/test.csv'
test_data = pd.read_csv(test_data_path)
 test_X = test_data[features]
test_preds = rf_model_on_full_data.predict(test_X)
 output = pd.DataFrame({'Id': test_data.Id,'SalePrice': test_preds})
 output.to_csv('submission.csv', index=False )