Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<choose_model_class>
T = torch.Tensor
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d6')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/gwd-efficientdetd6-weights/fold0-efficientdetd6-PL-last-checkpoint1.bin' )<categorify>
SIZE = 800
EPOCHS = 30
CONV_OUT_1 = 50
CONV_OUT_2 = 100
BATCH_SIZE = 32
TEST_PATH = Path('.. /input/abstraction-and-reasoning-challenge/')
 SUBMISSION_PATH = Path('.. /input/abstraction-and-reasoning-challenge/')
 TEST_PATH = TEST_PATH / 'test'
SUBMISSION_PATH = SUBMISSION_PATH / 'sample_submission.csv'
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
test_task_files = sorted(os.listdir(TEST_PATH))
test_tasks = []
for task_file in test_task_files:
with open(str(TEST_PATH / task_file), 'r')as f:
task = json.load(f)
 test_tasks.append(task )
def process_det(index, det, score_threshold=0.33):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=511 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<concatenate>
Xs_test, Xs_train, ys_train = [], [], []
for task in test_tasks:
X_test, X_train, y_train = [], [], []
for pair in task["test"]:
X_test.append(pair["input"])
 for pair in task["train"]:
X_train.append(pair["input"])
 y_train.append(pair["output"])
 Xs_test.append(X_test)
 Xs_train.append(X_train)
 ys_train.append(y_train )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
matrices = []
for X_test in Xs_test:
for X in X_test:
matrices.append(X)
 values = []
for matrix in matrices:
for row in matrix:
for value in row:
values.append(value)
 df = pd.DataFrame(values)
 df.columns = ["values"]
def make_tta_predictions(images, score_threshold=0.33):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
different_y = 0
different_xy = 0
k = []
for i in range(len(training_tasks)) :
task_file = str(training_path / training_tasks[i])
 with open(task_file, 'r')as f:
task = json.load(f)
 outs = [np.array(task['train'][i]['output'])for i in range(len(task['train'])) ]
inps = [np.array(task['train'][i]['input'])for i in range(len(task['train'])) ]
if len(set([x.shape for x in outs])) > 1:
different_y += 1
if any([x.shape != y.shape for x, y in zip(outs, inps)]):
different_xy +=1
k.append(i)
 print(different_y, different_xy)
 print(k )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
def replace_values(a, d):
return np.array([d.get(i, i)for i in range(a.min() , a.max() + 1)])[a - a.min() ]
def repeat_matrix(a):
return np.concatenate([a]*(( SIZE // len(a)) + 1)) [:SIZE]
def get_new_matrix(X):
if len(set([np.array(x ).shape for x in X])) > 1:
X = np.array([X[0]])
 return X
def get_outp(outp, dictionary=None, replace=True):
if replace:
outp = replace_values(outp, dictionary)
 outp_matrix_dims = outp.shape
outp_probs_len = outp.shape[0]*outp.shape[1]*10
outp = to_categorical(outp.flatten() ,
num_classes=10 ).flatten()
return outp, outp_probs_len, outp_matrix_dims
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
class Mutator:
def __init__(self):
pass
def mutate(self, X, y=None):
return X, y
class OneOfMutator(Mutator):
def __init__(self, mutators):
self.mutators = mutators
def mutate(self, X, y=None):
return np.random.choice(self.mutators ).mutate(X, y)
 class MutationPipeline(Mutator):
def __init__(self, mutators):
self.mutators = mutators
def mutate(self, X, y=None):
for mutator in self.mutators:
X, y = mutator.mutate(X, y)
 return X, y
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<load_from_csv>
class Shifter(Mutator):

 def __init__(self, sigma):
self.sigma = sigma
def mutate(self, X, y=None):
direction = np.random.randint(8)
 shift = np.random.randint(self.sigma)
 def do_shift(picture):
shifted = np.zeros(picture.shape, dtype=int)
 if shift == 0:
return picture
if direction == 0:
shifted[:,:-shift] = picture[:,shift:]
if direction == 1:
shifted[:,shift:] = picture[:,:-shift]
if direction == 2:
shifted[:-shift,:] = picture[shift:,:]
if direction == 3:
shifted[shift:,:] = picture[:-shift,:]
if direction == 4:
shifted[:-shift,:-shift] = picture[shift:,shift:]
if direction == 5:
shifted[:-shift,shift:] = picture[shift:,:-shift]
if direction == 6:
shifted[shift:,shift:] = picture[:-shift,:-shift]
if direction == 7:
shifted[shift:,:-shift] = picture[:-shift,shift:]
return shifted
return do_shift(X),(None if y is None else do_shift(y))
mutator = Shifter(2)
 fig, ax = plt.subplots(1, 4, figsize=(15,15))
plot_matrix(np.array(Xs_train[0][0]), ax[0])
 plot_matrix(np.array(ys_train[0][0]), ax[1])
 mutated = mutator.mutate(np.array(Xs_train[0][0]), np.array(ys_train[0][0]))
plot_matrix(mutated[0], ax[2])
 plot_matrix(mutated[1], ax[3] )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
class Flipper(Mutator):

 def __init__(self, do_t=False):
self.do_t = do_t
def mutate(self, X, y=None):
direction = np.random.randint(3-self.do_t)
 def do_mirror(picture):
if direction == 0:
return np.flipud(picture)
 if direction == 1:
return np.fliplr(picture)
 if direction == 2:
return picture.T
return picture
return do_mirror(X),(None if y is None else do_mirror(y))
mutator = Flipper(False)
 fig, ax = plt.subplots(1, 4, figsize=(15,15))
plot_matrix(np.array(X_train[0]), ax[0])
 plot_matrix(np.array(y_train[0]), ax[1])
 mutated = mutator.mutate(np.array(X_train[0]), np.array(y_train[0]))
plot_matrix(mutated[0], ax[2])
 plot_matrix(mutated[1], ax[3] )
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()<categorify>
def get_borders(X):
result = []
max_ = np.max([np.array(x ).shape for x in X])
 for x in X:
x = np.array(x)
 extended = np.zeros(( max_, max_))
diffw = max_ - x.shape[0]
diffh = max_ - x.shape[1]
def get_borders_add(diff):
addl = diff // 2
if addl != diff / 2.0:
addr = addl + 1
else:
addr = addl
return addl, addr
addl, addr = get_borders_add(diffw)
 addt, addb = get_borders_add(diffh)
 result.append(( addl, max_ - addr, addt, max_ - addb))
return result
def extend_matrices_to_max(X):
if len(set([np.array(x ).shape for x in X])) == 1:
return X
result = []
borders = get_borders(X)
 max_ = np.max([np.array(x ).shape for x in X])
 for i, x in enumerate(X):
x = np.array(x)
 extended = np.zeros(( max_, max_))
extended[borders[i][0]:borders[i][1], borders[i][2]:borders[i][3]] = x[:, :]
result.append(extended.astype(int))
return np.array(result)
 def narrow_prediction(prediction, orig_X, orig_y):
different_y_sizes = False
different_with_x_y = False
if len(set([x.shape for x in orig_y])) > 1:
different_y_sizes = True
if any([x.shape != y.shape for x, y in zip(orig_y, orig_X)]):
different_with_x_y = True
if not different_with_x_y:
borders = get_borders(orig_y)
 return np.array([y[borders[i][0]:borders[i][1], borders[i][2]:borders[i][3]] for i,y in enumerate(prediction)])
 result = []
for y in prediction:
for i in range(y.shape[0]):
if np.sum(y[i])> 1e-5:
left = i
for i in reversed(range(y.shape[0])) :
if np.sum(y[i])> 1e-5:
right = i + 1
for i in range(y.shape[1]):
if np.sum(y[:,i])> 1e-5:
bot = i
for i in reversed(range(y.shape[1])) :
if np.sum(y[:,i])> 1e-5:
top = i + 1
result.append(y[left:right, bot:top])
 return np.array(result)
def fill_age_random(df, age_column):

 if age_column+"_filled" not in df:
df.insert(( df.columns.get_loc(age_column)+1), age_column+"_filled", df[age_column])
 keys = []
values = []
for key, value in df[age_column].value_counts().items() :
keys.append(key)
 values.append(value)
 indices = [idx for idx, b in enumerate(df[age_column].isna())if b == True]
for idx, series in df.iterrows() :
if idx in indices:
df.loc[idx, age_column+"_filled"] = random.choices(keys, values)
 return df<prepare_x_and_y>
class Extender:
def __init__(self, X_train, X_test, y_train):
self.X_train = np.array(X_train)
 self.X_test = np.array(X_test)
 self.y_train = np.array(y_train)
 def get_max_shape(X):
shapes1 = [[len(x)] for x in X]
shapes2 = [[len(x[0])] for x in X]
return np.max(( np.max(shapes1),np.max(shapes2)))
 max_train = get_max_shape(X_train)
 max_test = get_max_shape(X_test)
 max_ = np.max([max_train, max_test])
 self.X_train_borders = self.get_borders(self.X_train, max_)
 self.X_test_borders = self.get_borders(self.X_test, max_)
 self.different_y_sizes = False
self.different_with_x_y = False
if len(set([np.shape(x)for x in self.y_train])) > 1:
self.different_y_sizes = True
if any([np.shape(x)!= np.shape(y)for x, y in zip(self.y_train, self.X_train)]):
self.different_with_x_y = True
max_y_train = get_max_shape(self.y_train)
 self.y_train_borders = self.get_borders(self.y_train, max_y_train)
 self.extended_X_train = self.extend_matrices_to_max(self.X_train, max_, self.X_train_borders)
 self.extended_X_test = self.extend_matrices_to_max(self.X_test, max_, self.X_test_borders)
 self.extended_y_train = self.extend_matrices_to_max(self.y_train, max_y_train, self.y_train_borders)
 def get_borders(self, X, max_):
result = []
for x in X:
x = np.array([np.array(k)for k in x])
 extended = np.zeros(( max_, max_))
diffw = max_ - x.shape[0]
diffh = max_ - x.shape[1]
def get_borders_add(diff):
addl = diff // 2
if addl != diff / 2.0:
addr = addl + 1
else:
addr = addl
return addl, addr
addl, addr = get_borders_add(diffw)
 addt, addb = get_borders_add(diffh)
 result.append(( addl, max_ - addr, addt, max_ - addb))
return result
def extend_matrices_to_max(self, X, max_, borders):
shapes = [np.array(x ).shape for x in X]
result = []
for i, x in enumerate(X):
x = np.array([np.array(k,dtype=int)for k in x])
 extended = np.zeros(( max_, max_), dtype=int)
 extended[borders[i][0]:borders[i][1], borders[i][2]:borders[i][3]] = x[:, :]
result.append(extended)
 return np.array(result)
 def narrow_prediction(self, prediction, idxs):
if not self.different_with_x_y:
borders = self.y_train_borders
return np.array([y[borders[i][0]:borders[i][1], borders[i][2]:borders[i][3]] for y,i in zip(prediction, idxs)])
 result = []
for y in prediction:
left, bot=0,0
right, top = y.shape
for i in range(y.shape[0]):
if np.sum(y[i])> 1e-5:
left = i
break
for i in reversed(range(y.shape[0])) :
if np.sum(y[i])> 1e-5:
right = i + 1
break
for i in range(y.shape[1]):
if np.sum(y[:,i])> 1e-5:
bot = i
break
for i in reversed(range(y.shape[1])) :
if np.sum(y[:,i])> 1e-5:
top = i + 1
break
result.append(y[left:right, bot:top])
 return np.array(result)
fill_age_random(test_data, "Age")
 test_data.info()<prepare_x_and_y>
ext = Extender(X_train, X_test,y_train )
y_train = train_data["Survived"]
train_data["Sex_cat"] = train_data["Sex"].astype("category")
 train_data["Sex_code"] = train_data["Sex_cat"].cat.codes
test_data["Sex_cat"] = test_data["Sex"].astype("category")
 test_data["Sex_code"] = test_data["Sex_cat"].cat.codes
features = ["Pclass", "Sex_code", "Age_filled"]
X_test = test_data[features].copy()
X_train = train_data[features].copy()
model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(X_train, y_train)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission is succesfully saved!" )<import_modules>
class ARCDataset(Dataset):
def __init__(self, X_train, X_test, y_train, mutation=ColorSwitcher()):
self.mutation = mutation
self.extender = Extender(X_train, X_test, y_train)
 self.X_train = repeat_matrix(self.extender.extended_X_train)
 self.y_train = repeat_matrix(self.extender.extended_y_train)
 self.colors = []
for x in X_train:
self.colors += np.unique(x ).tolist()
for x in X_test:
self.colors += np.unique(x ).tolist()
for x in y_train:
self.colors += np.unique(x ).tolist()
def __len__(self):
return SIZE
def get_input_dimension(self):
return self.X_train[0].shape
def get_output_dimension(self):
return self.y_train[0].shape
def get_output_prob_dimension(self):
return self.y_train[0].shape
def narrow_prediction(self, prediction, idxs):
return self.extender.narrow_prediction(prediction, idxs)
 def get_extended_test(self):
return self.extender.extended_X_test
def round_colors(self, prediction):
@np.vectorize
def find_nearest(value):
array = np.array(self.colors)
 idx =(np.abs(array - value)).argmin()
return array[idx]
return find_nearest(prediction)
 def __getitem__(self, idx):
in_, out_ = self.mutation.mutate(self.X_train[idx], self.y_train[idx])
 return torch.FloatTensor(in_.copy()), torch.LongTensor(out_.copy()), self.get_output_prob_dimension()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
from sklearn.model_selection import GridSearchCV,StratifiedKFold,cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier,IsolationForest, RandomForestClassifier, VotingClassifier<set_options>
class DoubleConv(nn.Module):

 def __init__(self, in_channels, out_channels, mid_channels=None):
super().__init__()
if not mid_channels:
mid_channels = out_channels
self.double_conv = nn.Sequential(
 nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1),
nn.BatchNorm2d(mid_channels),
nn.ReLU(inplace=True),
nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)
)
 def forward(self, x):
return self.double_conv(x)
 class Down(nn.Module):

 def __init__(self, in_channels, out_channels):
super().__init__()
self.maxpool_conv = nn.Sequential(
 nn.MaxPool2d(2),
DoubleConv(in_channels, out_channels)
)
 def forward(self, x):
return self.maxpool_conv(x)
 class Up(nn.Module):

 def __init__(self, in_channels, out_channels, bilinear=True):
super().__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
 self.conv = DoubleConv(in_channels, out_channels, in_channels // 2)
 else:
self.up = nn.ConvTranspose2d(in_channels , in_channels // 2, kernel_size=2, stride=2)
 self.conv = DoubleConv(in_channels, out_channels)
 def forward(self, x1, x2):
x1 = self.up(x1)
 diffY = x2.size() [2] - x1.size() [2]
diffX = x2.size() [3] - x1.size() [3]
x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,
diffY // 2, diffY - diffY // 2])
 x = torch.cat([x2, x1], dim=1)
 return self.conv(x)
 class OutConv(nn.Module):
def __init__(self, in_channels, out_channels):
super(OutConv, self ).__init__()
self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)
 def forward(self, x):
return self.conv(x)
 def shape(self):
self.conv.shape
sns.set(rc={'figure.figsize':(8,6)})
 sns.set_style("darkgrid" )<load_from_csv>

 class UNet(nn.Module):
def __init__(self, n_channels, n_classes, in_d, out_d, bilinear=True):
super(UNet, self ).__init__()
self.n_channels = n_channels
self.n_classes = n_classes
self.bilinear = bilinear
self.inc = DoubleConv(n_channels, 64)
 self.down1 = Down(64, 128)
 factor = 2 if bilinear else 1
self.up4 = Up(192, 64, bilinear)
 self.outc = OutConv(64, n_classes)
 self.conv = nn.Conv2d(n_classes, n_classes, kernel_size=np.abs(in_d[0]-out_d[0])+1, padding=-np.min(( in_d[0]-out_d[0], 0)))
 def forward(self, x):
x1 = self.inc(x)
 x2 = self.down1(x1)
 x = self.up4(x2, x1)
 logits = self.outc(x)
 logits = self.conv(logits)
 return logits
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )<define_variables>
class BasicCNNModel(nn.Module):
def __init__(self, inp_dim=(10, 10), outp_dim=(10, 10)) :
super(BasicCNNModel, self ).__init__()
self.network = UNet(1, 10, inp_dim, outp_dim)
 def forward(self, x):
x_ = self.network.forward(x)
 return x_
test_ids = test['PassengerId']<predict_on_test>
a=BasicCNNModel()
a.parameters
rng = np.random.RandomState(42)
 clf = IsolationForest(max_samples=100, random_state=rng,contamination = 0.05)
 clf.fit(train[["Age","SibSp","Parch","Fare"]].dropna())
 prediction = clf.predict(train[["Age","SibSp","Parch","Fare"]].dropna())
 df_outliers = train[["Age","SibSp","Parch","Fare"]].dropna()
df_outliers['Outlier'] = prediction
index_list = df_outliers[df_outliers['Outlier'] == -1].index
train.drop(index_list, inplace=True )<concatenate>
def transform_dim(inp_dim, outp_dim, test_dim):
return(test_dim[0]*outp_dim[0]/inp_dim[0],
test_dim[1]*outp_dim[1]/inp_dim[1])
 def resize(x, test_dim, inp_dim):
if inp_dim == test_dim:
return x
else:
return cv2.resize(flt(x), inp_dim,
interpolation=cv2.INTER_AREA)
 def flt(x): return np.float32(x)
 def npy(x): return x.cpu().detach().numpy()
def itg(x): return np.int32(np.round(x))
df = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True )<count_missing_values>
def train_model(model, train_loader, dataset, loss, optimizer, num_epochs, scheduler=None):
loss_history = []
train_history = []
val_history = []
for epoch in range(num_epochs):
model.train()
loss_accum = 0
correct_samples = 0
total_samples = 0
for i_step, train_batch in enumerate(train_loader):
x, y, prob_d = train_batch
prediction = model(x.unsqueeze(1))
loss_value = loss(prediction, y)
 optimizer.zero_grad()
loss_value.backward()
loss_accum += loss_value
optimizer.step()
ave_loss = loss_accum /(i_step if i_step else 1)
 loss_history.append(float(ave_loss))
if scheduler is not None:
scheduler.step()
print("Average loss: %f" %(ave_loss))
if len(loss_history)> 5 and np.mean(np.abs(np.array(loss_history[-3:])- np.array(loss_history[-4:-1])))< epoch*1e-5:
break
return loss_history
df.isnull().sum()<count_missing_values>
idx = 0
start = time.time()
test_predictions = []
class Flattener(nn.Module):
def forward(self, x):
batch_size, *_ = x.shape
return x.view(batch_size, -1)
 def do_train(dataset, do_test=False):
train_loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)
 in_d = dataset.get_input_dimension()
prob_d = dataset.get_output_prob_dimension()
d = dataset.get_output_dimension()
network = BasicCNNModel(in_d, d)
 loss = nn.CrossEntropyLoss()
optimizer = Adam(network.parameters() , lr=1e-2, weight_decay=1e-3)
 step_lr = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.05)
 lh = train_model(network, train_loader, dataset, loss, optimizer, EPOCHS)
 preds = []
if do_test:
test_loader = DataLoader(np.array(dataset.extender.extended_X_test, dtype=np.single), batch_size=1, shuffle=False)
 network.eval()
for i, test_batch in enumerate(test_loader):
test_batch = test_batch
test_preds = network(torch.FloatTensor(test_batch ).unsqueeze(1))
_, indices = torch.max(test_preds, 1)
 test_preds = dataset.narrow_prediction(indices.detach().numpy() , [i])
 preds.append(dataset.round_colors(test_preds))
return lh[-1], preds
for idx,(X_train, y_train)in enumerate(zip(Xs_train, ys_train)) :
X_test = Xs_test[idx]
print("TASK " + str(idx + 1))
mutator_losses = {}
losses = []
EPOCHS = 70
if len(mutators[idx].mutators)!= 0:
dataset = ARCDataset(X_train, X_test, y_train, mutators[idx])
 train_loss, predictions = do_train(dataset, do_test=True)
 test_predictions += predictions
else:
test_predictions += X_test
end = time.time()
print("Train loss: " + str(np.round(train_loss, 3)) + " " +\
"Total time: " + str(np.round(end - start, 1)) + " s" + "
")
 fig, ax = plt.subplots(1, 4, figsize=(15,15))
plot_matrix(np.array(X_train[0]), ax[0])
 plot_matrix(np.array(y_train[0]), ax[1])
 plot_matrix(X_test[0], ax[2])
 plot_matrix(test_predictions[-1][0], ax[3] )
df["Fare"].isnull().sum()<correct_missing_values>
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
df["Fare"].fillna(df["Fare"].median() ,inplace=True )<feature_engineering>
pp = 0
for k in test_predictions:
pp+=1
pp
df["Fare"] = np.log(df["Fare"] ).replace(-np.inf, 0 )<count_missing_values>
test_predictions = [[list(pred)for pred in test_pred] for test_pred in test_predictions]
for idx, pred in enumerate(test_predictions):
test_predictions[idx] = flattener(pred)
 submission = pd.read_csv(SUBMISSION_PATH)
 submission["output"] = test_predictions
df["Embarked"].isnull().sum()<feature_engineering>
submission.to_csv("submission.csv", index=False )
<count_missing_values><EOS> 
submission.to_csv("submission.csv", index=False )