Kaggle competition: Titanic - Machine Learning from Disaster
def input_output_shape_is_same(task):
return all([np.array(el['input'] ).shape == np.array(el['output'] ).shape for el in task['train']])
 def calk_score(task_test, predict):
return [int(np.equal(sample['output'], pred ).all())for sample, pred in zip(task_test, predict)]<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 dataset = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True)
 dataset = dataset.fillna(np.nan)
 dataset.head(15 )
def evaluate(tasks):
ts = TaskSolver()
result = []
predictions = []
for task in tqdm(tasks):
if input_output_shape_is_same(task):
ts.train(task['train'])
 pred = ts.predict(task['test'])
 score = calk_score(task['test'], pred)
 else:
pred = [el['input'] for el in task['test']]
score = [0]*len(task['test'])
 predictions.append(pred)
 result.append(score)
 return result, predictions<compute_test_metric>
def find_title(name):
result = re.search(r'[A-Z][a-z]*[.]', name)
 if(result == None):
return ''
return result.group(0)
 dataset['Title'] = dataset.Name.apply(find_title)
 dataset.Title.unique()
train_result, train_predictions = evaluate(train_tasks)
 train_solved = [any(score)for score in train_result]
total = sum([len(score)for score in train_result])
 print(f"solved : {sum(train_solved)} from {total}({sum(train_solved)/total})" )<compute_test_metric>
def set_median_age_by_title(title):
title_age_mask = dataset.loc[(dataset['Title'] == title)&(dataset['Age'].isna())]
df = dataset.iloc[title_age_mask.index]
df.Age = dataset.Age.median()
dataset.iloc[title_age_mask.index] = df
for title in dataset.Title.unique() :
set_median_age_by_title(title )
evaluation_result, evaluation_predictions = evaluate(evaluation_tasks)
 evaluation_solved = [any(score)for score in evaluation_result]
total = sum([len(score)for score in evaluation_result])
 print(f"solved : {sum(evaluation_solved)} from {total}({sum(evaluation_solved)/total})" )<predict_on_test>
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
dataset.drop(['Name','Ticket'], axis=1, inplace=True)
 dataset.head()
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred
def make_pediction(tasks):
ts = TaskSolver()
result = pd.Series()
for idx, task in tqdm(test_tasks.iteritems()):
if input_output_shape_is_same(task):
ts.train(task['train'])
 pred = ts.predict(task['test'])
 else:
pred = [el['input'] for el in task['test']]
for i, p in enumerate(pred):
result[f'{idx}_{i}'] = flattener(np.array(p ).tolist())
 return result<prepare_output>
fare_midean = dataset['Fare'].median()
values = {'Cabin':'Z','Embarked':'S','Fare':fare_midean}
dataset.fillna(value=values,inplace=True)
 dataset['Cabin1'] = dataset['Cabin'].str[0]
dataset.drop(['Cabin'], axis=1, inplace=True)
 dataset['Sex'] = LabelEncoder().fit_transform(dataset['Sex'].values)
 dataset['Cabin1'] = LabelEncoder().fit_transform(dataset['Cabin1'].values)
 dataset['Embarked'] = LabelEncoder().fit_transform(dataset['Embarked'].values)
 dataset['Title'] = LabelEncoder().fit_transform(dataset['Title'].values)
 dataset.head()
submission = make_pediction(test_tasks)
 submission.head()<save_to_csv>
working_columns = ['Pclass','Sex','Age','Fare','Embarked','FamilySize','Cabin1','Title','SibSp','Parch']
columns_to_scale = ['Pclass','Sex','Age','Fare','Embarked','FamilySize','Cabin1','Title','SibSp','Parch']
def scale_dataset(dataset):
scalers = {}
for column in columns_to_scale:
scaler = StandardScaler()
dataset[column] = scaler.fit_transform(dataset[column].values.reshape(-1, 1))
scalers[column] = scaler
return dataset, scalers
dataset, scalers = scale_dataset(dataset )
submission = submission.reset_index()
submission.columns = ['output_id', 'output']
submission.to_csv('submission.csv', index=False)
 submission<import_modules>
columns = [c for c in train_set.columns if c not in ["Survived", "PassengerId", "Pclass", "Sex", "Cabin1", "Embarked", 'FamilySize']]
clf = LocalOutlierFactor(n_neighbors=5, contamination=0.5)
 y_pred = clf.fit_predict(train_set[columns].values)
 X_scores = clf.negative_outlier_factor_
clf_df = pd.DataFrame(columns=['X_scores', 'y_pred'])
 clf_df.X_scores = X_scores
clf_df.y_pred = y_pred
outlier_index = clf_df.index[clf_df['y_pred'] == -1].tolist()
train_set.drop(outlier_index )
import os
import json
from pathlib import Path
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np
from pathlib import Path<define_variables>
y_source = train_set['Survived'].values.astype(int)
 X_source = train_set[working_columns].values
X_test = test_set[working_columns].values
y_test = test_set['Survived'].values.astype(int )
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
training_tasks = sorted(os.listdir(training_path))
evaluation_tasks = sorted(os.listdir(evaluation_path))
test_tasks = sorted(os.listdir(test_path))
print(len(training_tasks), len(evaluation_tasks), len(test_tasks))<load_from_disk>
X_train, X_test, y_train, y_test = train_test_split(X_source, y_source, test_size = 0.2)
 print('X_train: {}'.format(X_train.shape))
print('y_train: {}'.format(y_train.shape))
print('X_test: {}'.format(X_test.shape))
print('y_test: {}'.format(y_test.shape))
task_file = str(training_path / '00d62c1b.json')
 with open(task_file, 'r')as f:
task = json.load(f)
 print(task.keys() )<define_variables>
def create_model2(optimizer, units = 16, init = 'glorot_uniform', activation = 'relu', l2_init = 0.001, dropout=0.1):
model = Sequential()
model.add(Dense(units=units, input_dim=10, activation=activation, kernel_initializer=init, kernel_regularizer=l2(l2_init)))
 model.add(Dropout(dropout))
model.add(Dense(units=units, activation=activation, kernel_initializer=init, kernel_regularizer=l2(l2_init)))
 model.add(BatchNormalization())
 model.add(Dropout(dropout))
model.add(Dense(units=units, activation=activation, kernel_initializer=init))
model.add(Dense(1, activation='sigmoid',kernel_initializer=init))
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 return model
n_train_pairs = len(task['train'])
 n_test_pairs = len(task['test'])
 print(f'task contains {n_train_pairs} training pairs')
 print(f'task contains {n_test_pairs} test pairs' )<load_from_csv>
callbacks = [EarlyStopping(monitor='val_loss', patience=100)]
adam = optimizers.Adam(lr=3e-4, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
 best_model = create_model2(units=16, optimizer=adam, init='glorot_uniform', activation='relu', l2_init=0.3)
 history = best_model.fit(X_train, y_train,
validation_data=[X_test, y_test],
batch_size=64,
epochs=1000,
callbacks=callbacks,
verbose=1 )
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )<categorify>
scores = best_model.evaluate(X_train, y_train, verbose=0)
 print("%s: %.2f%%" %(best_model.metrics_names[1], scores[1]*100))
scores = best_model.evaluate(X_test, y_test, verbose=0)
 print("%s: %.2f%%" %(best_model.metrics_names[1], scores[1]*100))
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred 
<concatenate>
predictions = best_model.predict_classes(test_set[working_columns].values)
 submission = pd.DataFrame({
"PassengerId": test_set["PassengerId"],
"Survived": predictions.ravel()
})
 submission.to_csv('submission.csv', index=False )