Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
data_train = pd.read_csv('.. /input/train.csv')
 data_test = pd.read_csv('.. /input/test.csv' )
y_test_sorted = np.zeros(( 3, probs.shape[1]))
test = test.reset_index()
test = test.set_index('image_name')
 i = 0
ds_test = getTestDataset(filenames_test, cfg)
 for img, imgid in tqdm(iter(ds_test.unbatch())) :
imgid = imgid.numpy().decode('utf-8')
 y_test_sorted[:, test.loc[imgid]['index']] = probs[:, i, 0]
i += 1
for i in range(y_test_sorted.shape[0]):
submission = sample
submission['target'] = y_test_sorted[i]
submission.to_csv('submission_model_%s.csv' % i, index=False)
 submission = sample
submission['target'] = np.mean(y_test_sorted, axis=0)
 submission.to_csv('blended_effnets.csv', index=False )<load_pretrained>
data_train = data_train.drop(columns=['Name', 'Ticket', 'Fare', 'Cabin'])
 data_test = data_test.drop(columns=['Name', 'Ticket', 'Fare', 'Cabin'])
!gzip submission_model_0.csv
!gzip submission_model_1.csv
!gzip submission_model_2.csv
!gzip blended_effnets.csv<load_from_csv>
display(data_train.Age.value_counts(dropna=False ).sort_index())
 display(data_test.Age.value_counts(dropna=False ).sort_index() )
def MinMaxBestBaseStacking(input_folder, best_base, output_path):
sub_base = pd.read_csv(best_base)
 all_files = os.listdir(input_folder)
 outs = [pd.read_csv(os.path.join(input_folder, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "target" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub['is_iceberg_max'] = concat_sub.iloc[:, 1:6].max(axis=1)
 concat_sub['is_iceberg_min'] = concat_sub.iloc[:, 1:6].min(axis=1)
 concat_sub['is_iceberg_mean'] = concat_sub.iloc[:, 1:6].mean(axis=1)
 concat_sub['is_iceberg_median'] = concat_sub.iloc[:, 1:6].median(axis=1)
 cutoff_lo = 0.85
cutoff_hi = 0.17
concat_sub['is_iceberg_base'] = sub_base['target']
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:, 1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:, 1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_base']))
concat_sub[['image_name', 'target']].to_csv(output_path,
index=False, float_format='%.12f')
<define_variables>
data_train.Age = data_train.Age.fillna(data_train.Age.mean())
 data_test.Age = data_test.Age.fillna(data_test.Age.mean())
MinMaxBestBaseStacking('.. /input/melanoma-ensemble-files/', '.. /input/melanoma-ensemble-files/blend_sub.csv', 'submission.csv' )<load_from_csv>
display(data_train.Embarked.value_counts(dropna=False))
display(data_test.Embarked.value_counts(dropna=False))
!ls /kaggle/input/20201124-ensemble-1-testcsv<import_modules>
data_train.Embarked = data_train.Embarked.fillna('S')
warnings.filterwarnings("ignore" )<import_modules>
b = data_train.pop('Survived')
 data_train = pd.concat([data_train, b], axis=1)
 display(data_train.head())
l5kit.__version__<set_options>
display(data_train.Age.value_counts(dropna=False ).sort_index())
 display(data_test.Age.value_counts(dropna=False ).sort_index() )
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 set_seed(42 )<init_hyperparams>
display(data_train.Sex.value_counts(dropna=False ).sort_index())
 display(data_test.Sex.value_counts(dropna=False ).sort_index() )
cfg = {
'format_version': 4,
'data_path': "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
'model_params': {
'model_architecture': 'resnet50',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1,
'model_name': "model_resnet50_output",
'lr': 1e-3,
'weight_path': False,
'train': True,
'predict': True
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 16,
'shuffle': True,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 101,
'checkpoint_every_n_steps': 20,
}
}<feature_engineering>
display(( data_train.Pclass.value_counts(dropna=False ).sort_index()))
display(( data_test.Pclass.value_counts(dropna=False ).sort_index()))
DIR_INPUT = cfg["data_path"]
os.environ["L5KIT_DATA_FOLDER"] = DIR_INPUT
dm = LocalDataManager(None )<create_dataframe>
display(data_train.Survived.value_counts(dropna=False ).sort_index())
train_cfg = cfg["train_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 train_zarr = ChunkedDataset(dm.require(train_cfg["key"])).open()
train_dataset = AgentDataset(cfg, train_zarr, rasterizer)
 train_dataloader = DataLoader(train_dataset, shuffle=train_cfg["shuffle"], batch_size=train_cfg["batch_size"],
num_workers=train_cfg["num_workers"])
 print("==================================TRAIN DATA==================================")
 print(train_dataset )<create_dataframe>
display(data_train.SibSp.value_counts(dropna=False ).sort_index())
 display(data_test.SibSp.value_counts(dropna=False ).sort_index() )
test_cfg = cfg["test_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open()
test_mask = np.load(f"{DIR_INPUT}/scenes/mask.npz")["arr_0"]
test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataloader = DataLoader(test_dataset,shuffle=test_cfg["shuffle"],batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"])
 print("==================================TEST DATA==================================")
 print(test_dataset )<choose_model_class>
display(data_train.Parch.value_counts(dropna=False ).sort_index())
 display(data_test.Parch.value_counts(dropna=False ).sort_index() )
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
architecture = cfg["model_params"]["model_architecture"]
backbone = eval(architecture )(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 if architecture == "resnet50":
backbone_out_features = 2048
else:
backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences<compute_train_metric>
display(data_train.Embarked.value_counts(dropna=False ).sort_index())
 display(data_test.Embarked.value_counts(dropna=False ).sort_index() )
def forward(data, model, device, criterion = pytorch_neg_multi_log_likelihood_batch):
inputs = data["image"].to(device)
 target_availabilities = data["target_availabilities"].to(device)
 targets = data["target_positions"].to(device)
 preds, confidences = model(inputs)
 loss = criterion(targets, preds, confidences, target_availabilities)
 return loss, preds, confidences<choose_model_class>
import seaborn as sns
import matplotlib.pyplot as plt
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = LyftMultiModel(cfg)
 weight_path = cfg["model_params"]["weight_path"]
if weight_path:
model.load_state_dict(torch.load(weight_path))
model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg["model_params"]["lr"])
 print(f'device {device}' )<train_model>
data_test[(data_test.Embarked=='S')].groupby(['Pclass', 'Sex'] ).size()
print(model )<init_hyperparams>
data_train = pd.get_dummies(data_train, columns=['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked'])
 data_train.info()
if cfg["model_params"]["train"]:
tr_it = iter(train_dataloader)
 progress_bar = tqdm(range(cfg["train_params"]["max_num_steps"]))
num_iter = cfg["train_params"]["max_num_steps"]
losses_train = []
iterations = []
metrics = []
times = []
model_name = cfg["model_params"]["model_name"]
start = time.time()
for i in progress_bar:
try:
data = next(tr_it)
 except StopIteration:
tr_it = iter(train_dataloader)
 data = next(tr_it)
 model.train()
torch.set_grad_enabled(True)
 loss, _, _ = forward(data, model, device)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
losses_train.append(loss.item())
 progress_bar.set_description(f"loss: {loss.item() } loss(avg): {np.mean(losses_train)}")
 if i % cfg['train_params']['checkpoint_every_n_steps'] == 0:
torch.save(model.state_dict() , f'{model_name}_{i}.pth')
 iterations.append(i)
 metrics.append(np.mean(losses_train))
times.append(( time.time() -start)/60)
 results = pd.DataFrame({'iterations': iterations, 'metrics(avg)': metrics, 'elapsed_time(mins)': times})
 results.to_csv(f"train_metrics_{model_name}_{num_iter}.csv", index = False)
 print(f"Total training time is {(time.time() -start)/60} mins")
 print(results.head() )<save_to_csv>
data_test = pd.get_dummies(data_test, columns=['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked'])
 data_test.info()
pred_path = 'submission.csv'
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs = np.concatenate(confidences_list)
 )<set_options>
data_train['Age'] =(data_train.Age//10*10 )
sns.set()
%matplotlib inline
EPOCHS = 150
BATCH_SIZE = 16
SEED = 20031976
LRATE = 0.0001
VERBOSE=0 
<load_from_csv>
data_test['Age'] =(data_test.Age//10*10 )
np.random.seed(SEED)
 tf.set_random_seed(SEED)
 train_df = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/test.csv')
 print("Datasets loaded.. ")
 print('Train DF Shape', train_df.shape )<train_model>
data_train = pd.get_dummies(data_train, columns=['Age'] )
x_resampled, y_resampled = SMOTE(random_state=SEED ).fit_sample(x_train.reshape(x_train.shape[0], -1), train_df['diagnosis'].ravel())
 print("x_resampled.shape=",x_resampled.shape)
 print("y_resampled.shape=",y_resampled.shape)
 x_train = x_resampled.reshape(x_resampled.shape[0], 224, 224, 3)
 y_train = pd.get_dummies(y_resampled ).values
print("x_train.shape=",x_train.shape)
 print("y_train.shape=",y_train.shape )<categorify>
data_test = pd.get_dummies(data_test, columns=['Age'] )
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
b = data_train.pop('Survived')
 data_train = pd.concat([data_train, b], axis=1)
 data_train.head()
x_sptrain, x_spval, y_sptrain, y_spval = train_test_split(
 x_train, y_train_multi,
test_size=0.10,
random_state=SEED 
)
 print("train-validation splitted..." )<train_model>
X = data_train.drop(columns = ['Survived', 'PassengerId'], axis=1 )
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.10,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_sptrain, y_sptrain, batch_size=BATCH_SIZE, seed=SEED)
 print("Image data augmentated..." )<compute_test_metric>
y = data_train.Survived
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def fbeta_score(y_true, y_pred, beta=1):
if beta < 0:
raise ValueError('The lowest choosable beta is zero(only precision ).')
 if K.sum(K.round(K.clip(y_true, 0, 1)))== 0:
return 0
p = precision(y_true, y_pred)
 r = recall(y_true, y_pred)
 bb = beta ** 2
fbeta_score =(1 + bb)*(p * r)/(bb * p + r + K.epsilon())
 return fbeta_score
def fmeasure(y_true, y_pred):
return fbeta_score(y_true, y_pred, beta=1)
 def mean_pred(y_true, y_pred):
return K.mean(y_pred)
 def f1_score(y_true, y_pred):
p = precision(y_true, y_pred)
 r = recall(y_true, y_pred)
 return 2*(p*r)/(p+r+K.epsilon())
 print("Evaluation metrics defined...")
<import_modules>
from sklearn.model_selection import train_test_split
from keras.applications import DenseNet169,DenseNet121 
<choose_model_class>
from sklearn.model_selection import train_test_split
densenet = DenseNet121(
 weights='/kaggle/input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
)
 model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.2))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=LRATE),
metrics=['accuracy',mean_pred, precision, recall, f1_score, fbeta_score, fmeasure] 
)
 model.summary()<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=2 )
class KappaMetrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"Epoch: {epoch+1} val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
kappa_score = KappaMetrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=EPOCHS,
validation_data=(x_spval, y_spval),
callbacks=[kappa_score],
verbose=VERBOSE
)<save_to_csv>
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score
with open('history.json', 'w')as f:
json.dump(history.history, f)
 history_df = pd.DataFrame(history.history)
 history_df.head(EPOCHS )<save_to_csv>
num_trees = 1000
max_features = 3
kfold = KFold(n_splits=10, random_state=7)
 rfc = RandomForestClassifier(n_estimators=num_trees, max_features=max_features )
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False)
 display(test_df.head(5)) 
<set_options>
rfc.fit(X_train, y_train )
sns.set()
%matplotlib inline
EPOCHS = 50
BATCH_SIZE = 16
SEED = 20031976
LRATE = 0.00005
VERBOSE=0 
<load_from_csv>
rfc.score(X_train, y_train )
np.random.seed(SEED)
 tf.set_random_seed(SEED)
 train_df = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/test.csv')
 print("Datasets loaded.. ")
 print('Train DF Shape', train_df.shape )<train_model>
rfc.score(X_test, y_test )
x_resampled, y_resampled = SMOTE(random_state=SEED ).fit_sample(x_train.reshape(x_train.shape[0], -1), train_df['diagnosis'].ravel())
 print("x_resampled.shape=",x_resampled.shape)
 print("y_resampled.shape=",y_resampled.shape)
 x_train = x_resampled.reshape(x_resampled.shape[0], 224, 224, 3)
 y_train = pd.get_dummies(y_resampled ).values
print("x_train.shape=",x_train.shape)
 print("y_train.shape=",y_train.shape )<categorify>
y_pred = rfc.predict(X_test )
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
from sklearn.metrics import accuracy_score
x_sptrain, x_spval, y_sptrain, y_spval = train_test_split(
 x_train, y_train_multi,
test_size=0.10,
random_state=SEED 
)
 print("train-validation splitted..." )<train_model>
acc = accuracy_score(y_test, y_pred )
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.10,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_sptrain, y_sptrain, batch_size=BATCH_SIZE, seed=SEED)
 print("Image data augmentated..." )<compute_test_metric>
data_test.isnull().sum()
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def fbeta_score(y_true, y_pred, beta=1):
if beta < 0:
raise ValueError('The lowest choosable beta is zero(only precision ).')
 if K.sum(K.round(K.clip(y_true, 0, 1)))== 0:
return 0
p = precision(y_true, y_pred)
 r = recall(y_true, y_pred)
 bb = beta ** 2
fbeta_score =(1 + bb)*(p * r)/(bb * p + r + K.epsilon())
 return fbeta_score
def fmeasure(y_true, y_pred):
return fbeta_score(y_true, y_pred, beta=1)
 def mean_pred(y_true, y_pred):
return K.mean(y_pred)
 def f1_score(y_true, y_pred):
p = precision(y_true, y_pred)
 r = recall(y_true, y_pred)
 return 2*(p*r)/(p+r+K.epsilon())
 print("Evaluation metrics defined...")
<import_modules>
data_test = data_test.drop(columns=['PassengerId'] )
from keras.applications import DenseNet169 
<choose_model_class>
df = pd.DataFrame({'PassengerId': range(892, 1310), 'Survived':(rfc.predict(data_test)) } )
<compute_train_metric><EOS> 
df.to_csv('TitanicDataSetKaggleVersion2.csv', index=False )