Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
!pip uninstall -y dataclasses
LOGITS = []
LOGITS2 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu())/ 2
PREDS = LOGITS.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<set_options>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
warnings.filterwarnings("ignore")
 sys.path.insert(0, '.. /input/semisupervised-imagenet-models/semi-supervised-ImageNet1K-models-master/')
<define_variables>
features = ["Pclass", "Sex", "SibSp", "Parch", "Fare", "Embarked", "Age"]
label = ["Survived"]
X_train = pd.get_dummies(train_data[features + label])
 X_test = pd.get_dummies(test_data[features])
 X_train.head()
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
TEST = '.. /input/prostate-cancer-grade-assessment/test.csv'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
MODELS = [f'.. /input/panda-init-class-model1/RNXT50_128krnew1_3featureB_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128krnew1_2featureB_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128krnew1_1featureB_{i}.pth' for i in range(4)]
MODELS1 = [f'.. /input/panda-init-class-model1/RNXT50_128krnew_2_{i}.pth' for i in range(4)]
sz = 128
bs = 1
N_max = 3*128
N_max1 = 256
nworkers = 2<choose_model_class>
X_train.isna().sum()
class Modelm1(nn.Module):
def __init__(self, arch='resnext50_32x4d', n=6, pre=True):
super().__init__()
m = _resnext(semi_supervised_model_urls[arch], Bottleneck, [3, 4, 6, 3], False, progress=False,\
groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(AdaptiveConcatPool2d() ,Flatten() ,
nn.Linear(2*nc,512),Mish() ,nn.GroupNorm(32,512),
nn.Dropout(0.5),nn.Linear(512,n))
def forward(self, x):
shape = x.shape
n = shape[1]
x = x.view(-1,shape[2],shape[3],shape[4])
 x = self.enc(x)
 shape = x.shape
x = x.view(-1,n,shape[1],shape[2],shape[3] ).permute(0,2,1,3,4 ).contiguous() \ 
.view(-1,shape[1],shape[2]*n,shape[3])
 x = self.head(x)
 return x[:,:1]<choose_model_class>
X_test.isna().sum()
class AdaptiveConcatPool2dm1(Module):
"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`."
def __init__(self, sz:Optional[int]=None):
"Output will be 2*sz or 2 if sz is None"
self.output_size = sz or 1
self.ap = nn.AdaptiveAvgPool2d(self.output_size)
 self.mp = nn.AdaptiveMaxPool2d(self.output_size)
 def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)
 def _resnext(url, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 return model
class Model(nn.Module):
def __init__(self, arch='resnext50_32x4d', n=6, pre=True):
super().__init__()
m = _resnext(semi_supervised_model_urls[arch], Bottleneck, [3, 4, 6, 3], False, progress=False,\
groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(AdaptiveConcatPool2d() ,Flatten() ,
nn.Linear(2*nc,512),Mish() ,nn.GroupNorm(32,512),
nn.Dropout(0.5),nn.Linear(512,n))
def forward(self, x):
shape = x.shape
n = shape[1]
x = x.view(-1,shape[2],shape[3],shape[4])
 x = self.enc(x)
 shape = x.shape
x = x.view(-1,n,shape[1],shape[2],shape[3] ).permute(0,2,1,3,4 ).contiguous() \ 
.view(-1,shape[1],shape[2]*n,shape[3])
 x = self.head(x)
 return x[:,:1]<train_on_grid>
X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean() )
class AdaptiveConcatPool2dm(Module):
"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`."
def __init__(self, sz:Optional[int]=None):
"Output will be 2*sz or 2 if sz is None"
self.output_size = sz or 1
self.ap = nn.AdaptiveAvgPool2d(self.output_size)
 self.mp = nn.AdaptiveMaxPool2d(self.output_size)
 def forward(self, x): return torch.cat([self.ap(x),self.mp(x)], 1)
 def _resnext(url, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 return model
class Modelm(nn.Module):
def __init__(self, arch='resnext50_32x4d', n=6, pre=True):
super().__init__()
m = _resnext(semi_supervised_model_urls[arch], Bottleneck, [3, 4, 6, 3], False, progress=False,\
groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(AdaptiveConcatPool2dm() ,Flatten() ,
nn.Linear(2*nc,512),Mish() ,nn.GroupNorm(32,512),
nn.Dropout(0.5),nn.Linear(512,n))
def forward(self, x):
shape = x.shape
n = shape[1]
x = x.view(-1,shape[2],shape[3],shape[4])
 x = self.enc(x)
 shape = x.shape
x = x.view(-1,n,shape[1],shape[2],shape[3] ).permute(0,2,1,3,4 ).contiguous() \ 
.view(-1,shape[1],shape[2]*n,shape[3])
 x = self.head(x)
 return x[:,:1]<find_best_params>
y = X_train[label].values.ravel()
X_train = X_train.drop(label, axis=1 )
models = []
for path in MODELS[:-4]:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = Model(n=1+10)
 model.load_state_dict(state_dict)
 model.float()
model.eval()
model.cuda()
models.append(model)
 for path in MODELS[-4:]:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = Modelm(n=1+10)
 model.load_state_dict(state_dict)
 model.float()
model.eval()
model.cuda()
models.append(model)
 models2 = []
for path in MODELS1:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = Modelm1(n=1+10)
 model.load_state_dict(state_dict)
 model.float()
model.eval()
model.cuda()
models2.append(model)
 del state_dict<load_from_csv>
dt_config = {
"class": DecisionTreeClassifier,
"criterion": tune.choice(['gini', 'entropy']),
"max_depth": tune.randint(2, 8),
"min_samples_split": tune.randint(2, 10),
'min_samples_leaf': tune.randint(1, 10),
"random_state": 1
}
rf_config = {
"class": RandomForestClassifier,
"max_depth": tune.randint(2, 8),
"n_estimators": tune.qrandint(lower=10, upper=200, q=10),
'random_state': 1
}
xgb_config = {
"class": xgb.XGBClassifier,
"max_depth": tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1),
'random_state': 1
}
svm_config = {
"class": SVC,
"kernel": tune.choice(['linear', 'rbf', 'sigmoid']),
"gamma": tune.choice(['scale', 'auto']),
"random_state": 1
}
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
ds = PandaDataset(DATA,TEST)
 dl = DataLoader(ds, batch_size=bs, num_workers=nworkers, shuffle=False)
 names,preds = [],[]
with torch.no_grad() :
for x,x2,y in tqdm(dl):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),x.transpose(-1,-2),\
x.transpose(-1,-2 ).flip(-1), x.transpose(-1,-2 ).flip(-2),\
x.transpose(-1,-2 ).flip(-1,-2)],1)
 x = x.view(8,-1,3,sz,sz)
 p = [model(x[a].unsqueeze(0)) for model in models for a in range(x.shape[0])]
x2 = x2.cuda()
x2 = torch.stack([x2,x2.flip(-1),x2.flip(-2),x2.flip(-1,-2),x2.transpose(-1,-2),\
x2.transpose(-1,-2 ).flip(-1), x2.transpose(-1,-2 ).flip(-2),\
x2.transpose(-1,-2 ).flip(-1,-2)],1)
 x2 = x2.view(8,-1,3,sz,sz)
 p += [model(x2[a].unsqueeze(0)) for model in models2 for a in range(x2.shape[0])]
p = torch.stack(p,1)
 p = p.view(bs,8*(len(models)+len(models2)) ).mean(1 ).cpu()
p = torch.clamp(( 6.0*torch.sigmoid(p)).long().view(-1),0,5)
 names.append(y)
 preds.append(p)
 names = np.concatenate(names)
 preds = torch.cat(preds ).numpy()
sub_df = pd.DataFrame({'image_id': names, 'isup_grade': preds})
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<save_to_csv>
methods = {"rf": rf_config, "xgb": xgb_config, "svm": svm_config, "dt": dt_config}
def export_csv(predictions, name:str):
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 filename = f'{name}_submission.csv'
output.to_csv(filename, index=False)
 print(f"Your submission({name})was successfully saved! {filename}")
 def run_tune(method: str, num_samples: int)-> tune.ExperimentAnalysis:
optuna_search = OptunaSearch(metric="mean_accuracy", mode="max")
 return tune.run(
 trial,
config=methods[method],
num_samples=num_samples,
search_alg=optuna_search,
verbose=-1 
)
 def trial(config: Dict[str, Any]):
_config = deepcopy(config)
 model_class = config["class"]
_config.pop("class", None)
 model = model_class(
 **_config 
)
 scores = cross_val_score(model, X_train, y, cv=5, scoring="accuracy")
 tune.report(
 mean_accuracy=np.mean(scores),
done=True 
)
 tree_config ={"max_depth":5}
model = methods["dt"]["class"](**tree_config)
 scores = cross_val_score(model, X_train, y, cv=5, scoring="accuracy")
 model = methods["rf"]["class"](**tree_config)
 scores = cross_val_score(model, X_train, y, cv=5, scoring="accuracy")
 model = methods["xgb"]["class"](**tree_config)
 scores = cross_val_score(model, X_train, y, cv=5, scoring="accuracy")
 model = methods["svm"]["class"](**{"kernel": "linear"})
 scores = cross_val_score(model, X_train, y, cv=5, scoring="accuracy" )
sub_df.to_csv("submission.csv", index=False)
 sub_df.head()<define_variables>
def run_experiment(method: str, num_samples: int=50)-> Dict[str, Any]:
result = run_tune(method, num_samples ).get_best_trial(metric="mean_accuracy", mode="max")
 _config = deepcopy(result.config)
 model_class = _config["class"]
_config.pop("class")
 model = model_class(**_config)
 model.fit(X_train, y)
 predictions = model.predict(X_test)
 export_csv(predictions, method)
 return result
package_path = '/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'
sys.path.append(package_path)
<define_variables>
dt = run_experiment(method="dt", num_samples=100)
 print(f"result {dt.last_result}")
 print(f"{dt.config}" )
<define_variables>
rf = run_experiment(method="rf", num_samples=100)
 print(f"result {rf.last_result}")
 print(f"{rf.config}" )
mean_224 = torch.tensor([1.0-0.82097102, 1.0-0.63302738, 1.0-0.75392824])
 std_224 = torch.tensor([0.37723779, 0.49839178, 0.4015415])
 first_resnext_pth_path_224 = ".. /input/lb91-224tile/LB91_224tile_best_resnext50_X20_30e_0.pth"
ensemble_1 ={"mean":mean_224,"std":std_224,"tileSize":224,"isExpandTile":False,"arch":"resnext50_32x4d","path":first_resnext_pth_path_224,"NN":64}
mean_288 = torch.tensor([1.0-0.82809473, 1.0-0.65345352, 1.0-0.76934528])
 std_288 = torch.tensor([0.40728608, 0.52557509, 0.42674383])
 second_resnext_pth_path_288 = ".. /input/cv8899-expand-288/cv8899_expand_288_tiles_resnext50_X20_30e_0.pth"
expand_sz = 288
ensemble_2 ={"mean":mean_288,"std":std_288,"tileSize":288,"isExpandTile":True,"arch":"resnext50_32x4d","path":second_resnext_pth_path_288,"NN":64}
ensemble_list = []
ensemble_list.append(ensemble_1)
 ensemble_list.append(ensemble_2)
 MODELS = []
for model_path_index in range(len(ensemble_list)) :
MODELS.append(ensemble_list[model_path_index]['path'])
 DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
TEST = '.. /input/prostate-cancer-grade-assessment/test.csv'
DATA2 = '.. /input/prostate-cancer-grade-assessment/train_images'
TEST2 = '.. /input/prostate-cancer-grade-assessment/train.csv'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
mixnet_pth=".. /input/segmmixnetv2/checkpoint_segm_mixnet_hardaug_14elr0005_acc88.59.pth"
bs = 2
sz2=224
NUM=20
NN=64
nworkers = 2
debug = False 
<set_options>
xgb = run_experiment("xgb", num_samples=100)
 print(f"result {xgb.last_result}")
 print(f"{xgb.config}" )
<choose_model_class><EOS> 
svm = run_experiment(method="svm", num_samples=8)
 print(f"result {svm.last_result}")
 print(f"{svm.config}" )