Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<concatenate>
import numpy as np
import pandas as pd
import os
from keras import models, layers, Sequential
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
test_tfms = A.Compose([
A.RandomRotate90(p=p),
A.Flip(p=p),
A.OneOf([
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, 
),
A.HueSaturationValue(
 hue_shift_limit=20,
sat_shift_limit=50,
val_shift_limit=50)
 ], p=p),
A.OneOf([
A.IAAAdditiveGaussianNoise() ,
A.GaussNoise() ,
], p=p),
ToTensor(normalize=imagenet_stats)
 ] )<load_from_csv>
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()
test_df = pd.read_csv(path/'test.csv')
 model, opt = get_model(model_name='efficientnet-b5',lr=1e-4,wd=1e-4)
 model.load_state_dict(torch.load(f'.. /input/melanomaefficientnetb5/effb5.pth',map_location=device))
test_ds = MelanomaDataset(df=test_df,im_path=path/'test',transforms=test_tfms,is_test=True)
 test_dl = DataLoader(dataset=test_ds,batch_size=bs*2,shuffle=False,num_workers=4 )<predict_on_test>
test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()
def get_preds(model,device=None,tta=3):
if device is None:
device = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 preds = np.zeros(len(test_ds))
for tta_id in range(tta):
test_preds = []
with torch.no_grad() :
for xb in test_dl:
xb = xb.to(device)
 out = model(xb)
 out = torch.sigmoid(out)
 test_preds.extend(out.cpu().numpy())
 preds += np.array(test_preds ).reshape(-1)
 print(f'TTA {tta_id}')
 preds /= tta
return preds
preds = get_preds(model,tta=25 )<save_to_csv>
merged_df = pd.concat(( train_df.drop(['Survived'], axis = 1), test_df))
merged_df.head()
subm = pd.read_csv(path/'sample_submission.csv')
 subm.target = preds
subm.to_csv('submission.csv',index=False )<import_modules>
import seaborn as sb
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
from glob import glob
import shutil<load_from_csv>
train_df.groupby(['Pclass'])['Survived'].sum() / train_df.groupby(['Pclass'])['Survived'].count()
raw_pred_2cls = pd.read_csv('.. /input/vinbigdata-2class-prediction/2-cls test pred.csv')
 other_pred_2cls = pd.read_csv('.. /input/temp-submission/2-cls test pred.csv' )<feature_engineering>
train_df.groupby(['Sex'])['Survived'].sum() / train_df.groupby(['Sex'])['Survived'].count()
tmp_pred_2cls=other_pred_2cls
tmp_pred_2cls['target']=1-other_pred_2cls['target']<merge>
train_df.groupby(['SibSp'])['Survived'].sum() / train_df.groupby(['SibSp'])['Survived'].count()
compare = pd.merge(raw_pred_2cls, other_pred_2cls, on = 'image_id', how = 'left')
 compare.head()<feature_engineering>
train_df.groupby(['Parch'])['Survived'].sum() / train_df.groupby(['Parch'])['Survived'].count()
compare['dif']=compare['target_x']-compare['target_y']<save_to_csv>
train_df['Family'] = train_df['SibSp'] + train_df['Parch']
compare.to_csv('compare.csv',index = False )<load_from_csv>
train_df.groupby(['Family'])['Survived'].sum() / train_df.groupby(['Family'])['Survived'].count()
pred_14cls = pd.read_csv('.. /input/vinbigdata-14-class-submission-lb0154/submission.csv')
 pred_2cls = pd.read_csv('.. /input/vinbigdata-2class-prediction/2-cls test pred.csv')
<save_to_csv>
train_df.drop(['Family'], axis = 1, inplace = True )
pred_2cls.to_csv('pred_2cls.csv',index = False )<merge>
ticket_num_records = train_df.groupby(['Ticket'] ).size().sort_values(ascending=False ).to_dict()
train_df.groupby(['Ticket'] ).size().sort_values(ascending=False ).head()
pred_raw = pd.merge(pred_14cls, pred_2cls, on = 'image_id', how = 'left')
 pred_raw.head()<merge>
train_df['Companion'] = train_df['Ticket'].apply(lambda x: ticket_num_records[x] )
pred = pd.merge(pred_14cls, tmp_pred_2cls, on = 'image_id', how = 'left')
 pred.head()<count_values>
train_df.groupby(['Companion'])['Survived'].sum() / train_df.groupby(['Companion'])['Survived'].count()
pred['PredictionString'].value_counts().iloc[[0]]<groupby>
train_df.drop(['Companion'], axis = 1, inplace = True )
count_low=0
count_mid=0
count_high=0
def filter_2cls_raw(row, low_thr=0.08, high_thr=0.95):
global count_low
global count_mid
global count_high
prob = row['target']
if prob<low_thr:
row['PredictionString'] = '14 1 0 0 1 1'
count_low+=1
elif low_thr<=prob<high_thr:
row['PredictionString']+=f' 14 {prob} 0 0 1 1'
count_mid+=1
elif high_thr<=prob:
row['PredictionString'] = row['PredictionString']
count_high+=1
else:
raise ValueError('Prediction must be from [0-1]')
 return row<prepare_output>
train_df.groupby(pd.cut(train_df["Fare"], np.arange(0, 350, 25)))['Survived'].sum() / train_df.groupby(pd.cut(train_df["Fare"], np.arange(0, 350, 25)))['Survived'].count()
sub_raw = pred_raw.apply(filter_2cls_raw, axis=1)
 print(count_low/3000,count_mid/3000,count_high/3000)
 sub_raw[60:63]<groupby>
train_df['CabinId'] = train_df['Cabin'].apply(lambda x: 'None' if pd.isna(x)else x[0])
 train_df.groupby(['CabinId'])['Survived'].sum() / train_df.groupby(['CabinId'])['Survived'].count()
count_low=0
count_mid=0
count_high=0
def filter_2cls(row, low_thr=0.05, high_thr=0.99):
global count_low
global count_mid
global count_high
prob = row['target']
if prob<low_thr:
row['PredictionString'] = '14 1 0 0 1 1'
count_low+=1
elif low_thr<=prob<high_thr:
row['PredictionString']+=f' 14 {prob} 0 0 1 1'
count_mid+=1
elif high_thr<=prob:
row['PredictionString']+=f' 14 {prob} 0 0 1 1'
count_high+=1
else:
raise ValueError('Prediction must be from [0-1]')
 return row<prepare_output>
train_df.drop(['CabinId'], axis = 1, inplace = True )
sub = pred.apply(filter_2cls, axis=1)
 print(count_low/3000,count_mid/3000,count_high/3000)
 sub[60:63]<merge>
train_df.groupby(['Embarked'])['Survived'].sum() / train_df.groupby(['Embarked'])['Survived'].count()
merge_sub = pd.merge(sub_raw, sub, on = 'image_id', how = 'left' )<save_to_csv>
train_df['Title'] = train_df['Name'].apply(lambda x: re.compile('.+?[,][\s] (.*?)[\.][\s].+' ).findall(x)[0])
 train_df.groupby(['Title'])['Survived'].sum() / train_df.groupby(['Title'])['Survived'].count()
merge_sub.to_csv('merge_sub.csv',index = False )<count_values>
train_df.groupby(['Title'])['Survived'].count()
sub['PredictionString'].value_counts().iloc[[0]]<save_to_csv>
train_df.drop(['Title'], axis = 1, inplace = True )
sub_raw[['image_id', 'PredictionString']].to_csv('submission_raw.csv',index = False )<save_to_csv>
np.nanmean(train_df[train_df['Name'].str.contains('Master')]['Age'] )
sub[['image_id', 'PredictionString']].to_csv('submission.csv',index = False )<import_modules>
np.nanmean(train_df[train_df['Name'].str.contains('Miss')]['Age'] )
import numpy as np, pandas as pd
from glob import glob
import shutil, os
import matplotlib.pyplot as plt
from sklearn.model_selection import GroupKFold
from tqdm.notebook import tqdm
import seaborn as sns<define_variables>
merged_df['Title'] = merged_df['Name'].apply(lambda x: re.compile('.+?[,][\s] (.*?)[\.][\s].+' ).findall(x)[0] )
dim = 512
test_dir = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test'
weights_dir = '/kaggle/input/chest-x-ray-abnormality-detection-with-yolo-v3/yolov3/runs/train/exp/weights/best.pt'<load_from_csv>
boymean = np.nanmean(train_df[train_df['Name'].str.contains('Master.')]['Age'])
 girlmean = np.nanmean(train_df[train_df['Name'].str.contains('Miss.')]['Age'])
 meanage = np.nanmean(train_df['Age'] )
test_df = pd.read_csv(f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test.csv')
 test_df.head()<set_options>
merged_df['Age'] = np.where(np.isnan(merged_df['Age'])&(merged_df['Title'] == 'Master'), boymean, merged_df['Age'])
 merged_df['Age'] = np.where(np.isnan(merged_df['Age'])&(merged_df['Title'] == 'Miss'), girlmean, merged_df['Age'])
 merged_df['Age'] = merged_df['Age'].fillna(meanage )
shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')
 os.chdir('/kaggle/working/yolov5')
 clear_output()
print('Setup complete.Using torch %s %s' %(torch.__version__, torch.cuda.get_device_properties(0)if torch.cuda.is_available() else 'CPU'))<install_modules>
merged_df['Fare'] = merged_df['Fare'].fillna(np.nanmedian(merged_df['Fare']))
merged_df['Fare'] = merged_df['Fare'].apply(lambda x: 1 if x > 75.0 else 0 )
!python detect.py --weights $weights_dir\
--img 512\
--conf 0.15\
--iou 0.4\
--source $test_dir\
--save-txt --save-conf --exist-ok<categorify>
merged_df['Cabin'] = merged_df['Cabin'].apply(lambda x: 0 if pd.isna(x)else 1 )
def yolo2voc(image_height, image_width, bboxes):

 bboxes = bboxes.copy().astype(float)
 bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width
bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height
bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2
bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]
return bboxes<define_variables>
merged_df['Embarked'] = merged_df['Embarked'].fillna('N' )
image_ids = []
PredictionStrings = []
for file_path in tqdm(glob('runs/detect/exp/labels/*txt')) :
image_id = file_path.split('/')[-1].split('.')[0]
w, h = test_df.loc[test_df.image_id==image_id,['width', 'height']].values[0]
f = open(file_path, 'r')
 data = np.array(f.read().replace('
', ' ' ).strip().split(' ')).astype(np.float32 ).reshape(-1, 6)
 data = data[:, [0, 5, 1, 2, 3, 4]]
bboxes = list(np.round(np.concatenate(( data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1 ).reshape(-1), 1 ).astype(str))
for idx in range(len(bboxes)) :
bboxes[idx] = str(int(float(bboxes[idx])))if idx%6!=1 else bboxes[idx]
image_ids.append(image_id)
 PredictionStrings.append(' '.join(bboxes))<save_to_csv>
merged_df['Family'] = merged_df['Parch'] + merged_df['SibSp']
pred_df = pd.DataFrame({'image_id':image_ids,
'PredictionString':PredictionStrings})
 sub_df = pd.merge(test_df, pred_df, on = 'image_id', how = 'left' ).fillna("14 1 0 0 1 1")
 sub_df = sub_df[['image_id', 'PredictionString']]
sub_df.to_csv('/kaggle/working/submission.csv',index = False)
 sub_df.tail()<install_modules>
merged_df.drop(['Name', 'Ticket', 'SibSp', 'Parch'], axis = 1, inplace = True )
shutil.rmtree('/kaggle/working/yolov5' )<set_options>
train_df.groupby(pd.cut(train_df["Age"], np.arange(0, 100, 20)))['Survived'].sum() / train_df.groupby(pd.cut(train_df["Age"], np.arange(0, 100, 20)))['Survived'].count()
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50)
<set_options>
maxAge = train_df['Age'].max()
minAge = train_df['Age'].min()
merged_df['Age'] =(merged_df['Age'] - minAge)/(maxAge - minAge )
!nvidia-smi<install_modules>
dummiesPclass = pd.get_dummies(merged_df['Pclass'], prefix = 'Pclass')
 merged_df = pd.concat([merged_df, dummiesPclass], axis=1)
 merged_df.head()
!nvcc --version<import_modules>
dummiesFare = pd.get_dummies(merged_df['Fare'], prefix = 'Fare')
 merged_df = pd.concat([merged_df, dummiesFare], axis=1)
 merged_df.head()
torch.__version__<install_modules>
merged_df.groupby(['Title'])['PassengerId'].count()
!pip install detectron2 -f \
https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html<define_variables>
merged_df['Title'] = merged_df['Title'].apply(lambda x: 'Miss' if(x in ['Mlle', 'Mme', 'Ms'])else x)
 merged_df['Title'] = merged_df['Title'].apply(lambda x: 'Mrs' if(x in ['Dona', 'Lady'])else x)
 merged_df['Title'] = merged_df['Title'].apply(lambda x: 'Mr' if(x == 'Rev')else x)
 merged_df['Title'] = merged_df['Title'].apply(lambda x: x if(x in ['Master', 'Mr', 'Mrs', 'Miss'])else 'Other' )
def get_vinbigdata_dicts(
 imgdir: Path,
train_df: pd.DataFrame,
train_data_type: str = "original",
use_cache: bool = True,
debug: bool = True,
target_indices: Optional[np.ndarray] = None,
use_class14: bool = False, 
):
debug_str = f"_debug{int(debug)}"
train_data_type_str = f"_{train_data_type}"
class14_str = f"_14class{int(use_class14)}"
cache_path = Path(".")/ f"dataset_dicts_cache{train_data_type_str}{class14_str}{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 train_meta = pd.read_csv(imgdir / "train_meta.csv")
 if debug:
train_meta = train_meta.iloc[:500]
image_id = train_meta.loc[0, "image_id"]
image_path = str(imgdir / "train" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, train_meta_row in tqdm(train_meta.iterrows() , total=len(train_meta)) :
record = {}
image_id, height, width = train_meta_row.values
filename = str(imgdir / "train" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
objs = []
for index2, row in train_df.query("image_id == @image_id" ).iterrows() :
class_id = row["class_id"]
if class_id == 14:
if use_class14:
bbox_resized = [0, 0, resized_width, resized_height]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 else:
pass
else:
h_ratio = resized_height / height
w_ratio = resized_width / width
bbox_resized = [
float(row["x_min"])* w_ratio,
float(row["y_min"])* h_ratio,
float(row["x_max"])* w_ratio,
float(row["y_max"])* h_ratio,
]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 record["annotations"] = objs
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 if target_indices is not None:
dataset_dicts = [dataset_dicts[i] for i in target_indices]
return dataset_dicts
def get_vinbigdata_dicts_test(
 imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True, 
):
debug_str = f"_debug{int(debug)}"
cache_path = Path(".")/ f"dataset_dicts_cache_test{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 if debug:
test_meta = test_meta.iloc[:500]
image_id = test_meta.loc[0, "image_id"]
image_path = str(imgdir / "test" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, test_meta_row in tqdm(test_meta.iterrows() , total=len(test_meta)) :
record = {}
image_id, height, width = test_meta_row.values
filename = str(imgdir / "test" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 return dataset_dicts<categorify>
merged_df.groupby(['Title'])['PassengerId'].count()
def format_pred(labels: ndarray, boxes: ndarray, scores: ndarray)-> str:
pred_strings = []
for label, score, bbox in zip(labels, scores, boxes):
xmin, ymin, xmax, ymax = bbox.astype(np.int64)
 pred_strings.append(f"{label} {score} {xmin} {ymin} {xmax} {ymax}")
 return " ".join(pred_strings)
 def predict_batch(predictor: DefaultPredictor, im_list: List[ndarray])-> List:
with torch.no_grad() :
inputs_list = []
for original_image in im_list:
if predictor.input_format == "RGB":
original_image = original_image[:, :, ::-1]
height, width = original_image.shape[:2]
image = original_image
image = torch.as_tensor(image.astype("float32" ).transpose(2, 0, 1))
inputs = {"image": image, "height": height, "width": width}
inputs_list.append(inputs)
 predictions = predictor.model(inputs_list)
 return predictions<load_pretrained>
le = LabelEncoder()
merged_df['Title'] = le.fit_transform(merged_df['Title'] )
def save_yaml(filepath: Union[str, Path], content: Any, width: int = 120):
with open(filepath, "w")as f:
yaml.dump(content, f, width=width)
 def load_yaml(filepath: Union[str, Path])-> Any:
with open(filepath, "r")as f:
content = yaml.full_load(f)
 return content 
<define_variables>
dummiesTitle = pd.get_dummies(merged_df['Title'], prefix = 'Title')
 merged_df = pd.concat([merged_df, dummiesTitle], axis=1)
 merged_df.head()
thing_classes = [
"Aortic enlargement",
"Atelectasis",
"Calcification",
"Cardiomegaly",
"Consolidation",
"ILD",
"Infiltration",
"Lung Opacity",
"Nodule/Mass",
"Other lesion",
"Pleural effusion",
"Pleural thickening",
"Pneumothorax",
"Pulmonary fibrosis"
]
category_name_to_id = {class_name: index for index, class_name in enumerate(thing_classes)} 
<define_variables>
merged_df['Sex'] = le.fit_transform(merged_df['Sex'])
 dummiesSex = pd.get_dummies(merged_df['Sex'], prefix = 'Sex')
 merged_df = pd.concat([merged_df, dummiesSex], axis=1)
 merged_df.head()
@dataclass
class Flags:
debug: bool = True
outdir: str = "results/det"
imgdir_name: str = "vinbigdata-chest-xray-resized-png-256x256"
split_mode: str = "all_train"
seed: int = 111
train_data_type: str = "original"
use_class14: bool = False
iter: int = 10000
ims_per_batch: int = 2
num_workers: int = 4
lr_scheduler_name: str = "WarmupMultiStepLR"
base_lr: float = 0.00025
roi_batch_size_per_image: int = 512
eval_period: int = 10000
aug_kwargs: Dict = field(default_factory=lambda: {})
 def update(self, param_dict: Dict)-> "Flags":
for key, value in param_dict.items() :
if not hasattr(self, key):
raise ValueError(f"[ERROR] Unexpected key for flag = {key}")
 setattr(self, key, value)
 return self<load_from_csv>
dummiesCabin = pd.get_dummies(merged_df['Cabin'], prefix = 'Cabin')
 merged_df = pd.concat([merged_df, dummiesCabin], axis=1)
 merged_df.head()
inputdir = Path("/kaggle/input")
 traineddir = inputdir / "vinbigdata-alb-aug-512-cos"
flags: Flags = Flags().update(load_yaml(str(traineddir/"flags.yaml")))
 print("flags", flags)
 debug = flags.debug
outdir = Path(flags.outdir)
 os.makedirs(str(outdir), exist_ok=True)
 datadir = inputdir / "vinbigdata-chest-xray-abnormalities-detection"
if flags.imgdir_name == "vinbigdata-chest-xray-resized-png-512x512":
imgdir = inputdir/ "vinbigdata"
else:
imgdir = inputdir / flags.imgdir_name
test_meta = pd.read_csv(inputdir / "vinbigdata-testmeta" / "test_meta.csv")
 sample_submission = pd.read_csv(datadir / "sample_submission.csv" )<load_pretrained>
merged_df['Embarked'] = le.fit_transform(merged_df['Embarked'])
 dummiesEmbarked = pd.get_dummies(merged_df['Embarked'], prefix = 'Embarked')
 merged_df = pd.concat([merged_df, dummiesEmbarked], axis=1)
 merged_df.head()
cfg = get_cfg()
original_output_dir = cfg.OUTPUT_DIR
cfg.OUTPUT_DIR = str(outdir)
 print(f"cfg.OUTPUT_DIR {original_output_dir} -> {cfg.OUTPUT_DIR}")
 cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN =("vinbigdata_train",)
 cfg.DATASETS.TEST =()
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml")
 cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = flags.base_lr
cfg.SOLVER.MAX_ITER = flags.iter
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = flags.roi_batch_size_per_image
cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(thing_classes)
 cfg.MODEL.WEIGHTS = str(traineddir/"model_final.pth")
 print("Original thresh", cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)
 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.0
print("Changed thresh", cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)
 predictor = DefaultPredictor(cfg)
 DatasetCatalog.register(
 "vinbigdata_test", lambda: get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
)
 MetadataCatalog.get("vinbigdata_test" ).set(thing_classes=thing_classes)
 metadata = MetadataCatalog.get("vinbigdata_test")
 dataset_dicts = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
 if debug:
dataset_dicts = dataset_dicts[:100]
results_list = []
index = 0
batch_size = 4
for i in tqdm(range(ceil(len(dataset_dicts)/ batch_size))):
inds = list(range(batch_size * i, min(batch_size *(i + 1), len(dataset_dicts))))
dataset_dicts_batch = [dataset_dicts[i] for i in inds]
im_list = [cv2.imread(d["file_name"])for d in dataset_dicts_batch]
outputs_list = predict_batch(predictor, im_list)
 for im, outputs, d in zip(im_list, outputs_list, dataset_dicts_batch):
resized_height, resized_width, ch = im.shape
if index < 5:
v = Visualizer(
 im[:, :, ::-1],
metadata=metadata,
scale=0.5,
instance_mode=ColorMode.IMAGE_BW 
)
 out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
cv2.imwrite(str(outdir / f"pred_{index}.jpg"), out.get_image() [:, :, ::-1])
 image_id, dim0, dim1 = test_meta.iloc[index].values
instances = outputs["instances"]
if len(instances)== 0:
result = {"image_id": image_id, "PredictionString": "14 1.0 0 0 1 1"}
else:
fields: Dict[str, Any] = instances.get_fields()
pred_classes = fields["pred_classes"]
pred_scores = fields["scores"]
pred_boxes = fields["pred_boxes"].tensor
h_ratio = dim0 / resized_height
w_ratio = dim1 / resized_width
pred_boxes[:, [0, 2]] *= w_ratio
pred_boxes[:, [1, 3]] *= h_ratio
pred_classes_array = pred_classes.cpu().numpy()
pred_boxes_array = pred_boxes.cpu().numpy()
pred_scores_array = pred_scores.cpu().numpy()
result = {
"image_id": image_id,
"PredictionString": format_pred(
 pred_classes_array, pred_boxes_array, pred_scores_array 
),
}
results_list.append(result)
 index += 1<save_to_csv>
merged_df['Family'] = merged_df['Family'].apply(lambda x: 'N' if x == 0 else('S' if x < 4 else 'L'))
submission_det = pd.DataFrame(results_list, columns=['image_id', 'PredictionString'])
 submission_det.to_csv(outdir/"submission.csv", index=False)
 submission_det<import_modules>
merged_df['Family'] = le.fit_transform(merged_df['Family'])
 dummiesFamily = pd.get_dummies(merged_df['Family'], prefix = 'Family')
 merged_df = pd.concat([merged_df, dummiesFamily], axis=1)
 merged_df.head()
import pandas as pd
import numpy as np
from glob import glob
import shutil<define_variables>
merged_df.drop(['Pclass', 'Sex', 'Fare', 'Cabin', 'Embarked', 'Title', 'Family'], axis = 1, inplace = True )
low_thr = 0.08
high_thr = 0.95<load_from_csv>
train_df_x = merged_df[:891]
test_df_x = merged_df[891:]
pred_14cls = pd.read_csv('.. /input/vinbigdata-14-class-submission-lb0154/submission.csv')
 pred_2cls = pd.read_csv('.. /input/vinbigdata-2class-prediction/2-cls test pred.csv' )<merge>
train_df_y = train_df['Survived']
pred = pd.merge(pred_14cls, pred_2cls, on = 'image_id', how = 'left')
 pred.head()<count_values>
train_df = train_df_x.copy()
train_df['Survived'] = train_df_y
train_df.drop(['PassengerId'], axis = 1, inplace = True)
 train_df.head()
pred['PredictionString'].value_counts().iloc[[0]]<feature_engineering>
def train_and_test(model_specific_tasks, df, it = 20):
accsum = 0
minacc = 1.0
maxacc = 0
for i in range(it):
print('Iteration: ',(i + 1), end = '\r')
 train, test = train_test_split(df, test_size=0.2)
 train_x = train.drop(['Survived'], axis=1)
 test_x = test.drop(['Survived'], axis=1)
 train_y = train['Survived']
test_y = test['Survived']
train_x = np.asarray(train_x ).astype('float32')
 train_y = np.asarray(train_y ).astype('float32')
 acc = model_specific_tasks(train_x, train_y, test_x, test_y)
 accsum += acc
minacc = acc if acc < minacc else minacc
maxacc = acc if acc > maxacc else maxacc
print('Avg.accuracy: ',(accsum / it))
print('Min.accuracy: ', minacc)
 print('Max.accuracy: ', maxacc )
def filter_2cls(row, low_thr=low_thr, high_thr=high_thr):
prob = row['target']
if prob<low_thr:
row['PredictionString'] = '14 1 0 0 1 1'
elif low_thr<=prob<high_thr:
row['PredictionString']+=f' 14 {prob} 0 0 1 1'
elif high_thr<=prob:
row['PredictionString'] = row['PredictionString']
else:
raise ValueError('Prediction must be from [0-1]')
 return row<prepare_output>
def logistic_reg(train_x, train_y, test_x, test_y):
model = LogisticRegression(solver='sag', max_iter=1000)
 model.fit(train_x, train_y)
 return model.score(test_x, test_y )
sub = pred.apply(filter_2cls, axis=1)
 sub.head()<count_values>
train_and_test(logistic_reg, train_df.copy() , it = 50 )
sub['PredictionString'].value_counts().iloc[[0]]<save_to_csv>
def rfc(train_x, train_y, test_x, test_y):
model = RandomForestClassifier(n_estimators=100)
 model.fit(train_x, train_y)
 return model.score(test_x, test_y )
sub[['image_id', 'PredictionString']].to_csv('submission.csv',index = False )<set_options>
train_and_test(rfc, train_df.copy() )
warnings.filterwarnings("ignore")
 DIR_INPUT = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection'
DIR_TRAIN = f'{DIR_INPUT}/train'
DIR_TEST = f'{DIR_INPUT}/test'
DIR_WEIGHTS = '.. /input/chest-x-ray-abnormalities-detection'
WEIGHTS_FILE = f'{DIR_WEIGHTS}/model_state.pth'<load_from_csv>
def nn(train_x, train_y, test_x, test_y):
model = Sequential()
model.add(layers.Dense(32, activation='relu', input_shape =(22,)))
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(8, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.fit(train_x, train_y, epochs=150, batch_size=16, verbose = 0)
 return model.evaluate(test_x, test_y, verbose = 0)[1]
<categorify><EOS> 
train_and_test(nn, train_df.copy() , it = 10 )