Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
Image(url= "https://i.ytimg.com/vi/1PhMWUoPDsk/maxresdefault.jpg" )
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.25
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
SEED = 42
EPO = 15
WEIGHTS = '.. /input/yolov5weight60/best.pt'
CONFIG = '.. /input/modelyolov5/yolov5x.yaml'
DATA = '.. /input/configyolo5/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True<set_options>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
set_seed(SEED )<load_from_csv>
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 survived = train_df['Survived']
passenger_id = test_df['PassengerId']
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
submission = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 submission.head()
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<concatenate>
print(train_df.isnull().sum())
 print(test_df.isnull().sum() )
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<normalization>
train_test_data = [train_df, test_df]
print(train_test_data)
 for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<train_model>
train_df['Title'].value_counts()
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
test_df['Title'].value_counts()
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
train_df.drop('Name', axis=1, inplace=True)
 test_df.drop('Name', axis=1, inplace=True )
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
train_df["Age"].fillna(train_df.groupby("Title")["Age"].transform("median"), inplace=True)
 test_df["Age"].fillna(test_df.groupby("Title")["Age"].transform("median"), inplace=True )
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<load_from_csv>
train_df.groupby("Title")["Age"].transform("median")
def load_dataset(root):
csv = pd.read_csv(os.path.join(root, "train.csv"))
data = {}
for i in csv.index:
key = csv["image_id"][i]
bbox = json.loads(csv["bbox"][i])
 bbox = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3], 0.0]
if key in data:
data[key].append(bbox)
 else:
data[key] = [bbox]
return sorted(
 [(k, os.path.join(root, "train", k + ".jpg"), v)for k, v in data.items() ],
key=lambda x: x[0] 
)
 def load_image(path):
with open(path, "rb")as f:
buf = f.read()
return mx.image.imdecode(buf)
 def get_batches(dataset, batch_size, width=512, height=512, net=None, ctx=mx.cpu()):
batches = len(dataset)// batch_size
sampler = Sampler(dataset, width, height, net)
 stack_fn = [gcv.data.batchify.Stack() ]
pad_fn = [gcv.data.batchify.Pad(pad_val=-1)]
if net is None:
batchify_fn = gcv.data.batchify.Tuple(*(stack_fn + pad_fn))
else:
batchify_fn = gcv.data.batchify.Tuple(*(stack_fn * 6 + pad_fn))
with Pool(cpu_count() * 2)as p:
for i in range(batches):
start = i * batch_size
samples = p.map(sampler, range(start, start + batch_size))
batch = batchify_fn(samples)
 yield [x.as_in_context(ctx)for x in batch]
def gauss_blur(image, level):
return cv2.blur(image,(level * 2 + 1, level * 2 + 1))
def gauss_noise(image):
for i in range(image.shape[2]):
c = image[:, :, i]
diff = 255 - c.max() ;
noise = np.random.normal(0, random.randint(1, 6), c.shape)
 noise =(noise - noise.min())/(noise.max() - noise.min())
 noise = diff * noise
image[:, :, i] = c + noise.astype(np.uint8)
 return image
class YOLO3TrainTransform:
def __init__(self, width, height, net, mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225), **kwargs):
self._width = width
self._height = height
self._mean = mean
self._std = std
self._fake_x = mx.nd.zeros(( 1, 3, height, width))
net = copy.deepcopy(net)
 net.collect_params().reset_ctx(None)
 with mx.autograd.train_mode() :
_, self._anchors, self._offsets, self._feat_maps, _, _, _, _ = net(self._fake_x)
 self._target_generator = gcv.model_zoo.yolo.yolo_target.YOLOV3PrefetchTargetGenerator(
 num_class=len(net.classes), **kwargs)
 def __call__(self, img, label):
if np.random.uniform(0, 1)> 0.5:
img, expand = gcv.data.transforms.image.random_expand(img, max_ratio=1.5, fill=114, keep_ratio=False)
 bbox = gcv.data.transforms.bbox.translate(label, x_offset=expand[0], y_offset=expand[1])
 else:
img, bbox = img, label
h, w, _ = img.shape
bbox, crop = gcv.data.transforms.experimental.bbox.random_crop_with_constraints(bbox,(w, h))
x0, y0, w, h = crop
img = mx.image.fixed_crop(img, x0, y0, w, h)
 h, w, _ = img.shape
interp = np.random.randint(0, 5)
 img = gcv.data.transforms.image.imresize(img, self._width, self._height, interp=interp)
 bbox = gcv.data.transforms.bbox.resize(bbox,(w, h),(self._width, self._height))
h, w, _ = img.shape
img, flips = gcv.data.transforms.image.random_flip(img, px=0.5, py=0.5)
 bbox = gcv.data.transforms.bbox.flip(bbox,(w, h), flip_x=flips[0], flip_y=flips[1])
 img = gcv.data.transforms.experimental.image.random_color_distort(img)
 img = mx.nd.image.to_tensor(img)
 img = mx.nd.image.normalize(img, mean=self._mean, std=self._std)
 gt_bboxes = mx.nd.array(bbox[np.newaxis, :, :4])
 gt_ids = mx.nd.array(bbox[np.newaxis, :, 4:5])
 gt_mixratio = mx.nd.array(bbox[np.newaxis, :, -1:])
 objectness, center_targets, scale_targets, weights, class_targets = self._target_generator(
 self._fake_x, self._feat_maps, self._anchors, self._offsets,
gt_bboxes, gt_ids, gt_mixratio)
 return(img, objectness[0], center_targets[0], scale_targets[0], weights[0],
class_targets[0], gt_bboxes[0])
 class Sampler:
def __init__(self, dataset, width, height, net=None, **kwargs):
self._dataset = dataset
if net is None:
self._training_mode = False
self._transform = gcv.data.transforms.presets.yolo.YOLO3DefaultValTransform(width, height, **kwargs)
 else:
self._training_mode = True
self._transform = YOLO3TrainTransform(width, height, net, **kwargs)
 def __call__(self, idx):
if self._training_mode:
raw, bboxes = self._load_mixup(idx)
 raw = raw.asnumpy()
blur = random.randint(0, 3)
 if blur > 0:
raw = gauss_blur(raw, blur)
 raw = gauss_noise(raw)
 h, w, _ = raw.shape
rot = random.randint(0, 3)
 if rot > 0:
raw = np.rot90(raw, k=rot)
 if rot == 1:
raw_bboxes = bboxes.copy()
bboxes[:, [0, 2]] = raw_bboxes[:, [1, 3]]
bboxes[:, [1, 3]] = w - raw_bboxes[:, [2, 0]]
elif rot == 2:
bboxes[:, [0, 1, 2, 3]] = np.array([[w, h, w, h]])- bboxes[:, [2, 3, 0, 1]]
elif rot == 3:
raw_bboxes = bboxes.copy()
bboxes[:, [0, 2]] = h - raw_bboxes[:, [1, 3]]
bboxes[:, [1, 3]] = raw_bboxes[:, [2, 0]]
raw_bboxes = bboxes.copy()
bboxes[:, 0] = np.min(raw_bboxes[:, [0, 2]], axis=1)
 bboxes[:, 1] = np.min(raw_bboxes[:, [1, 3]], axis=1)
 bboxes[:, 2] = np.max(raw_bboxes[:, [0, 2]], axis=1)
 bboxes[:, 3] = np.max(raw_bboxes[:, [1, 3]], axis=1)
 raw = mx.nd.array(raw)
 else:
raw = load_image(self._dataset[idx][1])
 bboxes = np.array(self._dataset[idx][2])
 res = self._transform(raw, bboxes)
 return [mx.nd.array(x)for x in res]
def _load_mixup(self, idx1):
r = random.gauss(0.5, 0.5 / 1.96)
 if r > 0.0:
raw1 = load_image(self._dataset[idx1][1])
 bboxes1 = np.array(self._dataset[idx1][2])
 if r >= 1.0:
return raw1, np.hstack([bboxes1, np.full(( bboxes1.shape[0], 1), 1.0)])
 idx2 = random.randint(0, len(self._dataset)- 1)
 raw2 = load_image(self._dataset[idx2][1])
 bboxes2 = np.array(self._dataset[idx2][2])
 if r <= 0.0:
return raw2, np.hstack([bboxes2, np.full(( bboxes2.shape[0], 1), 1.0)])
 h = max(raw1.shape[0], raw2.shape[0])
 w = max(raw1.shape[1], raw2.shape[1])
 mix_raw = mx.nd.zeros(shape=(h, w, 3), dtype="float32")
 mix_raw[:raw1.shape[0], :raw1.shape[1], :] += raw1.astype("float32")* r
mix_raw[:raw2.shape[0], :raw2.shape[1], :] += raw2.astype("float32")*(1.0 - r)
 mix_bboxes = np.vstack([
np.hstack([bboxes1, np.full(( bboxes1.shape[0], 1), r)]),
np.hstack([bboxes2, np.full(( bboxes2.shape[0], 1), 1.0 - r)])
 ])
 return mix_raw.astype("uint8"), mix_bboxes 
<load_pretrained>
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4
def load_model(path, ctx=mx.cpu()):
net = gcv.model_zoo.yolo3_darknet53_custom(["wheat"], pretrained_base=False)
 net.set_nms(post_nms=150)
 net.load_parameters(path, ctx=ctx)
 return net 
<normalization>
Pclass1 = train_df[train_df['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train_df[train_df['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train_df[train_df['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))
def inference(models, path):
raw = load_image(path)
 rh, rw, _ = raw.shape
classes_list = []
scores_list = []
bboxes_list = []
for _ in range(5):
img, flips = gcv.data.transforms.image.random_flip(raw, px=0.5, py=0.5)
 x, _ = gcv.data.transforms.presets.yolo.transform_test(img, short=img_s)
 _, _, xh, xw = x.shape
rot = random.randint(0, 3)
 if rot > 0:
x = np.rot90(x.asnumpy() , k=rot, axes=(2, 3))
for model in models:
classes, scores, bboxes = model(mx.nd.array(x, ctx=context))
if rot > 0:
if rot == 1:
raw_bboxes = bboxes.copy()
bboxes[0, :, [0, 2]] = xh - raw_bboxes[0, :, [1, 3]]
bboxes[0, :, [1, 3]] = raw_bboxes[0, :, [2, 0]]
elif rot == 2:
bboxes[0, :, [0, 1, 2, 3]] = mx.nd.array([[xw], [xh], [xw], [xh]], ctx=context)- bboxes[0, :, [2, 3, 0, 1]]
elif rot == 3:
raw_bboxes = bboxes.copy()
bboxes[0, :, [0, 2]] = raw_bboxes[0, :, [1, 3]]
bboxes[0, :, [1, 3]] = xw - raw_bboxes[0, :, [2, 0]]
raw_bboxes = bboxes.copy()
bboxes[0, :, 0] = raw_bboxes[0, :, [0, 2]].min(axis=0)
 bboxes[0, :, 1] = raw_bboxes[0, :, [1, 3]].min(axis=0)
 bboxes[0, :, 2] = raw_bboxes[0, :, [0, 2]].max(axis=0)
 bboxes[0, :, 3] = raw_bboxes[0, :, [1, 3]].max(axis=0)
 bboxes[0, :, :] = gcv.data.transforms.bbox.flip(bboxes[0, :, :],(xw, xh), flip_x=flips[0], flip_y=flips[1])
 bboxes[0, :, 0::2] =(bboxes[0, :, 0::2] /(xw - 1)).clip(0.0, 1.0)
 bboxes[0, :, 1::2] =(bboxes[0, :, 1::2] /(xh - 1)).clip(0.0, 1.0)
 classes_list.append([
int(classes[0, i].asscalar())for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 scores_list.append([
scores[0, i].asscalar() for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 bboxes_list.append([
bboxes[0, i].asnumpy().tolist() for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 bboxes, scores, classes = weighted_boxes_fusion(bboxes_list, scores_list, classes_list)
 bboxes[:, 0::2] *= rw - 1
bboxes[:, 1::2] *= rh - 1
return bboxes, scores, classes 
<load_pretrained>
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )
amp.init()
rounds = 3
max_epochs = 5
learning_rate = 0.001
batch_size = 8
img_s = 512
threshold = 0.1
context = mx.gpu()
print("Loading pre-trained model...")
 model = load_model("/kaggle/input/global-wheat-detection-private/global-wheat-yolo3-darknet53.params", ctx=context)
 print("Loading training set...")
 dataset = load_dataset("/kaggle/input/global-wheat-detection")
 print("Loading test images...")
 test_images = [ 
(os.path.join(dirname, filename), os.path.splitext(filename)[0])
 for dirname, _, filenames in os.walk('/kaggle/input/global-wheat-detection/test')for filename in filenames
]
best_score = 0.0
metrics = [gcv.utils.metrics.VOCMApMetric(iou_thresh=iou)for iou in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]]
for pseudo_models in [[model]] * rounds:
print("Pseudo labaling...")
 pseudo_set = []
for path, image_id in test_images:
print(path)
 bboxes, scores, classes = inference(pseudo_models, path)
 label = [
[round(x)for x in bboxes[i].tolist() ] + [0.0] for i in range(classes.shape[0])
 if model.classes[int(classes[i])] == "wheat" and scores[i] > threshold
]
if len(label)> 0:
pseudo_set.append(( image_id, path, label))
training_set = pseudo_set
validation_set = dataset[int(len(dataset)* 0.9):]
print("Training set: ", len(training_set))
print("Validation set: ", len(validation_set))
print("Re-training...")
 trainer = mx.gluon.Trainer(model.collect_params() , "Nadam", {
"learning_rate": learning_rate
}, kvstore='local', update_on_kvstore=False)
 amp.init_trainer(trainer)
 for epoch in range(max_epochs):
ts = time.time()
random.shuffle(training_set)
 training_total_L = 0.0
training_batches = 0
for x, objectness, center_targets, scale_targets, weights, class_targets, gt_bboxes in get_batches(training_set, batch_size, width=img_s, height=img_s, net=model, ctx=context):
training_batches += 1
with mx.autograd.record() :
obj_loss, center_loss, scale_loss, cls_loss = model(x, gt_bboxes, objectness, center_targets, scale_targets, weights, class_targets)
 L = obj_loss + center_loss + scale_loss + cls_loss
with amp.scale_loss(L, trainer)as scaled_L:
scaled_L.backward()
trainer.step(x.shape[0])
 training_batch_L = mx.nd.mean(L ).asscalar()
if training_batch_L != training_batch_L:
raise ValueError()
training_total_L += training_batch_L
print("[Epoch %d Batch %d] batch_loss %.10f average_loss %.10f elapsed %.2fs" %(
 epoch, training_batches, training_batch_L, training_total_L / training_batches, time.time() - ts 
))
training_avg_L = training_total_L / training_batches
for metric in metrics:
metric.reset()
for x, label in get_batches(validation_set, batch_size, width=img_s, height=img_s, ctx=context):
classes, scores, bboxes = model(x)
 for metric in metrics:
metric.update(
 bboxes,
classes.reshape(( 0, -1)) ,
scores.reshape(( 0, -1)) ,
label[:, :, :4],
label[:, :, 4:5].reshape(( 0, -1)) 
)
 score = mx.nd.array([metric.get() [1] for metric in metrics], ctx=context ).mean()
print("[Epoch %d] training_loss %.10f validation_score %.10f best_score %.10f duration %.2fs" %(
 epoch + 1, training_avg_L, score.asscalar() , best_score, time.time() - ts 
))
if score.asscalar() > best_score:
best_score = score.asscalar()
model.save_parameters("global-wheat-yolo3-darknet53.params")
 print("Loading re-trained model...")
 model = load_model("global-wheat-yolo3-darknet53.params", ctx=context)
 print("Inference...")
 results = []
for path, image_id in test_images:
print(path)
 bboxes, scores, classes = inference([model], path)
 bboxes[:, 2:4] -= bboxes[:, 0:2]
results.append({
"image_id": image_id,
"PredictionString": " ".join([
" ".join([str(x)for x in [scores[i]] + [round(x)for x in bboxes[i].tolist() ]])
 for i in range(classes.shape[0])
 if model.classes[int(classes[i])] == "wheat" and scores[i] > threshold
])
 })
 pd.DataFrame(results, columns=['image_id', 'PredictionString'] ).to_csv('submission.csv', index=False)
<install_modules>
embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )
!rm global-wheat-yolo3-darknet53.params<load_pretrained>
train_df["Fare"].fillna(train_df.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test_df["Fare"].fillna(test_df.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 train_df.head(5 )
def load_image(path):
with open(path, "rb")as f:
buf = f.read()
return mx.image.imdecode(buf)
<load_pretrained>
train_df.Cabin.value_counts()
def load_model(path, ctx=mx.cpu()):
net = gcv.model_zoo.yolo3_darknet53_custom(["wheat"], pretrained_base=False)
 net.set_nms(post_nms=150)
 net.load_parameters(path, ctx=ctx)
 return net 
<load_pretrained>
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].str[:1]
threshold = 0.1
img_s = 512
context = mx.gpu()
print("Loading test images...")
 images = [ 
(os.path.join(dirname, filename), os.path.splitext(filename)[0])
 for dirname, _, filenames in os.walk('/kaggle/input/global-wheat-detection/test')for filename in filenames
]
print("Loading model...")
 model = load_model("/kaggle/input/global-wheat-detection-private/global-wheat-yolo3-darknet53.params", ctx=context)
 print("Inference...")
 results = []
for path, image_id in images:
print(path)
 raw = load_image(path)
 rh, rw, _ = raw.shape
classes_list = []
scores_list = []
bboxes_list = []
for _ in range(5):
img, flips = gcv.data.transforms.image.random_flip(raw, px=0.5, py=0.5)
 x, _ = gcv.data.transforms.presets.yolo.transform_test(img, short=img_s)
 _, _, xh, xw = x.shape
rot = random.randint(0, 3)
 if rot > 0:
x = np.rot90(x.asnumpy() , k=rot, axes=(2, 3))
classes, scores, bboxes = model(mx.nd.array(x, ctx=context))
if rot > 0:
if rot == 1:
raw_bboxes = bboxes.copy()
bboxes[0, :, [0, 2]] = xh - raw_bboxes[0, :, [1, 3]]
bboxes[0, :, [1, 3]] = raw_bboxes[0, :, [2, 0]]
elif rot == 2:
bboxes[0, :, [0, 1, 2, 3]] = mx.nd.array([[xw], [xh], [xw], [xh]], ctx=context)- bboxes[0, :, [2, 3, 0, 1]]
elif rot == 3:
raw_bboxes = bboxes.copy()
bboxes[0, :, [0, 2]] = raw_bboxes[0, :, [1, 3]]
bboxes[0, :, [1, 3]] = xw - raw_bboxes[0, :, [2, 0]]
raw_bboxes = bboxes.copy()
bboxes[0, :, 0] = raw_bboxes[0, :, [0, 2]].min(axis=0)
 bboxes[0, :, 1] = raw_bboxes[0, :, [1, 3]].min(axis=0)
 bboxes[0, :, 2] = raw_bboxes[0, :, [0, 2]].max(axis=0)
 bboxes[0, :, 3] = raw_bboxes[0, :, [1, 3]].max(axis=0)
 bboxes[0, :, :] = gcv.data.transforms.bbox.flip(bboxes[0, :, :],(xw, xh), flip_x=flips[0], flip_y=flips[1])
 bboxes[0, :, 0::2] =(bboxes[0, :, 0::2] /(xw - 1)).clip(0.0, 1.0)
 bboxes[0, :, 1::2] =(bboxes[0, :, 1::2] /(xh - 1)).clip(0.0, 1.0)
 classes_list.append([
int(classes[0, i].asscalar())for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 scores_list.append([
scores[0, i].asscalar() for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 bboxes_list.append([
bboxes[0, i].asnumpy().tolist() for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 bboxes, scores, classes = weighted_boxes_fusion(bboxes_list, scores_list, classes_list)
 bboxes[:, 0::2] *= rw - 1
bboxes[:, 1::2] *= rh - 1
bboxes[:, 2:4] -= bboxes[:, 0:2]
results.append({
"image_id": image_id,
"PredictionString": " ".join([
" ".join([str(x)for x in [scores[i]] + [round(x)for x in bboxes[i].tolist() ]])
 for i in range(classes.shape[0])
 if model.classes[int(classes[i])] == "wheat" and scores[i] > threshold
])
 })
 pd.DataFrame(results, columns=['image_id', 'PredictionString'] ).to_csv('submission.csv', index=False)
<set_options>
cabin_mapping = {"A": 0, "B": 0.4, "C": 0.8, "D": 1.2, "E": 1.6, "F": 2, "G": 2.4, "T": 2.8}
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].map(cabin_mapping )
sns.set_style('darkgrid')
 warnings.filterwarnings('ignore')
<load_from_csv>
train_df["Cabin"].fillna(train_df.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test_df["Cabin"].fillna(test_df.groupby("Pclass")["Cabin"].transform("median"), inplace=True )
df=pd.read_csv('.. /input/kobe-bryant-shot-selection/data.csv', header=0,sep=',' )<count_values>
train_df["FamilySize"] = train_df["SibSp"] + train_df["Parch"] + 1
test_df["FamilySize"] = test_df["SibSp"] + test_df["Parch"] + 1
df.shot_made_flag.value_counts()<count_missing_values>
train_df.loc[train_df['FamilySize'] <= 0.0,'IsAlone'] = 0
train_df.loc[train_df['FamilySize'] > 0.0,'IsAlone'] = 1
test_df.loc[test_df['FamilySize'] <= 0.0,'IsAlone'] = 0
test_df.loc[test_df['FamilySize'] > 0.0,'IsAlone'] = 1
df.shot_made_flag.isnull().sum()<drop_column>
family_mapping = {1: 0, 2: 0.4, 3: 0.8, 4: 1.2, 5: 1.6, 6: 2, 7: 2.4, 8: 2.8, 9: 3.2, 10: 3.6, 11: 4}
for dataset in train_test_data:
dataset['FamilySize'] = dataset['FamilySize'].map(family_mapping )
if 'lat' in df.columns:
df.drop(labels='lat',axis=1,inplace=True)
 if 'lon' in df.columns:
df.drop(labels='lon',axis=1,inplace=True )<count_values>
features_drop = ['Ticket', 'Parch', ]
train_df = train_df.drop(features_drop, axis=1)
 test_df = test_df.drop(features_drop, axis=1)
 train_df = train_df.drop(['PassengerId'], axis=1 )
df.team_name.value_counts()<drop_column>
train_data = train_df.drop('Survived', axis=1)
 target = train_df['Survived']
train_data.shape, target.shape
if 'team_name' in df.columns:
df.drop(labels='team_name', inplace=True, axis=1 )<drop_column>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
if 'shot_id' in df.columns:
df.drop(labels='shot_id', inplace=True, axis=1 )<count_values>
clf = KNeighborsClassifier(n_neighbors = 13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
df.game_id.value_counts().head(10 )<drop_column>
round(np.mean(score)*100, 2 )
if 'game_id' in df.columns:
df.drop(labels='game_id', inplace=True, axis=1)
 if 'game_event_id' in df.columns:
df.drop(labels='game_event_id', inplace=True, axis=1 )<count_values>
clf = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
df.matchup.str.startswith('LAL' ).value_counts()<data_type_conversions>
round(np.mean(score)*100, 2 )
df['home_or_away']=df.matchup.apply(lambda x: 'home' if x.find('@')==-1 else 'away')
 df['home_or_away']=df['home_or_away'].astype('category' )<drop_column>
rand_clf = RandomForestClassifier(n_estimators=13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
if 'matchup' in df.columns:
df.drop(labels='matchup', axis=1,inplace=True )<data_type_conversions>
round(np.mean(score)*100, 2 )
df['opponent']=df['opponent'].astype('category' )<count_values>
clf = GaussianNB()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
df.team_id.value_counts()<drop_column>
round(np.mean(score)*100, 2 )
if 'team_id' in df.columns:
df.drop(labels='team_id', axis=1,inplace=True )<count_values>
svm = SVC()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
df.shot_type.value_counts()<data_type_conversions>
round(np.mean(score)*100,2 )
df['shot_type']=df.shot_type.astype('category' )<data_type_conversions>
svm = SVC()
clf.fit(train_data, target)
 test_data = test_df.drop("PassengerId", axis=1 ).copy()
prediction = clf.predict(test_data )
df['game_date']=pd.to_datetime(df['game_date'] )<data_type_conversions>
model = XGBClassifier()
model.fit(train_data, target )
df['season']=df.season.astype('category' )<feature_engineering>
y_pred = clf.predict(test_data)
 predictions = [round(value)for value in y_pred]
df['weekofyear']=df.game_date.apply(lambda x:x.weekofyear)
 df['dayofweek']=df.game_date.apply(lambda x:x.dayofweek)
 df['year']=df.game_date.apply(lambda x:x.year)
 df['month']=df.game_date.apply(lambda x:x.month)
 df['weekofyear']=df['weekofyear'].astype('category')
 df['dayofweek']=df['dayofweek'].astype('category')
 df['year']=df['year'].astype('category')
 df['month']=df['month'].astype('category')
<drop_column>
run_gs = False
if run_gs:
parameter_grid = {
'max_depth' : [4, 6, 8],
'n_estimators': [50, 10],
'max_features': ['sqrt', 'auto', 'log2'],
'min_samples_split': [2, 3, 10],
'min_samples_leaf': [1, 3, 10],
'bootstrap': [True, False],
}
forest = RandomForestClassifier()
cross_validation = StratifiedKFold(n_splits=5)
 grid_search = GridSearchCV(forest,
scoring='accuracy',
param_grid=parameter_grid,
cv=cross_validation,
verbose=1 
)
 grid_search.fit(train_data, target)
 model = grid_search
parameters = grid_search.best_params_
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
else:
parameters = {'bootstrap': False, 'min_samples_leaf': 3, 'n_estimators': 50,
'min_samples_split': 10, 'max_features': 'sqrt', 'max_depth': 6}
model = RandomForestClassifier(**parameters)
 model.fit(train_data, target)
if 'game_date' in df.columns:
df.drop(labels='game_date',axis=1,inplace=True)
<count_values>
output = model.predict(test_data ).astype(int )
df.shot_zone_range.value_counts()<data_type_conversions>
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": output
})
 submission.to_csv('submission.csv', index=False )
df['shot_zone_range']=df['shot_zone_range'].astype('category' )<count_values>
Image(url= "https://i.ytimg.com/vi/1PhMWUoPDsk/maxresdefault.jpg" )
df.shot_zone_basic.value_counts()<data_type_conversions>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
df['shot_zone_basic']=df['shot_zone_basic'].astype('category')
<count_values>
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 survived = train_df['Survived']
passenger_id = test_df['PassengerId']
df.shot_zone_area.value_counts()<data_type_conversions>
submission = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 submission.head()
df['shot_zone_area']=df['shot_zone_area'].astype('category' )<count_values>
print(train_df.isnull().sum())
 print(test_df.isnull().sum() )
df.action_type.value_counts()<data_type_conversions>
train_test_data = [train_df, test_df]
print(train_test_data)
 for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )
df['action_type']=df.action_type.astype('category' )<count_values>
train_df['Title'].value_counts()
df.combined_shot_type.value_counts()<data_type_conversions>
test_df['Title'].value_counts()
df['combined_shot_type']=df.combined_shot_type.astype('category' )<count_values>
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )
df.minutes_remaining.value_counts()<feature_engineering>
train_df.drop('Name', axis=1, inplace=True)
 test_df.drop('Name', axis=1, inplace=True )
df['total_seconds_remaining']=df[['minutes_remaining','seconds_remaining']].apply(lambda x:x[0]*60+x[1], axis=1 ).values
bins_=[0]+list(np.linspace(6,715,71))
df['time_intervals']=pd.cut(df.total_seconds_remaining,bins=bins_,labels=list(range(1,72)) ).values
plt.figure(figsize=(16,6))
df.groupby('time_intervals')['shot_made_flag'].mean().plot(kind='line')
 plt.show()<data_type_conversions>
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )
df['in_last_five_seconds']=[1 if val==1 else 0 for val in df.time_intervals.values]
df['in_last_five_seconds']=df['in_last_five_seconds'].astype('category')
 if 'minutes_remaining' in df.columns:
df.drop(labels='minutes_remaining',axis=1,inplace=True)
 if 'seconds_remaining' in df.columns:
df.drop(labels='seconds_remaining',axis=1,inplace=True)
 if 'time_intervals' in df.columns:
df.drop(labels='time_intervals',axis=1,inplace=True)
<count_values>
train_df["Age"].fillna(train_df.groupby("Title")["Age"].transform("median"), inplace=True)
 test_df["Age"].fillna(test_df.groupby("Title")["Age"].transform("median"), inplace=True )
df.period.value_counts()<count_values>
train_df.groupby("Title")["Age"].transform("median")
df.playoffs.value_counts()<data_type_conversions>
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4
df['period']=df['period'].astype('category')
 df['playoffs']=df['playoffs'].astype('category')
<count_unique_values>
Pclass1 = train_df[train_df['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train_df[train_df['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train_df[train_df['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))
df.shot_distance.nunique()<count_values>
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )
df.shot_distance.value_counts().head(15 )<data_type_conversions>
embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )
df['shot_made_flag']=df['shot_made_flag'].astype('category')
<prepare_x_and_y>
train_df["Fare"].fillna(train_df.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test_df["Fare"].fillna(test_df.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 train_df.head(5 )
df_with_dummies=pd.get_dummies(df.drop(labels='shot_made_flag',axis=1),drop_first=True)
 xtrain=df_with_dummies[df.shot_made_flag.notnull() ]
ytrain=df.shot_made_flag[df.shot_made_flag.notnull() ].values
test=df_with_dummies[df.shot_made_flag.isnull() ]<import_modules>
train_df.Cabin.value_counts()
from sklearn.ensemble import RandomForestClassifier 
<init_hyperparams>
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].str[:1]
def optimization_of_parameter_of_rf(X,y,dict_of_param,name_of_parameter, list_of_values, min_estimators, max_estimators):
list_of_parameter_dicts=[(value,{**dict_of_param,**{'n_estimators':100, 'warm_start':True, 'oob_score':True, 'n_jobs':-1,
'random_state':434,name_of_parameter:value}})for value in list_of_values]
ensemble_clfs = [("{x}={y}".format(x=name_of_parameter, y=value), RandomForestClassifier(**param_dict))
for value, param_dict in list_of_parameter_dicts]
error_rate = OrderedDict(( label, [])for label, _ in ensemble_clfs)
 for label, clf in ensemble_clfs:
for i in range(min_estimators, max_estimators + 1):
clf.set_params(n_estimators=i)
 clf.fit(X, y)
 oob_error = 1 - clf.oob_score_
error_rate[label].append(( i, oob_error))
plt.figure(figsize=(15,8))
for label, clf_err in error_rate.items() :
xs, ys = zip(*clf_err)
 plt.plot(xs, ys, label=label)
 plt.xlim(min_estimators, max_estimators)
 plt.xlabel("n_estimators")
 plt.ylabel("OOB error rate")
 plt.legend(loc="upper right")
 plt.show()<train_on_grid>
cabin_mapping = {"A": 0, "B": 0.4, "C": 0.8, "D": 1.2, "E": 1.6, "F": 2, "G": 2.4, "T": 2.8}
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].map(cabin_mapping )
optimization_of_parameter_of_rf(xtrain.values, ytrain,{'min_samples_leaf':9},'max_features',[50,70,90],30,250 )<train_on_grid>
train_df["Cabin"].fillna(train_df.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test_df["Cabin"].fillna(test_df.groupby("Pclass")["Cabin"].transform("median"), inplace=True )
optimization_of_parameter_of_rf(xtrain.values, ytrain,{'max_features':50},'min_samples_leaf',[5,9,11,13],30,250 )<find_best_params>
train_df["FamilySize"] = train_df["SibSp"] + train_df["Parch"] + 1
test_df["FamilySize"] = test_df["SibSp"] + test_df["Parch"] + 1
optimization_of_parameter_of_rf(xtrain.values, ytrain,{'max_features':50, 'min_samples_leaf':13},'max_depth',[15,20,25],30,250 )<train_model>
train_df.loc[train_df['FamilySize'] <= 0.0,'IsAlone'] = 0
train_df.loc[train_df['FamilySize'] > 0.0,'IsAlone'] = 1
test_df.loc[test_df['FamilySize'] <= 0.0,'IsAlone'] = 0
test_df.loc[test_df['FamilySize'] > 0.0,'IsAlone'] = 1
rfc=RandomForestClassifier(n_estimators=400,max_features=50,min_samples_leaf=13, max_depth=20)
 rfc.fit(xtrain.values, ytrain)
 preds=rfc.predict_proba(test)
<save_to_csv>
family_mapping = {1: 0, 2: 0.4, 3: 0.8, 4: 1.2, 5: 1.6, 6: 2, 7: 2.4, 8: 2.8, 9: 3.2, 10: 3.6, 11: 4}
for dataset in train_test_data:
dataset['FamilySize'] = dataset['FamilySize'].map(family_mapping )
preds_df=pd.DataFrame({'shot_made_flag':preds[:,1]},index=df[df.shot_made_flag.isnull() ].index+1)
 preds_df.index.name='shot_id'
preds_df.to_csv('submission.csv' )<set_options>
features_drop = ['Ticket', 'Parch', ]
train_df = train_df.drop(features_drop, axis=1)
 test_df = test_df.drop(features_drop, axis=1)
 train_df = train_df.drop(['PassengerId'], axis=1 )
%matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )<load_from_csv>
train_data = train_df.drop('Survived', axis=1)
 target = train_df['Survived']
train_data.shape, target.shape
dfBase = pd.read_csv('.. /input/kobe-bryant-shot-selection/data.csv')
 dfBase.dataframeName = 'kobe-bryant-shot-selection.csv'<create_dataframe>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
print(f'Dataset de treino tem {dfBase.shape[0]} linhas por {dfBase.shape[1]} colunas({dfBase.shape[0] * dfBase.shape[1]} celulas)' )<feature_engineering>
clf = KNeighborsClassifier(n_neighbors = 13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
dfPreprocess = dfBase.copy()
dfPreprocess['dist'] = np.sqrt(dfPreprocess['loc_x']**2 + dfPreprocess['loc_y']**2)
 loc_x_zero = dfPreprocess['loc_x'] == 0
dfPreprocess['angle'] = np.array([0]*len(dfPreprocess))
dfPreprocess['angle'][~loc_x_zero] = np.arctan(dfPreprocess['loc_y'][~loc_x_zero] / dfPreprocess['loc_x'][~loc_x_zero])
 dfPreprocess['angle'][loc_x_zero] = np.pi / 2<feature_engineering>
round(np.mean(score)*100, 2 )
dfPreprocess['remaining_time'] = dfPreprocess['minutes_remaining'] * 60 + dfPreprocess['seconds_remaining']<feature_engineering>
clf = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
dfPreprocess['match_elapsed_time'] =(dfPreprocess['period'] * 720)+(720 - dfPreprocess['remaining_time'] )<drop_column>
round(np.mean(score)*100, 2 )
dfPreprocess = dfPreprocess.drop(axis=1, columns=[
'shot_zone_range',
'shot_zone_area',
'shot_distance',
'lat',
'lon',
'loc_x',
'loc_y',
'shot_zone_basic',
'shot_type',
'team_name',
'team_id',
'matchup',
'game_event_id',
'game_id',
'season',
'game_date',
'seconds_remaining',
'minutes_remaining',
'period',
])
 dfPreprocess.set_index('shot_id', inplace=True )<drop_column>
rand_clf = RandomForestClassifier(n_estimators=13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
dfPreprocess = dfPreprocess[['dist','angle', 'action_type', 'combined_shot_type', 'playoffs', 'match_elapsed_time', 'remaining_time', 'opponent', 'shot_made_flag']]<rename_columns>
round(np.mean(score)*100, 2 )
dfPreprocess.columns = [
'dist',
'angle',
'action_type_cat',
'combined_shot_type_cat',
'playoffs_cat',
'match_elapsed_time',
'remaining_time',
'opponent_cat',
'target'
]<remove_duplicates>
clf = GaussianNB()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
print(f'Antes - Preprocess tem {dfPreprocess.shape[0]} linhas por {dfPreprocess.shape[1]} colunas({dfPreprocess.shape[0] * dfPreprocess.shape[1]} celulas)')
 dfPreprocess.drop_duplicates()
print(f'Depois - Preprocess tem {dfPreprocess.shape[0]} linhas por {dfPreprocess.shape[1]} colunas({dfPreprocess.shape[0] * dfPreprocess.shape[1]} celulas)' )<define_variables>
round(np.mean(score)*100, 2 )
def generateMetadata(dfInput):
data = []
for f in dfInput.columns:
if f == 'target':
role = 'target'
elif f == 'id':
role = 'id'
else:
role = 'input'
if f == 'target':
level = 'binary'
elif 'cat' in f or f == 'id':
level = 'nominal'
elif dfInput[f].dtype == float or dfInput[f].dtype == np.float64:
level = 'interval'
elif dfInput[f].dtype == int or dfInput[f].dtype == np.int64:
level = 'ordinal'
keep = True
if f == 'id':
keep = False
dtype = dfInput[f].dtype
f_dict = {
'varname': f,
'role': role,
'level': level,
'keep': keep,
'dtype': dtype
}
data.append(f_dict)
 meta = pd.DataFrame(data, columns=['varname', 'role', 'level', 'keep', 'dtype'])
 meta.set_index('varname', inplace=True)
 return meta<create_dataframe>
svm = SVC()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
meta_preprocess = generateMetadata(dfPreprocess )<filter>
round(np.mean(score)*100,2 )
print('Metadados categoricos da base pré processada')
 print(meta_preprocess[(meta_preprocess.level == 'nominal')&(meta_preprocess.keep)].index )<create_dataframe>
svm = SVC()
clf.fit(train_data, target)
 test_data = test_df.drop("PassengerId", axis=1 ).copy()
prediction = clf.predict(test_data )
print('Tipos e quantidade de features do dataset')
 display(pd.DataFrame({'count' : meta_preprocess.groupby(['role', 'level'])['role'].size() } ).reset_index() )<count_missing_values>
model = XGBClassifier()
model.fit(train_data, target )
def getMissingAttributes(dfInput):
atributos_missing = []
return_missing = []
for f in dfInput.columns:
missings = dfInput[f].isna().sum()
if missings > 0:
atributos_missing.append(f)
 missings_perc = missings/dfInput.shape[0]
return_missing.append([f, missings, missings_perc])
 print('Atributo {} tem {} amostras({:.2%})com valores faltantes'.format(f, missings, missings_perc))
print('No total, há {} atributos com valores faltantes'.format(len(atributos_missing)))
 return pd.DataFrame(return_missing ).rename(index=str, columns={0: "column_name", 1: "column_nulls", 2: "column_percentage"} )<filter>
y_pred = clf.predict(test_data)
 predictions = [round(value)for value in y_pred]
missing = getMissingAttributes(dfPreprocess[meta_preprocess[(meta_preprocess.role != 'target')].index])
 display(missing )<define_variables>
run_gs = False
if run_gs:
parameter_grid = {
'max_depth' : [4, 6, 8],
'n_estimators': [50, 10],
'max_features': ['sqrt', 'auto', 'log2'],
'min_samples_split': [2, 3, 10],
'min_samples_leaf': [1, 3, 10],
'bootstrap': [True, False],
}
forest = RandomForestClassifier()
cross_validation = StratifiedKFold(n_splits=5)
 grid_search = GridSearchCV(forest,
scoring='accuracy',
param_grid=parameter_grid,
cv=cross_validation,
verbose=1 
)
 grid_search.fit(train_data, target)
 model = grid_search
parameters = grid_search.best_params_
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
else:
parameters = {'bootstrap': False, 'min_samples_leaf': 3, 'n_estimators': 50,
'min_samples_split': 10, 'max_features': 'sqrt', 'max_depth': 6}
model = RandomForestClassifier(**parameters)
 model.fit(train_data, target)
remove_threshold = 0.425<filter>
output = model.predict(test_data ).astype(int )
<feature_engineering><EOS> 
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": output
})
 submission.to_csv('submission.csv', index=False )