Kaggle competition: Global Wheat Detection
train['popularity'] = np.log1p(train.popularity)
 test['popularity'] = np.log1p(test.popularity )<feature_engineering>
!pip install --no-deps '.. /input/wheatdep/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/wheatdep/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
train['film_that_has_homepage'] = train['homepage'].isnull().apply(lambda x: 0 if x==True else 1 ).copy()
test['film_that_has_homepage'] = test['homepage'].isnull().apply(lambda x: 0 if x==True else 1 ).copy()<feature_engineering>
NMS_THRESHOLD = 0.94
lang = train['original_language']
lang_more_17_samples = [x[0] for x in Counter(pd.DataFrame(lang ).stack() ).most_common(17)]
for col in lang_more_17_samples :
train[col] = train['original_language'].apply(lambda x: 1 if x == col else 0)
 for col in lang_more_17_samples :
test[col] = test['original_language'].apply(lambda x: 1 if x == col else 0)
<load_from_csv>
MIN_CLASS_SCORE = -5.0
_DUMMY_DETECTION_SCORE = -1e5
MAX_DETECTION_POINTS = 5000
MAX_DETECTIONS_PER_IMAGE = 100
def decode_box_outputs(rel_codes, anchors, output_xyxy=False):

 ycenter_a =(anchors[0] + anchors[2])/ 2
xcenter_a =(anchors[1] + anchors[3])/ 2
ha = anchors[2] - anchors[0]
wa = anchors[3] - anchors[1]
ty, tx, th, tw = rel_codes
w = torch.exp(tw)* wa
h = torch.exp(th)* ha
ycenter = ty * ha + ycenter_a
xcenter = tx * wa + xcenter_a
ymin = ycenter - h / 2.
 xmin = xcenter - w / 2.
 ymax = ycenter + h / 2.
 xmax = xcenter + w / 2.
 if output_xyxy:
out = torch.stack([xmin, ymin, xmax, ymax], dim=1)
 else:
out = torch.stack([ymin, xmin, ymax, xmax], dim=1)
 return out
def _generate_anchor_configs(min_level, max_level, num_scales, aspect_ratios):

 anchor_configs = {}
for level in range(min_level, max_level + 1):
anchor_configs[level] = []
for scale_octave in range(num_scales):
for aspect in aspect_ratios:
anchor_configs[level].append(( 2 ** level, scale_octave / float(num_scales), aspect))
return anchor_configs
def _generate_anchor_boxes(image_size, anchor_scale, anchor_configs):

 boxes_all = []
for _, configs in anchor_configs.items() :
boxes_level = []
for config in configs:
stride, octave_scale, aspect = config
if image_size % stride != 0:
raise ValueError("input size must be divided by the stride.")
 base_anchor_size = anchor_scale * stride * 2 ** octave_scale
anchor_size_x_2 = base_anchor_size * aspect[0] / 2.0
anchor_size_y_2 = base_anchor_size * aspect[1] / 2.0
x = np.arange(stride / 2, image_size, stride)
 y = np.arange(stride / 2, image_size, stride)
 xv, yv = np.meshgrid(x, y)
 xv = xv.reshape(-1)
 yv = yv.reshape(-1)
 boxes = np.vstack(( yv - anchor_size_y_2, xv - anchor_size_x_2,
yv + anchor_size_y_2, xv + anchor_size_x_2))
boxes = np.swapaxes(boxes, 0, 1)
 boxes_level.append(np.expand_dims(boxes, axis=1))
boxes_level = np.concatenate(boxes_level, axis=1)
 boxes_all.append(boxes_level.reshape([-1, 4]))
anchor_boxes = np.vstack(boxes_all)
 return anchor_boxes
def generate_detections(cls_outputs, box_outputs, anchor_boxes, indices, classes, image_scale):

 anchor_boxes = anchor_boxes[indices, :]
boxes = decode_box_outputs(box_outputs.T.float() , anchor_boxes.T, output_xyxy=True)
 scores = cls_outputs.sigmoid().squeeze(1 ).float()
top_detection_idx = batched_nms(boxes, scores, classes, iou_threshold=NMS_THRESHOLD)
 top_detection_idx = top_detection_idx[:MAX_DETECTIONS_PER_IMAGE]
boxes = boxes[top_detection_idx]
scores = scores[top_detection_idx, None]
classes = classes[top_detection_idx, None]
boxes[:, 2] -= boxes[:, 0]
boxes[:, 3] -= boxes[:, 1]
boxes *= image_scale
classes += 1
detections = torch.cat([boxes, scores, classes.float() ], dim=1)
 if len(top_detection_idx)< MAX_DETECTIONS_PER_IMAGE:
detections = torch.cat([
detections,
torch.zeros(
(MAX_DETECTIONS_PER_IMAGE - len(top_detection_idx), 6), device=detections.device, dtype=detections.dtype)
 ], dim=0)
 return detections
class Anchors(nn.Module):

 def __init__(self, min_level, max_level, num_scales, aspect_ratios, anchor_scale, image_size):

 super(Anchors, self ).__init__()
self.min_level = min_level
self.max_level = max_level
self.num_scales = num_scales
self.aspect_ratios = aspect_ratios
self.anchor_scale = anchor_scale
self.image_size = image_size
self.config = self._generate_configs()
self.register_buffer('boxes', self._generate_boxes())
 def _generate_configs(self):

 return _generate_anchor_configs(self.min_level, self.max_level, self.num_scales, self.aspect_ratios)
 def _generate_boxes(self):

 boxes = _generate_anchor_boxes(self.image_size, self.anchor_scale, self.config)
 boxes = torch.from_numpy(boxes ).float()
return boxes
def get_anchors_per_location(self):
return self.num_scales * len(self.aspect_ratios)
 class AnchorLabeler(nn.Module):

 def __init__(self, anchors, num_classes, match_threshold=0.5):

 super(AnchorLabeler, self ).__init__()
similarity_calc = region_similarity_calculator.IouSimilarity()
matcher = argmax_matcher.ArgMaxMatcher(
 match_threshold,
unmatched_threshold=match_threshold,
negatives_lower_than_unmatched=True,
force_match_for_each_row=True)
 box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder()
self.target_assigner = target_assigner.TargetAssigner(similarity_calc, matcher, box_coder)
 self.anchors = anchors
self.match_threshold = match_threshold
self.num_classes = num_classes
def _unpack_labels(self, labels):

 labels_unpacked = []
anchors = self.anchors
count = 0
for level in range(anchors.min_level, anchors.max_level + 1):
feat_size = int(anchors.image_size / 2 ** level)
 steps = feat_size ** 2 * anchors.get_anchors_per_location()
indices = torch.arange(count, count + steps, device=labels.device)
 count += steps
labels_unpacked.append(
 torch.index_select(labels, 0, indices ).view([feat_size, feat_size, -1]))
return labels_unpacked
def label_anchors(self, gt_boxes, gt_labels):

 gt_box_list = box_list.BoxList(gt_boxes)
 anchor_box_list = box_list.BoxList(self.anchors.boxes)
 cls_targets, _, box_targets, _, matches = self.target_assigner.assign(anchor_box_list, gt_box_list, gt_labels)
 cls_targets -= 1
cls_targets = cls_targets.long()
cls_targets_dict = self._unpack_labels(cls_targets)
 box_targets_dict = self._unpack_labels(box_targets)
 num_positives =(matches.match_results != -1 ).float().sum()
return cls_targets_dict, box_targets_dict, num_positives

 def _post_process(config, cls_outputs, box_outputs):

 batch_size = cls_outputs[0].shape[0]
cls_outputs_all = torch.cat([
cls_outputs[level].permute(0, 2, 3, 1 ).reshape([batch_size, -1, config.num_classes])
 for level in range(config.num_levels)], 1)
 box_outputs_all = torch.cat([
box_outputs[level].permute(0, 2, 3, 1 ).reshape([batch_size, -1, 4])
 for level in range(config.num_levels)], 1)
 _, cls_topk_indices_all = torch.topk(cls_outputs_all.reshape(batch_size, -1), dim=1, k=MAX_DETECTION_POINTS)
 indices_all = cls_topk_indices_all / config.num_classes
classes_all = cls_topk_indices_all % config.num_classes
box_outputs_all_after_topk = torch.gather(
 box_outputs_all, 1, indices_all.unsqueeze(2 ).expand(-1, -1, 4))
cls_outputs_all_after_topk = torch.gather(
 cls_outputs_all, 1, indices_all.unsqueeze(2 ).expand(-1, -1, config.num_classes))
cls_outputs_all_after_topk = torch.gather(
 cls_outputs_all_after_topk, 2, classes_all.unsqueeze(2))
return cls_outputs_all_after_topk, box_outputs_all_after_topk, indices_all, classes_all
class DetBenchEval(nn.Module):
def __init__(self, model, config):
super(DetBenchEval, self ).__init__()
self.config = config
self.model = model
self.anchors = Anchors(
 config.min_level, config.max_level,
config.num_scales, config.aspect_ratios,
config.anchor_scale, config.image_size)
 def forward(self, x, image_scales):
class_out, box_out = self.model(x)
 class_out, box_out, indices, classes = _post_process(self.config, class_out, box_out)
 batch_detections = []
for i in range(x.shape[0]):
detections = generate_detections(
 class_out[i], box_out[i], self.anchors.boxes, indices[i], classes[i], image_scales[i])
 batch_detections.append(detections)
 return torch.stack(batch_detections, dim=0)
 class DetBenchTrain(nn.Module):
def __init__(self, model, config):
super(DetBenchTrain, self ).__init__()
self.config = config
self.model = model
anchors = Anchors(
 config.min_level, config.max_level,
config.num_scales, config.aspect_ratios,
config.anchor_scale, config.image_size)
 self.anchor_labeler = AnchorLabeler(anchors, config.num_classes, match_threshold=0.5)
 self.loss_fn = DetectionLoss(self.config)
 def forward(self, x, gt_boxes, gt_labels):
class_out, box_out = self.model(x)
 cls_targets = []
box_targets = []
num_positives = []
for i in range(x.shape[0]):
gt_class_out, gt_box_out, num_positive = self.anchor_labeler.label_anchors(gt_boxes[i], gt_labels[i])
 cls_targets.append(gt_class_out)
 box_targets.append(gt_box_out)
 num_positives.append(num_positive)
 return self.loss_fn(class_out, box_out, cls_targets, box_targets, num_positives)
google_train_path = '.. /input/google-result/google_result_train.csv'
google_test_path = '.. /input/google-result/google_result_test.csv'
google_train = pd.read_csv(google_train_path)
 google_test = pd.read_csv(google_test_path)
 train['google_result'] = google_train['result']
test['google_result'] = google_test['result']<feature_engineering>
SIZE = 1024
WEIGHT_FILE1 = '.. /input/effdet5fold/stac2.bin'
WEIGHT_FILE2 = '.. /input/effdet5fold/splitbn.bin'
STAGE1_THRESHOLD = 0.25
STAGE1_IOU_THR = 0.475
STAGE1_SKIP_THR = 0.41
STAGE1_SUPPRESS = 0.125
STAGE2_THRESHOLD = 0.37
STAGE2_IOU_THR = 0.44
STAGE2_SKIP_THR = 0.43
STAGE2_SUPPRESS = 0.125
MAX_BOX = 240
MIN_BOX = 10
EPOCHS_1 = 13
EPOCHS_2 = 13
train['google_result']=np.log1p(train.google_result)
 test['google_result']=np.log1p(test.google_result )<feature_engineering>
__author__ = 'ZFTurbo: https://kaggle.com/zfturbo'
@jit(nopython=True)
 def bb_intersection_over_union(A, B)-> float:
xA = max(A[0], B[0])
 yA = max(A[1], B[1])
 xB = min(A[2], B[2])
 yB = min(A[3], B[3])
 interArea = max(0, xB - xA)* max(0, yB - yA)
 if interArea == 0:
return 0.0
boxAArea =(A[2] - A[0])*(A[3] - A[1])
 boxBArea =(B[2] - B[0])*(B[3] - B[1])
 iou = interArea / float(boxAArea + boxBArea - interArea)
 center_x1 =(A[2] + A[0])/ 2
center_y1 =(A[3] + A[1])/ 2
center_x2 =(B[2] + B[0])/ 2
center_y2 =(B[3] + B[1])/ 2
d_2 =(center_x1 - center_x2)** 2 +(center_y1 - center_y2)** 2
c_2 =(max(A[2], B[2])- min(A[0], B[0])) ** 2 +(
 max(A[3], B[3])- min(A[1], B[1])) ** 2
d_iou = max(min(iou - d_2 / c_2, 1),-1)
 return d_iou
def prefilter_boxes(boxes, scores, labels, weights, thr):
new_boxes = dict()
for t in range(len(boxes)) :
if len(boxes[t])!= len(scores[t]):
print('Error.Length of boxes arrays not equal to length of scores array: {} != {}'.format(len(boxes[t]), len(scores[t])))
 exit()
if len(boxes[t])!= len(labels[t]):
print('Error.Length of boxes arrays not equal to length of labels array: {} != {}'.format(len(boxes[t]), len(labels[t])))
 exit()
for j in range(len(boxes[t])) :
score = scores[t][j]
if score < thr:
continue
label = int(labels[t][j])
 box_part = boxes[t][j]
x1 = float(box_part[0])
 y1 = float(box_part[1])
 x2 = float(box_part[2])
 y2 = float(box_part[3])
 if x2 < x1:
warnings.warn('X2 < X1 value in box.Swap them.')
 x1, x2 = x2, x1
if y2 < y1:
warnings.warn('Y2 < Y1 value in box.Swap them.')
 y1, y2 = y2, y1
if x1 < 0:
warnings.warn('X1 < 0 in box.Set it to 0.')
 x1 = 0
if x1 > 1:
warnings.warn('X1 > 1 in box.Set it to 1.Check that you normalize boxes in [0, 1] range.')
 x1 = 1
if x2 < 0:
warnings.warn('X2 < 0 in box.Set it to 0.')
 x2 = 0
if x2 > 1:
warnings.warn('X2 > 1 in box.Set it to 1.Check that you normalize boxes in [0, 1] range.')
 x2 = 1
if y1 < 0:
warnings.warn('Y1 < 0 in box.Set it to 0.')
 y1 = 0
if y1 > 1:
warnings.warn('Y1 > 1 in box.Set it to 1.Check that you normalize boxes in [0, 1] range.')
 y1 = 1
if y2 < 0:
warnings.warn('Y2 < 0 in box.Set it to 0.')
 y2 = 0
if y2 > 1:
warnings.warn('Y2 > 1 in box.Set it to 1.Check that you normalize boxes in [0, 1] range.')
 y2 = 1
if(x2 - x1)*(y2 - y1)== 0.0:
warnings.warn("Zero area box skipped: {}.".format(box_part))
continue
b = [int(label), float(score)* weights[t], x1, y1, x2, y2]
if label not in new_boxes:
new_boxes[label] = []
new_boxes[label].append(b)
 for k in new_boxes:
current_boxes = np.array(new_boxes[k])
 new_boxes[k] = current_boxes[current_boxes[:, 1].argsort() [::-1]]
return new_boxes
def get_weighted_box(boxes, conf_type='avg'):

 box = np.zeros(6, dtype=np.float32)
 conf = 0
conf_list = []
for b in boxes:
box[2:] +=(b[1] * b[2:])
 conf += b[1]
conf_list.append(b[1])
 box[0] = boxes[0][0]
if conf_type == 'avg':
box[1] = conf / len(boxes)
 elif conf_type == 'max':
box[1] = np.array(conf_list ).max()
box[2:] /= conf
return box
def find_matching_box(boxes_list, new_box, match_iou):
best_iou = match_iou
best_index = -1
for i in range(len(boxes_list)) :
box = boxes_list[i]
if box[0] != new_box[0]:
continue
iou = bb_intersection_over_union(box[2:], new_box[2:])
 if iou > best_iou:
best_index = i
best_iou = iou
return best_index, best_iou
def weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=None, iou_thr=0.55, skip_box_thr=0.0, conf_type='avg', allows_overflow=False):

 if weights is None:
weights = np.ones(len(boxes_list))
if len(weights)!= len(boxes_list):
print('Warning: incorrect number of weights {}.Must be: {}.Set weights equal to 1.'.format(len(weights), len(boxes_list)))
 weights = np.ones(len(boxes_list))
weights = np.array(weights)
 if conf_type not in ['avg', 'max']:
print('Unknown conf_type: {}.Must be "avg" or "max"'.format(conf_type))
exit()
filtered_boxes = prefilter_boxes(boxes_list, scores_list, labels_list, weights, skip_box_thr)
 if len(filtered_boxes)== 0:
return np.zeros(( 0, 4)) , np.zeros(( 0,)) , np.zeros(( 0,))
overall_boxes = []
for label in filtered_boxes:
boxes = filtered_boxes[label]
new_boxes = []
weighted_boxes = []
for j in range(0, len(boxes)) :
index, best_iou = find_matching_box(weighted_boxes, boxes[j], iou_thr)
 if index != -1:
new_boxes[index].append(boxes[j])
 weighted_boxes[index] = get_weighted_box(new_boxes[index], conf_type)
 else:
new_boxes.append([boxes[j].copy() ])
 weighted_boxes.append(boxes[j].copy())
 for i in range(len(new_boxes)) :
if not allows_overflow:
weighted_boxes[i][1] = weighted_boxes[i][1] * min(weights.sum() , len(new_boxes[i])) / weights.sum()
else:
weighted_boxes[i][1] = weighted_boxes[i][1] * len(new_boxes[i])/ weights.sum()
overall_boxes.append(np.array(weighted_boxes))
overall_boxes = np.concatenate(overall_boxes, axis=0)
 overall_boxes = overall_boxes[overall_boxes[:, 1].argsort() [::-1]]
boxes = overall_boxes[:, 2:]
scores = overall_boxes[:, 1]
labels = overall_boxes[:, 0]
return boxes, scores, labels
train['film_belongs_to_collection'] = train['belongs_to_collection'].apply(lambda x: 0 if x == {} else 1)
 test['film_belongs_to_collection'] = test['belongs_to_collection'].apply(lambda x: 0 if x == {} else 1 )<feature_engineering>
def get_valid_transforms() :
return A.Compose([
A.Resize(height=SIZE, width=SIZE, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
t = train[['id','revenue', 'title', 'genres']].copy()
t['genres'] = [[y['name'] for y in x] for x in t['genres']]
genres = t['genres'].sum()
ctr = Counter(genres)
 df_genres = pd.DataFrame.from_dict(ctr, orient='index' ).reset_index().rename(columns={'index':'genre', 0:'count'})
 df_genres=df_genres.sort_values('count', ascending=False)
 df_genres = df_genres[df_genres['count'] > 1]
df_genres = df_genres.nlargest(20, 'count')
 genres = list(df_genres['genre'])
 dic={}
for a in genres:
mask = t.genres.apply(lambda x: a in x)
 dic[a] = t[mask]['revenue'].mean()
t = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'genre'})
 t['color'] = Category20c[len(t)]
hover1 = HoverTool(tooltips = [ 
('Genre','@genre'), 
('Genre mean revenue','@mean_revenue')
 ])
 p = figure(x_range=t.genre, plot_width=1400,plot_height=400, toolbar_location=None, title="Mean revenue per genre", tools=[hover1])
 p.vbar(x='genre', top='mean_revenue', width=0.9, source=t, legend='genre',
line_color='white', fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p)
<feature_engineering>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
train['genres_names'] = [[y['name'] for y in x] for x in train['genres']]
genres_list=['Action', 'Adventure', 'Science Fiction', 'Family', 'Fantasy','Animation']
for a in genres_list :
train['genre_'+a]=train['genres_names'].apply(lambda x: 1 if a in x else 0)
 train = train.drop(['genres_names'], axis=1)
 test['genres_names'] = [[y['name'] for y in x] for x in test['genres']]
for a in genres_list :
test['genre_'+a]=test['genres_names'].apply(lambda x: 1 if a in x else 0)
 test = test.drop(['genres_names'], axis=1 )<feature_engineering>
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)
def date_features(df):
df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 18)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 18)&(df['release_year'] < 100), "release_year"] += 1900
df['release_date'] = pd.to_datetime(df['release_date'])
 df['release_month'] = df['release_date'].dt.month
df['release_quarter'] = df['release_date'].dt.quarter
df.drop(columns=['release_date'], inplace=True)
 return df
train=date_features(train)
 test=date_features(test )<feature_engineering>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d6')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=SIZE
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 newc = {}
for k,v in checkpoint['model_state_dict'].items() :
if 'aux_bn' not in k:
newc[k] = v
net.load_state_dict(newc)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net(WEIGHT_FILE1 )
for col in range(1,12):
train['month'+str(col)] = train['release_month'].apply(lambda x: 1 if x == col else 0)
 for col in range(1,12):
test['month'+str(col)] = test['release_month'].apply(lambda x: 1 if x == col else 0)
 for col in range(1,4):
train['quarter'+str(col)] = train['release_quarter'].apply(lambda x: 1 if x == col else 0)
 for col in range(1,4):
test['quarter'+str(col)] = test['release_quarter'].apply(lambda x: 1 if x == col else 0)
<feature_engineering>
class BaseWheatTTA:

 image_size = SIZE
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
<drop_column>
tta_combinations = [[TTARotate270() ],[None],[TTARotate90() ],[TTARotate180() ],[TTAVerticalFlip() ,TTARotate270() ],[TTAVerticalFlip() ,TTARotate90() ],[TTAVerticalFlip() ],[TTAHorizontalFlip() ] ]
tta_transforms = []
for tta_combination in tta_combinations:
print([tta_transform for tta_transform in tta_combination if tta_transform])
 tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
train = train.drop(['release_month', 'release_quarter'], axis=1)
 test = test.drop(['release_month', 'release_quarter'], axis=1 )<feature_engineering>
def make_tta_predictions(images, score_threshold=STAGE1_THRESHOLD):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 del images
gc.collect()
return predictions
def run_wbf(predictions, image_index, image_size=SIZE, iou_thr=STAGE1_IOU_THR, skip_box_thr=STAGE1_SKIP_THR, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
if(len(boxes)==0): return boxes, scores, labels
try:
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 except:
return [], [], []
boxes = boxes*(image_size-1)
 return boxes, scores, labels
df_names = df_names[df_names['count'] > 11]
names_list = list(df_names['actor'])
 train['cast_names']=[[y['name'] for y in x] for x in train['cast']]
train['cast_names'] = train['cast_names'].apply(lambda x: x[:3])
 dic={}
for a in names_list:
mask = train['cast_names'].apply(lambda x: a in x)
 dic[a] = train[mask]['revenue'].mean()
actors_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'actor'})
 names_list = list(actors_mean_revenue.nlargest(40, 'mean_revenue')['actor'])
 train['actors_mean_revenue'] = train['cast_names'].apply(lambda x: actors_mean_revenue[actors_mean_revenue['actor'].isin(x)].mean() ['mean_revenue'])
 train['actors_mean_revenue'].fillna(0,inplace=True)
 train['total_top_actors_revenue']=train['cast_names'].apply(lambda x: sum([1 for i in x if i in names_list]))
train = train.drop(['cast_names'], axis=1)
 test['cast_names']=[[y['name'] for y in x] for x in test['cast']]
test['cast_names'] = test['cast_names'].apply(lambda x: x[:3])
 test['actors_mean_revenue'] = test['cast_names'].apply(lambda x: actors_mean_revenue[actors_mean_revenue['actor'].isin(x)].mean() ['mean_revenue'])
 test['actors_mean_revenue'].fillna(0,inplace=True)
 test['total_top_actors_revenue']=test['cast_names'].apply(lambda x: sum([1 for i in x if i in names_list]))
test = test.drop(['cast_names'], axis=1)
<feature_engineering>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("[{1}, {2}, {3}, {4}]".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return pred_strings
t = train[['id','revenue', 'title', 'cast']].copy()
t['cast'] = [[y['name'] for y in x] for x in t['cast']]
t['cast'] = t['cast'].apply(lambda x: x[:3])
 names = t['cast'].sum()
ctr = Counter(names)
 names=[n for n in ctr if ctr[n] > 0]
unique_names = pd.Series(names ).unique()
dic={}
for a in unique_names:
mask = t.cast.apply(lambda x: a in x)
 dic[a] = t[mask]['revenue'].mean()
actors_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'actor'})
 t = actors_mean_revenue.nlargest(20, 'mean_revenue')
 t['color'] = Category20c[20]
hover1 = HoverTool(tooltips = [ 
('Actor','@actor'), 
('Revenue','@mean_revenue')
 ])
 p = figure(x_range=t.actor, plot_width=1400,plot_height=400, toolbar_location=None, title="20 actors with highest mean revenue", tools=[hover1])
 p.vbar(x='actor', top='mean_revenue', width=0.9, source=t, legend='actor',
line_color='white', fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p)
<feature_engineering>
results = []
for images, image_ids in tqdm(data_loader):
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
score_threshold = STAGE1_SUPPRESS
indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
bboxes =(format_prediction_string(boxes, scores))
for bbox in bboxes:
result = {
'image_id': image_id,
'width': 1024,
'height': 1024,
'bbox': bbox,
'source': 'pseudo'
}
results.append(result )
t = train[['id','popularity', 'title', 'cast']].copy()
t['popularity'] = np.expm1(t['popularity'])
 t['cast'] = [[y['name'] for y in x] for x in t['cast']]
t['cast'] = t['cast'].apply(lambda x: x[:3])
 names = t['cast'].sum()
ctr = Counter(names)
 names=[n for n in ctr if ctr[n] > 0]
unique_names = pd.Series(names ).unique()
dic={}
for a in unique_names:
mask = t.cast.apply(lambda x: a in x)
 dic[a] = t[mask]['popularity'].mean()
t = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'actor'})
 t = t.nlargest(20, 'mean_revenue')
 t['color'] = Category20c[20]
hover1 = HoverTool(tooltips = [ 
('Actor','@actor'), 
('Revenue','@mean_revenue')
 ])
 p = figure(x_range=t.actor, plot_width=1400,plot_height=400, toolbar_location=None, title="20 actors with highest mean popularity", tools=[hover1])
 p.vbar(x='actor', top='mean_revenue', width=0.9, source=t, legend='actor',
line_color='white', fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p )<feature_engineering>
test_df = pd.DataFrame(results, columns=['image_id', 'width', 'height', 'bbox', 'source'])
 test_df.to_csv('pseudo.csv', index=False)
 test_df.head()
print(len(test_df))
df_has_nominated_actor['nominations'] = df_has_nominated_actor['nominations'].apply(lambda x: 1 if x == 'True' else 0)
 train['has_nominated_actor'] = df_has_nominated_actor['nominations']
test['cast_ids']=[[y['id'] for y in x] for x in test['cast']]
test['cast_ids'] = test['cast_ids'].apply(lambda x: x[:3])
 test['has_nominated_actor'] = test['cast_ids'].apply(lambda x: 0 if(nominations[nominations['id'].isin(x)]['nominations'] != 0 ).any() else 1)
 test = test.drop(['cast_ids'], axis=1)
 train['cast_ids']=[[y['id'] for y in x] for x in train['cast']]
train['cast_ids'] = train['cast_ids'].apply(lambda x: x[:4])
 train['nominated_actors'] = train['cast_ids'].apply(lambda x:(nominations[nominations['id'].isin(x)]['nominations'] != 0 ).sum())
 test['cast_ids']=[[y['id'] for y in x] for x in test['cast']]
test['cast_ids'] = test['cast_ids'].apply(lambda x: x[:4])
 test['nominated_actors'] = test['cast_ids'].apply(lambda x:(nominations[nominations['id'].isin(x)]['nominations'] != 0 ).sum())
<feature_engineering>
del net
gc.collect()
df_names = df_names[df_names['count'] > 10]
names_list = list(df_names['actor'])
 train['crew_names'] = [[y['name'] for y in x if y['department']=='Directing'] for x in train['crew'] ]
train['crew_names'] = train['crew_names'].apply(lambda x: x[:3])
 dic={}
for a in names_list:
mask = train['crew_names'].apply(lambda x: a in x)
 dic[a] = train[mask]['revenue'].mean()
directors_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'director'})
 names_list = list(directors_mean_revenue.nlargest(40, 'mean_revenue')['director'])
 for a in names_list :
train['director_'+a]=train['crew_names'].apply(lambda x: 1 if a in x else 0)
 train = train.drop(['crew_names'], axis=1)
 test['crew_names'] = [[y['name'] for y in x if y['department']=='Directing'] for x in test['crew'] ]
test['crew_names'] = test['crew_names'].apply(lambda x: x[:3])
 for a in names_list :
test['director_'+a]=test['crew_names'].apply(lambda x: 1 if a in x else 0)
 test = test.drop(['crew_names'], axis=1)
<feature_engineering>
net = load_net(WEIGHT_FILE2)
 results = []
for images, image_ids in tqdm(data_loader):
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
score_threshold = STAGE1_SUPPRESS
indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
bboxes =(format_prediction_string(boxes, scores))
for bbox in bboxes:
result = {
'image_id': image_id,
'width': 1024,
'height': 1024,
'bbox': bbox,
'source': 'pseudo'
}
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'width', 'height', 'bbox', 'source'])
 test_df.to_csv('pseudo2.csv', index=False)
 test_df.head()
print(len(test_df))
del net
gc.collect()
df_names = df_names[df_names['count'] > 9]
names_list = list(df_names['actor'])
 train['production_companies'] = [[y['name'] for y in x] for x in train['production_companies'] ]
train['production_companies'] = train['production_companies'].apply(lambda x: x[:3])
 dic={}
for a in names_list:
mask = train['production_companies'].apply(lambda x: a in x)
 dic[a] = train[mask]['revenue'].mean()
companies_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'company'})
 names_list = list(companies_mean_revenue.nlargest(20, 'mean_revenue')['company'])
 for a in names_list :
train['production_'+a]=train['production_companies'].apply(lambda x: 1 if a in x else 0)
 train = train.drop(['production_companies'], axis=1)
 test['production_companies'] = [[y['name'] for y in x] for x in test['production_companies'] ]
test['production_companies'] = test['production_companies'].apply(lambda x: x[:3])
 for a in names_list :
test['production_'+a]=test['production_companies'].apply(lambda x: 1 if a in x else 0)
 test = test.drop(['production_companies'], axis=1 )<prepare_x_and_y>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
y = np.log1p(train.revenue )<split>
test_df = pd.read_csv('pseudo.csv')
 bad_boxes = [3687, 117344,173,113947,52868,2159,2169,121633,121634,147504,118211,52727,147552]
marking = pd.concat([pd.read_csv('.. /input/global-wheat-detection/train.csv'),test_df])
 if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10:
marking.drop(bad_boxes)
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 targets = set(marking.source.values)
 if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10:
targets.remove('arvalis_3')
 targets.remove('usask_1')
 skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
print(set(df_folds.source.values))
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds = pd.concat([ df_folds.loc[df_folds['source'] == i, :] for i in targets])
 df_folds.loc[df_folds['source']=='pseudo','fold'] = 1
df_folds.loc[df_folds['fold']==4,'fold'] = 1
X = train.drop(['id','runtime', 'release_day'], axis=1)
 test_X = test.drop(['id','runtime', 'release_day'], axis=1 ).select_dtypes(exclude=['object'])
 train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1,test_size=0.33)
 train_X=train_X.drop(['revenue'], axis=1 ).select_dtypes(exclude=['object'])
 X=X.drop(['revenue'], axis=1 ).select_dtypes(exclude=['object'])
 val_X_revenue=val_X.pop('revenue')
 val_X_title=val_X.pop('title')
 val_X=val_X.select_dtypes(exclude=['object'])
 xgb_model = XGBRegressor(learning_rate=0.05,
n_estimators=10000,max_depth=4)
 xgb_model.fit(train_X, train_y, early_stopping_rounds=100,
eval_set=[(val_X, val_y)], eval_metric = 'rmse')
 xbg_val_predictions=xgb_model.predict(val_X)
<save_to_csv>
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 1024), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.RandomRotate90(p=0.5),
A.Transpose(p=0.5),
A.JpegCompression(quality_lower=85, quality_upper=95, p=0.2),
A.OneOf([
A.Blur(blur_limit=3, p=1.0),
A.MedianBlur(blur_limit=3, p=1.0)
 ],p=0.1),
A.Resize(height=SIZE, width=SIZE, p=1),
A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=SIZE, width=SIZE, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
xgb_model_full = XGBRegressor(n_estimators=145, learning_rate=0.05,max_depth=4)
 xgb_model_full.fit(X, y)
 test_preds=xgb_model_full.predict(test_X)
 output = pd.DataFrame({'id': test.id,
'revenue': np.expm1(test_preds)})
 output.to_csv('submission.csv', index=False )<set_options>
TRAIN_ROOT_PATH = '/kaggle/input/global-wheat-detection/train'
TEST_ROOT_PATH = '/kaggle/input/global-wheat-detection/test'
DATASET_PATH = '/kaggle/working/pseudoset'
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.33:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
records = self.marking[self.marking['image_id'] == image_id]
PATH = TEST_ROOT_PATH if(records.iloc[0].source == 'pseudo')else TRAIN_ROOT_PATH
image = cv2.imread(f'{PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_mixup_image_and_boxes(self, index):
image, boxes = self.load_image_and_boxes(index)
 r_image, r_boxes = self.load_image_and_boxes(random.randint(0, self.image_ids.shape[0] - 1))
return(image+r_image)/2, np.vstack(( boxes, r_boxes)).astype(np.int32)
 def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 opt_level = 'O1'
def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) == 10: break
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 if(step+1)% 1 == 0:
self.optimizer.step()
self.optimizer.zero_grad()
if self.config.step_scheduler:
self.scheduler.step()
if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) == 10 and step == 5: break
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
')
 def collate_fn(batch):
return tuple(zip(*batch))
warnings.filterwarnings("ignore" )<load_from_csv>
def get_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d6')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size= SIZE
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 newc = {}
for k,v in checkpoint['model_state_dict'].items() :
if 'aux_bn' not in k: newc[k]=v
net.load_state_dict(newc)
 del checkpoint
gc.collect()
for param in net.backbone.parameters() :
param.requires_grad = False
net = DetBenchTrain(net, config)
 return net.cuda()
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train.info()<load_from_csv>
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 1].index.values,
marking=marking,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 0].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 class TrainGlobalConfig:
num_workers = 4
batch_size = 4
n_epochs = EPOCHS_1
lr = 0.0001 * 0.05
folder = 'pseudo'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=6,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
 def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test.info()<count_missing_values>
torch.cuda.empty_cache()
train.isna().sum()<count_missing_values>
net = get_net(WEIGHT_FILE1 )
test.isna().sum()<feature_engineering>
run_training()
train[['release_month','release_day','release_year']]=train['release_date'].str.split('/',expand=True ).replace(np.nan, -1 ).astype(int)
 train.loc[(train['release_year'] <= 19)&(train['release_year'] < 100), "release_year"] += 2000
train.loc[(train['release_year'] > 19)&(train['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(train['release_date'])
 train['release_dayofweek'] = releaseDate.dt.dayofweek
train['release_quarter'] = releaseDate.dt.quarter<categorify>
del net
gc.collect()
torch.cuda.empty_cache()
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
train = train
train['genres'] = train['genres'].map(lambda x: sorted([d['name'] for d in get_dictionary(x)])).map(lambda x: ','.join(map(str, x)))
 genres = train.genres.str.get_dummies(sep=',')
 train = pd.concat([train, genres], axis=1, sort=False)
 print("Action Genres Movie ", train[train.Action == 1].shape[0])
 print("Adventure Genres Movie ", train[train.Adventure == 1].shape[0])
 print("Animation Genres Movie ", train[train.Animation == 1].shape[0])
 print("Comedy Genres Movie ", train[train.Comedy == 1].shape[0])
 print("Crime Genres Movie ", train[train.Crime == 1].shape[0])
 print("Documentary Genres Movie ", train[train.Documentary == 1].shape[0])
 print("Drama Genres Movie ", train[train.Drama == 1].shape[0])
 print("Family Genres Movie ", train[train.Family == 1].shape[0])
 print("Fantasy Genres Movie ", train[train.Fantasy == 1].shape[0])
 print("Foreign Genres Movie ", train[train.Foreign == 1].shape[0])
 print("History Genres Movie ", train[train.History == 1].shape[0])
 print("Music Genres Movie ", train[train.Music == 1].shape[0])
 print("Mystery Genres Movie ", train[train.Mystery == 1].shape[0])
 print("Romance Genres Movie ", train[train.Romance == 1].shape[0])
 print("Science Fiction Genres Movie ", train[train['Science Fiction'] == 1].shape[0])
 print("TV Movie Genres Movie ", train[train['TV Movie'] == 1].shape[0])
 print("Thriller Genres Movie ", train[train.Thriller == 1].shape[0])
 print("War Genres Movie ", train[train.War == 1].shape[0])
 print("Western Genres Movie ", train[train.Western == 1].shape[0] )<categorify>
test_df = pd.read_csv('pseudo2.csv')
 bad_boxes = [3687, 117344,173,113947,52868,2159,2169,121633,121634,147504,118211,52727,147552]
marking = pd.concat([pd.read_csv('.. /input/global-wheat-detection/train.csv'),test_df])if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10 else pd.concat([test_df, test_df])
 if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10:
marking.drop(bad_boxes)
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 targets = set(marking.source.values)
 if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10:
targets.remove('arvalis_3')
 targets.remove('usask_1')
 skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
print(set(df_folds.source.values))
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds = pd.concat([ df_folds.loc[df_folds['source'] == i, :] for i in targets])
 df_folds.loc[df_folds['source']=='pseudo','fold'] = 2
df_folds.loc[df_folds['fold']==3,'fold'] = 2
test = test
test['genres'] = test['genres'].map(lambda x: sorted([d['name'] for d in get_dictionary(x)])).map(lambda x: ','.join(map(str, x)))
 genres = test.genres.str.get_dummies(sep=',')
 test = pd.concat([test, genres], axis=1, sort=False)
 print("Action Genres Movie ", test[test.Action == 1].shape[0])
 print("Adventure Genres Movie ", test[test.Adventure == 1].shape[0])
 print("Animation Genres Movie ", test[test.Animation == 1].shape[0])
 print("Comedy Genres Movie ", test[test.Comedy == 1].shape[0])
 print("Crime Genres Movie ", test[test.Crime == 1].shape[0])
 print("Documentary Genres Movie ", test[test.Documentary == 1].shape[0])
 print("Drama Genres Movie ", test[test.Drama == 1].shape[0])
 print("Family Genres Movie ", test[test.Family == 1].shape[0])
 print("Fantasy Genres Movie ", test[test.Fantasy == 1].shape[0])
 print("Foreign Genres Movie ", test[test.Foreign == 1].shape[0])
 print("History Genres Movie ", test[test.History == 1].shape[0])
 print("Music Genres Movie ", test[test.Music == 1].shape[0])
 print("Mystery Genres Movie ", test[test.Mystery == 1].shape[0])
 print("Romance Genres Movie ", test[test.Romance == 1].shape[0])
 print("Science Fiction Genres Movie ", test[test['Science Fiction'] == 1].shape[0])
 print("TV Movie Genres Movie 0")
 print("Thriller Genres Movie ", test[test.Thriller == 1].shape[0])
 print("War Genres Movie ", test[test.War == 1].shape[0])
 print("Western Genres Movie ", test[test.Western == 1].shape[0] )<count_values>
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 2].index.values,
marking=marking,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 0].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 class TrainGlobalConfig:
num_workers = 4
batch_size = 4
n_epochs = EPOCHS_2
lr = 0.0001 * 0.05
folder = 'pseudo2'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=6,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
 def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )
train['status'].value_counts()<filter>
net = get_net(WEIGHT_FILE2)
 run_training()
train.loc[train['status'] == "Rumored"][['status','revenue']]<count_values>
del net
gc.collect()
torch.cuda.empty_cache()
test['status'].value_counts()<load_from_csv>
def load_net() :
config = get_efficientdet_config('tf_efficientdet_d6')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=SIZE
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net = DetBenchEval(net, config)
 net.eval()
net.model.load_state_dict(torch.load('./pseudo2/best-checkpoint.bin')['model_state_dict'])
 return net.cuda()
net2 = load_net()
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.merge(train, trainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, testAdditionalFeatures, how='left', on=['imdb_id'] )<count_missing_values>
def load_net() :
config = get_efficientdet_config('tf_efficientdet_d6')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=SIZE
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net = DetBenchEval(net, config)
 net.eval()
net.model.load_state_dict(torch.load('./pseudo/best-checkpoint.bin')['model_state_dict'])
 return net.cuda()
net = load_net()
print("Missing rating in Train set", train['rating'].isna().sum())
 print("Missing total Votes in Train set", train['totalVotes'].isna().sum())
 print("")
 print("Missing rating in Test set", test['rating'].isna().sum())
 print("Missing total Votes in Test set", test['totalVotes'].isna().sum() )<data_type_conversions>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
tta_combinations = [[TTARotate270() ],[None],[TTARotate90() ],[TTARotate180() ],[TTAVerticalFlip() ,TTARotate270() ],[TTAVerticalFlip() ,TTARotate90() ],[TTAVerticalFlip() ],[TTAHorizontalFlip() ] ]
tta_transforms = []
for tta_combination in tta_combinations:
print([tta_transform for tta_transform in tta_combination if tta_transform])
 tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
def make_tta_predictions(images, score_threshold=STAGE2_THRESHOLD):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
predictions2 = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 for tta_transform in tta_transforms:
result = []
det = net2(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions2.append(result)
 return predictions, predictions2
WEIGHTS = [3] * 16 + [2] * 16
def run_wbf(predictions, predictions2, image_index, image_size=SIZE, iou_thr=STAGE2_IOU_THR, skip_box_thr=STAGE2_SKIP_THR, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes2 = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions2]
scores2 = [prediction[image_index]['scores'].tolist() for prediction in predictions2]
labels2 = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions2]
if(len(boxes)==0)and(len(boxes2)==0): return boxes, scores, labels
try:
boxes, scores, labels = weighted_boxes_fusion(boxes+boxes2, scores+scores2, labels+labels2, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 except:
return [], [], []
boxes = boxes*(image_size-1)
 return boxes, scores, labels
train['rating'] = train['rating'].fillna(1.5)
 train['totalVotes'] = train['totalVotes'].fillna(6)
 test['rating'] = test['rating'].fillna(1.5)
 test['totalVotes'] = test['totalVotes'].fillna(6 )<feature_engineering>
def get_valid_transforms() :
return A.Compose([
A.Resize(height=SIZE, width=SIZE, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn 
)
 def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 config = get_efficientdet_config('tf_efficientdet_d6')
 config.num_classes = 1
config.image_size= SIZE
net.model.load_state_dict(torch.load('./pseudo/best-checkpoint.bin')['model_state_dict'])
 net = DetBenchEval(net.model, config ).cuda()
net.eval()
results = []
for images, image_ids in data_loader:
predictions, predictions2 = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, predictions2, image_index=i)
 boxes =(boxes ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
score_threshold = STAGE2_SUPPRESS
indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
if(len(boxes)== 0):
result = {
'image_id': image_id,
'PredictionString': ""
}
else:
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indexes = np.where(boxes[:,2]*boxes[:,3] < MAX_BOX*MAX_BOX)
 boxes = boxes[indexes]
scores = scores[indexes]
indexes = np.where(boxes[:,2]*boxes[:,3] > MIN_BOX*MIN_BOX)
 boxes = boxes[indexes]
scores = scores[indexes]
if(len(boxes)== 0):
result = {
'image_id': image_id,
'PredictionString': ""
}
else:
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()
def prepare(df):
global json_cols
global train_dict
df['rating'] = df['rating'].fillna(1.5)
 df['totalVotes'] = df['totalVotes'].fillna(6)
 df['weightedRating'] =(df['rating']*df['totalVotes'] + 6.367 * 1000)/(df['totalVotes'] + 1000)
 df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 19)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 19)&(df['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['originalBudget'] = df['budget']
df['inflationBudget'] = df['budget'] + df['budget']*1.8/100*(2018-df['release_year'])
 df['budget'] = np.log1p(df['budget'])
 df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df['collection_name'] = df['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['collection_name'].fillna('')))
 df['collection_name'] = le.transform(df['collection_name'].fillna('' ).astype(str))
df['num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 df['num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
 df['_popularity_mean_year'] = df['popularity'] / df.groupby("release_year")["popularity"].transform('mean')
 df['_budget_runtime_ratio'] = df['budget']/df['runtime']
df['_budget_popularity_ratio'] = df['budget']/df['popularity']
df['_budget_year_ratio'] = df['budget']/(df['release_year']*df['release_year'])
 df['_releaseYear_popularity_ratio'] = df['release_year']/df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity']/df['release_year']
df['_popularity_totalVotes_ratio'] = df['totalVotes']/df['popularity']
df['_rating_popularity_ratio'] = df['rating']/df['popularity']
df['_rating_totalVotes_ratio'] = df['totalVotes']/df['rating']
df['_totalVotes_releaseYear_ratio'] = df['totalVotes']/df['release_year']
df['_budget_rating_ratio'] = df['budget']/df['rating']
df['_runtime_rating_ratio'] = df['runtime']/df['rating']
df['_budget_totalVotes_ratio'] = df['budget']/df['totalVotes']
df['has_homepage'] = 0
df.loc[pd.isnull(df['homepage']),"has_homepage"] = 1
df['isbelongs_to_collectionNA'] = 0
df.loc[pd.isnull(df['belongs_to_collection']),"isbelongs_to_collectionNA"] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'] == 0 ,"isTaglineNA"] = 1
df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'] == "en" ,"isOriginalLanguageEng"] = 1
df['isTitleDifferent'] = 1
df.loc[ df['original_title'] == df['title'] ,"isTitleDifferent"] = 0
df['isMovieReleased'] = 1
df.loc[ df['status'] != "Released" ,"isMovieReleased"] = 0
df['collection_id'] = df['belongs_to_collection'].apply(lambda x : np.nan if len(x)==0 else x[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['cast'].apply(lambda x : len(x))
df['meanruntimeByYear'] = df.groupby("release_year")["runtime"].aggregate('mean')
 df['meanPopularityByYear'] = df.groupby("release_year")["popularity"].aggregate('mean')
 df['meanBudgetByYear'] = df.groupby("release_year")["budget"].aggregate('mean')
 df['meantotalVotesByYear'] = df.groupby("release_year")["totalVotes"].aggregate('mean')
 df['meanTotalVotesByRating'] = df.groupby("rating")["totalVotes"].aggregate('mean')
 for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies'] :
df[col] = df[col].map(lambda x: sorted(list(set([n if n in train_dict[col] else col+'_etc' for n in [d['name'] for d in x]])))).map(lambda x: ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis = 1, inplace = True)
 df = df.drop(['id', 'revenue','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline', 'collection_id'
],axis=1)
 df.fillna(value=0.0, inplace = True)
 return df
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test['revenue'] = np.nan
train = pd.merge(train, pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 test = pd.merge(test, pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 additionalTrainData = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/additionalTrainData.csv')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].astype('str')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].str.replace('-', '/')
 train = pd.concat([train, additionalTrainData])
 print(train.columns)
 print(train.shape)
 train['revenue'] = np.log1p(train['revenue'])
 y = train['revenue'].values
json_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(json_cols + ['belongs_to_collection']):
train[col] = train[col].apply(lambda x : get_dictionary(x))
test[col] = test[col].apply(lambda x : get_dictionary(x))
def get_json_dict(df):
global json_cols
result = dict()
for e_col in json_cols :
d = dict()
rows = df[e_col].values
for row in rows :
if row is None : continue
for i in row :
if i['name'] not in d :
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols :
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]<choose_model_class>
!rm pseudo.csv
!rm -r pseudo
random_seed = 2019
k = 10
fold = list(KFold(k, shuffle = True, random_state = random_seed ).split(train))
np.random.seed(random_seed )<train_model>
!rm pseudo2.csv
!rm pseudo2
def xgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'subsample': 0.6,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': random_seed,
'silent': True,
}
record = dict()
model = xgb.train(params
, xgb.DMatrix(trn_x, trn_y)
 , 1000000
, [(xgb.DMatrix(trn_x, trn_y), 'train'),(xgb.DMatrix(val_x, val_y), 'valid')]
, verbose_eval=verbose
, early_stopping_rounds=500
, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid']['rmse'][best_idx], 'importance':[i for k, i in model.get_score().items() ]}<train_model>
fig, ax = plt.subplots(3, 3, figsize=(30, 30))
DIR_TEST = '.. /input/global-wheat-detection/test'
for i1 in range(3):
for j1 in range(3):
num = j1*3 + i1
img_pth = os.path.join(DIR_TEST, test_df.image_id[num]+'.jpg')
 image = cv2.imread(img_pth)
 b = [float(i)for i in test_df.PredictionString[num].split(' ')]
boxes = []
i = 0
while i < len(b):
boxes.append([int(b[i+1]),int(b[i+2]),int(b[i+3]),int(b[i+4]),float(b[i])])
 i+=5
boxes = np.array(boxes)
 for box in boxes.astype(float):
cv2.rectangle(image,(int(box[0]), int(box[1])) ,(int(box[0])+int(box[2]), int(box[1])+int(box[3])) ,(0, 32, 255), 3)
 cv2.putText(image, str(box[4]),(int(box[0]), int(box[1])+20), cv2.FONT_HERSHEY_SIMPLEX, 0.9,(36,255,12), 2)
 ax[i1][j1].imshow(image )