Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<choose_model_class>
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import cv2
import glob
import os
import sys
from tqdm import tqdm
from itertools import product
import torch
import albumentations as A
from albumentations.pytorch.transforms import ToTensorV2
from IPython.display import clear_output
catmodel = cat.CatBoostRegressor(iterations=10000,
learning_rate=0.01,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.8,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200,
random_seed=random_seed)
<train_model>
!pip install /wbf
!cd /cocoapi/PythonAPI/ && pip install -e.
 sys.path.append('/cocoapi/PythonAPI/')
 !pip install /addict
!pip install /mmcv
!pip install /terminaltables
!pip install -v -e /mmdetection
sys.path.append('/mmdetection' )
Kfolder.validate(train, test, features , catmodel, name="catfinal", prepare_stacking=True,
fit_params={"use_best_model": True, "verbose": 100} )<feature_engineering>
yolo_iou_threshold = 0.6
yolo_score_threshold = 0.25
yolov4_cfgfile = '.. /input/yolov4weights/yolov4-wheat.cfg'
yolov4_weightfile = '.. /input/yolov4weights/yolov4-wheat_40000.weights'
yolo_m = Darknet(yolov4_cfgfile)
 yolo_m.load_weights(yolov4_weightfile)
 yolo_m.cuda()
yolo_num_classes = yolo_m.num_classes
yolo_models = [yolo_m]
yolo_transform = A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def convert_image_to_yolo(img):
image = img.copy()
image = cv2.resize(image,(1024, 1024))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 sample = {'image': image}
sample = yolo_transform(**sample)
 image = sample['image']
return(image,)
 class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 def process_det(index, det, score_threshold=0.25):
scores = det[index][:, 5].copy()
det = det[index][:, :4].copy()
bboxes = np.zeros(( det.shape))
multiplier = 1024
bboxes[:, 0] =(( det[:, 0])* multiplier ).astype(int)
 bboxes[:, 1] =(( det[:, 1])* multiplier ).astype(int)
 bboxes[:, 2] =(det[:, 0] +(det[:, 2] * multiplier)).astype(int)
 bboxes[:, 3] =(det[:, 1] +(det[:, 3] * multiplier)).astype(int)
 bboxes =(bboxes ).clip(min = 0, max = multiplier-1 ).astype(int)
 indexes = np.where(scores>score_threshold)
 bboxes = bboxes[indexes]
scores = scores[indexes]
return bboxes, scores
def make_tta_predictions(yolo_model, image, score_threshold=0.25, iou_th=0.6):
predictions = np.array([])
 with torch.no_grad() :
images = torch.stack(image ).float().cuda()
for tta_transform in yolo_tta_transforms:
input_img = tta_transform.batch_augment(images.clone() ).permute(0,2,3,1 ).cpu().numpy()
det = do_detect(yolo_model, input_img, 0.0001, iou_th)
 det_new = []
for i_i in det:
if i_i is not None:
det_new.append(np.array(i_i))
else:
det_new.append(np.array([]))
det = det_new
if det[0].size !=0:
boxes, scores = process_det(0, det)
 if len(boxes)> 0:
boxes = tta_transform.deaugment_boxes(boxes.copy())
 boxes =(boxes ).round().astype(np.int32 ).clip(min=0, max=1023)
 if len(boxes)> 0:
predictions_curr = np.zeros(( len(boxes), 5))
predictions_curr[:,:4] = boxes
predictions_curr[:,4] = scores
if predictions.size==0:
predictions = predictions_curr
else:
predictions = np.concatenate(( predictions, predictions_curr), axis=0)
 return predictions
yolo_tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
yolo_tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
train['Revenue_Dragon1'] = 0.4 * train["lgbfinal"] + \
0.2 * train["xgbfinal"] + \
0.4 * train["catfinal"]<save_to_csv>
def format_prediction_string(boxes, scores):
pred_strings = []
for sc, bbox in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(sc, bbox[0], bbox[1], bbox[2], bbox[3]))
return " ".join(pred_strings)
 def predict(models,
test_files,
confidence_thresh=0.5,
nms_thresh = 0.5,
pseudolabelling_thresh = 0.8,
with_tta=False,
multiscale=False,
with_wbf=False,
wbf_weights=None,
clip_val=None,
with_yolo=True):
results = []
pseudo_labels = []
for img_path in test_files:
models_results = []
img = cv2.imread(img_path)
 im_w, im_h = img.shape[:2]
if multiscale:
resize_scales = [0.8, 0.7]
resized_imgs = []
for resize_scale in resize_scales:
img_resized = cv2.resize(img,(int(resize_scale*im_h), int(resize_scale*im_w)) , cv2.INTER_AREA)
 img_resized = cv2.resize(img_resized,(im_h, im_w), cv2.INTER_CUBIC)
 resized_imgs.append(img_resized)
 if with_yolo:
image_yolo = convert_image_to_yolo(img)
 for yolo_model in yolo_models:
yolo_predictions = make_tta_predictions(yolo_model, image_yolo, score_threshold=yolo_score_threshold, iou_th=yolo_iou_threshold)
 if yolo_predictions.size!=0:
models_results.append(yolo_predictions)
 clear_output(wait=True)
 for model_i, model in enumerate(models):
if with_tta:
wheat_result_i = detect_tta(model, img)
 if multiscale:
for resized_img in resized_imgs:
wheat_result_j = detect_tta(model, resized_img)
 wheat_result_i = np.concatenate(( wheat_result_i, wheat_result_j), axis=0)
 else:
res_values = inference_detector(model, img)
 wheat_result_i = res_values[0]
if model_i==0:
wheat_result = wheat_result_i
else:
wheat_result = np.concatenate(( wheat_result, wheat_result_i), axis=0)
 models_results.append(wheat_result_i)
 if with_wbf:
wbf_weights = [1] * len(models_results)
 bboxes_wbf = [model_result[:,:4]/1024 for model_result in models_results]
scores_wbf = [model_result[:,4] for model_result in models_results]
labels_wbf = [np.zeros_like(score_wbf)for score_wbf in scores_wbf]
wheat_bboxes, wheat_scores, _ = weighted_boxes_fusion(bboxes_wbf, scores_wbf, labels_wbf, weights=wbf_weights, iou_thr=nms_thresh, skip_box_thr=confidence_thresh, allows_overflow=False)
 wheat_bboxes *= 1024
else:
wheat_bboxes, wheat_scores = nms(wheat_result[:,:4], wheat_result[:,4], nms_thresh)
 res_scores = []
res_bboxes = []
image_id = img_path.split('/')[-1].replace('.jpg','')
 for bbox, score in zip(wheat_bboxes, wheat_scores):
if score>=confidence_thresh:
if clip_val:
if score>clip_val:
score = clip_val
res_scores.append(score)
 bbx_int = [int(bbx)for bbx in bbox]
bbx_int[2] = bbx_int[2] - bbx_int[0]
bbx_int[3] = bbx_int[3] - bbx_int[1]
res_bboxes.append(bbx_int)
 if score>=pseudolabelling_thresh:
pseudo_labels.append([image_id, 1024, 1024, f'[{bbx_int[0]}, {bbx_int[1]}, {bbx_int[2]}, {bbx_int[3]}]', 'nvnn'])
 result = {
'image_id': image_id,
'PredictionString': format_prediction_string(res_bboxes, res_scores)
 }
results.append(result)
 return results, pseudo_labels
def rot_bboxes_90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def rot_bboxes_90_n(boxes, im_w, im_h, n=1):
for _ in range(n):
boxes = rot_bboxes_90(boxes, im_w, im_h)
 return boxes
def flip_bboxes_lr(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1a, y1a, x2a, y2a = im_w-x2, y1, im_w-x1, y2
ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def flip_bboxes_ud(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1a, y1a, x2a, y2a = x1, im_h - y2, x2, im_h - y1
ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect_tta(model, image):
res_values = inference_detector(model, image)
 all_results = res_values[0]
image1 = image.copy()
im_w, im_h = image.shape[:2]
fliplr_image = cv2.flip(image1, 1)
 res_values = inference_detector(model, fliplr_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = flip_bboxes_lr(boxes, im_w, im_h)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 image1 = image.copy()
im_w, im_h = image.shape[:2]
flipud_image = cv2.flip(image1, 0)
 res_values = inference_detector(model, flipud_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = flip_bboxes_ud(boxes, im_w, im_h)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 image1 = image.copy()
im_w, im_h = image.shape[:2]
fliplr_image = cv2.flip(image1, 1)
 flipud_image = cv2.flip(fliplr_image, 0)
 res_values = inference_detector(model, flipud_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = flip_bboxes_ud(boxes, im_w, im_h)
 boxes = flip_bboxes_lr(boxes, im_w, im_h)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 image1 = image.copy()
im_w, im_h = image.shape[:2]
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 res_values = inference_detector(model, rotated_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = rot_bboxes_90_n(boxes, im_w, im_h, n=3)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 image1 = image.copy()
im_w, im_h = image.shape[:2]
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_COUNTERCLOCKWISE)
 res_values = inference_detector(model, rotated_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = rot_bboxes_90_n(boxes, im_w, im_h, n=1)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 image1 = image.copy()
im_w, im_h = image.shape[:2]
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rot_flip_ud_image = cv2.flip(rotated_image, 0)
 res_values = inference_detector(model, rot_flip_ud_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = flip_bboxes_ud(boxes, im_w, im_h)
 boxes = rot_bboxes_90_n(boxes, im_w, im_h, n=3)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 image1 = image.copy()
im_w, im_h = image.shape[:2]
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_COUNTERCLOCKWISE)
 rot_flip_ud_image = cv2.flip(rotated_image, 0)
 res_values = inference_detector(model, rot_flip_ud_image)
 bboxes_scores = res_values[0]
if bboxes_scores.size!=0:
boxes = bboxes_scores[:,:4]
boxes = flip_bboxes_ud(boxes, im_w, im_h)
 boxes = rot_bboxes_90_n(boxes, im_w, im_h, n=1)
 bboxes_scores[:,:4] = boxes
all_results = np.concatenate(( all_results, bboxes_scores), axis=0)
 del image1
return all_results
test['revenue'] = np.expm1(0.4* test["lgbfinal"]+ 0.4 * test["catfinal"] + 0.2 * test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_Dragon1.csv', index=False)
 test[['id','revenue']].head()<set_options>
MMDETECTION_PATH = '/mmdetection/'
TEST_IMAGES_DIR = '/kaggle/input/global-wheat-detection/test/'
test_files=glob.glob(f'{TEST_IMAGES_DIR}*.jpg' )
output_notebook() 
<load_from_csv>
CONFIGS_PATHS = ['/kaggle/input/glf-configs/',
'/kaggle/input/glf-configs/',
'/kaggle/input/glf-configs/']
WEIGHTS_PATHS = ['/kaggle/input/glf-weights/',
'/kaggle/input/glf-weights/',
'/kaggle/input/glf-weights/']
WEIGHTS_NAMES = ['m1_s2_epoch_24.pth',
'm2_s7_epoch_30_multiscale.pth',
'm2_s3_epoch_24.pth']
MODEL_NAMES = ['gfl_x101',
'gfl_x101',
'gfl_x101']
models = []
for CONFIGS_PATH, WEIGHTS_PATH, WEIGHTS_NAME, MODEL_NAME in zip(CONFIGS_PATHS, WEIGHTS_PATHS, WEIGHTS_NAMES, MODEL_NAMES):
config_file = os.path.join(CONFIGS_PATH, MODEL_NAME + '.py')
 checkpoint_file = os.path.join(WEIGHTS_PATH,WEIGHTS_NAME)
 models.append(init_detector(config_file, checkpoint_file, device='cuda:0'))
if len(models)==1:
wbf_weights = None
else:
wbf_weights = [1] *(len(models)+1 )
train_path = '.. /input/tmdb-box-office-prediction/train.csv'
test_path = '.. /input/tmdb-box-office-prediction/test.csv'
nominations_path='.. /input/nominations/nominations.csv'
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path)
 nominations=pd.read_csv(nominations_path)
<categorify>
confidence_thresh = 0.4
nms_thresh = 0.5
pseudolabelling_thresh = 0.4
with_tta = True
with_yolo = True
colons_in_Json = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew','belongs_to_collection']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in colons_in_Json :
train[col] = train[col].apply(lambda x : get_dictionary(x))
for col in colons_in_Json :
test[col] = test[col].apply(lambda x : get_dictionary(x)) 
<save_to_csv>
file_name = test_files[9]
image = cv2.imread(file_name)
 models_results = []
if with_yolo:
image_yolo = convert_image_to_yolo(image)
 yolo_predictions = make_tta_predictions(image_yolo, score_threshold=yolo_score_threshold, iou_th=yolo_iou_threshold)
 models_results.append(yolo_predictions)
 for model_i, model in enumerate(models):
if with_tta:
wheat_result_i = detect_tta(model, image)
 else:
res_values = inference_detector(model, image)
 wheat_result_i = res_values[0]
models_results.append(wheat_result_i)
 bboxes_wbf = [model_result[:,:4]/1024 for model_result in models_results]
scores_wbf = [model_result[:,4] for model_result in models_results]
labels_wbf = [np.zeros_like(score_wbf)for score_wbf in scores_wbf]
wheat_bboxes, wheat_scores, _ = weighted_boxes_fusion(bboxes_wbf, scores_wbf, labels_wbf, weights=wbf_weights, iou_thr=nms_thresh, skip_box_thr=confidence_thresh)
 print('Before WBF')
 print(f'scores: {scores_wbf}')
 print('After WBF')
 print(f'scores: {wheat_scores}')
 wheat_bboxes *= 1024
print(f'Before wbf {sum([len(model_result)for model_result in models_results])}')
 print(f'After wbf {len(wheat_bboxes)}')
 num_bboxes = 0
print(f'Before confidence thresholding {len(wheat_bboxes)}')
 for bbox, score in zip(wheat_bboxes, wheat_scores):
if score>=confidence_thresh:
bbx_int = [int(bbx)for bbx in bbox]
cv2.rectangle(image,(bbx_int[0],bbx_int[1]),(bbx_int[2],bbx_int[3]),(255,0,0), 2)
 num_bboxes+=1
print(f'After confidence thresholding {num_bboxes}')
 fig=plt.figure(figsize=(18, 16))
plt.imshow(image)
 plt.show()
%%script false
path = './train.csv'
data = pd.read_csv(path)
 cast_dict = data['cast'].apply(lambda x: ast.literal_eval(x)if pd.notnull(x)else {})
 cast_ids=[[y['id'] for y in x] for x in cast_dict]
unique_cast = pd.DataFrame(cast_ids ).stack().unique()
nominations=pd.DataFrame(columns=['id', 'nominations'])
 nominations['id'] = [x for x in unique_cast]
i=0
with open('nominations.csv', 'a')as csv:
for x in unique_cast[i:]:
start_time = time.time()
print(x)
 try:
with urlopen("https://www.themoviedb.org/person/"+str(x)) as page:
soup = BeautifulSoup(page, 'html.parser')
 name=soup.find('h2' ).text
name = unidecode.unidecode(name)
 name=name.replace(" ", "%20")
 print(name)
 url="https://sg.media-imdb.com/suggests/"+name[0].lower() +"/"+name.lower() +".json"
print(url)
 page = urlopen(url)
 soup = BeautifulSoup(page, 'html.parser')
 id=soup.text.find('"id":"nm')
 id=soup.text[id+6:id+15]
print(id)
 print(i)
 page = urlopen("https://www.imdb.com/name/"+id+"/awards?ref_=nm_ql_2")
 soup = BeautifulSoup(page, 'html.parser')
 a = 0
academy=0
golden=0
if soup.find_all('h3', string="Academy Awards, USA"):
soupCarotte = soup.find_all('table', attrs={'class': 'awards'})[a]
academy=len(soupCarotte.find_all('td', attrs={'class': 'award_outcome'}))
a+=1
if soup.find_all('h3', string="Golden Globes, USA"):
soupSoup = soup.find_all('table', attrs={'class': 'awards'})[a]
golden=len(soupSoup.find_all('td', attrs={'class': 'award_outcome'}))
nominations.loc[i,'nominations']=golden+academy
except urllib.error.HTTPError:
nominations.loc[i,'nominations']=0
print(nominations)
 output = pd.DataFrame({'id': nominations.loc[i, 'id'],
'nominations': nominations.loc[i, 'nominations']}, index=[0])
 output.to_csv(csv, header=False, index=False)
 i+=1
print(time.time() - start_time)
<feature_engineering>
results, pseudo_labels = predict(models,
test_files,
confidence_thresh=confidence_thresh,
nms_thresh = nms_thresh,
pseudolabelling_thresh = pseudolabelling_thresh,
with_tta=with_tta,
with_wbf=True,
wbf_weights=wbf_weights,
with_yolo=with_yolo )
%%script false
warnings.filterwarnings("ignore")
 path = './train.csv'
data = pd.read_csv(path)
 print(data['belongs_to_collection'].iloc[0])
 colons_in_Json = ['belongs_to_collection']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(colons_in_Json):
data[col] = data[col].apply(lambda x : get_dictionary(x))
google_result=data[['id','title','belongs_to_collection']]
google_result['google_result']=""
movie_title=google_result['title']
list_movie_title=list(movie_title)
 google_result['film_belongs_to_collection'] = google_result['belongs_to_collection'].apply(lambda x: 0 if x == {} else 1)
 word_movie=' movie'
with open('google_result.csv', 'a')as csv:
for index, r in google_result.iloc[0:].iterrows() :
if google_result.iloc[index]['film_belongs_to_collection']==0:
search=google_result['title'].iloc[index]
search=search+word_movie
else:
words = ['(Theatrical)','(1958 series)','(Series)','- Коллекция','Collection','(Animation)','(Universal Series)','(Heisei)',': The Original Series','- Collezione','(Original)','(Hammer Series)','(Remake)','(Reboot)','(Series)','Trilogy','(1976 series)','(Original Series)','(Universal Series)','Anthology','(Universal)','() ','The Klapisch ','(Коллекция)']
search = []
belongs_to_collection_line=google_result.iloc[index]['belongs_to_collection']
collection_name = belongs_to_collection_line[0]['name']
for w in words:
collection_name = collection_name.replace(w, '')
 search.append(collection_name)
 search=(search[0])
 search=search+word_movie
try:
print(search)
 r = requests.get("https://www.google.com/search", params={'q':search})
 soup = BeautifulSoup(r.text, "lxml")
 res = soup.find("div", {"id": "resultStats"})
 nb_result = ''.join(x for x in res.text if x.isdigit())
 print(nb_result)
 google_result.loc[index, 'google_result']=nb_result
except urllib.error.HTTPError:
google_result.loc[index,'google_result']=0
output = pd.DataFrame({'id': google_result.loc[index, 'title'],
'google_result': google_result.loc[index, 'google_result']}, index=[0])
 output.to_csv(csv, header=False, index=False)
<feature_engineering>
PSEUDOLABEL = False
t= train[['id','title','runtime','revenue']].copy()
t.iloc[1335]=t.iloc[1335].replace(np.nan, int(120))
t.iloc[2302]=t.iloc[2302].replace(np.nan, int(90))
t['runtime_cat_min_60'] = t['runtime'].apply(lambda x: 1 if(x <=60)else 0)
 t['runtime_cat_61_80'] = t['runtime'].apply(lambda x: 1 if(x >60)&(x<=80)else 0)
 t['runtime_cat_81_100'] = t['runtime'].apply(lambda x: 1 if(x >80)&(x<=100)else 0)
 t['runtime_cat_101_120'] = t['runtime'].apply(lambda x: 1 if(x >100)&(x<=120)else 0)
 t['runtime_cat_121_140'] = t['runtime'].apply(lambda x: 1 if(x >120)&(x<=140)else 0)
 t['runtime_cat_141_170'] = t['runtime'].apply(lambda x: 1 if(x >140)&(x<=170)else 0)
 t['runtime_cat_171_max'] = t['runtime'].apply(lambda x: 1 if(x >=170)else 0)
 t.loc[t.runtime_cat_min_60 == 1,'runtime_category'] = 'cat_min-60'
t.loc[t.runtime_cat_61_80 == 1,'runtime_category'] = 'cat_61-80'
t.loc[t.runtime_cat_81_100 == 1,'runtime_category'] = 'cat_81-100'
t.loc[t.runtime_cat_101_120 == 1,'runtime_category'] = 'cat_101-120'
t.loc[t.runtime_cat_121_140 == 1,'runtime_category'] = 'cat_121-140'
t.loc[t.runtime_cat_141_170 == 1,'runtime_category'] = 'cat_141-170'
t.loc[t.runtime_cat_171_max == 1,'runtime_category'] = 'cat_171-max'
cat = t['runtime_category']
ctr = Counter(cat)
 cat = [x for x in ctr]
unique_names = pd.Series(cat ).unique()
dic={}
for a in unique_names:
mask = t.runtime_category.apply(lambda x: a in x)
 dic[a] = t[mask]['revenue'].mean()
t = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'runtime_cat'})
 t = t.nlargest(6, 'mean_revenue')
 t['color'] = Category20c[6]
hover1 = HoverTool(tooltips = [ 
('Runtime_category','@runtime_cat'), 
('Revenue','@mean_revenue')
 ])
 p = figure(x_range=t.runtime_cat, plot_width=800,plot_height=400, toolbar_location=None, title="Revenue per runtime category", tools=[hover1])
 p.vbar(x='runtime_cat', top='mean_revenue', width=0.9, source=t, legend='runtime_cat',
line_color='white',fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p )<feature_engineering>
if PSEUDOLABEL:
!mkdir -p /pseudo/images
!cp /kaggle/input/global-wheat-detection/train/* /pseudo/images
!cp /kaggle/input/global-wheat-detection/test/* /pseudo/images
!ls -l /pseudo/images | wc -l
train_df = pd.read_csv('/kaggle/input/global-wheat-detection/train.csv')
 pseudolabels_file_name = '/pseudo/train_with_pseudo.csv'
pseudolabels_df = pd.DataFrame(pseudo_labels, columns =['image_id', 'width', 'height', 'bbox', 'source'])
 total_df = pd.concat([pseudolabels_df,train_df], ignore_index=True)
 total_df.to_csv(pseudolabels_file_name, index=False)
 total_df.head()
train['runtime_cat_min_60'] = train['runtime'].apply(lambda x: 1 if(x <=60)else 0)
 train['runtime_cat_61_80'] = train['runtime'].apply(lambda x: 1 if(x >60)&(x<=80)else 0)
 train['runtime_cat_81_100'] = train['runtime'].apply(lambda x: 1 if(x >80)&(x<=100)else 0)
 train['runtime_cat_101_120'] = train['runtime'].apply(lambda x: 1 if(x >100)&(x<=120)else 0)
 train['runtime_cat_121_140'] = train['runtime'].apply(lambda x: 1 if(x >120)&(x<=140)else 0)
 train['runtime_cat_141_170'] = train['runtime'].apply(lambda x: 1 if(x >140)&(x<=170)else 0)
 train['runtime_cat_171_max'] = train['runtime'].apply(lambda x: 1 if(x >=170)else 0)
 test['runtime_cat_min_60'] = test['runtime'].apply(lambda x: 1 if(x <=60)else 0)
 test['runtime_cat_61_80'] = test['runtime'].apply(lambda x: 1 if(x >60)&(x<=80)else 0)
 test['runtime_cat_81_100'] = test['runtime'].apply(lambda x: 1 if(x >80)&(x<=100)else 0)
 test['runtime_cat_101_120'] = test['runtime'].apply(lambda x: 1 if(x >100)&(x<=120)else 0)
 test['runtime_cat_121_140'] = test['runtime'].apply(lambda x: 1 if(x >120)&(x<=140)else 0)
 test['runtime_cat_141_170'] = test['runtime'].apply(lambda x: 1 if(x >140)&(x<=170)else 0)
 test['runtime_cat_171_max'] = test['runtime'].apply(lambda x: 1 if(x >=170)else 0 )<feature_engineering>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()
train['budget'] = np.log1p(train.budget)
 test['budget'] = np.log1p(test.budget )<feature_engineering>
test_df.to_csv('/kaggle/working/submission.csv', index=False )