Kaggle competition: APTOS 2019 Blindness Detection
df.dropna(how='any', axis='rows', inplace=True)
 mask = df['pickup_longitude'].between(-75, -73)
 mask &= df['dropoff_longitude'].between(-75, -73)
 mask &= df['pickup_latitude'].between(40, 42)
 mask &= df['dropoff_latitude'].between(40, 42)
 mask &= df['passenger_count'].between(0, 8)
 mask &= df['fare_amount'].between(0, 250)
 df = df[mask]<compute_test_metric>
from __future__ import print_function, absolute_import
import os
import sys
import time
import datetime
import argparse
import os.path as osp
import numpy as np
import random
from PIL import Image
import tqdm
import cv2
import csv
import math
import torchvision as tv
import torchvision
import torch.nn.functional as F
import torch.optim as optim
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from sklearn.metrics import f1_score
from torch.utils.data import DataLoader
from torch.autograd import Variable
from torch.optim import lr_scheduler
from tqdm import tqdm
from torch.utils.data import Dataset
import torchvision.transforms as transforms
from tensorboardX import SummaryWriter
def dist(pickup_lat, pickup_long, dropoff_lat, dropoff_long):
distance = np.abs(dropoff_lat - pickup_lat)+ np.abs(dropoff_long - pickup_long)
 return distance<feature_engineering>
name_file='.. /input/aptos2019-blindness-detection/test.csv'
csv_file=csv.reader(open(name_file,'r'))
content=[]
for line in csv_file:
content.append(line[0]+'.png')
 content=content[1:]
def transform(data):
data['hour'] = data['pickup_datetime'].dt.hour
data['day'] = data['pickup_datetime'].dt.day
data['month'] = data['pickup_datetime'].dt.month
data['year'] = data['pickup_datetime'].dt.year
data = data.drop('pickup_datetime', axis=1)
 nyc =(-74.0063889, 40.7141667)
 jfk =(-73.7822222222, 40.6441666667)
 ewr =(-74.175, 40.69)
 lgr =(-73.87, 40.77)
 data['distance_to_center'] = dist(nyc[1], nyc[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['pickup_distance_to_jfk'] = dist(jfk[1], jfk[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_jfk'] = dist(jfk[1], jfk[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_ewr'] = dist(ewr[1], ewr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_ewr'] = dist(ewr[1], ewr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_lgr'] = dist(lgr[1], lgr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_lgr'] = dist(lgr[1], lgr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['long_dist'] = data['pickup_longitude'] - data['dropoff_longitude']
data['lat_dist'] = data['pickup_latitude'] - data['dropoff_latitude']
data['dist'] = dist(data['pickup_latitude'], data['pickup_longitude'],
data['dropoff_latitude'], data['dropoff_longitude'])
 return data
df = transform(df )<import_modules>
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM,self ).__init__()
self.p = Parameter(torch.ones(1)*p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(self.eps)+ ')'
class Baseline_single_gem(nn.Module):
def __init__(self, num_classes, loss_type="single BCE", **kwargs):
super(Baseline_single_gem, self ).__init__()
self.loss_type = loss_type
resnet50 = torchvision.models.densenet201(pretrained=False)
 self.base = nn.Sequential(*list(resnet50.children())[:-1])
 self.feature_dim = 1920
if self.loss_type == "single BCE":
self.ap = GeM()
self.classifiers = nn.Linear(in_features=self.feature_dim, out_features=num_classes)
 self.sigmoid = nn.Sigmoid()
self.dropout=nn.Dropout(0.5)
 self.cal_score=nn.Linear(in_features=num_classes, out_features=1)
 def freeze_base(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze_all(self):
for p in self.parameters() :
p.requires_grad = True
def forward(self, x1):
x = self.base(x1)
 map_feature=torch.tensor(x)
 if self.loss_type == "single BCE":
x = self.ap(x)
 x = self.dropout(x)
 x = x.view(x.size(0), -1)
 feat_m=torch.tensor(x)
 ys = self.classifiers(x)
 return ys
class Baseline_single(nn.Module):
def __init__(self, num_classes, loss_type="single BCE", **kwargs):
super(Baseline_single, self ).__init__()
self.loss_type = loss_type
resnet50 = torchvision.models.densenet201(pretrained=False)
 self.base = nn.Sequential(*list(resnet50.children())[:-1])
 self.feature_dim = 1920
if self.loss_type == "single BCE":
self.ap = nn.AdaptiveAvgPool2d(1)
 self.classifiers = nn.Linear(in_features=self.feature_dim, out_features=num_classes)
 self.sigmoid = nn.Sigmoid()
self.dropout=nn.Dropout(0.5)
 self.cal_score=nn.Linear(in_features=num_classes, out_features=1)
 def freeze_base(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze_all(self):
for p in self.parameters() :
p.requires_grad = True
def forward(self, x1):
x = self.base(x1)
 map_feature=torch.tensor(x)
 if self.loss_type == "single BCE":
x = self.ap(x)
 x = self.dropout(x)
 x = x.view(x.size(0), -1)
 feat_m=torch.tensor(x)
 ys = self.classifiers(x)
 return ys
import xgboost as xgb
from bayes_opt import BayesianOptimization
from sklearn.metrics import mean_squared_error<split>
def cv_imread(file_path):
	cv_img=cv2.imdecode(np.fromfile(file_path,dtype=np.uint8),-1)
 	return cv_img
def change_size(image):
	
	b=cv2.threshold(image,15,255,cv2.THRESH_BINARY)
 	binary_image=b[1]
	binary_image=cv2.cvtColor(binary_image,cv2.COLOR_BGR2GRAY)
 	print(binary_image.shape)
 	x=binary_image.shape[0]
	print("高度x=",x)
 	y=binary_image.shape[1]
	print("宽度y=",y)
 	edges_x=[]
	edges_y=[]
	for i in range(x):
		for j in range(y):
			if binary_image[i][j]==255:
			
			
			 edges_x.append(i)
 			 edges_y.append(j)
 	left=min(edges_x)
 	right=max(edges_x)
 	width=right-left
	bottom=min(edges_y)
 	top=max(edges_y)
 	height=top-bottom
	pre1_picture=image[left:left+width,bottom:bottom+height]
	return pre1_picture
def crop_image1(img,tol=7):
	
	
		
	mask = img>tol
	return img[np.ix_(mask.any(1),mask.any(0)) ]
def crop_image_from_gray(img,tol=7):
	if img.ndim ==2:
		mask = img>tol
		return img[np.ix_(mask.any(1),mask.any(0)) ]
	elif img.ndim==3:
		gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 		mask = gray_img>tol
		
		check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0)) ].shape[0]
		if(check_shape == 0):
			return img
		else:
			img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0)) ]
			img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0)) ]
			img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0)) ]
	
			img = np.stack([img1,img2,img3],axis=-1)
 	
		return img
def load_ben_color(image, sigmaX=10):
	image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 	image = crop_image_from_gray(image)
 	image = cv2.resize(image,(492, 492))
	image=cv2.addWeighted(image,4, cv2.GaussianBlur(image ,(0,0), sigmaX),-4 ,128)
 		
	return image
def findCircle(image):
	
	hsv_img=cv2.cvtColor(image,cv2.COLOR_BGR2HSV)
 	h_img=image[:,:,0]
	s_img=image[:,:,1]
	v_img=image[:,:,2]
	height,width=v_img.shape
	
	mask_v_a=cv2.adaptiveThreshold(v_img,255,cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY_INV,int(max(height,width)/16)*2+1,1)
 	
	
	
	ratio=128/min(height,width)
 	msk=cv2.resize(mask_v_a,(int(width*ratio),int(height*ratio)) ,interpolation=cv2.INTER_CUBIC)
 	h,w=msk.shape
	msk_expand=np.zeros(( 3*h,3*w),np.uint8)
 	msk_expand[h:2*h,w:2*w]=msk
	
	
	long_edge=max(h,w)
 	r0=round(0.3*long_edge)
 	r1=round(0.7*long_edge)
 	
	circles=cv2.HoughCircles(msk_expand,cv2.HOUGH_GRADIENT,1,90,param1 = 50,param2 = 5,minRadius = r0,maxRadius = r1)
 	
	if circles is None:
		c_x=width/2
		c_y=height/2
		radius=0.55*max(height,width)
 	else:	
		
		circles = np.uint16(np.around(circles))
		c_x=(circles[0,0,0]-w)/ratio
		c_y=(circles[0,0,1]-h)/ratio
		radius=circles[0,0,2]/ratio
	
	return c_x,c_y,radius
def circleCrop(c_x,c_y,radius,height,width):
	if math.floor(radius+c_y)>height:
		y0=max(math.ceil(c_y-radius),0);
		y1=height;
		if math.floor(radius+c_x)>width:
			x1=width
		else:
			x1=math.floor(radius+c_x)
 		if math.floor(c_x-radius<0):
			x0=0
		else:
			x0=math.floor(c_x-radius)
 	elif math.ceil(c_y-radius)<0:
		y0=0
		y1=min(math.floor(c_y+radius),height)
 		if math.floor(radius+c_x)>width:
			x1=width
		else:
			x1=math.floor(radius+c_x)
 		if math.floor(c_x-radius<0):
			x0=0
		else:
			x0=math.floor(c_x-radius)
 	else:
		y0=math.ceil(c_y-radius)
 		y1=math.floor(c_y+radius)
 		x0=math.ceil(c_x-radius)
 		x1=math.floor(c_x+radius)
 	return x0,x1,y0,y1
			
def trimFundus(image):
	c_x,c_y,radius=findCircle(image)
 	height=image.shape[0]
	width=image.shape[1]
	x0,x1,y0,y1=circleCrop(c_x,c_y,radius,height,width)
 	
	trimed=image[y0:y1,x0:x1,:]
	return trimed
def load_ben_yuan(image,sigmaX=10):
	image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 	image = crop_image_from_gray(image)
 	image = cv2.resize(image,(512, 512))
	return image
PARAM = 92
def Radius_Reduction(img,PARAM):
h,w,c=img.shape
Frame=np.zeros(( h,w,c),dtype=np.uint8)
 cv2.circle(Frame,(int(math.floor(w/2)) ,int(math.floor(h/2))),int(math.floor(( h*PARAM)/float(2*100))),(255,255,255), -1)
 Frame1=cv2.cvtColor(Frame, cv2.COLOR_BGR2GRAY)
 img1 =cv2.bitwise_and(img,img,mask=Frame1)
 return img1
def info_image(im):
cy = im.shape[0]//2
midline = im[cy,:]
midline = np.where(midline>midline.mean() /3)[0]
if len(midline)>im.shape[1]//2:
x_start, x_end = np.min(midline), np.max(midline)
 else:
x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10
cx =(x_start + x_end)/2
r =(x_end - x_start)/2
return cx, cy, r
def resize_image(im, img_size, augmentation=False):
cx, cy, r = info_image(im)
 scaling = img_size/(2*r)
 rotation = 0
if augmentation:
scaling *= 1 + 0.3 *(np.random.rand() -0.5)
 rotation = 360 * np.random.rand()
M = cv2.getRotationMatrix2D(( cx,cy), rotation, scaling)
 M[0,2] -= cx - img_size/2
M[1,2] -= cy - img_size/2
return cv2.warpAffine(im, M,(img_size, img_size))
def subtract_median_bg_image(im):
k = np.max(im.shape)//20*2+1
bg = cv2.medianBlur(im, k)
 return cv2.addWeighted(im, 4, bg, -4, 128)
 def subtract_gaussian_bg_image(im):
bg = cv2.GaussianBlur(im ,(0,0), 10)
 return cv2.addWeighted(im, 4, bg, -4, 128)
 def open_img(fn, size):
"Open image in `fn`, subclass and overwrite for custom behavior."
image = cv2.imread(fn)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = resize_image(image, size)
 image = subtract_gaussian_bg_image(image)
 image = Radius_Reduction(image, PARAM)
 image = crop_image_from_gray(image)
 image = cv2.resize(image,(512,512))
return image
def get_preds(arr):
mask = arr == 0
return np.clip(np.where(mask.any(1), mask.argmax(1), 5)- 1, 0, 4)
 cnt_t=0
class eye_dataset(Dataset):
	
	def __init__(self, txt_path,transform=None,transform2=None):
		imgs = []
		for img in txt_path:
			imgs.append(img)
 		self.imgs = imgs
		self.transform = transform
		self.transform2 = transform2
	def __getitem__(self, index):
		fn= self.imgs[index]
		
		
		
		img=cv2.imread('/kaggle/input/aptos2019-blindness-detection/test_images/'+fn)
 		
		img_copy=img.copy()
		img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 		img_copy=img.copy()
		try:
			img=trimFundus(img)
 			img = cv2.resize(img,(512, 512))
		except Exception as e:
			
			print(e)
 			img = crop_image_from_gray(img_copy)
 			img = cv2.resize(img,(512, 512))
		
		
		img = Image.fromarray(img)
 		
		
		if self.transform is not None:
			img1 = self.transform(img)
 			img2 = self.transform2(img)
 		
		return img1,img2, fn[:-4]
	def __len__(self):
		return len(self.imgs)
 class eye_dataset_circle(Dataset):
	
	def __init__(self, txt_path,transform=None):
		imgs = []
		for img in txt_path:
			imgs.append(img)
 		self.imgs = imgs
		self.transform = transform
	def __getitem__(self, index):
		fn= self.imgs[index]
		
		
		
		
		img=open_img('/kaggle/input/aptos2019-blindness-detection/test_images/'+fn,530)
 		img = Image.fromarray(img)
 		
		
		if self.transform is not None:
			img = self.transform(img)
 		
		
		return img, fn[:-4]
	def __len__(self):
		return len(self.imgs)
 class eye_dataset_orl(Dataset):
	
	def __init__(self, txt_path,transform=None,transform2=None,transform3=None,transform4=None):
		imgs = []
		for img in txt_path:
			imgs.append(img)
 		self.imgs = imgs
		self.transform = transform
		self.transform2 = transform2
		self.transform3 = transform3
		self.transform4 = transform4
	def __getitem__(self, index):
		fn= self.imgs[index]
		
		
		
		img=cv2.imread('/kaggle/input/aptos2019-blindness-detection/test_images/'+fn)
 		img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 		img=crop_image_from_gray(img)
 		img3_=open_img('/kaggle/input/aptos2019-blindness-detection/test_images/'+fn,530)
 		img4_=open_img('/kaggle/input/aptos2019-blindness-detection/test_images/'+fn,530)
 		img3_ = Image.fromarray(img3_)
 		img4_ = Image.fromarray(img4_)
 		img=cv2.resize(img,(512, 512))
		img = Image.fromarray(img)
 		
		
		if self.transform is not None:
			img1 = self.transform(img)
 			img2 = self.transform2(img)
 			img3 = self.transform3(img3_)
 			img4 = self.transform4(img4_)
 			
		
		
		return img1,img2,img3,img4, fn[:-4]
	def __len__(self):
		return len(self.imgs )
X_train, X_test, y_train, y_test = train_test_split(df.drop('fare_amount', axis=1),
df['fare_amount'], test_size=0.25)
 del(df)
 dtrain = xgb.DMatrix(X_train, label=y_train)
 del(X_train)
 dtest = xgb.DMatrix(X_test)
 del(X_test )<compute_train_metric>
def load_para_dict(model1):
state_dict_1=torch.load(model1)
 new_state_dict = OrderedDict()
for k, v in state_dict_1.items() :
if 'module' in k:
name = k[7:]
else:
name=k
new_state_dict[name] = v
return new_state_dict