Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<load_from_csv>
%matplotlib inline
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 by_sex_class_test = test_data.groupby(['Sex', 'Pclass'])
 test_data["Fare"].fillna(test_data.Fare.mean() ,inplace=True)
 test_data["Embarked"].fillna(test_data.Embarked.mode() ,inplace=True)
 test_data["Age"].fillna(by_sex_class_test['Age'].transform(impute_median),inplace=True)
<define_variables>
isascii = lambda s: len(s)== len(s.encode())
 tknzr = TweetTokenizer()
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english" )
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )<define_variables>
from torch import nn
import torch
from torch.nn import functional as F
from torchvision.models import resnet50, resnet34, densenet201, densenet121
from torch.optim.lr_scheduler import StepLR
from torch.autograd import Variable
from torch.utils.data import TensorDataset
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )<save_to_csv>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
min_rating, max_rating =None, None
rater_a, rater_b = np.array(y, dtype=int), np.array(y_pred, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b, min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator, denominator = 0.0, 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j] / num_scored_items)
 d = np.square(i - j)/ np.square(num_ratings - 1)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 def val_kappa(preds, train_data):
labels = train_data.get_label()
preds = np.argmax(preds.reshape(( -1,5)) , axis=1)
 return 'qwk', quadratic_weighted_kappa(labels, preds), True
def val_kappa_reg(preds, train_data, cdf):
labels = train_data.get_label()
preds = getTestScore2(preds, cdf)
 return 'qwk', quadratic_weighted_kappa(labels, preds), True
def get_cdf(hist):
return np.cumsum(hist/np.sum(hist))
def getScore(pred, cdf, valid=False):
num = pred.shape[0]
output = np.asarray([4]*num, dtype=int)
 rank = pred.argsort()
output[rank[:int(num*cdf[0]-1)]] = 0
output[rank[int(num*cdf[0]):int(num*cdf[1]-1)]] = 1
output[rank[int(num*cdf[1]):int(num*cdf[2]-1)]] = 2
output[rank[int(num*cdf[2]):int(num*cdf[3]-1)]] = 3
if valid:
cutoff = [ pred[rank[int(num*cdf[i]-1)]] for i in range(4)]
return output, cutoff
return output
def getTestScore(pred, cutoff):
num = pred.shape[0]
output = np.asarray([4]*num, dtype=int)
 for i in range(num):
if pred[i] <= cutoff[0]:
output[i] = 0
elif pred[i] <= cutoff[1]:
output[i] = 1
elif pred[i] <= cutoff[2]:
output[i] = 2
elif pred[i] <= cutoff[3]:
output[i] = 3
return output
def getTestScore2(pred, cdf):
num = pred.shape[0]
rank = pred.argsort()
output = np.asarray([4]*num, dtype=int)
 output[rank[:int(num*cdf[0]-1)]] = 0
output[rank[int(num*cdf[0]):int(num*cdf[1]-1)]] = 1
output[rank[int(num*cdf[1]):int(num*cdf[2]-1)]] = 2
output[rank[int(num*cdf[2]):int(num*cdf[3]-1)]] = 3
return output
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
isascii = lambda s: len(s)== len(s.encode())
 def custom_tokenizer(text):
init_doc = tknzr.tokenize(text)
 retval = []
for t in init_doc:
if isascii(t):
retval.append(t)
 else:
for w in t:
retval.append(w)
 return retval
def build_emb_matrix(word_dict, emb_dict):
embed_size = 300
nb_words = len(word_dict)+1000
nb_oov = 0
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 for key in tqdm(word_dict):
word = key
embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = emb_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
nb_oov+=1
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words, nb_oov
def _init_esim_weights(module):
if isinstance(module, nn.Linear):
nn.init.kaiming_normal_(module.weight.data)
 nn.init.constant_(module.bias.data, 0.0)
 elif isinstance(module, nn.LSTM)or isinstance(module, nn.GRU):
if isinstance(module, nn.LSTM):
hidden_size = module.bias_hh_l0.data.shape[0] // 4
else:
hidden_size = module.bias_hh_l0.data.shape[0] // 3
for name, param in module.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif '_ih_' in name:
nn.init.xavier_normal_(param)
 elif '_hh_' in name:
nn.init.orthogonal_(param)
 param.data[hidden_size:(2 * hidden_size)] = 1.0
y = train_data["Survived"]
features = ["Pclass", "Sex", "SibSp", "Parch","Age", "Embarked"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission_14.csv', index=False)
 print("Your submission 14 was successfully saved!" )<import_modules>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
import numpy as np
import pandas as pd
import os
import json
from pathlib import Path
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np<define_variables>
class ResnetModel(nn.Module):
def __init__(self, resnet_fun = resnet50, freeze_basenet = True):
super(ResnetModel, self ).__init__()
self.resnet = resnet_fun(pretrained=False)
 if freeze_basenet:
for p in self.resnet.parameters() :
p.requires_grad = False
def init_resnet(self, path):
state = torch.load(path)
 self.resnet.load_state_dict(state)
 def forward(self, x):
batch_size = x.shape[0]
x = x/255.0
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
x = torch.cat([ 
(x[:, [0]] - mean[0])/ std[0], 
(x[:, [1]] - mean[1])/ std[1], 
(x[:, [2]] - mean[2])/ std[2],
], 1)
 x = self.resnet.conv1(x)
 x = self.resnet.bn1(x)
 x = self.resnet.relu(x)
 x = self.resnet.maxpool(x)
 x = self.resnet.layer1(x)
 x = self.resnet.layer2(x)
 x = self.resnet.layer3(x)
 x = self.resnet.layer4(x)
 x = F.adaptive_avg_pool2d(x, output_size=1 ).view(batch_size, -1)
 return x
class DenseModel(nn.Module):
def __init__(self, dense_func = densenet201, freeze_basenet = True):
super(DenseModel, self ).__init__()
self.densenet = dense_func(pretrained=False)
 if freeze_basenet:
for p in self.densenet.parameters() :
p.requires_grad = False
def init_densenet(self, path):
state = torch.load(path)
 self.densenet.load_state_dict(state)
 def forward(self, x):
batch_size = x.shape[0]
x = x/255.0
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
x = torch.cat([ 
(x[:, [0]] - mean[0])/ std[0], 
(x[:, [1]] - mean[1])/ std[1], 
(x[:, [2]] - mean[2])/ std[2],
], 1)
 x = self.densenet.features(x)
 x = F.adaptive_avg_pool2d(F.relu(x,inplace=True), output_size=1 ).view(batch_size, -1)
 return x
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
train_tasks = sorted(os.listdir(training_path))
eval_tasks = sorted(os.listdir(evaluation_path))<load_from_disk>
labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
train_file = str(training_path / train_tasks[15])
 with open(train_file, 'r')as f:
task = json.load(f )<categorify>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv')
 train_len = len(train)
 data_df = pd.concat([train, test], sort=False ).reset_index(drop=True)
 data_df['Breed_full'] = data_df['Breed1'].astype(str)+'_'+data_df['Breed2'].astype(str)
 data_df['Color_full'] = data_df['Color1'].astype(str)+'_'+data_df['Color2'].astype(str)+'_'+data_df['Color3'].astype(str)
 data_df['Breed_full'],_ = pd.factorize(data_df['Breed_full'])
 data_df['Color_full'],_ = pd.factorize(data_df['Color_full'])
 data_df['State'],_ = pd.factorize(data_df['State'])
 data_df['hard_interaction'] = data_df['Type'].astype(str)+data_df['Gender'].astype(str)+ \
data_df['Vaccinated'].astype(str)+'_'+ \
data_df['Dewormed'].astype(str)+'_'+data_df['Sterilized'].astype(str)
 data_df['hard_interaction'],_ = pd.factorize(data_df['hard_interaction'])
 data_df['MaturitySize'] = data_df['MaturitySize'].replace(0, np.nan)
 data_df['FurLength'] = data_df['FurLength'].replace(0, np.nan)
 data_df['Vaccinated'] = data_df['Vaccinated'].replace(3, np.nan)
 data_df['Vaccinated'] = data_df['Vaccinated'].replace(2, 0)
 data_df['Dewormed'] = data_df['Dewormed'].replace(3, np.nan)
 data_df['Dewormed'] = data_df['Dewormed'].replace(2, 0)
 data_df['Sterilized'] = data_df['Sterilized'].replace(3, np.nan)
 data_df['Sterilized'] = data_df['Sterilized'].replace(2, 0)
 data_df['Health'] = data_df['Health'].replace(0, np.nan)
 data_df['age_in_year'] =(data_df['Age']//12 ).astype(np.int8)
 data_df['avg_fee'] = data_df['Fee']/data_df['Quantity']
data_df['avg_photo'] = data_df['PhotoAmt']/data_df['Quantity']
pattern = re.compile(r"[0-9\.:!]")
 data_df['empty_name'] = data_df['Name'].isnull().astype(np.int8)
 data_df['Name'] =data_df['Name'].fillna('')
 data_df['name_len'] = data_df['Name'].apply(lambda x: len(x))
data_df['strange_name'] = data_df['Name'].apply(lambda x: len(pattern.findall(x)) >0 ).astype(np.int8 )
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<compute_test_metric>
data_df['color_num'] = 1
data_df['color_num'] += data_df['Color2'].apply(lambda x: 1 if x!=0 else 0)
 data_df['color_num'] += data_df['Color3'].apply(lambda x: 1 if x!=0 else 0 )
def find_count_tow_list(a,b):
count = 0
for i,j in enumerate(a):
if j == b[i]:
count+=1
return count<define_variables>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_breed.rename(index=str, columns={'BreedName':'Breed1Name'},inplace=True)
 labels_breed['Breed2Name'] = labels_breed['Breed1Name'].values
replace_color = {}
for k in range(len(task['train'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
data = [[replace_color[i] for i in j] for j in task['test'][0]['input']]<define_variables>
data_df = data_df.merge(labels_breed[['BreedID','Breed1Name']], left_on='Breed1', right_on='BreedID', how='left')
 data_df.drop('BreedID',axis=1,inplace=True)
 data_df = data_df.merge(labels_breed[['BreedID','Breed2Name']], left_on='Breed2', right_on='BreedID', how='left')
 data_df.drop('BreedID',axis=1,inplace=True)
 data_df['Breed2Name'].fillna('',inplace=True)
 data_df['BreedName_full'] = data_df['Breed1Name']+' '+data_df['Breed2Name']
train_correct = len(train_tasks)
 for i in range(len(train_tasks)) :
train_file = str(training_path / train_tasks[i])
 with open(train_file, 'r')as f:
task = json.load(f)
 replace_color = {}
for k in range(len(task['train'])) :
if len(flattener(task['train'][k]['output'])) == len(flattener(task['train'][k]['input'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
else:
break
data = [[replace_color[i] if i in replace_color else i for i in j] for j in task['test'][0]['input']]
pred_1 = flattener(data)
 if pred_1 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 train_correct-=1
for j in task['test'][0]['input']:
if not np.any(j==0):
temp_data = j
data = [temp_data for j in range(len(task['test'][0]['input'])) ]
pred_2 = flattener(data)
 if pred_2 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 train_correct-=1<train_model>
data_df['breed_noname'] = data_df['BreedName_full'].isnull().astype(np.int8)
 data_df['BreedName_full'].fillna('',inplace=True)
 data_df['BreedName_full'] = data_df['BreedName_full'].str.lower()
print("Training set error rate ==> {}, detect tasks {} / {}".format(train_correct/len(train_tasks),len(train_tasks)- train_correct,len(train_tasks)) )<define_variables>
data_df['breed_num'] = 1
data_df['breed_num'] += data_df['Breed2'].apply(lambda x: 1 if x!=0 else 0)
 data_df['breed_mixed'] = data_df['BreedName_full'].apply(lambda x: x.find('mixed')>=0 ).astype(np.int8)
 data_df['breed_Domestic'] = data_df['BreedName_full'].apply(lambda x: x.find('domestic')>=0 ).astype(np.int8)
 data_df['pure_breed'] =(( data_df['breed_num']==1)&(data_df['breed_mixed']==0)).astype(np.int8 )
eval_correct = len(eval_tasks)
 for i in range(len(eval_tasks)) :
eval_file = str(evaluation_path / eval_tasks[i])
 with open(eval_file, 'r')as f:
task = json.load(f)
 replace_color = {}
for k in range(len(task['train'])) :
if len(flattener(task['train'][k]['output'])) == len(flattener(task['train'][k]['input'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
else:
break
data = [[replace_color[i] if i in replace_color else i for i in j] for j in task['test'][0]['input']]
pred_1 = flattener(data)
 if pred_1 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 eval_correct-=1
for j in task['test'][0]['input']:
if not np.any(j==0):
temp_data = j
data = [temp_data for j in range(len(task['test'][0]['input'])) ]
pred_2 = flattener(data)
 if pred_2 == flattener(task['test'][0]['output']):
print('file name = '+train_tasks[i])
 plot_task(task)
 eval_correct-=1<compute_test_metric>
print("Evaluate set error rate ==> {}, detect tasks {} / {}".format(eval_correct/len(eval_tasks),len(eval_tasks)- eval_correct,len(eval_tasks)) )<load_from_csv>
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f )
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )<define_variables>
cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']
catdog_ratings = {**cat_ratings, **dog_ratings}
count = 0
for output_id in submission.index:
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 f = str(test_path / str(task_id + '.json'))
with open(f, 'r')as read_file:
task = json.load(read_file)
 replace_color = {}
for k in range(len(task['train'])) :
if len(flattener(task['train'][k]['output'])) == len(flattener(task['train'][k]['input'])) :
target = flattener(task['train'][k]['output'])
 input_str = flattener(task['train'][k]['input'])
 base_count = find_count_tow_list(input_str,target)
 for i in range(10):
for j in range(10):
input_strr = input_str.replace(str(i),str(j))
new_count = find_count_tow_list(target,input_strr)
 if new_count>base_count:
replace_color[i]=j
else:
break
data = [[replace_color[i] if i in replace_color else i for i in j] for j in task['test'][pair_id]['input']]
pred_1 = flattener(data)
 for j in task['test'][0]['input']:
if not np.any(j==0):
temp_data = j
data = [temp_data for j in range(len(task['test'][0]['input'])) ]
pred_2 = flattener(data)
 data = [[0 for i in j] for j in data]
pred_3 = flattener(data)
 pred = pred_1 + ' ' + pred_2 + ' ' + pred_3 + ' '
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv' )<set_options>
parameters_df=pd.DataFrame()
i = 0
for breed in catdog_ratings.keys() :
for key in catdog_ratings[breed].keys() :
parameters_df.at[i,'breed'] = breed
parameters_df.at[i,key] = catdog_ratings[breed][key]
i = i+1
%matplotlib inline
fig = plt.figure(figsize=(8., 6.) )<categorify>
parameters_df.rename(index=str, columns={'breed':'Breed1Name'},inplace=True )
def mask_match(task):
result = True
for i in range(len(task)) :
if not np.array_equal(np.clip(np.array(task[i]['input']), 0, 1), np.clip(np.array(task[i]['output']), 0, 1)) :
result = False
break
return result
def shape_match(task):
result = True
for i in range(len(task)) :
if not np.array_equal(np.array(task[i]['input'] ).shape, np.array(task[i]['output'] ).shape):
result = False
break
return result
def rotation_match(task):
r_ = {r:0 for r in range(1,4)}
result = 0
for i in range(len(task)) :
for r in range(3):
rot = np.rot90(np.array(task[i]['input']), r+1)
 rot = np.flip(rot, axis=1)
 if np.array_equal(np.array(task[i]['output']), rot):
r_[r + 1] += 1
for r in r_:
if r_[r]==len(task):
result = r
return result
def color_match(task):
result = True
for i in range(len(task)) :
if not np.array_equal(np.unique(np.array(task[i]['input'])) , np.unique(np.array(task[i]['output']))):
result = False
break
return result<feature_engineering>
data_df = data_df.merge(parameters_df[['Breed1Name','Affectionate with Family','Amount of Shedding',
'Easy to Groom','General Health','Intelligence','Kid Friendly',
'Pet Friendly','Potential for Playfulness']], on='Breed1Name', how='left' )
path = '/kaggle/input/abstraction-and-reasoning-challenge/'
tasks = pd.DataFrame(glob.glob(path + '**/**'), columns=['path'])
 tasks['tte'] = tasks['path'].map(lambda x: x.split('/')[-2])
 tasks['output_id'] = tasks['path'].map(lambda x: x.split('/')[-1].split('.')[0])
 tasks['file'] = tasks['path'].map(lambda x: eval(open(x ).read()))
tasks['train'] = tasks['file'].map(lambda x: x['train'])
 tasks['test'] = tasks['file'].map(lambda x: x['test'])
 tasks.drop(columns=['file'], inplace=True)
 tasks['l'] = tasks.apply(lambda r:(len(r['train']), len(r['test'])) , axis=1)
 tasks.tte.value_counts()<feature_engineering>
data_df['Description'] = data_df['Description'].fillna(' ' )
tasks['mask_match'] = tasks['train'].map(lambda x: mask_match(x))
tasks['shape_match'] = tasks['train'].map(lambda x: shape_match(x))
tasks['rotation_match'] = tasks['train'].map(lambda x: rotation_match(x))
tasks['color_match'] = tasks['train'].map(lambda x: color_match(x))
tasks.head()<define_variables>
english_desc, chinese_desc = [], []
tokens = set()
word_dict = {}
pos_count, word_count = 1, 1
pos_dict = {}
eng_sequences = []
pos_sequences = []
for i in range(len(data_df)) :
e_d, c_d, eng_seq, pos_seq = [], [], [], []
doc = custom_tokenizer(data_df['Description'].iloc[i])
 for token in doc:
if not isascii(token):
c_d.append(token)
 else:
e_d.append(token)
 if token not in word_dict:
word_dict[token] = word_count
word_count +=1
english_desc.append(' '.join(e_d))
chinese_desc.append(' '.join(c_d))
pos_seq = nltk.pos_tag(e_d)
 for t in pos_seq:
if t[1] not in pos_dict:
pos_dict[t[1]] = pos_count
pos_count += 1
pos_seq = [pos_dict[t[1]] for t in pos_seq]
eng_seq = [word_dict[t] for t in e_d]
if len(eng_seq)==0:
eng_seq.append(0)
 pos_seq.append(0)
 eng_sequences.append(eng_seq)
 pos_sequences.append(pos_seq )
def flattener(pred):
str_pred = '|'+ '|'.join([''.join([str(v)for v in row])for row in pred])+'|'
str_pred = ' '.join([str_pred for i in range(2)])
 str_pred += ' |'+ '|'.join([''.join([str(0)for v in row])for row in pred])+'|'
return str_pred<define_variables>
data_df['English_desc'] = english_desc
data_df['Chinese_desc'] = chinese_desc
data_df['e_description_len'] = data_df['English_desc'].apply(lambda x:len(x))
data_df['e_description_word_len'] = data_df['English_desc'].apply(lambda x: len(x.split(' ')))
 data_df['e_description_word_unique'] = data_df['English_desc'].apply(lambda x: len(set(x.split(' '))))
data_df['c_description_len'] = data_df['Chinese_desc'].apply(lambda x:len(x))
data_df['c_description_word_len'] = data_df['Chinese_desc'].apply(lambda x:len(x.split(' ')))
 data_df['c_description_word_unique'] = data_df['Chinese_desc'].apply(lambda x: len(set(x)))
 data_df['description_len'] = data_df['Description'].apply(lambda x:len(x))
data_df['description_word_len'] = data_df['Description'].apply(lambda x: len(x.split(' ')) )
evaluation = tasks[tasks['tte']=='evaluation'].reset_index(drop=True)
 score = 0.
 denom = 0.
 for i in range(len(evaluation)) :
for j in range(len(evaluation['test'][i])) :
denom += 1
if evaluation['rotation_match'][i] > 0:
rot = np.array(evaluation['test'][i][j]['input'])
 rot = np.flip(np.rot90(rot, evaluation['rotation_match'][i]), axis=1)
 if not np.array_equal(np.array(evaluation['test'][i][j]['output']), rot):
score += 1
elif evaluation['shape_match'][i] == True:
if not np.array_equal(np.array(evaluation['test'][i][j]['output']), np.array(evaluation['test'][i][j]['input'])) : score += 1
else:
if not np.array_equal(np.array(evaluation['test'][i][j]['output']), np.array(evaluation['train'][i][0]['output'])) : score += 1
print(score/denom )<save_to_csv>
def load_glove() :
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in(open(EMBEDDING_FILE)))
 return embeddings_index
glove_emb = load_glove()
embedding_matrix, nb_words, nb_oov = build_emb_matrix(word_dict, glove_emb)
 print(nb_words, nb_oov)
 del glove_emb
gc.collect()
test = tasks[tasks['tte']=='test'].reset_index(drop=True)
 sub = open('submission.csv','w')
 sub.write('output_id,output
')
 for i in range(len(test)) :
for j in range(len(test['test'][i])) :
if test['rotation_match'][i] > 0:
rot = np.array(test['test'][i][j]['input'])
 pred = np.flip(np.rot90(rot, test['rotation_match'][i]), axis=1)
 elif test['shape_match'][i] == True:
pred = np.array(test['test'][i][j]['input'])
 else:
pred = np.array(test['train'][i][0]['output'])
 sub.write(test['output_id'][i]+ '_' + str(j)+ ',' + flattener(pred)+'
')
 sub.close()<define_variables>
len(set(train.index.tolist()))
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.5
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
EPO = 15
WEIGHTS = '.. /input/best-yolov5x-fold0pt/best_yolov5x_fold0.pt'
CONFIG = '.. /input/best-yolov5x-fold0pt/yolov5x.yaml'
DATA = '.. /input/best-yolov5x-fold0pt/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)))
 image_files = train_image_files+test_image_files
metadata_files = train_metadata_files+test_metadata_files
sentiment_files = train_sentiment_files+test_sentiment_files
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<concatenate>
IMG_SIZE = 256
BATCH_SIZE = 256
def get_petid(path):
basename = os.path.basename(path)
 return basename.split('-')[0]
def get_picid(path):
basename = os.path.splitext(os.path.basename(path)) [0]
return basename.split('-')[1]
image_df = pd.DataFrame(image_files, columns=['path'])
 image_df['PetID'] = image_df['path'].apply(get_petid)
 image_df['PicID'] = image_df['path'].apply(get_picid)
 gc.collect()
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<normalization>
pet_image_dataset = PetDataset(image_df)
 pet_image_loader = DataLoader(pet_image_dataset, batch_size=BATCH_SIZE, shuffle=False,
num_workers=1, pin_memory=True )
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<train_model>
resnet34_feature = []
model = ResnetModel(resnet_fun=resnet34)
 model.init_resnet('.. /input/pretrained-pytorch-dog-and-cat-models/resnet34.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 resnet34_feature.append(y_pred.cpu().numpy())
 resnet34_feature = np.vstack(resnet34_feature)
 del model
gc.collect()
torch.cuda.empty_cache()
densenet121_feature = []
model = DenseModel(dense_func=densenet121)
 model.init_densenet('.. /input/pretrained-pytorch-dog-and-cat-models/densenet121.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 densenet121_feature.append(y_pred.cpu().numpy())
 densenet121_feature = np.vstack(densenet121_feature)
 del model
gc.collect()
torch.cuda.empty_cache()
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
densenet_feature = []
model = DenseModel()
model.init_densenet('.. /input/pytorch-pretrained-image-models/densenet201.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 densenet_feature.append(y_pred.cpu().numpy())
 densenet_feature = np.vstack(densenet_feature)
 del model
gc.collect()
torch.cuda.empty_cache()
resnet50_feature = []
model = ResnetModel()
model.init_resnet('.. /input/pytorch-pretrained-image-models/resnet50.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 resnet50_feature.append(y_pred.cpu().numpy())
 resnet50_feature = np.vstack(resnet50_feature )
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
del img_batch
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
del model
del pet_image_loader
del pet_image_dataset
gc.collect()
torch.cuda.empty_cache()
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
RES50_IMG_FEATURE_DIM = resnet50_feature.shape[1]
DENSE_IMG_FEATURE_DIM = densenet_feature.shape[1]
RES34_IMG_FEATURE_DIM = resnet34_feature.shape[1]
DENSE121_IMG_FEATURE_DIM = densenet121_feature.shape[1]
print(RES50_IMG_FEATURE_DIM)
 print(DENSE_IMG_FEATURE_DIM)
 print(RES34_IMG_FEATURE_DIM)
 print(DENSE121_IMG_FEATURE_DIM )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
resnet50_feature_df = pd.DataFrame(resnet50_feature, dtype=np.float32,
columns=['resnet50_%d'%i for i in range(RES50_IMG_FEATURE_DIM)])
 resnet50_feature_df['PetID'] = image_df['PetID'].values
resnet50_feature_df['PicID'] = image_df['PicID'].values
resnet50_feature_df_avg = resnet50_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
resnet50_feature_df_avg.columns = ['PetID']+['resnet50_mean_%d'%i for i in range(RES50_IMG_FEATURE_DIM)]
resnet50_feature_df_1 = resnet50_feature_df[resnet50_feature_df['PicID']=='1'].drop('PicID', axis=1)
 resnet50_feature_df = resnet50_feature_df_1.merge(resnet50_feature_df_avg, on='PetID', how='outer')
 resnet50_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(resnet50_feature_df, on='PetID', how='left')
 for c in resnet50_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
resnet50_feature_df[c] = resnet50_feature_df[c].fillna(-1)
 del resnet50_feature_df_avg
del resnet50_feature_df_1
gc.collect()
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
dense_feature_df = pd.DataFrame(densenet_feature, dtype=np.float32,
columns=['densenet_%d'%i for i in range(DENSE_IMG_FEATURE_DIM)])
 dense_feature_df['PetID'] = image_df['PetID'].values
dense_feature_df['PicID'] = image_df['PicID'].values
dense_feature_df_avg = dense_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
dense_feature_df_avg.columns = ['PetID']+['densenet_mean_%d'%i for i in range(DENSE_IMG_FEATURE_DIM)]
dense_feature_df_1 = dense_feature_df[dense_feature_df['PicID']=='1'].drop('PicID', axis=1)
 dense_feature_df = dense_feature_df_1.merge(dense_feature_df_avg, on='PetID', how='left')
 dense_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(dense_feature_df, on='PetID', how='left')
 for c in dense_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
dense_feature_df[c] = dense_feature_df[c].fillna(-1)
 del dense_feature_df_1
del dense_feature_df_avg
gc.collect()
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<categorify>
res34_feature_df = pd.DataFrame(resnet34_feature, dtype=np.float32,
columns=['densenet_%d'%i for i in range(RES34_IMG_FEATURE_DIM)])
 res34_feature_df['PetID'] = image_df['PetID'].values
res34_feature_df['PicID'] = image_df['PicID'].values
res34_feature_df_avg = res34_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
res34_feature_df_avg.columns = ['PetID']+['res34_mean_%d'%i for i in range(RES34_IMG_FEATURE_DIM)]
res34_feature_df_1 = res34_feature_df[res34_feature_df['PicID']=='1'].drop('PicID', axis=1)
 res34_feature_df = res34_feature_df_1.merge(res34_feature_df_avg, on='PetID', how='left')
 res34_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(res34_feature_df, on='PetID', how='left')
 for c in res34_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
res34_feature_df[c] = res34_feature_df[c].fillna(-1)
 del res34_feature_df_1
del res34_feature_df_avg
gc.collect()
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
dense121_feature_df = pd.DataFrame(densenet121_feature, dtype=np.float32,
columns=['densenet_%d'%i for i in range(DENSE121_IMG_FEATURE_DIM)])
 dense121_feature_df['PetID'] = image_df['PetID'].values
dense121_feature_df['PicID'] = image_df['PicID'].values
dense121_feature_df_avg = dense121_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
dense121_feature_df_avg.columns = ['PetID']+['dense121_mean_%d'%i for i in range(DENSE121_IMG_FEATURE_DIM)]
dense121_feature_df_1 = dense121_feature_df[dense121_feature_df['PicID']=='1'].drop('PicID', axis=1)
 dense121_feature_df = dense121_feature_df_1.merge(dense121_feature_df_avg, on='PetID', how='left')
 dense121_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(dense121_feature_df, on='PetID', how='left')
 for c in dense121_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
dense121_feature_df[c] = dense121_feature_df[c].fillna(-1)
 del dense121_feature_df_1
del dense121_feature_df_avg
gc.collect()
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
class ImgExtractor(nn.Module):
def __init__(self, raw_feature_dim = 4096, feature_dim=64):
super(ImgExtractor, self ).__init__()
self.extractor = nn.Sequential(
 nn.BatchNorm1d(raw_feature_dim),
nn.Dropout(0.5),
nn.Linear(raw_feature_dim, raw_feature_dim//4),
nn.ELU(inplace=True),
nn.Dropout(0.1),
nn.Linear(raw_feature_dim//4, raw_feature_dim//8),
nn.ELU(inplace=True),
nn.Linear(raw_feature_dim//8, feature_dim),
nn.ELU(inplace=True)
)
 self.logit = nn.Sequential(
 nn.Linear(feature_dim, 1)
)
 def forward(self, x):
batch_size = x.shape[0]
feat = self.extractor(x)
 out = self.logit(feat)
 return out, feat
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
IMG_FEATURE_DIM_NN = 128
IMG_FEATURE_DIM_NN2 = 64
def get_image_feature(img_train_data, img_test_data,img_feature_dim):
loss_fn = torch.nn.MSELoss().cuda()
oof_train_img = np.zeros(( img_train_data.shape[0], img_feature_dim+1))
oof_test_img = []
test_dataset = TensorDataset(torch.tensor(img_test_data))
test_loader = DataLoader(test_dataset, batch_size=1024, shuffle=False, num_workers=1, pin_memory=True)
 qwks = []
rmses = []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
print('fold:',n_fold)
 X_tr = img_train_data[train_idx]
X_val = img_train_data[valid_idx]
y_tr = train.iloc[train_idx]['AdoptionSpeed'].values
y_val = train.iloc[valid_idx]['AdoptionSpeed'].values
hist = histogram(y_tr.astype(int),
int(np.min(train['AdoptionSpeed'])) ,
int(np.max(train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 tra_dataset = TensorDataset(torch.tensor(X_tr), torch.tensor(y_tr))
val_dataset = TensorDataset(torch.tensor(X_val), torch.tensor(y_val))
training_loader = DataLoader(tra_dataset, batch_size=1024, shuffle=True, num_workers=1, pin_memory=True)
 validation_loader = DataLoader(val_dataset, batch_size=1024, shuffle=False, num_workers=1, pin_memory=True)
 model = ImgExtractor(raw_feature_dim=img_train_data.shape[1], feature_dim=img_feature_dim)
 model.cuda()
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.002)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=7, eta_min=0.0003)
 iteration = 0
min_val_loss = 100
since = time.time()
test_feats = None
for epoch in range(15):
scheduler.step()
model.train()
for x, y in training_loader:
iteration += 1
x = x.cuda()
y = y.type(torch.FloatTensor ).cuda().view(-1, 1)
 pred, feat = model(x)
 loss = loss_fn(pred, y)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
val_predicts = []
val_feats = []
with torch.no_grad() :
for x, y in validation_loader:
x = x.cuda()
y = y.type(torch.FloatTensor ).cuda()
v_pred, feat = model(x)
 val_predicts.append(v_pred.cpu().numpy())
 val_feats.append(feat.cpu().numpy())
 val_predicts = np.concatenate(val_predicts)
 val_feats = np.vstack(val_feats)
 pred_test_y_k = getTestScore2(val_predicts.flatten() , tr_cdf)
 qwk = quadratic_weighted_kappa(y_val, pred_test_y_k)
 val_loss = rmse(y_val,val_predicts)
 if val_loss<min_val_loss:
min_val_loss = val_loss
oof_train_img[valid_idx,:] = np.hstack([val_feats,val_predicts])
 test_feats = []
test_preds = []
with torch.no_grad() :
for x, in test_loader:
x = x.cuda()
v_pred, feat = model(x)
 test_preds.append(v_pred.cpu().numpy())
 test_feats.append(feat.cpu().numpy())
 test_feats = np.hstack([np.vstack(test_feats), np.concatenate(test_preds)])
 print(epoch, "best loss! val loss:", val_loss, 'qwk:', qwk, "elapsed time:", time.time() -since)
 oof_test_img.append(test_feats)
 rmses.append(min_val_loss)
 qwks.append(qwk)
 del model
del x
del y
del tra_dataset
del val_dataset
del training_loader
del validation_loader
gc.collect()
torch.cuda.empty_cache()
print('overall rmse: %.5f'%rmse(oof_train_img[:,-1], train['AdoptionSpeed']))
print('mean rmse =', np.mean(rmses), 'rmse std =', np.std(rmses))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))
del test_loader
del test_dataset
gc.collect()
return oof_train_img, np.mean(oof_test_img, axis=0 )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d4')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/effdet4/effdet5-cutmix-augmix/best-checkpoint-018epoch.bin' )<predict_on_test>
res34_array = res34_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
res34_tra = res34_array[0:train_len]
res34_test = res34_array[train_len:]
oof_train_res34, oof_test_res34 = get_image_feature(res34_tra, res34_test, IMG_FEATURE_DIM_NN2)
 del res34_tra
del res34_test
gc.collect()
dnn_resnet34_features = np.vstack([oof_train_res34, oof_test_res34])
 dnn_resnet34_features = pd.DataFrame(dnn_resnet34_features, columns=['res34_%d'%c for c in range(dnn_resnet34_features.shape[1])])
 dnn_resnet34_features['PetID'] = data_df['PetID'].values
del oof_train_res34
del oof_test_res34
gc.collect()
def make_predictions(images, score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 return [predictions]
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
stdscaler = RobustScaler()
res50_array = resnet50_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
res50_tra = res50_array[0:train_len]
res50_test = res50_array[train_len:]
oof_train_res, oof_test_res = get_image_feature(res50_tra, res50_test, IMG_FEATURE_DIM_NN)
 del res50_tra
del res50_test
gc.collect()
dnn_resnet50_features = np.vstack([oof_train_res, oof_test_res])
 dnn_resnet50_features = pd.DataFrame(dnn_resnet50_features, columns=['resnet_%d'%c for c in range(dnn_resnet50_features.shape[1])])
 dnn_resnet50_features['PetID'] = data_df['PetID'].values
del oof_train_res
del oof_test_res
gc.collect()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<predict_on_test>
dense121_array = dense121_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
dense121_tra = dense121_array[0:train_len]
dense121_test = dense121_array[train_len:]
oof_train_dense121, oof_test_dense121 = get_image_feature(dense121_tra, dense121_test, IMG_FEATURE_DIM_NN)
 del dense121_tra
del dense121_test
gc.collect()
dnn_dense121_features = np.vstack([oof_train_dense121, oof_test_dense121])
 dnn_dense121_features = pd.DataFrame(dnn_dense121_features, columns=['dense121_%d'%c for c in range(dnn_dense121_features.shape[1])])
 dnn_dense121_features['PetID'] = data_df['PetID'].values
del oof_train_dense121
del oof_test_dense121
gc.collect()
results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
dense_array = dense_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
dense_tra = dense_array[0:train_len]
dense_test = dense_array[train_len:]
oof_train_dense, oof_test_dense = get_image_feature(dense_tra, dense_test, IMG_FEATURE_DIM_NN)
 del dense_tra
del dense_test
gc.collect()
dnn_dense_features = np.vstack([oof_train_dense, oof_test_dense])
 dnn_dense_features = pd.DataFrame(dnn_dense_features, columns=['dense_%d'%c for c in range(dnn_dense_features.shape[1])])
 dnn_dense_features['PetID'] = data_df['PetID'].values
del oof_train_dense
del oof_test_dense
gc.collect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
n_components = 64
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_resnet50_features = svd_.fit_transform(res50_array)
 svd_resnet50_features = pd.DataFrame(svd_resnet50_features)
 svd_resnet50_features = svd_resnet50_features.add_prefix('resnet50_SVD_')
 svd_resnet50_features['PetID'] = resnet50_feature_df['PetID'].values
import numpy as np
import pandas as pd
import os
from tqdm.auto import tqdm
import shutil as sh<install_modules>
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_resnet34_features = svd_.fit_transform(res34_array)
 svd_resnet34_features = pd.DataFrame(svd_resnet34_features)
 svd_resnet34_features = svd_resnet34_features.add_prefix('resnet34_SVD_')
 svd_resnet34_features['PetID'] = resnet50_feature_df['PetID'].values
!cp -r.. /input/yolov5train/* .<install_modules>
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_dense121_features = svd_.fit_transform(dense121_array)
 svd_dense121_features = pd.DataFrame(svd_dense121_features)
 svd_dense121_features = svd_dense121_features.add_prefix('dense121_SVD_')
 svd_dense121_features['PetID'] = resnet50_feature_df['PetID'].values
!pip install --no-deps '.. /input/weightedboxesfusion/' > /dev/null<feature_engineering>
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_dense_features = svd_.fit_transform(dense_array)
 svd_dense_features = pd.DataFrame(svd_dense_features)
 svd_dense_features = svd_dense_features.add_prefix('dense_SVD_')
 svd_dense_features['PetID'] = dense_feature_df['PetID'].values
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name,mini in tqdm(df.groupby('image_id')) :
if name in val_index:
path2save = 'val2017/'
else:
path2save = 'train2017/'
if not os.path.exists('convertor/fold{}/labels/'.format(fold)+path2save):
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 if not os.path.exists('convertor/fold{}/images/{}'.format(fold,path2save)) :
os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save))
sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<compute_train_metric>
svd_ = TruncatedSVD(n_components=64, random_state=1337)
 svd_img_features = svd_.fit_transform(np.hstack([res50_array, dense_array, dense121_array, res34_array]))
svd_img_features = pd.DataFrame(svd_img_features)
 svd_img_features = svd_img_features.add_prefix('IMG_SVD_')
 svd_img_features['PetID'] = resnet50_feature_df['PetID'].values
def run_wbf(boxes, scores, image_size=1023, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(im0, imgsz, model, device, conf_thres, iou_thres):
img = letterbox(im0, new_shape=imgsz)[0]
img = img[:, :, ::-1].transpose(2, 0, 1)
 img = np.ascontiguousarray(img)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=False)[0]
pred = non_max_suppression(pred, conf_thres, iou_thres)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<load_pretrained>
features = svd_img_features.drop(['PetID'], axis=1 ).values
cluster = KMeans(n_clusters=32)
 cluster_label = cluster.fit_predict(features)
 cluster_img_features = pd.DataFrame(cluster_label)
 cluster_img_features = cluster_img_features.add_prefix('img_CLUSTER_')
 cluster_img_features['PetID'] = dense_feature_df['PetID'].values
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = '.. /input/yolov5/yolov5-master/weights/best_yolov5x_fold0.pt'
imgsz = 1024
conf_thres = 0.5
iou_thres = 0.6
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for name in imagenames:
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
if im01.shape[0]!=1024 or im01.shape[1]!=1024:
continue
assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.6, skip_box_thr=0.43)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.05].astype(np.int32)
 scores = scores[scores >=float(0.05)]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<define_variables>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = '; '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 ret_val = {}
ret_val['doc_mag'] = file['documentSentiment']['magnitude']
ret_val['doc_score']= file['documentSentiment']['score']
ret_val['doc_language'] = file['language']
ret_val['doc_stcs_len'] = len(file['sentences'])
 if ret_val['doc_stcs_len']>0:
ret_val['doc_first_score'] = file['sentences'][0]['sentiment']['score']
ret_val['doc_first_mag'] = file['sentences'][0]['sentiment']['magnitude']
ret_val['doc_last_score'] = file['sentences'][-1]['sentiment']['score']
ret_val['doc_last_mag'] = file['sentences'][-1]['sentiment']['magnitude']
else:
ret_val['doc_first_score'] = np.nan
ret_val['doc_first_mag'] = np.nan
ret_val['doc_last_score'] = np.nan
ret_val['doc_last_mag'] = np.nan
ret_val['doc_ent_num'] = len(file['entities'])
 mags, scores = [], []
for s in file['sentences']:
mags.append(s['sentiment']['magnitude'])
 scores.append(s['sentiment']['score'])
 if len(scores)==0:
ret_val['doc_score_sum'] = np.nan
ret_val['doc_mag_sum'] = np.nan
ret_val['doc_score_mena'] = np.nan
ret_val['doc_mag_mean'] = np.nan
ret_val['doc_score_max'] = np.nan
ret_val['doc_mag_max'] = np.nan
ret_val['doc_score_min'] = np.nan
ret_val['doc_mag_min'] = np.nan
ret_val['doc_score_std'] = np.nan
ret_val['doc_mag_std'] = np.nan
else:
ret_val['doc_score_sum'] = np.sum(scores)
 ret_val['doc_mag_sum'] = np.sum(mags)
 ret_val['doc_score_mena'] = np.mean(scores)
 ret_val['doc_mag_mean'] = np.mean(mags)
 ret_val['doc_score_max'] = np.max(scores)
 ret_val['doc_mag_max'] = np.max(mags)
 ret_val['doc_score_min'] = np.min(scores)
 ret_val['doc_mag_min'] = np.min(mags)
 ret_val['doc_score_std'] = np.std(scores)
 ret_val['doc_mag_std'] = np.std(mags)
 ret_val['sentiment_entities'] = []
ret_val['doc_ent_person_count'] = 0
ret_val['doc_ent_location_count'] = 0
ret_val['doc_ent_org_count'] = 0
ret_val['doc_ent_event_count'] = 0
ret_val['doc_ent_woa_count'] = 0
ret_val['doc_ent_good_count'] = 0
ret_val['doc_ent_other_count'] = 0
key_mapper = {
'PERSON':'doc_ent_person_count',
'LOCATION':'doc_ent_location_count',
'ORGANIZATION':'doc_ent_org_count',
'EVENT':'doc_ent_event_count',
'WORK_OF_ART':'doc_ent_woa_count',
'CONSUMER_GOOD':'doc_ent_good_count',
'OTHER':'doc_ent_other_count'
}
for e in file['entities']:
ret_val['sentiment_entities'].append(e['name'])
 if e['type'] in key_mapper:
ret_val[key_mapper[e['type']]]+=1
ret_val['sentiment_entities'] = ' '.join(ret_val['sentiment_entities'])
 return ret_val
def parse_metadata_file(self, file, img):

 file_keys = list(file.keys())
 if 'textAnnotations' in file_keys:
textblock_num = len(file['textAnnotations'])
 textlen = np.sum([len(text['description'])for text in file['textAnnotations']])
 else:
textblock_num = 0
textlen = 0
if 'faceAnnotations' in file_keys:
faceanno = 1
else:
faceanno = 0
if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
if len(file_annots)==0:
file_label_score_mean = np.nan
file_label_score_max = np.nan
file_label_score_min = np.nan
else:
temp = np.asarray([x['score'] for x in file_annots])
 file_label_score_mean = temp.mean()
file_label_score_max = temp.max()
file_label_score_min = temp.min()
file_top_desc = [x['description'] for x in file_annots]
else:
file_label_score_mean = np.nan
file_label_score_max = np.nan
file_label_score_min = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
if len(file_colors)==0:
file_color_score = np.nan
file_color_pixelfrac = np.nan
color_red_mean = np.nan
color_green_mean = np.nan
color_blue_mean = np.nan
color_red_std = np.nan
color_green_std = np.nan
color_blue_std = np.nan
else:
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_color_red = np.asarray([x['color']['red'] if 'red' in x['color'] else 0 for x in file_colors])
 file_color_green = np.asarray([x['color']['green'] if 'green' in x['color'] else 0for x in file_colors])
 file_color_blue = np.asarray([x['color']['blue'] if 'blue' in x['color'] else 0 for x in file_colors])
 color_red_mean = file_color_red.mean()
color_green_mean = file_color_green.mean()
color_blue_mean = file_color_blue.mean()
color_red_std = file_color_red.std()
color_green_std = file_color_green.std()
color_blue_std = file_color_blue.std()
if len(file_crops)==0:
file_crop_conf=np.nan
file_crop_importance = np.nan
file_crop_fraction_mean = np.nan
file_crop_fraction_sum = np.nan
file_crop_fraction_std = np.nan
file_crop_num = 0
else:
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
file_crop_num = len(file_crops)
 if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
crop_areas = []
image_area = img.shape[0]*img.shape[1]
for crophint in file_crops:
v_x, v_y = [], []
for vertices in crophint['boundingPoly']['vertices']:
if 'x' not in vertices:
v_x.append(0)
 else:
v_x.append(vertices['x'])
 if 'y' not in vertices:
v_y.append(0)
 else:
v_y.append(vertices['y'])
 crop_areas.append(( max(v_x)-min(v_x)) *(max(v_y)-min(v_y)) /image_area)
 file_crop_fraction_mean = np.mean(crop_areas)
 file_crop_fraction_sum = np.sum(crop_areas)
 file_crop_fraction_std = np.std(crop_areas)
 df_metadata = {
'label_score_mean': file_label_score_mean,
'label_score_max': file_label_score_max,
'label_score_min': file_label_score_min,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'color_red_mean':color_red_mean,
'color_green_mean':color_green_mean,
'color_blue_mean':color_blue_mean,
'color_red_std':color_red_std,
'color_green_std':color_green_std,
'color_blue_std':color_blue_std,
'crop_area_sum':file_crop_fraction_sum,
'annots_top_desc': self.sentence_sep.join(file_top_desc),
'img_aratio':img.shape[0]/img.shape[1],
'text_len':textlen,
'textblock_num':textblock_num,
'face_annotation':faceanno
}
return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = None
dfs_metadata = []
for ind in range(1,200):
metadata_filename = '.. /input/petfinder-adoption-prediction/{}_metadata/{}-{}.json'.format(mode, pet_id, ind)
 image_filename = '.. /input/petfinder-adoption-prediction/{}_images/{}-{}.jpg'.format(mode, pet_id, ind)
 try:
image = cv2.imread(image_filename)
 metadata_file = pet_parser.open_metadata_file(metadata_filename)
 df_metadata = pet_parser.parse_metadata_file(metadata_file, image)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 except FileNotFoundError:
break
return [df_sentiment, dfs_metadata]
pet_parser = PetFinderParser()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<load_pretrained>
dfs_test = Parallel(n_jobs=n_jobs, verbose=1 )(
 delayed(extract_additional_features )(i, mode='test')for i in test_pet_ids)
 test_dicts_sentiment = [x[0] for x in dfs_test if x[0] is not None]
test_dfs_metadata = [x[1] for x in dfs_test if len(x[1])>0]
test_dfs_sentiment = pd.DataFrame(test_dicts_sentiment)
 test_dfs_metadata = list(itertools.chain.from_iterable(test_dfs_metadata))
test_dfs_metadata = pd.DataFrame(test_dfs_metadata)
 print(test_dfs_sentiment.shape, test_dfs_metadata.shape )
def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = '.. /input/yolov5/yolov5-master/weights/best_yolov5x_fold0.pt'
if not os.path.exists(weights):
weights = '.. /input/yolov5/yolov5-master/weights/best_yolov5x_fold0.pt'
imgsz = 1024
conf_thres = 0.5
iou_thres = 0.6
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for name in imagenames:
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 if 1:
enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.6, skip_box_thr=0.5)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.05].astype(np.int32)
 scores = scores[scores >=float(0.05)]
if count<10:
for box, score in zip(boxes,scores):
cv2.rectangle(im0, 
(box[0], box[1]), 
(box[2]+box[0], box[3]+box[1]), 
(220, 0, 0), 2)
 cv2.putText(im0, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX ,
0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(im0)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
meta_df = pd.concat([train_dfs_metadata, test_dfs_metadata], sort=False ).reset_index(drop=True)
 senti_df = pd.concat([train_dfs_sentiment, test_dfs_sentiment], sort=False ).reset_index(drop=True )
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<load_from_csv>
metadata_desc = meta_df.groupby(['PetID'])['annots_top_desc'].unique().reset_index()
metadata_desc['meta_annots_top_desc'] = metadata_desc['annots_top_desc'].apply(lambda x: '; '.join(x))
metadata_desc.drop('annots_top_desc', axis=1, inplace=True)
 possible_annots = set()
for i in range(len(meta_df)) :
possible_annots = possible_annots.union(set(meta_df['annots_top_desc'].iloc[i].split('; ')))
 annot_mapper = {}
for idx, a in enumerate(possible_annots):
annot_mapper[a] = str(idx)
 metadata_desc['meta_desc'] = metadata_desc['meta_annots_top_desc'].apply(lambda x: ' '.join(annot_mapper[i] for i in x.split('; ')) )
def load_dataset(root):
csv = pd.read_csv(os.path.join(root, "train.csv"))
data = {}
for i in csv.index:
key = csv["image_id"][i]
bbox = json.loads(csv["bbox"][i])
 bbox = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3], 0.0]
if key in data:
data[key].append(bbox)
 else:
data[key] = [bbox]
return sorted(
 [(k, os.path.join(root, "train", k + ".jpg"), v)for k, v in data.items() ],
key=lambda x: x[0] 
)
 def load_image(path):
with open(path, "rb")as f:
buf = f.read()
return mx.image.imdecode(buf)
 def get_batches(dataset, batch_size, width=512, height=512, net=None, ctx=mx.cpu()):
batches = len(dataset)// batch_size
sampler = Sampler(dataset, width, height, net)
 stack_fn = [gcv.data.batchify.Stack() ]
pad_fn = [gcv.data.batchify.Pad(pad_val=-1)]
if net is None:
batchify_fn = gcv.data.batchify.Tuple(*(stack_fn + pad_fn))
else:
batchify_fn = gcv.data.batchify.Tuple(*(stack_fn * 6 + pad_fn))
with Pool(cpu_count() * 2)as p:
for i in range(batches):
start = i * batch_size
samples = p.map(sampler, range(start, start + batch_size))
batch = batchify_fn(samples)
 yield [x.as_in_context(ctx)for x in batch]
def gauss_blur(image, level):
return cv2.blur(image,(level * 2 + 1, level * 2 + 1))
def gauss_noise(image):
for i in range(image.shape[2]):
c = image[:, :, i]
diff = 255 - c.max() ;
noise = np.random.normal(0, random.randint(1, 6), c.shape)
 noise =(noise - noise.min())/(noise.max() - noise.min())
 noise = diff * noise
image[:, :, i] = c + noise.astype(np.uint8)
 return image
class YOLO3TrainTransform:
def __init__(self, width, height, net, mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225), **kwargs):
self._width = width
self._height = height
self._mean = mean
self._std = std
self._fake_x = mx.nd.zeros(( 1, 3, height, width))
net = copy.deepcopy(net)
 net.collect_params().reset_ctx(None)
 with mx.autograd.train_mode() :
_, self._anchors, self._offsets, self._feat_maps, _, _, _, _ = net(self._fake_x)
 self._target_generator = gcv.model_zoo.yolo.yolo_target.YOLOV3PrefetchTargetGenerator(
 num_class=len(net.classes), **kwargs)
 def __call__(self, img, label):
if np.random.uniform(0, 1)> 0.5:
img, expand = gcv.data.transforms.image.random_expand(img, max_ratio=1.5, fill=114, keep_ratio=False)
 bbox = gcv.data.transforms.bbox.translate(label, x_offset=expand[0], y_offset=expand[1])
 else:
img, bbox = img, label
h, w, _ = img.shape
bbox, crop = gcv.data.transforms.experimental.bbox.random_crop_with_constraints(bbox,(w, h))
x0, y0, w, h = crop
img = mx.image.fixed_crop(img, x0, y0, w, h)
 h, w, _ = img.shape
interp = np.random.randint(0, 5)
 img = gcv.data.transforms.image.imresize(img, self._width, self._height, interp=interp)
 bbox = gcv.data.transforms.bbox.resize(bbox,(w, h),(self._width, self._height))
h, w, _ = img.shape
img, flips = gcv.data.transforms.image.random_flip(img, px=0.5, py=0.5)
 bbox = gcv.data.transforms.bbox.flip(bbox,(w, h), flip_x=flips[0], flip_y=flips[1])
 img = gcv.data.transforms.experimental.image.random_color_distort(img)
 img = mx.nd.image.to_tensor(img)
 img = mx.nd.image.normalize(img, mean=self._mean, std=self._std)
 gt_bboxes = mx.nd.array(bbox[np.newaxis, :, :4])
 gt_ids = mx.nd.array(bbox[np.newaxis, :, 4:5])
 gt_mixratio = mx.nd.array(bbox[np.newaxis, :, -1:])
 objectness, center_targets, scale_targets, weights, class_targets = self._target_generator(
 self._fake_x, self._feat_maps, self._anchors, self._offsets,
gt_bboxes, gt_ids, gt_mixratio)
 return(img, objectness[0], center_targets[0], scale_targets[0], weights[0],
class_targets[0], gt_bboxes[0])
 class Sampler:
def __init__(self, dataset, width, height, net=None, **kwargs):
self._dataset = dataset
if net is None:
self._training_mode = False
self._transform = gcv.data.transforms.presets.yolo.YOLO3DefaultValTransform(width, height, **kwargs)
 else:
self._training_mode = True
self._transform = YOLO3TrainTransform(width, height, net, **kwargs)
 def __call__(self, idx):
if self._training_mode:
raw, bboxes = self._load_mixup(idx)
 raw = raw.asnumpy()
blur = random.randint(0, 3)
 if blur > 0:
raw = gauss_blur(raw, blur)
 raw = gauss_noise(raw)
 raw = mx.nd.array(raw)
 else:
raw = load_image(self._dataset[idx][1])
 bboxes = np.array(self._dataset[idx][2])
 res = self._transform(raw, bboxes)
 return [mx.nd.array(x)for x in res]
def _load_mixup(self, idx1):
r = random.gauss(0.5, 0.5 / 1.96)
 if r > 0.0:
raw1 = load_image(self._dataset[idx1][1])
 bboxes1 = np.array(self._dataset[idx1][2])
 if r >= 1.0:
return raw1, np.hstack([bboxes1, np.full(( bboxes1.shape[0], 1), 1.0)])
 idx2 = random.randint(0, len(self._dataset)- 1)
 raw2 = load_image(self._dataset[idx2][1])
 bboxes2 = np.array(self._dataset[idx2][2])
 if r <= 0.0:
return raw2, np.hstack([bboxes2, np.full(( bboxes2.shape[0], 1), 1.0)])
 h = max(raw1.shape[0], raw2.shape[0])
 w = max(raw1.shape[1], raw2.shape[1])
 mix_raw = mx.nd.zeros(shape=(h, w, 3), dtype="float32")
 mix_raw[:raw1.shape[0], :raw1.shape[1], :] += raw1.astype("float32")* r
mix_raw[:raw2.shape[0], :raw2.shape[1], :] += raw2.astype("float32")*(1.0 - r)
 mix_bboxes = np.vstack([
np.hstack([bboxes1, np.full(( bboxes1.shape[0], 1), r)]),
np.hstack([bboxes2, np.full(( bboxes2.shape[0], 1), 1.0 - r)])
 ])
 return mix_raw.astype("uint8"), mix_bboxes 
<load_pretrained>
senti_df['sentiment_entities'].fillna('', inplace=True)
 senti_df['sentiment_entities'] = senti_df['sentiment_entities'].str.lower()
senti_df['sentiment_len'] = senti_df['sentiment_entities'].apply(lambda x:len(x))
senti_df['sentiment_word_len'] = senti_df['sentiment_entities'].apply(lambda x: len(x.replace(';',' ' ).split(' ')))
 senti_df['sentiment_word_unique'] = senti_df['sentiment_entities'].apply(lambda x: len(set(x.replace(';',' ' ).split(' '))))
senti_df['doc_language'] = pd.factorize(senti_df['doc_language'])[0]
def load_model(path, ctx=mx.cpu()):
net = gcv.model_zoo.yolo3_darknet53_custom(["wheat"], pretrained_base=False)
 net.set_nms(post_nms=150)
 net.load_parameters(path, ctx=ctx)
 return net 
<load_pretrained>
aggregates = {
'color_blue_mean':['mean','std'],
'color_blue_std':['mean'],
'color_green_mean':['mean','std'],
'color_green_std':['mean'],
'color_pixelfrac':['mean','std'],
'color_red_mean':['mean','std'],
'color_red_std':['mean'],
'color_score':['mean','max'],
'crop_area_sum':['mean','std','min'],
'crop_conf':['mean','std','max'],
'crop_importance':['mean','std'],
'label_score_max':['mean','std','max'],
'label_score_mean':['mean','max','std'],
'label_score_min':['mean','max','std'],
'img_aratio':['nunique','std','max','min'],
'textblock_num':['mean','max'],
'face_annotation':['mean','nunique']
}
metadata_gr = meta_df.drop(['annots_top_desc'], axis=1)
 for i in metadata_gr.columns:
if 'PetID' not in i:
metadata_gr[i] = metadata_gr[i].astype(float)
 metadata_gr = metadata_gr.groupby(['PetID'] ).agg(aggregates)
 metadata_gr.columns = pd.Index(['{}_{}_{}'.format('meta', c[0], c[1].upper())for c in metadata_gr.columns.tolist() ])
 metadata_gr = metadata_gr.reset_index()
max_epochs = 6
learning_rate = 0.001
batch_size = 8
img_s = 512
threshold = 0.1
context = mx.gpu()
print("Loading model...")
 model = load_model("/kaggle/input/global-wheat-detection-yolo-darknet-weights/global-wheat-yolo3-darknet53.params", ctx=context)
 print("Loading test images...")
 test_images = [ 
(os.path.join(dirname, filename), os.path.splitext(filename)[0])
 for dirname, _, filenames in os.walk('/kaggle/input/global-wheat-detection/test')for filename in filenames
]
print("Pseudo labaling...")
 pseudo_set = []
for path, image_id in test_images:
print(path)
 raw = load_image(path)
 x, _ = gcv.data.transforms.presets.yolo.transform_test(raw, short=img_s)
 classes, scores, bboxes = model(x.as_in_context(context))
bboxes[0, :, 0::2] =(bboxes[0, :, 0::2] / x.shape[3] ).clip(0.0, 1.0)* raw.shape[1]
bboxes[0, :, 1::2] =(bboxes[0, :, 1::2] / x.shape[2] ).clip(0.0, 1.0)* raw.shape[0]
label = [
[round(x)for x in bboxes[0, i].asnumpy().tolist() ] + [0.0] for i in range(classes.shape[1])
 if model.classes[int(classes[0, i].asscalar())] == "wheat" and scores[0, i].asscalar() > threshold
]
if len(label)> 0:
pseudo_set.append(( image_id, path, label))
print("Loading training set...")
 dataset = load_dataset("/kaggle/input/global-wheat-detection")
 split = int(len(dataset)* 0.9)
 training_set = dataset[:split] + pseudo_set
print("Training set: ", len(training_set))
validation_set = dataset[split:]
print("Validation set: ", len(validation_set))
print("Re-training...")
 trainer = mx.gluon.Trainer(model.collect_params() , "Nadam", {
"learning_rate": learning_rate
})
 metrics = [gcv.utils.metrics.VOCMApMetric(iou_thresh=iou)for iou in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]]
best_score = 0.0
for epoch in range(max_epochs):
ts = time.time()
random.shuffle(training_set)
 training_total_L = 0.0
training_batches = 0
for x, objectness, center_targets, scale_targets, weights, class_targets, gt_bboxes in get_batches(training_set, batch_size, width=img_s, height=img_s, net=model, ctx=context):
training_batches += 1
with mx.autograd.record() :
obj_loss, center_loss, scale_loss, cls_loss = model(x, gt_bboxes, objectness, center_targets, scale_targets, weights, class_targets)
 L = obj_loss + center_loss + scale_loss + cls_loss
L.backward()
trainer.step(x.shape[0])
 training_batch_L = mx.nd.mean(L ).asscalar()
if training_batch_L != training_batch_L:
raise ValueError()
training_total_L += training_batch_L
print("[Epoch %d Batch %d] batch_loss %.10f average_loss %.10f elapsed %.2fs" %(
 epoch, training_batches, training_batch_L, training_total_L / training_batches, time.time() - ts 
))
training_avg_L = training_total_L / training_batches
for metric in metrics:
metric.reset()
for x, label in get_batches(validation_set, batch_size, width=img_s, height=img_s, ctx=context):
classes, scores, bboxes = model(x)
 for metric in metrics:
metric.update(
 bboxes,
classes.reshape(( 0, -1)) ,
scores.reshape(( 0, -1)) ,
label[:, :, :4],
label[:, :, 4:5].reshape(( 0, -1)) 
)
 score = mx.nd.array([metric.get() [1] for metric in metrics], ctx=context ).mean()
print("[Epoch %d] training_loss %.10f validation_score %.10f best_score %.10f duration %.2fs" %(
 epoch + 1, training_avg_L, score.asscalar() , best_score, time.time() - ts 
))
if score.asscalar() > best_score:
best_score = score.asscalar()
model.save_parameters("global-wheat-yolo3-darknet53.params")
 print("Loading re-trained model...")
 model = load_model("global-wheat-yolo3-darknet53.params", ctx=context)
 print("Inference...")
 results = []
for path, image_id in test_images:
print(path)
 raw = load_image(path)
 x, _ = gcv.data.transforms.presets.yolo.transform_test(raw, short=img_s)
 classes, scores, bboxes = model(x.as_in_context(context))
bboxes[0, :, 0::2] =(bboxes[0, :, 0::2] / x.shape[3] ).clip(0.0, 1.0)* raw.shape[1]
bboxes[0, :, 1::2] =(bboxes[0, :, 1::2] / x.shape[2] ).clip(0.0, 1.0)* raw.shape[0]
bboxes[0, :, 2:4] -= bboxes[0, :, 0:2]
results.append({
"image_id": image_id,
"PredictionString": " ".join([
" ".join([str(x)for x in [scores[0, i].asscalar() ] + [round(x)for x in bboxes[0, i].asnumpy().tolist() ]])
 for i in range(classes.shape[1])
 if model.classes[int(classes[0, i].asscalar())] == "wheat" and scores[0, i].asscalar() > threshold
])
 })
 pd.DataFrame(results, columns=['image_id', 'PredictionString'] ).to_csv('submission.csv', index=False)
<load_from_csv>
meta_df = metadata_desc.merge(metadata_gr, on='PetID', how='left' )
BE_data = pd.read_csv('.. /input/kobe-bryant-shot-selection/data.csv.zip')
 BE_sample = pd.read_csv('.. /input/kobe-bryant-shot-selection/sample_submission.csv.zip' )<categorify>
feat_df = data_df[['PetID','Color1','Breed1','State','RescuerID','Name','Breed_full','Color_full','hard_interaction']]
delete_columns = ['shot_distance',
'game_date']
BE_data.drop(delete_columns, axis=1, inplace=True)
 le = LabelEncoder()
BE_data["action_type"] = le.fit_transform(BE_data["action_type"])
 BE_data["season"] = le.fit_transform(BE_data["season"])
 BE_data["combined_shot_type"] = le.fit_transform(BE_data["combined_shot_type"])
 BE_data["shot_type"] = le.fit_transform(BE_data["shot_type"])
 BE_data["shot_zone_area"] = le.fit_transform(BE_data["shot_zone_area"])
 BE_data["shot_zone_basic"] = le.fit_transform(BE_data["shot_zone_basic"])
 BE_data["shot_zone_range"] = le.fit_transform(BE_data["shot_zone_range"])
 BE_data["team_name"] = le.fit_transform(BE_data["team_name"])
 BE_data["opponent"] = le.fit_transform(BE_data["opponent"] )<categorify>
agg = {
'Fee':['mean','std','max'],
'avg_fee':['mean','std','max'],
'Breed1':['nunique'],
'Age':['mean','std','max'],
'Quantity':['std'],
'PetID':['nunique']
}
feat = data_df.groupby('Color1' ).agg(agg)
 feat.columns = pd.Index(['COLOR_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Color1', how='left')
 agg = {
'Fee':['mean','std','max'],
'avg_fee':['mean','std','max'],
'Breed_full':['nunique'],
'Quantity':['sum'],
}
feat = data_df.groupby('Color_full' ).agg(agg)
 feat.columns = pd.Index(['COLORfull_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Color_full', how='left' )
BE_data["matchup"]=BE_data["matchup"].map({ "LAL @ ATL": 3111.71, "LAL @ BKN" : 3940.98, "LAL @ BOS" : 4171.64,
"LAL @ CHA" : 3406.51, "LAL @ CHH" : 3406.51, "LAL @ CHI" : 2802.76,
"LAL @ CLE" : 3293.8, "LAL @ DAL" : 1992.7, "LAL @ DEN" : 1336.99,
"LAL @ DET" : 3187.13, "LAL @ GSW" : 556.05, "LAL @ HOU" : 2209.31,
"LAL @ IND" : 2908.74, "LAL @ LAC" : 0, "LAL @ MEM" : 2577.17,
"LAL @ MIA" : 3760.32, "LAL @ MIL" : 2804.07, "LAL @ MIN" : 2450.08,
"LAL @ NJN" : 3941, "LAL @ NOH" : 2687.88, "LAL @ NOP" : 2687.88,
"LAL @ NYK" : 3938.9, "LAL @ OKC" : 1898.97, "LAL @ ORL" : 3538.33,
"LAL @ PHI" : 3845.69, "LAL @ PHX" : 576.62, "LAL @ POR" : 1331.11,
"LAL @ SAC" : 581.69, "LAL @ SAS" : 1940.71, "LAL @ SEA" : 1898.99,
"LAL @ TOR" : 3497.01, "LAL @ UTA" : 935.07, "LAL @ VAN" : 3695.84,
"LAL @ WAS" : 3695.84
})
<drop_column>
agg = {
'Color_full':['nunique'],
'Breed2':['nunique'],
'FurLength':['nunique'],
'Fee':['mean','max'],
'avg_fee':['mean','std','max'],
'Age':['mean','std','min','max'],
'Quantity':['mean','std','max','sum'],
'PetID':['nunique'],
'FurLength':['mean'],
'Health':['mean'],
'MaturitySize':['mean','std','min','max'],
'Vaccinated':['mean'],
'Dewormed':['mean'],
'Sterilized':['mean']
}
feat = data_df.groupby('Breed1' ).agg(agg)
 feat.columns = pd.Index(['BREED1_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Breed1', how='left')
 agg = {
'Color_full':['nunique'],
'Fee':['mean','min','max'],
'avg_fee':['mean','std','max'],
'Quantity':['sum'],
'PetID':['nunique']
}
feat = data_df.groupby('Breed_full' ).agg(agg)
 feat.columns = pd.Index(['BREEDfull_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Breed_full', how='left' )
BE_test = BE_data
BE_train = BE_data
BE_test = BE_test[BE_test["shot_made_flag"].isnull() ]
BE_train.dropna(subset=['shot_made_flag'], inplace=True)
 X_train = BE_train.drop('shot_made_flag', axis=1 )<prepare_x_and_y>
agg = {
'Color_full':['nunique'],
'Breed_full':['nunique'],
'PetID':['nunique'],
'RescuerID':['nunique'],
'Fee':['mean','max'],
'avg_fee':['mean','std','max'],
'Age':['mean','std','max'],
'Quantity':['mean','std','max'],
'FurLength':['mean','std'],
'Health':['mean'],
'MaturitySize':['mean','std'],
'Vaccinated':['mean'],
'Dewormed':['mean'],
'Sterilized':['mean'],
'VideoAmt':['mean','std'],
'PhotoAmt':['mean','std'],
'avg_photo':['mean','std']
}
feat = data_df.groupby('State' ).agg(agg)
 feat.columns = pd.Index(['STATE_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='State', how='left' )
y_train = BE_train['shot_made_flag']
X_test = BE_test.drop('shot_made_flag', axis=1)
 Y_train = BE_train['shot_made_flag']<train_model>
feat = data_df.groupby('Name')['PetID'].agg({'name_count':'nunique'} ).reset_index()
feat_df = feat_df.merge(feat, on='Name', how='left' )
model = XGBClassifier(n_estimators=20, random_state=71)
 model.fit(X_train, Y_train)
 pred = model.predict_proba(X_test)[:, 1]
sub_data = pd.DataFrame({'shot_id':X_test['shot_id'], 'shot_made_flag':pred} )<save_to_csv>
agg = {
'PetID':['nunique'],
'Breed_full':['nunique'],
'VideoAmt':['mean','std'],
'PhotoAmt':['mean','std'],
'avg_photo':['mean','std'],
'Sterilized':['mean'],
'Dewormed':['mean'],
'Vaccinated':['mean']
}
rescuer_count = data_df.groupby(['RescuerID'] ).agg(agg)
 rescuer_count.columns = pd.Index(['RESCUER_' + e[0] + "_" + e[1].upper() for e in rescuer_count.columns.tolist() ])
 rescuer_count.reset_index(inplace=True)
 feat_df = feat_df.merge(rescuer_count, how='left', on='RescuerID' )
submission = sub_data
submission.to_csv('submission_first.csv', index=False )<set_options>
agg = {
'Fee':['mean','min','max'],
'avg_fee':['mean','std','max']
}
feat = data_df.groupby('hard_interaction' ).agg(agg)
 feat.columns = pd.Index(['INTERACTION_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='hard_interaction', how='left' )
pd.options.display.max_rows = 500
pd.options.display.max_columns = 50
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
feat_df.drop(['Color1','Breed1','State','RescuerID','Name','Breed_full','Color_full','hard_interaction'], axis=1, inplace=True )
df = pd.read_csv('.. /input/data.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv' )<drop_column>
X_text = data_df[['PetID','Description']].merge(data_df[['PetID','Chinese_desc']], on='PetID', how='left')
 X_text = X_text.merge(senti_df[['PetID','sentiment_entities']], on='PetID', how='left')
 X_text = X_text.merge(metadata_desc[['PetID','meta_annots_top_desc','meta_desc']], on='PetID', how='left')
 text_columns = ['Description','Chinese_desc','sentiment_entities','meta_annots_top_desc','meta_desc']
print(text_columns)
 X_text['meta_annots_top_desc'].fillna(' ',inplace=True)
 X_text['meta_desc'].fillna(' ',inplace=True)
 X_text['sentiment_entities'].fillna(' ',inplace=True )
df_test = df.loc[df.shot_made_flag.isnull() , :]
df_test.index = range(len(df_test))
df.dropna(inplace=True)
 df.index = range(len(df))<count_values>
text_features = []
ngram_ranges = [(1,3),(1,3),(1,1),(1,3),(1,1)]
n_components = [80, 24, 10, 32,16]
for idx, i in enumerate(text_columns):
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components[idx], random_state=1337)
 tfidf_col = TfidfVectorizer(ngram_range = ngram_ranges[idx], stop_words = 'english',
tokenizer=custom_tokenizer,
strip_accents='unicode' ).fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
text_features.append(svd_col)
c= df.loc[(df.loc_x==0)&(df.loc_y==0)]
c.shot_made_flag.value_counts()<define_variables>
docs = [custom_tokenizer(x)for x in data_df['Description'].values]
dictionary = corpora.Dictionary(docs)
 name = 'Description'
corpus = [dictionary.doc2bow(text)for text in docs]
lda = LdaMulticore(corpus, id2word=dictionary, num_topics=20, random_state = 999)
 docres = [dict(lda[doc_bow])for doc_bow in corpus]
doc_df = pd.DataFrame(docres,dtype=np.float16 ).fillna(0.001)
 doc_df.columns = ['%s_lda_%d'%(name,x)for x in range(20)]
def color_generator(num_colors):
colors = []
for i in range(num_colors):
colors.append(( np.random.rand() , np.random.rand() , np.random.rand()))
return colors<define_variables>
text_features.append(doc_df )
colors = color_generator(100 )<load_from_csv>
X_text_char = data_df[['PetID','Name','BreedName_full']].merge(metadata_desc[['PetID','meta_annots_top_desc']], on='PetID', how='left')
 X_text_char['meta_annots_top_desc'].fillna(' ',inplace=True)
 for c in ['Name','BreedName_full','meta_annots_top_desc']:
svd_ = TruncatedSVD(
 n_components=16, random_state=1337)
 tfidf_col = TfidfVectorizer(ngram_range =(1,5), analyzer='char',
strip_accents='unicode' ).fit_transform(X_text_char[c])
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_CHAR_{}_'.format(c))
text_features.append(svd_col)
 del X_text_char
gc.collect()
df = pd.read_csv('.. /input/data.csv' )<feature_engineering>
text_features = pd.concat(text_features, axis=1 )
df['angle'] = df.apply(lambda row: 90 if row['loc_y']==0 else m.degrees(m.atan(row['loc_x']/abs(row['loc_y']))),axis=1 )<data_type_conversions>
text_features['PetID'] = X_text['PetID'].values
df['angle_bin'] = pd.cut(df.angle, 7, labels=range(7))
df['angle_bin'] = df.angle_bin.astype(int )<feature_engineering>
def get_lda_feature(data, target, source, name, n_topic = 10, random_state = 999):
retval = pd.DataFrame(data[[target, source]])
 x = retval.groupby(target, as_index=False)[source].agg({'list':(lambda x: list(x)) })
 x['sentence'] = x['list'].apply(lambda x: list(map(str,x)))
 docs = x['sentence'].tolist()
dictionary = corpora.Dictionary(docs)
 corpus = [dictionary.doc2bow(text)for text in docs]
lda = LdaMulticore(corpus, id2word=dictionary, num_topics=n_topic, random_state = random_state)
 docres = [dict(lda[doc_bow])for doc_bow in corpus]
doc_df = pd.DataFrame(docres,dtype=np.float16 ).fillna(0.001)
 doc_df.columns = ['%s_lda_%d'%(name,x)for x in range(n_topic)]
doc_df[target] = x[target]
return doc_df
df['matchup_code'] = df.matchup.apply(lambda x: 0 if(x.split(' ')[1])=='@' else 1 )<feature_engineering>
breed_color_lda = get_lda_feature(data_df, 'Breed1', 'Color_full', 'breed', n_topic=5)
 breed_breed_lda = get_lda_feature(data_df, 'Breed1', 'Breed2', 'breed_breed', n_topic=5)
 state_breed_lda = get_lda_feature(data_df, 'State', 'Breed_full', 'State_breed', n_topic=5)
 state_color_lda = get_lda_feature(data_df, 'State', 'Color_full', 'State_color',n_topic=5)
df['action_type'] = df.action_type.apply(lambda x: x.replace('-', ''))
df['action_type'] = df.action_type.apply(lambda x: x.replace('Follow Up', 'followup'))
df['action_type'] = df.action_type.apply(lambda x: x.replace('Finger Roll','fingerroll'))<feature_engineering>
data_df_proc = data_df.copy()
data_df_proc = data_df_proc.merge(senti_df, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(meta_df, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(feat_df, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_resnet50_features[['PetID','resnet_%d'%IMG_FEATURE_DIM_NN]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_dense_features[['PetID','dense_%d'%IMG_FEATURE_DIM_NN]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_resnet34_features[['PetID','res34_%d'%IMG_FEATURE_DIM_NN2]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_dense121_features[['PetID','dense121_%d'%IMG_FEATURE_DIM_NN]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_resnet34_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_resnet50_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_dense_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_dense121_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(cluster_img_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(breed_color_lda, how='left', on='Breed1')
 data_df_proc = data_df_proc.merge(breed_breed_lda, how='left', on='Breed1')
 data_df_proc = data_df_proc.merge(state_breed_lda, how='left', on='State')
 data_df_proc = data_df_proc.merge(state_color_lda, how='left', on='State')
 data_df_proc = data_df_proc.merge(text_features, how='left', on='PetID')
 print(data_df_proc.shape)
 assert data_df_proc.shape[0] == data_df.shape[0]
cv = CountVectorizer(max_features=50, stop_words=['shot'] )<create_dataframe>
early_stop = 300
verbose_eval = 100
num_rounds = 10000
to_drop_columns = ['PetID', 'Name', 'RescuerID', 'AdoptionSpeed', 'target2',
'main_breed_Type', 'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName',
'Description', 'sentiment_entities', 'meta_annots_top_desc','meta_desc',
'Chinese_desc', 'English_desc','BreedName_full','Breed1Name','Breed2Name']
shot_features = cv.fit_transform(df['action_type'] ).toarray()
shot_features = pd.DataFrame(shot_features, columns=cv.get_feature_names() )<concatenate>
torch.manual_seed(1991)
 torch.cuda.manual_seed(1991)
 torch.backends.cudnn.deterministic = True
df = pd.concat([df,shot_features], axis=1 )<data_type_conversions>
fm_cols = ['Type','age_in_year','Breed1','Breed2','Gender','Color1','Color2','Color3','MaturitySize',
'FurLength','Vaccinated','Dewormed','Sterilized','Health','State','Breed_full',
'Color_full', 'hard_interaction','img_CLUSTER_0']
fm_data = data_df_proc[fm_cols]
fm_values = []
for c in fm_cols:
fm_data.loc[:,c] = fm_data[c].fillna(0)
 fm_data.loc[:,c] = c+'_'+fm_data[c].astype(str)
 fm_values+=fm_data[c].unique().tolist()
df['game_date'] = pd.to_datetime(df.game_date )<feature_engineering>
lbe = LabelEncoder()
lbe.fit(fm_values)
 for c in fm_cols:
fm_data.loc[:,c] = lbe.transform(fm_data[c] )
df['game_date_month'] = df.game_date.dt.month
df['game_date_quarter'] = df.game_date.dt.quarter<feature_engineering>
numerical_cols = [x for x in X_train.columns if x not in to_drop_columns+fm_cols+svd_dense_features.columns.tolist() ]
df['time_remaining'] = df.apply(lambda row: row['minutes_remaining']*60+row['seconds_remaining'], axis=1 )<feature_engineering>
numerical_feats = []
for c in numerical_cols:
numerical_feats.append(data_df_proc[c].fillna(0))
numerical_feats = np.vstack(numerical_feats ).T
df['timeUnder4'] = df.time_remaining.apply(lambda x: 1 if x<4 else 0 )<feature_engineering>
MAX_LEN = 400
class PetDesDataset(Dataset):
def __init__(self, sentences, pos, fm_data, numerical_feat,
mode='train', target=None):
super(PetDesDataset, self ).__init__()
self.data = sentences
self.pos = pos
self.target = target
self.mode = mode
self.fm_data = fm_data
self.fm_dim = fm_data.shape[1]
self.numerical_feat = numerical_feat
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
if index not in range(0, self.__len__()):
return self.__getitem__(np.random.randint(0, self.__len__()))
sentence_len = min(MAX_LEN, len(self.data[index]))
sentence = torch.tensor(self.data[index][:sentence_len])
 fm_data = self.fm_data[index,:]
pos = torch.tensor(self.pos[index][:sentence_len])
 if self.mode != 'test':
return sentence, pos, sentence_len, fm_data, self.numerical_feat[index], self.target[index]
else:
return sentence, pos, sentence_len, fm_data, self.numerical_feat[index]
def nn_collate(batch):
has_label = len(batch[0])== 6
if has_label:
sentences, poses, lengths, fm_data, numerical_feats, label = zip(*batch)
 sentences = nn.utils.rnn.pad_sequence(sentences, batch_first=True ).type(torch.LongTensor)
 poses = nn.utils.rnn.pad_sequence(poses, batch_first=True ).type(torch.LongTensor)
 lengths = torch.LongTensor(lengths)
 fm_data = torch.LongTensor(fm_data)
 numerical_feats = torch.FloatTensor(numerical_feats)
 label = torch.FloatTensor(label)
 return sentences, poses, lengths, fm_data, numerical_feats, label
else:
sentences, poses, lengths, fm_data, numerical_feats = zip(*batch)
 sentences = nn.utils.rnn.pad_sequence(sentences, batch_first=True ).type(torch.LongTensor)
 poses = nn.utils.rnn.pad_sequence(poses, batch_first=True ).type(torch.LongTensor)
 lengths = torch.LongTensor(lengths)
 fm_data = torch.LongTensor(fm_data)
 numerical_feats = torch.FloatTensor(numerical_feats)
 return sentences, poses, lengths, fm_data, numerical_feats
df['distance_bin'] = pd.cut(df.shot_distance, bins=10, labels=range(10))<groupby>
def get_mask(sequences_batch, sequences_lengths, cpu=False):
batch_size = sequences_batch.size() [0]
max_length = torch.max(sequences_lengths)
 mask = torch.ones(batch_size, max_length, dtype=torch.float)
 mask[sequences_batch[:, :max_length] == 0] = 0.0
if cpu:
return mask
else:
return mask.cuda()
class Attention(nn.Module):
def __init__(self, feature_dim, bias=True, head_num=1, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.head_num = head_num
weight = torch.zeros(feature_dim, self.head_num)
 bias = torch.zeros(( 1, 1, self.head_num))
nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 self.b = nn.Parameter(bias)
 def forward(self, x, mask=None):
batch_size, step_dim, feature_dim = x.size()
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim, self.head_num)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 if mask is not None:
eij = eij * mask - 99999.9 *(1 - mask)
 a = torch.softmax(eij, dim=1)
 weighted_input = torch.bmm(x.permute(( 0,2,1)) ,
a ).view(batch_size, -1)
 return weighted_input
ang_dist = df.groupby(['angle_bin', 'distance_bin'])['shot_made_flag'].agg([np.mean],as_index= False ).reset_index()<drop_column>
embed_size = 300
class FM(nn.Module):
def __init__(self, max_features, feat_len, embed_size):
super(FM, self ).__init__()
self.bias_emb = nn.Embedding(max_features, 1)
 self.fm_emb = nn.Embedding(max_features, embed_size)
 self.feat_len = feat_len
self.embed_size = embed_size
def forward(self, x):
bias = self.bias_emb(x)
 bias = torch.sum(bias,1)
 emb = self.fm_emb(x)
 sum_feature_emb = torch.sum(emb, 1)
 square_sum_feature_emb = sum_feature_emb*sum_feature_emb
square_feature_emb = emb * emb
sum_square_feature_emb = torch.sum(square_feature_emb, 1)
 second_order = 0.5*(square_sum_feature_emb-sum_square_feature_emb)
 return bias+second_order, emb.view(-1, self.feat_len*self.embed_size)
ang_dist['group'] = range(len(ang_dist))
ang_dist.drop('mean', inplace=True, axis=1 )<merge>
class FmNlpModel(nn.Module):
def turn_on_embedding(self):
self.embedding.weight.requires_grad = True
def __init__(self, hidden_size=64, init_embedding=None, head_num=3,
fm_embed_size=8, fm_feat_len=10, fm_max_feature = 300, numerical_dim = 300,
nb_word = 40000, nb_pos = 200, pos_emb_size = 10):
super(FmNlpModel, self ).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(nb_word, 300, padding_idx=0)
 self.pos_embedding = nn.Embedding(nb_pos+100, pos_emb_size, padding_idx=0)
 if init_embedding is not None:
self.embedding.weight.data.copy_(torch.from_numpy(init_embedding))
self.embedding.weight.requires_grad = False
self.fm = FM(fm_max_feature, fm_feat_len, fm_embed_size)
 self.dropout = nn.Dropout(0.1)
 self.attention_gru = Attention(feature_dim=self.hidden_size * 2, head_num=head_num)
 self.gru = nn.GRU(embed_size+pos_emb_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru2 = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.dnn = nn.Sequential(
 nn.BatchNorm1d(numerical_dim),
nn.Dropout(0.1),
nn.Linear(numerical_dim, 256),
nn.ELU(inplace=True),
nn.Dropout(0.1),
nn.Linear(256, 128),
nn.ELU(inplace=True), 
)
 self.rnn_dnn = nn.Sequential(
 nn.BatchNorm1d(fm_embed_size+fm_feat_len*fm_embed_size +2*head_num * hidden_size+128),
nn.Dropout(0.3),
nn.Linear(fm_embed_size+fm_feat_len*fm_embed_size+2*head_num * hidden_size+128, 32),
nn.ELU(inplace=True), 
)
 self.logit = nn.Sequential(
 nn.Linear(32,1)
)
 def forward(self, x, pos_x, len_x, fm_x, numerical_x):
fm_result, fm_embed = self.fm(fm_x)
 sentence_mask = get_mask(x, len_x)
 x = x * sentence_mask.long()
sentence_mask = torch.unsqueeze(sentence_mask, -1)
 h_embedding = self.embedding(x)
 h_pos_embedding = self.pos_embedding(pos_x)
 h_embedding = torch.cat([h_embedding, h_pos_embedding],2)
 h_embedding = self.dropout(h_embedding)
 sorted_seq_lengths, indices = torch.sort(len_x, descending=True)
 _, desorted_indices = torch.sort(indices, descending=False)
 h_embedding = h_embedding[indices]
packed_inputs = nn.utils.rnn.pack_padded_sequence(h_embedding, sorted_seq_lengths, batch_first=True)
 h_gru, _ = self.gru(packed_inputs)
 h_gru2, _ = self.gru2(h_gru)
 h_gru2, _ = nn.utils.rnn.pad_packed_sequence(h_gru2, batch_first=True)
 h_gru2 = h_gru2[desorted_indices]
att_pool_gru = self.attention_gru(h_gru2, sentence_mask)
 numerical_x = self.dnn(numerical_x)
 x = torch.cat([att_pool_gru,fm_result,fm_embed,numerical_x],1)
 feat = self.rnn_dnn(x)
 out = self.logit(feat)
 return out, feat
df = df.merge(ang_dist, 'left', ['angle_bin', 'distance_bin'] )<drop_column>
X_train_numerical = numerical_feats[0:len(train)]
X_test_numerical = numerical_feats[len(train):]
X_train_seq = pd.Series(eng_sequences[0:len(train)])
 X_test_seq = pd.Series(eng_sequences[len(train):])
 X_train_pos_seq = pd.Series(pos_sequences[0:len(train)])
 X_test_pos_seq = pd.Series(pos_sequences[len(train):])
 X_train_fm = fm_data.iloc[0:len(train)].values
X_test_fm = fm_data.iloc[len(train):].values
Y_train = data_df.iloc[0:len(train)]['AdoptionSpeed'].values
predictors = df.columns.drop(['game_event_id'
, 'shot_made_flag'
, 'game_id'
, 'shot_id'
, 'game_date'
, 'minutes_remaining'
, 'seconds_remaining'
,'lat', 'lon'
, 'playoffs'
, 'team_id', 'team_name'
, 'matchup'
] )<define_variables>
train_epochs = 6
loss_fn = torch.nn.MSELoss().cuda()
oof_train_nlp = np.zeros(( X_train.shape[0], 32+1))
oof_test_nlp = []
test_set = PetDesDataset(X_test_seq.tolist() , X_test_pos_seq.tolist() , X_test_fm, X_test_numerical, mode='test')
 test_loader = DataLoader(test_set, batch_size=512, shuffle=False, num_workers=1, pin_memory=True,
collate_fn=nn_collate)
 qwks = []
rmses = []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
print('fold:', n_fold)
 hist = histogram(Y_train[train_idx].astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 training_set = PetDesDataset(X_train_seq[train_idx].tolist() ,
X_train_pos_seq[train_idx].tolist() ,
X_train_fm[train_idx],
X_train_numerical[train_idx], target = Y_train[train_idx])
 validation_set = PetDesDataset(X_train_seq[valid_idx].tolist() ,
X_train_pos_seq[valid_idx].tolist() ,
X_train_fm[valid_idx],
X_train_numerical[valid_idx],target = Y_train[valid_idx])
 training_loader = DataLoader(training_set, batch_size=512, shuffle=True, num_workers=1,
collate_fn=nn_collate)
 validation_loader = DataLoader(validation_set, batch_size=512, shuffle=False, num_workers=1,
collate_fn=nn_collate)
 model = FmNlpModel(hidden_size=48, init_embedding=embedding_matrix, head_num=10,
fm_embed_size=10, fm_feat_len=X_train_fm.shape[1], fm_max_feature=len(fm_values),
numerical_dim=X_train_numerical.shape[1],
nb_word=nb_words, nb_pos=nb_pos, pos_emb_size=10)
 model.cuda()
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.01)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=4, eta_min=0.0001)
 iteration = 0
min_val_loss = 100
since = time.time()
for epoch in range(train_epochs):
scheduler.step()
model.train()
for sentences, poses, lengths, x_fm, x_numerical, labels in training_loader:
iteration += 1
sentences = sentences.cuda()
poses = poses.cuda()
lengths = lengths.cuda()
x_fm = x_fm.cuda()
x_numerical = x_numerical.cuda()
labels = labels.type(torch.FloatTensor ).cuda().view(-1, 1)
 pred,_ = model(sentences, poses, lengths, x_fm, x_numerical)
 loss = loss_fn(pred, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
val_predicts = []
val_feats = []
with torch.no_grad() :
for sentences, poses, lengths, x_fm, x_numerical, labels in validation_loader:
sentences = sentences.cuda()
poses = poses.cuda()
lengths = lengths.cuda()
x_fm = x_fm.cuda()
x_numerical = x_numerical.cuda()
labels = labels.type(torch.FloatTensor ).cuda()
v_pred, v_feat = model(sentences, poses, lengths, x_fm, x_numerical)
 val_predicts.append(v_pred.cpu().numpy())
 val_feats.append(v_feat.cpu().numpy())
 val_predicts = np.concatenate(val_predicts)
 val_feats = np.vstack(val_feats)
 val_loss = rmse(Y_train[valid_idx], val_predicts)
 if val_loss<min_val_loss:
min_val_loss = val_loss
oof_train_nlp[valid_idx,:] = np.hstack([val_feats, val_predicts])
 test_feats = []
test_preds = []
with torch.no_grad() :
for sentences, poses, lengths, x_fm, x_numerical in test_loader:
sentences = sentences.cuda()
poses = poses.cuda()
lengths = lengths.cuda()
x_fm = x_fm.cuda()
x_numerical = x_numerical.cuda()
v_pred, feat = model(sentences, poses, lengths, x_fm, x_numerical)
 test_preds.append(v_pred.cpu().numpy())
 test_feats.append(feat.cpu().numpy())
 test_feats = np.hstack([np.vstack(test_feats), np.concatenate(test_preds)])
 pred_test_y_k = getTestScore2(val_predicts.flatten() , tr_cdf)
 qwk = quadratic_weighted_kappa(Y_train[valid_idx], pred_test_y_k)
 print(epoch, "val loss:", val_loss, "val QWK_2 = ", qwk, "elapsed time:", time.time() -since)
 oof_test_nlp.append(test_feats)
 del model
del training_set
del validation_set
del sentences
del lengths
del x_fm
del x_numerical
del poses
gc.collect()
torch.cuda.empty_cache()
qwks.append(qwk)
 rmses.append(min_val_loss )
predictors = ['action_type', 'combined_shot_type', 'shot_distance',
'shot_zone_basic', 'opponent', 'matchup_code',
'bank', 'driving', 'dunk', 'fadeaway', 'jump', 'pullup', 'running',
'slam', 'turnaround','timeUnder4', 'angle_bin','loc_x', 'loc_y','period', 'season']<categorify>
print('overall rmse: %.5f'%rmse(oof_train_nlp[:,-1], X_train['AdoptionSpeed']))
print('mean rmse =', np.mean(rmses), 'rmse std =', np.std(rmses))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))
le = LabelEncoder()
for col in predictors:
if df[col].dtype=='object':
df[col] = le.fit_transform(df[col] )<split>
oof_test_nlp = np.mean(oof_test_nlp, axis=0 )
df_test = df.loc[df.shot_made_flag.isnull() , :]
df_test.index = range(len(df_test))
df.dropna(inplace=True)
 df.index = range(len(df))<choose_model_class>
features = [x for x in X_train.columns if x not in to_drop_columns+svd_resnet50_features.columns.tolist() ]
xgb = xgboost.XGBClassifier(seed=1, learning_rate=0.01, n_estimators=500,silent=False, max_depth=7, subsample=0.6, colsample_bytree=0.6 )<choose_model_class>
def run_lgb(X_train, X_test, features, split_index):
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.02,
'subsample': 0.85,
'feature_fraction': 0.7,
'lambda_l1':0.01,
'verbosity': -1,
}
oof_train_lgb = np.zeros(( X_train.shape[0]))
oof_test_lgb = []
qwks = []
rmses = []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
since = time.time()
X_tr = X_train.iloc[train_idx]
X_val = X_train.iloc[valid_idx]
y_tr = X_tr['AdoptionSpeed'].values
y_val = X_val['AdoptionSpeed'].values
d_train = lgb.Dataset(X_tr[features], label=y_tr, 
)
 d_valid = lgb.Dataset(X_val[features], label=y_val, reference=d_train)
 watchlist = [d_valid]
print('training LGB:')
 lgb_model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=500,
early_stopping_rounds=100, 
)
 val_pred = lgb_model.predict(X_val[features])
 test_pred = lgb_model.predict(X_test[features])
 train_pred = lgb_model.predict(X_tr[features])
 oof_train_lgb[valid_idx] = val_pred
oof_test_lgb.append(test_pred)
 hist = histogram(X_tr['AdoptionSpeed'].astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 _, cutoff = getScore(train_pred, tr_cdf, True)
 pred_test_y_k = getTestScore2(val_pred, tr_cdf)
 qwk = quadratic_weighted_kappa(X_val['AdoptionSpeed'].values, pred_test_y_k)
 qwks.append(qwk)
 rmses.append(rmse(X_val['AdoptionSpeed'].values, val_pred))
print("QWK_2 = ", qwk,'elapsed time:',time.time() -since)
 oof_test_lgb = np.mean(oof_test_lgb, axis=0)
 print('overall rmse: %.5f'%rmse(oof_train_lgb, X_train['AdoptionSpeed']))
print('mean rmse =', np.mean(rmses), 'rmse std =', np.std(rmses))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))
return oof_train_lgb, oof_test_lgb
kf = KFold(n=len(df), shuffle=True,n_folds=5, random_state=50 )<compute_train_metric>
base_cols = [x for x in senti_df.columns.tolist() +meta_df.columns.tolist() +feat_df.columns.tolist() \
+cluster_img_features.columns.tolist() \
+breed_color_lda.columns.tolist() +breed_breed_lda.columns.tolist() \
+state_breed_lda.columns.tolist() \
+state_color_lda.columns.tolist() +text_features.columns.tolist() if x not in to_drop_columns]
def run_test(predictors):
all_score = []
for train_index, test_index in kf:
xgb.fit(df.loc[train_index, predictors], df.loc[train_index, 'shot_made_flag'])
 score = metrics.log_loss(df.loc[test_index, 'shot_made_flag'], xgb.predict_proba(df.loc[test_index, predictors])[:,1])
 all_score.append(score)
 print(score)
 print('Mean score =', np.array(all_score ).mean() )<train_model>
cols34 = base_cols+['res34_%d'%IMG_FEATURE_DIM_NN2]+['resnet34_SVD_%d'%i for i in range(32)]
lgb34_oof, lgb34_test = run_lgb(X_train, X_test, cols34, split_index )
run_test(predictors )<train_model>
cols50 = base_cols+['resnet_%d'%IMG_FEATURE_DIM_NN]+['resnet50_SVD_%d'%i for i in range(32)]
lgb50_oof, lgb50_test = run_lgb(X_train, X_test, cols50, split_index )
xgb.fit(df[predictors], df['shot_made_flag'] )<predict_on_test>
cols121 = base_cols+['dense121_%d'%IMG_FEATURE_DIM_NN]+['dense121_SVD_%d'%i for i in range(32)]
lgb121_oof, lgb121_test = run_lgb(X_train, X_test, cols121, split_index )
xgb.predict_proba(df_test[predictors])[:,1]<predict_on_test>
cols201 = base_cols+['dense_%d'%IMG_FEATURE_DIM_NN]+['dense_SVD_%d'%i for i in range(32)]
lgb201_oof, lgb201_test = run_lgb(X_train, X_test, cols201, split_index )
preds = xgb.predict_proba(df_test[predictors] )<predict_on_test>
df_test['shot_made_flag'] = xgb.predict_proba(df_test[predictors])[:,1]<define_variables>
submission = df_test[['shot_id', 'shot_made_flag']]<save_to_csv>
def mean_encode(train_data, test_data, columns, target_col, reg_method=None,
alpha=0, add_random=False, rmean=0, rstd=0.1, folds=1):

 encoded_cols = []
target_mean_global = train_data[target_col].mean()
for col in columns:
nrows_cat = train_data.groupby(col)[target_col].count()
target_means_cats = train_data.groupby(col)[target_col].mean()
target_means_cats_adj =(target_means_cats*nrows_cat +
target_mean_global*alpha)/(nrows_cat+alpha)
 encoded_col_test = test_data[col].map(target_means_cats_adj)
 if reg_method == 'expanding_mean':
train_data_shuffled = train_data.sample(frac=1, random_state=1)
 cumsum = train_data_shuffled.groupby(col)[target_col].cumsum() - train_data_shuffled[target_col]
cumcnt = train_data_shuffled.groupby(col ).cumcount()
encoded_col_train = cumsum/(cumcnt)
 encoded_col_train.fillna(target_mean_global, inplace=True)
 if add_random:
encoded_col_train = encoded_col_train + normal(loc=rmean, scale=rstd,
size=(encoded_col_train.shape[0]))
elif(reg_method == 'k_fold')and(folds > 1):
kf = StratifiedKFold(folds, shuffle=True, random_state=1)
 kfold = kf.split(train_data, train_data[target_col].values)
 parts = []
for tr_in, val_ind in kfold:
df_for_estimation, df_estimated = train_data.iloc[tr_in], train_data.iloc[val_ind]
nrows_cat = df_for_estimation.groupby(col)[target_col].count()
target_means_cats = df_for_estimation.groupby(col)[target_col].mean()
target_means_cats_adj =(target_means_cats*nrows_cat +
target_mean_global*alpha)/(nrows_cat+alpha)
 encoded_col_train_part = df_estimated[col].map(target_means_cats_adj)
 if add_random:
encoded_col_train_part = encoded_col_train_part + normal(loc=rmean, scale=rstd,
size=(encoded_col_train_part.shape[0]))
parts.append(encoded_col_train_part)
 encoded_col_train = pd.concat(parts, axis=0)
 encoded_col_train.fillna(target_mean_global, inplace=True)
 else:
encoded_col_train = train_data[col].map(target_means_cats_adj)
 if add_random:
encoded_col_train = encoded_col_train + normal(loc=rmean, scale=rstd,
size=(encoded_col_train.shape[0]))
encoded_col = pd.concat([encoded_col_train, encoded_col_test], axis=0)
 encoded_col[encoded_col.isnull() ] = target_mean_global
encoded_cols.append(pd.DataFrame({'mean_'+target_col+'_'+col:encoded_col}))
all_encoded = pd.concat(encoded_cols, axis=1)
 return(all_encoded[:train_data.shape[0]],
all_encoded[train_data.shape[0]:] )
submission.to_csv('last_submission.csv', index=False )<set_options>
features = [x for x in X_train.columns if x not in to_drop_columns+svd_dense_features.columns.tolist() ]
%matplotlib inline
get_ipython().run_line_magic('config', "InlineBackend.figure_format ='retina'")
 sns.set(style='white', context='notebook', palette='deep')
 sns.set(rc={'figure.figsize':(11.7,8.27)})
<load_from_csv>
xgb_features = features
train = pd.read_csv('.. /input/titanic/train.csv', index_col=0)
 test = pd.read_csv('.. /input/titanic/test.csv', index_col=0 )<prepare_x_and_y>
params = {
'objective': 'reg:linear',
'eval_metric':'rmse',
'eta': 0.01,
'tree_method':'gpu_hist',
'max_depth': 9,
'subsample': 0.85,
'colsample_bytree': 0.7,
'alpha': 0.01,
}
oof_train_xgb = np.zeros(( X_train.shape[0]))
oof_test_xgb = []
qwks = []
i = 0
test_set = xgb.DMatrix(X_test[xgb_features])
 for n_fold,(train_idx, valid_idx)in enumerate(split_index):
X_tr = X_train.iloc[train_idx]
X_val = X_train.iloc[valid_idx]
y_tr = X_tr['AdoptionSpeed'].values
y_val = X_val['AdoptionSpeed'].values
d_train = xgb.DMatrix(X_tr[xgb_features], y_tr)
 d_valid = xgb.DMatrix(X_val[xgb_features], y_val)
 watchlist = [d_valid]
since = time.time()
print('training XGB:')
 model = xgb.train(params, d_train, num_boost_round = 10000, evals=[(d_valid,'val')],
early_stopping_rounds=100,
verbose_eval=500)
 val_pred = model.predict(d_valid)
 test_pred = model.predict(test_set)
 oof_train_xgb[valid_idx] = val_pred
oof_test_xgb.append(test_pred)
 tr_cdf = get_cdf(hist)
 pred_test_y_k = getTestScore2(val_pred, tr_cdf)
 qwk = quadratic_weighted_kappa(X_val['AdoptionSpeed'].values, pred_test_y_k)
 qwks.append(qwk)
 print("QWK_2 = ", qwk,'elapsed time:',time.time() -since )
train = pd.read_csv('.. /input/titanic/train.csv', index_col=0)
 test = pd.read_csv('.. /input/titanic/test.csv', index_col=0)
 X_train = train.drop('Survived',axis=1 ).copy()
y_train = train['Survived'].copy()
X_test = test.copy()
X = pd.concat([X_train, X_test], axis=0)
 X.fillna(-999, inplace=True)
 X = pd.get_dummies(X ).astype('int')
 X = X.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
X_train = X.iloc[:len(train)]
X_test = X.iloc[len(train):]
def getModelLGBM() :
model = LGBMClassifier(objective='binary', seed=42)
 return model
def fitModelLGBM(model, X_train, y_train):
model.fit(X_train, y_train)
 return model
model = getModelLGBM()
model.fit(X_train, y_train)
 preds_lgb = model.predict(X_test )<prepare_x_and_y>
print('overall rmse: %.5f'%rmse(oof_train_xgb, X_train['AdoptionSpeed']))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))
train = pd.read_csv('.. /input/titanic/train.csv', index_col=0)
 test = pd.read_csv('.. /input/titanic/test.csv', index_col=0)
 X_train = train.drop('Survived',axis=1 ).copy()
y_train = train['Survived'].copy()
X_test = test.copy()
X = pd.concat([X_train, X_test], axis=0)
 X.fillna(-999, inplace=True)
 X = pd.get_dummies(X ).astype('int')
 X = X.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
X_train = X.iloc[:len(train)]
X_test = X.iloc[len(train):]
def getModelXGB() :
model = XGBClassifier(objective='binary:hinge',
max_depth=20,
n_estimators=2000,
gamma=0,
eta=0.05,
seed=42)
 return model
def fitModelXGB(model, X_train, y_train):
model.fit(X_train, y_train)
 return model
model = getModelXGB()
model.fit(X_train, y_train)
 preds_xgb = model.predict(X_test )<load_from_csv>
final_pred = pd.DataFrame()
final_pred['lgb34'] = getTestScore2(lgb34_oof, tr_cdf)
 final_pred['lgb50'] = getTestScore2(lgb50_oof, tr_cdf)
 final_pred['lgb121'] = getTestScore2(lgb121_oof, tr_cdf)
 final_pred['lgb201'] = getTestScore2(lgb201_oof, tr_cdf)
 final_pred['xgb'] = getTestScore2(oof_train_xgb, tr_cdf)
 final_pred['nlp_pred'] = getTestScore2(oof_train_nlp[:,-1], tr_cdf )
train = pd.read_csv('.. /input/titanic/train.csv', index_col=0)
 test = pd.read_csv('.. /input/titanic/test.csv', index_col=0)
 X_train = train.drop('Survived',axis=1 ).copy()
y_train = train['Survived'].copy()
X_test = test.copy()
X_train.fillna(-999, inplace=True)
 X_test.fillna(-999, inplace=True)
 def getModelCat() :
model = CatBoostClassifier(custom_loss=['Accuracy'],
random_seed=42,
logging_level='Silent')
 return model
def fitModelCat(model, X_train, y_train):
categorical_features_indices = np.where(X_train.dtypes != np.float)[0]
model.fit(X_train, y_train,
cat_features=categorical_features_indices,
plot=False);
return model
categorical_features_indices = np.where(X_train.dtypes != np.float)[0]
model = getModelCat()
model.fit(X_train, y_train,
cat_features = categorical_features_indices)
 preds_cat = model.predict(X_test )<train_on_grid>
final_pred['pred'] = final_pred[['lgb34','lgb50','lgb121','lgb201','xgb','nlp_pred']].mode(axis=1)[0]
skf = StratifiedKFold(n_splits=5)
 scores = np.array([])
 for train_idx, val_idx in skf.split(X_train, y_train):
cv_train_x = X_train.iloc[train_idx]
cv_train_y = y_train.iloc[train_idx]
cv_val_x = X_train.iloc[val_idx]
cv_val_y = y_train.iloc[val_idx]
model = getModelXGB()
model = fitModelXGB(model, cv_train_x, cv_train_y)
 preds = model.predict(cv_val_x)
 score = accuracy_score(cv_val_y, preds)
 scores = np.append(scores, score)
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std() * 2)) 
<load_from_csv>
qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, final_pred['pred'])
 print(qwk )
train = pd.read_csv('.. /input/titanic/train.csv', index_col=0)
 test = pd.read_csv('.. /input/titanic/test.csv', index_col=0)
 X = pd.concat([train.copy() , test.copy() ],axis=0)
 X = X.drop(['Survived','Name', 'Ticket', 'Cabin'], axis=1)
 X['Pclass'] = X['Pclass'] * -1
X['Age'] = X['Age'].fillna(X['Age'].median())
 X['Fare'] = X['Fare'].fillna(X['Fare'].median())
 X['Embarked'] = X['Embarked'].fillna(X['Embarked'].mode() [0])
 X['Fare'] = np.log1p(X['Fare'])
 X['Age'] = pd.cut(X['Age'], bins=10)
 X = pd.get_dummies(X)
 X_train = X.iloc[:len(train)]
X_test = X.iloc[len(train):]
def getModelSVM() :
model = make_pipeline(StandardScaler() ,
SGDClassifier(loss='hinge',max_iter=1000, tol=1e-3, random_state=1305))
return model
def fitModelSVM(model, X_train, y_train):
model.fit(X_train, y_train)
 return model
model = getModelSVM()
model = fitModelSVM(model, X_train, y_train)
 preds_svm = model.predict(X_test)
 X_test['Survived'] = preds_svm<load_from_csv>
final_pred = pd.DataFrame()
final_pred['lgb34'] = getTestScore2(lgb34_test, tr_cdf)
 final_pred['lgb50'] = getTestScore2(lgb50_test, tr_cdf)
 final_pred['lgb121'] = getTestScore2(lgb121_test, tr_cdf)
 final_pred['lgb201'] = getTestScore2(lgb201_test, tr_cdf)
 final_pred['xgb'] = getTestScore2(np.mean(oof_test_xgb,axis=0), tr_cdf)
 final_pred['nlp_pred'] = getTestScore2(oof_test_nlp[:,-1], tr_cdf)
 final_pred['pred'] = final_pred[['lgb34','lgb50','lgb121','lgb201','xgb','nlp_pred']].mode(axis=1)[0]
train = pd.read_csv('.. /input/titanic/train.csv', index_col=0)
 test = pd.read_csv('.. /input/titanic/test.csv', index_col=0)
 X = pd.concat([train.copy() , test.copy() ],axis=0)
 X = X.drop(['Survived','Name', 'Ticket', 'Cabin'], axis=1)
 X['Pclass'] = X['Pclass'] * -1
X['Age'] = X['Age'].fillna(X['Age'].median())
 X['Fare'] = X['Fare'].fillna(X['Fare'].median())
 X['Embarked'] = X['Embarked'].fillna(X['Embarked'].mode() [0])
 X['Fare'] = np.log1p(X['Fare'])
 X['Age'] = pd.cut(X['Age'], bins=10)
 X = pd.get_dummies(X)
 X_train = X.iloc[:len(train)]
X_test = X.iloc[len(train):]
transformer = StandardScaler()
X_train = transformer.fit_transform(X_train)
 X_test = transformer.transform(X_test)
 def getModelNN() :
model = tf.keras.models.Sequential([tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')])
 return model
def fitModelNN(model, X_train, y_train):
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 model.fit(X_train, y_train, epochs=50)
 return model
model = getModelNN()
model = fitModelNN(model, X_train, y_train)
 preds_nn = model.predict(X_test)
 preds_nn = np.round(preds_nn)
 test['Survived'] = preds_nn
test['Survived'] = test['Survived'].astype('int' )<data_type_conversions>
np.corrcoef([lgb34_test,
lgb50_test,
lgb121_test,
lgb201_test,
np.mean(oof_test_xgb,axis=0),
oof_test_nlp[:,-1]] )
preds_nn = preds_nn.reshape(418, ).astype('int')
 preds = preds_nn + preds_cat + preds_svm + preds_xgb + preds_lgb
preds = np.round(preds / 5)
 test['Survived'] = preds_xgb.astype('int' )<data_type_conversions>
X_train_stacking = np.vstack([lgb34_oof,
lgb50_oof,
lgb121_oof,
lgb201_oof,
oof_train_xgb,
oof_train_nlp[:,-1]] ).T
X_test_stacking = np.vstack([lgb34_test,
lgb50_test,
lgb121_test,
lgb201_test,
np.mean(oof_test_xgb,axis=0),
oof_test_nlp[:,-1]] ).T
stacking_train = np.zeros(( X_train.shape[0]))
stacking_test = []
rmses, qwks = [], []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
X_tr = X_train_stacking[train_idx]
X_val = X_train_stacking[valid_idx]
y_tr = X_train.iloc[train_idx]['AdoptionSpeed'].values
y_val = X_train.iloc[valid_idx]['AdoptionSpeed'].values
since = time.time()
print('training Ridge:')
 model = Ridge(alpha=1)
 model.fit(X_tr, y_tr)
 print(model.coef_)
 val_pred = model.predict(X_val)
 test_pred = model.predict(X_test_stacking)
 stacking_train[valid_idx] = val_pred
stacking_test.append(test_pred)
 loss = rmse(Y_train[valid_idx], val_pred)
 hist = histogram(y_tr.astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 pred_test_y_k = getTestScore2(val_pred, tr_cdf)
 qwk = quadratic_weighted_kappa(y_val, pred_test_y_k)
 qwks.append(qwk)
 rmses.append(loss)
 print("RMSE=",loss, "QWK_2 = ", qwk,'elapsed time:',time.time() -since)
 stacking_test = np.mean(stacking_test, axis=0)
 print('mean rmse:',np.mean(rmses), 'rmse std:', np.std(rmses))
print('mean qwk:', np.mean(qwks), 'qwk std:', np.std(qwks))
test['Survived'] = preds_xgb.astype('int' )<data_type_conversions>
hist = histogram(X_train['AdoptionSpeed'].astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 train_predictions = getTestScore2(stacking_train, tr_cdf)
 test_predictions = getTestScore2(stacking_test, tr_cdf)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, train_predictions)
 print("QWK = ", qwk )
<save_to_csv><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': stacking_test.astype(np.int32)})
 submission.to_csv('submission.csv', index=False )