Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<count_values>
list_of_languages = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_languages for i in j] ).most_common(15 )<feature_engineering>
dataset_path = ".. /input/global-wheat-detection/"
trainImg_path = dataset_path+"train/"
train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )<define_variables>
f = open('wheat.yaml', 'w')
 f.write('train:./training_data
')
 f.write('val:./training_data
')
 f.write('test:./testing_data
')
 f.write('
')
 f.write('nc: 1
')
 f.write('
')
 f.write("names: ['wheat']
")
 f.close()
for i, e in enumerate(train['Keywords'][:5]):
print(i, e )<count_values>
print('Number of Keywords in films')
 train['Keywords'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )<string_transform>
!rm submission.csv
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 plt.figure(figsize =(16, 12))
text = ' '.join(['_'.join(i.split(' ')) for j in list_of_keywords for i in j])
 wordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,
width=1200, height=1000 ).generate(text)
 plt.imshow(wordcloud)
 plt.title('Top keywords')
 plt.axis("off")
 plt.show()<feature_engineering>
','')
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )<define_variables>
!cat./submission.csv
for i, e in enumerate(train['cast'][:1]):
print(i, e )<count_values>
df = pd.read_csv('submission.csv')
 print(df )
print('Number of casted persons in films')
 train['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )<count_values>
train_dir ='/kaggle/input/global-wheat-detection/train/'
test_dir = '/kaggle/input/global-wheat-detection/test/'
train = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 weights = '.. /input/fasterrcnn-resnet50-fpn/fasterrcnn_resnet50_fpn(1 ).pth'
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(15 )<count_unique_values>
sample = pd.read_csv('.. /input/global-wheat-detection/sample_submission.csv')
 sample.shape
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_genders for i in j] ).most_common()<count_values>
class Wheatdatasets(Dataset):
def __init__(self,dataframe,image_dir,transforms = None):
super().__init__()
self.image_id = dataframe['image_id'].unique()
self.df = dataframe
self.img_dir = image_dir
self.transforms = transforms
def __getitem__(self,index:int):
image_id = self.image_id[index]
record = self.df[self.df['image_id']==image_id]
image = cv2.imread(self.img_dir+image_id+'.jpg',cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB ).astype(np.float32)
 image = image/255.0
if self.transforms:
sample ={
'image':image
}
sample =self.transforms(**sample)
 image = sample['image']
return image,image_id
def __len__(self)-> int:
return self.image_id.shape[0]
list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )<feature_engineering>
def get_transform() :
return al.Compose([
ToTensorV2(p=1.0)
 ] )
train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )<count_values>
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained = False,pretrained_backbone=False )
print('Number of casted persons in films')
 train['crew'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )<count_values>
x = model.to(device )
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )<count_values>
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = Wheatdatasets(sample,test_dir,get_transform())
 test_dataloader = DataLoader(
 test_dataset,
batch_size=4,
shuffle =False,
num_workers =4,drop_last = False,
collate_fn = collate_fn)
list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15 )<count_values>
def format_string(boxes,score):
pred_strings = []
for j in zip(score,boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_genders for i in j] ).most_common(15 )<count_values>
detection_threshold = 0.5
results = []
for images, image_ids in test_dataloader:
images = list(image.to(device)for image in images)
 outputs = model(images)
 for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
boxes = boxes[scores >= detection_threshold].astype(np.int32)
 scores = scores[scores >= detection_threshold]
image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_string(boxes, scores)
 }
results.append(result )
list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_departments for i in j] ).most_common(14 )<count_values>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()
list_of_crew_names = train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values
Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )<feature_engineering>
train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
for g in top_crew_names:
train['crew_name_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_crew_names:
test['crew_name_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )<feature_engineering>
test_df.to_csv('submission.csv', index=False )
train['log_revenue'] = np.log1p(train['revenue'] )<feature_engineering>
img_id = list(test_df['image_id'])
 print(img_id )
train['log_budget'] = np.log1p(train['budget'])
 test['log_budget'] = np.log1p(test['budget'] )<feature_engineering>
for i in img_id:
q = df.loc[df['image_id'] == i]
if q.empty:
print(i)
 p = test_df.loc[test_df['image_id'] == i]['PredictionString'].item()
df2 = pd.DataFrame([[i, p]], columns=['image_id','PredictionString'])
 print(df2)
 df = df.append(df2 )
train['has_homepage'] = 0
train.loc[train['homepage'].isnull() == False, 'has_homepage'] = 1
test['has_homepage'] = 0
test.loc[test['homepage'].isnull() == False, 'has_homepage'] = 1<train_model>
df.to_csv('submission.csv', index=False )