Kaggle competition: Natural Language Processing with Disaster Tweets
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
nltk.download('wordnet')
 nltk.download('punkt')
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None], [TTAVerticalFlip() , None],[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df.sample(10 )
def make_tta_predictions(images, score_threshold=0.5):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda() , torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
'labels': labels[indexes],
})
 predictions.append(result)
 return predictions<categorify>
print("Training :")
 print("Length of the data :", len(df))
print(df.isnull().sum() )
test_transforms = Compose([
Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
print("Test :")
 print("Length of the data :", len(df_test))
print(df_test.isnull().sum() )
class TestDataset(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[2]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self):
return self.image_ids.shape[0]<concatenate>
tokens = word_tokenize(df["text"][0])
 tokens = [word.lower() for word in tokens]
print(df["text"][0])
 print(tokens )
def collate_fn(batch):
return tuple(zip(*batch))<create_dataframe>
words = [word for word in tokens if word.isalpha() ]
print(words )
test_set = TestDataset(image_ids=np.array([path.split('/')[-1][:-4] for path in test_paths]),transforms=test_transforms)
 test_loader = DataLoader(test_set,batch_size=4,shuffle=False,num_workers=2,drop_last=False,collate_fn=collate_fn )<define_variables>
stop_words = set(stopwords.words("english"))
words = [word for word in words if not word in stop_words]
print(words )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<save_to_csv>
porter = PorterStemmer()
stemmed = [porter.stem(word)for word in words]
print(stemmed )
SUBMISSION_PATH = '/kaggle/working'
submission_id = 'submission'
submission_path = os.path.join(SUBMISSION_PATH, '{}.csv'.format(submission_id))
sample_submission = pd.DataFrame(submission, columns=["image_id","PredictionString"])
 sample_submission.to_csv(submission_path, index=False)
 submission_df = pd.read_csv(submission_path)
 submission_df<define_variables>
lemmatizer = WordNetLemmatizer()
lemmatized = [lemmatizer.lemmatize(word)for word in words]
print(lemmatized )
all_path = glob('.. /input/global-wheat-detection/test/*')
 DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'<categorify>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 stop_words = set(stopwords.words("english"))
def mytokenize(text, normalize=None, stop_words=stop_words):
text = remove_html(text)
 text = remove_URL(text)
 text = remove_emoji(text)
 text = text.lower()
text = word_tokenize(text)
 text = [word for word in text if word.isalpha() ]
text = [word for word in text if not word in stop_words]
if normalize == "s":
porter = PorterStemmer()
text = [porter.stem(word)for word in text]
if normalize == "l":
lemmatizer = WordNetLemmatizer()
text = [lemmatizer.lemmatize(word)for word in text]
return text
mytokenize_s = partial(mytokenize, normalize='s')
 mytokenize_l = partial(mytokenize, normalize='l')
 def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 df['tokens'] = df['text'].apply(mytokenize_l)
 df_test['tokens'] = df_test['text'].apply(mytokenize_l )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
def join_list(tab):
return " ".join(tab)
 df["text_preprocessed"] = df["tokens"].apply(join_list)
 df_test["text_preprocessed"] = df_test["tokens"].apply(join_list)
 def transform_keyword(word):
return word.split('%20')
 df["keyword"] = df.keyword.fillna(" ")
 df_test["keyword"] = df_test.keyword.fillna(" ")
 df["keyword"] = df["keyword"].apply(transform_keyword ).apply(join_list)
 df_test["keyword"] = df_test["keyword"].apply(transform_keyword ).apply(join_list)
 df["text_preprocessed"] = df["keyword"] + " " + df["text_preprocessed"]
df_test["text_preprocessed"] = df_test["keyword"] + " " + df_test["text_preprocessed"]
df["tokens"] = df["text_preprocessed"].apply(str.split)
 df_test["tokens"] = df_test["text_preprocessed"].apply(str.split)
 df.sample()
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 sk_doc2bow = CountVectorizer()
sk_doc2bow.fit(X_all)
 del X_all
X = sk_doc2bow.transform(df["text_preprocessed"])
 X_test = sk_doc2bow.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_d2b.csv",index=False )
class CrossEntropyLabelSmooth(nn.Module):

 def __init__(self, num_classes, epsilon=0.1, use_gpu=True):
super(CrossEntropyLabelSmooth, self ).__init__()
self.num_classes = num_classes
self.epsilon = epsilon
self.use_gpu = use_gpu
self.logsoftmax = nn.LogSoftmax(dim=1)
 def forward(self, inputs, targets):

 log_probs = self.logsoftmax(inputs)
 targets = torch.zeros(log_probs.size() ).scatter_(1, targets.unsqueeze(1 ).data.cpu() , 1)
 if self.use_gpu: targets = targets.cuda()
targets =(1 - self.epsilon)* targets + self.epsilon / self.num_classes
loss =(- targets * log_probs ).mean(0 ).sum()
return loss<compute_train_metric>
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 tfidf = TfidfVectorizer(stop_words = 'english')
 tfidf.fit(X_all)
 del X_all
X = tfidf.transform(df["text_preprocessed"])
 X_test = tfidf.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )
def fastrcnn_loss(class_logits, box_regression, labels, regression_targets):

 labels = torch.cat(labels, dim=0)
 regression_targets = torch.cat(regression_targets, dim=0)
 labal_smooth_loss = CrossEntropyLabelSmooth(2)
 classification_loss = labal_smooth_loss(class_logits, labels)
 sampled_pos_inds_subset = torch.nonzero(labels > 0 ).squeeze(1)
 labels_pos = labels[sampled_pos_inds_subset]
N, num_classes = class_logits.shape
box_regression = box_regression.reshape(N, -1, 4)
 box_loss = det_utils.smooth_l1_loss(
 box_regression[sampled_pos_inds_subset, labels_pos],
regression_targets[sampled_pos_inds_subset],
beta=1 / 9,
size_average=False, 
)
 box_loss = box_loss / labels.numel()
return classification_loss, box_loss<choose_model_class>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_tf.csv",index=False )
def fpn_backbone_269(pretrained, norm_layer=misc_nn_ops.FrozenBatchNorm2d, trainable_layers=3):
print(f'
Pretarined is {pretrained}')
 backbone = resnest269e(pretrained=pretrained)
 assert trainable_layers <= 5 and trainable_layers >= 0
layers_to_train = ['layer4', 'layer3', 'layer2', 'layer1', 'conv1'][:trainable_layers]
for name, parameter in backbone.named_parameters() :
if all([not name.startswith(layer)for layer in layers_to_train]):
parameter.requires_grad_(False)
 return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}
in_channels_stage2 = backbone.inplanes // 8
in_channels_list = [
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
]
out_channels = 256
return BackboneWithFPN(backbone, return_layers, in_channels_list, out_channels)
 class WheatDetector_269(nn.Module):
def __init__(self, **kwargs):
super(WheatDetector_269, self ).__init__()
self.backbone = fpn_backbone_269(pretrained=False)
 self.base = FasterRCNN(self.backbone, num_classes = 2, **kwargs)
 self.base.roi_heads.fastrcnn_loss = self.fastrcnn_loss
def fastrcnn_loss(self, class_logits, box_regression, labels, regression_targets):

 labels = torch.cat(labels, dim=0)
 regression_targets = torch.cat(regression_targets, dim=0)
 labal_smooth_loss = CrossEntropyLabelSmooth(2)
 classification_loss = labal_smooth_loss(class_logits, labels)
 sampled_pos_inds_subset = torch.nonzero(labels > 0 ).squeeze(1)
 labels_pos = labels[sampled_pos_inds_subset]
N, num_classes = class_logits.shape
box_regression = box_regression.reshape(N, -1, 4)
 box_loss = det_utils.smooth_l1_loss(
 box_regression[sampled_pos_inds_subset, labels_pos],
regression_targets[sampled_pos_inds_subset],
beta=1 / 9,
size_average=False, 
)
 box_loss = box_loss / labels.numel()
return classification_loss, box_loss
def forward(self, images, targets=None):
return self.base(images, targets )<choose_model_class>
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 tfidf = TfidfVectorizer(stop_words = 'english', min_df=10)
 tfidf.fit(X_all)
 del X_all
X = tfidf.transform(df["text_preprocessed"])
 X_test = tfidf.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )
def fpn_backbone_101(pretrained, norm_layer=misc_nn_ops.FrozenBatchNorm2d, trainable_layers=3):
print(f'
Pretarined is {pretrained}')
 backbone = resnest101e(pretrained=pretrained)
 assert trainable_layers <= 5 and trainable_layers >= 0
layers_to_train = ['layer4', 'layer3', 'layer2', 'layer1', 'conv1'][:trainable_layers]
for name, parameter in backbone.named_parameters() :
if all([not name.startswith(layer)for layer in layers_to_train]):
parameter.requires_grad_(False)
 return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}
in_channels_stage2 = backbone.inplanes // 8
in_channels_list = [
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
]
out_channels = 256
return BackboneWithFPN(backbone, return_layers, in_channels_list, out_channels)
 class WheatDetector_101(nn.Module):
def __init__(self, **kwargs):
super(WheatDetector_101, self ).__init__()
self.backbone = fpn_backbone_101(pretrained=False)
 self.base = FasterRCNN(self.backbone, num_classes = 2, **kwargs)
 self.base.roi_heads.fastrcnn_loss = self.fastrcnn_loss
def fastrcnn_loss(self, class_logits, box_regression, labels, regression_targets):

 labels = torch.cat(labels, dim=0)
 regression_targets = torch.cat(regression_targets, dim=0)
 labal_smooth_loss = CrossEntropyLabelSmooth(2)
 classification_loss = labal_smooth_loss(class_logits, labels)
 sampled_pos_inds_subset = torch.nonzero(labels > 0 ).squeeze(1)
 labels_pos = labels[sampled_pos_inds_subset]
N, num_classes = class_logits.shape
box_regression = box_regression.reshape(N, -1, 4)
 box_loss = det_utils.smooth_l1_loss(
 box_regression[sampled_pos_inds_subset, labels_pos],
regression_targets[sampled_pos_inds_subset],
beta=1 / 9,
size_average=False, 
)
 box_loss = box_loss / labels.numel()
return classification_loss, box_loss
def forward(self, images, targets=None):
return self.base(images, targets )<choose_model_class>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_tf.csv",index=False )
def load_net_269(checkpoint_path):
model = WheatDetector_269()
checkpoint=torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model.eval() ;
return model.cuda()
def load_net_101(checkpoint_path):
model = WheatDetector_101()
checkpoint=torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model.eval() ;
return model.cuda()
models = [
load_net_101('.. /input/resnest101/fold0-best-checkpoint.bin'),
load_net_101('.. /input/resnest101/fold1-best-checkpoint.bin'),
load_net_101('.. /input/resnest101/fold2-best-checkpoint.bin'),
load_net_101('.. /input/resnest101/fold3-best-checkpoint.bin'),
load_net_101('.. /input/resnest101/fold4-best-checkpoint.bin'),
load_net_269('.. /input/resnest269/fold0-best-checkpoint.bin'),
load_net_269('.. /input/resnest269/fold1-best-checkpoint.bin'),
load_net_269('.. /input/resnest269/fold2-best-checkpoint.bin'),
load_net_269('.. /input/resnest269/fold3-best-checkpoint.bin'),
load_net_269('.. /input/resnest269/fold4-best-checkpoint.bin')
 ]<categorify>
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 tfidf = TfidfVectorizer(stop_words = 'english', min_df=5, ngram_range=(1, 3))
tfidf.fit(X_all)
 del X_all
X = tfidf.transform(df["text_preprocessed"])
 X_test = tfidf.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42)
 row_tf = X_train[0].toarray() [0]
print(row_tf.shape)
 print(row_tf)
 row_tf[row_tf != 0]
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_tf.csv",index=False )
def process_det(index, outputs, score_threshold=0.5):
boxes = outputs[index]['boxes'].data.cpu().numpy()
scores = outputs[index]['scores'].data.cpu().numpy()
boxes =(boxes ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<concatenate>
X_all = pd.concat([df["tokens"], df_test["tokens"]] ).reset_index(drop=True)
 print(len(X_all))
mydict = Dictionary(X_all)
 corpus = [mydict.doc2bow(text)for text in X_all]
tf_model = TfidfModel(corpus)
 corpus_tf = tf_model[corpus]
print(len(corpus_tf))
lsi_model = LsiModel(corpus_tf, id2word=mydict, num_topics=200 )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
mydict.num_docs
def make_tta_predictions(images,net, score_threshold=0.1):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = net(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.5, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
for i, word in enumerate(mydict.items()):
print(word)
 if i > 9:
break
fold1 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[0])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold1[image_id] = [boxes, scores, labels]
print('
Completed')
 fold2 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[1])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold2[image_id] = [boxes, scores, labels]
print('
Completed')
 fold3 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[2])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold3[image_id] = [boxes, scores, labels]
print('
Completed')
 fold4 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[3])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold4[image_id] = [boxes, scores, labels]
print('
Completed')
<predict_on_test>
lsi_model.num_topics
fold5 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[4])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold5[image_id] = [boxes, scores, labels]
print('
Completed')
 fold6 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[5])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold6[image_id] = [boxes, scores, labels]
print('
Completed')
 fold7 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[6])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold7[image_id] = [boxes, scores, labels]
print('
Completed')
 fold8 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[7])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold8[image_id] = [boxes, scores, labels]
print('
Completed')
 fold9 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[8])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold9[image_id] = [boxes, scores, labels]
print('
Completed')
 fold10 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[9])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold10[image_id] = [boxes, scores, labels]
print('
Completed')
<train_model>
def transform(df):
corpus = [mydict.doc2bow(text)for text in df]
corpus = tf_model[corpus]
corpus = lsi_model[corpus]
return corpus
print(len(df["tokens"]))
X = transform(df["tokens"])
 print(len(X))
X_test = transform(df_test["tokens"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )
def run_last_wbf(model1,model2,model3,model4,
model5,model6,model7,model8,model9,model10,
iou_thr=0.5,skip_box_thr=0.43):
box1,scores1,labels1 = model1
box2,scores2,labels2 = model2
box3,scores3,labels3 = model3
box4,scores4,labels4 = model4
box1 = box1/1023
box2 = box2/1023
box3 = box3/1023
box4 = box4/1023
box5,scores5,labels5 = model5
box6,scores6,labels6 = model6
box7,scores7,labels7 = model7
box8,scores8,labels8 = model8
box9,scores9,labels9 = model9
box10,scores10,labels10 = model10
box5 = box5/1023
box6 = box6/1023
box7 = box7/1023
box8 = box8/1023
box9 = box9/1023
box10 = box10/1023
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion([box1,box2,box3,box4,box5,box6,box7,box8,box9,box10],
[scores1,scores2,scores3,scores4,scores5,scores6,scores7,scores8,scores9,scores10],
[labels1,labels2,labels3,labels4,labels5,labels6,labels7,labels8,labels9,labels10],
weights=None,iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes,scores,labels<feature_engineering>
def make_vec(X, num_top):
matrix = np.zeros(( len(X), num_top))
for i, row in enumerate(X):
matrix[i, list(map(lambda tup: tup[0], row)) ] = list(map(lambda tup: tup[1], row))
return matrix
make_vec(X_train, lsi_model.num_topics ).shape
w = {}
for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes,scores,labels = run_last_wbf(fold1[image_id],fold2[image_id],fold3[image_id],fold4[image_id],
fold5[image_id],fold6[image_id],fold7[image_id],fold8[image_id],fold9[image_id],fold10[image_id])
 boxes =(boxes*1023)
 indexes = np.where(scores > 0.50)[0]
w[image_id] = [boxes,scores,labels]<categorify>
def transform(df, tf_model, model):
corpus = [mydict.doc2bow(text)for text in df]
corpus = tf_model[corpus]
corpus = model[corpus]
corpus = make_vec(corpus, model.num_topics)
 return corpus
print(len(df["tokens"]))
X = transform(df["tokens"], tf_model, lsi_model)
 print(len(X))
X_test = transform(df_test["tokens"], tf_model, lsi_model)
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lsi.csv",index=False )
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
model = RandomForestClassifier(n_estimators=200, max_depth=None, random_state=42, n_jobs=-1 ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = RandomForestClassifier(n_estimators=200, max_depth=None, random_state=42, n_jobs=-1 ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lsi_rf.csv",index=False )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
lda_model = LdaModel(corpus_tf, id2word=mydict, num_topics=100, dtype=np.float64 )
class CrossEntropyLabelSmooth(nn.Module):

 def __init__(self, num_classes, epsilon=0.1, use_gpu=True):
super(CrossEntropyLabelSmooth, self ).__init__()
self.num_classes = num_classes
self.epsilon = epsilon
self.use_gpu = use_gpu
self.logsoftmax = nn.LogSoftmax(dim=1)
 def forward(self, inputs, targets):

 log_probs = self.logsoftmax(inputs)
 targets = torch.zeros(log_probs.size() ).scatter_(1, targets.unsqueeze(1 ).data.cpu() , 1)
 if self.use_gpu: targets = targets.cuda()
targets =(1 - self.epsilon)* targets + self.epsilon / self.num_classes
loss =(- targets * log_probs ).mean(0 ).sum()
return loss
def fastrcnn_loss(class_logits, box_regression, labels, regression_targets):

 labels = torch.cat(labels, dim=0)
 regression_targets = torch.cat(regression_targets, dim=0)
 labal_smooth_loss = CrossEntropyLabelSmooth(2)
 classification_loss = labal_smooth_loss(class_logits, labels)
 sampled_pos_inds_subset = torch.nonzero(labels > 0 ).squeeze(1)
 labels_pos = labels[sampled_pos_inds_subset]
N, num_classes = class_logits.shape
box_regression = box_regression.reshape(N, -1, 4)
 box_loss = det_utils.smooth_l1_loss(
 box_regression[sampled_pos_inds_subset, labels_pos],
regression_targets[sampled_pos_inds_subset],
beta=1 / 9,
size_average=False, 
)
 box_loss = box_loss / labels.numel()
return classification_loss, box_loss
def fpn_backbone(pretrained, norm_layer=misc_nn_ops.FrozenBatchNorm2d, trainable_layers=3):
print(f'
Pretarined is {pretrained}')
 backbone = resnest101e(pretrained=pretrained)
 assert trainable_layers <= 5 and trainable_layers >= 0
layers_to_train = ['layer4', 'layer3', 'layer2', 'layer1', 'conv1'][:trainable_layers]
for name, parameter in backbone.named_parameters() :
if all([not name.startswith(layer)for layer in layers_to_train]):
parameter.requires_grad_(False)
 return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}
in_channels_stage2 = backbone.inplanes // 8
in_channels_list = [
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
]
out_channels = 256
return BackboneWithFPN(backbone, return_layers, in_channels_list, out_channels)
 class WheatDetector(nn.Module):
def __init__(self, **kwargs):
super(WheatDetector, self ).__init__()
self.backbone = fpn_backbone(pretrained=False)
 self.base = FasterRCNN(self.backbone, num_classes = 2, **kwargs)
 self.base.roi_heads.fastrcnn_loss = self.fastrcnn_loss
def fastrcnn_loss(self, class_logits, box_regression, labels, regression_targets):

 labels = torch.cat(labels, dim=0)
 regression_targets = torch.cat(regression_targets, dim=0)
 labal_smooth_loss = CrossEntropyLabelSmooth(2)
 classification_loss = labal_smooth_loss(class_logits, labels)
 sampled_pos_inds_subset = torch.nonzero(labels > 0 ).squeeze(1)
 labels_pos = labels[sampled_pos_inds_subset]
N, num_classes = class_logits.shape
box_regression = box_regression.reshape(N, -1, 4)
 box_loss = det_utils.smooth_l1_loss(
 box_regression[sampled_pos_inds_subset, labels_pos],
regression_targets[sampled_pos_inds_subset],
beta=1 / 9,
size_average=False, 
)
 box_loss = box_loss / labels.numel()
return classification_loss, box_loss
def forward(self, images, targets=None):
return self.base(images, targets)
 def load_net(checkpoint_path):
model = WheatDetector()
checkpoint=torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model.eval() ;
return model.cuda()
models = [
load_net('.. /input/fold-1-resnest101-512/best-checkpoint.bin'),
load_net('.. /input/fold2resnest101512/best-checkpoint.bin'),
load_net('.. /input/fold3resnest101512/best-checkpoint.bin'),
load_net('.. /input/resnest101fold4/best-checkpoint.bin'),
load_net('.. /input/resnest101fold3/best-checkpoint.bin')
 ]<categorify>
lda_model.num_topics
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
print(len(df["tokens"]))
X = transform(df["tokens"], tf_model, lda_model)
 print(len(X))
X_test = transform(df_test["tokens"], tf_model, lda_model)
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )
def make_tta_predictions(images,net, score_threshold=0.1):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = net(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.5, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*1023
return boxes, scores, labels<predict_on_test>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lda.csv",index=False )
fold0 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[0])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold0[image_id] = [boxes, scores, labels]
print('
Completed')
 fold1 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[1])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold1[image_id] = [boxes, scores, labels]
print('
Completed')
 fold2 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[2])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold2[image_id] = [boxes, scores, labels]
print('
Completed')
 fold3 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[3])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold3[image_id] = [boxes, scores, labels]
print('
Completed')
 fold4 = {}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,models[4])
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 image_id = image_ids[i]
fold4[image_id] = [boxes, scores, labels]
print('
Completed' )<load_pretrained>
model = RandomForestClassifier(n_estimators=200, max_depth=None, random_state=42, n_jobs=-1 ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = RandomForestClassifier(n_estimators=200, max_depth=None, random_state=42, n_jobs=-1 ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lda_rf.csv",index=False )
def run_last_wbf(model1,model2,model3,model4,model5,
iou_thr=0.5,skip_box_thr=0.43):
box1,scores1,labels1 = model1
box2,scores2,labels2 = model2
box3,scores3,labels3 = model3
box4,scores4,labels4 = model4
box5,scores5,labels5 = model5
box1 = box1/1023
box2 = box2/1023
box3 = box3/1023
box4 = box4/1023
box5 = box5/1023
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion([box1,box2,box3,box4,box5],
[scores1,scores2,scores3,scores4,scores5],
[labels1,labels2,labels3,labels4,labels5],
weights=None,iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes,scores,labels
x = {}
for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes,scores,labels = run_last_wbf(fold1[image_id],fold2[image_id],fold0[image_id],fold3[image_id],fold4[image_id])
 boxes =(boxes*1023)
 x[image_id] = [boxes,scores,labels]<load_pretrained>
print(len(df["tokens"]))
X_1 = transform(df["tokens"], tf_model, lda_model)
 X_2 = transform(df["tokens"], tf_model, lsi_model)
 X = np.hstack(( X_1, X_2))
print(len(X))
X_test = transform(df_test["tokens"], tf_model, lda_model)
 X_1 = transform(df_test["tokens"], tf_model, lda_model)
 X_2 = transform(df_test["tokens"], tf_model, lsi_model)
 X_test = np.hstack(( X_1, X_2))
X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42)
 X_train.shape
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/efficientdetd51/0.bin'),
load_net('.. /input/efficientdetd51/1.bin'),
load_net('.. /input/efficientdetd51/2.bin'),
load_net('.. /input/efficientdetd51/3.bin'),
load_net('.. /input/efficientdetd51/4.bin'),
load_net('.. /input/efficientdetd51/5.bin'),
load_net('.. /input/efficientdetd51/6.bin'),
load_net('.. /input/efficientdetd51/7.bin'),
]<data_type_conversions>
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lda_lsi.csv",index=False )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test/'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_predictions(images, score_threshold=0.35):
images = torch.stack(images ).cuda().float()
predictions = []
for net in models:
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<compute_train_metric>
model = RandomForestClassifier(n_estimators=200, max_depth=None, random_state=42, n_jobs=-1 ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = RandomForestClassifier(n_estimators=200, max_depth=None, random_state=42, n_jobs=-1 ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lda_lsi_rf.csv",index=False )
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.432, skip_box_thr=0.397, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
X_all = pd.concat([df["tokens"], df_test["tokens"]] ).reset_index(drop=True)
 model_w2v = Word2Vec(sentences=X_all, size=50, window=3, min_count=1, workers=-1)
 del X_all
y={}
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i,)
 boxes =(boxes*2)
 y[image_ids[i]] = [boxes,scores,labels]<load_pretrained>
vector = model_w2v.wv['armageddon']
print(vector )
models = [
load_net('.. /input/efficientdetd52/0.bin'),
load_net('.. /input/efficientdetd52/1.bin'),
load_net('.. /input/efficientdetd52/2.bin'),
load_net('.. /input/efficientdetd52/3.bin'),
load_net('.. /input/efficientdetd52/4.bin'),
load_net('.. /input/kaggleeffnet/plabel_model/last-checkpoint1.bin'),
load_net('.. /input/kaggleeffnet/plabel_model/best-checkpoint-004epoch.bin')
 ]<predict_on_test>
print(len(df["tokens"]))
X = [np.mean([model_w2v.wv[text] for text in texts], axis=0)for texts in df["tokens"]]
X = np.array(X)
 print(len(X))
X_test = [np.mean([model_w2v.wv[text] for text in texts], axis=0)if len(texts)!= 0 else np.zeros(50)for texts in df_test["tokens"]]
X_test = np.array(X_test)
 X_test.shape
z={}
for images, image_ids in data_loader:
predictions = make_predictions(images,0.4337)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i,iou_thr=0.4637,skip_box_thr=0.12)
 boxes =(boxes*2)
 z[image_ids[i]] = [boxes,scores,labels]<define_variables>
X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42)
 print(X_train.shape)
 model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_w2v.csv",index=False )
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.432, skip_box_thr=0.397, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<categorify>
w2v_model = gensim.downloader.load("word2vec-google-news-300")
 type(w2v_model )
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<choose_model_class>
X_all = pd.concat([df["tokens"], df_test["tokens"]] ).reset_index(drop=True)
 documents = [TaggedDocument(doc, [i])for i, doc in enumerate(X_all)]
del X_all
model_d2v = Doc2Vec(documents, vector_size=500, window=2, min_count=1, workers=-1 )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/effdet1024/best-checkpoint-1024.bin' )<predict_on_test>
print(len(df["tokens"]))
X = [model_d2v.infer_vector(texts)for texts in df["tokens"]]
X = np.array(X)
 print(len(X))
X_test = [model_d2v.infer_vector(texts)for texts in df_test["tokens"]]
X_test = np.array(X_test )
l={}
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 l[image_ids[i]] = [boxes, scores, labels]<feature_engineering>
X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42)
 print(X_train.shape)
 model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_lda_lsi.csv",index=False )