Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
!pip install pytorch-pretrained-bert pytorch-nlp
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
warnings.filterwarnings('ignore')
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<concatenate>
nltk.download('punkt' )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
pd.options.display.max_colwidth = 100
seed_val=42
tf.random.set_seed(seed_val)
 random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 train_df.head()
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
print(train_df.info())
 print("
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
train_df['target'].value_counts(normalize=True )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<set_options>
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test_df.head()
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<define_variables>
train_df[~train_df['keyword'].isnull() ][['keyword', 'text']]
TRAIN_DATA_PATH = '.. /input/global-wheat-detection/train/'
TRAIN_CSV_PATH = '.. /input/global-wheat-detection/train.csv'
TEST_DATA_PATH = '.. /input/global-wheat-detection/test/'
if len(os.listdir('.. /input/global-wheat-detection/test/')) >11:
PL_OPT = True
else:
PL_OPT = False
warmup_opt = True
warmup_epoch = 1
PL_lr_sche = 'plat'
cos_lr_min = 1e-8
PL_lr = 0.0001
PL_batchsize = 1
PL_epoch = 5
PL_thr = 0.25
img_size = 1024
OOF_thr = 0.3
WBF_iou_thr = 0.44
WBF_skip_thr = 0.43
path1 = [
'.. /input/212121/21_all - 0.39332 train - epoch89.bin'
]
path2 = [
'.. /input/292929/29_0 - 0.40681 train - 0.37596 val - epoch94.bin',
]<define_search_model>
len(train_df['location'].unique() )
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=img_size, width=img_size, p=1),
A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=img_size, width=img_size, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transforms() :
return A.Compose([
A.Resize(height=img_size, width=img_size, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<load_from_csv>
train_df.drop(columns=['keyword', 'location'], inplace=True )
marking = pd.read_csv(TRAIN_CSV_PATH)
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<data_type_conversions>
spacy_en = spacy.load('en_core_web_sm', disable=['parser','ner'])
 bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
class DatasetT(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{TEST_DATA_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
dataset = DatasetT(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_DATA_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<categorify>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes<choose_model_class>
special_characters = {
"SuruÌ¤":"Suruc",
"JapÌ_n":"Japan" ,
"\x89ÛÏWhen":"When",
"å£3million":"3 million",
"fromåÊwounds":"from wounds",
"mÌ¼sica":"music",
"donå«t":"do not",
"didn`t":"did not",
"i\x89Ûªm":"I am",
"I\x89Ûªm":"I am",
"it\x89Ûªs":"it is",
"It\x89Ûªs":"It is",
"i\x89Ûªd":"I would",
"I\x89Ûªd":"I would",
"i\x89Ûªve":"I have",
"I\x89Ûªve":"I have",
"let\x89Ûªs":"let us",
"don\x89Ûªt":"do not",
"Don\x89Ûªt":"Do not",
"can\x89Ûªt":"cannot",
"Can\x89Ûªt":"Cannot",
"that\x89Ûªs":"that is",
"That\x89Ûªs":"That is",
"here\x89Ûªs":"here is",
"Here\x89Ûªs":"Here is",
"you\x89Ûªre":"you are",
"You\x89Ûªre":"You are",
"you\x89Ûªve":"you have",
"You\x89Ûªve":"You have",
"you\x89Ûªll":"you will",
"You\x89Ûªll":"You will",
"China\x89Ûªs":"China's",
"doesn\x89Ûªt":"does not",
"wouldn\x89Ûªt":"would not",
"\x89Û_":"",
"\x89Û¢":"",
"\x89ÛÒ":"",
"\x89ÛÓ":"",
"\x89ÛÏ":"",
"\x89Û÷":"",
"\x89Ûª":"",
"\x89Û¢åÊ":"",
"\x89Û\x9d":"",
"å_":"",
"å¨":"",
"åÀ":"",
"åÇ":"",
"åÊ":"",
"åÈ":"" ,
"Ì©":"",
"&lt;":"<",
"&gt;":">",
"&amp;":"&"
}
def load_test_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=img_size
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()<load_pretrained>
expand_contractions = {
"I'm":"I am",
"I'M":"I am",
"i'm":"I am",
"i'M":"I am",
"i'd":"I would",
"I'd":"I would",
"i'll":"I will",
"I'll":"I will",
"i've":"I have",
"I've":"I have",
"you're":"you are",
"You're":"You are",
"you'd":"you would",
"You'd":"You would",
"you've":"you have",
"You've":"You have",
"you'll":"you will",
"You'll":"You will" ,
"y'know":"you know" ,
"Y'know":"You know" ,
"y'all":"you all",
"Y'all":"You all",
"we're":"we are",
"We're":"We are",
"we've":"we have",
"We've":"We have" ,
"we'd":"we would",
"We'd":"We would",
"WE'VE":"We have",
"we'll":"we will",
"We'll":"We will",
"they're":"they are",
"They're":"They are",
"they'd":"they would",
"They'd":"They would" ,
"they've":"they have",
"They've":"They have",
"they'll":"they will",
"They'll":"They will",
"he's":"he is",
"He's":"He is",
"he'll":"he will",
"He'll":"He will",
"she's":"she is",
"She's":"She is",
"she'll":"she will",
"She'll":"She will",
"it's":"it is",
"It's":"It is",
"it'll":"it will",
"It'll":"It will",
"isn't":"is not",
"Isn't":"Is not",
"who's":"who is",
"Who's":"Who is",
"what's":"what is",
"What's":"What is",
"that's":"that is",
"That's":"That is",
"here's":"here is",
"Here's":"Here is",
"there's":"there is",
"There's":"There is",
"where's":"where is",
"Where's":"Where is" ,
"wHeRE's":"where is" ,
"how's":"how is" ,
"How's":"How is" ,
"how're":"how are" ,
"How're":"How are" ,
"let's":"let us",
"Let's":"Let us",
"won't":"will not",
"wasn't":"was not",
"aren't":"are not",
"couldn't":"could not",
"shouldn't":"should not",
"haven't":"have not",
"Haven't":"Have not",
"hasn't":"has not",
"wouldn't":"would not",
"weren't":"were not",
"Weren't":"Were not",
"ain't":"am not",
"Ain't":"am not",
"don't":"do not",
"Don't":"do not",
"DON'T":"Do not",
"didn't":"did not",
"Didn't":"Did not",
"DIDN'T":"Did not",
"doesn't":"does not",
"can't":"cannot",
"Can't":"Cannot",
"Could've":"Could have",
"should've":"should have",
"would've":"would have"
}
def load_train_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = img_size
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
return DetBenchTrain(net, config )<categorify>
informal_abbreviations = {
"b/c":"because",
"w/e":"whatever",
"w/out":"without",
"w/o":"without",
"w/":"with ",
"<3":"love",
"c/o":"care of",
"p/u":"pick up",
"
":" "
}
class BaseWheatTTA:

 image_size = img_size
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
def clean_text(text):
cleaned_text = text.lower()
cleaned_text = re.sub(r'https?:\S+|www\.\S+', '', cleaned_text)
 cleaned_text = re.sub(r'<.*?>', '', cleaned_text)
 cleaned_text = ''.join(ch for ch in cleaned_text if ch in string.printable)
 cleaned_text = ' '.join(abbreviations[word] if word in abbreviations else word for word in cleaned_text.split())
 cleaned_text = re.sub(r'@\S+', 'USER', cleaned_text)
 cleaned_text = re.sub(r'\d+?[.,\d]*', 'NUMBER', cleaned_text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 cleaned_text = emoji_pattern.sub(r'', cleaned_text)
 cleaned_text = re.sub(r'\s+', ' ', cleaned_text ).strip()
cleaned_text = word_tokenize(cleaned_text)
 cleaned_text = [abbreviations[word] if word in abbreviations else word for word in cleaned_text]
cleaned_text = [special_characters[word] if word in special_characters else word for word in cleaned_text]
cleaned_text = [expand_contractions[word] if word in expand_contractions else word for word in cleaned_text]
cleaned_text = [informal_abbreviations[word] if word in informal_abbreviations else word for word in cleaned_text]
cleaned_text = [smileys[word] if word in smileys else word for word in cleaned_text]
cleaned_text = ' '.join([word for word in cleaned_text])
 doc = spacy_en(cleaned_text)
 cleaned_text = ' '.join([token.lemma_ for token in doc if token.lemma_ not in ['-PRON-'] and token.pos_ not in ['DET','-PRON']])
 return cleaned_text
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<find_best_model_class>
def add_special_token(text):
cleaned_text = "[CLS] " + text + " [SEP]"
return cleaned_text
def make_tta_predictions(images, models, score_threshold):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
assert images.shape[0] == 1
predictions = []
boxes_All = []
scores_All = []
for tta_transform in tta_transforms:
for net in models:
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 boxes = det[0].detach().cpu().numpy() [:,:4]
scores = det[0].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
scores = scores[indexes]
if len(boxes_All)== 0:
boxes_All = boxes
scores_All = scores
else:
boxes_All = np.concatenate(( boxes_All,boxes),0)
 scores_All = np.concatenate(( scores_All,scores),0)
 result = {
'boxes': boxes_All,
'scores': scores_All,
}
predictions.append([result])
 return predictions<init_hyperparams>
train_df['cleaned_text'] = np.vectorize(clean_text )(train_df['text'])
 train_df['cleaned_text'] = np.vectorize(add_special_token )(train_df['cleaned_text'] )
class TrainGlobalConfig:
num_workers = 2
batch_size = PL_batchsize
n_epochs = PL_epoch
lr = PL_lr
folder = 'plabel_model'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
if PL_lr_sche == 'cos':
SchedulerClass = torch.optim.lr_scheduler.CosineAnnealingLR
scheduler_params = dict(
 T_max=PL_epoch,
eta_min=cos_lr_min, 
)
 elif PL_lr_sche == 'plat':
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<choose_model_class>
MAX_LEN = 128
BATCH_SZ=32
class WarmUp(_LRScheduler):

 def __init__(self, optimizer, total_iters, last_epoch=-1):
self.total_iters = total_iters
super(WarmUp, self ).__init__(optimizer, last_epoch)
 def get_lr(self):

 return [base_lr * self.last_epoch /(self.total_iters + 1e-8)for base_lr in self.base_lrs]<init_hyperparams>
def generate_input_attention_mask(tweets):
tokenized_tweets = [bert_tokenizer.tokenize(tweet)for tweet in tweets]
input_ids = [bert_tokenizer.convert_tokens_to_ids(x)for x in tokenized_tweets]
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 attention_masks = []
for seq in input_ids:
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask)
 return input_ids, attention_masks
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config, train_loader_length):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 if warmup_opt:
iter_per_epoch = train_loader_length
total_iters = iter_per_epoch * warmup_epoch
self.WarmUp_scheduler = WarmUp(self.optimizer, total_iters)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint1.bin')
 if warmup_opt and(self.epoch+1)<= warmup_epoch:
pass
else:
if self.config.validation_scheduler:
if PL_lr_sche == 'plat':
self.scheduler.step(metrics=summary_loss.avg)
 elif PL_lr_sche == 'cos':
self.scheduler.step()
self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if warmup_opt and(self.epoch+1)<= warmup_epoch:
self.WarmUp_scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<normalization>
train_input_ids, train_attention_masks = generate_input_attention_mask(train_df['cleaned_text'] )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<categorify>
def recall(y_true, y_pred):
true_positives = K.sum(K.round(y_true * y_pred))
possible_positives = K.sum(y_true)
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(y_true * y_pred))
predicted_positives = K.sum(K.round(y_pred))
precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def f1_score(y_true, y_pred):
prec = precision(y_true, y_pred)
 rec = recall(y_true, y_pred)
 return 2*(( prec*rec)/(prec+rec+K.epsilon()))
if PL_OPT:
test_models = []
for p in path2:
test_models.append(load_test_net(p))
results_plabel = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images, test_models, PL_thr)
 for i, image in enumerate(images):
assert i == 0
image_id = image_ids[i]
image_ = cv2.imread(f'{TEST_DATA_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, img_size, WBF_iou_thr, WBF_skip_thr)
 if img_size == 512:
boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 elif img_size == 1024:
boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p)
 del test_models
gc.collect()
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df['image_id'] = results_df['image_id'].apply(lambda x: TEST_DATA_PATH+'/'+ x+'.jpg')
 marking['image_id'] = marking['image_id'].apply(lambda x: TRAIN_DATA_PATH+'/'+ x+'.jpg')
 train_data_plabel = pd.concat([results_df,marking], axis=0)
 skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = train_data_plabel[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
fold_number = 0
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True, 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 train_models = []
for p in path1:
train_models.append(load_train_net(p))
device = torch.device('cuda:0')
 fitter = Fitter(model=train_models[0].to(device), device=device, config=TrainGlobalConfig, train_loader_length=len(train_loader))
fitter.fit(train_loader, val_loader)
 del train_models
gc.collect()<define_variables>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(train_input_ids, train_df['target'], train_size=0.8, random_state=100)
 train_masks, validation_masks, _, _ = train_test_split(train_attention_masks, train_df['target'], train_size=0.8, random_state=100)
 train_inputs = torch.tensor(train_inputs, dtype=torch.long)
 validation_inputs = torch.tensor(validation_inputs, dtype=torch.long)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels.values, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<load_pretrained>
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SZ)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=BATCH_SZ )
if PL_OPT:
final_models = [
load_test_net(f'plabel_model/last-checkpoint1.bin')
 ]
else:
final_models = []
for p in path1:
final_models.append(load_test_net(p))<predict_on_test>
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)
 model.cuda()
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images, final_models, OOF_thr)
 for i, image in enumerate(images):
assert i == 0
boxes, scores, labels = run_wbf(predictions, img_size, WBF_iou_thr, WBF_skip_thr)
 if img_size == 512:
boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 elif img_size == 1024:
boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.0}
]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=2e-5,
warmup=.1)
 def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head(10 )<import_modules>
train_loss_set = []
epochs = 4
for _ in range(epochs):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
optimizer.zero_grad()
loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
 train_loss_set.append(loss.item())
 loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += b_input_ids.size(0)
 nb_tr_steps += 1
print("Train loss: {}".format(tr_loss/nb_tr_steps))
from object_detection_utils import show_Nimages<set_options>
model.eval()
eval_accuracy = 0
nb_eval_steps = 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print("Validation Accuracy: {}".format(eval_accuracy/nb_eval_steps))
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True 
<load_pretrained>
test_df.drop(columns=['keyword', 'location'], inplace=True)
 test_df['cleaned_text'] = np.vectorize(clean_text )(test_df['text'])
 test_df['cleaned_text'] = np.vectorize(add_special_token )(test_df['cleaned_text'])
 test_input_ids, test_attention_masks = generate_input_attention_mask(test_df['cleaned_text'] )
BEST_PATHS = ["/kaggle/input/best-models-frcnn/F0_68_nofinetune_clear_best.bin",
"/kaggle/input/best-models-frcnn/F1_68_nofinetune_clear_best.bin",
"/kaggle/input/5fold-68-clear/F2_68_nofinetune_clear_best.bin",
"/kaggle/input/5fold-68-clear/F3_68_nofinetune_clear_best.bin"]
for BEST_PATH in BEST_PATHS:
ckp = torch.load(BEST_PATH)
 print(ckp.keys())
 print(ckp['epoch'], ckp['best_valid_loss'] )<init_hyperparams>
test_inputs = torch.tensor(test_input_ids, dtype=torch.long)
 test_attention = torch.tensor(test_attention_masks, dtype=torch.long)
 test_data = TensorDataset(test_inputs, test_attention)
 test_sampler = SequentialSampler(test_data)
 test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size = BATCH_SZ )
SEED=42
seed_everything(SEED)
 APEX = False
ACCUM = 2
SWAP_VALID_AND_TRAIN = False
N_VIZ = 10

 USE_NMS = False
SCORE_THRESHOLD = 0.65
NMS_IOU_THRESHOLD = 0.5
IMG_SIZE = 1024
WBF_IOU, WBF_SKIP_BOX = 0.44, 0.38
PP_SHRINK = [-1,0]
WBF_SCORE_THRESHOLD = 0.265
USE_BOUNDS_FILTER = True
LOWER_BOUND, UPPER_BOUND = 70, 175000

 PSEUDO_EPOCHS = 4
PSEUDO_EPOCHS_COMMIT = 0
HSV_H = 0.03
HSV_S = 0.68
HSV_V = 0.36
BC_B = 0.1
BC_C = 0.1
BASE_LR = 0.00135
BIAS_LR_FACTOR = 0.5
MOMENTUM = 0.75
WARMUP_EPOCHS = 200<install_modules>
model.eval()
predictions = []
for batch in test_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_mask = batch
with torch.no_grad() :
logits = model(b_input_ids, token_type_ids=None, attention_mask = b_mask)
 logits = logits.detach().cpu().numpy()
predictions.append(logits)
 test_predictions = [item for sublist in predictions for item in sublist]
test_predictions = np.argmax(test_predictions, axis=1 ).flatten()
<import_modules><EOS> 
submission = pd.DataFrame({'id':test_df['id'], 'target':test_predictions})
 submission.to_csv('./submission.csv', index=False )