Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_pretrained>
nltk.download('stopwords', quiet=True)
 stopwords = stopwords.words('english')
 sns.set(style="white", font_scale=1.2)
 plt.rcParams["figure.figsize"] = [10,8]
pd.set_option.display_max_columns = 0
pd.set_option.display_max_rows = 0
<define_variables>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<import_modules>
null_counts = pd.DataFrame({"Num_Null": train.isnull().sum() })
 null_counts["Pct_Null"] = null_counts["Num_Null"] / train.count() * 100
null_counts
from utils.datasets import *
from utils.utils import *<train_on_grid>
len(train["keyword"].value_counts() )
def detect1Image_aug(im0, imgsz, model, device, conf_thres, iou_thres):
img = letterbox(im0, new_shape=imgsz)[0]
img = img[:, :, ::-1].transpose(2, 0, 1)
 img = np.ascontiguousarray(img)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=True)[0]
pred = non_max_suppression(pred, conf_thres, iou_thres)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<train_model>
def keyword_disaster_probabilities(x):
tweets_w_keyword = np.sum(train["keyword"].fillna("" ).str.contains(x))
tweets_w_keyword_disaster = np.sum(train["keyword"].fillna("" ).str.contains(x)& train["target"] == 1)
 return tweets_w_keyword_disaster / tweets_w_keyword
keywords_vc["Disaster_Probability"] = keywords_vc.index.map(keyword_disaster_probabilities)
 keywords_vc.head()
def clip_coords2(boxes, img_shape):
boxes[:, 0].clamp_(0, img_shape[1])
 boxes[:, 1].clamp_(0, img_shape[0])
 boxes[:, 2].clamp_(0, img_shape[1])
 boxes[:, 3].clamp_(0, img_shape[0])
 def scale_coords2(coords,factorx,factory, img0_shape):
coords[:, 0::2] *= factorx
coords[:, 1::2] *= factory
clip_coords2(coords, img0_shape)
 return coords<install_modules>
keywords_vc.sort_values(by="Disaster_Probability", ascending=False ).head(10 )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<install_modules>
len(train["location"].value_counts() )
!pip install /kaggle/input/orkatzfdata/yacs-0.1.7-py3-none-any.whl
!mkdir fvcore
!cp -R '/kaggle/input/orkatzfdata/fvcore-0.1.dev200407/fvcore-0.1.dev200407/'./fvcore
!pip install fvcore/fvcore-0.1.dev200407/.
 !mkdir detectron2-ResNeSt
!cp -R /kaggle/input/orkatzfdata/detectron2-ResNeSt/*./detectron2-ResNeSt/
!pip install detectron2-ResNeSt/ .<choose_model_class>
def create_corpus(target):
corpus = []
for w in train.loc[train["target"] == target]["text"].str.split() :
for i in w:
corpus.append(i)
 return corpus
def create_corpus_dict(target):
corpus = create_corpus(target)
 stop_dict = defaultdict(int)
 for word in corpus:
if word in stopwords:
stop_dict[word] += 1
return sorted(stop_dict.items() , key=lambda x:x[1], reverse=True )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
def load_net7(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net = torch.nn.DataParallel(net)
 checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()<load_pretrained>
corpus_disaster, corpus_non_disaster = create_corpus(1), create_corpus(0)
 counter_disaster, counter_non_disaster = Counter(corpus_disaster), Counter(corpus_non_disaster)
 x_disaster, y_disaster, x_non_disaster, y_non_disaster = [], [], [], []
counter = 0
for word, count in counter_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_disaster.append(word)
 y_disaster.append(count)
 counter = 0
for word, count in counter_non_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_non_disaster.append(word)
 y_non_disaster.append(count)
 fig, ax = plt.subplots(1,2, figsize=(20,8))
sns.barplot(x=y_disaster, y=x_disaster, orient='h', palette="Reds_d", ax=ax[0])
 sns.barplot(x=y_non_disaster, y=x_non_disaster, orient='h', palette="Blues_d", ax=ax[1])
 ax[0].set_title("Top 15 Non-Stopwords - Disaster Tweets")
 ax[0].set_xlabel("Word Frequency")
 ax[1].set_title("Top 15 Non-Stopwords - Non-Disaster Tweets")
 ax[1].set_xlabel("Word Frequency")
 plt.tight_layout()
plt.show()
models2 = [load_net7('.. /input/tempb7/best-checkpoint-015epoch.bin'),
load_net7('.. /input/tempb7/best-checkpoint-020epoch.bin'),
load_net7('.. /input/tempb7/best-checkpoint-022epoch.bin'),]
models = [
load_net('.. /input/effdetbestpth/best-fold0-augmix.pth'),
load_net('.. /input/effdetbestpth/best-fold3.pth'),
load_net('.. /input/effdetbestpth/best-fold4.pth'),
load_net('.. /input/effdetbestpth/best-fold1-augmix.pth'),
load_net('.. /input/effdetbestpth/best-fold2-augmix.pth'),
load_net('.. /input/effdetbestpth/fold0-best.bin')
 ]<set_options>
def bigrams(target):
corpus = train[train["target"] == target]["text"]
count_vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = count_vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in count_vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_cascade_rcnn_ResNeSt_101_FPN_syncbn_range-scale_1x.yaml"))
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
cfg.MODEL.WEIGHTS = os.path.join('/kaggle/input/best-inrae-1/', "model_final.pth")
 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.47
cfg.DATASETS.TEST =("m5_val",)
 predictor1 = DefaultPredictor(cfg )<define_variables>
def remove_pattern(input_txt, pattern):
r = re.findall(pattern, input_txt)
 for i in r:
input_txt = re.sub(i, '', input_txt)
 return input_txt
train['tweet'] = np.vectorize(remove_pattern )(train['text'], "
test['tweet'] = np.vectorize(remove_pattern )(test['text'], "
train.head()
train['tweet'] = train['tweet'].str.replace("[^a-zA-Z
test['tweet'] = test['tweet'].str.replace("[^a-zA-Z
train.head()
train['tweet'] = train['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
test['tweet'] = test['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
train.head()
train['tweet'] = train['tweet'].str.lower()
test['tweet'] = test['tweet'].str.lower()
set(stopwords.words('english'))
stops = set(stopwords.words('english'))
train['tokenized_sents'] = train.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 test['tokenized_sents'] = test.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 def remove_stops(row):
my_list = row['tokenized_sents']
meaningful_words = [w for w in my_list if not w in stops]
return(meaningful_words)
 train['clean_tweet'] = train.apply(remove_stops, axis=1)
 test['clean_tweet'] = test.apply(remove_stops, axis=1)
 train.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 test.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 def rejoin_words(row):
my_list = row['clean_tweet']
joined_words =(" ".join(my_list))
return joined_words
train['clean_tweet'] = train.apply(rejoin_words, axis=1)
 test['clean_tweet'] = test.apply(rejoin_words, axis=1)
 train.head()
models3 =[predictor1]<data_type_conversions>
warnings.filterwarnings("ignore")
 tqdm.pandas()
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
 random_state = 29
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None,transforms2=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
self.transforms2 = transforms2
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image0 = sample['image']
if self.transforms2:
sample = {'image': image}
sample = self.transforms2(**sample)
 image1 = sample['image']
return image0,image1, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 def get_valid_transforms2() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() ,transforms2=get_valid_transforms2() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<init_hyperparams>
!pip install GPUtil
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()
def make_predictions(
 images, images1,image_ids,
score_threshold=0.25, 
):
images = images.cuda().float()
images1 = images1.cuda().float()
image_id = image_ids
rh,rw,_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg' ).shape
Hscale512 = rh/512
Wscale512 = rw/512
Hscale1024 = rh/1024
Wscale1024 = rw/1024
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes[:, 0::2] *= Wscale512
boxes[:, 1::2] *= Hscale512
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 for fold_number, net in enumerate(models2):
with torch.no_grad() :
det = net(images1, torch.tensor([1]*images1.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes[:, 0::2] *= Wscale1024
boxes[:, 1::2] *= Hscale1024
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 for fold_number, net in enumerate(models3):
image_id = image_ids
result =[]
with torch.no_grad() :
im = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 outputs = predictor1(im)
 out = outputs["instances"].to("cpu")
 scores = out.get_fields() ['scores'].numpy()
boxes = out.get_fields() ['pred_boxes'].tensor.numpy().astype(int)
 labels= out.get_fields() ['scores'].numpy()
result.append({
'boxes': boxes,
'scores': scores,
})
 predictions.append(result)
 predictions.append(result)
 return predictions<categorify>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sub= pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
def clip_coords3(boxes, img_shape):
boxes[:, 0].clamp_(0, img_shape[1])
 boxes[:, 1].clamp_(0, img_shape[0])
 boxes[:, 2].clamp_(0, img_shape[1])
 boxes[:, 3].clamp_(0, img_shape[0])
 return boxes<compute_test_metric>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
def run_wbf2(predictions, image_index, image_size=1024, iou_thr=0.34, skip_box_thr=0.33, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<load_pretrained>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'URL',text)
 def remove_HTML(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_not_ASCII(text):
text = ''.join([word for word in text if word in string.printable])
 return text
def word_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def replace_abbrev(text):
string = ""
for word in text.split() :
string += word_abbrev(word)+ " "
return string
def remove_mention(text):
at=re.compile(r'@\S+')
 return at.sub(r'USER',text)
 def remove_number(text):
num = re.compile(r'[-+]?[.\d]*[\d]+[:,.\d]*')
 return num.sub(r'NUMBER', text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'EMOJI', text)
 def transcription_sad(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[(\\/]')
 return smiley.sub(r'SADFACE', text)
 def transcription_smile(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[)dDp]')
 return smiley.sub(r'SMILE', text)
 def transcription_heart(text):
heart = re.compile(r'<3')
 return heart.sub(r'HEART', text)
 def remove_elongated_words(text):
rep = re.compile(r'\b(\S*? )([a-z])\2{2,}\b')
 return rep.sub(r'\1\2 ELONG', text)
 def remove_repeat_punct(text):
rep = re.compile(r'([!?.]){2,}')
 return rep.sub(r'\1 REPEAT', text)
 def remove_all_punct(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_punct(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
def remove_stopwords(text):
text = ' '.join([word for word in text.split() if word not in(stopwords)])
 return text
def detect() :
transforms = get_valid_transforms()
transforms2 = get_valid_transforms2()
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
weights800 = '.. /input/yolo800/best_yolov5x_fold0_800.pt'
if not os.path.exists(weights):
weights = '.. /input/yolov5pth/weightsbest_yolov5x_fold3.pt'
imgsz = 1024
conf_thres = 0.34
iou_thres = 0.45
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
model800 = torch.load(weights800, map_location=device)['model'].float()
model800.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for name in tqdm(imagenames):
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
image = cv2.cvtColor(im01, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
sample = {'image': image}
sample = transforms(**sample)
 image0 = sample['image']
sample = {'image': image}
sample = transforms2(**sample)
 image1 = sample['image']
if image1.ndimension() == 3:
image1 = image1.unsqueeze(0)
 image0 = image0.unsqueeze(0)
 assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 if 1:
enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 enboxes.append(boxes)
 enscores.append(scores)
 enboxes.append(boxes)
 enscores.append(scores)
 predictions = make_predictions(image0,image1,image_id)
 boxes, scores, labels = run_wbf2(predictions, image_index=0)
 enboxes.append(boxes)
 enscores.append(scores)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.48, skip_box_thr=0.575)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.05].astype(np.int32)
 scores = scores[scores >=float(0.05)]
if count<10:
for box, score in zip(boxes,scores):
cv2.rectangle(im0, 
(box[0], box[1]), 
(box[2]+box[0], box[3]+box[1]), 
(220, 0, 0), 2)
 cv2.putText(im0, '%.2f'%(score),(box[0]+np.random.randint(20), box[1]), cv2.FONT_HERSHEY_SIMPLEX ,
0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(im0)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
def clean_tweet(text):
text = remove_URL(text)
 text = remove_HTML(text)
 text = remove_not_ASCII(text)
 text = text.lower()
text = replace_abbrev(text)
 text = remove_mention(text)
 text = remove_number(text)
 text = remove_emoji(text)
 text = transcription_sad(text)
 text = transcription_smile(text)
 text = transcription_heart(text)
 text = remove_elongated_words(text)
 text = remove_repeat_punct(text)
 return text
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
train["clean_text"] = train["text"].apply(clean_tweet)
 test["clean_text"] = test["text"].apply(clean_tweet)
 train["clean_tokens"] = train["clean_text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["clean_text"].apply(lambda x: word_tokenize(x))
! pip install --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/nvidiaapex/<install_modules>
skip_gram_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2,sg=1)
 skip_gram_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 cbow_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2)
 cbow_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 train["clean_tokens"] = train["text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["text"].apply(lambda x: word_tokenize(x))
def get_word_embeddings(token_list,vector,k=150):
if len(token_list)< 1:
return np.zeros(k)
 else:
vectorized = [vector[word] if word in vector else np.random.rand(k)for word in token_list]
sum = np.sum(vectorized,axis=0)
 return sum/len(vectorized )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<define_variables>
max_features=5000
count_vectorizer = CountVectorizer(max_features=max_features)
 sparce_matrix_train=count_vectorizer.fit_transform(train['clean_text'])
 sparce_matrix_test=count_vectorizer.fit_transform(train['clean_text'])
 def count_vector(data):
count_vectorizer = CountVectorizer()
vect = count_vectorizer.fit_transform(data)
 return vect, count_vectorizer
def tfidf_vector(data):
tfidf_vectorizer = TfidfVectorizer()
vect = tfidf_vectorizer.fit_transform(data)
 return vect, tfidf_vectorizer
def skip_gram_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, skip_gram_model))
return list(embeddings)
 def cbow_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, cbow_model))
return list(embeddings)
 X_train_count, count_vectorizer = count_vector(train['clean_text'])
 X_train_tfidf, tfidf_vectorizer = tfidf_vector(train['clean_text'])
 X_train_skip_gram = skip_gram_vectorizer(train['clean_tokens'],skip_gram_model)
 X_train_cbow = cbow_vectorizer(train['clean_tokens'],cbow_model)
 X_test_count = count_vectorizer.transform(test['clean_text'])
 X_test_tfidf = tfidf_vectorizer.transform(test['clean_text'])
 X_test_skip_gram = skip_gram_vectorizer(test['clean_tokens'],skip_gram_model)
 X_test_cbow = cbow_vectorizer(test['clean_tokens'],cbow_model)
 X_train_skip_gram = np.array(X_train_skip_gram)
 X_train_cbow = np.array(X_train_cbow)
 X_test_skip_gram = np.array(X_test_skip_gram)
 X_test_cbow = np.array(X_test_cbow)
 with open('count_vectorizer.pickle', 'wb')as handle:
pickle.dump(count_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('tfidf_vectorizer.pickle', 'wb')as handle:
pickle.dump(tfidf_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('skip_gram_model.pickle', 'wb')as handle:
pickle.dump(skip_gram_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
look_at_on_kernel = 1<set_options>
metrics = pd.DataFrame(columns=['model' ,'vectoriser', 'f1 score', 'train accuracy','test accuracy'] )
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<load_from_csv>
models=[
XGBClassifier(max_depth=6, n_estimators=1000),
LogisticRegression(random_state=random_state),
SVC(random_state=random_state),
MultinomialNB() ,
DecisionTreeClassifier(random_state = random_state),
KNeighborsClassifier() ,
RandomForestClassifier(random_state=random_state),
]
marking = pd.read_csv('.. /input/pure-box/cleanedTrainNoIndexOnLimit.csv' )<define_search_model>
for model in models:
y = train.target
x = X_train_count
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test,'Count vector')
 x = X_train_tfidf
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'Tfidf vector')
 x = X_train_skip_gram
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'skip gram vector')
 x = X_train_cbow
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'cbow vector' )
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=1024, width=1024, p=1),
A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<init_hyperparams>
metrics = metrics.sort_values('f1 score',ascending=False )
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 if apex_on:
self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level="O1")
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint1.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
try:
loss, _, _ = self.model(images, boxes, labels)
 if apex_on:
with amp.scale_loss(loss, self.optimizer)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
except RuntimeError as e:
if 'out of memory' in str(e):
print('| WARNING: ran out of memory')
 if hasattr(torch.cuda, 'empty_cache'):
torch.cuda.empty_cache()
else:
raise e
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<init_hyperparams>
free_gpu_cache()
class TrainGlobalConfig:
num_workers = 2
batch_size = 1
if apex_on:
batch_size *= 2
n_epochs = 3
lr = 0.0001
folder = 'plabel_model'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<data_type_conversions>
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM,GRU, Dropout, Activation, Input, Flatten, Bidirectional, Conv1D, MaxPooling1D
from tensorflow.keras import layers
from keras import optimizers
from functools import reduce
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
def train_lstm(x_train,x_test,y_train,y_test,vectorizer_name,vocab_size,input_length):
epochs = 1
verbose = 1
batch_size = 32
embed_dim = 32
optimizer = optimizers.Adam(lr=0.002)
 model = Sequential()
model.add(Embedding(vocab_size, embed_dim,input_length = input_length))
model.add(Dropout(0.2))
model.add(LSTM(32, dropout=0.2, recurrent_dropout=0.4))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss = 'binary_crossentropy', optimizer=optimizer, metrics = ['accuracy'])
 print(model.summary())
 model.fit(x_train, y_train, epochs = epochs, verbose = verbose, batch_size=batch_size, validation_data=(x_test, y_test))
model_name = "LSTM using "+ vectorizer_name
model.save(model_name)
 y_pred = model.predict(x_test)
 f1score = f1_score(y_test,y_pred.round() ,average='weighted')
 train_accuracy = round(accuracy_score(y_train,reduce(list.__add__, model.predict(x_train ).round().tolist())) *100)
 test_accuracy = round(accuracy_score(y_test,y_pred.round())*100)
 global metrics
metrics = metrics.append({
'model': model_name,
'f1 score': f1score,
'train accuracy': train_accuracy,
'test accuracy': test_accuracy,
'vectoriser': vectorizer_name,
},
ignore_index=True 
)
 print(model_name)
 print(classification_report(y_test,y_pred.round()))
print('Accuracy of classifier on training set:{}%'.format(train_accuracy))
print('Accuracy of classifier on test set:{}%'.format(test_accuracy))
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
y = train['target'].values
x_train, x_test, y_train, y_test = train_test_split(X_train_skip_gram,y, test_size = 0.3)
 train_lstm(x_train,x_test,y_train,y_test, 'skip gram vector',5329,150)
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/weight1024-5/2bs48epoch38383.bin' )<categorify>
%reset -f
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
!pip install GPUtil
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()
free_gpu_cache()
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
import re
import torch
from transformers import ElectraTokenizer, ElectraForSequenceClassification,AdamW
import torch
from sklearn.metrics import classification_report
import random
import time
import datetime
import numpy as np
import pandas as pd
from transformers import get_linear_schedule_with_warmup
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from sklearn.model_selection import train_test_split
def make_tta_predictions(images, score_threshold=0.01):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.2, skip_box_thr=0.33, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 df_train= train
df_test= test
results_plabel = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
image_id = image_ids[i]
image_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p )<prepare_output>
def preprocess(text):
text=text.lower()
text = re.sub(r'https?:\/\/.*[\r
]*', '', text)
 text = re.sub(r'http?:\/\/.*[\r
]*', '', text)
 text=text.replace(r'&amp;?',r'and')
 text=text.replace(r'&lt;',r'<')
 text=text.replace(r'&gt;',r'>')
 text = re.sub(r"(?:\@)\w+", '', text)
 text=text.encode("ascii",errors="ignore" ).decode()
text=re.sub(r'[:"
text=re.sub(r'[!]+','!',text)
 text=re.sub(r'[?]+','?',text)
 text=re.sub(r'[.]+','.',text)
 text=re.sub(r"'","",text)
 text=re.sub(r"\(","",text)
 text=re.sub(r"\)","",text)
 text=" ".join(text.split())
 return text
df_train['text'] = df_train['text'].apply(preprocess)
 df_test['text'] = df_test['text'].apply(preprocess)
 df_train=df_train[df_train["text"]!='']
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df.head()<feature_engineering>
df_train=df_train[["text","target"]]
results_df['image_id'] = results_df['image_id'].apply(lambda x: DATA_ROOT_PATH+'/'+ x+'.jpg' )<feature_engineering>
texts = df_train.text.values
labels = df_train.target.values
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
marking['image_id'] = marking['image_id'].apply(lambda x: TRAIN_ROOT_PATH+'/'+ x+'.jpg' )<concatenate>
torch.cuda.empty_cache()
tokenizer = ElectraTokenizer.from_pretrained('google/electra-base-discriminator')
 model = ElectraForSequenceClassification.from_pretrained('google/electra-base-discriminator',num_labels=2)
 model.cuda()
if len(os.listdir('.. /input/global-wheat-detection/test/')) <11:
train_data_plabel = results_df
else:
train_data_plabel = pd.concat([results_df, marking], axis=0 )<feature_engineering>
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = train_data_plabel[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<categorify>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes<create_dataframe>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )
fold_number = 0
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True,
)<load_pretrained>
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<choose_model_class>
optimizer = AdamW(model.parameters() ,
lr = 6e-6,
eps = 1e-8 
)
 epochs = 5
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load('.. /input/weight1024-5/2bs48epoch38383.bin')
 net.load_state_dict(checkpoint['model_state_dict'])
 return DetBenchTrain(net, config)
 net = get_net()<normalization>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<train_model>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
if len(os.listdir('.. /input/global-wheat-detection/test/')) <1:
pass
else:
run_training()<set_options>
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
preds=[]
true=[]
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
preds.append(logits)
 true.append(label_ids)
 tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print("Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print("Validation took: {:}".format(format_time(time.time() - t0)) )
time.sleep(1)
 def memory_cleanup() :

 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
torch.cuda.empty_cache()
memory_cleanup()<categorify>
report = {}
report['model'] = 'Electra'
report['test accuracy'] = 0.82
metrics = metrics.append(report,ignore_index=True )
results_plabel = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
image_id = image_ids[i]
image_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p )<prepare_output>
flat_predictions = [item for sublist in preds for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
flat_true_labels = [item for sublist in true for item in sublist]
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df.head()<feature_engineering>
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
results_df['image_id'] = results_df['image_id'].apply(lambda x: DATA_ROOT_PATH+'/'+ x+'.jpg' )<define_variables>
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train' 
<concatenate>
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':flat_predictions} )
<feature_engineering><EOS> 
df_leak = pd.read_csv('/kaggle/input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding ='ISO-8859-1')[['choose_one', 'text']]
df_leak['target'] =(df_leak['choose_one'] == 'Relevant' ).astype(np.int8)
 df_leak['id'] = df_leak.index.astype(np.int16)
 df_leak.drop(columns=['choose_one', 'text'], inplace=True)
 df_test = df_test.merge(df_leak, on=['id'], how='left')
 print('Leaked Data Set Shape = {}'.format(df_leak.shape))
print('Leaked Data Set Memory Usage = {:.2f} MB'.format(df_leak.memory_usage().sum() / 1024**2))
print(df_test)
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = df_test['target'].values
submission.to_csv('submission.csv', index=False)
 print(submission )