Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+cached_dev_deberta-mlm_128_atomic filter=lfs diff=lfs merge=lfs -text
+cached_train_deberta-mlm_128_atomic filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

cached_dev_deberta-mlm_128_atomic

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de2aeddcf9134d495f3461fedebd548864b101e1920dcc1facf27a6790e27e75
+size 4501475

cached_train_deberta-mlm_128_atomic

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be198ba32b721dd13ee5c578e9de10d3e49aaa3f273d7407b847bad3ae39e1b
+size 365724007

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-large",
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": "atomic",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

data_utils.py

@@ -0,0 +1,236 @@
+import json
+import logging
+
+import nltk
+from nltk.corpus import stopwords
+from tqdm import tqdm
+from tqdm import tqdm
+
+skip_words = set(stopwords.words('english'))
+skip_words.add('\'s')
+skip_words.add('.')
+skip_words.add(',')
+PERSON_NAMES = ['Alex', 'Ash', 'Aspen', 'Bali', 'Berkeley', 'Cameron', 'Chris', 'Cody', 'Dana', 'Drew', 'Emory',
+                'Flynn', 'Gale', 'Jamie', 'Jesse',
+                'Kai', 'Kendall', 'Kyle', 'Lee', 'Logan', 'Max', 'Morgan', 'Nico', 'Paris', 'Pat', 'Quinn', 'Ray',
+                'Robin', 'Rowan', 'Rudy', 'Sam', 'Skylar', 'Sydney',
+                'Taylor', 'Tracy', 'West', 'Wynne']
+logger = logging.getLogger(__name__)
+
+
+def accuracy(out, labels):
+    return {'acc': (out == labels).mean()}
+
+
+def handle_words(span, tokenizer, keywords=None, is_start=False):
+    inputs = []
+    labels = []
+    words = nltk.word_tokenize(span)
+    for w_i, w in enumerate(words):
+        if (w_i == 0 and is_start) or w == '.' or w == ',' or w.startswith('\''):
+            w_bpes = tokenizer.tokenize(w)
+        else:
+            w_bpes = tokenizer.tokenize(w, add_prefix_space=True)
+        inputs.extend(w_bpes)
+        if keywords != None:
+            if w in keywords:
+                labels.extend(w_bpes)
+            else:
+                labels.extend([-100] * len(w_bpes))
+        else:
+            if w not in PERSON_NAMES and w not in skip_words and w.lower() not in skip_words:
+                labels.extend(w_bpes)
+            else:
+                labels.extend([-100] * len(w_bpes))
+    return inputs, labels
+
+
+def handle_underscores(suffix, tokenizer, keywords=None, prefix=False):
+    inputs = []
+    labels = []
+    if '_' in suffix:
+        suffix_parts = [i.strip() for i in suffix.split('___')]
+        for i, part in enumerate(suffix_parts):
+            if part:
+                tmp_inputs, tmp_labels = handle_words(part, tokenizer, keywords=keywords, is_start=(i == 0 and prefix))
+                inputs += tmp_inputs
+                labels += tmp_labels
+
+                if i != len(suffix_parts) - 1 and suffix_parts[i + 1]:
+                    inputs.append(tokenizer.mask_token)
+                    labels.append(-100)
+            else:
+                inputs.append(tokenizer.mask_token)
+                labels.append(-100)
+    else:
+        inputs, labels = handle_words(suffix, tokenizer, keywords=keywords, is_start=prefix)
+    return inputs, labels
+
+from tqdm import tqdm
+def convert_examples_to_features(examples, tokenizer, max_length=512):
+    data = []
+    for example in tqdm(examples, desc="converting examples to features"):
+        inputs, labels = handle_underscores(example['context'], tokenizer, keywords=example.get('keywords', None), prefix=True)
+        choices = [handle_underscores(cand, tokenizer) for cand in example['candidates']]
+        input_ids = [inputs + cand[0] for cand in choices]
+        input_ids = [tokenizer.convert_tokens_to_ids(cand) for cand in input_ids]
+        label_ids = [labels + cand[1] for cand in choices]
+        label_ids = [[t if t == -100 else input_ids[i][t_i] for t_i, t in enumerate(cand)] for i, cand in
+                     enumerate(label_ids)]
+        label_ids = [[-100] + cand + [-100] for cand in label_ids]
+        input_ids = [tokenizer.prepare_for_model(cand, max_length=max_length, truncation=True)['input_ids'] for cand in
+                     input_ids]
+        data.append([input_ids, label_ids, example['correct']])
+    return data
+
+
+class ATOMICMLMProcessor(object):
+    def __init__(self, args):
+        self.D = []
+        self.filelist = [args.train_file, args.dev_file]
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0])
+        return self.D
+
+    def get_dev_examples(self):
+        data = []
+        with open(self.filelist[1], 'r') as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                data.append(sample)
+        print(len(data))
+        return data
+
+    def load_data(self, filename):
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                self.D.append({'id': sample['id'], 'context': sample['context'],
+                               'ending': sample['candidates'][sample['correct']], 'keywords': sample.get('keywords', None)})
+            print(len(self.D))
+
+
+class ATOMICProcessor(object):
+    def __init__(self, args):
+        print('loading from %s %s' % (args.train_file, args.dev_file))
+        self.filelist = [args.train_file, args.dev_file]
+        self.D = [[], []]
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0], 0)
+        return self.D[0]
+
+    def get_dev_examples(self):
+        self.load_data(self.filelist[1], 1)
+        return self.D[1]
+
+    def load_data(self, filename, sid):
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                self.D[sid].append(sample)
+            print(len(self.D[sid]))
+
+
+class CWWVProcessor(object):
+    def __init__(self, args):
+        self.answerKey_mapping = {'A': 0, 'B': 1, 'C': 2}
+        self.D = [[], []]
+        if args.task_name == 'cskg':
+            print('loading from %s %s' % (args.second_train_file, args.second_dev_file))
+            self.filelist = [args.second_train_file, args.second_dev_file]
+        else:
+            print('loading from %s %s' % (args.train_file, args.dev_file))
+            self.filelist = [args.train_file, args.dev_file]
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0], 0)
+        return self.D[0]
+
+    def get_dev_examples(self):
+        self.load_data(self.filelist[1], 1)
+        return self.D[1]
+
+    def load_data(self, filename, sid):
+        skipped = 0
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                context = sample['question']['stem']
+                if context.endswith('.'):
+                    context = context[:-1]
+                if not context.endswith('[MASK]'):
+                    skipped += 1
+                    context_parts = context.split('[MASK]')
+                    context = context_parts[0].strip()
+                    candidates = [c['text'] + context_parts[1] + '.' for c in sample['question']['choices']]
+                else:
+                    context = context[:-7]
+                    candidates = [c['text'] + '.' for c in sample['question']['choices']]
+                label = self.answerKey_mapping[sample['answerKey']]
+                keywords = nltk.word_tokenize(sample['question']['head'])
+                keywords = [w for w in keywords if w not in skip_words and w.lower() not in skip_words]
+                self.D[sid].append({'id': sample['id'], 'context': context, 'correct': label, 'candidates': candidates,
+                                    'keywords': keywords})
+            print(len(self.D[sid]), skipped)
+
+
+class CWWVMLMProcessor(object):
+    def __init__(self, args):
+        self.answerKey_mapping = {'A': 0, 'B': 1, 'C': 2}
+        self.D = []
+        self.filelist = [args.train_file, args.dev_file]
+        self.args = args
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0])
+        return self.D
+
+    def get_dev_examples(self):
+        processor = CSKGProcessor(self.args)
+        return processor.get_dev_examples()
+
+    def load_data(self, filename):
+        skipped = 0
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                context = sample['question']['stem']
+                if context.endswith('.'):
+                    context = context[:-1]
+                assert context.endswith('[MASK]')
+                context = context[:-7]
+                candidates = [c['text'] + '.' for c in sample['question']['choices']]
+                label = self.answerKey_mapping[sample['answerKey']]
+                keywords = nltk.word_tokenize(sample['question']['head'])
+                keywords = [w for w in keywords if w not in skip_words and w.lower() not in skip_words]
+                self.D.append(
+                    {'id': sample['id'], 'context': context, 'ending': candidates[label], 'keywords': keywords})
+            print(len(self.D))
+
+
+class CSKGProcessor(object):
+    def __init__(self, args):
+        # CWWV set always uses second train/dev file params
+        self.atomicprocessor = ATOMICProcessor(args)
+        self.cwwvprocessor = CWWVProcessor(args)
+
+    def get_train_examples(self):
+        cwwv_questions = self.cwwvprocessor.get_train_examples()
+        atomic_questions = self.atomicprocessor.get_train_examples()
+        return cwwv_questions + atomic_questions
+
+    def get_dev_examples(self):
+        cwwv_questions = self.cwwvprocessor.get_dev_examples()
+        atomic_questions = self.atomicprocessor.get_dev_examples()
+        return cwwv_questions + atomic_questions
+
+
+myprocessors = {
+    "atomic": ATOMICProcessor,
+    "cwwv": CWWVProcessor,
+    "atomicmlm": ATOMICMLMProcessor,
+    "cwwvmlm": CWWVMLMProcessor,
+    "cskg": CSKGProcessor
+}

eval_results.txt

@@ -0,0 +1 @@
+acc = 0.525

logits_test.txt

@@ -0,0 +1,120 @@
+-13.485865592956543 -12.554702758789062 -16.843822479248047 -17.034975051879883 -15.528239250183105
+-11.945438385009766 -15.764880180358887 -15.65064811706543 -17.049840927124023 -15.45334243774414
+-11.587425231933594 -16.005020141601562 -18.591140747070312 -13.294342041015625 -16.296398162841797
+-10.009797096252441 -15.341827392578125 -10.539949417114258 -10.960021018981934 -14.047414779663086
+-12.25959300994873 -10.891813278198242 -14.401248931884766 -11.447187423706055 -13.685133934020996
+-12.804305076599121 -15.47064208984375 -12.90679931640625 -12.644388198852539 -16.551565170288086
+-12.235984802246094 -17.26417350769043 -15.814994812011719 -14.540316581726074 -16.57213020324707
+-13.064817428588867 -14.498727798461914 -12.224983215332031 -18.461196899414062 -15.455927848815918
+-9.443912506103516 -15.756866455078125 -14.860000610351562 -11.844358444213867 -16.200241088867188
+-11.700218200683594 -15.54508113861084 -18.18265724182129 -17.967411041259766 -15.61465072631836
+-11.174132347106934 -17.90937614440918 -17.444185256958008 -14.454703330993652 -14.554675102233887
+-12.42685604095459 -12.927202224731445 -14.598489761352539 -13.699914932250977 -16.133480072021484
+-10.828542709350586 -11.421213150024414 -14.742795944213867 -16.211345672607422 -15.933958053588867
+-9.981731414794922 -13.114375114440918 -10.851600646972656 -13.30392074584961 -15.448455810546875
+-12.899808883666992 -12.504766464233398 -11.667335510253906 -11.96485710144043 -16.87687110900879
+-10.543583869934082 -10.229654312133789 -11.832517623901367 -13.934309005737305 -14.101360321044922
+-12.628169059753418 -14.552963256835938 -15.033495903015137 -14.496928215026855 -14.277275085449219
+-10.540155410766602 -17.203994750976562 -15.181567192077637 -13.87678337097168 -16.33222198486328
+-12.89242935180664 -12.802375793457031 -11.98313045501709 -12.95468807220459 -14.210517883300781
+-13.707990646362305 -15.822447776794434 -14.352869033813477 -16.99176597595215 -15.332679748535156
+-18.221946716308594 -14.591471672058105 -11.751192092895508 -15.28943157196045 -15.79006576538086
+-10.999690055847168 -17.102828979492188 -11.563447952270508 -16.147525787353516 -15.312467575073242
+-12.119155883789062 -13.997671127319336 -14.05282974243164 -15.500701904296875 -15.024360656738281
+-12.406027793884277 -13.902008056640625 -14.89413070678711 -13.278053283691406 -16.163454055786133
+-12.729238510131836 -15.645377159118652 -14.512473106384277 -16.292064666748047 -14.945101737976074
+-11.565582275390625 -15.689852714538574 -15.196950912475586 -16.58242416381836 -16.84659194946289
+-9.898529052734375 -12.66911506652832 -12.563604354858398 -15.67273235321045 -14.335538864135742
+-9.923110961914062 -11.773245811462402 -11.786827087402344 -17.258573532104492 -15.622135162353516
+-10.604410171508789 -18.388168334960938 -17.512317657470703 -19.24336051940918 -14.125904083251953
+-16.949708938598633 -13.706928253173828 -16.456981658935547 -14.97553539276123 -14.764923095703125
+-11.796066284179688 -17.549203872680664 -16.29804229736328 -14.065374374389648 -14.975845336914062
+-16.109678268432617 -10.775965690612793 -14.450605392456055 -14.85343074798584 -14.199943542480469
+-12.714162826538086 -14.389898300170898 -14.631216049194336 -13.585458755493164 -14.533248901367188
+-11.866647720336914 -14.02271842956543 -16.863229751586914 -16.302730560302734 -15.423881530761719
+-15.34256362915039 -13.611021995544434 -16.341060638427734 -18.729846954345703 -16.023725509643555
+-13.729169845581055 -13.358748435974121 -16.144556045532227 -15.138229370117188 -15.170283317565918
+-15.203313827514648 -15.215215682983398 -15.796660423278809 -13.29316520690918 -14.272794723510742
+-10.206670761108398 -14.634307861328125 -16.95503807067871 -15.288354873657227 -14.735435485839844
+-15.958247184753418 -11.922269821166992 -15.554479598999023 -16.529380798339844 -14.621512413024902
+-11.756235122680664 -13.470741271972656 -14.978282928466797 -12.765861511230469 -15.564665794372559
+-12.600726127624512 -15.769315719604492 -15.375072479248047 -18.390806198120117 -15.738916397094727
+-15.593986511230469 -14.472156524658203 -13.198034286499023 -14.962379455566406 -16.735809326171875
+-14.267927169799805 -12.278310775756836 -14.038803100585938 -15.891279220581055 -15.780318260192871
+-12.514389038085938 -17.2401180267334 -14.838483810424805 -15.404850006103516 -15.5819091796875
+-12.566061019897461 -12.968563079833984 -10.93451976776123 -13.795123100280762 -14.745218276977539
+-21.77191162109375 -12.907783508300781 -14.735198974609375 -16.297962188720703 -16.843549728393555
+-12.247344970703125 -13.780832290649414 -12.38991928100586 -14.87220573425293 -16.25807762145996
+-10.589265823364258 -14.178890228271484 -15.336084365844727 -12.817083358764648 -15.201112747192383
+-11.625978469848633 -12.869548797607422 -12.684764862060547 -13.059203147888184 -14.451128959655762
+-15.250967979431152 -15.940725326538086 -12.647708892822266 -17.80953598022461 -14.153979301452637
+-13.933172225952148 -11.741453170776367 -16.345247268676758 -15.836057662963867 -14.906820297241211
+-11.782374382019043 -14.776283264160156 -14.698659896850586 -18.39876365661621 -15.45709228515625
+-11.998146057128906 -16.717647552490234 -17.300548553466797 -18.50290298461914 -15.252758026123047
+-13.041389465332031 -14.309408187866211 -16.311140060424805 -19.84041404724121 -15.644210815429688
+-11.62677001953125 -14.909978866577148 -18.91885757446289 -16.421764373779297 -15.917211532592773
+-15.454143524169922 -14.28858757019043 -14.329549789428711 -13.516512870788574 -15.161291122436523
+-14.137840270996094 -14.48200511932373 -14.00358772277832 -17.134559631347656 -14.228231430053711
+-14.116720199584961 -15.66433334350586 -12.732897758483887 -13.650927543640137 -13.985754013061523
+-12.114367485046387 -13.667549133300781 -16.13338279724121 -16.44911766052246 -13.523371696472168
+-13.346145629882812 -18.75728988647461 -17.335689544677734 -17.35544204711914 -14.020357131958008
+-12.425326347351074 -16.646942138671875 -14.301322937011719 -15.461494445800781 -15.571342468261719
+-12.522256851196289 -11.32237434387207 -12.392147064208984 -10.751494407653809 -13.668184280395508
+-11.427050590515137 -12.420162200927734 -12.610843658447266 -13.642584800720215 -13.698797225952148
+-10.28366470336914 -10.129068374633789 -17.50717544555664 -18.64311408996582 -14.564188957214355
+-12.855567932128906 -15.540145874023438 -19.281057357788086 -16.280914306640625 -15.508527755737305
+-13.631608963012695 -13.476893424987793 -10.74915599822998 -17.62717056274414 -15.61255168914795
+-14.164091110229492 -14.324302673339844 -17.250370025634766 -13.515758514404297 -15.604305267333984
+-15.42667293548584 -18.33716583251953 -14.98896598815918 -17.703462600708008 -14.412519454956055
+-11.1312255859375 -13.484000205993652 -10.89915657043457 -14.660863876342773 -14.351375579833984
+-11.8013916015625 -15.06019115447998 -14.530506134033203 -14.725985527038574 -15.17402458190918
+-11.380867004394531 -16.774526596069336 -19.0806941986084 -14.300642013549805 -14.787707328796387
+-15.317098617553711 -10.536006927490234 -16.74585723876953 -17.00075340270996 -14.233205795288086
+-12.836723327636719 -14.365041732788086 -13.245519638061523 -14.606501579284668 -15.848045349121094
+-14.671722412109375 -12.97309398651123 -18.96438980102539 -18.358306884765625 -14.383865356445312
+-12.006148338317871 -19.101789474487305 -18.057790756225586 -17.27611541748047 -14.254199981689453
+-12.239377975463867 -16.24175262451172 -18.39486312866211 -17.79523468017578 -15.77902889251709
+-11.026527404785156 -17.164255142211914 -16.89369773864746 -14.13223648071289 -14.647666931152344
+-13.34439468383789 -15.085588455200195 -14.015231132507324 -14.57590389251709 -15.548398971557617
+-8.840858459472656 -13.32087230682373 -11.948570251464844 -12.7437744140625 -12.76047420501709
+-15.12006664276123 -14.209697723388672 -16.21258544921875 -15.09268569946289 -15.725016593933105
+-12.218896865844727 -19.344924926757812 -14.896772384643555 -15.128798484802246 -15.085726737976074
+-13.855364799499512 -14.396207809448242 -14.475112915039062 -16.274978637695312 -16.561267852783203
+-12.931468963623047 -20.261959075927734 -15.067851066589355 -18.67504119873047 -15.390913963317871
+-10.541189193725586 -17.481229782104492 -16.9975643157959 -19.102977752685547 -14.517946243286133
+-12.27475643157959 -16.989179611206055 -16.667797088623047 -15.321989059448242 -13.522154808044434
+-13.157089233398438 -16.034408569335938 -18.0081729888916 -12.339900016784668 -14.611282348632812
+-11.697013854980469 -18.469486236572266 -17.101667404174805 -18.21478271484375 -15.592073440551758
+-10.410983085632324 -12.342016220092773 -19.91473960876465 -13.985015869140625 -13.253973007202148
+-11.477904319763184 -10.292394638061523 -14.22382926940918 -14.867258071899414 -14.294317245483398
+-13.928434371948242 -15.648571014404297 -13.287254333496094 -18.776403427124023 -14.822330474853516
+-15.338903427124023 -17.95416831970215 -19.120832443237305 -16.684314727783203 -16.52161407470703
+-11.756458282470703 -15.748832702636719 -16.6794376373291 -14.992359161376953 -14.095579147338867
+-12.05335807800293 -16.373342514038086 -17.24437713623047 -16.61781883239746 -13.433090209960938
+-10.43917465209961 -17.29708480834961 -14.298304557800293 -18.689228057861328 -13.535030364990234
+-12.81536865234375 -16.188444137573242 -13.543594360351562 -17.28397560119629 -13.897306442260742
+-11.524691581726074 -16.597545623779297 -16.99277114868164 -17.697744369506836 -15.411514282226562
+-12.537017822265625 -15.651408195495605 -17.060205459594727 -16.015666961669922 -14.72602367401123
+-16.58844757080078 -12.648893356323242 -12.805540084838867 -16.793424606323242 -14.052775382995605
+-8.198223114013672 -14.490667343139648 -15.456783294677734 -18.407733917236328 -13.935432434082031
+-12.235084533691406 -13.254622459411621 -17.220083236694336 -17.194355010986328 -16.625526428222656
+-11.645500183105469 -12.380645751953125 -16.97612762451172 -16.918365478515625 -16.081634521484375
+-15.551166534423828 -18.355005264282227 -17.210491180419922 -16.315765380859375 -16.989500045776367
+-11.072067260742188 -14.111711502075195 -14.02056884765625 -13.801989555358887 -14.380411148071289
+-10.74884033203125 -17.039485931396484 -16.053722381591797 -17.099529266357422 -14.157671928405762
+-14.432212829589844 -15.668070793151855 -17.897315979003906 -19.030807495117188 -16.767248153686523
+-10.875746726989746 -16.729921340942383 -16.830699920654297 -10.96748161315918 -14.065666198730469
+-12.992372512817383 -13.389198303222656 -15.847464561462402 -18.74746322631836 -14.213224411010742
+-16.27631378173828 -15.575210571289062 -15.83560562133789 -19.659807205200195 -15.64935302734375
+-14.469172477722168 -17.462440490722656 -13.705839157104492 -14.783185958862305 -15.626546859741211
+-12.304758071899414 -11.911407470703125 -15.557670593261719 -12.007896423339844 -13.70199966430664
+-12.623868942260742 -13.375839233398438 -17.584022521972656 -14.576876640319824 -15.300482749938965
+-12.100661277770996 -15.983474731445312 -13.280969619750977 -16.395549774169922 -13.329227447509766
+-11.029035568237305 -17.723522186279297 -16.722801208496094 -16.074941635131836 -14.450968742370605
+-14.947690963745117 -13.623953819274902 -17.58797836303711 -18.477333068847656 -14.811092376708984
+-15.929779052734375 -12.438506126403809 -13.058927536010742 -18.52623748779297 -14.129800796508789
+-14.234972953796387 -16.270557403564453 -16.478979110717773 -16.46598243713379 -16.054523468017578
+-11.928365707397461 -12.55320930480957 -17.857826232910156 -15.371994018554688 -15.036209106445312
+-13.607194900512695 -15.683792114257812 -16.34781837463379 -15.106184005737305 -13.208913803100586
+-12.801432609558105 -18.26355743408203 -17.48211669921875 -12.184883117675781 -15.798808097839355
+-14.617281913757324 -14.461203575134277 -18.154417037963867 -14.951545715332031 -14.540334701538086

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f9e50c31777d2402062072d7ea15663f5a6b50395c09328957f06df9b7f7138
+size 1740904889

run_pretrain.py

@@ -0,0 +1,651 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import json
+import logging
+import os
+import random
+import wandb
+
+import numpy as np
+import torch
+from torch.optim import AdamW
+from torch.utils.data import DataLoader
+from torch.utils.data import RandomSampler
+from torch.utils.data import SequentialSampler
+from torch.utils.data.distributed import DistributedSampler
+from torch.utils.tensorboard import SummaryWriter
+from tqdm import tqdm
+from tqdm import trange
+from transformers import DebertaV2Config
+from transformers import DebertaV2ForMaskedLM
+from transformers import DebertaV2Tokenizer
+from transformers import RobertaConfig
+from transformers import RobertaForMaskedLM
+from transformers import RobertaTokenizer
+from transformers import get_linear_schedule_with_warmup
+
+from data_utils import accuracy
+from data_utils import convert_examples_to_features
+from data_utils import myprocessors
+
+from evaluate_DeBERTa import eval_tasks
+from evaluate_DeBERTa import main as evaluate_main
+
+logger = logging.getLogger(__name__)
+
+from transformers import MODEL_WITH_LM_HEAD_MAPPING
+
+MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+MODEL_CLASSES = {
+    'roberta-mlm': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
+    'deberta-mlm': (DebertaV2Config, DebertaV2ForMaskedLM, DebertaV2Tokenizer)
+}
+
+
+class MyDataset(torch.utils.data.Dataset):
+
+    def __init__(self, data, pad_token, mask_token, max_words_to_mask):
+        self.data = data
+        self.pad_token = pad_token
+        self.mask_token = mask_token
+        self.max_words_to_mask = max_words_to_mask
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sample = self.data[idx]
+        return sample, self.pad_token, self.mask_token, self.max_words_to_mask
+
+
+def mCollateFn(batch):
+    batch_input_ids = []
+    batch_input_mask = []
+    batch_input_labels = []
+    batch_label_ids = []
+    features = [b[0] for b in batch]
+    pad_token = batch[0][1]
+    mask_token = batch[0][2]
+    MAX_WORDS_TO_MASK = batch[0][3]
+    max_len = max([len(cand) for f in features for cand in f[0]])
+    for f in features:
+        batch_input_ids.append([])
+        batch_input_mask.append([])
+        batch_input_labels.append([])
+        batch_label_ids.append(f[2])
+        for i in range(len(f[0])):
+            masked_sequences = []
+            masked_labels = []
+            this_att_mask = []
+            sequence = f[0][i] + [pad_token] * (max_len - len(f[0][i]))
+            label_sequence = f[1][i] + [-100] * (max_len - len(f[1][i]))
+            valid_indices = [l_i for l_i, l in enumerate(label_sequence) if l != -100]
+            if len(valid_indices) > MAX_WORDS_TO_MASK:
+                rm_indices = random.sample(valid_indices, (len(valid_indices) - MAX_WORDS_TO_MASK))
+                label_sequence = [-100 if l_i in rm_indices else l for l_i, l in enumerate(label_sequence)]
+            for j, t in enumerate(label_sequence):
+                if t == -100:
+                    continue
+                    masked_sequences.append(sequence)
+                    masked_labels.append([-100] * max_len)
+                else:
+                    masked_sequences.append(sequence[:j] + [mask_token] + sequence[j + 1:])
+                    masked_labels.append([-100] * j + [sequence[j]] + [-100] * (max_len - j - 1))
+                this_att_mask.append([1] * len(f[0][i]) + [0] * (max_len - len(f[0][i])))
+            batch_input_ids[-1].append(torch.tensor(masked_sequences, dtype=torch.long))
+            batch_input_mask[-1].append(torch.tensor(this_att_mask, dtype=torch.long))
+            batch_input_labels[-1].append(torch.tensor(masked_labels, dtype=torch.long))
+    return batch_input_ids, batch_input_mask, batch_input_labels, torch.tensor(batch_label_ids, dtype=torch.long)
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def count_parameters(model):
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+def train(args, train_dataset, model, tokenizer, eval_dataset):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter(os.path.join(args.output_dir, 'runs'))
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size,
+                                  collate_fn=mCollateFn)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+         'weight_decay': args.weight_decay},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
+    ]
+
+    warmup_steps = args.warmup_steps if args.warmup_steps != 0 else int(args.warmup_proportion * t_total)
+    logger.info("warm up steps = %d", warmup_steps)
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon, betas=(0.9, 0.98))
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=warmup_steps, num_training_steps=t_total)
+
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
+                                                          output_device=args.local_rank,
+                                                          find_unused_parameters=True)
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
+                args.train_batch_size * args.gradient_accumulation_steps * (
+                    torch.distributed.get_world_size() if args.local_rank != -1 else 1))
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    curr_best = 0.0
+    CE = torch.nn.CrossEntropyLoss(reduction='none')
+    loss_fct = torch.nn.MultiMarginLoss(margin=args.margin)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in tqdm(enumerate(epoch_iterator), desc=f"Train Epoch {_}"):
+            model.train()
+            num_cand = len(batch[0][0])
+            choice_loss = []
+            choice_seq_lens = np.array([0] + [len(c) for sample in batch[0] for c in sample])
+            choice_seq_lens = np.cumsum(choice_seq_lens)
+            input_ids = torch.cat([c for sample in batch[0] for c in sample], dim=0).to(args.device)
+            att_mask = torch.cat([c for sample in batch[1] for c in sample], dim=0).to(args.device)
+            input_labels = torch.cat([c for sample in batch[2] for c in sample], dim=0).to(args.device)
+
+            if len(input_ids) < args.max_sequence_per_time:
+                inputs = {'input_ids': input_ids,
+                          'attention_mask': att_mask}
+                outputs = model(**inputs)
+                ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)), input_labels.view(-1))
+                ce_loss = ce_loss.view(outputs[0].size(0), -1).sum(1)
+            else:
+                ce_loss = []
+                for chunk in range(0, len(input_ids), args.max_sequence_per_time):
+                    inputs = {'input_ids': input_ids[chunk:chunk + args.max_sequence_per_time],
+                              'attention_mask': att_mask[chunk:chunk + args.max_sequence_per_time]}
+                    outputs = model(**inputs)
+                    tmp_ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)),
+                                     input_labels[chunk:chunk + args.max_sequence_per_time].view(-1))
+                    tmp_ce_loss = tmp_ce_loss.view(outputs[0].size(0), -1).sum(1)
+                    ce_loss.append(tmp_ce_loss)
+                ce_loss = torch.cat(ce_loss, dim=0)
+            # all tokens are valid
+            for c_i in range(len(choice_seq_lens) - 1):
+                start = choice_seq_lens[c_i]
+                end = choice_seq_lens[c_i + 1]
+                choice_loss.append(-ce_loss[start:end].sum() / (end - start))
+
+            choice_loss = torch.stack(choice_loss)
+            choice_loss = choice_loss.view(-1, num_cand)
+            loss = loss_fct(choice_loss, batch[3].to(args.device))
+
+            if args.n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    tb_writer.add_scalar('lr', scheduler.get_last_lr()[0], global_step)
+                    tb_writer.add_scalar('loss', (tr_loss - logging_loss) / args.logging_steps, global_step)
+                    tb_writer.add_scalar('Batch_loss', loss.item() * args.gradient_accumulation_steps, global_step)
+                    logger.info(" global_step = %s, average loss = %s", global_step,
+                                (tr_loss - logging_loss) / args.logging_steps)
+                    wandb.log({"train/loss":loss.item()})
+                    logging_loss = tr_loss
+
+                if args.local_rank == -1 and args.evaluate_during_training and global_step % args.save_steps == 0:
+                    torch.cuda.empty_cache()
+                    results = evaluate(args, model, tokenizer, eval_dataset)
+                    wandb.log({"eval/"+k:v for k,v in results.items()})
+                    for key, value in results.items():
+                        tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+                    if results['acc'] > curr_best:
+                        curr_best = results['acc']
+                        print("At iteration {}, best acc is {}".format(global_step, curr_best))
+                        # Save model checkpoint
+                        output_dir = args.output_dir
+                        if not os.path.exists(output_dir):
+                            os.makedirs(output_dir)
+                        model_to_save = model.module if hasattr(model,
+                                                                'module') else model  # Take care of distributed/parallel training
+                        model_to_save.save_pretrained(output_dir)
+                        tokenizer.save_pretrained(output_dir)
+                        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                        logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+    results = evaluate(args, model, tokenizer, eval_dataset)
+    for key, value in results.items():
+        tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+    if results['acc'] > curr_best:
+        curr_best = results['acc']
+        # Save model checkpoint
+        output_dir = args.output_dir
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        model_to_save = model.module if hasattr(model,
+                                                'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(output_dir)
+        tokenizer.save_pretrained(output_dir)
+        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+        logger.info("Saving model checkpoint to %s", output_dir)
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+    return global_step, tr_loss / global_step
+
+
+def save_logits(logits_all, filename):
+    with open(filename, "w") as f:
+        for i in range(len(logits_all)):
+            for j in range(len(logits_all[i])):
+                f.write(str(logits_all[i][j]))
+                if j == len(logits_all[i]) - 1:
+                    f.write("\n")
+                else:
+                    f.write(" ")
+
+
+def evaluate(args, model, tokenizer, eval_dataset):
+    results = {}
+    if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+        os.makedirs(args.output_dir)
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+    # Note that DistributedSampler samples randomly
+    eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
+    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size,
+                                 collate_fn=mCollateFn)
+
+    # Eval!
+    logger.info("***** Running evaluation *****")
+    logger.info("  Num examples = %d", len(eval_dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    CE = torch.nn.CrossEntropyLoss(reduction='none')
+    preds = []
+    out_label_ids = []
+    for batch in tqdm(eval_dataloader, desc="Evaluating"):
+        model.eval()
+        with torch.no_grad():
+            num_cand = len(batch[0][0])
+            choice_loss = []
+            choice_seq_lens = np.array([0] + [len(c) for sample in batch[0] for c in sample])
+            choice_seq_lens = np.cumsum(choice_seq_lens)
+            input_ids = torch.cat([c for sample in batch[0] for c in sample], dim=0).to(args.device)
+            att_mask = torch.cat([c for sample in batch[1] for c in sample], dim=0).to(args.device)
+            input_labels = torch.cat([c for sample in batch[2] for c in sample], dim=0).to(args.device)
+            if len(input_ids) < args.max_sequence_per_time:
+                inputs = {'input_ids': input_ids,
+                          'attention_mask': att_mask}
+                outputs = model(**inputs)
+                ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)), input_labels.view(-1))
+                ce_loss = ce_loss.view(outputs[0].size(0), -1).sum(1)
+            else:
+                ce_loss = []
+                for chunk in range(0, len(input_ids), args.max_sequence_per_time):
+                    inputs = {'input_ids': input_ids[chunk:chunk + args.max_sequence_per_time],
+                              'attention_mask': att_mask[chunk:chunk + args.max_sequence_per_time]}
+                    outputs = model(**inputs)
+                    tmp_ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)),
+                                     input_labels[chunk:chunk + args.max_sequence_per_time].view(-1))
+                    tmp_ce_loss = tmp_ce_loss.view(outputs[0].size(0), -1).sum(1)
+                    ce_loss.append(tmp_ce_loss)
+                ce_loss = torch.cat(ce_loss, dim=0)
+            for c_i in range(len(choice_seq_lens) - 1):
+                start = choice_seq_lens[c_i]
+                end = choice_seq_lens[c_i + 1]
+                choice_loss.append(-ce_loss[start:end].sum() / (end - start))
+            choice_loss = torch.stack(choice_loss)
+            choice_loss = choice_loss.view(-1, num_cand)
+        preds.append(choice_loss)
+        out_label_ids.append(batch[3].numpy())
+    preds = torch.cat(preds, dim=0).cpu().numpy()
+    save_logits(preds.tolist(), os.path.join(args.output_dir, args.logits_file))
+    preds = np.argmax(preds, axis=1)
+    result = accuracy(preds, np.concatenate(out_label_ids, axis=0))
+    results.update(result)
+    output_eval_file = os.path.join(args.output_dir, args.results_file)
+    with open(output_eval_file, "w") as writer:
+        logger.info("***** Eval results *****")
+        for key in sorted(result.keys()):
+            print("%s = %s\n" % (key, str(result[key])))
+            logger.info("  %s = %s", key, str(result[key]))
+            writer.write("%s = %s\n" % (key, str(result[key])))
+    return results
+
+
+def write_data(filename, data):
+    with open(filename, 'w') as fout:
+        for sample in data:
+            fout.write(json.dumps(sample))
+            fout.write('\n')
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+    processor = myprocessors[task](args)
+    cached_features_file = os.path.join(args.output_dir, 'cached_{}_{}_{}_{}'.format(
+        'dev' if evaluate else 'train',
+        str(args.model_type),
+        str(args.max_seq_length),
+        str(task)))
+    if os.path.exists(cached_features_file): # remove evaluate
+        features = torch.load(cached_features_file)
+    else:
+        examples = processor.get_dev_examples() if evaluate else processor.get_train_examples()
+        features = convert_examples_to_features(examples, tokenizer, max_length=args.max_seq_length)
+        # if evaluate:
+        torch.save(features, cached_features_file)
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+    print('max_words_to_mask is %s for pretraining tasks %s' % (args.max_words_to_mask, task))
+    return MyDataset(features, tokenizer.pad_token_id, tokenizer.mask_token_id, args.max_words_to_mask)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    ## Required parameters
+    parser.add_argument("--train_file", default=None, type=str, required=True,
+                        help="The train file name")
+    parser.add_argument("--dev_file", default=None, type=str, required=True,
+                        help="The dev file name")
+    parser.add_argument("--model_type", default=None, type=str, required=True,
+                        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
+                        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(
+                            MODEL_TYPES))
+    parser.add_argument("--config_name", default="", type=str,
+                        help="Pretrained config name or path if not the same as model_name")
+    parser.add_argument("--tokenizer_name", default="", type=str,
+                        help="Pretrained tokenizer name or path if not the same as model_name")
+    parser.add_argument("--cache_dir", default=".cache", type=str,
+                        help="Where do you want to store the pre-trained models downloaded")
+    parser.add_argument("--task_name", default=None, type=str, required=True,
+                        help="The name of the task to train selected in the list: " + ", ".join(myprocessors.keys()))
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+
+    ## Other parameters
+    parser.add_argument("--second_train_file", default=None, type=str,
+                        help="Used when combining ATOMIC and CWWV")
+    parser.add_argument("--second_dev_file", default=None, type=str,
+                        help="Used when combining ATOMIC and CWWV")
+    parser.add_argument("--max_seq_length", default=128, type=int,
+                        help="The maximum total input sequence length after tokenization. Sequences longer "
+                             "than this will be truncated, sequences shorter will be padded.")
+    parser.add_argument("--max_words_to_mask", default=6, type=int,
+                        help="The maximum number of tokens to mask when computing scores")
+    parser.add_argument("--max_sequence_per_time", default=80, type=int,
+                        help="The maximum number of sequences to feed into the model")
+    parser.add_argument("--do_train", action='store_true',
+                        help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true',
+                        help="Whether to run eval on the dev set.")
+    parser.add_argument("--do_ext_eval", action='store_true',
+                        help="Whether to run external eval on the downstream mcqa datasets.")                    
+    parser.add_argument("--evaluate_during_training", action='store_true',
+                        help="Run evaluation during training at each logging step.")
+    parser.add_argument("--do_lower_case", action='store_true',
+                        help="Set this flag if you are using an uncased model.")
+    parser.add_argument("--per_gpu_train_batch_size", default=1, type=int,
+                        help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=1, type=int,
+                        help="Batch size per GPU/CPU for evaluation.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument("--margin", default=1.0, type=float,
+                        help="The margin for ranking loss")
+    parser.add_argument("--learning_rate", default=1e-5, type=float,
+                        help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.01, type=float,
+                        help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-6, type=float,
+                        help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float,
+                        help="Max gradient norm.")
+    parser.add_argument("--num_train_epochs", default=1.0, type=float,
+                        help="Total number of training epochs to perform.")
+    parser.add_argument("--max_steps", default=-1, type=int,
+                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
+    parser.add_argument("--warmup_steps", default=0, type=int,
+                        help="Linear warmup over warmup_steps.")
+    parser.add_argument("--warmup_proportion", default=0.05, type=float,
+                        help="Linear warmup over warmup proportion.")
+    parser.add_argument('--logging_steps', type=int, default=50,
+                        help="Log every X updates steps.")
+    parser.add_argument('--save_steps', type=int, default=50,
+                        help="Save checkpoint every X updates steps.")
+    parser.add_argument("--logits_file", default='logits_test.txt', type=str,
+                        help="The file where prediction logits will be written")
+    parser.add_argument("--results_file", default='eval_results.txt', type=str,
+                        help="The file where eval results will be written")
+    parser.add_argument("--no_cuda", action='store_true',
+                        help="Avoid using CUDA when available")
+    parser.add_argument('--overwrite_output_dir', action='store_true',
+                        help="Overwrite the content of the output directory")
+    parser.add_argument('--seed', type=int, default=42,
+                        help="random seed for initialization")
+    parser.add_argument('--fp16', action='store_true',
+                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
+    parser.add_argument('--fp16_opt_level', type=str, default='O1',
+                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                             "See details at https://nvidia.github.io/apex/amp.html")
+    parser.add_argument("--local_rank", type=int, default=-1,
+                        help="For distributed training: local_rank")
+    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
+    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")
+
+    ### for extrinsic evaluation
+
+    parser.add_argument("--eval_output_dir", default="./output/eval_results", type=str, required=True,
+                    help="output of the predictions")
+
+    args = parser.parse_args()
+
+    wandb.init(project="car_mcqa", config=args)
+
+    if os.path.exists(args.output_dir) and os.listdir(
+            args.output_dir) and not args.overwrite_output_dir and args.do_train:
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
+                args.output_dir))
+    if not os.path.exists(args.output_dir):
+        os.makedirs(args.output_dir)
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl')
+        args.n_gpu = 1
+    args.device = device
+
+    if args.do_train:
+        for handler in logging.root.handlers[:]:
+            logging.root.removeHandler(handler)
+    # Setup logging
+    if args.do_train:
+        log_file = os.path.join(args.output_dir, 'train.log')
+        logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                            datefmt='%m/%d/%Y %H:%M:%S',
+                            level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
+                            filename=log_file)
+        logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+                       args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
+        os.system("cp run_pretrain.py %s" % os.path.join(args.output_dir, 'run_pretrain.py'))
+        os.system("cp data_utils.py %s" % os.path.join(args.output_dir, 'data_utils.py'))
+
+    # Set seed
+    set_seed(args)
+    args.task_name = args.task_name.lower()
+    if args.task_name not in myprocessors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          finetuning_task=args.task_name, cache_dir=args.cache_dir)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case, cache_dir=args.cache_dir)
+    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config, cache_dir=args.cache_dir)
+
+    count = count_parameters(model)
+    print("number of params", count)
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+
+    eval_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=True)
+    print("num of eval set", len(eval_dataset))
+
+    if args.do_train:
+        init_result = evaluate(args, model, tokenizer, eval_dataset)
+        print(init_result)
+    
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        print("num train examples", len(train_dataset))
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer, eval_dataset)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+    
+    # Evaluation
+    
+    results = {}
+    if args.do_eval:
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = model_class.from_pretrained(args.output_dir)
+        model.eval()
+        model.to(args.device)
+        result = evaluate(args, model, tokenizer, eval_dataset)
+
+
+    # do extrinsic evaluation
+
+    if args.do_ext_eval:
+        del model
+        import gc
+        gc.collect()
+        torch.cuda.empty_cache()
+
+
+        ext_results = {}
+
+        for task_name, dataset_path in eval_tasks:
+            eval_args = argparse.Namespace()
+            eval_args.dataset_file = dataset_path
+            eval_args.lm = args.output_dir
+            eval_args.out_dir = os.path.join(args.eval_output_dir, os.path.basename( args.output_dir))
+            eval_args.device = 0
+            eval_args.reader = task_name
+            eval_args.overwrite_output_dir = args.overwrite_output_dir
+            eval_args.cache_dir = None
+            if task_name in ["socialiqa", "winogrande", "piqa", "commonsenseqa", "anli"]:
+                acc = evaluate_main(eval_args)
+                ext_results[task_name] = acc
+            else:
+                tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+                model = model_class.from_pretrained(args.output_dir)
+                model.eval()
+                model.to(args.device)
+                
+                # load data
+                examples = []
+                with open(dataset_path, "r") as f:
+                    for row in tqdm(f):
+                        sample = json.loads(row)
+                        examples.append(sample)
+                features = convert_examples_to_features(examples, tokenizer, max_length=args.max_seq_length)
+                eval_dataset = MyDataset(features, tokenizer.pad_token_id, tokenizer.mask_token_id, args.max_words_to_mask)
+                result = evaluate(args, model, tokenizer, eval_dataset)
+                ext_results[task_name] = result['acc']
+
+        wandb.log({"ext/"+task_name:acc for task_name, acc in ext_results.items()})
+        
+    # return results
+
+if __name__ == "__main__":
+    main()

runs/events.out.tfevents.1695471913.car-atm-2i-half-sample-name-1-0-0.28.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a757fc252e0640c729dd7f1f2fcf08dadb6184a9d3570e98e3011cc339dbd081
+size 46125

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

train.log

@@ -0,0 +1,557 @@
+09/23/2023 12:10:45 - WARNING - __main__ -   Process rank: -1, device: cuda, n_gpu: 1, distributed training: False, 16-bits training: False
+09/23/2023 12:11:04 - INFO - __main__ -   Training/evaluation parameters Namespace(train_file='../../../data/mcqa/atomic/train_atm_n_2i_half_sample_name.jsonl', dev_file='../../../data/mcqa/atomic/dev_random_10k.jsonl', model_type='deberta-mlm', model_name_or_path='microsoft/deberta-v3-large', config_name='', tokenizer_name='', cache_dir='.cache', task_name='atomic', output_dir='output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6', second_train_file=None, second_dev_file=None, max_seq_length=128, max_words_to_mask=6, max_sequence_per_time=80, do_train=True, do_eval=True, do_ext_eval=True, evaluate_during_training=True, do_lower_case=False, per_gpu_train_batch_size=2, per_gpu_eval_batch_size=16, gradient_accumulation_steps=16, margin=1.0, learning_rate=5e-06, weight_decay=0.01, adam_epsilon=1e-06, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, warmup_steps=0, warmup_proportion=0.05, logging_steps=50, save_steps=500, logits_file='logits_test.txt', results_file='eval_results.txt', no_cuda=False, overwrite_output_dir=False, seed=42, fp16=False, fp16_opt_level='O1', local_rank=-1, server_ip='', server_port='', eval_output_dir='./eval_results', n_gpu=1, device=device(type='cuda'))
+09/23/2023 12:11:13 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 12:11:13 - INFO - __main__ -     Num examples = 10000
+09/23/2023 12:11:13 - INFO - __main__ -     Batch size = 16
+09/23/2023 12:15:11 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 12:15:11 - INFO - __main__ -     acc = 0.3392
+09/23/2023 12:25:13 - INFO - __main__ -   warm up steps = 835
+09/23/2023 12:25:13 - INFO - __main__ -   ***** Running training *****
+09/23/2023 12:25:13 - INFO - __main__ -     Num examples = 534833
+09/23/2023 12:25:13 - INFO - __main__ -     Num Epochs = 1
+09/23/2023 12:25:13 - INFO - __main__ -     Instantaneous batch size per GPU = 2
+09/23/2023 12:25:13 - INFO - __main__ -     Total train batch size (w. parallel, distributed & accumulation) = 32
+09/23/2023 12:25:13 - INFO - __main__ -     Gradient Accumulation steps = 16
+09/23/2023 12:25:13 - INFO - __main__ -     Total optimization steps = 16713
+09/23/2023 12:28:54 - INFO - __main__ -    global_step = 50, average loss = 0.6903331369534135
+09/23/2023 12:32:33 - INFO - __main__ -    global_step = 100, average loss = 0.6819266405794769
+09/23/2023 12:36:13 - INFO - __main__ -    global_step = 150, average loss = 0.6690767159638926
+09/23/2023 12:39:56 - INFO - __main__ -    global_step = 200, average loss = 0.6476348407182377
+09/23/2023 12:43:39 - INFO - __main__ -    global_step = 250, average loss = 0.6220815655076877
+09/23/2023 12:47:19 - INFO - __main__ -    global_step = 300, average loss = 0.5299683179453859
+09/23/2023 12:50:56 - INFO - __main__ -    global_step = 350, average loss = 0.39345016410181416
+09/23/2023 12:54:38 - INFO - __main__ -    global_step = 400, average loss = 0.31127411118301096
+09/23/2023 12:58:19 - INFO - __main__ -    global_step = 450, average loss = 0.25150225180907
+09/23/2023 13:02:00 - INFO - __main__ -    global_step = 500, average loss = 0.22586858159028453
+09/23/2023 13:02:01 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 13:02:01 - INFO - __main__ -     Num examples = 10000
+09/23/2023 13:02:01 - INFO - __main__ -     Batch size = 16
+09/23/2023 13:05:56 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 13:05:56 - INFO - __main__ -     acc = 0.6996
+09/23/2023 13:06:23 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 13:10:02 - INFO - __main__ -    global_step = 550, average loss = 0.22251796642665794
+09/23/2023 13:13:46 - INFO - __main__ -    global_step = 600, average loss = 0.19366045010890956
+09/23/2023 13:17:29 - INFO - __main__ -    global_step = 650, average loss = 0.18587105088678071
+09/23/2023 13:21:15 - INFO - __main__ -    global_step = 700, average loss = 0.1760789550206391
+09/23/2023 13:24:59 - INFO - __main__ -    global_step = 750, average loss = 0.18312411408871412
+09/23/2023 13:28:42 - INFO - __main__ -    global_step = 800, average loss = 0.15576540186157217
+09/23/2023 13:32:25 - INFO - __main__ -    global_step = 850, average loss = 0.16302873345994157
+09/23/2023 13:36:07 - INFO - __main__ -    global_step = 900, average loss = 0.15725697406036487
+09/23/2023 13:39:46 - INFO - __main__ -    global_step = 950, average loss = 0.15640976145299645
+09/23/2023 13:43:33 - INFO - __main__ -    global_step = 1000, average loss = 0.15606625928507128
+09/23/2023 13:43:34 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 13:43:34 - INFO - __main__ -     Num examples = 10000
+09/23/2023 13:43:34 - INFO - __main__ -     Batch size = 16
+09/23/2023 13:47:30 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 13:47:30 - INFO - __main__ -     acc = 0.7961
+09/23/2023 13:47:58 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 13:51:41 - INFO - __main__ -    global_step = 1050, average loss = 0.14431810150181262
+09/23/2023 13:55:20 - INFO - __main__ -    global_step = 1100, average loss = 0.15233074207513708
+09/23/2023 13:59:01 - INFO - __main__ -    global_step = 1150, average loss = 0.1404175848151772
+09/23/2023 14:02:44 - INFO - __main__ -    global_step = 1200, average loss = 0.12134294869215864
+09/23/2023 14:06:20 - INFO - __main__ -    global_step = 1250, average loss = 0.1363200130731275
+09/23/2023 14:09:59 - INFO - __main__ -    global_step = 1300, average loss = 0.13769450530940958
+09/23/2023 14:13:43 - INFO - __main__ -    global_step = 1350, average loss = 0.12156560226379952
+09/23/2023 14:17:18 - INFO - __main__ -    global_step = 1400, average loss = 0.12623315585107775
+09/23/2023 14:20:59 - INFO - __main__ -    global_step = 1450, average loss = 0.14377202547417256
+09/23/2023 14:24:33 - INFO - __main__ -    global_step = 1500, average loss = 0.1286695548933858
+09/23/2023 14:24:34 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 14:24:34 - INFO - __main__ -     Num examples = 10000
+09/23/2023 14:24:34 - INFO - __main__ -     Batch size = 16
+09/23/2023 14:28:29 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 14:28:29 - INFO - __main__ -     acc = 0.8048
+09/23/2023 14:28:56 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 14:32:42 - INFO - __main__ -    global_step = 1550, average loss = 0.1198868363915244
+09/23/2023 14:36:24 - INFO - __main__ -    global_step = 1600, average loss = 0.12324378551486007
+09/23/2023 14:40:00 - INFO - __main__ -    global_step = 1650, average loss = 0.11938468464672042
+09/23/2023 14:43:41 - INFO - __main__ -    global_step = 1700, average loss = 0.14236379045556533
+09/23/2023 14:47:22 - INFO - __main__ -    global_step = 1750, average loss = 0.13320694023670512
+09/23/2023 14:51:02 - INFO - __main__ -    global_step = 1800, average loss = 0.13622453257718006
+09/23/2023 14:54:42 - INFO - __main__ -    global_step = 1850, average loss = 0.13987649206645072
+09/23/2023 14:58:22 - INFO - __main__ -    global_step = 1900, average loss = 0.12299754774277971
+09/23/2023 15:02:05 - INFO - __main__ -    global_step = 1950, average loss = 0.11868109124743569
+09/23/2023 15:05:47 - INFO - __main__ -    global_step = 2000, average loss = 0.1415042275990345
+09/23/2023 15:05:47 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 15:05:47 - INFO - __main__ -     Num examples = 10000
+09/23/2023 15:05:47 - INFO - __main__ -     Batch size = 16
+09/23/2023 15:09:43 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 15:09:43 - INFO - __main__ -     acc = 0.8063
+09/23/2023 15:10:10 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 15:13:51 - INFO - __main__ -    global_step = 2050, average loss = 0.11399275673671581
+09/23/2023 15:17:31 - INFO - __main__ -    global_step = 2100, average loss = 0.1065546132405143
+09/23/2023 15:21:11 - INFO - __main__ -    global_step = 2150, average loss = 0.12809142941467144
+09/23/2023 15:24:51 - INFO - __main__ -    global_step = 2200, average loss = 0.12454848410692648
+09/23/2023 15:28:34 - INFO - __main__ -    global_step = 2250, average loss = 0.10986286829065647
+09/23/2023 15:32:14 - INFO - __main__ -    global_step = 2300, average loss = 0.11237965747121052
+09/23/2023 15:35:56 - INFO - __main__ -    global_step = 2350, average loss = 0.10897610924319451
+09/23/2023 15:39:41 - INFO - __main__ -    global_step = 2400, average loss = 0.12056981857070241
+09/23/2023 15:43:24 - INFO - __main__ -    global_step = 2450, average loss = 0.13911059297635803
+09/23/2023 15:47:10 - INFO - __main__ -    global_step = 2500, average loss = 0.11335444856034883
+09/23/2023 15:47:10 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 15:47:10 - INFO - __main__ -     Num examples = 10000
+09/23/2023 15:47:10 - INFO - __main__ -     Batch size = 16
+09/23/2023 15:51:06 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 15:51:06 - INFO - __main__ -     acc = 0.8234
+09/23/2023 15:51:32 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 15:55:10 - INFO - __main__ -    global_step = 2550, average loss = 0.12103958850973867
+09/23/2023 15:58:57 - INFO - __main__ -    global_step = 2600, average loss = 0.11913071399074397
+09/23/2023 16:02:38 - INFO - __main__ -    global_step = 2650, average loss = 0.11255583499452769
+09/23/2023 16:06:28 - INFO - __main__ -    global_step = 2700, average loss = 0.1006322616293619
+09/23/2023 16:10:12 - INFO - __main__ -    global_step = 2750, average loss = 0.0932968783121487
+09/23/2023 16:13:51 - INFO - __main__ -    global_step = 2800, average loss = 0.11056979637924087
+09/23/2023 16:17:38 - INFO - __main__ -    global_step = 2850, average loss = 0.12318793082176853
+09/23/2023 16:21:21 - INFO - __main__ -    global_step = 2900, average loss = 0.10864610994302439
+09/23/2023 16:25:03 - INFO - __main__ -    global_step = 2950, average loss = 0.11261582636667299
+09/23/2023 16:28:40 - INFO - __main__ -    global_step = 3000, average loss = 0.12150005620278534
+09/23/2023 16:28:40 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 16:28:40 - INFO - __main__ -     Num examples = 10000
+09/23/2023 16:28:40 - INFO - __main__ -     Batch size = 16
+09/23/2023 16:32:35 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 16:32:35 - INFO - __main__ -     acc = 0.8261
+09/23/2023 16:33:02 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 16:36:46 - INFO - __main__ -    global_step = 3050, average loss = 0.10565035182957218
+09/23/2023 16:40:30 - INFO - __main__ -    global_step = 3100, average loss = 0.10429829731896462
+09/23/2023 16:44:14 - INFO - __main__ -    global_step = 3150, average loss = 0.10812272985053824
+09/23/2023 16:47:54 - INFO - __main__ -    global_step = 3200, average loss = 0.12238092143270478
+09/23/2023 16:51:33 - INFO - __main__ -    global_step = 3250, average loss = 0.10868940783606376
+09/23/2023 16:55:14 - INFO - __main__ -    global_step = 3300, average loss = 0.1209917226509424
+09/23/2023 16:58:59 - INFO - __main__ -    global_step = 3350, average loss = 0.1191260662042896
+09/23/2023 17:02:41 - INFO - __main__ -    global_step = 3400, average loss = 0.1174743126919202
+09/23/2023 17:06:26 - INFO - __main__ -    global_step = 3450, average loss = 0.100895225374843
+09/23/2023 17:10:02 - INFO - __main__ -    global_step = 3500, average loss = 0.0931866138278565
+09/23/2023 17:10:03 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 17:10:03 - INFO - __main__ -     Num examples = 10000
+09/23/2023 17:10:03 - INFO - __main__ -     Batch size = 16
+09/23/2023 17:13:58 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 17:13:58 - INFO - __main__ -     acc = 0.8229
+09/23/2023 17:17:45 - INFO - __main__ -    global_step = 3550, average loss = 0.10633477224648231
+09/23/2023 17:21:30 - INFO - __main__ -    global_step = 3600, average loss = 0.1021722938354651
+09/23/2023 17:25:11 - INFO - __main__ -    global_step = 3650, average loss = 0.10295378862727375
+09/23/2023 17:28:50 - INFO - __main__ -    global_step = 3700, average loss = 0.1024187771679135
+09/23/2023 17:32:34 - INFO - __main__ -    global_step = 3750, average loss = 0.09922411829451448
+09/23/2023 17:36:14 - INFO - __main__ -    global_step = 3800, average loss = 0.11105157318372222
+09/23/2023 17:39:57 - INFO - __main__ -    global_step = 3850, average loss = 0.12378941989987652
+09/23/2023 17:43:42 - INFO - __main__ -    global_step = 3900, average loss = 0.1034327056143593
+09/23/2023 17:47:25 - INFO - __main__ -    global_step = 3950, average loss = 0.09697925167827634
+09/23/2023 17:51:09 - INFO - __main__ -    global_step = 4000, average loss = 0.11230336717126192
+09/23/2023 17:51:09 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 17:51:09 - INFO - __main__ -     Num examples = 10000
+09/23/2023 17:51:09 - INFO - __main__ -     Batch size = 16
+09/23/2023 17:55:05 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 17:55:05 - INFO - __main__ -     acc = 0.8371
+09/23/2023 17:55:32 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 17:59:12 - INFO - __main__ -    global_step = 4050, average loss = 0.10925351051962934
+09/23/2023 18:03:00 - INFO - __main__ -    global_step = 4100, average loss = 0.09795216493275802
+09/23/2023 18:06:43 - INFO - __main__ -    global_step = 4150, average loss = 0.09962472554965643
+09/23/2023 18:10:25 - INFO - __main__ -    global_step = 4200, average loss = 0.10342389734141762
+09/23/2023 18:14:05 - INFO - __main__ -    global_step = 4250, average loss = 0.09674815248567029
+09/23/2023 18:17:48 - INFO - __main__ -    global_step = 4300, average loss = 0.10319628210134396
+09/23/2023 18:21:33 - INFO - __main__ -    global_step = 4350, average loss = 0.09340641272166977
+09/23/2023 18:25:14 - INFO - __main__ -    global_step = 4400, average loss = 0.10845618240913608
+09/23/2023 18:28:59 - INFO - __main__ -    global_step = 4450, average loss = 0.11604906246473547
+09/23/2023 18:32:43 - INFO - __main__ -    global_step = 4500, average loss = 0.09590314964269055
+09/23/2023 18:32:43 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 18:32:43 - INFO - __main__ -     Num examples = 10000
+09/23/2023 18:32:43 - INFO - __main__ -     Batch size = 16
+09/23/2023 18:36:38 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 18:36:38 - INFO - __main__ -     acc = 0.8305
+09/23/2023 18:40:22 - INFO - __main__ -    global_step = 4550, average loss = 0.09955280199857952
+09/23/2023 18:44:07 - INFO - __main__ -    global_step = 4600, average loss = 0.09018894311768236
+09/23/2023 18:47:49 - INFO - __main__ -    global_step = 4650, average loss = 0.11624654464081687
+09/23/2023 18:51:30 - INFO - __main__ -    global_step = 4700, average loss = 0.11213955332923434
+09/23/2023 18:55:07 - INFO - __main__ -    global_step = 4750, average loss = 0.11335175217776851
+09/23/2023 18:58:47 - INFO - __main__ -    global_step = 4800, average loss = 0.10374061681199237
+09/23/2023 19:02:34 - INFO - __main__ -    global_step = 4850, average loss = 0.09650620453016018
+09/23/2023 19:06:16 - INFO - __main__ -    global_step = 4900, average loss = 0.1034209698169434
+09/23/2023 19:09:53 - INFO - __main__ -    global_step = 4950, average loss = 0.10046588191311458
+09/23/2023 19:13:34 - INFO - __main__ -    global_step = 5000, average loss = 0.10752027794980677
+09/23/2023 19:13:34 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 19:13:34 - INFO - __main__ -     Num examples = 10000
+09/23/2023 19:13:34 - INFO - __main__ -     Batch size = 16
+09/23/2023 19:17:29 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 19:17:29 - INFO - __main__ -     acc = 0.8355
+09/23/2023 19:21:19 - INFO - __main__ -    global_step = 5050, average loss = 0.10195030277842307
+09/23/2023 19:24:58 - INFO - __main__ -    global_step = 5100, average loss = 0.10987481483532065
+09/23/2023 19:28:41 - INFO - __main__ -    global_step = 5150, average loss = 0.10906005093554995
+09/23/2023 19:32:23 - INFO - __main__ -    global_step = 5200, average loss = 0.09835696181547973
+09/23/2023 19:36:06 - INFO - __main__ -    global_step = 5250, average loss = 0.10181126694624254
+09/23/2023 19:39:52 - INFO - __main__ -    global_step = 5300, average loss = 0.08663028705283068
+09/23/2023 19:43:30 - INFO - __main__ -    global_step = 5350, average loss = 0.10507196654667496
+09/23/2023 19:47:18 - INFO - __main__ -    global_step = 5400, average loss = 0.108608085659871
+09/23/2023 19:51:03 - INFO - __main__ -    global_step = 5450, average loss = 0.099619501844536
+09/23/2023 19:54:49 - INFO - __main__ -    global_step = 5500, average loss = 0.10225338533447939
+09/23/2023 19:54:49 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 19:54:49 - INFO - __main__ -     Num examples = 10000
+09/23/2023 19:54:49 - INFO - __main__ -     Batch size = 16
+09/23/2023 19:58:45 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 19:58:45 - INFO - __main__ -     acc = 0.8279
+09/23/2023 20:02:26 - INFO - __main__ -    global_step = 5550, average loss = 0.10436682683890468
+09/23/2023 20:06:11 - INFO - __main__ -    global_step = 5600, average loss = 0.10477761221260153
+09/23/2023 20:09:52 - INFO - __main__ -    global_step = 5650, average loss = 0.09326410317778937
+09/23/2023 20:13:31 - INFO - __main__ -    global_step = 5700, average loss = 0.11269167278223904
+09/23/2023 20:17:16 - INFO - __main__ -    global_step = 5750, average loss = 0.10188864256499074
+09/23/2023 20:21:00 - INFO - __main__ -    global_step = 5800, average loss = 0.10433580860199981
+09/23/2023 20:24:43 - INFO - __main__ -    global_step = 5850, average loss = 0.08972063858884212
+09/23/2023 20:28:22 - INFO - __main__ -    global_step = 5900, average loss = 0.1065664726671821
+09/23/2023 20:32:07 - INFO - __main__ -    global_step = 5950, average loss = 0.10174332244623656
+09/23/2023 20:35:49 - INFO - __main__ -    global_step = 6000, average loss = 0.08872646622621687
+09/23/2023 20:35:49 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 20:35:49 - INFO - __main__ -     Num examples = 10000
+09/23/2023 20:35:49 - INFO - __main__ -     Batch size = 16
+09/23/2023 20:39:45 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 20:39:45 - INFO - __main__ -     acc = 0.8363
+09/23/2023 20:43:29 - INFO - __main__ -    global_step = 6050, average loss = 0.10705330887685705
+09/23/2023 20:47:16 - INFO - __main__ -    global_step = 6100, average loss = 0.09171272950654384
+09/23/2023 20:50:59 - INFO - __main__ -    global_step = 6150, average loss = 0.0861645900901567
+09/23/2023 20:54:46 - INFO - __main__ -    global_step = 6200, average loss = 0.08994678908144124
+09/23/2023 20:58:32 - INFO - __main__ -    global_step = 6250, average loss = 0.08786970607354305
+09/23/2023 21:02:13 - INFO - __main__ -    global_step = 6300, average loss = 0.09656520821336016
+09/23/2023 21:05:56 - INFO - __main__ -    global_step = 6350, average loss = 0.09620310332989902
+09/23/2023 21:09:42 - INFO - __main__ -    global_step = 6400, average loss = 0.09152124080545036
+09/23/2023 21:13:22 - INFO - __main__ -    global_step = 6450, average loss = 0.09472263304131047
+09/23/2023 21:17:06 - INFO - __main__ -    global_step = 6500, average loss = 0.10554198697194807
+09/23/2023 21:17:06 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 21:17:06 - INFO - __main__ -     Num examples = 10000
+09/23/2023 21:17:06 - INFO - __main__ -     Batch size = 16
+09/23/2023 21:21:01 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 21:21:01 - INFO - __main__ -     acc = 0.841
+09/23/2023 21:21:28 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 21:25:14 - INFO - __main__ -    global_step = 6550, average loss = 0.09830655160796596
+09/23/2023 21:28:55 - INFO - __main__ -    global_step = 6600, average loss = 0.09539545015402837
+09/23/2023 21:32:40 - INFO - __main__ -    global_step = 6650, average loss = 0.09118585625503328
+09/23/2023 21:36:18 - INFO - __main__ -    global_step = 6700, average loss = 0.09700520555491493
+09/23/2023 21:40:03 - INFO - __main__ -    global_step = 6750, average loss = 0.105271778342576
+09/23/2023 21:43:45 - INFO - __main__ -    global_step = 6800, average loss = 0.10975144471223758
+09/23/2023 21:47:28 - INFO - __main__ -    global_step = 6850, average loss = 0.09920243133579788
+09/23/2023 21:51:11 - INFO - __main__ -    global_step = 6900, average loss = 0.09791661702009151
+09/23/2023 21:54:51 - INFO - __main__ -    global_step = 6950, average loss = 0.08630025177910283
+09/23/2023 21:58:29 - INFO - __main__ -    global_step = 7000, average loss = 0.09660528897402401
+09/23/2023 21:58:29 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 21:58:29 - INFO - __main__ -     Num examples = 10000
+09/23/2023 21:58:29 - INFO - __main__ -     Batch size = 16
+09/23/2023 22:02:25 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 22:02:25 - INFO - __main__ -     acc = 0.843
+09/23/2023 22:02:51 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 22:06:33 - INFO - __main__ -    global_step = 7050, average loss = 0.10305566756385814
+09/23/2023 22:10:07 - INFO - __main__ -    global_step = 7100, average loss = 0.10687436608219286
+09/23/2023 22:13:47 - INFO - __main__ -    global_step = 7150, average loss = 0.0946133067667688
+09/23/2023 22:17:27 - INFO - __main__ -    global_step = 7200, average loss = 0.09795189084834419
+09/23/2023 22:21:17 - INFO - __main__ -    global_step = 7250, average loss = 0.09060888570308634
+09/23/2023 22:24:59 - INFO - __main__ -    global_step = 7300, average loss = 0.0877145413684775
+09/23/2023 22:28:35 - INFO - __main__ -    global_step = 7350, average loss = 0.10495714643941029
+09/23/2023 22:32:21 - INFO - __main__ -    global_step = 7400, average loss = 0.07401456630654138
+09/23/2023 22:36:03 - INFO - __main__ -    global_step = 7450, average loss = 0.09523518772701209
+09/23/2023 22:39:41 - INFO - __main__ -    global_step = 7500, average loss = 0.10137952610446518
+09/23/2023 22:39:41 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 22:39:41 - INFO - __main__ -     Num examples = 10000
+09/23/2023 22:39:41 - INFO - __main__ -     Batch size = 16
+09/23/2023 22:43:37 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 22:43:37 - INFO - __main__ -     acc = 0.846
+09/23/2023 22:44:03 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 22:47:46 - INFO - __main__ -    global_step = 7550, average loss = 0.09563293447645264
+09/23/2023 22:51:31 - INFO - __main__ -    global_step = 7600, average loss = 0.09618103489105125
+09/23/2023 22:55:13 - INFO - __main__ -    global_step = 7650, average loss = 0.08849806944810552
+09/23/2023 22:58:54 - INFO - __main__ -    global_step = 7700, average loss = 0.10007433392238455
+09/23/2023 23:02:36 - INFO - __main__ -    global_step = 7750, average loss = 0.09035434001329122
+09/23/2023 23:06:24 - INFO - __main__ -    global_step = 7800, average loss = 0.09338357288788757
+09/23/2023 23:10:04 - INFO - __main__ -    global_step = 7850, average loss = 0.09912064949181514
+09/23/2023 23:13:47 - INFO - __main__ -    global_step = 7900, average loss = 0.08827902228244057
+09/23/2023 23:17:27 - INFO - __main__ -    global_step = 7950, average loss = 0.11218067690118914
+09/23/2023 23:21:09 - INFO - __main__ -    global_step = 8000, average loss = 0.08588292430682486
+09/23/2023 23:21:09 - INFO - __main__ -   ***** Running evaluation *****
+09/23/2023 23:21:09 - INFO - __main__ -     Num examples = 10000
+09/23/2023 23:21:09 - INFO - __main__ -     Batch size = 16
+09/23/2023 23:25:05 - INFO - __main__ -   ***** Eval results *****
+09/23/2023 23:25:05 - INFO - __main__ -     acc = 0.8472
+09/23/2023 23:25:31 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/23/2023 23:29:08 - INFO - __main__ -    global_step = 8050, average loss = 0.09245043838061974
+09/23/2023 23:32:54 - INFO - __main__ -    global_step = 8100, average loss = 0.08283289226481429
+09/23/2023 23:36:34 - INFO - __main__ -    global_step = 8150, average loss = 0.08407623038449856
+09/23/2023 23:40:17 - INFO - __main__ -    global_step = 8200, average loss = 0.09736820162237564
+09/23/2023 23:44:06 - INFO - __main__ -    global_step = 8250, average loss = 0.08463705457368632
+09/23/2023 23:47:50 - INFO - __main__ -    global_step = 8300, average loss = 0.10010304888644896
+09/23/2023 23:51:35 - INFO - __main__ -    global_step = 8350, average loss = 0.09222401980725409
+09/23/2023 23:55:17 - INFO - __main__ -    global_step = 8400, average loss = 0.08634746881416504
+09/23/2023 23:58:59 - INFO - __main__ -    global_step = 8450, average loss = 0.08723288500368653
+09/24/2023 00:02:37 - INFO - __main__ -    global_step = 8500, average loss = 0.10130320921433394
+09/24/2023 00:02:37 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 00:02:37 - INFO - __main__ -     Num examples = 10000
+09/24/2023 00:02:37 - INFO - __main__ -     Batch size = 16
+09/24/2023 00:06:32 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 00:06:32 - INFO - __main__ -     acc = 0.8452
+09/24/2023 00:10:13 - INFO - __main__ -    global_step = 8550, average loss = 0.0889340414837352
+09/24/2023 00:13:53 - INFO - __main__ -    global_step = 8600, average loss = 0.0960574367789377
+09/24/2023 00:17:37 - INFO - __main__ -    global_step = 8650, average loss = 0.07860265792332939
+09/24/2023 00:21:20 - INFO - __main__ -    global_step = 8700, average loss = 0.09233207383847912
+09/24/2023 00:25:05 - INFO - __main__ -    global_step = 8750, average loss = 0.09803196908305836
+09/24/2023 00:28:44 - INFO - __main__ -    global_step = 8800, average loss = 0.08913468146740343
+09/24/2023 00:32:26 - INFO - __main__ -    global_step = 8850, average loss = 0.0880054514182666
+09/24/2023 00:36:11 - INFO - __main__ -    global_step = 8900, average loss = 0.0839999437017832
+09/24/2023 00:39:52 - INFO - __main__ -    global_step = 8950, average loss = 0.10094311676693905
+09/24/2023 00:43:32 - INFO - __main__ -    global_step = 9000, average loss = 0.10011614485312748
+09/24/2023 00:43:32 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 00:43:32 - INFO - __main__ -     Num examples = 10000
+09/24/2023 00:43:32 - INFO - __main__ -     Batch size = 16
+09/24/2023 00:47:27 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 00:47:27 - INFO - __main__ -     acc = 0.8463
+09/24/2023 00:51:10 - INFO - __main__ -    global_step = 9050, average loss = 0.09407024829903093
+09/24/2023 00:54:48 - INFO - __main__ -    global_step = 9100, average loss = 0.09510339217069032
+09/24/2023 00:58:27 - INFO - __main__ -    global_step = 9150, average loss = 0.09413513723055075
+09/24/2023 01:02:10 - INFO - __main__ -    global_step = 9200, average loss = 0.08488880819528276
+09/24/2023 01:05:47 - INFO - __main__ -    global_step = 9250, average loss = 0.09847264970565447
+09/24/2023 01:09:28 - INFO - __main__ -    global_step = 9300, average loss = 0.08640140883806452
+09/24/2023 01:13:08 - INFO - __main__ -    global_step = 9350, average loss = 0.07884123000112594
+09/24/2023 01:16:54 - INFO - __main__ -    global_step = 9400, average loss = 0.0831154512307694
+09/24/2023 01:20:32 - INFO - __main__ -    global_step = 9450, average loss = 0.09913980022422038
+09/24/2023 01:24:11 - INFO - __main__ -    global_step = 9500, average loss = 0.09805536182444484
+09/24/2023 01:24:11 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 01:24:11 - INFO - __main__ -     Num examples = 10000
+09/24/2023 01:24:11 - INFO - __main__ -     Batch size = 16
+09/24/2023 01:28:07 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 01:28:07 - INFO - __main__ -     acc = 0.8463
+09/24/2023 01:31:55 - INFO - __main__ -    global_step = 9550, average loss = 0.0912455873134968
+09/24/2023 01:35:38 - INFO - __main__ -    global_step = 9600, average loss = 0.10278063782119716
+09/24/2023 01:39:12 - INFO - __main__ -    global_step = 9650, average loss = 0.08788584528032516
+09/24/2023 01:42:53 - INFO - __main__ -    global_step = 9700, average loss = 0.08058010207216285
+09/24/2023 01:46:34 - INFO - __main__ -    global_step = 9750, average loss = 0.08765123128723644
+09/24/2023 01:50:14 - INFO - __main__ -    global_step = 9800, average loss = 0.09005017607181799
+09/24/2023 01:54:03 - INFO - __main__ -    global_step = 9850, average loss = 0.07892634223760979
+09/24/2023 01:57:44 - INFO - __main__ -    global_step = 9900, average loss = 0.07999062808303278
+09/24/2023 02:01:26 - INFO - __main__ -    global_step = 9950, average loss = 0.09494447313452838
+09/24/2023 02:05:06 - INFO - __main__ -    global_step = 10000, average loss = 0.0841888710015337
+09/24/2023 02:05:06 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 02:05:06 - INFO - __main__ -     Num examples = 10000
+09/24/2023 02:05:06 - INFO - __main__ -     Batch size = 16
+09/24/2023 02:09:01 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 02:09:01 - INFO - __main__ -     acc = 0.8471
+09/24/2023 02:12:40 - INFO - __main__ -    global_step = 10050, average loss = 0.08929907138342968
+09/24/2023 02:16:20 - INFO - __main__ -    global_step = 10100, average loss = 0.10172551687661326
+09/24/2023 02:20:00 - INFO - __main__ -    global_step = 10150, average loss = 0.09577305402533966
+09/24/2023 02:23:46 - INFO - __main__ -    global_step = 10200, average loss = 0.09480085656211486
+09/24/2023 02:27:27 - INFO - __main__ -    global_step = 10250, average loss = 0.07956519629078684
+09/24/2023 02:31:05 - INFO - __main__ -    global_step = 10300, average loss = 0.08291967767250753
+09/24/2023 02:34:47 - INFO - __main__ -    global_step = 10350, average loss = 0.09592102762369904
+09/24/2023 02:38:29 - INFO - __main__ -    global_step = 10400, average loss = 0.08570889301292482
+09/24/2023 02:42:13 - INFO - __main__ -    global_step = 10450, average loss = 0.07362440132081247
+09/24/2023 02:45:58 - INFO - __main__ -    global_step = 10500, average loss = 0.08574875552483718
+09/24/2023 02:45:58 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 02:45:58 - INFO - __main__ -     Num examples = 10000
+09/24/2023 02:45:58 - INFO - __main__ -     Batch size = 16
+09/24/2023 02:49:53 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 02:49:53 - INFO - __main__ -     acc = 0.8524
+09/24/2023 02:50:20 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 02:54:03 - INFO - __main__ -    global_step = 10550, average loss = 0.08846153970320302
+09/24/2023 02:57:43 - INFO - __main__ -    global_step = 10600, average loss = 0.08381684645668429
+09/24/2023 03:01:26 - INFO - __main__ -    global_step = 10650, average loss = 0.09288432469184045
+09/24/2023 03:05:08 - INFO - __main__ -    global_step = 10700, average loss = 0.08199916316298186
+09/24/2023 03:08:56 - INFO - __main__ -    global_step = 10750, average loss = 0.09068042659768252
+09/24/2023 03:12:37 - INFO - __main__ -    global_step = 10800, average loss = 0.08719110449641448
+09/24/2023 03:16:20 - INFO - __main__ -    global_step = 10850, average loss = 0.09036207084544003
+09/24/2023 03:20:04 - INFO - __main__ -    global_step = 10900, average loss = 0.095746248819637
+09/24/2023 03:23:45 - INFO - __main__ -    global_step = 10950, average loss = 0.1019882604497252
+09/24/2023 03:27:25 - INFO - __main__ -    global_step = 11000, average loss = 0.08660416512644588
+09/24/2023 03:27:25 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 03:27:25 - INFO - __main__ -     Num examples = 10000
+09/24/2023 03:27:25 - INFO - __main__ -     Batch size = 16
+09/24/2023 03:31:21 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 03:31:21 - INFO - __main__ -     acc = 0.8521
+09/24/2023 03:35:00 - INFO - __main__ -    global_step = 11050, average loss = 0.07959849048202158
+09/24/2023 03:38:42 - INFO - __main__ -    global_step = 11100, average loss = 0.08480279741248524
+09/24/2023 03:42:25 - INFO - __main__ -    global_step = 11150, average loss = 0.07940411141982623
+09/24/2023 03:46:06 - INFO - __main__ -    global_step = 11200, average loss = 0.08627346496621613
+09/24/2023 03:49:48 - INFO - __main__ -    global_step = 11250, average loss = 0.08515130840663915
+09/24/2023 03:53:28 - INFO - __main__ -    global_step = 11300, average loss = 0.08047833000106039
+09/24/2023 03:57:07 - INFO - __main__ -    global_step = 11350, average loss = 0.08884227124826338
+09/24/2023 04:00:47 - INFO - __main__ -    global_step = 11400, average loss = 0.09542614945773494
+09/24/2023 04:04:26 - INFO - __main__ -    global_step = 11450, average loss = 0.08332637125422479
+09/24/2023 04:08:07 - INFO - __main__ -    global_step = 11500, average loss = 0.09769482501476887
+09/24/2023 04:08:07 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 04:08:07 - INFO - __main__ -     Num examples = 10000
+09/24/2023 04:08:07 - INFO - __main__ -     Batch size = 16
+09/24/2023 04:12:02 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 04:12:02 - INFO - __main__ -     acc = 0.851
+09/24/2023 04:15:51 - INFO - __main__ -    global_step = 11550, average loss = 0.09137944790694746
+09/24/2023 04:19:38 - INFO - __main__ -    global_step = 11600, average loss = 0.07454582622590351
+09/24/2023 04:23:20 - INFO - __main__ -    global_step = 11650, average loss = 0.08284565404814202
+09/24/2023 04:26:59 - INFO - __main__ -    global_step = 11700, average loss = 0.0969824349215196
+09/24/2023 04:30:41 - INFO - __main__ -    global_step = 11750, average loss = 0.09389037321489013
+09/24/2023 04:34:23 - INFO - __main__ -    global_step = 11800, average loss = 0.08608788483528769
+09/24/2023 04:38:05 - INFO - __main__ -    global_step = 11850, average loss = 0.09322659247220144
+09/24/2023 04:41:49 - INFO - __main__ -    global_step = 11900, average loss = 0.09286965438863262
+09/24/2023 04:45:31 - INFO - __main__ -    global_step = 11950, average loss = 0.08214385434631367
+09/24/2023 04:49:12 - INFO - __main__ -    global_step = 12000, average loss = 0.09392224536069989
+09/24/2023 04:49:12 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 04:49:12 - INFO - __main__ -     Num examples = 10000
+09/24/2023 04:49:12 - INFO - __main__ -     Batch size = 16
+09/24/2023 04:53:07 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 04:53:07 - INFO - __main__ -     acc = 0.8514
+09/24/2023 04:56:53 - INFO - __main__ -    global_step = 12050, average loss = 0.08019034011129406
+09/24/2023 05:00:34 - INFO - __main__ -    global_step = 12100, average loss = 0.08210711618239656
+09/24/2023 05:04:16 - INFO - __main__ -    global_step = 12150, average loss = 0.08764273267355747
+09/24/2023 05:08:02 - INFO - __main__ -    global_step = 12200, average loss = 0.08758470895321807
+09/24/2023 05:11:48 - INFO - __main__ -    global_step = 12250, average loss = 0.07766548367973883
+09/24/2023 05:15:27 - INFO - __main__ -    global_step = 12300, average loss = 0.08148344823415755
+09/24/2023 05:19:08 - INFO - __main__ -    global_step = 12350, average loss = 0.08814196670609817
+09/24/2023 05:22:50 - INFO - __main__ -    global_step = 12400, average loss = 0.08936668847491092
+09/24/2023 05:26:29 - INFO - __main__ -    global_step = 12450, average loss = 0.08240065188347216
+09/24/2023 05:30:12 - INFO - __main__ -    global_step = 12500, average loss = 0.08683115135392655
+09/24/2023 05:30:12 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 05:30:12 - INFO - __main__ -     Num examples = 10000
+09/24/2023 05:30:12 - INFO - __main__ -     Batch size = 16
+09/24/2023 05:34:07 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 05:34:07 - INFO - __main__ -     acc = 0.8515
+09/24/2023 05:37:53 - INFO - __main__ -    global_step = 12550, average loss = 0.08871277472944712
+09/24/2023 05:41:34 - INFO - __main__ -    global_step = 12600, average loss = 0.08797626828309149
+09/24/2023 05:45:11 - INFO - __main__ -    global_step = 12650, average loss = 0.10095825259459616
+09/24/2023 05:48:58 - INFO - __main__ -    global_step = 12700, average loss = 0.07953012495926487
+09/24/2023 05:52:41 - INFO - __main__ -    global_step = 12750, average loss = 0.08843418272979761
+09/24/2023 05:56:19 - INFO - __main__ -    global_step = 12800, average loss = 0.07413991435227217
+09/24/2023 05:59:59 - INFO - __main__ -    global_step = 12850, average loss = 0.07519575585451094
+09/24/2023 06:03:48 - INFO - __main__ -    global_step = 12900, average loss = 0.08996981896292709
+09/24/2023 06:07:28 - INFO - __main__ -    global_step = 12950, average loss = 0.08996171029284597
+09/24/2023 06:11:11 - INFO - __main__ -    global_step = 13000, average loss = 0.08077499923689174
+09/24/2023 06:11:11 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 06:11:11 - INFO - __main__ -     Num examples = 10000
+09/24/2023 06:11:11 - INFO - __main__ -     Batch size = 16
+09/24/2023 06:15:06 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 06:15:06 - INFO - __main__ -     acc = 0.8527
+09/24/2023 06:15:33 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 06:19:13 - INFO - __main__ -    global_step = 13050, average loss = 0.08447560470420284
+09/24/2023 06:22:54 - INFO - __main__ -    global_step = 13100, average loss = 0.08299598100831646
+09/24/2023 06:26:32 - INFO - __main__ -    global_step = 13150, average loss = 0.08393764879734135
+09/24/2023 06:30:08 - INFO - __main__ -    global_step = 13200, average loss = 0.09848508099505125
+09/24/2023 06:33:47 - INFO - __main__ -    global_step = 13250, average loss = 0.09162080157435412
+09/24/2023 06:37:28 - INFO - __main__ -    global_step = 13300, average loss = 0.0914362099875143
+09/24/2023 06:41:09 - INFO - __main__ -    global_step = 13350, average loss = 0.07781068138462616
+09/24/2023 06:44:55 - INFO - __main__ -    global_step = 13400, average loss = 0.08868030074576382
+09/24/2023 06:48:36 - INFO - __main__ -    global_step = 13450, average loss = 0.08357623873533157
+09/24/2023 06:52:18 - INFO - __main__ -    global_step = 13500, average loss = 0.08828085365807055
+09/24/2023 06:52:18 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 06:52:18 - INFO - __main__ -     Num examples = 10000
+09/24/2023 06:52:18 - INFO - __main__ -     Batch size = 16
+09/24/2023 06:56:14 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 06:56:14 - INFO - __main__ -     acc = 0.8499
+09/24/2023 06:59:57 - INFO - __main__ -    global_step = 13550, average loss = 0.08140521681067185
+09/24/2023 07:03:37 - INFO - __main__ -    global_step = 13600, average loss = 0.08341409597109305
+09/24/2023 07:07:17 - INFO - __main__ -    global_step = 13650, average loss = 0.08142950747031136
+09/24/2023 07:10:56 - INFO - __main__ -    global_step = 13700, average loss = 0.09089667504686076
+09/24/2023 07:14:45 - INFO - __main__ -    global_step = 13750, average loss = 0.07177684095106088
+09/24/2023 07:18:24 - INFO - __main__ -    global_step = 13800, average loss = 0.08592368463818274
+09/24/2023 07:22:01 - INFO - __main__ -    global_step = 13850, average loss = 0.08120634569131653
+09/24/2023 07:25:48 - INFO - __main__ -    global_step = 13900, average loss = 0.08909589071197843
+09/24/2023 07:29:30 - INFO - __main__ -    global_step = 13950, average loss = 0.08629100337015189
+09/24/2023 07:33:10 - INFO - __main__ -    global_step = 14000, average loss = 0.07722124511306902
+09/24/2023 07:33:10 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 07:33:10 - INFO - __main__ -     Num examples = 10000
+09/24/2023 07:33:10 - INFO - __main__ -     Batch size = 16
+09/24/2023 07:37:05 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 07:37:05 - INFO - __main__ -     acc = 0.8533
+09/24/2023 07:37:32 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 07:41:11 - INFO - __main__ -    global_step = 14050, average loss = 0.08182521525057382
+09/24/2023 07:44:48 - INFO - __main__ -    global_step = 14100, average loss = 0.0902410151962249
+09/24/2023 07:48:28 - INFO - __main__ -    global_step = 14150, average loss = 0.07409664937826164
+09/24/2023 07:52:12 - INFO - __main__ -    global_step = 14200, average loss = 0.08879891355274594
+09/24/2023 07:55:53 - INFO - __main__ -    global_step = 14250, average loss = 0.09268313445325475
+09/24/2023 07:59:30 - INFO - __main__ -    global_step = 14300, average loss = 0.08798344542199629
+09/24/2023 08:03:13 - INFO - __main__ -    global_step = 14350, average loss = 0.09607475698139752
+09/24/2023 08:06:59 - INFO - __main__ -    global_step = 14400, average loss = 0.07222031111843535
+09/24/2023 08:10:40 - INFO - __main__ -    global_step = 14450, average loss = 0.07480319764195884
+09/24/2023 08:14:19 - INFO - __main__ -    global_step = 14500, average loss = 0.0838716509303049
+09/24/2023 08:14:19 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 08:14:19 - INFO - __main__ -     Num examples = 10000
+09/24/2023 08:14:19 - INFO - __main__ -     Batch size = 16
+09/24/2023 08:18:16 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 08:18:16 - INFO - __main__ -     acc = 0.8542
+09/24/2023 08:18:42 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 08:22:18 - INFO - __main__ -    global_step = 14550, average loss = 0.08034001361316769
+09/24/2023 08:25:55 - INFO - __main__ -    global_step = 14600, average loss = 0.07689567271547276
+09/24/2023 08:29:37 - INFO - __main__ -    global_step = 14650, average loss = 0.09093381941405823
+09/24/2023 08:33:25 - INFO - __main__ -    global_step = 14700, average loss = 0.07569706412876258
+09/24/2023 08:37:04 - INFO - __main__ -    global_step = 14750, average loss = 0.07479940189456101
+09/24/2023 08:40:47 - INFO - __main__ -    global_step = 14800, average loss = 0.08522207450543647
+09/24/2023 08:44:34 - INFO - __main__ -    global_step = 14850, average loss = 0.0889268495763099
+09/24/2023 08:48:16 - INFO - __main__ -    global_step = 14900, average loss = 0.08616152721479012
+09/24/2023 08:51:56 - INFO - __main__ -    global_step = 14950, average loss = 0.07867321850848384
+09/24/2023 08:55:39 - INFO - __main__ -    global_step = 15000, average loss = 0.08426695556714549
+09/24/2023 08:55:39 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 08:55:39 - INFO - __main__ -     Num examples = 10000
+09/24/2023 08:55:39 - INFO - __main__ -     Batch size = 16
+09/24/2023 08:59:34 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 08:59:34 - INFO - __main__ -     acc = 0.8542
+09/24/2023 09:03:12 - INFO - __main__ -    global_step = 15050, average loss = 0.07868185437655484
+09/24/2023 09:07:00 - INFO - __main__ -    global_step = 15100, average loss = 0.08520105790423259
+09/24/2023 09:10:42 - INFO - __main__ -    global_step = 15150, average loss = 0.09536004922925713
+09/24/2023 09:14:19 - INFO - __main__ -    global_step = 15200, average loss = 0.08502999547665241
+09/24/2023 09:17:58 - INFO - __main__ -    global_step = 15250, average loss = 0.08957034896484402
+09/24/2023 09:21:34 - INFO - __main__ -    global_step = 15300, average loss = 0.07968287494033575
+09/24/2023 09:25:14 - INFO - __main__ -    global_step = 15350, average loss = 0.08545487473544199
+09/24/2023 09:28:55 - INFO - __main__ -    global_step = 15400, average loss = 0.08528959889241378
+09/24/2023 09:32:38 - INFO - __main__ -    global_step = 15450, average loss = 0.08095955706679887
+09/24/2023 09:36:19 - INFO - __main__ -    global_step = 15500, average loss = 0.08725373520917856
+09/24/2023 09:36:19 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 09:36:19 - INFO - __main__ -     Num examples = 10000
+09/24/2023 09:36:19 - INFO - __main__ -     Batch size = 16
+09/24/2023 09:40:15 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 09:40:15 - INFO - __main__ -     acc = 0.8545
+09/24/2023 09:40:42 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 09:44:22 - INFO - __main__ -    global_step = 15550, average loss = 0.0843266883040269
+09/24/2023 09:48:03 - INFO - __main__ -    global_step = 15600, average loss = 0.07855528741223679
+09/24/2023 09:51:47 - INFO - __main__ -    global_step = 15650, average loss = 0.09478737017554523
+09/24/2023 09:55:32 - INFO - __main__ -    global_step = 15700, average loss = 0.08910313490487169
+09/24/2023 09:59:16 - INFO - __main__ -    global_step = 15750, average loss = 0.07736712342710234
+09/24/2023 10:02:53 - INFO - __main__ -    global_step = 15800, average loss = 0.08501649839432503
+09/24/2023 10:06:37 - INFO - __main__ -    global_step = 15850, average loss = 0.08495221398276044
+09/24/2023 10:10:23 - INFO - __main__ -    global_step = 15900, average loss = 0.08510145512744202
+09/24/2023 10:14:07 - INFO - __main__ -    global_step = 15950, average loss = 0.08335533107921947
+09/24/2023 10:17:49 - INFO - __main__ -    global_step = 16000, average loss = 0.09103241352764599
+09/24/2023 10:17:49 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 10:17:49 - INFO - __main__ -     Num examples = 10000
+09/24/2023 10:17:49 - INFO - __main__ -     Batch size = 16
+09/24/2023 10:21:45 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 10:21:45 - INFO - __main__ -     acc = 0.8549
+09/24/2023 10:22:12 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 10:25:53 - INFO - __main__ -    global_step = 16050, average loss = 0.0808029190406296
+09/24/2023 10:29:33 - INFO - __main__ -    global_step = 16100, average loss = 0.0950222506766113
+09/24/2023 10:33:15 - INFO - __main__ -    global_step = 16150, average loss = 0.08560644885961664
+09/24/2023 10:36:53 - INFO - __main__ -    global_step = 16200, average loss = 0.07925290400889935
+09/24/2023 10:40:34 - INFO - __main__ -    global_step = 16250, average loss = 0.08252620983123052
+09/24/2023 10:44:15 - INFO - __main__ -    global_step = 16300, average loss = 0.08747977073326182
+09/24/2023 10:47:55 - INFO - __main__ -    global_step = 16350, average loss = 0.08805208059333382
+09/24/2023 10:51:41 - INFO - __main__ -    global_step = 16400, average loss = 0.07935831163018064
+09/24/2023 10:55:23 - INFO - __main__ -    global_step = 16450, average loss = 0.0807358610859228
+09/24/2023 10:59:03 - INFO - __main__ -    global_step = 16500, average loss = 0.0775301494665473
+09/24/2023 10:59:03 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 10:59:03 - INFO - __main__ -     Num examples = 10000
+09/24/2023 10:59:03 - INFO - __main__ -     Batch size = 16
+09/24/2023 11:02:59 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 11:02:59 - INFO - __main__ -     acc = 0.8532
+09/24/2023 11:06:39 - INFO - __main__ -    global_step = 16550, average loss = 0.06899339191091712
+09/24/2023 11:10:25 - INFO - __main__ -    global_step = 16600, average loss = 0.08612027997849508
+09/24/2023 11:14:10 - INFO - __main__ -    global_step = 16650, average loss = 0.08232147437905951
+09/24/2023 11:17:50 - INFO - __main__ -    global_step = 16700, average loss = 0.08530993062430753
+09/24/2023 11:18:50 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 11:18:50 - INFO - __main__ -     Num examples = 10000
+09/24/2023 11:18:50 - INFO - __main__ -     Batch size = 16
+09/24/2023 11:22:45 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 11:22:45 - INFO - __main__ -     acc = 0.8533
+09/24/2023 11:22:45 - INFO - __main__ -    global_step = 16713, average loss = 0.11041826268834619
+09/24/2023 11:23:18 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 11:23:18 - INFO - __main__ -     Num examples = 10000
+09/24/2023 11:23:18 - INFO - __main__ -     Batch size = 16
+09/24/2023 11:27:13 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 11:27:13 - INFO - __main__ -     acc = 0.8549
+09/24/2023 11:27:16 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/socialiqa_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6', out_dir='./eval_results/deberta-v3-large_2i_atm_half_sample_name_5e-6', device=0, reader='socialiqa', overwrite_output_dir=False, cache_dir=None)
+09/24/2023 11:27:16 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 11:34:38 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/winogrande_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6', out_dir='./eval_results/deberta-v3-large_2i_atm_half_sample_name_5e-6', device=0, reader='winogrande', overwrite_output_dir=False, cache_dir=None)
+09/24/2023 11:34:38 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 11:37:05 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/piqa_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6', out_dir='./eval_results/deberta-v3-large_2i_atm_half_sample_name_5e-6', device=0, reader='piqa', overwrite_output_dir=False, cache_dir=None)
+09/24/2023 11:37:05 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 11:43:59 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/commonsenseqa_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6', out_dir='./eval_results/deberta-v3-large_2i_atm_half_sample_name_5e-6', device=0, reader='commonsenseqa', overwrite_output_dir=False, cache_dir=None)
+09/24/2023 11:43:59 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 11:49:43 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/anli_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6', out_dir='./eval_results/deberta-v3-large_2i_atm_half_sample_name_5e-6', device=0, reader='anli', overwrite_output_dir=False, cache_dir=None)
+09/24/2023 11:49:43 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/deberta-v3-large_2i_atm_half_sample_name_5e-6
+09/24/2023 11:54:31 - INFO - __main__ -   ***** Running evaluation *****
+09/24/2023 11:54:31 - INFO - __main__ -     Num examples = 120
+09/24/2023 11:54:31 - INFO - __main__ -     Batch size = 16
+09/24/2023 11:54:47 - INFO - __main__ -   ***** Eval results *****
+09/24/2023 11:54:47 - INFO - __main__ -     acc = 0.525

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb357eae91ca6dee772e1aa051d51d1ac15dfb3d6939fc85c99c233728675db4
+size 1915