Batch upload part 18

nl_tasks/exp100/run_ex08/ft/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": false,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex08/ft/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

nl_tasks/exp100/run_ex08/ft2/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": true,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex08/ft2/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc7597ccf1305d10c57f61a9c789f5d7a5cc15cc8e54fdc0806057df1fe03a3b
+size 33602915

nl_tasks/exp100/run_ex09/ft/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": false,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex09/ft/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nl_tasks/exp100/run_ex09/ft/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

nl_tasks/exp100/run_ex09/ft/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 512,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

nl_tasks/exp100/run_ex09/ft2/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": true,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex09/ft2/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e041e22247e003d3fa1f62f968d3096e9383222bfc93bfd5deee072308dba1e8
+size 33602915

nl_tasks/exp100/run_ex09/trainer_state.json

@@ -0,0 +1,260 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 6252,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.09596928982725528,
+      "grad_norm": 0.06508654356002808,
+      "learning_rate": 0.019976180419211866,
+      "loss": 0.5532,
+      "step": 200
+    },
+    {
+      "epoch": 0.19193857965451055,
+      "grad_norm": 0.0456949919462204,
+      "learning_rate": 0.01985490438184627,
+      "loss": 0.283,
+      "step": 400
+    },
+    {
+      "epoch": 0.28790786948176583,
+      "grad_norm": 0.0334312878549099,
+      "learning_rate": 0.019632144212142762,
+      "loss": 0.2612,
+      "step": 600
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.04754582419991493,
+      "learning_rate": 0.01931019385651278,
+      "loss": 0.2547,
+      "step": 800
+    },
+    {
+      "epoch": 0.4798464491362764,
+      "grad_norm": 0.02298681065440178,
+      "learning_rate": 0.018892368705063736,
+      "loss": 0.242,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5758157389635317,
+      "grad_norm": 0.021144121885299683,
+      "learning_rate": 0.018382971450274496,
+      "loss": 0.2388,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6717850287907869,
+      "grad_norm": 0.023693973198533058,
+      "learning_rate": 0.01778724777859868,
+      "loss": 0.2355,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.019977454096078873,
+      "learning_rate": 0.017111332351276085,
+      "loss": 0.2275,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8637236084452975,
+      "grad_norm": 0.019784899428486824,
+      "learning_rate": 0.01636218563063265,
+      "loss": 0.2254,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9596928982725528,
+      "grad_norm": 0.020388498902320862,
+      "learning_rate": 0.015547522202421078,
+      "loss": 0.2216,
+      "step": 2000
+    },
+    {
+      "epoch": 1.055662188099808,
+      "grad_norm": 0.017220880836248398,
+      "learning_rate": 0.014675731332326341,
+      "loss": 0.2087,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1516314779270633,
+      "grad_norm": 0.018207907676696777,
+      "learning_rate": 0.013755790574731894,
+      "loss": 0.2038,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2476007677543186,
+      "grad_norm": 0.016460182145237923,
+      "learning_rate": 0.012797173323388642,
+      "loss": 0.202,
+      "step": 2600
+    },
+    {
+      "epoch": 1.3435700575815739,
+      "grad_norm": 0.013017100282013416,
+      "learning_rate": 0.011809751256014321,
+      "loss": 0.2012,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4395393474088292,
+      "grad_norm": 0.018365703523159027,
+      "learning_rate": 0.010803692677432199,
+      "loss": 0.2005,
+      "step": 3000
+    },
+    {
+      "epoch": 1.5355086372360844,
+      "grad_norm": 0.01587655022740364,
+      "learning_rate": 0.009789357808094205,
+      "loss": 0.1964,
+      "step": 3200
+    },
+    {
+      "epoch": 1.6314779270633397,
+      "grad_norm": 0.01589033380150795,
+      "learning_rate": 0.008777192096289453,
+      "loss": 0.198,
+      "step": 3400
+    },
+    {
+      "epoch": 1.727447216890595,
+      "grad_norm": 0.014157130382955074,
+      "learning_rate": 0.007777618652691038,
+      "loss": 0.1951,
+      "step": 3600
+    },
+    {
+      "epoch": 1.8234165067178503,
+      "grad_norm": 0.013822129927575588,
+      "learning_rate": 0.006800930914931747,
+      "loss": 0.1941,
+      "step": 3800
+    },
+    {
+      "epoch": 1.9193857965451055,
+      "grad_norm": 0.013463828712701797,
+      "learning_rate": 0.005857186647530748,
+      "loss": 0.1892,
+      "step": 4000
+    },
+    {
+      "epoch": 2.015355086372361,
+      "grad_norm": 0.015684494748711586,
+      "learning_rate": 0.004956104368742014,
+      "loss": 0.1853,
+      "step": 4200
+    },
+    {
+      "epoch": 2.111324376199616,
+      "grad_norm": 0.01606130413711071,
+      "learning_rate": 0.004106963270903152,
+      "loss": 0.1642,
+      "step": 4400
+    },
+    {
+      "epoch": 2.2072936660268714,
+      "grad_norm": 0.015006215311586857,
+      "learning_rate": 0.0033185076648879854,
+      "loss": 0.164,
+      "step": 4600
+    },
+    {
+      "epoch": 2.3032629558541267,
+      "grad_norm": 0.013956602662801743,
+      "learning_rate": 0.0025988569326776123,
+      "loss": 0.165,
+      "step": 4800
+    },
+    {
+      "epoch": 2.399232245681382,
+      "grad_norm": 0.01500143762677908,
+      "learning_rate": 0.0019554219153431287,
+      "loss": 0.1593,
+      "step": 5000
+    },
+    {
+      "epoch": 2.495201535508637,
+      "grad_norm": 0.016031745821237564,
+      "learning_rate": 0.0013948285974623588,
+      "loss": 0.1621,
+      "step": 5200
+    },
+    {
+      "epoch": 2.5911708253358925,
+      "grad_norm": 0.01416528970003128,
+      "learning_rate": 0.0009228498738555002,
+      "loss": 0.161,
+      "step": 5400
+    },
+    {
+      "epoch": 2.6871401151631478,
+      "grad_norm": 0.01726922020316124,
+      "learning_rate": 0.0005443461012942996,
+      "loss": 0.1576,
+      "step": 5600
+    },
+    {
+      "epoch": 2.783109404990403,
+      "grad_norm": 0.015128469094634056,
+      "learning_rate": 0.00026321504737310985,
+      "loss": 0.1558,
+      "step": 5800
+    },
+    {
+      "epoch": 2.8790786948176583,
+      "grad_norm": 0.016625599935650826,
+      "learning_rate": 8.23517519598982e-05,
+      "loss": 0.1565,
+      "step": 6000
+    },
+    {
+      "epoch": 2.9750479846449136,
+      "grad_norm": 0.015019167214632034,
+      "learning_rate": 3.618714567147352e-06,
+      "loss": 0.1576,
+      "step": 6200
+    },
+    {
+      "epoch": 3.0,
+      "step": 6252,
+      "total_flos": 6.0970588176384e+18,
+      "train_loss": 0.2099078561889004,
+      "train_runtime": 8160.2371,
+      "train_samples_per_second": 36.764,
+      "train_steps_per_second": 0.766
+    }
+  ],
+  "logging_steps": 200,
+  "max_steps": 6252,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.0970588176384e+18,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

nl_tasks/exp100/run_ex10/ft/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": false,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex10/ft/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nl_tasks/exp100/run_ex10/ft/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

nl_tasks/exp100/run_ex10/ft/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 512,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

nl_tasks/exp100/run_ex10/ft2/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": true,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex10/ft2/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff09e3f976b0f890a445477281ac6c563f8b2b11869aff99580213720ae3ec8f
+size 33602915

nl_tasks/exp100/run_ex10/trainer_state.json

@@ -0,0 +1,183 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 4168,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.09596928982725528,
+      "grad_norm": 0.17983072996139526,
+      "learning_rate": 0.029892950505116346,
+      "loss": 5.4284,
+      "step": 200
+    },
+    {
+      "epoch": 0.19193857965451055,
+      "grad_norm": 0.02686592936515808,
+      "learning_rate": 0.02944923894162051,
+      "loss": 0.3284,
+      "step": 400
+    },
+    {
+      "epoch": 0.28790786948176583,
+      "grad_norm": 0.02659301459789276,
+      "learning_rate": 0.028671096149031867,
+      "loss": 0.2782,
+      "step": 600
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.021944062784314156,
+      "learning_rate": 0.027576532435823177,
+      "loss": 0.2639,
+      "step": 800
+    },
+    {
+      "epoch": 0.4798464491362764,
+      "grad_norm": 0.019736966118216515,
+      "learning_rate": 0.02619088175137459,
+      "loss": 0.2518,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5758157389635317,
+      "grad_norm": 0.020629985257983208,
+      "learning_rate": 0.024546215325414244,
+      "loss": 0.2478,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6717850287907869,
+      "grad_norm": 0.022958872839808464,
+      "learning_rate": 0.022680599371429494,
+      "loss": 0.2429,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.020077615976333618,
+      "learning_rate": 0.020637214034687996,
+      "loss": 0.2336,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8637236084452975,
+      "grad_norm": 0.018682507798075676,
+      "learning_rate": 0.018463353977035808,
+      "loss": 0.2302,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9596928982725528,
+      "grad_norm": 0.02266324870288372,
+      "learning_rate": 0.016209333730185887,
+      "loss": 0.225,
+      "step": 2000
+    },
+    {
+      "epoch": 1.055662188099808,
+      "grad_norm": 0.01823570765554905,
+      "learning_rate": 0.013927323153367477,
+      "loss": 0.2111,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1516314779270633,
+      "grad_norm": 0.02119363099336624,
+      "learning_rate": 0.011670139948958654,
+      "loss": 0.2052,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2476007677543186,
+      "grad_norm": 0.018385590985417366,
+      "learning_rate": 0.009490027183628048,
+      "loss": 0.2016,
+      "step": 2600
+    },
+    {
+      "epoch": 1.3435700575815739,
+      "grad_norm": 0.01590455323457718,
+      "learning_rate": 0.0074374441095630085,
+      "loss": 0.1987,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4395393474088292,
+      "grad_norm": 0.017842255532741547,
+      "learning_rate": 0.0055598982725275775,
+      "loss": 0.1957,
+      "step": 3000
+    },
+    {
+      "epoch": 1.5355086372360844,
+      "grad_norm": 0.018860826268792152,
+      "learning_rate": 0.0039008459378997943,
+      "loss": 0.1901,
+      "step": 3200
+    },
+    {
+      "epoch": 1.6314779270633397,
+      "grad_norm": 0.018248997628688812,
+      "learning_rate": 0.002498686284601174,
+      "loss": 0.1899,
+      "step": 3400
+    },
+    {
+      "epoch": 1.727447216890595,
+      "grad_norm": 0.01603817380964756,
+      "learning_rate": 0.0013858726465499599,
+      "loss": 0.1865,
+      "step": 3600
+    },
+    {
+      "epoch": 1.8234165067178503,
+      "grad_norm": 0.016947340220212936,
+      "learning_rate": 0.0005881613721758754,
+      "loss": 0.1848,
+      "step": 3800
+    },
+    {
+      "epoch": 1.9193857965451055,
+      "grad_norm": 0.018619216978549957,
+      "learning_rate": 0.00012401568732964163,
+      "loss": 0.1808,
+      "step": 4000
+    },
+    {
+      "epoch": 2.0,
+      "step": 4168,
+      "total_flos": 4.0647058784256e+18,
+      "train_loss": 0.47151951963750505,
+      "train_runtime": 5453.3681,
+      "train_samples_per_second": 36.675,
+      "train_steps_per_second": 0.764
+    }
+  ],
+  "logging_steps": 200,
+  "max_steps": 4168,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.0647058784256e+18,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

nl_tasks/exp100/run_ex11/ft/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": false,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex11/ft/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nl_tasks/exp100/run_ex11/ft/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

nl_tasks/exp100/run_ex11/ft/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 512,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

nl_tasks/exp100/run_ex11/ft2/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": true,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex11/ft2/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:669642b5dfd24d7a899a1a21a69a5a9cf6d0170c2609f2a54c664123864585da
+size 33602915

nl_tasks/exp100/run_ex11/trainer_state.json

@@ -0,0 +1,183 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 4168,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.09596928982725528,
+      "grad_norm": 0.09253966063261032,
+      "learning_rate": 0.007971453468031025,
+      "loss": 3.4503,
+      "step": 200
+    },
+    {
+      "epoch": 0.19193857965451055,
+      "grad_norm": 0.06041109189391136,
+      "learning_rate": 0.007853130384432137,
+      "loss": 0.2986,
+      "step": 400
+    },
+    {
+      "epoch": 0.28790786948176583,
+      "grad_norm": 0.05962882563471794,
+      "learning_rate": 0.007645625639741832,
+      "loss": 0.2678,
+      "step": 600
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.05638430267572403,
+      "learning_rate": 0.007353741982886181,
+      "loss": 0.2562,
+      "step": 800
+    },
+    {
+      "epoch": 0.4798464491362764,
+      "grad_norm": 0.043786656111478806,
+      "learning_rate": 0.0069842351336998915,
+      "loss": 0.2442,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5758157389635317,
+      "grad_norm": 0.04289592057466507,
+      "learning_rate": 0.006545657420110465,
+      "loss": 0.2413,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6717850287907869,
+      "grad_norm": 0.044572457671165466,
+      "learning_rate": 0.006048159832381199,
+      "loss": 0.237,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.03955984488129616,
+      "learning_rate": 0.0055032570759168,
+      "loss": 0.2277,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8637236084452975,
+      "grad_norm": 0.03934319689869881,
+      "learning_rate": 0.004923561060542882,
+      "loss": 0.2243,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9596928982725528,
+      "grad_norm": 0.034399278461933136,
+      "learning_rate": 0.004322488994716237,
+      "loss": 0.2193,
+      "step": 2000
+    },
+    {
+      "epoch": 1.055662188099808,
+      "grad_norm": 0.037286121398210526,
+      "learning_rate": 0.003713952840897994,
+      "loss": 0.2051,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1516314779270633,
+      "grad_norm": 0.03788420185446739,
+      "learning_rate": 0.0031120373197223083,
+      "loss": 0.1981,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2476007677543186,
+      "grad_norm": 0.036435652524232864,
+      "learning_rate": 0.0025306739156341464,
+      "loss": 0.1947,
+      "step": 2600
+    },
+    {
+      "epoch": 1.3435700575815739,
+      "grad_norm": 0.03084568865597248,
+      "learning_rate": 0.0019833184292168023,
+      "loss": 0.1924,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4395393474088292,
+      "grad_norm": 0.03642988204956055,
+      "learning_rate": 0.0014826395393406876,
+      "loss": 0.1901,
+      "step": 3000
+    },
+    {
+      "epoch": 1.5355086372360844,
+      "grad_norm": 0.03472171723842621,
+      "learning_rate": 0.0010402255834399453,
+      "loss": 0.1851,
+      "step": 3200
+    },
+    {
+      "epoch": 1.6314779270633397,
+      "grad_norm": 0.03253506124019623,
+      "learning_rate": 0.0006663163425603131,
+      "loss": 0.1857,
+      "step": 3400
+    },
+    {
+      "epoch": 1.727447216890595,
+      "grad_norm": 0.036016546189785004,
+      "learning_rate": 0.0003695660390799893,
+      "loss": 0.1827,
+      "step": 3600
+    },
+    {
+      "epoch": 1.8234165067178503,
+      "grad_norm": 0.032651085406541824,
+      "learning_rate": 0.00015684303258023348,
+      "loss": 0.1821,
+      "step": 3800
+    },
+    {
+      "epoch": 1.9193857965451055,
+      "grad_norm": 0.0359911285340786,
+      "learning_rate": 3.30708499545711e-05,
+      "loss": 0.1782,
+      "step": 4000
+    },
+    {
+      "epoch": 2.0,
+      "step": 4168,
+      "total_flos": 4.0647058784256e+18,
+      "train_loss": 0.3700037432723677,
+      "train_runtime": 5461.8246,
+      "train_samples_per_second": 36.618,
+      "train_steps_per_second": 0.763
+    }
+  ],
+  "logging_steps": 200,
+  "max_steps": 4168,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.0647058784256e+18,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

nl_tasks/exp100/run_ex12/ft/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": false,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex12/ft/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nl_tasks/exp100/run_ex12/ft/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

nl_tasks/exp100/run_ex12/ft/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 512,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

nl_tasks/exp100/run_ex12/ft2/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "T": 1.0,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "inference_mode": true,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "CAUSAL_LM"
+}

nl_tasks/exp100/run_ex12/ft2/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86f326abe5a1f356d70a24ce7bab2ee7dd2bcb059d6f2282d04dc2f86fee6dc1
+size 33602915

nl_tasks/exp100/run_ex12/trainer_state.json

@@ -0,0 +1,260 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 6252,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.09596928982725528,
+      "grad_norm": 0.13730373978614807,
+      "learning_rate": 0.007990472167684746,
+      "loss": 0.4657,
+      "step": 200
+    },
+    {
+      "epoch": 0.19193857965451055,
+      "grad_norm": 0.0858209878206253,
+      "learning_rate": 0.007941961752738508,
+      "loss": 0.2803,
+      "step": 400
+    },
+    {
+      "epoch": 0.28790786948176583,
+      "grad_norm": 0.06287017464637756,
+      "learning_rate": 0.007852857684857105,
+      "loss": 0.2572,
+      "step": 600
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.053079769015312195,
+      "learning_rate": 0.007724077542605112,
+      "loss": 0.2477,
+      "step": 800
+    },
+    {
+      "epoch": 0.4798464491362764,
+      "grad_norm": 0.03740881383419037,
+      "learning_rate": 0.007556947482025495,
+      "loss": 0.2368,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5758157389635317,
+      "grad_norm": 0.034000739455223083,
+      "learning_rate": 0.007353188580109798,
+      "loss": 0.2351,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6717850287907869,
+      "grad_norm": 0.04023474082350731,
+      "learning_rate": 0.007114899111439472,
+      "loss": 0.231,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.030921513214707375,
+      "learning_rate": 0.006844532940510433,
+      "loss": 0.223,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8637236084452975,
+      "grad_norm": 0.03130370005965233,
+      "learning_rate": 0.006544874252253061,
+      "loss": 0.221,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9596928982725528,
+      "grad_norm": 0.029764752835035324,
+      "learning_rate": 0.006219008880968432,
+      "loss": 0.2163,
+      "step": 2000
+    },
+    {
+      "epoch": 1.055662188099808,
+      "grad_norm": 0.027886036783456802,
+      "learning_rate": 0.0058702925329305366,
+      "loss": 0.2034,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1516314779270633,
+      "grad_norm": 0.026955854147672653,
+      "learning_rate": 0.005502316229892758,
+      "loss": 0.1974,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2476007677543186,
+      "grad_norm": 0.024037910625338554,
+      "learning_rate": 0.005118869329355457,
+      "loss": 0.1958,
+      "step": 2600
+    },
+    {
+      "epoch": 1.3435700575815739,
+      "grad_norm": 0.02323935180902481,
+      "learning_rate": 0.004723900502405729,
+      "loss": 0.195,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4395393474088292,
+      "grad_norm": 0.023859383538365364,
+      "learning_rate": 0.00432147707097288,
+      "loss": 0.1937,
+      "step": 3000
+    },
+    {
+      "epoch": 1.5355086372360844,
+      "grad_norm": 0.025017334148287773,
+      "learning_rate": 0.0039157431232376815,
+      "loss": 0.1901,
+      "step": 3200
+    },
+    {
+      "epoch": 1.6314779270633397,
+      "grad_norm": 0.024762826040387154,
+      "learning_rate": 0.0035108768385157816,
+      "loss": 0.1915,
+      "step": 3400
+    },
+    {
+      "epoch": 1.727447216890595,
+      "grad_norm": 0.023029997944831848,
+      "learning_rate": 0.0031110474610764154,
+      "loss": 0.189,
+      "step": 3600
+    },
+    {
+      "epoch": 1.8234165067178503,
+      "grad_norm": 0.023539869114756584,
+      "learning_rate": 0.0027203723659726987,
+      "loss": 0.1881,
+      "step": 3800
+    },
+    {
+      "epoch": 1.9193857965451055,
+      "grad_norm": 0.023440731689333916,
+      "learning_rate": 0.002342874659012299,
+      "loss": 0.1832,
+      "step": 4000
+    },
+    {
+      "epoch": 2.015355086372361,
+      "grad_norm": 0.024401186034083366,
+      "learning_rate": 0.0019824417474968055,
+      "loss": 0.1798,
+      "step": 4200
+    },
+    {
+      "epoch": 2.111324376199616,
+      "grad_norm": 0.02438773214817047,
+      "learning_rate": 0.001642785308361261,
+      "loss": 0.1588,
+      "step": 4400
+    },
+    {
+      "epoch": 2.2072936660268714,
+      "grad_norm": 0.023876527324318886,
+      "learning_rate": 0.0013274030659551942,
+      "loss": 0.1585,
+      "step": 4600
+    },
+    {
+      "epoch": 2.3032629558541267,
+      "grad_norm": 0.021032139658927917,
+      "learning_rate": 0.001039542773071045,
+      "loss": 0.1594,
+      "step": 4800
+    },
+    {
+      "epoch": 2.399232245681382,
+      "grad_norm": 0.023737894371151924,
+      "learning_rate": 0.0007821687661372514,
+      "loss": 0.1545,
+      "step": 5000
+    },
+    {
+      "epoch": 2.495201535508637,
+      "grad_norm": 0.02574790269136429,
+      "learning_rate": 0.0005579314389849435,
+      "loss": 0.1574,
+      "step": 5200
+    },
+    {
+      "epoch": 2.5911708253358925,
+      "grad_norm": 0.022647960111498833,
+      "learning_rate": 0.00036913994954220007,
+      "loss": 0.157,
+      "step": 5400
+    },
+    {
+      "epoch": 2.6871401151631478,
+      "grad_norm": 0.024767184630036354,
+      "learning_rate": 0.00021773844051771986,
+      "loss": 0.1537,
+      "step": 5600
+    },
+    {
+      "epoch": 2.783109404990403,
+      "grad_norm": 0.02572454698383808,
+      "learning_rate": 0.00010528601894924394,
+      "loss": 0.1524,
+      "step": 5800
+    },
+    {
+      "epoch": 2.8790786948176583,
+      "grad_norm": 0.024511748924851418,
+      "learning_rate": 3.294070078395928e-05,
+      "loss": 0.1536,
+      "step": 6000
+    },
+    {
+      "epoch": 2.9750479846449136,
+      "grad_norm": 0.02418585494160652,
+      "learning_rate": 1.447485826858941e-06,
+      "loss": 0.1544,
+      "step": 6200
+    },
+    {
+      "epoch": 3.0,
+      "step": 6252,
+      "total_flos": 6.0970588176384e+18,
+      "train_loss": 0.2022177925951879,
+      "train_runtime": 8189.2736,
+      "train_samples_per_second": 36.633,
+      "train_steps_per_second": 0.763
+    }
+  ],
+  "logging_steps": 200,
+  "max_steps": 6252,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.0970588176384e+18,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

nl_tasks/expsBOFT/seed43/trainer_state.json

@@ -0,0 +1,218 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.08375173062086105,
+      "learning_rate": 0.000392,
+      "loss": 0.5193,
+      "step": 50
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.09268203377723694,
+      "learning_rate": 0.0007920000000000001,
+      "loss": 0.3316,
+      "step": 100
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.08198747783899307,
+      "learning_rate": 0.0007964216926581925,
+      "loss": 0.304,
+      "step": 150
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.0816216915845871,
+      "learning_rate": 0.0007854602918076551,
+      "loss": 0.2918,
+      "step": 200
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.07457849383354187,
+      "learning_rate": 0.0007673184950396212,
+      "loss": 0.274,
+      "step": 250
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.07685171067714691,
+      "learning_rate": 0.0007423342497022817,
+      "loss": 0.2687,
+      "step": 300
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.07849128544330597,
+      "learning_rate": 0.0007109729650142636,
+      "loss": 0.2651,
+      "step": 350
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.07266736030578613,
+      "learning_rate": 0.0006738188423714755,
+      "loss": 0.2575,
+      "step": 400
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.06927025318145752,
+      "learning_rate": 0.0006315639927804526,
+      "loss": 0.2525,
+      "step": 450
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.08536054193973541,
+      "learning_rate": 0.00058499554413983,
+      "loss": 0.2494,
+      "step": 500
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.07602768391370773,
+      "learning_rate": 0.000534980978536894,
+      "loss": 0.2429,
+      "step": 550
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.07055249065160751,
+      "learning_rate": 0.00048245197269763485,
+      "loss": 0.2457,
+      "step": 600
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.07144515216350555,
+      "learning_rate": 0.00042838704261214224,
+      "loss": 0.2292,
+      "step": 650
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.07937044650316238,
+      "learning_rate": 0.00037379331563313267,
+      "loss": 0.2169,
+      "step": 700
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.07409252226352692,
+      "learning_rate": 0.00031968776959892677,
+      "loss": 0.2098,
+      "step": 750
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.07844420522451401,
+      "learning_rate": 0.00026707828846051743,
+      "loss": 0.2145,
+      "step": 800
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.07791652530431747,
+      "learning_rate": 0.00021694488731055218,
+      "loss": 0.2082,
+      "step": 850
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.0782908946275711,
+      "learning_rate": 0.00017022145655641685,
+      "loss": 0.2077,
+      "step": 900
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.0826650932431221,
+      "learning_rate": 0.00012777836530893536,
+      "loss": 0.2137,
+      "step": 950
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.0696156919002533,
+      "learning_rate": 9.040624805263558e-05,
+      "loss": 0.2076,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.06966507434844971,
+      "learning_rate": 5.880127662124091e-05,
+      "loss": 0.2108,
+      "step": 1050
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.08326321095228195,
+      "learning_rate": 3.355219183361582e-05,
+      "loss": 0.2106,
+      "step": 1100
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.0792745053768158,
+      "learning_rate": 1.512933636625089e-05,
+      "loss": 0.2073,
+      "step": 1150
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.07648582756519318,
+      "learning_rate": 3.8758931591217575e-06,
+      "loss": 0.209,
+      "step": 1200
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.0787830799818039,
+      "learning_rate": 1.4925668450960217e-09,
+      "loss": 0.2124,
+      "step": 1250
+    },
+    {
+      "epoch": 2.0,
+      "step": 1250,
+      "total_flos": 1.62594677587968e+18,
+      "train_loss": 0.25041088790893556,
+      "train_runtime": 3374.0916,
+      "train_samples_per_second": 23.71,
+      "train_steps_per_second": 0.37
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1250,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.62594677587968e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

nl_tasks/expsBOFT/seed44/ft/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nl_tasks/expsBOFT/seed44/ft/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

nl_tasks/expsBOFT/seed44/ft/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 512,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

nl_tasks/expsBOFT/seed44/ft2/README.md

@@ -0,0 +1,205 @@
+---
+base_model: meta-llama/Llama-2-7b-hf
+library_name: peft
+tags:
+- base_model:adapter:meta-llama/Llama-2-7b-hf
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

nl_tasks/expsBOFT/seed44/ft2/adapter_config.json

@@ -0,0 +1,27 @@
+{
+  "auto_mapping": {
+    "base_model_class": "LlamaForCausalLM",
+    "parent_library": "transformers.models.llama.modeling_llama"
+  },
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "boft_block_num": 0,
+  "boft_block_size": 16,
+  "boft_dropout": 0.05,
+  "boft_n_butterfly_factor": 2,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "modules_to_save": null,
+  "peft_type": "BOFT",
+  "peft_version": "0.18.0",
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": null
+}

nl_tasks/expsBOFT/seed44/ft2/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:584526a06a1f45f2f77e6a89a7201b05aa25a3d6be60f231b255a32c48c4b261
+size 34619504

nl_tasks/expsBOFT/seed44/trainer_state.json

@@ -0,0 +1,218 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.08375173062086105,
+      "learning_rate": 0.000392,
+      "loss": 0.5193,
+      "step": 50
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.09268203377723694,
+      "learning_rate": 0.0007920000000000001,
+      "loss": 0.3316,
+      "step": 100
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.08198747783899307,
+      "learning_rate": 0.0007964216926581925,
+      "loss": 0.304,
+      "step": 150
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.0816216915845871,
+      "learning_rate": 0.0007854602918076551,
+      "loss": 0.2918,
+      "step": 200
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.07457849383354187,
+      "learning_rate": 0.0007673184950396212,
+      "loss": 0.274,
+      "step": 250
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.07685171067714691,
+      "learning_rate": 0.0007423342497022817,
+      "loss": 0.2687,
+      "step": 300
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.07849128544330597,
+      "learning_rate": 0.0007109729650142636,
+      "loss": 0.2651,
+      "step": 350
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.07266736030578613,
+      "learning_rate": 0.0006738188423714755,
+      "loss": 0.2575,
+      "step": 400
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.06927025318145752,
+      "learning_rate": 0.0006315639927804526,
+      "loss": 0.2525,
+      "step": 450
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.08536054193973541,
+      "learning_rate": 0.00058499554413983,
+      "loss": 0.2494,
+      "step": 500
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.07602768391370773,
+      "learning_rate": 0.000534980978536894,
+      "loss": 0.2429,
+      "step": 550
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.07055249065160751,
+      "learning_rate": 0.00048245197269763485,
+      "loss": 0.2457,
+      "step": 600
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.07144515216350555,
+      "learning_rate": 0.00042838704261214224,
+      "loss": 0.2292,
+      "step": 650
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.07937044650316238,
+      "learning_rate": 0.00037379331563313267,
+      "loss": 0.2169,
+      "step": 700
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.07409252226352692,
+      "learning_rate": 0.00031968776959892677,
+      "loss": 0.2098,
+      "step": 750
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.07844420522451401,
+      "learning_rate": 0.00026707828846051743,
+      "loss": 0.2145,
+      "step": 800
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.07791652530431747,
+      "learning_rate": 0.00021694488731055218,
+      "loss": 0.2082,
+      "step": 850
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.0782908946275711,
+      "learning_rate": 0.00017022145655641685,
+      "loss": 0.2077,
+      "step": 900
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.0826650932431221,
+      "learning_rate": 0.00012777836530893536,
+      "loss": 0.2137,
+      "step": 950
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.0696156919002533,
+      "learning_rate": 9.040624805263558e-05,
+      "loss": 0.2076,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.06966507434844971,
+      "learning_rate": 5.880127662124091e-05,
+      "loss": 0.2108,
+      "step": 1050
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.08326321095228195,
+      "learning_rate": 3.355219183361582e-05,
+      "loss": 0.2106,
+      "step": 1100
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.0792745053768158,
+      "learning_rate": 1.512933636625089e-05,
+      "loss": 0.2073,
+      "step": 1150
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.07648582756519318,
+      "learning_rate": 3.8758931591217575e-06,
+      "loss": 0.209,
+      "step": 1200
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.0787830799818039,
+      "learning_rate": 1.4925668450960217e-09,
+      "loss": 0.2124,
+      "step": 1250
+    },
+    {
+      "epoch": 2.0,
+      "step": 1250,
+      "total_flos": 1.62594677587968e+18,
+      "train_loss": 0.25041088790893556,
+      "train_runtime": 3377.6799,
+      "train_samples_per_second": 23.685,
+      "train_steps_per_second": 0.37
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1250,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.62594677587968e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

nl_tasks/inference/MATH_infer.py

@@ -0,0 +1,132 @@
+import argparse
+import json
+import pdb
+import jsonlines
+
+import util
+from vllm import LLM, SamplingParams
+import sys
+MAX_INT = sys.maxsize
+INVALID_ANS = "[invalid]"
+MAX_TOKEN = 1408
+
+import random
+import numpy as np
+import torch
+import os
+
+invalid_outputs = []
+def remove_boxed(s):
+    left = "\\boxed{"
+    try:
+        assert s[:len(left)] == left
+        assert s[-1] == "}"
+        return s[len(left):-1]
+    except:
+        return None
+
+def process_results(doc, completion, answer):
+    split_ans = completion.split('The answer is: ')
+    if len(split_ans) > 1:
+        ans = split_ans[-1]
+        extract_ans_temp = ans.split('.\n')[0]
+        extract_ans_temp = extract_ans_temp.strip()
+        if len(extract_ans_temp)>0 and extract_ans_temp[-1] == '.':
+            extract_ans = extract_ans_temp[0:-1]
+        else:
+            extract_ans = extract_ans_temp
+        extract_ans = extract_ans.strip()
+        if util.is_equiv(extract_ans, answer):
+            return True
+        else:
+            return False
+    else:
+        temp = {'question': doc, 'output': completion, 'answer': answer}
+        invalid_outputs.append(temp)
+        return False
+def batch_data(data_list, batch_size=1):
+    n = len(data_list) // batch_size
+    batch_data = []
+    for i in range(n-1):
+        start = i * batch_size
+        end = (i+1)*batch_size
+        batch_data.append(data_list[start:end])
+
+    last_start = (n-1) * batch_size
+    last_end = MAX_INT
+    batch_data.append(data_list[last_start:last_end])
+    return batch_data
+
+def test_hendrycks_math(model, data_path, start=0, end=MAX_INT, batch_size=1, tensor_parallel_size=1):
+    hendrycks_math_ins = []
+    hendrycks_math_answers = []
+    problem_prompt = (
+        "Below is an instruction that describes a task. "
+        "Write a response that appropriately completes the request.\n\n"
+        "### Instruction:\n{instruction}\n\n### Response: Let's think step by step."
+    )
+    print('promt =====', problem_prompt)
+    with open(data_path, "r+", encoding="utf8") as f:
+        for idx, item in enumerate(jsonlines.Reader(f)):
+            temp_instr = problem_prompt.format(instruction=item["instruction"])
+            hendrycks_math_ins.append(temp_instr)
+            solution = item['output']
+            temp_ans = remove_boxed(util.last_boxed_only_string(solution))
+            hendrycks_math_answers.append(temp_ans)
+
+
+    print('total length ===', len(hendrycks_math_ins))
+    hendrycks_math_ins = hendrycks_math_ins[start:end]
+    hendrycks_math_answers = hendrycks_math_answers[start:end]
+    print('lenght ====', len(hendrycks_math_ins))
+    # batch_hendrycks_math_ins = batch_data(hendrycks_math_ins, batch_size=batch_size)
+
+    stop_tokens = ["Instruction:", "Instruction", "Response:", "Response"]
+    sampling_params = SamplingParams(temperature=0, top_p=1, max_tokens=MAX_TOKEN, stop=stop_tokens)
+    print('sampleing =====', sampling_params)
+    llm = LLM(model=model,tensor_parallel_size=tensor_parallel_size, gpu_memory_utilization=0.95)
+
+    outputs = llm.generate(hendrycks_math_ins, sampling_params)
+    res_completions = [output.outputs[0].text for output in outputs]
+
+    results = []
+    for idx, (prompt, completion, prompt_answer) in enumerate(zip(hendrycks_math_ins, res_completions, hendrycks_math_answers)):
+        res = process_results(prompt, completion, prompt_answer)
+        results.append(res)
+
+    acc = sum(results) / len(results)
+    print('len invalid outputs ====', len(invalid_outputs), ', invalid_outputs===', len(invalid_outputs))
+    # print('start===', start, ', end====',end)
+    print('length====', len(results), ', acc====', acc*100)
+
+    current_path = args.model
+    parent_dir = os.path.dirname(current_path.rstrip('/'))
+    output_filename = os.path.join(parent_dir, 'output.txt')
+    # output_filename = args.model + 'output.txt'
+    with open(output_filename, "a", encoding="utf-8") as f:
+        print(f'\n MATH math MAX TOKEN = {MAX_TOKEN}, length==== {len(results)}, math acc %====, {acc*100}', file=f)
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=str, default=0)  # model path
+    parser.add_argument("--data_file", type=str, default='data/MATH_test.jsonl')  # data path
+    parser.add_argument("--start", type=int, default=0) #start index
+    parser.add_argument("--end", type=int, default=MAX_INT)  # end index
+    parser.add_argument("--batch_size", type=int, default=50)  # batch_size
+    parser.add_argument("--tensor_parallel_size", type=int, default=1)  # tensor_parallel_size
+    return parser.parse_args()
+
+def set_deterministic_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    # torch.backends.cudnn.deterministic = True
+    # torch.backends.cudnn.benchmark = False
+    
+if __name__ == "__main__":
+    args = parse_args()
+    set_deterministic_seed()
+    test_hendrycks_math(model=args.model, data_path=args.data_file, start=args.start, end=args.end, batch_size=args.batch_size, tensor_parallel_size=args.tensor_parallel_size)
+    print('math ends', args.model)
+

nl_tasks/inference/grader.py

@@ -0,0 +1,141 @@
+"""
+This logic is largely copied from the Hendrycks' MATH release (math_equivalence), and borrowed from:
+- https://github.com/microsoft/ProphetNet/tree/master/CRITIC
+"""
+import multiprocessing
+from math import isclose
+from typing import Union
+
+from sympy import simplify, N
+from sympy.parsing.sympy_parser import parse_expr
+from sympy.parsing.latex import parse_latex
+
+
+def is_digit(s):
+    try:
+        float(str(s).replace(",", ""))
+        return True
+    except ValueError:
+        return False
+
+def math_equal(prediction: Union[bool, float, str],
+                reference: Union[float, str],
+                include_percentage: bool = True,
+                is_close: bool = True,
+                timeout: bool = False,
+                ) -> bool:
+    """
+    Exact match of math if and only if:
+    1. numerical equal: both can convert to float and are equal
+    2. symbolic equal: both can convert to sympy expression and are equal
+    """
+    try: # 1. numerical equal
+        if is_digit(prediction) and is_digit(reference):
+            prediction = float(str(prediction).replace(",", ""))
+            reference = float(str(reference).replace(",", ""))
+            # number questions
+            if include_percentage:
+                gt_result = [reference / 100, reference, reference * 100]
+            else:
+                gt_result = [reference]
+            for item in gt_result:
+                try:
+                    if is_close:
+                        if isclose(item, prediction, rel_tol=1e-4):
+                            return True
+                    else:
+                        if item == prediction:
+                            return True
+                except Exception:
+                    continue
+            return False
+    except:
+        pass
+
+    if not prediction and prediction not in [0, False]:
+        return False
+
+    # 2. symbolic equal
+    reference = str(reference).strip()
+    prediction = str(prediction).strip()
+
+    ## deal with [], (), {}
+    pred_str, ref_str = prediction, reference
+    if (prediction.startswith("[") and prediction.endswith("]") and not reference.startswith("(")) or \
+        (prediction.startswith("(") and prediction.endswith(")") and not reference.startswith("[")):
+        pred_str = pred_str.strip("[]()")
+        ref_str = ref_str.strip("[]()")
+    for s in ['{', "}", "(", ")"]:
+        ref_str = ref_str.replace(s, "")
+        pred_str = pred_str.replace(s, "")
+    if pred_str == ref_str:
+        return True
+
+    ## [a, b] vs. [c, d], return a==c and b==d
+    if (prediction.startswith("[") and prediction.endswith("]")) and (reference.startswith("[") and reference.endswith("]")) or \
+        (prediction.startswith("(") and prediction.endswith(")")) and (reference.startswith("(") and reference.endswith(")")):
+        pred_parts = prediction[1:-1].split(",")
+        ref_parts = reference[1:-1].split(",")
+        if len(pred_parts) == len(ref_parts):
+            if all([math_equal(pred_parts[i], ref_parts[i], include_percentage, is_close) for i in range(len(pred_parts))]):
+                return True
+
+    # symbolic equal with sympy
+    if timeout:
+        if call_with_timeout(symbolic_equal_process, prediction, reference):
+            return True
+    else:
+        if symbolic_equal(prediction, reference):
+            return True
+
+    return False
+
+
+def math_equal_process(param):
+    return math_equal(param[-2], param[-1])
+
+
+def symbolic_equal(a, b):
+    def _parse(s):
+        for f in [parse_latex, parse_expr]:
+            try:
+                return f(s)
+            except:
+                pass
+        return s
+    a = _parse(a)
+    b = _parse(b)
+
+    try:
+        if simplify(a-b) == 0:
+            return True
+    except:
+        pass
+
+    try:
+        if isclose(N(a), N(b), rel_tol=1e-3):
+            return True
+    except:
+        pass
+    return False
+
+
+def symbolic_equal_process(a, b, output_queue):  
+    result = symbolic_equal(a, b)
+    output_queue.put(result)  
+
+
+def call_with_timeout(func, *args, timeout=1, **kwargs):  
+    output_queue = multiprocessing.Queue()  
+    process_args = args + (output_queue,)  
+    process = multiprocessing.Process(target=func, args=process_args, kwargs=kwargs)  
+    process.start()  
+    process.join(timeout)  
+  
+    if process.is_alive():  
+        process.terminate()  
+        process.join()  
+        return False  
+  
+    return output_queue.get()
+

nl_tasks/inference/gsm8k_infer.py

@@ -0,0 +1,157 @@
+import argparse
+import json
+import re
+import jsonlines
+from fraction import Fraction
+from vllm import LLM, SamplingParams
+import sys
+from grader import math_equal
+MAX_INT = sys.maxsize
+MAX_TOKEN = 1024
+
+import random
+import numpy as np
+import torch
+import os
+
+def is_number(s):
+    try:
+        float(s)
+        return True
+    except ValueError:
+        pass
+    try:
+        import unicodedata
+        unicodedata.numeric(s)
+        return True
+    except (TypeError, ValueError):
+        pass
+    return False
+
+def extract_answer_number(completion):
+    text = completion.split('The answer is: ')
+    if len(text) > 1:
+        extract_ans = text[-1].strip()
+        match = re.search(r'[\-+]?\d*[\.,/]?\d+', extract_ans)
+        if match:
+            if '/' in match.group():
+                denominator = match.group().split('/')[1]
+                numerator = match.group().split('/')[0]
+                if is_number(denominator) == True and is_number(numerator) == True:
+                    if denominator == '0':
+                        return round(float(numerator.replace(',', '')))
+                    else:
+                        frac = Fraction(match.group().replace(',', ''))
+                        num_numerator = frac.numerator
+                        num_denominator = frac.denominator
+                        return round(float(num_numerator / num_denominator))
+                else:
+                    return None
+            else:
+                if float(match.group().replace(',', '')) == float('inf'):
+                    return None
+                return round(float(match.group().replace(',', '')))
+        else:
+            return None
+    else:
+        return None
+    
+def batch_data(data_list, batch_size=1):
+    n = len(data_list) // batch_size
+    batch_data = []
+    for i in range(n-1):
+        start = i * batch_size
+        end = (i+1)*batch_size
+        batch_data.append(data_list[start:end])
+
+    last_start = (n-1) * batch_size
+    last_end = MAX_INT
+    batch_data.append(data_list[last_start:last_end])
+    return batch_data
+
+def gsm8k_test(model, data_path, start=0, end=MAX_INT, batch_size=1, tensor_parallel_size=1):
+    INVALID_ANS = "[invalid]"
+    gsm8k_ins = []
+    gsm8k_answers = []
+    problem_prompt = (
+        "Below is an instruction that describes a task. "
+        "Write a response that appropriately completes the request.\n\n"
+        "### Instruction:\n{instruction}\n\n### Response: Let's think step by step."
+    )
+    print('prompt =====', problem_prompt)
+    with open(data_path,"r+", encoding="utf8") as f:
+        for idx, item in enumerate(jsonlines.Reader(f)):
+            temp_instr = problem_prompt.format(instruction=item["question"])
+            gsm8k_ins.append(temp_instr)
+            temp_ans = item['answer'].split('#### ')[1]
+            temp_ans = int(temp_ans.replace(',', ''))
+            gsm8k_answers.append(temp_ans)
+
+    gsm8k_ins = gsm8k_ins[start:end]
+    gsm8k_answers = gsm8k_answers[start:end]
+    print('lenght ====', len(gsm8k_ins))
+    # batch_gsm8k_ins = batch_data(gsm8k_ins, batch_size=batch_size)
+
+    stop_tokens = ["Instruction:", "Instruction", "Response:", "Response"]
+    sampling_params = SamplingParams(temperature=0, top_p=1, max_tokens=MAX_TOKEN, stop=stop_tokens)
+    print('sampleing =====', sampling_params)
+    llm = LLM(model=model,tensor_parallel_size=tensor_parallel_size, gpu_memory_utilization=0.90)
+
+    result = []
+
+    outputs = llm.generate(gsm8k_ins, sampling_params)
+    res_completions = [output.outputs[0].text for output in outputs]
+
+    invalid_outputs = []
+    for idx, (prompt, completion, prompt_answer) in enumerate(zip(gsm8k_ins, res_completions, gsm8k_answers)):
+        doc = {'question': prompt}
+        y_pred = extract_answer_number(completion)
+        # print('\n y_pred', y_pred, type(y_pred))
+        # print('ans', prompt_answer, type(prompt_answer))
+        if y_pred != None:
+            result.append(float(y_pred) == float(prompt_answer) or math_equal(y_pred, prompt_answer))
+        else:
+            result.append(False)
+            temp = {'question': prompt, 'output': completion, 'answer': prompt_answer}
+            invalid_outputs.append(temp)
+
+    acc = sum(result) / len(result)
+    print('len invalid outputs ====', len(invalid_outputs), ', invalid_outputs===', len(invalid_outputs))
+    # print('start===', start, ', end====', end)
+    print('gsm8k length====', len(result), ', gsm8k acc %====', acc*100)
+
+    
+    current_path = args.model
+    parent_dir = os.path.dirname(current_path.rstrip('/'))
+    output_filename = os.path.join(parent_dir, 'output.txt')
+    # output_filename = args.model + 'output.txt'
+    with open(output_filename, "a", encoding="utf-8") as f:
+        print(f'\n gsm8k MAX TOKEN = {MAX_TOKEN}, length==== {len(result)}, gsm8k acc %====, {acc*100}', file=f)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=str)  # model path
+    parser.add_argument("--data_file", type=str, default='data/gsm8k_test.jsonl')  # data path
+    parser.add_argument("--start", type=int, default=0) #start index
+    parser.add_argument("--end", type=int, default=MAX_INT)  # end index
+    parser.add_argument("--batch_size", type=int, default=60)  # batch_size
+    parser.add_argument("--tensor_parallel_size", type=int, default=1)  # tensor_parallel_size
+    return parser.parse_args()
+
+def set_deterministic_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    # torch.backends.cudnn.deterministic = True
+    # torch.backends.cudnn.benchmark = False
+
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    set_deterministic_seed()
+    gsm8k_test(model=args.model, data_path=args.data_file, start=args.start, end=args.end, batch_size=args.batch_size, tensor_parallel_size=args.tensor_parallel_size)
+    print('gsm ends', args.model)
+

nl_tasks/inference/util.py

@@ -0,0 +1,253 @@
+import pprint
+from grader import math_equal
+
+def last_boxed_only(sample):
+    q, a = sample
+    a = last_boxed_only_string(a)
+    if a == None:
+        return None
+    return (q, a)
+
+def last_boxed_only_string(string):
+    idx = string.rfind("\\boxed")
+    if idx < 0:
+        idx = string.rfind("\\fbox")
+        if idx < 0:
+            return None
+
+    i = idx
+    right_brace_idx = None
+    num_left_braces_open = 0
+    while i < len(string):
+        if string[i] == "{":
+            num_left_braces_open += 1
+        if string[i] == "}":
+            num_left_braces_open -= 1
+            if num_left_braces_open == 0:
+                right_brace_idx = i
+                break
+        i += 1
+    
+    if right_brace_idx == None:
+        retval = None
+    else:
+        retval = string[idx:right_brace_idx + 1]
+    
+    return retval
+
+def only_until_first_boxed_from_tokens(string, tokens):
+    idx = string.find("\\boxed")
+    if idx < 0:
+        idx = string.find("\\fbox")
+        if idx < 0:
+            return None
+    
+    cum_length = 0
+    for i, t in enumerate(tokens):
+        cum_length += len(t)
+        if cum_length >= idx:
+            break
+    
+    return tokens[:i]
+
+
+
+def clean_numbers(sample):
+    if not sample:
+        return None
+    new_sample = list()
+    for s in sample:
+        new_sample.append(_clean_numbers(s))
+
+    return tuple(new_sample)
+
+def _clean_numbers(string):
+    """
+    Clean Numbers in the given string
+
+    >>> _clean_numbers(None, "Hello 123")
+    'Hello 123'
+    >>> _clean_numbers(None, "Hello 1234")
+    'Hello 1,234'
+    >>> _clean_numbers(None, "Hello 1234324asdasd")
+    'Hello 1,234,324asdasd'
+    """
+    num_prev_digits = 0
+    new_string = ""
+    for i, c in enumerate(string):
+        # isdigit() doesnt work here because of weird unicode chars.
+        if c in {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0'}:
+            num_prev_digits += 1
+        else:
+            if num_prev_digits > 3:
+                # Some fixing
+                string_number = new_string[-num_prev_digits:]
+                new_string = new_string[:-num_prev_digits] + "{0:,}".format(int(string_number))
+            num_prev_digits = 0
+        new_string += c
+
+    if num_prev_digits > 3:
+        # Some fixing
+        string_number = new_string[-num_prev_digits:]
+        new_string = new_string[:-num_prev_digits] + "{0:,}".format(int(string_number))
+
+    return new_string
+
+def fix_fracs(string):
+    substrs = string.split("\\frac")
+    new_str = substrs[0]
+    if len(substrs) > 1:
+        substrs = substrs[1:]
+        for substr in substrs:
+            new_str += "\\frac"
+            if substr[0] == "{":
+                new_str += substr
+            else:
+                try:
+                    assert len(substr) >= 2
+                except AssertionError:
+                    return string
+                a = substr[0]
+                b = substr[1]
+                if b != "{":
+                    if len(substr) > 2:
+                        post_substr = substr[2:]
+                        new_str += "{" + a + "}{" + b + "}" + post_substr
+                    else:
+                        new_str += "{" + a + "}{" + b + "}"
+                else:
+                    if len(substr) > 2:
+                        post_substr = substr[2:]
+                        new_str += "{" + a + "}" + b + post_substr
+                    else:
+                        new_str += "{" + a + "}" + b
+    string = new_str
+    return string
+
+def fix_a_slash_b(string):
+    if len(string.split("/")) != 2:
+        return string
+    a = string.split("/")[0]
+    b = string.split("/")[1]
+    try:
+        a = int(a)
+        b = int(b)
+        assert string == "{}/{}".format(a, b)
+        new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"
+        return new_string
+    except AssertionError:
+        return string
+
+def remove_right_units(string):
+    # "\\text{ " only ever occurs (at least in the val set) when describing units
+    if "\\text{ " in string:
+        splits = string.split("\\text{ ")
+        assert len(splits) == 2
+        return splits[0]
+    else:
+        return string
+
+def fix_sqrt(string):
+    if "\\sqrt" not in string:
+        return string
+    splits = string.split("\\sqrt")
+    new_string = splits[0]
+    for split in splits[1:]:
+        if split[0] != "{":
+            a = split[0]
+            new_substr = "\\sqrt{" + a + "}" + split[1:]
+        else:
+            new_substr = "\\sqrt" + split
+        new_string += new_substr
+    return new_string
+
+
+def strip_string(string):
+    # linebreaks
+    string = string.replace("\n", "")
+
+    # remove inverse spaces
+    string = string.replace("\\!", "")
+
+    # replace \\ with \
+    string = string.replace("\\\\", "\\")
+
+    # replace tfrac and dfrac with frac
+    string = string.replace("tfrac", "frac")
+    string = string.replace("dfrac", "frac")
+
+    # remove \left and \right
+    string = string.replace("\\left", "")
+    string = string.replace("\\right", "")
+
+    # Remove circ (degrees)
+    string = string.replace("^{\\circ}", "")
+    string = string.replace("^\\circ", "")
+
+    # remove dollar signs
+    string = string.replace("\\$", "")
+
+    # remove units (on the right)
+    string = remove_right_units(string)
+
+    # remove percentage
+    string = string.replace("\\%", "")
+    string = string.replace("\%", "")  # noqa: W605
+
+    # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
+    string = string.replace(" .", " 0.")
+    string = string.replace("{.", "{0.")
+    # if empty, return empty string
+    if len(string) == 0:
+        return string
+    if string[0] == ".":
+        string = "0" + string
+
+    # to consider: get rid of e.g. "k = " or "q = " at beginning
+    if len(string.split("=")) == 2:
+        if len(string.split("=")[0]) <= 2:
+            string = string.split("=")[1]
+
+    # fix sqrt3 --> sqrt{3}
+    string = fix_sqrt(string)
+
+    # remove spaces
+    string = string.replace(" ", "")
+
+    # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
+    string = fix_fracs(string)
+
+    # manually change 0.5 --> \frac{1}{2}
+    if string == "0.5":
+        string = "\\frac{1}{2}"
+
+    # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
+    string = fix_a_slash_b(string)
+
+    return string
+
+
+def is_equiv(str1, str2, verbose=False):
+    if str1 is None and str2 is None:
+        print("WARNING: Both None")
+        return True
+    if str1 is None or str2 is None:
+        return False
+
+    try:
+        ss1 = strip_string(str1)
+        ss2 = strip_string(str2)
+        #pdb.set_trace()
+        if verbose:
+            print(ss1, ss2)
+        #return ss1 == ss2
+        res = math_equal(ss1,ss2) or ss1 == ss2
+        return res
+    except Exception:
+        #return str1 == str2
+        res = math_equal(str1,str1) or str1 == str2
+        return res
+
+class NotEqual:
+    def __eq__(self, other):
+        return False