Upload folder using huggingface_hub

eq_bench.json/openai-community__gpt2/results_2024-10-27T17-03-52.150542.json

@@ -0,0 +1,122 @@
+{
+  "results": {
+    "eq_bench": {
+      "alias": "eq_bench",
+      "eqbench,none": 0.0,
+      "eqbench_stderr,none": 0.0,
+      "percent_parseable,none": 0.0,
+      "percent_parseable_stderr,none": 0.0
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1
+  },
+  "n-shot": {
+    "eq_bench": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    }
+  },
+  "n-samples": {
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=openai-community/gpt2,dtype=auto,trust_remote_code=True",
+    "model_num_parameters": 124439808,
+    "model_dtype": "torch.float32",
+    "model_revision": "main",
+    "model_sha": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "7882043b",
+  "date": 1730073796.3589597,
+  "pretty_env_info": "PyTorch version: 2.5.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 10.5.0-1ubuntu1~22.04) 10.5.0\nClang version: 15.0.7\nCMake version: version 3.26.3\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.8.0-47-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 11.5.119\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3080\nNvidia driver version: 555.58.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               24\nOn-line CPU(s) list:                  0-23\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 9 3900X 12-Core Processor\nCPU family:                           23\nModel:                                113\nThread(s) per core:                   2\nCore(s) per socket:                   12\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU max MHz:                          4672.0698\nCPU min MHz:                          2200.0000\nBogoMIPS:                             7585.66\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            384 KiB (12 instances)\nL1i cache:                            384 KiB (12 instances)\nL2 cache:                             6 MiB (12 instances)\nL3 cache:                             64 MiB (4 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-23\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.1.2\n[pip3] torch==2.5.0\n[pip3] torchmetrics==1.4.0\n[pip3] torchvision==0.20.0\n[pip3] triton==3.1.0\n[conda] blas                      1.0                         mkl  \n[conda] mkl                       2021.4.0           h06a4308_640  \n[conda] mkl-service               2.4.0            py39h7f8727e_0  \n[conda] mkl_fft                   1.3.1            py39hd3c417c_0  \n[conda] mkl_random                1.2.2            py39h51133e4_0  \n[conda] numpy                     1.21.5           py39he7a7128_1  \n[conda] numpy-base                1.21.5           py39hf524024_1  \n[conda] numpydoc                  1.2                pyhd3eb1b0_0  ",
+  "transformers_version": "4.45.0",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "50256"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "50256"
+  ],
+  "tokenizer_bos_token": [
+    "<|endoftext|>",
+    "50256"
+  ],
+  "eot_token_id": 50256,
+  "max_length": 1024,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "openai-community/gpt2",
+  "model_name_sanitized": "openai-community__gpt2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 6865.042045323,
+  "end_time": 6907.059375842,
+  "total_evaluation_time_seconds": "42.0173305190001"
+}