update

.gitignore

@@ -11,3 +11,5 @@ __pycache__/
 #eval-queue-bk/
 #eval-results-bk/
 logs/
+.idea/
+

.idea/OmniGenomeLeaderboard.iml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module version="4">
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,88 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="54" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="70">
+            <item index="0" class="java.lang.String" itemvalue="ftfy" />
+            <item index="1" class="java.lang.String" itemvalue="gensim" />
+            <item index="2" class="java.lang.String" itemvalue="diffusers" />
+            <item index="3" class="java.lang.String" itemvalue="tensorflow_text" />
+            <item index="4" class="java.lang.String" itemvalue="tensorflow" />
+            <item index="5" class="java.lang.String" itemvalue="TextAttack" />
+            <item index="6" class="java.lang.String" itemvalue="tensorflow_hub" />
+            <item index="7" class="java.lang.String" itemvalue="pyabsa" />
+            <item index="8" class="java.lang.String" itemvalue="protobuf" />
+            <item index="9" class="java.lang.String" itemvalue="networkx" />
+            <item index="10" class="java.lang.String" itemvalue="update-checker" />
+            <item index="11" class="java.lang.String" itemvalue="boostaug" />
+            <item index="12" class="java.lang.String" itemvalue="pandas" />
+            <item index="13" class="java.lang.String" itemvalue="termcolor" />
+            <item index="14" class="java.lang.String" itemvalue="metric-visualizer" />
+            <item index="15" class="java.lang.String" itemvalue="spacy" />
+            <item index="16" class="java.lang.String" itemvalue="seqeval" />
+            <item index="17" class="java.lang.String" itemvalue="autocuda" />
+            <item index="18" class="java.lang.String" itemvalue="sentencepiece" />
+            <item index="19" class="java.lang.String" itemvalue="findfile" />
+            <item index="20" class="java.lang.String" itemvalue="gitpython" />
+            <item index="21" class="java.lang.String" itemvalue="pytorch_warmup" />
+            <item index="22" class="java.lang.String" itemvalue="torchtext" />
+            <item index="23" class="java.lang.String" itemvalue="googledrivedownloader" />
+            <item index="24" class="java.lang.String" itemvalue="opennmt-py" />
+            <item index="25" class="java.lang.String" itemvalue="opencv-contrib-python" />
+            <item index="26" class="java.lang.String" itemvalue="timm" />
+            <item index="27" class="java.lang.String" itemvalue="pytorch_lightning" />
+            <item index="28" class="java.lang.String" itemvalue="invisible-watermark" />
+            <item index="29" class="java.lang.String" itemvalue="tqdm" />
+            <item index="30" class="java.lang.String" itemvalue="tokenizers" />
+            <item index="31" class="java.lang.String" itemvalue="gradio" />
+            <item index="32" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="33" class="java.lang.String" itemvalue="nltk" />
+            <item index="34" class="java.lang.String" itemvalue="arxiv" />
+            <item index="35" class="java.lang.String" itemvalue="imblearn" />
+            <item index="36" class="java.lang.String" itemvalue="packaging" />
+            <item index="37" class="java.lang.String" itemvalue="setuptools" />
+            <item index="38" class="java.lang.String" itemvalue="numpy" />
+            <item index="39" class="java.lang.String" itemvalue="requests" />
+            <item index="40" class="java.lang.String" itemvalue="nlpaug" />
+            <item index="41" class="java.lang.String" itemvalue="yacs" />
+            <item index="42" class="java.lang.String" itemvalue="tensorboardX" />
+            <item index="43" class="java.lang.String" itemvalue="rouge" />
+            <item index="44" class="java.lang.String" itemvalue="datasets" />
+            <item index="45" class="java.lang.String" itemvalue="transformers" />
+            <item index="46" class="java.lang.String" itemvalue="typing_extensions" />
+            <item index="47" class="java.lang.String" itemvalue="torch" />
+            <item index="48" class="java.lang.String" itemvalue="cmudict" />
+            <item index="49" class="java.lang.String" itemvalue="pykakasi" />
+            <item index="50" class="java.lang.String" itemvalue="fastapi" />
+            <item index="51" class="java.lang.String" itemvalue="rouge-chinese" />
+            <item index="52" class="java.lang.String" itemvalue="peft" />
+            <item index="53" class="java.lang.String" itemvalue="uvicorn" />
+            <item index="54" class="java.lang.String" itemvalue="sse-starlette" />
+            <item index="55" class="java.lang.String" itemvalue="trl" />
+            <item index="56" class="java.lang.String" itemvalue="tiktoken" />
+            <item index="57" class="java.lang.String" itemvalue="scipy" />
+            <item index="58" class="java.lang.String" itemvalue="pydantic" />
+            <item index="59" class="java.lang.String" itemvalue="jieba" />
+            <item index="60" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="61" class="java.lang.String" itemvalue="transformers_stream_generator" />
+            <item index="62" class="java.lang.String" itemvalue="accelerate" />
+            <item index="63" class="java.lang.String" itemvalue="optimum" />
+            <item index="64" class="java.lang.String" itemvalue="auto-gptq" />
+            <item index="65" class="java.lang.String" itemvalue="bitsandbytes" />
+            <item index="66" class="java.lang.String" itemvalue="deepspeed" />
+            <item index="67" class="java.lang.String" itemvalue="evaluate" />
+            <item index="68" class="java.lang.String" itemvalue="tensorboardx" />
+            <item index="69" class="java.lang.String" itemvalue="sklearn" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="C:\Users\chuan\miniconda3" />
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -5,8 +5,57 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="4dc9d937-d789-48c5-9ba5-fe08d01bc11f" name="Changes" comment="">
+      <change afterPath="$PROJECT_DIR$/.idea/OmniGenomeLeaderboard.iml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/inspectionProfiles/Project_Default.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/inspectionProfiles/profiles_settings.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/GB/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/GB/kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/GUE/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/GUE/kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/PGB/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/PGB/kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-queue/RGB/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/3UTRBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/Caduceus.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/DNABERT-2-117M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/HyenaDNA.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/NT-V2-100M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/OmniGenome186M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GB/SpliceBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/3UTRBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/Caduceus.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/DNABERT-2-117M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/HyenaDNA.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/NT-V2-100M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/OmniGenome186M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/GUE/SpliceBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/3UTRBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/Agro-NT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/CDSBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/Caduceus.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/DNABERT-2-117M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/HyenaDNA.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/NT-V2-100M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/OmniGenome186M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/RNA-BERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/RNA-FM.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/RNA-MSM.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/PGB/SpliceBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/3UTRBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/Agro-NT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/CDSBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/DNABERT-2-117M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/HyenaDNA.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/NT-V2-100M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/OmniGenome186M.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/SpliceBERT.json" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/eval-results/RGB/yangheng/results_OmniGenome-52M.json" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/about.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/about.py" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -26,10 +75,10 @@
   </component>
   <component name="PropertiesComponent"><![CDATA[{
   "keyToString": {
-    "Python.app.executor": "Debug",
+    "Python.app.executor": "Run",
     "RunOnceActivity.ShowReadmeOnStart": "true",
     "git-widget-placeholder": "main",
-    "last_opened_file_path": "C:/Users/chuan/OneDrive - University of Exeter/AIProjects/OmniGenomeLeaderboard/eval-queue",
+    "last_opened_file_path": "C:/Users/chuan/OneDrive - University of Exeter/AIProjects/OmniGenomeLeaderboard",
     "node.js.detected.package.eslint": "true",
     "node.js.detected.package.tslint": "true",
     "node.js.selected.package.eslint": "(autodetect)",
@@ -40,9 +89,9 @@
 }]]></component>
   <component name="RecentsManager">
     <key name="CopyFile.RECENT_KEYS">
+      <recent name="C:\Users\chuan\OneDrive - University of Exeter\AIProjects\OmniGenomeLeaderboard" />
       <recent name="C:\Users\chuan\OneDrive - University of Exeter\AIProjects\OmniGenomeLeaderboard\eval-queue" />
       <recent name="C:\Users\chuan\OneDrive - University of Exeter\AIProjects\OmniGenomeLeaderboard\eval-queue\.cache\huggingface\download\GB\InstaDeepAI" />
-      <recent name="C:\Users\chuan\OneDrive - University of Exeter\AIProjects\OmniGenomeLeaderboard" />
     </key>
   </component>
   <component name="RunManager">
@@ -94,7 +143,9 @@
       <workItem from="1726143220520" duration="3239000" />
       <workItem from="1726146471494" duration="2362000" />
       <workItem from="1726223901200" duration="2056000" />
-      <workItem from="1726434553745" duration="4460000" />
+      <workItem from="1726434553745" duration="6270000" />
+      <workItem from="1726515056209" duration="632000" />
+      <workItem from="1726687104634" duration="322000" />
     </task>
     <servers />
   </component>
@@ -134,6 +185,6 @@
     </breakpoint-manager>
   </component>
   <component name="com.intellij.coverage.CoverageDataManagerImpl">
-    <SUITE FILE_PATH="coverage/OmniGenomeLeaderboard$app.coverage" NAME="app Coverage Results" MODIFIED="1726486285408" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/OmniGenomeLeaderboard$app.coverage" NAME="app Coverage Results" MODIFIED="1726687258224" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
   </component>
 </project>

app.py

@@ -35,31 +35,30 @@ def restart_space():
 
 
 ### Space initialisation
-
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO,
-        local_dir=EVAL_RESULTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO,
+#         local_dir=EVAL_REQUESTS_PATH,
+#         repo_type="dataset",
+#         tqdm_class=None,
+#         etag_timeout=30,
+#         token=TOKEN,
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print(EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO,
+#         local_dir=EVAL_RESULTS_PATH,
+#         repo_type="dataset",
+#         tqdm_class=None,
+#         etag_timeout=30,
+#         token=TOKEN,
+#     )
+# except Exception:
+#     restart_space()
 
 RGB_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH+"/RGB/", EVAL_REQUESTS_PATH+"/RGB/", RGB_COLS, RGB_BENCHMARK_COLS)
 PGB_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH+"/PGB/", EVAL_REQUESTS_PATH+"/PGB/", PGB_COLS, PGB_BENCHMARK_COLS)

eval-queue/GB/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "InstaDeepAI/nucleotide-transformer-v2-100m-multi-species", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 96, "license": "custom"}

eval-queue/GB/kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 7.73, "license": "custom"}

eval-queue/GUE/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "InstaDeepAI/nucleotide-transformer-v2-100m-multi-species", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 96, "license": "custom"}

eval-queue/GUE/kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 7.73, "license": "custom"}

eval-queue/PGB/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "InstaDeepAI/nucleotide-transformer-v2-100m-multi-species", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 96, "license": "custom"}

eval-queue/PGB/kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 7.73, "license": "custom"}

eval-queue/RGB/InstaDeepAI/nucleotide-transformer-v2-100m-multi-species_eval_request_False_bfloat16_Original.json

@@ -0,0 +1 @@
+{"model": "InstaDeepAI/nucleotide-transformer-v2-100m-multi-species", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 100, "license": "custom"}

eval-results/GB/3UTRBERT.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"multimolecule/utrbert-4mer",
+      "model_args":"pretrained=multimolecule/utrbert-4mer,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"86M", "Pretraining Data":"20,362 Sequences", "Species":"Multi-Species", "Nucleic Acid":"mRNA 3'UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/utrbert-4mer",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.8950
+      },
+      "DOW":{
+         "F1":0.9022
+      },
+      "DRE":{
+         "F1":0.7435
+      },
+      "DME":{
+         "F1":0.8014
+      },
+      "HCE":{
+         "F1":0.7023
+      },
+      "HEE":{
+         "F1":0.7633
+      },
+      "HRE":{
+         "F1":0.9847
+      },
+      "HNP":{
+         "F1":0.8249
+      },
+      "HOR":{
+         "F1":0.6678
+      }
+   }
+}

eval-results/GB/Caduceus.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16",
+      "model_args":"pretrained= kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"7.73M", "Pretraining Data":"35 billion nucleotide base pairs", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.9213
+      },
+      "DOW":{
+         "F1":0.9474
+      },
+      "DRE":{
+         "F1":0.7203
+      },
+      "DME":{
+         "F1":0.7561
+      },
+      "HCE":{
+         "F1":0.7020
+      },
+      "HEE":{
+         "F1":0.7647
+      },
+      "HRE":{
+         "F1":0.7916
+      },
+      "HNP":{
+         "F1":0.8436
+      },
+      "HOR":{
+         "F1":0.6317
+      }
+   }
+}

eval-results/GB/DNABERT-2-117M.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"zhihan1996/DNABERT-2-117M",
+      "model_args":"pretrained=zhihan1996/DNABERT-2-117M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"BPE", "# of Params":"117M", "Pretraining Data":"32.49B Tokens", "Species":"Human + 135 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"zhihan1996/DNABERT-2-117M",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.9267
+      },
+      "DOW":{
+         "F1":0.9517
+      },
+      "DRE":{
+         "F1":0.4377
+      },
+      "DME":{
+         "F1":0.7721
+      },
+      "HCE":{
+         "F1":0.7558
+      },
+      "HEE":{
+         "F1":0.8066
+      },
+      "HRE":{
+         "F1":0.7814
+      },
+      "HNP":{
+         "F1":0.8580
+      },
+      "HOR":{
+         "F1":0.6803
+      }
+   }
+}

eval-results/GB/HyenaDNA.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_args":"pretrained=LongSafari/hyenadna-large-1m-seqlen-hf,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"47M", "Pretraining Data":"~3.2B Tokens", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.8821
+      },
+      "DOW":{
+         "F1":0.9413
+      },
+      "DRE":{
+         "F1":0.7011
+      },
+      "DME":{
+         "F1":0.7644
+      },
+      "HCE":{
+         "F1":0.7038
+      },
+      "HEE":{
+         "F1":0.7958
+      },
+      "HRE":{
+         "F1":0.9633
+      },
+      "HNP":{
+         "F1":0.8599
+      },
+      "HOR":{
+         "F1":0.6703
+      }
+   }
+}

eval-results/GB/NT-V2-100M.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_args":"pretrained=InstaDeepAI/nucleotide-transformer-v2-100m-multi-species,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"96M", "Pretraining Data":"300B Tokens", "Species":"Human + 850 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.9166
+      },
+      "DOW":{
+         "F1":0.9432
+      },
+      "DRE":{
+         "F1":0.7820
+      },
+      "DME":{
+         "F1":0.8172
+      },
+      "HCE":{
+         "F1":0.7198
+      },
+      "HEE":{
+         "F1":0.7985
+      },
+      "HRE":{
+         "F1":0.9330
+      },
+      "HNP":{
+         "F1":0.8530
+      },
+      "HOR":{
+         "F1":0.6853
+      }
+   }
+}

eval-results/GB/OmniGenome186M.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"yangheng/omnigenome-186M",
+      "model_args":"pretrained=yangheng/omnigenome-186M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"186M", "Pretraining Data":"54.2B Tokens", "Species":"1124 Plant Species", "Nucleic Acid":"mRNA, CDS, UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"yangheng/omnigenome-186M",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.9416
+      },
+      "DOW":{
+         "F1":0.9349
+      },
+      "DRE":{
+         "F1":0.7717
+      },
+      "DME":{
+         "F1":0.8034
+      },
+      "HCE":{
+         "F1":0.7351
+      },
+      "HEE":{
+         "F1":0.8223
+      },
+      "HRE":{
+         "F1":0.9566
+      },
+      "HNP":{
+         "F1":0.8787
+      },
+      "HOR":{
+         "F1":0.6897
+      }
+   }
+}

eval-results/GB/SpliceBERT.json

@@ -0,0 +1,48 @@
+{
+   "config":{
+      "model":"multimolecule/splicebert",
+      "model_args":"pretrained=multimolecule/splicebert,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"19.7M", "Pretraining Data":"65 billion nucleotides", "Species":"Multi-Species", "Nucleic Acid":"mRNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/splicebert",
+      "model_sha":"main"
+   },
+   "results":{
+      "DEM":{
+         "F1":0.9472
+      },
+      "DOW":{
+         "F1":0.9642
+      },
+      "DRE":{
+         "F1":0.7229
+      },
+      "DME":{
+         "F1":0.7470
+      },
+      "HCE":{
+         "F1":0.7350
+      },
+      "HEE":{
+         "F1":0.7960
+      },
+      "HRE":{
+         "F1":0.9523
+      },
+      "HNP":{
+         "F1":0.8957
+      },
+      "HOR":{
+         "F1":0.6889
+      }
+   }
+}

eval-results/GUE/3UTRBERT.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"multimolecule/utrbert-4mer",
+      "model_args":"pretrained=multimolecule/utrbert-4mer,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"86M", "Pretraining Data":"20,362 Sequences", "Species":"Multi-Species", "Nucleic Acid":"mRNA 3'UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/utrbert-4mer",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7189
+      },
+      "Mouse TF-M":{
+         "F1":0.7146
+      },
+      "Virus CVC":{
+         "F1":0.6871
+      },
+      "Human TF-H":{
+         "F1":0.7485
+      },
+      "Human PD":{
+         "F1":0.8237
+      },
+      "Human CPD":{
+         "F1":0.9051
+      },
+      "Human SSP":{
+         "F1":0.8195
+      }
+   }
+}

eval-results/GUE/Caduceus.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16",
+      "model_args":"pretrained= kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"7.73M", "Pretraining Data":"35 billion nucleotide base pairs", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7349
+      },
+      "Mouse TF-M":{
+         "F1":0.7818
+      },
+      "Virus CVC":{
+         "F1":0.4909
+      },
+      "Human TF-H":{
+         "F1":0.7956
+      },
+      "Human PD":{
+         "F1":0.8913
+      },
+      "Human CPD":{
+         "F1":0.8509
+      },
+      "Human SSP":{
+         "F1":0.8182
+      }
+   }
+}

eval-results/GUE/DNABERT-2-117M.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"zhihan1996/DNABERT-2-117M",
+      "model_args":"pretrained=zhihan1996/DNABERT-2-117M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"BPE", "# of Params":"117M", "Pretraining Data":"32.49B Tokens", "Species":"Human + 135 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"zhihan1996/DNABERT-2-117M",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7585
+      },
+      "Mouse TF-M":{
+         "F1":0.8623
+      },
+      "Virus CVC":{
+         "F1":0.6890
+      },
+      "Human TF-H":{
+         "F1":0.8180
+      },
+      "Human PD":{
+         "F1":0.9017
+      },
+      "Human CPD":{
+         "F1":0.8257
+      },
+      "Human SSP":{
+         "F1":0.8521
+      }
+   }
+}

eval-results/GUE/HyenaDNA.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_args":"pretrained=LongSafari/hyenadna-large-1m-seqlen-hf,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"47M", "Pretraining Data":"~3.2B Tokens", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7308
+      },
+      "Mouse TF-M":{
+         "F1":0.7344
+      },
+      "Virus CVC":{
+         "F1":0.6637
+      },
+      "Human TF-H":{
+         "F1":0.7762
+      },
+      "Human PD":{
+         "F1":0.9119
+      },
+      "Human CPD":{
+         "F1":0.8431
+      },
+      "Human SSP":{
+         "F1":0.8334
+      }
+   }
+}

eval-results/GUE/NT-V2-100M.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_args":"pretrained=InstaDeepAI/nucleotide-transformer-v2-100m-multi-species,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"96M", "Pretraining Data":"300B Tokens", "Species":"Human + 850 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7493
+      },
+      "Mouse TF-M":{
+         "F1":0.7810
+      },
+      "Virus CVC":{
+         "F1":0.5923
+      },
+      "Human TF-H":{
+         "F1":0.7912
+      },
+      "Human PD":{
+         "F1":0.9087
+      },
+      "Human CPD":{
+         "F1":0.8470
+      },
+      "Human SSP":{
+         "F1":0.8413
+      }
+   }
+}

eval-results/GUE/OmniGenome186M.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"yangheng/omnigenome-186M",
+      "model_args":"pretrained=yangheng/omnigenome-186M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"186M", "Pretraining Data":"54.2B Tokens", "Species":"1124 Plant Species", "Nucleic Acid":"mRNA, CDS, UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"yangheng/omnigenome-186M",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7851
+      },
+      "Mouse TF-M":{
+         "F1":0.8472
+      },
+      "Virus CVC":{
+         "F1":0.7472
+      },
+      "Human TF-H":{
+         "F1":0.8173
+      },
+      "Human PD":{
+         "F1":0.9004
+      },
+      "Human CPD":{
+         "F1":0.8522
+      },
+      "Human SSP":{
+         "F1":0.9039
+      }
+   }
+}

eval-results/GUE/SpliceBERT.json

@@ -0,0 +1,42 @@
+{
+   "config":{
+      "model":"multimolecule/splicebert",
+      "model_args":"pretrained=multimolecule/splicebert,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"19.7M", "Pretraining Data":"65 billion nucleotides", "Species":"Multi-Species", "Nucleic Acid":"mRNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/splicebert",
+      "model_sha":"main"
+   },
+   "results":{
+      "Yeast EMP":{
+         "F1":0.7766
+      },
+      "Mouse TF-M":{
+         "F1":0.8497
+      },
+      "Virus CVC":{
+         "F1":0.5624
+      },
+      "Human TF-H":{
+         "F1":0.8277
+      },
+      "Human PD":{
+         "F1":0.9224
+      },
+      "Human CPD":{
+         "F1":0.8396
+      },
+      "Human SSP":{
+         "F1":0.9381
+      }
+   }
+}

eval-results/PGB/3UTRBERT.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"multimolecule/utrbert-4mer",
+      "model_args":"pretrained=multimolecule/utrbert-4mer,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"86M", "Pretraining Data":"20,362 Sequences", "Species":"Multi-Species", "Nucleic Acid":"mRNA 3'UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/utrbert-4mer",
+      "model_sha":"main"
+   },
+"results":{
+      "PolyA":{
+         "F1":0.7648
+      },
+      "LncRNA":{
+         "F1":0.7075
+      },
+      "Chrom Acc":{
+         "F1":0.6371
+      },
+      "Prom Str":{
+         "RMSE":1.04
+      },
+      "Term Str":{
+         "RMSE":0.36
+      },
+      "Splice":{
+         "F1":0.9444
+      },
+      "Gene Exp":{
+         "RMSE":14.87
+      },
+      "Enhancer":{
+         "F1":0.7167
+      }
+   }
+}

eval-results/PGB/Agro-NT.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"InstaDeepAI/agro-nucleotide-transformer-1b",
+      "model_args":"pretrained=InstaDeepAI/agro-nucleotide-transformer-1b,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"985M", "Pretraining Data":"472.5B Tokens", "Species":"48 Edible Plants", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"InstaDeepAI/agro-nucleotide-transformer-1b",
+      "model_sha":"main"
+   },
+"results":{
+      "PolyA":{
+         "F1":0.7889
+      },
+      "LncRNA":{
+         "F1":0.6724
+      },
+      "Chrom Acc":{
+         "F1":0.6327
+      },
+      "Prom Str":{
+         "RMSE":0.94
+      },
+      "Term Str":{
+         "RMSE":0.78
+      },
+      "Splice":{
+         "F1":0.8845
+      },
+      "Gene Exp":{
+         "RMSE":15.56
+      },
+      "Enhancer":{
+         "F1":0.6283
+      }
+   }
+}

eval-results/PGB/CDSBERT.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"GleghornLab/cdsBERT",
+      "model_args":"pretrained=GleghornLab/cdsBERT,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"420M", "Pretraining Data":"4M Sequences", "Species":"4,069 RNA families", "Nucleic Acid":"CDS"},
+      "model_dtype":"bfloat16",
+      "model_name":"GleghornLab/cdsBERT",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.3972
+      },
+      "LncRNA":{
+         "F1":0.3306
+      },
+      "Chrom Acc":{
+         "F1":0.4895
+      },
+      "Prom Str":{
+         "RMSE":2.19
+      },
+      "Term Str":{
+         "RMSE":0.59
+      },
+      "Splice":{
+         "F1":0.5220
+      },
+      "Gene Exp":{
+         "RMSE":14.77
+      },
+      "Enhancer":{
+         "F1":0.3393
+      }
+   }
+}

eval-results/PGB/Caduceus.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16",
+      "model_args":"pretrained= kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"7.73M", "Pretraining Data":"35 billion nucleotide base pairs", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"kuleshov-group/caduceus-ps_seqlen-131k_d_model-256_n_layer-16",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.7089
+      },
+      "LncRNA":{
+         "F1":0.6840
+      },
+      "Chrom Acc":{
+         "F1":0.6453
+      },
+      "Prom Str":{
+         "RMSE":0.91
+      },
+      "Term Str":{
+         "RMSE":0.26
+      },
+      "Splice":{
+         "F1":0.7951
+      },
+      "Gene Exp":{
+         "RMSE":14.72
+      },
+      "Enhancer":{
+         "F1":0.6083
+      }
+   }
+}

eval-results/PGB/DNABERT-2-117M.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"zhihan1996/DNABERT-2-117M",
+      "model_args":"pretrained=zhihan1996/DNABERT-2-117M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"BPE", "# of Params":"117M", "Pretraining Data":"32.49B Tokens", "Species":"Human + 135 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"zhihan1996/DNABERT-2-117M",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.4135
+      },
+      "LncRNA":{
+         "F1":0.7255
+      },
+      "Chrom Acc":{
+         "F1":0.6149
+      },
+      "Prom Str":{
+         "RMSE":0.99
+      },
+      "Term Str":{
+         "RMSE":0.24
+      },
+      "Splice":{
+         "F1":0.4534
+      },
+      "Gene Exp":{
+         "RMSE":14.78
+      },
+      "Enhancer":{
+         "F1":0.3640
+      }
+   }
+}

eval-results/PGB/HyenaDNA.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_args":"pretrained=LongSafari/hyenadna-large-1m-seqlen-hf,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"47M", "Pretraining Data":"~3.2B Tokens", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.8311
+      },
+      "LncRNA":{
+         "F1":0.5821
+      },
+      "Chrom Acc":{
+         "F1":0.5220
+      },
+      "Prom Str":{
+         "RMSE":0.88
+      },
+      "Term Str":{
+         "RMSE":0.26
+      },
+      "Splice":{
+         "F1":0.9028
+      },
+      "Gene Exp":{
+         "RMSE":14.76
+      },
+      "Enhancer":{
+         "F1":0.6617
+      }
+   }
+}

eval-results/PGB/NT-V2-100M.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_args":"pretrained=InstaDeepAI/nucleotide-transformer-v2-100m-multi-species,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"96M", "Pretraining Data":"300B Tokens", "Species":"Human + 850 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.7126
+      },
+      "LncRNA":{
+         "F1":0.7308
+      },
+      "Chrom Acc":{
+         "F1":0.6571
+      },
+      "Prom Str":{
+         "RMSE":0.81
+      },
+      "Term Str":{
+         "RMSE":0.27
+      },
+      "Splice":{
+         "F1":0.9505
+      },
+      "Gene Exp":{
+         "RMSE":14.69
+      },
+      "Enhancer":{
+         "F1":0.7389
+      }
+   }
+}

eval-results/PGB/OmniGenome186M.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"yangheng/omnigenome-186M",
+      "model_args":"pretrained=yangheng/omnigenome-186M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"186M", "Pretraining Data":"54.2B Tokens", "Species":"1124 Plant Species", "Nucleic Acid":"mRNA, CDS, UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"yangheng/omnigenome-186M",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.8755
+      },
+      "LncRNA":{
+         "F1":0.7796
+      },
+      "Chrom Acc":{
+         "F1":0.6769
+      },
+      "Prom Str":{
+         "RMSE":0.59
+      },
+      "Term Str":{
+         "RMSE":0.18
+      },
+      "Splice":{
+         "F1":0.9841
+      },
+      "Gene Exp":{
+         "RMSE":14.71
+      },
+      "Enhancer":{
+         "F1":0.7977
+      }
+   }
+}

eval-results/PGB/RNA-BERT.json

@@ -0,0 +1,46 @@
+{
+   "config":{
+      "model":"multimolecule/rnabert",
+      "model_args":"pretrained=multimolecule/rnabert,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"0.48M", "Pretraining Data":"76,237 human ncRNA sequences", "Species":"Human", "Nucleic Acid":"ncRNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/rnabert",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.7854
+      },
+      "LncRNA":{
+         "F1":0.6199
+      },
+      "Chrom Acc":{
+         "F1":0.4894
+      },
+      "Prom Str":{
+         "RMSE":1.81
+      },
+      "Term Str":{
+         "RMSE":0.38
+      },
+      "Splice":{
+         "F1":0.9445
+      },
+      "Gene Exp":{
+         "RMSE":14.89
+      },
+      "Enhancer":{
+         "F1":0.5761
+      }
+
+   }
+}

eval-results/PGB/RNA-FM.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"multimolecule/rnafm",
+      "model_args":"pretrained=multimolecule/rnafm,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"99.52M", "Pretraining Data":"23.7 million non-redundant RNA sequences", "Species":"Multi-Species", "Nucleic Acid":"ncRNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/rnafm",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.8494
+      },
+      "LncRNA":{
+         "F1":0.6875
+      },
+      "Chrom Acc":{
+         "F1":0.5492
+      },
+      "Prom Str":{
+         "RMSE":0.95
+      },
+      "Term Str":{
+         "RMSE":0.27
+      },
+      "Splice":{
+         "F1":0.9595
+      },
+      "Gene Exp":{
+         "RMSE":14.83
+      },
+      "Enhancer":{
+         "F1":0.5714
+      }
+   }
+}

eval-results/PGB/RNA-MSM.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"multimolecule/rnamsm",
+      "model_args":"pretrained=multimolecule/rnamsm,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"96.5M", "Pretraining Data":"3,932 RNA families", "Species":"Multi-Species", "Nucleic Acid":"RNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/rnamsm",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.8425
+      },
+      "LncRNA":{
+         "F1":0.6749
+      },
+      "Chrom Acc":{
+         "F1":0.5352
+      },
+      "Prom Str":{
+         "RMSE":1.28
+      },
+      "Term Str":{
+         "RMSE":0.28
+      },
+      "Splice":{
+         "F1":0.9549
+      },
+      "Gene Exp":{
+         "RMSE":14.87
+      },
+      "Enhancer":{
+         "F1":0.6145
+      }
+   }
+}

eval-results/PGB/SpliceBERT.json

@@ -0,0 +1,45 @@
+{
+   "config":{
+      "model":"multimolecule/splicebert",
+      "model_args":"pretrained=multimolecule/splicebert,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"19.7M", "Pretraining Data":"65 billion nucleotides", "Species":"Multi-Species", "Nucleic Acid":"mRNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/splicebert",
+      "model_sha":"main"
+   },
+   "results":{
+      "PolyA":{
+         "F1":0.6523
+      },
+      "LncRNA":{
+         "F1":0.7188
+      },
+      "Chrom Acc":{
+         "F1":0.6362
+      },
+      "Prom Str":{
+         "RMSE":0.75
+      },
+      "Term Str":{
+         "RMSE":0.22
+      },
+      "Splice":{
+         "F1":0.9645
+      },
+      "Gene Exp":{
+         "RMSE":14.70
+      },
+      "Enhancer":{
+         "F1":0.6971
+      }
+   }
+}

eval-results/RGB/yangheng/3UTRBERT.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"multimolecule/utrbert-4mer",
+      "model_args":"pretrained=multimolecule/utrbert-4mer,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"86M", "Pretraining Data":"20,362 Sequences", "Species":"Multi-Species", "Nucleic Acid":"mRNA 3'UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/utrbert-4mer",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7772
+      },
+      "SNMD":{
+         "AUC":0.5002
+      },
+      "SNMR":{
+         "F1":0.2401
+      },
+      "ArchiveII":{
+         "F1":0.7898
+      },
+      "bpRNA":{
+         "F1":0.5693
+      },
+      "RNAStralign":{
+         "F1":0.9203
+      }
+   }
+}

eval-results/RGB/yangheng/Agro-NT.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"InstaDeepAI/agro-nucleotide-transformer-1b",
+      "model_args":"pretrained=InstaDeepAI/agro-nucleotide-transformer-1b,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"985M", "Pretraining Data":"472.5B Tokens", "Species":"48 Edible Plants", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"InstaDeepAI/agro-nucleotide-transformer-1b",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7830
+      },
+      "SNMD":{
+         "AUC":0.4999
+      },
+      "SNMR":{
+         "F1":0.2638
+      },
+      "ArchiveII":{
+         "F1":0.7013
+      },
+      "bpRNA":{
+         "F1":0.4871
+      },
+      "RNAStralign":{
+         "F1":0.7521
+      }
+   }
+}

eval-results/RGB/yangheng/CDSBERT.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"GleghornLab/cdsBERT",
+      "model_args":"pretrained=GleghornLab/cdsBERT,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"420M", "Pretraining Data":"4M Sequences", "Species":"4,069 RNA families", "Nucleic Acid":"CDS"},
+      "model_dtype":"bfloat16",
+      "model_name":"GleghornLab/cdsBERT",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7468
+      },
+      "SNMD":{
+         "AUC":0.5503
+      },
+      "SNMR":{
+         "F1":0.3616
+      },
+      "ArchiveII":{
+         "F1":0.8934
+      },
+      "bpRNA":{
+         "F1":0.7001
+      },
+      "RNAStralign":{
+         "F1":0.9715
+      }
+   }
+}

eval-results/RGB/yangheng/DNABERT-2-117M.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"zhihan1996/DNABERT-2-117M",
+      "model_args":"pretrained=zhihan1996/DNABERT-2-117M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"BPE", "# of Params":"117M", "Pretraining Data":"32.49B Tokens", "Species":"Human + 135 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"zhihan1996/DNABERT-2-117M",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.8158
+      },
+      "SNMD":{
+         "AUC":0.4994
+      },
+      "SNMR":{
+         "F1":0.1586
+      },
+      "ArchiveII":{
+         "F1":0.5982
+      },
+      "bpRNA":{
+         "F1":0.4340
+      },
+      "RNAStralign":{
+         "F1":0.6549
+      }
+   }
+}

eval-results/RGB/yangheng/HyenaDNA.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_args":"pretrained=LongSafari/hyenadna-large-1m-seqlen-hf,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"47M", "Pretraining Data":"~3.2B Tokens", "Species":"Human", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"LongSafari/hyenadna-large-1m-seqlen-hf",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.8056
+      },
+      "SNMD":{
+         "AUC":0.5332
+      },
+      "SNMR":{
+         "F1":0.3980
+      },
+      "ArchiveII":{
+         "F1":0.8423
+      },
+      "bpRNA":{
+         "F1":0.5662
+      },
+      "RNAStralign":{
+         "F1":0.9542
+      }
+   }
+}

eval-results/RGB/yangheng/NT-V2-100M.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_args":"pretrained=InstaDeepAI/nucleotide-transformer-v2-100m-multi-species,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"k-mers", "# of Params":"96M", "Pretraining Data":"300B Tokens", "Species":"Human + 850 Species", "Nucleic Acid":"DNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"InstaDeepAI/nucleotide-transformer-v2-100m-multi-species",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7826
+      },
+      "SNMD":{
+         "AUC":0.5049
+      },
+      "SNMR":{
+         "F1":0.2601
+      },
+      "ArchiveII":{
+         "F1":0.7990
+      },
+      "bpRNA":{
+         "F1":0.5660
+      },
+      "RNAStralign":{
+         "F1":0.9084
+      }
+   }
+}

eval-results/RGB/yangheng/OmniGenome186M.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"yangheng/omnigenome-186M",
+      "model_args":"pretrained=yangheng/omnigenome-186M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"186M", "Pretraining Data":"54.2B Tokens", "Species":"1124 Plant Species", "Nucleic Acid":"mRNA, CDS, UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"yangheng/omnigenome-186M",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7164
+      },
+      "SNMD":{
+         "AUC":0.6381
+      },
+      "SNMR":{
+         "F1":0.4980
+      },
+      "ArchiveII":{
+         "F1":0.9520
+      },
+      "bpRNA":{
+         "F1":0.8248
+      },
+      "RNAStralign":{
+         "F1":0.9912
+      }
+   }
+}

eval-results/RGB/yangheng/SpliceBERT.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"multimolecule/splicebert",
+      "model_args":"pretrained=multimolecule/splicebert,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"19.7M", "Pretraining Data":"65 billion nucleotides", "Species":"Multi-Species", "Nucleic Acid":"mRNA"},
+      "model_dtype":"bfloat16",
+      "model_name":"multimolecule/splicebert",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7340
+      },
+      "SNMD":{
+         "AUC":0.5811
+      },
+      "SNMR":{
+         "F1":0.4644
+      },
+      "ArchiveII":{
+         "F1":0.8905
+      },
+      "bpRNA":{
+         "F1":0.6910
+      },
+      "RNAStralign":{
+         "F1":0.9697
+      }
+   }
+}

eval-results/RGB/yangheng/results_OmniGenome-52M.json

@@ -0,0 +1,39 @@
+{
+   "config":{
+      "model":"yangheng/omnigenome-52M",
+      "model_args":"pretrained=yangheng/omnigenome-52M,revision=main,dtype=bfloat16",
+      "num_fewshot":0,
+      "batch_size":1,
+      "batch_sizes":[
+         
+      ],
+      "device":"cpu",
+      "no_cache":true,
+      "limit":20,
+      "bootstrap_iters":100000,
+      "description_dict":{"Tokenization":"SNT", "# of Params":"52M", "Pretraining Data":"54.2B Tokens", "Species":"1124 Plant Species", "Nucleic Acid":"mRNA, CDS, UTR"},
+      "model_dtype":"bfloat16",
+      "model_name":"yangheng/omnigenome-52M",
+      "model_sha":"main"
+   },
+   "results":{
+      "mRNA":{
+         "RMSE":0.7191
+      },
+      "SNMD":{
+         "AUC":0.6244
+      },
+      "SNMR":{
+         "F1":0.4891
+      },
+      "ArchiveII":{
+         "F1":0.9498
+      },
+      "bpRNA":{
+         "F1":0.8234
+      },
+      "RNAStralign":{
+         "F1":0.9901
+      }
+   }
+}