Upload folder using huggingface_hub

data_download.py

@@ -0,0 +1,23 @@
+from datasets import load_dataset
+from pathlib import Path
+
+datasets = [
+    "demo_human_or_worm",
+    "dummy_mouse_enhancers_ensembl",
+    "human_enhancers_ensembl",
+    "human_nontata_promoters",
+    "demo_coding_vs_intergenomic_seqs",
+    "drosophila_enhancers_stark",
+    "human_enhancers_cohn",
+    "human_ensembl_regulatory",
+    "human_ocr_ensembl",
+]
+
+out_root = Path("hf_raw")
+out_root.mkdir(parents=True, exist_ok=True)
+
+for name in datasets:
+    hf_id = f"katarinagresova/Genomic_Benchmarks_{name}"
+    ds = load_dataset(hf_id)  # downloads to the HF cache
+    ds.save_to_disk(out_root / name)  # optional: persist locally for reuse
+    print(f"downloaded {hf_id}")

data_split.py

@@ -0,0 +1,28 @@
+from datasets import load_from_disk, concatenate_datasets
+from pathlib import Path
+import pandas as pd
+
+src_root = Path("hf_raw")
+dst_root = Path("ft_data")
+seed = 42
+
+for ds_dir in src_root.iterdir():
+    if not ds_dir.is_dir():
+        continue
+    ds = load_from_disk(ds_dir)
+
+    # Combine all available splits, shuffle, then 80/10/10
+    full = concatenate_datasets([ds[s] for s in ds.keys()])
+    full = full.shuffle(seed=seed)
+    split1 = full.train_test_split(test_size=0.2, seed=seed)
+    train = split1["train"]
+    split2 = split1["test"].train_test_split(test_size=0.5, seed=seed)
+    dev, test = split2["train"], split2["test"]
+
+    out = dst_root / ds_dir.name / "split"
+    out.mkdir(parents=True, exist_ok=True)
+    for name, subset in [("train", train), ("dev", dev), ("test", test)]:
+        subset.to_pandas()[["seq", "label"]].rename(
+            columns={"seq": "sequence", "label": "labels"}
+        ).to_csv(out / f"{name}.csv", sep="\t", index=False)
+        print(f"Wrote {out/f'{name}.csv'}")

full_output_multi_epoch/collect_results.py

@@ -0,0 +1,48 @@
+import json
+from pathlib import Path
+import pandas as pd
+
+ROOT = Path(".")  # full_output_multi_epoch
+OUTPUT_DIR = ROOT / "tables"
+OUTPUT_DIR.mkdir(exist_ok=True)
+
+all_rows = []
+
+for task_dir in sorted(p for p in ROOT.iterdir() if p.is_dir() and p.name != "tables"):
+    task_name = task_dir.name
+    task_rows = []
+
+    for model_dir in sorted(p for p in task_dir.iterdir() if p.is_dir()):
+        model_name = model_dir.name
+        results_dir = model_dir / "results"
+        if not results_dir.exists():
+            continue
+
+        # Usually only one run folder inside results/
+        for run_dir in results_dir.iterdir():
+            eval_file = run_dir / "eval_results.json"
+            if not eval_file.exists():
+                continue
+
+            with open(eval_file) as f:
+                metrics = json.load(f)
+
+            row = {
+                "task": task_name,
+                "model": model_name,
+                **metrics
+            }
+
+            task_rows.append(row)
+            all_rows.append(row)
+
+    if task_rows:
+        df_task = pd.DataFrame(task_rows).sort_values("model")
+        df_task.to_csv(OUTPUT_DIR / f"results_{task_name}.csv", index=False)
+        print(f"Saved: results_{task_name}.csv")
+
+# Optional: global table
+if all_rows:
+    df_all = pd.DataFrame(all_rows)
+    df_all.to_csv(OUTPUT_DIR / "results_all_tasks.csv", index=False)
+    print("Saved: results_all_tasks.csv")

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/root/NaN/dna-tokenizer/pretrain/models/base_3072/checkpoint-100000",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 3072
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a6a82cc56bde43624d024a36dc776ca84a353a85d62a95cab3e0f0dc237234e
+size 353632152

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5e6dd1dfc2e9d5f8f1a799a64224eb8fd8aea63758b054765b6a8753036d8c0
+size 707385995

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9af56506fc0b71f6f6e9ee81e8a4836588c94483bc8810d576e0f47b0223ac05
+size 14709

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e32828859e9d069315b3f3e9abbd7af9b79ba7a4ee7041fec91396a6f9e6768
+size 1465

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "cache_dir": null,
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 100,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "trust_remote_code": true,
+  "unk_token": "[UNK]",
+  "use_fast": true
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/trainer_state.json

@@ -0,0 +1,110 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.24,
+  "eval_steps": 200,
+  "global_step": 1400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.32,
+      "eval_accuracy": 0.9009,
+      "eval_f1": 0.9008999752249938,
+      "eval_loss": 0.24717512726783752,
+      "eval_matthews_correlation": 0.8019013074464254,
+      "eval_precision": 0.9009536610343063,
+      "eval_recall": 0.9009476464346753,
+      "eval_runtime": 0.728,
+      "eval_samples_per_second": 13736.193,
+      "eval_steps_per_second": 108.516,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_accuracy": 0.9041,
+      "eval_f1": 0.9040877529651761,
+      "eval_loss": 0.24045702815055847,
+      "eval_matthews_correlation": 0.8087813953002411,
+      "eval_precision": 0.9045580575877361,
+      "eval_recall": 0.9042234069469404,
+      "eval_runtime": 0.7007,
+      "eval_samples_per_second": 14272.015,
+      "eval_steps_per_second": 112.749,
+      "step": 400
+    },
+    {
+      "epoch": 0.96,
+      "eval_accuracy": 0.9037,
+      "eval_f1": 0.9035290138594547,
+      "eval_loss": 0.23689354956150055,
+      "eval_matthews_correlation": 0.8092946059534654,
+      "eval_precision": 0.9058485926765776,
+      "eval_recall": 0.9034495690288824,
+      "eval_runtime": 0.7017,
+      "eval_samples_per_second": 14251.42,
+      "eval_steps_per_second": 112.586,
+      "step": 600
+    },
+    {
+      "epoch": 1.28,
+      "eval_accuracy": 0.9056,
+      "eval_f1": 0.9054084521155339,
+      "eval_loss": 0.23994144797325134,
+      "eval_matthews_correlation": 0.8134456166108831,
+      "eval_precision": 0.9081210458851026,
+      "eval_recall": 0.9053293611386959,
+      "eval_runtime": 0.7009,
+      "eval_samples_per_second": 14267.83,
+      "eval_steps_per_second": 112.716,
+      "step": 800
+    },
+    {
+      "epoch": 1.6,
+      "eval_accuracy": 0.9105,
+      "eval_f1": 0.910486894386207,
+      "eval_loss": 0.22719423472881317,
+      "eval_matthews_correlation": 0.8216385258625271,
+      "eval_precision": 0.911009292886067,
+      "eval_recall": 0.910629320836721,
+      "eval_runtime": 0.7015,
+      "eval_samples_per_second": 14254.403,
+      "eval_steps_per_second": 112.61,
+      "step": 1000
+    },
+    {
+      "epoch": 1.92,
+      "eval_accuracy": 0.9131,
+      "eval_f1": 0.9130232787039119,
+      "eval_loss": 0.22087331116199493,
+      "eval_matthews_correlation": 0.8269895715265106,
+      "eval_precision": 0.9140534915148619,
+      "eval_recall": 0.9129368339048614,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14284.915,
+      "eval_steps_per_second": 112.851,
+      "step": 1200
+    },
+    {
+      "epoch": 2.24,
+      "eval_accuracy": 0.9137,
+      "eval_f1": 0.9136644219716503,
+      "eval_loss": 0.22041191160678864,
+      "eval_matthews_correlation": 0.8276523914394088,
+      "eval_precision": 0.9140498457138171,
+      "eval_recall": 0.91360266653066,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14286.336,
+      "eval_steps_per_second": 112.862,
+      "step": 1400
+    }
+  ],
+  "logging_steps": 100000,
+  "max_steps": 1875,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "total_flos": 4972799101763584.0,
+  "trial_name": null,
+  "trial_params": null
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f558acf951dddcf698771a7d98f4feaca4a4b912d76fa9c7ec16ebe809283c
+size 5329

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/root/NaN/dna-tokenizer/pretrain/models/base_3072/checkpoint-100000",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 3072
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49fce4efe8374b1208584285c2051dcfa80394add10213127dc3ef177a73a08
+size 353632152

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05b46437dee9879cd4392837c38cd80da8cf6894ea365c9440e53b4fa5bf754e
+size 707385995

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49465b41dedc595a1cf05bd2e4f847a88fe0d62e74d64c40d13068f732aaee3
+size 14709

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:778fe40a0c11980abd25e31db8ab18a2f8d910488102d622b91f45279b3ff175
+size 1465

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "cache_dir": null,
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 100,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "trust_remote_code": true,
+  "unk_token": "[UNK]",
+  "use_fast": true
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/trainer_state.json

@@ -0,0 +1,123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.56,
+  "eval_steps": 200,
+  "global_step": 1600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.32,
+      "eval_accuracy": 0.9009,
+      "eval_f1": 0.9008999752249938,
+      "eval_loss": 0.24717512726783752,
+      "eval_matthews_correlation": 0.8019013074464254,
+      "eval_precision": 0.9009536610343063,
+      "eval_recall": 0.9009476464346753,
+      "eval_runtime": 0.728,
+      "eval_samples_per_second": 13736.193,
+      "eval_steps_per_second": 108.516,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_accuracy": 0.9041,
+      "eval_f1": 0.9040877529651761,
+      "eval_loss": 0.24045702815055847,
+      "eval_matthews_correlation": 0.8087813953002411,
+      "eval_precision": 0.9045580575877361,
+      "eval_recall": 0.9042234069469404,
+      "eval_runtime": 0.7007,
+      "eval_samples_per_second": 14272.015,
+      "eval_steps_per_second": 112.749,
+      "step": 400
+    },
+    {
+      "epoch": 0.96,
+      "eval_accuracy": 0.9037,
+      "eval_f1": 0.9035290138594547,
+      "eval_loss": 0.23689354956150055,
+      "eval_matthews_correlation": 0.8092946059534654,
+      "eval_precision": 0.9058485926765776,
+      "eval_recall": 0.9034495690288824,
+      "eval_runtime": 0.7017,
+      "eval_samples_per_second": 14251.42,
+      "eval_steps_per_second": 112.586,
+      "step": 600
+    },
+    {
+      "epoch": 1.28,
+      "eval_accuracy": 0.9056,
+      "eval_f1": 0.9054084521155339,
+      "eval_loss": 0.23994144797325134,
+      "eval_matthews_correlation": 0.8134456166108831,
+      "eval_precision": 0.9081210458851026,
+      "eval_recall": 0.9053293611386959,
+      "eval_runtime": 0.7009,
+      "eval_samples_per_second": 14267.83,
+      "eval_steps_per_second": 112.716,
+      "step": 800
+    },
+    {
+      "epoch": 1.6,
+      "eval_accuracy": 0.9105,
+      "eval_f1": 0.910486894386207,
+      "eval_loss": 0.22719423472881317,
+      "eval_matthews_correlation": 0.8216385258625271,
+      "eval_precision": 0.911009292886067,
+      "eval_recall": 0.910629320836721,
+      "eval_runtime": 0.7015,
+      "eval_samples_per_second": 14254.403,
+      "eval_steps_per_second": 112.61,
+      "step": 1000
+    },
+    {
+      "epoch": 1.92,
+      "eval_accuracy": 0.9131,
+      "eval_f1": 0.9130232787039119,
+      "eval_loss": 0.22087331116199493,
+      "eval_matthews_correlation": 0.8269895715265106,
+      "eval_precision": 0.9140534915148619,
+      "eval_recall": 0.9129368339048614,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14284.915,
+      "eval_steps_per_second": 112.851,
+      "step": 1200
+    },
+    {
+      "epoch": 2.24,
+      "eval_accuracy": 0.9137,
+      "eval_f1": 0.9136644219716503,
+      "eval_loss": 0.22041191160678864,
+      "eval_matthews_correlation": 0.8276523914394088,
+      "eval_precision": 0.9140498457138171,
+      "eval_recall": 0.91360266653066,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14286.336,
+      "eval_steps_per_second": 112.862,
+      "step": 1400
+    },
+    {
+      "epoch": 2.56,
+      "eval_accuracy": 0.9143,
+      "eval_f1": 0.9142909071223366,
+      "eval_loss": 0.2254679948091507,
+      "eval_matthews_correlation": 0.8286017317454041,
+      "eval_precision": 0.9143290342981838,
+      "eval_recall": 0.9142726993622687,
+      "eval_runtime": 0.6994,
+      "eval_samples_per_second": 14297.498,
+      "eval_steps_per_second": 112.95,
+      "step": 1600
+    }
+  ],
+  "logging_steps": 100000,
+  "max_steps": 1875,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "total_flos": 5683198973444096.0,
+  "trial_name": null,
+  "trial_params": null
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1600/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f558acf951dddcf698771a7d98f4feaca4a4b912d76fa9c7ec16ebe809283c
+size 5329

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/root/NaN/dna-tokenizer/pretrain/models/base_3072/checkpoint-100000",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 3072
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4bcd93b1e022098504062a855e3a51a7d0612b9d1095e3f104a4fc2db387e3
+size 353632152

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32defe438c9f8f46bbe218a6fa6185cb17ffcac4156f82723cb71898ea5a84bc
+size 707385995

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b352d03af2b56a998f314da069a38bc56dcbf7905c5abcaa5b1ad04b523e04
+size 14709

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:084228aa33f0ae6a0339955a663a055ad6d466f21dead5e3294bff0e025f5dd1
+size 1465

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "cache_dir": null,
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 100,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "trust_remote_code": true,
+  "unk_token": "[UNK]",
+  "use_fast": true
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/trainer_state.json

@@ -0,0 +1,136 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.88,
+  "eval_steps": 200,
+  "global_step": 1800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.32,
+      "eval_accuracy": 0.9009,
+      "eval_f1": 0.9008999752249938,
+      "eval_loss": 0.24717512726783752,
+      "eval_matthews_correlation": 0.8019013074464254,
+      "eval_precision": 0.9009536610343063,
+      "eval_recall": 0.9009476464346753,
+      "eval_runtime": 0.728,
+      "eval_samples_per_second": 13736.193,
+      "eval_steps_per_second": 108.516,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_accuracy": 0.9041,
+      "eval_f1": 0.9040877529651761,
+      "eval_loss": 0.24045702815055847,
+      "eval_matthews_correlation": 0.8087813953002411,
+      "eval_precision": 0.9045580575877361,
+      "eval_recall": 0.9042234069469404,
+      "eval_runtime": 0.7007,
+      "eval_samples_per_second": 14272.015,
+      "eval_steps_per_second": 112.749,
+      "step": 400
+    },
+    {
+      "epoch": 0.96,
+      "eval_accuracy": 0.9037,
+      "eval_f1": 0.9035290138594547,
+      "eval_loss": 0.23689354956150055,
+      "eval_matthews_correlation": 0.8092946059534654,
+      "eval_precision": 0.9058485926765776,
+      "eval_recall": 0.9034495690288824,
+      "eval_runtime": 0.7017,
+      "eval_samples_per_second": 14251.42,
+      "eval_steps_per_second": 112.586,
+      "step": 600
+    },
+    {
+      "epoch": 1.28,
+      "eval_accuracy": 0.9056,
+      "eval_f1": 0.9054084521155339,
+      "eval_loss": 0.23994144797325134,
+      "eval_matthews_correlation": 0.8134456166108831,
+      "eval_precision": 0.9081210458851026,
+      "eval_recall": 0.9053293611386959,
+      "eval_runtime": 0.7009,
+      "eval_samples_per_second": 14267.83,
+      "eval_steps_per_second": 112.716,
+      "step": 800
+    },
+    {
+      "epoch": 1.6,
+      "eval_accuracy": 0.9105,
+      "eval_f1": 0.910486894386207,
+      "eval_loss": 0.22719423472881317,
+      "eval_matthews_correlation": 0.8216385258625271,
+      "eval_precision": 0.911009292886067,
+      "eval_recall": 0.910629320836721,
+      "eval_runtime": 0.7015,
+      "eval_samples_per_second": 14254.403,
+      "eval_steps_per_second": 112.61,
+      "step": 1000
+    },
+    {
+      "epoch": 1.92,
+      "eval_accuracy": 0.9131,
+      "eval_f1": 0.9130232787039119,
+      "eval_loss": 0.22087331116199493,
+      "eval_matthews_correlation": 0.8269895715265106,
+      "eval_precision": 0.9140534915148619,
+      "eval_recall": 0.9129368339048614,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14284.915,
+      "eval_steps_per_second": 112.851,
+      "step": 1200
+    },
+    {
+      "epoch": 2.24,
+      "eval_accuracy": 0.9137,
+      "eval_f1": 0.9136644219716503,
+      "eval_loss": 0.22041191160678864,
+      "eval_matthews_correlation": 0.8276523914394088,
+      "eval_precision": 0.9140498457138171,
+      "eval_recall": 0.91360266653066,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14286.336,
+      "eval_steps_per_second": 112.862,
+      "step": 1400
+    },
+    {
+      "epoch": 2.56,
+      "eval_accuracy": 0.9143,
+      "eval_f1": 0.9142909071223366,
+      "eval_loss": 0.2254679948091507,
+      "eval_matthews_correlation": 0.8286017317454041,
+      "eval_precision": 0.9143290342981838,
+      "eval_recall": 0.9142726993622687,
+      "eval_runtime": 0.6994,
+      "eval_samples_per_second": 14297.498,
+      "eval_steps_per_second": 112.95,
+      "step": 1600
+    },
+    {
+      "epoch": 2.88,
+      "eval_accuracy": 0.9147,
+      "eval_f1": 0.9146887175829004,
+      "eval_loss": 0.22356006503105164,
+      "eval_matthews_correlation": 0.8294144901528736,
+      "eval_precision": 0.9147501760450676,
+      "eval_recall": 0.914664318551609,
+      "eval_runtime": 0.7,
+      "eval_samples_per_second": 14286.322,
+      "eval_steps_per_second": 112.862,
+      "step": 1800
+    }
+  ],
+  "logging_steps": 100000,
+  "max_steps": 1875,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "total_flos": 6393598845124608.0,
+  "trial_name": null,
+  "trial_params": null
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/checkpoint-1800/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f558acf951dddcf698771a7d98f4feaca4a4b912d76fa9c7ec16ebe809283c
+size 5329

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_3072/results/base_3072_hg38_BPE_3e-5_demo_coding_vs_intergenomic_seqs_seed42/eval_results.json

@@ -0,0 +1 @@
+{"eval_loss": 0.23370100557804108, "eval_accuracy": 0.9143, "eval_f1": 0.9142994643716523, "eval_matthews_correlation": 0.8286761912329407, "eval_precision": 0.9143515624765606, "eval_recall": 0.9143246291940671, "eval_runtime": 0.6978, "eval_samples_per_second": 14330.746, "eval_steps_per_second": 113.213, "epoch": 3.0}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/root/NaN/dna-tokenizer/pretrain/models/base_4096/checkpoint-100000",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 4096
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:238ba1e2b02bba89f3c53efa289d3b86bf72e3e2221144b3b08c0ba88aa3f592
+size 356777880

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818f86f31a65892b5378bc0961089a1e5b20e82e36b9753753b4202848248115
+size 713677451

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881f77abcbcf9e83c940223f5ca9133f12d2be418768a67a53945677a75907b9
+size 14709

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e32828859e9d069315b3f3e9abbd7af9b79ba7a4ee7041fec91396a6f9e6768
+size 1465

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "cache_dir": null,
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 100,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "trust_remote_code": true,
+  "unk_token": "[UNK]",
+  "use_fast": true
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/trainer_state.json

@@ -0,0 +1,110 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.24,
+  "eval_steps": 200,
+  "global_step": 1400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.32,
+      "eval_accuracy": 0.9004,
+      "eval_f1": 0.900398243025007,
+      "eval_loss": 0.2527843713760376,
+      "eval_matthews_correlation": 0.8010224138324217,
+      "eval_precision": 0.9005488941757427,
+      "eval_recall": 0.9004735232026368,
+      "eval_runtime": 0.7219,
+      "eval_samples_per_second": 13852.527,
+      "eval_steps_per_second": 109.435,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_accuracy": 0.9033,
+      "eval_f1": 0.903172411254586,
+      "eval_loss": 0.23981137573719025,
+      "eval_matthews_correlation": 0.8079007693175992,
+      "eval_precision": 0.9048124537208442,
+      "eval_recall": 0.9030901514174194,
+      "eval_runtime": 0.695,
+      "eval_samples_per_second": 14387.745,
+      "eval_steps_per_second": 113.663,
+      "step": 400
+    },
+    {
+      "epoch": 0.96,
+      "eval_accuracy": 0.9036,
+      "eval_f1": 0.9035254406015145,
+      "eval_loss": 0.2367754429578781,
+      "eval_matthews_correlation": 0.8078341688032824,
+      "eval_precision": 0.9043850412921304,
+      "eval_recall": 0.9034496690337827,
+      "eval_runtime": 0.6971,
+      "eval_samples_per_second": 14344.287,
+      "eval_steps_per_second": 113.32,
+      "step": 600
+    },
+    {
+      "epoch": 1.28,
+      "eval_accuracy": 0.9109,
+      "eval_f1": 0.9108989085116292,
+      "eval_loss": 0.2278522253036499,
+      "eval_matthews_correlation": 0.8218201348399584,
+      "eval_precision": 0.9109,
+      "eval_recall": 0.9109201350866192,
+      "eval_runtime": 0.6987,
+      "eval_samples_per_second": 14312.505,
+      "eval_steps_per_second": 113.069,
+      "step": 800
+    },
+    {
+      "epoch": 1.6,
+      "eval_accuracy": 0.9115,
+      "eval_f1": 0.9114986538945258,
+      "eval_loss": 0.2427944540977478,
+      "eval_matthews_correlation": 0.8232139301065716,
+      "eval_precision": 0.9116419660562449,
+      "eval_recall": 0.9115719670263842,
+      "eval_runtime": 0.6945,
+      "eval_samples_per_second": 14397.949,
+      "eval_steps_per_second": 113.744,
+      "step": 1000
+    },
+    {
+      "epoch": 1.92,
+      "eval_accuracy": 0.9083,
+      "eval_f1": 0.9082222817103751,
+      "eval_loss": 0.22559477388858795,
+      "eval_matthews_correlation": 0.8173335971525737,
+      "eval_precision": 0.9091934761461085,
+      "eval_recall": 0.9081407988991461,
+      "eval_runtime": 0.6941,
+      "eval_samples_per_second": 14407.148,
+      "eval_steps_per_second": 113.816,
+      "step": 1200
+    },
+    {
+      "epoch": 2.24,
+      "eval_accuracy": 0.9094,
+      "eval_f1": 0.909320550293345,
+      "eval_loss": 0.23093822598457336,
+      "eval_matthews_correlation": 0.8195720153678601,
+      "eval_precision": 0.910335398306722,
+      "eval_recall": 0.9092373526302788,
+      "eval_runtime": 0.6996,
+      "eval_samples_per_second": 14293.8,
+      "eval_steps_per_second": 112.921,
+      "step": 1400
+    }
+  ],
+  "logging_steps": 100000,
+  "max_steps": 1875,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "total_flos": 4880710238208000.0,
+  "trial_name": null,
+  "trial_params": null
+}

full_output_multi_epoch/demo_coding_vs_intergenomic_seqs/base_4096/checkpoint-1400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b30702f12d1c6578823ea18d25e59c6b65c7d877145f8e9aed3cb60df7adec9
+size 5329