Training in progress, epoch 1

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "uw-madison/yoso-4096",
+  "architectures": [
+    "YosoForMultipleChoice"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "conv_window": null,
+  "eos_token_id": 2,
+  "hash_code_len": 9,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "lsh_backward": true,
+  "max_position_embeddings": 4096,
+  "model_type": "yoso",
+  "num_attention_heads": 12,
+  "num_hash": 64,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "AlbertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.39.0.dev0",
+  "type_vocab_size": 1,
+  "use_expectation": true,
+  "use_fast_hash": true,
+  "vocab_size": 50265
+}

log.txt

@@ -0,0 +1,9 @@
+[1/4] /home/ludaze/usr/local/cuda/cuda-11.8/bin/nvcc --generate-dependencies-with-compile --dependency-output fast_lsh_cumulation_cuda.cuda.o.d -DTORCH_EXTENSION_NAME=fast_lsh_cumulation -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/TH -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/THC -isystem /home/ludaze/usr/local/cuda/cuda-11.8/include -isystem /home/ludaze/.conda/envs/axolotl/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 --compiler-options '-fPIC' -std=c++17 -c /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/transformers/kernels/yoso/fast_lsh_cumulation_cuda.cu -o fast_lsh_cumulation_cuda.cuda.o 
+[2/4] c++ -MMD -MF fast_lsh_cumulation_torch.o.d -DTORCH_EXTENSION_NAME=fast_lsh_cumulation -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/TH -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/THC -isystem /home/ludaze/usr/local/cuda/cuda-11.8/include -isystem /home/ludaze/.conda/envs/axolotl/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++17 -c /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/transformers/kernels/yoso/fast_lsh_cumulation_torch.cpp -o fast_lsh_cumulation_torch.o 
+[3/4] /home/ludaze/usr/local/cuda/cuda-11.8/bin/nvcc --generate-dependencies-with-compile --dependency-output fast_lsh_cumulation.cuda.o.d -DTORCH_EXTENSION_NAME=fast_lsh_cumulation -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/TH -isystem /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/include/THC -isystem /home/ludaze/usr/local/cuda/cuda-11.8/include -isystem /home/ludaze/.conda/envs/axolotl/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 --compiler-options '-fPIC' -std=c++17 -c /home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/transformers/kernels/yoso/fast_lsh_cumulation.cu -o fast_lsh_cumulation.cuda.o 
+[4/4] c++ fast_lsh_cumulation_torch.o fast_lsh_cumulation.cuda.o fast_lsh_cumulation_cuda.cuda.o -shared -L/home/ludaze/.conda/envs/axolotl/lib/python3.10/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda -ltorch -ltorch_python -L/home/ludaze/usr/local/cuda/cuda-11.8/lib64 -lcudart -o fast_lsh_cumulation.so
+{'loss': 1.3875, 'grad_norm': 0.290032297372818, 'learning_rate': 4.708318749270797e-05, 'epoch': 0.18}
+{'loss': 1.387, 'grad_norm': 0.24705088138580322, 'learning_rate': 4.416637498541594e-05, 'epoch': 0.35}
+{'loss': 1.387, 'grad_norm': 0.2913823425769806, 'learning_rate': 4.124956247812391e-05, 'epoch': 0.53}
+{'loss': 1.3867, 'grad_norm': 0.2294648140668869, 'learning_rate': 3.8332749970831875e-05, 'epoch': 0.7}
+{'loss': 1.3868, 'grad_norm': 0.25805965065956116, 'learning_rate': 3.5415937463539846e-05, 'epoch': 0.88}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7c1e52f9a3606b166333f65d696a3aa6724be199025da91d8b3ecc092550652
+size 509619172

runs/Mar13_10-51-07_cehsc-app-001.ad.unlv.edu/events.out.tfevents.1710352269.cehsc-app-001.ad.unlv.edu.2985100.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb81185d7f756e1bec10b8d2accf8c710d80db11def9d501623caf2a3945903d
+size 6151

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc7ea0d9c64fc59dd649552dd9900385ad550a2f37beed7e40b9a68601284e2c
+size 4920