diff --git "a/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/memory-bench/Benchmark-V5headsize2x.ipynb" "b/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/memory-bench/Benchmark-V5headsize2x.ipynb" deleted file mode 100644--- "a/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/memory-bench/Benchmark-V5headsize2x.ipynb" +++ /dev/null @@ -1,5195 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "d75336f4", - "metadata": { - "papermill": { - "duration": 0.003004, - "end_time": "2023-08-28T19:46:29.311058", - "exception": false, - "start_time": "2023-08-28T19:46:29.308054", - "status": "completed" - }, - "tags": [] - }, - "source": [ - "## Custom Experimental Memory benchmarking\n", - "\n", - "The following is meant to observe the memory performance in a more verbose CSV logging mode, for the baseline raven models" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "d5383b16", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T19:46:29.319989Z", - "iopub.status.busy": "2023-08-28T19:46:29.319499Z", - "iopub.status.idle": "2023-08-28T19:47:04.154153Z", - "shell.execute_reply": "2023-08-28T19:47:04.153258Z" - }, - "papermill": { - "duration": 34.841164, - "end_time": "2023-08-28T19:47:04.156753", - "exception": false, - "start_time": "2023-08-28T19:46:29.315589", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2023-08-28 19:46:29-- https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-headsize2x/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\r\n", - "Resolving huggingface.co (huggingface.co)... 18.154.227.7, 18.154.227.67, 18.154.227.87, ...\r\n", - "Connecting to huggingface.co (huggingface.co)|18.154.227.7|:443... connected.\r\n", - "HTTP request sent, awaiting response... " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "302 Found\r\n", - "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/fae9799e56fd469c6c1694dae6ae77f78130c06d340e945a3dc8e8ea2bd4e5bb?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%22%3B&Expires=1693511189&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5MzUxMTE4OX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkL2ZhZTk3OTllNTZmZDQ2OWM2YzE2OTRkYWU2YWU3N2Y3ODEzMGMwNmQzNDBlOTQ1YTNkYzhlOGVhMmJkNGU1YmI%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=rM%7EHHa4Yu7UYwTUaL0xc04UGrh%7EU8JD89eIsLA8BGvLhoz427ftItG7QPZK9gZxdzehxjOm2VSccDl27cpCVLClcTCdMQBxqt1kdhjheVfDNzPuYTg7QOpteaiq5Q3PoZBIY8DuYGgFRtBHFilPh3KxD-bwFDGLQOieIkFXuhQ8AaEzC8CWpv1bjstpugZWcOvPkT4nLgsK2jHyUufEHfMRldwGp9rvaJC96F6PUFOInUh-Sf4vzY2jpp04htm4nakH%7EbqZdQHh3lkNn7az9VGBkFib4oHdSI4mj2stDA5alnTA32-eUuWcG4-E1IXW7fHAKbXTOgb1ktghAWbOeTQ__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n", - "--2023-08-28 19:46:29-- https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/fae9799e56fd469c6c1694dae6ae77f78130c06d340e945a3dc8e8ea2bd4e5bb?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%22%3B&Expires=1693511189&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5MzUxMTE4OX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkL2ZhZTk3OTllNTZmZDQ2OWM2YzE2OTRkYWU2YWU3N2Y3ODEzMGMwNmQzNDBlOTQ1YTNkYzhlOGVhMmJkNGU1YmI%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=rM%7EHHa4Yu7UYwTUaL0xc04UGrh%7EU8JD89eIsLA8BGvLhoz427ftItG7QPZK9gZxdzehxjOm2VSccDl27cpCVLClcTCdMQBxqt1kdhjheVfDNzPuYTg7QOpteaiq5Q3PoZBIY8DuYGgFRtBHFilPh3KxD-bwFDGLQOieIkFXuhQ8AaEzC8CWpv1bjstpugZWcOvPkT4nLgsK2jHyUufEHfMRldwGp9rvaJC96F6PUFOInUh-Sf4vzY2jpp04htm4nakH%7EbqZdQHh3lkNn7az9VGBkFib4oHdSI4mj2stDA5alnTA32-eUuWcG4-E1IXW7fHAKbXTOgb1ktghAWbOeTQ__&Key-Pair-Id=KVTP0A1DKRTAX\r\n", - "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 108.138.64.111, 108.138.64.36, 108.138.64.49, ...\r\n", - "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|108.138.64.111|:443... connected.\r\n", - "HTTP request sent, awaiting response... " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "200 OK\r\n", - "Length: 3441599109 (3.2G) [binary/octet-stream]\r\n", - "Saving to: ‘v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth’\r\n", - "\r\n", - "\r", - " v5-hs2x-L 0%[ ] 0 --.-KB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6 1%[ ] 40.63M 203MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6- 2%[ ] 95.45M 239MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D 4%[ ] 142.13M 237MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D4 6%[> ] 204.10M 255MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D40 8%[> ] 276.32M 276MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D409 10%[=> ] 336.37M 280MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D4096 12%[=> ] 405.00M 289MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D4096- 13%[=> ] 458.17M 286MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D4096-E 15%[==> ] 507.75M 282MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "v5-hs2x-L6-D4096-E0 16%[==> ] 531.80M 265MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "5-hs2x-L6-D4096-E0_ 17%[==> ] 560.12M 254MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-hs2x-L6-D4096-E0_1 18%[==> ] 608.44M 253MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "hs2x-L6-D4096-E0_1- 20%[===> ] 669.42M 257MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "s2x-L6-D4096-E0_1-m 21%[===> ] 702.78M 251MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "2x-L6-D4096-E0_1-me 23%[===> ] 756.78M 252MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "x-L6-D4096-E0_1-mem 24%[===> ] 814.39M 256MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-L6-D4096-E0_1-mem- 26%[====> ] 879.63M 261MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "L6-D4096-E0_1-mem-c 28%[====> ] 927.40M 261MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "6-D4096-E0_1-mem-ct 29%[====> ] 983.76M 260MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-D4096-E0_1-mem-ctx 31%[=====> ] 1.01G 255MB/s eta 9s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "D4096-E0_1-mem-ctx- 32%[=====> ] 1.05G 244MB/s eta 9s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "4096-E0_1-mem-ctx-8 33%[=====> ] 1.08G 235MB/s eta 9s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "096-E0_1-mem-ctx-8k 35%[======> ] 1.14G 236MB/s eta 9s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "96-E0_1-mem-ctx-8k. 37%[======> ] 1.22G 245MB/s eta 9s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "6-E0_1-mem-ctx-8k.p 40%[=======> ] 1.29G 257MB/s eta 7s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-E0_1-mem-ctx-8k.pt 41%[=======> ] 1.34G 268MB/s eta 7s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "E0_1-mem-ctx-8k.pth 43%[=======> ] 1.40G 275MB/s eta 7s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "0_1-mem-ctx-8k.pth 45%[========> ] 1.45G 271MB/s eta 7s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "_1-mem-ctx-8k.pth 47%[========> ] 1.51G 276MB/s eta 7s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "1-mem-ctx-8k.pth 49%[========> ] 1.58G 285MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-mem-ctx-8k.pth 51%[=========> ] 1.65G 290MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "mem-ctx-8k.pth 53%[=========> ] 1.71G 292MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "em-ctx-8k.pth 55%[==========> ] 1.78G 294MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "m-ctx-8k.pth 57%[==========> ] 1.84G 298MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-ctx-8k.pth 59%[==========> ] 1.91G 307MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "ctx-8k.pth 61%[===========> ] 1.98G 312MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "tx-8k.pth 63%[===========> ] 2.05G 331MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "x-8k.pth 64%[===========> ] 2.08G 321MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-8k.pth 66%[============> ] 2.13G 315MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "8k.pth 67%[============> ] 2.18G 300MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "k.pth 69%[============> ] 2.22G 297MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - ".pth 71%[=============> ] 2.29G 300MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "pth 73%[=============> ] 2.36G 306MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "th 75%[==============> ] 2.41G 299MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "h 76%[==============> ] 2.46G 297MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " 79%[==============> ] 2.54G 300MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v 80%[===============> ] 2.58G 291MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5 82%[===============> ] 2.66G 293MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5- 84%[===============> ] 2.72G 291MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-h 87%[================> ] 2.79G 295MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs 88%[================> ] 2.84G 288MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2 90%[=================> ] 2.89G 284MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x 92%[=================> ] 2.97G 296MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x- 93%[=================> ] 2.99G 287MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L 95%[==================> ] 3.06G 289MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6 97%[==================> ] 3.12G 300MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6- 99%[==================> ] 3.19G 303MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "v5-hs2x-L6-D4096-E0 100%[===================>] 3.21G 304MB/s in 12s \r\n", - "\r\n", - "2023-08-28 19:46:41 (285 MB/s) - ‘v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth’ saved [3441599109/3441599109]\r\n", - "\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2023-08-28 19:46:41-- https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-headsize2x/v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth\r\n", - "Resolving huggingface.co (huggingface.co)... 18.154.227.7, 18.154.227.87, 18.154.227.69, ...\r\n", - "Connecting to huggingface.co (huggingface.co)|18.154.227.7|:443... connected.\r\n", - "HTTP request sent, awaiting response... 302 Found\r\n", - "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/c8e3cb04e7c511cb2a2756ba15243c0ef7c725d520fcbe4489f86ea6cb59ccbd?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth%22%3B&Expires=1693511201&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5MzUxMTIwMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkL2M4ZTNjYjA0ZTdjNTExY2IyYTI3NTZiYTE1MjQzYzBlZjdjNzI1ZDUyMGZjYmU0NDg5Zjg2ZWE2Y2I1OWNjYmQ%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=GleIBxLYsSmY1UDcS0YISvtXWBXA7jeAaL52F1ufgoi4VHZcWYZAP295z24p9veFq8BlZgSjiWup8lLhpUx8ohzq8K-NJ6OfmMyvQKFurUv28nrS3UxPSSFKi2Ciq243qDJTq9js-QEofqJ9hx-xSS49AhCseWvkf1C0EGKwkjRr4FAIqBMdpMKNW3Jb3PgDIt6U8hD5uP0yK98mOREsf6%7EvFQAOlGL2CBuct88Cx41TRzolSiwstPKtSUcz4xpL6%7EbVoCncXmIqQqV5IUBrKH00YDJZfcxYxMXKufxzGSNQI6YpOdDdnLv8M6xqT663mjWhH6nwT7XzHJ1hqFtEvA__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n", - "--2023-08-28 19:46:41-- https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/c8e3cb04e7c511cb2a2756ba15243c0ef7c725d520fcbe4489f86ea6cb59ccbd?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth%22%3B&Expires=1693511201&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5MzUxMTIwMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkL2M4ZTNjYjA0ZTdjNTExY2IyYTI3NTZiYTE1MjQzYzBlZjdjNzI1ZDUyMGZjYmU0NDg5Zjg2ZWE2Y2I1OWNjYmQ%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=GleIBxLYsSmY1UDcS0YISvtXWBXA7jeAaL52F1ufgoi4VHZcWYZAP295z24p9veFq8BlZgSjiWup8lLhpUx8ohzq8K-NJ6OfmMyvQKFurUv28nrS3UxPSSFKi2Ciq243qDJTq9js-QEofqJ9hx-xSS49AhCseWvkf1C0EGKwkjRr4FAIqBMdpMKNW3Jb3PgDIt6U8hD5uP0yK98mOREsf6%7EvFQAOlGL2CBuct88Cx41TRzolSiwstPKtSUcz4xpL6%7EbVoCncXmIqQqV5IUBrKH00YDJZfcxYxMXKufxzGSNQI6YpOdDdnLv8M6xqT663mjWhH6nwT7XzHJ1hqFtEvA__&Key-Pair-Id=KVTP0A1DKRTAX\r\n", - "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 108.138.64.121, 108.138.64.49, 108.138.64.36, ...\r\n", - "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|108.138.64.121|:443... connected.\r\n", - "HTTP request sent, awaiting response... " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "200 OK\r\n", - "Length: 1066508165 (1017M) [binary/octet-stream]\r\n", - "Saving to: ‘v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth’\r\n", - "\r\n", - "\r", - " v5-hs2x-L 0%[ ] 0 --.-KB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6 1%[ ] 14.74M 65.5MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6- 2%[ ] 30.00M 68.3MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D 3%[ ] 33.98M 53.1MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2 4%[ ] 45.26M 51.0MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D20 5%[> ] 54.71M 47.9MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D204 6%[> ] 61.03M 44.7MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2048 7%[> ] 75.78M 47.8MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2048- 8%[> ] 91.03M 49.8MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2048-E 9%[> ] 97.96M 48.3MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "v5-hs2x-L6-D2048-E0 10%[=> ] 106.29M 44.9MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "5-hs2x-L6-D2048-E0_ 10%[=> ] 110.99M 42.5MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-hs2x-L6-D2048-E0_1 12%[=> ] 122.07M 42.6MB/s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "hs2x-L6-D2048-E0_1- 13%[=> ] 137.33M 43.7MB/s eta 20s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "s2x-L6-D2048-E0_1-m 15%[==> ] 152.59M 43.8MB/s eta 20s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "2x-L6-D2048-E0_1-me 16%[==> ] 167.33M 45.2MB/s eta 20s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "x-L6-D2048-E0_1-mem 17%[==> ] 179.02M 45.8MB/s eta 20s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-L6-D2048-E0_1-mem- 18%[==> ] 183.10M 43.1MB/s eta 19s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "L6-D2048-E0_1-mem-c 19%[==> ] 198.36M 43.2MB/s eta 19s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "6-D2048-E0_1-mem-ct 21%[===> ] 213.62M 43.1MB/s eta 19s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-D2048-E0_1-mem-ctx 22%[===> ] 228.87M 40.8MB/s eta 18s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "D2048-E0_1-mem-ctx- 24%[===> ] 244.13M 41.8MB/s eta 18s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "2048-E0_1-mem-ctx-8 25%[====> ] 258.89M 44.6MB/s eta 18s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "048-E0_1-mem-ctx-8k 25%[====> ] 261.46M 42.0MB/s eta 18s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "48-E0_1-mem-ctx-8k. 27%[====> ] 274.66M 42.0MB/s eta 17s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "8-E0_1-mem-ctx-8k.p 28%[====> ] 288.09M 42.3MB/s eta 17s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-E0_1-mem-ctx-8k.pt 28%[====> ] 289.92M 42.1MB/s eta 17s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "E0_1-mem-ctx-8k.pth 30%[=====> ] 305.95M 42.4MB/s eta 17s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "0_1-mem-ctx-8k.pth 31%[=====> ] 320.43M 42.5MB/s eta 16s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "_1-mem-ctx-8k.pth 32%[=====> ] 335.18M 42.2MB/s eta 16s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "1-mem-ctx-8k.pth 34%[=====> ] 350.44M 42.8MB/s eta 16s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-mem-ctx-8k.pth 36%[======> ] 366.20M 44.6MB/s eta 16s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "mem-ctx-8k.pth 38%[======> ] 394.69M 47.4MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "em-ctx-8k.pth 39%[======> ] 397.08M 45.4MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "m-ctx-8k.pth 40%[=======> ] 407.66M 46.7MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-ctx-8k.pth 40%[=======> ] 411.99M 45.4MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "ctx-8k.pth 42%[=======> ] 427.25M 44.1MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "tx-8k.pth 43%[=======> ] 442.51M 44.6MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "x-8k.pth 44%[=======> ] 457.24M 47.4MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-8k.pth 46%[========> ] 468.82M 46.4MB/s eta 13s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "8k.pth 46%[========> ] 473.02M 46.5MB/s eta 12s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "k.pth 48%[========> ] 488.28M 47.2MB/s eta 12s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - ".pth 49%[========> ] 503.67M 50.4MB/s eta 12s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "pth 51%[=========> ] 518.80M 48.6MB/s eta 12s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "th 52%[=========> ] 534.05M 49.2MB/s eta 11s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "h 54%[=========> ] 549.31M 48.6MB/s eta 11s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " 55%[==========> ] 564.06M 50.8MB/s eta 11s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v 56%[==========> ] 571.01M 48.9MB/s eta 11s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5 57%[==========> ] 579.83M 47.1MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5- 58%[==========> ] 595.09M 46.0MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-h 60%[===========> ] 610.35M 46.8MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs 61%[===========> ] 625.61M 47.9MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2 62%[===========> ] 634.36M 49.9MB/s eta 10s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x 63%[===========> ] 643.80M 48.4MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x- 63%[===========> ] 650.43M 47.0MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L 64%[===========> ] 655.80M 47.2MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6 65%[============> ] 661.61M 48.1MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6- 65%[============> ] 667.93M 47.2MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D 66%[============> ] 674.24M 44.2MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2 66%[============> ] 680.36M 43.5MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D20 67%[============> ] 688.05M 41.0MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D204 69%[============> ] 702.19M 44.0MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2048 71%[=============> ] 731.91M 49.5MB/s eta 8s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2048- 73%[=============> ] 747.17M 49.1MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - " v5-hs2x-L6-D2048-E 75%[==============> ] 762.94M 49.6MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "v5-hs2x-L6-D2048-E0 76%[==============> ] 776.37M 49.0MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "5-hs2x-L6-D2048-E0_ 77%[==============> ] 784.51M 46.4MB/s eta 6s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-hs2x-L6-D2048-E0_1 78%[==============> ] 795.10M 46.8MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "hs2x-L6-D2048-E0_1- 78%[==============> ] 802.39M 44.8MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "s2x-L6-D2048-E0_1-m 80%[===============> ] 819.31M 47.6MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "2x-L6-D2048-E0_1-me 81%[===============> ] 823.97M 46.6MB/s eta 5s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "x-L6-D2048-E0_1-mem 82%[===============> ] 839.23M 47.8MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-L6-D2048-E0_1-mem- 84%[===============> ] 854.49M 48.4MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "L6-D2048-E0_1-mem-c 85%[================> ] 869.75M 48.9MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "6-D2048-E0_1-mem-ct 87%[================> ] 885.01M 50.3MB/s eta 4s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-D2048-E0_1-mem-ctx 88%[================> ] 900.27M 50.9MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "D2048-E0_1-mem-ctx- 90%[=================> ] 915.53M 51.0MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "2048-E0_1-mem-ctx-8 91%[=================> ] 930.78M 45.3MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "048-E0_1-mem-ctx-8k 93%[=================> ] 946.04M 45.3MB/s eta 3s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "48-E0_1-mem-ctx-8k. 94%[=================> ] 959.48M 46.7MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "8-E0_1-mem-ctx-8k.p 95%[==================> ] 976.05M 48.0MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "-E0_1-mem-ctx-8k.pt 96%[==================> ] 976.55M 45.1MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "E0_1-mem-ctx-8k.pth 97%[==================> ] 991.82M 45.0MB/s eta 1s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "0_1-mem-ctx-8k.pth 99%[==================> ] 1007M 46.2MB/s eta 0s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "_1-mem-ctx-8k.pth 99%[==================> ] 1009M 45.2MB/s eta 0s " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "v5-hs2x-L6-D2048-E0 100%[===================>] 1017M 46.2MB/s in 22s \r\n", - "\r\n", - "2023-08-28 19:47:04 (45.9 MB/s) - ‘v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth’ saved [1066508165/1066508165]\r\n", - "\r\n" - ] - } - ], - "source": [ - "# Lets download the custom models\n", - "!mkdir -p ../../../model/\n", - "!cd ../../../model/ && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-headsize2x/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\"\n", - "!cd ../../../model/ && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-headsize2x/v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "f9464dc8", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T19:47:04.180963Z", - "iopub.status.busy": "2023-08-28T19:47:04.180147Z", - "iopub.status.idle": "2023-08-28T19:47:04.435465Z", - "shell.execute_reply": "2023-08-28T19:47:04.434302Z" - }, - "papermill": { - "duration": 0.270186, - "end_time": "2023-08-28T19:47:04.437768", - "exception": false, - "start_time": "2023-08-28T19:47:04.167582", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "total 4.2G\r\n", - "-rw-r--r-- 1 root root 1018M Aug 26 02:06 v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth\r\n", - "-rw-r--r-- 1 root root 3.3G Aug 25 07:41 v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\r\n" - ] - } - ], - "source": [ - "!ls -lh ../../../model/" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "22fab9c1", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T19:47:04.462341Z", - "iopub.status.busy": "2023-08-28T19:47:04.461407Z", - "iopub.status.idle": "2023-08-28T19:47:07.766804Z", - "shell.execute_reply": "2023-08-28T19:47:07.765698Z" - }, - "papermill": { - "duration": 3.320526, - "end_time": "2023-08-28T19:47:07.769118", - "exception": false, - "start_time": "2023-08-28T19:47:04.448592", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: aiocsv in /usr/local/lib/python3.10/dist-packages (1.2.4)\r\n", - "Requirement already satisfied: aiofiles in /usr/local/lib/python3.10/dist-packages (23.2.1)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n", - "\u001b[0m" - ] - } - ], - "source": [ - "!python3 -m pip install aiocsv aiofiles" - ] - }, - { - "cell_type": "markdown", - "id": "32f7a478", - "metadata": { - "papermill": { - "duration": 0.010518, - "end_time": "2023-08-28T19:47:07.790575", - "exception": false, - "start_time": "2023-08-28T19:47:07.780057", - "status": "completed" - }, - "tags": [] - }, - "source": [ - "# headsize 2x (128) - L6-D4096" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "dfefa1bc", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T19:47:07.814819Z", - "iopub.status.busy": "2023-08-28T19:47:07.814046Z", - "iopub.status.idle": "2023-08-28T19:48:23.590356Z", - "shell.execute_reply": "2023-08-28T19:48:23.589469Z" - }, - "papermill": { - "duration": 75.79119, - "end_time": "2023-08-28T19:48:23.592568", - "exception": false, - "start_time": "2023-08-28T19:47:07.801378", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-08-28 19:47:12,418] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation start ###\r\n", - "###\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5 tokens : 100.0% similarity, with 5 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10 tokens : 100.0% similarity, with 10 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15 tokens : 100.0% similarity, with 15 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 20 tokens : 100.0% similarity, with 20 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 25 tokens : 100.0% similarity, with 25 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 30 tokens : 100.0% similarity, with 30 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 35 tokens : 100.0% similarity, with 35 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 40 tokens : 100.0% similarity, with 40 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 45 tokens : 100.0% similarity, with 45 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 50 tokens : 100.0% similarity, with 50 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 55 tokens : 100.0% similarity, with 55 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 60 tokens : 98.33333333333333% similarity, with 59 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 65 tokens : 98.46153846153847% similarity, with 64 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 70 tokens : 98.57142857142858% similarity, with 69 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 75 tokens : 98.66666666666667% similarity, with 74 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 80 tokens : 96.25% similarity, with 77 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 85 tokens : 96.47058823529412% similarity, with 82 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 90 tokens : 96.66666666666667% similarity, with 87 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 95 tokens : 97.89473684210527% similarity, with 93 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 100 tokens : 99.0% similarity, with 99 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 105 tokens : 99.04761904761905% similarity, with 104 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 110 tokens : 98.18181818181819% similarity, with 108 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 115 tokens : 98.26086956521739% similarity, with 113 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 120 tokens : 98.33333333333333% similarity, with 118 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 125 tokens : 99.2% similarity, with 124 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 130 tokens : 99.23076923076923% similarity, with 129 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 135 tokens : 98.51851851851852% similarity, with 133 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 140 tokens : 98.57142857142858% similarity, with 138 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 145 tokens : 98.62068965517241% similarity, with 143 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 150 tokens : 98.66666666666667% similarity, with 148 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 160 tokens : 98.125% similarity, with 157 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 170 tokens : 97.6470588235294% similarity, with 166 matched token, and 4 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 180 tokens : 97.77777777777777% similarity, with 176 matched token, and 4 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 190 tokens : 98.42105263157895% similarity, with 187 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 200 tokens : 98.5% similarity, with 197 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 210 tokens : 99.04761904761905% similarity, with 208 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 220 tokens : 98.63636363636363% similarity, with 217 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 230 tokens : 99.1304347826087% similarity, with 228 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 240 tokens : 99.58333333333333% similarity, with 239 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 250 tokens : 99.6% similarity, with 249 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 260 tokens : 99.23076923076923% similarity, with 258 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 270 tokens : 98.14814814814815% similarity, with 265 matched token, and 5 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 280 tokens : 98.21428571428571% similarity, with 275 matched token, and 5 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 290 tokens : 97.93103448275862% similarity, with 284 matched token, and 6 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 300 tokens : 98.0% similarity, with 294 matched token, and 6 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 325 tokens : 98.15384615384616% similarity, with 319 matched token, and 6 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 350 tokens : 97.42857142857143% similarity, with 341 matched token, and 9 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 375 tokens : 96.0% similarity, with 360 matched token, and 15 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 400 tokens : 96.25% similarity, with 385 matched token, and 15 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 425 tokens : 95.52941176470588% similarity, with 406 matched token, and 19 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 450 tokens : 95.77777777777777% similarity, with 431 matched token, and 19 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 475 tokens : 95.36842105263158% similarity, with 453 matched token, and 22 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 500 tokens : 95.19999999999999% similarity, with 476 matched token, and 24 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 525 tokens : 94.47619047619048% similarity, with 496 matched token, and 29 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 550 tokens : 94.72727272727272% similarity, with 521 matched token, and 29 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 575 tokens : 94.78260869565217% similarity, with 545 matched token, and 30 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 600 tokens : 94.83333333333334% similarity, with 569 matched token, and 31 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 625 tokens : 94.88% similarity, with 593 matched token, and 32 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 650 tokens : 94.0% similarity, with 611 matched token, and 39 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 675 tokens : 94.37037037037037% similarity, with 637 matched token, and 38 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 700 tokens : 93.28571428571428% similarity, with 653 matched token, and 47 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 750 tokens : 93.2% similarity, with 699 matched token, and 51 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 800 tokens : 91.25% similarity, with 730 matched token, and 70 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 850 tokens : 90.23529411764706% similarity, with 767 matched token, and 83 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 900 tokens : 90.0% similarity, with 810 matched token, and 90 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 950 tokens : 89.57894736842105% similarity, with 851 matched token, and 99 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1000 tokens : 88.5% similarity, with 885 matched token, and 115 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Finished baseline model to eval output predictive matching (aka 0 memory?), for 1000 tokens\r\n", - "###\r\n", - "### Model validation end ###\r\n", - "###\r\n" - ] - } - ], - "source": [ - "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D4096-E0_1-1k.csv\"" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "9bf65b9d", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T19:48:23.692599Z", - "iopub.status.busy": "2023-08-28T19:48:23.692144Z", - "iopub.status.idle": "2023-08-28T19:52:36.142124Z", - "shell.execute_reply": "2023-08-28T19:52:36.141237Z" - }, - "papermill": { - "duration": 252.468321, - "end_time": "2023-08-28T19:52:36.144355", - "exception": false, - "start_time": "2023-08-28T19:48:23.676034", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-08-28 19:48:28,178] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation start ###\r\n", - "###\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1100 tokens : 86.18181818181819% similarity, with 948 matched token, and 152 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1150 tokens : 85.04347826086956% similarity, with 978 matched token, and 172 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1200 tokens : 84.66666666666667% similarity, with 1016 matched token, and 184 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1250 tokens : 84.24000000000001% similarity, with 1053 matched token, and 197 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1300 tokens : 83.07692307692308% similarity, with 1080 matched token, and 220 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1350 tokens : 82.2962962962963% similarity, with 1111 matched token, and 239 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1400 tokens : 81.28571428571428% similarity, with 1138 matched token, and 262 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1450 tokens : 80.55172413793103% similarity, with 1168 matched token, and 282 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1500 tokens : 79.86666666666666% similarity, with 1198 matched token, and 302 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1550 tokens : 79.16129032258064% similarity, with 1227 matched token, and 323 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1600 tokens : 77.3125% similarity, with 1237 matched token, and 363 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1650 tokens : 75.63636363636364% similarity, with 1248 matched token, and 402 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1700 tokens : 74.41176470588235% similarity, with 1265 matched token, and 435 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1750 tokens : 72.39999999999999% similarity, with 1267 matched token, and 483 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1800 tokens : 70.22222222222221% similarity, with 1264 matched token, and 536 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1850 tokens : 69.62162162162161% similarity, with 1288 matched token, and 562 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1900 tokens : 68.84210526315789% similarity, with 1308 matched token, and 592 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1950 tokens : 67.02564102564102% similarity, with 1307 matched token, and 643 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2000 tokens : 66.3% similarity, with 1326 matched token, and 674 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2050 tokens : 64.04878048780488% similarity, with 1313 matched token, and 737 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2100 tokens : 63.66666666666667% similarity, with 1337 matched token, and 763 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2150 tokens : 62.83720930232558% similarity, with 1351 matched token, and 799 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2200 tokens : 61.68181818181818% similarity, with 1357 matched token, and 843 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2250 tokens : 60.84444444444445% similarity, with 1369 matched token, and 881 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2300 tokens : 58.434782608695656% similarity, with 1344 matched token, and 956 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2350 tokens : 57.48936170212766% similarity, with 1351 matched token, and 999 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2400 tokens : 56.458333333333336% similarity, with 1355 matched token, and 1045 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2450 tokens : 55.224489795918366% similarity, with 1353 matched token, and 1097 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2500 tokens : 54.400000000000006% similarity, with 1360 matched token, and 1140 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2550 tokens : 53.333333333333336% similarity, with 1360 matched token, and 1190 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2600 tokens : 51.76923076923077% similarity, with 1346 matched token, and 1254 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2650 tokens : 50.79245283018869% similarity, with 1346 matched token, and 1304 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2700 tokens : 49.81481481481482% similarity, with 1345 matched token, and 1355 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2750 tokens : 49.163636363636364% similarity, with 1352 matched token, and 1398 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2800 tokens : 48.285714285714285% similarity, with 1352 matched token, and 1448 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2850 tokens : 47.40350877192982% similarity, with 1351 matched token, and 1499 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2900 tokens : 46.310344827586206% similarity, with 1343 matched token, and 1557 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2950 tokens : 45.52542372881356% similarity, with 1343 matched token, and 1607 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3000 tokens : 44.56666666666666% similarity, with 1337 matched token, and 1663 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3050 tokens : 43.77049180327869% similarity, with 1335 matched token, and 1715 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3100 tokens : 42.45161290322581% similarity, with 1316 matched token, and 1784 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3150 tokens : 41.84126984126984% similarity, with 1318 matched token, and 1832 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3200 tokens : 41.1875% similarity, with 1318 matched token, and 1882 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3250 tokens : 40.43076923076923% similarity, with 1314 matched token, and 1936 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3300 tokens : 39.484848484848484% similarity, with 1303 matched token, and 1997 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3350 tokens : 38.776119402985074% similarity, with 1299 matched token, and 2051 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3400 tokens : 37.6764705882353% similarity, with 1281 matched token, and 2119 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3450 tokens : 36.608695652173914% similarity, with 1263 matched token, and 2187 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3500 tokens : 35.97142857142857% similarity, with 1259 matched token, and 2241 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3550 tokens : 35.267605633802816% similarity, with 1252 matched token, and 2298 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3600 tokens : 34.72222222222222% similarity, with 1250 matched token, and 2350 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3650 tokens : 33.91780821917808% similarity, with 1238 matched token, and 2412 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3700 tokens : 33.37837837837838% similarity, with 1235 matched token, and 2465 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3750 tokens : 32.4% similarity, with 1215 matched token, and 2535 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3800 tokens : 31.5% similarity, with 1197 matched token, and 2603 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3850 tokens : 30.80519480519481% similarity, with 1186 matched token, and 2664 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3900 tokens : 29.897435897435898% similarity, with 1166 matched token, and 2734 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3950 tokens : 29.012658227848103% similarity, with 1146 matched token, and 2804 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4000 tokens : 28.025% similarity, with 1121 matched token, and 2879 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation end ###\r\n", - "###\r\n" - ] - } - ], - "source": [ - "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D4096-E0_1-4k.csv\" 1100 4000" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "f6ebc262", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T19:52:36.182721Z", - "iopub.status.busy": "2023-08-28T19:52:36.181796Z", - "iopub.status.idle": "2023-08-28T20:55:06.419130Z", - "shell.execute_reply": "2023-08-28T20:55:06.418217Z" - }, - "papermill": { - "duration": 3750.258844, - "end_time": "2023-08-28T20:55:06.421163", - "exception": false, - "start_time": "2023-08-28T19:52:36.162319", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-08-28 19:52:40,787] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation start ###\r\n", - "###\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4100 tokens : 26.195121951219512% similarity, with 1074 matched token, and 3026 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4200 tokens : 24.952380952380953% similarity, with 1048 matched token, and 3152 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4300 tokens : 23.13953488372093% similarity, with 995 matched token, and 3305 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4400 tokens : 21.386363636363637% similarity, with 941 matched token, and 3459 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4500 tokens : 19.2% similarity, with 864 matched token, and 3636 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4600 tokens : 16.97826086956522% similarity, with 781 matched token, and 3819 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4700 tokens : 15.319148936170212% similarity, with 720 matched token, and 3980 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4800 tokens : 13.645833333333332% similarity, with 655 matched token, and 4145 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4900 tokens : 12.306122448979592% similarity, with 603 matched token, and 4297 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5000 tokens : 10.92% similarity, with 546 matched token, and 4454 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5100 tokens : 9.450980392156863% similarity, with 482 matched token, and 4618 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5200 tokens : 7.98076923076923% similarity, with 415 matched token, and 4785 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5300 tokens : 7.283018867924529% similarity, with 386 matched token, and 4914 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5400 tokens : 6.12962962962963% similarity, with 331 matched token, and 5069 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5500 tokens : 5.50909090909091% similarity, with 303 matched token, and 5197 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5600 tokens : 5.017857142857142% similarity, with 281 matched token, and 5319 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5700 tokens : 4.385964912280701% similarity, with 250 matched token, and 5450 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5800 tokens : 4.068965517241379% similarity, with 236 matched token, and 5564 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5900 tokens : 3.8474576271186436% similarity, with 227 matched token, and 5673 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6000 tokens : 3.716666666666667% similarity, with 223 matched token, and 5777 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6100 tokens : 3.4918032786885247% similarity, with 213 matched token, and 5887 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6200 tokens : 3.3225806451612905% similarity, with 206 matched token, and 5994 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6300 tokens : 3.2222222222222223% similarity, with 203 matched token, and 6097 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6400 tokens : 3.2031249999999996% similarity, with 205 matched token, and 6195 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6500 tokens : 3.1846153846153844% similarity, with 207 matched token, and 6293 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6600 tokens : 3.1363636363636362% similarity, with 207 matched token, and 6393 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6700 tokens : 3.119402985074627% similarity, with 209 matched token, and 6491 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6800 tokens : 3.088235294117647% similarity, with 210 matched token, and 6590 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6900 tokens : 3.173913043478261% similarity, with 219 matched token, and 6681 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7000 tokens : 3.1285714285714286% similarity, with 219 matched token, and 6781 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7100 tokens : 3.140845070422535% similarity, with 223 matched token, and 6877 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7200 tokens : 3.111111111111111% similarity, with 224 matched token, and 6976 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7300 tokens : 3.1643835616438354% similarity, with 231 matched token, and 7069 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7400 tokens : 3.1621621621621623% similarity, with 234 matched token, and 7166 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7500 tokens : 3.1466666666666665% similarity, with 236 matched token, and 7264 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7600 tokens : 3.1578947368421053% similarity, with 240 matched token, and 7360 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7700 tokens : 3.1558441558441555% similarity, with 243 matched token, and 7457 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7800 tokens : 3.1538461538461537% similarity, with 246 matched token, and 7554 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7900 tokens : 3.151898734177215% similarity, with 249 matched token, and 7651 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8000 tokens : 3.1375% similarity, with 251 matched token, and 7749 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8100 tokens : 3.185185185185185% similarity, with 258 matched token, and 7842 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8200 tokens : 3.1707317073170733% similarity, with 260 matched token, and 7940 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8300 tokens : 3.156626506024096% similarity, with 262 matched token, and 8038 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8400 tokens : 3.130952380952381% similarity, with 263 matched token, and 8137 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8500 tokens : 3.2% similarity, with 272 matched token, and 8228 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8600 tokens : 3.1976744186046515% similarity, with 275 matched token, and 8325 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8700 tokens : 3.206896551724138% similarity, with 279 matched token, and 8421 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8800 tokens : 3.193181818181818% similarity, with 281 matched token, and 8519 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8900 tokens : 3.191011235955056% similarity, with 284 matched token, and 8616 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9000 tokens : 3.188888888888889% similarity, with 287 matched token, and 8713 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9100 tokens : 3.208791208791209% similarity, with 292 matched token, and 8808 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9200 tokens : 3.2282608695652173% similarity, with 297 matched token, and 8903 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9300 tokens : 3.21505376344086% similarity, with 299 matched token, and 9001 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9400 tokens : 3.1914893617021276% similarity, with 300 matched token, and 9100 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9500 tokens : 3.2105263157894735% similarity, with 305 matched token, and 9195 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9600 tokens : 3.21875% similarity, with 309 matched token, and 9291 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9700 tokens : 3.216494845360825% similarity, with 312 matched token, and 9388 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9800 tokens : 3.2040816326530615% similarity, with 314 matched token, and 9486 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9900 tokens : 3.2222222222222223% similarity, with 319 matched token, and 9581 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10000 tokens : 3.25% similarity, with 325 matched token, and 9675 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10100 tokens : 3.2574257425742577% similarity, with 329 matched token, and 9771 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10200 tokens : 3.2549019607843137% similarity, with 332 matched token, and 9868 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10300 tokens : 3.2427184466019416% similarity, with 334 matched token, and 9966 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10400 tokens : 3.240384615384615% similarity, with 337 matched token, and 10063 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10500 tokens : 3.257142857142857% similarity, with 342 matched token, and 10158 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10600 tokens : 3.2547169811320753% similarity, with 345 matched token, and 10255 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10700 tokens : 3.2710280373831773% similarity, with 350 matched token, and 10350 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10800 tokens : 3.2870370370370368% similarity, with 355 matched token, and 10445 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10900 tokens : 3.2660550458715596% similarity, with 356 matched token, and 10544 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11000 tokens : 3.2818181818181817% similarity, with 361 matched token, and 10639 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11100 tokens : 3.288288288288288% similarity, with 365 matched token, and 10735 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11200 tokens : 3.2857142857142856% similarity, with 368 matched token, and 10832 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11300 tokens : 3.2920353982300887% similarity, with 372 matched token, and 10928 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11400 tokens : 3.324561403508772% similarity, with 379 matched token, and 11021 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11500 tokens : 3.3391304347826085% similarity, with 384 matched token, and 11116 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11600 tokens : 3.3448275862068964% similarity, with 388 matched token, and 11212 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11700 tokens : 3.3247863247863245% similarity, with 389 matched token, and 11311 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11800 tokens : 3.347457627118644% similarity, with 395 matched token, and 11405 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11900 tokens : 3.3361344537815127% similarity, with 397 matched token, and 11503 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12000 tokens : 3.3416666666666663% similarity, with 401 matched token, and 11599 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12100 tokens : 3.3636363636363638% similarity, with 407 matched token, and 11693 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12200 tokens : 3.344262295081967% similarity, with 408 matched token, and 11792 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12300 tokens : 3.3577235772357725% similarity, with 413 matched token, and 11887 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12400 tokens : 3.362903225806452% similarity, with 417 matched token, and 11983 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12500 tokens : 3.3680000000000003% similarity, with 421 matched token, and 12079 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12600 tokens : 3.3650793650793656% similarity, with 424 matched token, and 12176 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12700 tokens : 3.362204724409449% similarity, with 427 matched token, and 12273 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12800 tokens : 3.359375% similarity, with 430 matched token, and 12370 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12900 tokens : 3.3643410852713176% similarity, with 434 matched token, and 12466 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13000 tokens : 3.3692307692307697% similarity, with 438 matched token, and 12562 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13100 tokens : 3.381679389312977% similarity, with 443 matched token, and 12657 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13200 tokens : 3.3939393939393945% similarity, with 448 matched token, and 12752 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13300 tokens : 3.37593984962406% similarity, with 449 matched token, and 12851 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13400 tokens : 3.395522388059701% similarity, with 455 matched token, and 12945 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13500 tokens : 3.3851851851851853% similarity, with 457 matched token, and 13043 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13600 tokens : 3.4044117647058822% similarity, with 463 matched token, and 13137 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13700 tokens : 3.386861313868613% similarity, with 464 matched token, and 13236 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13800 tokens : 3.398550724637681% similarity, with 469 matched token, and 13331 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13900 tokens : 3.3884892086330933% similarity, with 471 matched token, and 13429 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14000 tokens : 3.4357142857142855% similarity, with 481 matched token, and 13519 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14100 tokens : 3.4255319148936167% similarity, with 483 matched token, and 13617 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14200 tokens : 3.408450704225352% similarity, with 484 matched token, and 13716 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14300 tokens : 3.4055944055944054% similarity, with 487 matched token, and 13813 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14400 tokens : 3.4166666666666665% similarity, with 492 matched token, and 13908 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14500 tokens : 3.420689655172414% similarity, with 496 matched token, and 14004 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14600 tokens : 3.404109589041096% similarity, with 497 matched token, and 14103 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14700 tokens : 3.4421768707482996% similarity, with 506 matched token, and 14194 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14800 tokens : 3.4391891891891895% similarity, with 509 matched token, and 14291 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14900 tokens : 3.422818791946309% similarity, with 510 matched token, and 14390 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15000 tokens : 3.42% similarity, with 513 matched token, and 14487 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15100 tokens : 3.4172185430463573% similarity, with 516 matched token, and 14584 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15200 tokens : 3.4078947368421058% similarity, with 518 matched token, and 14682 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15300 tokens : 3.4117647058823533% similarity, with 522 matched token, and 14778 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15400 tokens : 3.4090909090909087% similarity, with 525 matched token, and 14875 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15500 tokens : 3.4000000000000004% similarity, with 527 matched token, and 14973 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15600 tokens : 3.4038461538461537% similarity, with 531 matched token, and 15069 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15700 tokens : 3.4012738853503186% similarity, with 534 matched token, and 15166 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15800 tokens : 3.3924050632911396% similarity, with 536 matched token, and 15264 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15900 tokens : 3.3962264150943398% similarity, with 540 matched token, and 15360 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 16000 tokens : 3.3875% similarity, with 542 matched token, and 15458 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation end ###\r\n", - "###\r\n" - ] - } - ], - "source": [ - "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D4096-E0_1-16k.csv\" 4100 16000" - ] - }, - { - "cell_type": "markdown", - "id": "99ff749b", - "metadata": { - "papermill": { - "duration": 0.025441, - "end_time": "2023-08-28T20:55:06.471214", - "exception": false, - "start_time": "2023-08-28T20:55:06.445773", - "status": "completed" - }, - "tags": [] - }, - "source": [ - "# headsize 2x (128) - L6-D2048" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "9e2e63f0", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T20:55:06.524222Z", - "iopub.status.busy": "2023-08-28T20:55:06.523703Z", - "iopub.status.idle": "2023-08-28T20:56:03.919486Z", - "shell.execute_reply": "2023-08-28T20:56:03.918445Z" - }, - "papermill": { - "duration": 57.425007, - "end_time": "2023-08-28T20:56:03.921687", - "exception": false, - "start_time": "2023-08-28T20:55:06.496680", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-08-28 20:55:11,095] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation start ###\r\n", - "###\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5 tokens : 100.0% similarity, with 5 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10 tokens : 100.0% similarity, with 10 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15 tokens : 100.0% similarity, with 15 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 20 tokens : 100.0% similarity, with 20 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 25 tokens : 100.0% similarity, with 25 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 30 tokens : 100.0% similarity, with 30 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 35 tokens : 100.0% similarity, with 35 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 40 tokens : 100.0% similarity, with 40 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 45 tokens : 100.0% similarity, with 45 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 50 tokens : 100.0% similarity, with 50 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 55 tokens : 100.0% similarity, with 55 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 60 tokens : 100.0% similarity, with 60 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 65 tokens : 100.0% similarity, with 65 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 70 tokens : 100.0% similarity, with 70 matched token, and 0 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 75 tokens : 98.66666666666667% similarity, with 74 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 80 tokens : 98.75% similarity, with 79 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 85 tokens : 98.82352941176471% similarity, with 84 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 90 tokens : 98.88888888888889% similarity, with 89 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 95 tokens : 98.94736842105263% similarity, with 94 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 100 tokens : 99.0% similarity, with 99 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 105 tokens : 99.04761904761905% similarity, with 104 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 110 tokens : 99.0909090909091% similarity, with 109 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 115 tokens : 99.1304347826087% similarity, with 114 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 120 tokens : 99.16666666666667% similarity, with 119 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 125 tokens : 99.2% similarity, with 124 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 130 tokens : 99.23076923076923% similarity, with 129 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 135 tokens : 99.25925925925925% similarity, with 134 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 140 tokens : 99.28571428571429% similarity, with 139 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 145 tokens : 99.3103448275862% similarity, with 144 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 150 tokens : 99.33333333333333% similarity, with 149 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 160 tokens : 99.375% similarity, with 159 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 170 tokens : 99.41176470588235% similarity, with 169 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 180 tokens : 99.44444444444444% similarity, with 179 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 190 tokens : 99.47368421052632% similarity, with 189 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 200 tokens : 99.5% similarity, with 199 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 210 tokens : 99.52380952380952% similarity, with 209 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 220 tokens : 99.54545454545455% similarity, with 219 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 230 tokens : 99.1304347826087% similarity, with 228 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 240 tokens : 99.58333333333333% similarity, with 239 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 250 tokens : 99.6% similarity, with 249 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 260 tokens : 99.61538461538461% similarity, with 259 matched token, and 1 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 270 tokens : 99.25925925925925% similarity, with 268 matched token, and 2 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 280 tokens : 98.92857142857143% similarity, with 277 matched token, and 3 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 290 tokens : 98.27586206896551% similarity, with 285 matched token, and 5 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 300 tokens : 98.33333333333333% similarity, with 295 matched token, and 5 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 325 tokens : 98.46153846153847% similarity, with 320 matched token, and 5 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 350 tokens : 98.0% similarity, with 343 matched token, and 7 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 375 tokens : 97.06666666666666% similarity, with 364 matched token, and 11 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 400 tokens : 97.25% similarity, with 389 matched token, and 11 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 425 tokens : 96.70588235294117% similarity, with 411 matched token, and 14 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 450 tokens : 95.33333333333334% similarity, with 429 matched token, and 21 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 475 tokens : 95.15789473684211% similarity, with 452 matched token, and 23 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 500 tokens : 95.39999999999999% similarity, with 477 matched token, and 23 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 525 tokens : 94.47619047619048% similarity, with 496 matched token, and 29 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 550 tokens : 94.9090909090909% similarity, with 522 matched token, and 28 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 575 tokens : 93.73913043478261% similarity, with 539 matched token, and 36 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 600 tokens : 93.16666666666666% similarity, with 559 matched token, and 41 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 625 tokens : 93.44% similarity, with 584 matched token, and 41 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 650 tokens : 92.46153846153847% similarity, with 601 matched token, and 49 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 675 tokens : 92.14814814814815% similarity, with 622 matched token, and 53 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 700 tokens : 92.14285714285714% similarity, with 645 matched token, and 55 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 750 tokens : 91.73333333333333% similarity, with 688 matched token, and 62 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 800 tokens : 91.125% similarity, with 729 matched token, and 71 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 850 tokens : 90.58823529411765% similarity, with 770 matched token, and 80 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 900 tokens : 88.66666666666667% similarity, with 798 matched token, and 102 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 950 tokens : 87.78947368421053% similarity, with 834 matched token, and 116 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1000 tokens : 87.5% similarity, with 875 matched token, and 125 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Finished baseline model to eval output predictive matching (aka 0 memory?), for 1000 tokens\r\n", - "###\r\n", - "### Model validation end ###\r\n", - "###\r\n" - ] - } - ], - "source": [ - "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D2048-E0_1-1k.csv\"" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "1dc2f595", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T20:56:03.982184Z", - "iopub.status.busy": "2023-08-28T20:56:03.981058Z", - "iopub.status.idle": "2023-08-28T20:59:22.917254Z", - "shell.execute_reply": "2023-08-28T20:59:22.915980Z" - }, - "papermill": { - "duration": 198.96905, - "end_time": "2023-08-28T20:59:22.919486", - "exception": false, - "start_time": "2023-08-28T20:56:03.950436", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-08-28 20:56:08,589] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation start ###\r\n", - "###\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1100 tokens : 85.81818181818181% similarity, with 944 matched token, and 156 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1150 tokens : 84.69565217391303% similarity, with 974 matched token, and 176 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1200 tokens : 82.75% similarity, with 993 matched token, and 207 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1250 tokens : 82.16% similarity, with 1027 matched token, and 223 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1300 tokens : 80.92307692307692% similarity, with 1052 matched token, and 248 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1350 tokens : 79.4074074074074% similarity, with 1072 matched token, and 278 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1400 tokens : 78.21428571428571% similarity, with 1095 matched token, and 305 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1450 tokens : 77.10344827586208% similarity, with 1118 matched token, and 332 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1500 tokens : 76.33333333333333% similarity, with 1145 matched token, and 355 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1550 tokens : 75.74193548387098% similarity, with 1174 matched token, and 376 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1600 tokens : 74.1875% similarity, with 1187 matched token, and 413 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1650 tokens : 72.96969696969697% similarity, with 1204 matched token, and 446 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1700 tokens : 71.23529411764706% similarity, with 1211 matched token, and 489 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1750 tokens : 68.8% similarity, with 1204 matched token, and 546 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1800 tokens : 68.44444444444444% similarity, with 1232 matched token, and 568 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1850 tokens : 66.97297297297298% similarity, with 1239 matched token, and 611 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1900 tokens : 66.26315789473685% similarity, with 1259 matched token, and 641 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 1950 tokens : 65.07692307692308% similarity, with 1269 matched token, and 681 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2000 tokens : 64.45% similarity, with 1289 matched token, and 711 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2050 tokens : 62.48780487804878% similarity, with 1281 matched token, and 769 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2100 tokens : 61.38095238095238% similarity, with 1289 matched token, and 811 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2150 tokens : 59.627906976744185% similarity, with 1282 matched token, and 868 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2200 tokens : 58.77272727272727% similarity, with 1293 matched token, and 907 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2250 tokens : 57.99999999999999% similarity, with 1305 matched token, and 945 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2300 tokens : 56.869565217391305% similarity, with 1308 matched token, and 992 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2350 tokens : 54.55319148936171% similarity, with 1282 matched token, and 1068 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2400 tokens : 53.95833333333333% similarity, with 1295 matched token, and 1105 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2450 tokens : 52.53061224489796% similarity, with 1287 matched token, and 1163 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2500 tokens : 51.160000000000004% similarity, with 1279 matched token, and 1221 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2550 tokens : 49.68627450980392% similarity, with 1267 matched token, and 1283 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2600 tokens : 49.15384615384615% similarity, with 1278 matched token, and 1322 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2650 tokens : 47.84905660377359% similarity, with 1268 matched token, and 1382 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2700 tokens : 46.925925925925924% similarity, with 1267 matched token, and 1433 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2750 tokens : 46.47272727272727% similarity, with 1278 matched token, and 1472 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2800 tokens : 45.5% similarity, with 1274 matched token, and 1526 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2850 tokens : 44.70175438596492% similarity, with 1274 matched token, and 1576 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2900 tokens : 44.06896551724138% similarity, with 1278 matched token, and 1622 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 2950 tokens : 43.28813559322034% similarity, with 1277 matched token, and 1673 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3000 tokens : 42.36666666666667% similarity, with 1271 matched token, and 1729 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3050 tokens : 41.63934426229508% similarity, with 1270 matched token, and 1780 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3100 tokens : 41.12903225806452% similarity, with 1275 matched token, and 1825 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3150 tokens : 40.15873015873016% similarity, with 1265 matched token, and 1885 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3200 tokens : 39.53125% similarity, with 1265 matched token, and 1935 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3250 tokens : 38.95384615384616% similarity, with 1266 matched token, and 1984 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3300 tokens : 38.27272727272727% similarity, with 1263 matched token, and 2037 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3350 tokens : 37.5223880597015% similarity, with 1257 matched token, and 2093 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3400 tokens : 37.23529411764706% similarity, with 1266 matched token, and 2134 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3450 tokens : 36.289855072463766% similarity, with 1252 matched token, and 2198 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3500 tokens : 35.8% similarity, with 1253 matched token, and 2247 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3550 tokens : 34.901408450704224% similarity, with 1239 matched token, and 2311 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3600 tokens : 34.61111111111111% similarity, with 1246 matched token, and 2354 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3650 tokens : 33.89041095890411% similarity, with 1237 matched token, and 2413 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3700 tokens : 33.189189189189186% similarity, with 1228 matched token, and 2472 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3750 tokens : 32.77333333333333% similarity, with 1229 matched token, and 2521 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3800 tokens : 32.23684210526316% similarity, with 1225 matched token, and 2575 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3850 tokens : 31.61038961038961% similarity, with 1217 matched token, and 2633 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3900 tokens : 31.23076923076923% similarity, with 1218 matched token, and 2682 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 3950 tokens : 30.531645569620252% similarity, with 1206 matched token, and 2744 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4000 tokens : 29.849999999999998% similarity, with 1194 matched token, and 2806 token mismatch\r\n", - "###\r\n", - "### Model validation end ###\r\n", - "###\r\n" - ] - } - ], - "source": [ - "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D2048-E0_1-4k.csv\" 1100 4000" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "629f8e65", - "metadata": { - "execution": { - "iopub.execute_input": "2023-08-28T20:59:22.986171Z", - "iopub.status.busy": "2023-08-28T20:59:22.985519Z", - "iopub.status.idle": "2023-08-28T21:57:15.296164Z", - "shell.execute_reply": "2023-08-28T21:57:15.295296Z" - }, - "papermill": { - "duration": 3472.346593, - "end_time": "2023-08-28T21:57:15.298359", - "exception": false, - "start_time": "2023-08-28T20:59:22.951766", - "status": "completed" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-08-28 20:59:27,613] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation start ###\r\n", - "###\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4100 tokens : 28.41463414634146% similarity, with 1165 matched token, and 2935 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4200 tokens : 27.11904761904762% similarity, with 1139 matched token, and 3061 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4300 tokens : 25.813953488372093% similarity, with 1110 matched token, and 3190 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4400 tokens : 24.06818181818182% similarity, with 1059 matched token, and 3341 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4500 tokens : 22.466666666666665% similarity, with 1011 matched token, and 3489 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4600 tokens : 20.58695652173913% similarity, with 947 matched token, and 3653 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4700 tokens : 19.382978723404257% similarity, with 911 matched token, and 3789 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4800 tokens : 17.791666666666668% similarity, with 854 matched token, and 3946 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 4900 tokens : 16.28571428571429% similarity, with 798 matched token, and 4102 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5000 tokens : 15.2% similarity, with 760 matched token, and 4240 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5100 tokens : 13.76470588235294% similarity, with 702 matched token, and 4398 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5200 tokens : 12.153846153846153% similarity, with 632 matched token, and 4568 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5300 tokens : 10.528301886792454% similarity, with 558 matched token, and 4742 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5400 tokens : 9.24074074074074% similarity, with 499 matched token, and 4901 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5500 tokens : 8.436363636363636% similarity, with 464 matched token, and 5036 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5600 tokens : 7.642857142857143% similarity, with 428 matched token, and 5172 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5700 tokens : 6.859649122807017% similarity, with 391 matched token, and 5309 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5800 tokens : 6.241379310344827% similarity, with 362 matched token, and 5438 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 5900 tokens : 5.508474576271186% similarity, with 325 matched token, and 5575 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6000 tokens : 5.033333333333333% similarity, with 302 matched token, and 5698 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6100 tokens : 4.557377049180328% similarity, with 278 matched token, and 5822 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6200 tokens : 4.274193548387097% similarity, with 265 matched token, and 5935 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6300 tokens : 3.984126984126984% similarity, with 251 matched token, and 6049 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6400 tokens : 3.9218749999999996% similarity, with 251 matched token, and 6149 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6500 tokens : 3.8307692307692305% similarity, with 249 matched token, and 6251 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6600 tokens : 3.621212121212121% similarity, with 239 matched token, and 6361 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6700 tokens : 3.6865671641791047% similarity, with 247 matched token, and 6453 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6800 tokens : 3.514705882352941% similarity, with 239 matched token, and 6561 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 6900 tokens : 3.5652173913043477% similarity, with 246 matched token, and 6654 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7000 tokens : 3.471428571428571% similarity, with 243 matched token, and 6757 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7100 tokens : 3.408450704225352% similarity, with 242 matched token, and 6858 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7200 tokens : 3.375% similarity, with 243 matched token, and 6957 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7300 tokens : 3.383561643835616% similarity, with 247 matched token, and 7053 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7400 tokens : 3.3648648648648654% similarity, with 249 matched token, and 7151 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7500 tokens : 3.32% similarity, with 249 matched token, and 7251 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7600 tokens : 3.3684210526315788% similarity, with 256 matched token, and 7344 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7700 tokens : 3.3636363636363638% similarity, with 259 matched token, and 7441 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7800 tokens : 3.346153846153846% similarity, with 261 matched token, and 7539 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 7900 tokens : 3.329113924050633% similarity, with 263 matched token, and 7637 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8000 tokens : 3.3125% similarity, with 265 matched token, and 7735 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8100 tokens : 3.4074074074074074% similarity, with 276 matched token, and 7824 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8200 tokens : 3.3658536585365857% similarity, with 276 matched token, and 7924 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8300 tokens : 3.337349397590361% similarity, with 277 matched token, and 8023 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8400 tokens : 3.3214285714285716% similarity, with 279 matched token, and 8121 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8500 tokens : 3.3882352941176466% similarity, with 288 matched token, and 8212 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8600 tokens : 3.383720930232558% similarity, with 291 matched token, and 8309 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8700 tokens : 3.3908045977011496% similarity, with 295 matched token, and 8405 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8800 tokens : 3.375% similarity, with 297 matched token, and 8503 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 8900 tokens : 3.359550561797753% similarity, with 299 matched token, and 8601 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9000 tokens : 3.3666666666666663% similarity, with 303 matched token, and 8697 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9100 tokens : 3.3626373626373627% similarity, with 306 matched token, and 8794 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9200 tokens : 3.369565217391304% similarity, with 310 matched token, and 8890 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9300 tokens : 3.344086021505376% similarity, with 311 matched token, and 8989 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9400 tokens : 3.3297872340425534% similarity, with 313 matched token, and 9087 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9500 tokens : 3.3684210526315788% similarity, with 320 matched token, and 9180 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9600 tokens : 3.3541666666666665% similarity, with 322 matched token, and 9278 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9700 tokens : 3.350515463917526% similarity, with 325 matched token, and 9375 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9800 tokens : 3.3265306122448983% similarity, with 326 matched token, and 9474 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 9900 tokens : 3.3636363636363638% similarity, with 333 matched token, and 9567 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10000 tokens : 3.4000000000000004% similarity, with 340 matched token, and 9660 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10100 tokens : 3.425742574257426% similarity, with 346 matched token, and 9754 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10200 tokens : 3.4117647058823533% similarity, with 348 matched token, and 9852 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10300 tokens : 3.388349514563107% similarity, with 349 matched token, and 9951 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10400 tokens : 3.3942307692307696% similarity, with 353 matched token, and 10047 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10500 tokens : 3.419047619047619% similarity, with 359 matched token, and 10141 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10600 tokens : 3.3962264150943398% similarity, with 360 matched token, and 10240 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10700 tokens : 3.4392523364485985% similarity, with 368 matched token, and 10332 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10800 tokens : 3.4351851851851847% similarity, with 371 matched token, and 10429 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 10900 tokens : 3.431192660550459% similarity, with 374 matched token, and 10526 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11000 tokens : 3.4454545454545453% similarity, with 379 matched token, and 10621 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11100 tokens : 3.432432432432433% similarity, with 381 matched token, and 10719 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11200 tokens : 3.428571428571429% similarity, with 384 matched token, and 10816 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11300 tokens : 3.433628318584071% similarity, with 388 matched token, and 10912 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11400 tokens : 3.482456140350877% similarity, with 397 matched token, and 11003 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11500 tokens : 3.5130434782608697% similarity, with 404 matched token, and 11096 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11600 tokens : 3.508620689655172% similarity, with 407 matched token, and 11193 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11700 tokens : 3.4957264957264957% similarity, with 409 matched token, and 11291 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11800 tokens : 3.5254237288135593% similarity, with 416 matched token, and 11384 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 11900 tokens : 3.5126050420168067% similarity, with 418 matched token, and 11482 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12000 tokens : 3.5249999999999995% similarity, with 423 matched token, and 11577 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12100 tokens : 3.5371900826446283% similarity, with 428 matched token, and 11672 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12200 tokens : 3.524590163934426% similarity, with 430 matched token, and 11770 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12300 tokens : 3.528455284552846% similarity, with 434 matched token, and 11866 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12400 tokens : 3.5241935483870965% similarity, with 437 matched token, and 11963 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12500 tokens : 3.512% similarity, with 439 matched token, and 12061 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12600 tokens : 3.507936507936508% similarity, with 442 matched token, and 12158 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12700 tokens : 3.5196850393700787% similarity, with 447 matched token, and 12253 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12800 tokens : 3.515625% similarity, with 450 matched token, and 12350 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 12900 tokens : 3.5193798449612403% similarity, with 454 matched token, and 12446 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13000 tokens : 3.523076923076923% similarity, with 458 matched token, and 12542 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13100 tokens : 3.5038167938931295% similarity, with 459 matched token, and 12641 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13200 tokens : 3.5151515151515147% similarity, with 464 matched token, and 12736 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13300 tokens : 3.496240601503759% similarity, with 465 matched token, and 12835 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13400 tokens : 3.544776119402985% similarity, with 475 matched token, and 12925 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13500 tokens : 3.5111111111111115% similarity, with 474 matched token, and 13026 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13600 tokens : 3.5367647058823533% similarity, with 481 matched token, and 13119 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13700 tokens : 3.5328467153284673% similarity, with 484 matched token, and 13216 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13800 tokens : 3.5144927536231885% similarity, with 485 matched token, and 13315 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 13900 tokens : 3.553956834532374% similarity, with 494 matched token, and 13406 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14000 tokens : 3.564285714285714% similarity, with 499 matched token, and 13501 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14100 tokens : 3.5673758865248226% similarity, with 503 matched token, and 13597 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14200 tokens : 3.549295774647887% similarity, with 504 matched token, and 13696 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14300 tokens : 3.5454545454545454% similarity, with 507 matched token, and 13793 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14400 tokens : 3.5624999999999996% similarity, with 513 matched token, and 13887 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14500 tokens : 3.5586206896551724% similarity, with 516 matched token, and 13984 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14600 tokens : 3.5479452054794516% similarity, with 518 matched token, and 14082 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14700 tokens : 3.5918367346938775% similarity, with 528 matched token, and 14172 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14800 tokens : 3.567567567567568% similarity, with 528 matched token, and 14272 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 14900 tokens : 3.577181208053691% similarity, with 533 matched token, and 14367 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15000 tokens : 3.56% similarity, with 534 matched token, and 14466 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15100 tokens : 3.562913907284768% similarity, with 538 matched token, and 14562 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15200 tokens : 3.55921052631579% similarity, with 541 matched token, and 14659 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15300 tokens : 3.5620915032679736% similarity, with 545 matched token, and 14755 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15400 tokens : 3.5584415584415585% similarity, with 548 matched token, and 14852 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15500 tokens : 3.561290322580645% similarity, with 552 matched token, and 14948 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15600 tokens : 3.5705128205128203% similarity, with 557 matched token, and 15043 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15700 tokens : 3.579617834394905% similarity, with 562 matched token, and 15138 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15800 tokens : 3.5632911392405062% similarity, with 563 matched token, and 15237 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 15900 tokens : 3.5534591194968552% similarity, with 565 matched token, and 15335 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Model validation for 16000 tokens : 3.5374999999999996% similarity, with 566 matched token, and 15434 token mismatch\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "###\r\n", - "### Model validation end ###\r\n", - "###\r\n" - ] - } - ], - "source": [ - "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D2048-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D2048-E0_1-16k.csv\" 4100 16000" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "rwkv-infctx", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "papermill": { - "default_parameters": {}, - "duration": 7847.626274, - "end_time": "2023-08-28T21:57:15.669712", - "environment_variables": {}, - "exception": null, - "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/memory-bench/Benchmark-V5headsize2x.ipynb", - "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/memory-bench/Benchmark-V5headsize2x.ipynb", - "parameters": {}, - "start_time": "2023-08-28T19:46:28.043438", - "version": "2.4.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file