{ "cells": [ { "cell_type": "markdown", "id": "8f5f2ffe", "metadata": { "papermill": { "duration": 0.002629, "end_time": "2023-08-26T01:23:07.096418", "exception": false, "start_time": "2023-08-26T01:23:07.093789", "status": "completed" }, "tags": [] }, "source": [ "## Custom Experimental Memory benchmarking\n", "\n", "The following is meant to observe the memory performance in a more verbose CSV logging mode, for the baseline raven models" ] }, { "cell_type": "code", "execution_count": 1, "id": "5e19b2cd", "metadata": { "execution": { "iopub.execute_input": "2023-08-26T01:23:07.102616Z", "iopub.status.busy": "2023-08-26T01:23:07.102019Z", "iopub.status.idle": "2023-08-26T01:24:15.273307Z", "shell.execute_reply": "2023-08-26T01:24:15.272409Z" }, "papermill": { "duration": 68.176689, "end_time": "2023-08-26T01:24:15.275361", "exception": false, "start_time": "2023-08-26T01:23:07.098672", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2023-08-26 01:23:07-- https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-headsize2x/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\r\n", "Resolving huggingface.co (huggingface.co)... 18.154.227.67, 18.154.227.69, 18.154.227.87, ...\r\n", "Connecting to huggingface.co (huggingface.co)|18.154.227.67|:443... connected.\r\n", "HTTP request sent, awaiting response... 302 Found\r\n", "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/fae9799e56fd469c6c1694dae6ae77f78130c06d340e945a3dc8e8ea2bd4e5bb?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%22%3B&Expires=1693272187&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5MzI3MjE4N319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkL2ZhZTk3OTllNTZmZDQ2OWM2YzE2OTRkYWU2YWU3N2Y3ODEzMGMwNmQzNDBlOTQ1YTNkYzhlOGVhMmJkNGU1YmI%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=y4AlHINm1RFbemOtm%7ER7DeItTbEeD1%7ECx1PV7P%7EEJjhErMBHqakRcU%7EM25XaFxRLsfQAC0Vv-ZS6DxiWkUQlW4Ku6kuOoU2cdCSvonLreanJ2ZRA-%7EZmrsX7Fi5lNhkuA9SEaeQTAQCCx1C-mc82E6LWXduPSnJy4ZcHFl-vWY2MT5kTLTuzEnyqfszLzVzw5Hd9RyewdNyLpKnoyoQO1fi5hAFnZacM72uhhIa7aGvI5ciXe4bsjg37RDw87fDdX3TRoxl3jadcJbxOGBguIwp5nbBdU6gVQSThsBgLUb4k6tAfVVlVG5XHaIjWRXaNy7A6oiyd9heIEmRyRvVZJQ__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n", "--2023-08-26 01:23:07-- https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/fae9799e56fd469c6c1694dae6ae77f78130c06d340e945a3dc8e8ea2bd4e5bb?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth%22%3B&Expires=1693272187&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5MzI3MjE4N319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkL2ZhZTk3OTllNTZmZDQ2OWM2YzE2OTRkYWU2YWU3N2Y3ODEzMGMwNmQzNDBlOTQ1YTNkYzhlOGVhMmJkNGU1YmI%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=y4AlHINm1RFbemOtm%7ER7DeItTbEeD1%7ECx1PV7P%7EEJjhErMBHqakRcU%7EM25XaFxRLsfQAC0Vv-ZS6DxiWkUQlW4Ku6kuOoU2cdCSvonLreanJ2ZRA-%7EZmrsX7Fi5lNhkuA9SEaeQTAQCCx1C-mc82E6LWXduPSnJy4ZcHFl-vWY2MT5kTLTuzEnyqfszLzVzw5Hd9RyewdNyLpKnoyoQO1fi5hAFnZacM72uhhIa7aGvI5ciXe4bsjg37RDw87fDdX3TRoxl3jadcJbxOGBguIwp5nbBdU6gVQSThsBgLUb4k6tAfVVlVG5XHaIjWRXaNy7A6oiyd9heIEmRyRvVZJQ__&Key-Pair-Id=KVTP0A1DKRTAX\r\n", "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... " ] }, { "name": "stdout", "output_type": "stream", "text": [ "108.138.64.111, 108.138.64.36, 108.138.64.121, ...\r\n", "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|108.138.64.111|:443... connected.\r\n", "HTTP request sent, awaiting response... " ] }, { "name": "stdout", "output_type": "stream", "text": [ "200 OK\r\n", "Length: 3441599109 (3.2G) [binary/octet-stream]\r\n", "Saving to: ‘v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth’\r\n", "\r\n", "\r", " v5-hs2x-L 0%[ ] 0 --.-KB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6 0%[ ] 14.74M 64.4MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6- 0%[ ] 17.60M 41.0MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D 0%[ ] 30.52M 41.5MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4 1%[ ] 52.47M 55.7MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D40 1%[ ] 61.03M 51.7MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D409 2%[ ] 75.78M 52.3MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096 2%[ ] 91.03M 52.7MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096- 3%[ ] 106.29M 54.8MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096-E 3%[ ] 109.17M 51.0MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-hs2x-L6-D4096-E0 3%[ ] 122.07M 52.0MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-hs2x-L6-D4096-E0_ 4%[ ] 136.81M 53.2MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-hs2x-L6-D4096-E0_1 4%[ ] 147.28M 53.1MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "hs2x-L6-D4096-E0_1- 4%[ ] 152.59M 49.3MB/s eta 63s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "s2x-L6-D4096-E0_1-m 5%[> ] 167.80M 51.0MB/s eta 63s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "2x-L6-D4096-E0_1-me 5%[> ] 174.32M 49.9MB/s eta 63s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-L6-D4096-E0_1-mem 5%[> ] 183.10M 49.5MB/s eta 63s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L6-D4096-E0_1-mem- 6%[> ] 198.36M 49.7MB/s eta 63s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L6-D4096-E0_1-mem-c 6%[> ] 213.11M 51.5MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D4096-E0_1-mem-ct 6%[> ] 220.57M 49.0MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D4096-E0_1-mem-ctx 6%[> ] 228.87M 49.8MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D4096-E0_1-mem-ctx- 7%[> ] 244.13M 48.5MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4096-E0_1-mem-ctx-8 8%[> ] 263.16M 51.4MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "096-E0_1-mem-ctx-8k 8%[> ] 274.66M 50.4MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-E0_1-mem-ctx-8k. 8%[> ] 289.92M 48.3MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-E0_1-mem-ctx-8k.p 9%[> ] 305.18M 50.7MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 9%[> ] 320.43M 48.7MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 10%[=> ] 335.69M 50.6MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 10%[=> ] 342.92M 51.9MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 10%[=> ] 349.13M 48.2MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 10%[=> ] 355.14M 49.1MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 11%[=> ] 366.20M 47.4MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 11%[=> ] 381.33M 48.0MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 11%[=> ] 385.65M 45.6MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 12%[=> ] 396.73M 44.6MB/s eta 60s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 12%[=> ] 411.99M 44.4MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 13%[=> ] 426.75M 43.9MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 13%[=> ] 441.98M 44.4MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 13%[=> ] 457.24M 44.2MB/s eta 61s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 14%[=> ] 471.24M 43.6MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 14%[=> ] 473.02M 40.1MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 14%[=> ] 483.37M 41.3MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 14%[=> ] 488.28M 38.6MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 15%[==> ] 503.54M 40.8MB/s eta 59s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 15%[==> ] 518.29M 43.3MB/s eta 59s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 16%[==> ] 525.85M 42.2MB/s eta 59s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 16%[==> ] 534.05M 39.9MB/s eta 59s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 16%[==> ] 548.80M 42.1MB/s eta 59s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 17%[==> ] 562.75M 43.4MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 17%[==> ] 565.79M 42.8MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-h 17%[==> ] 585.78M 43.6MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs 18%[==> ] 595.09M 42.6MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2 18%[==> ] 608.53M 44.5MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x 18%[==> ] 612.97M 41.8MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x- 19%[==> ] 625.09M 42.9MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L 19%[==> ] 639.04M 43.7MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6 19%[==> ] 649.30M 42.2MB/s eta 58s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6- 19%[==> ] 656.13M 43.0MB/s eta 57s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D 20%[===> ] 671.38M 45.0MB/s eta 57s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4 20%[===> ] 684.82M 44.3MB/s eta 57s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D40 20%[===> ] 686.64M 41.8MB/s eta 57s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D409 21%[===> ] 701.39M 43.9MB/s eta 57s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096 21%[===> ] 714.11M 46.5MB/s eta 55s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096- 21%[===> ] 717.16M 44.4MB/s eta 55s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096-E 22%[===> ] 732.42M 42.9MB/s eta 55s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-hs2x-L6-D4096-E0 22%[===> ] 747.69M 44.7MB/s eta 55s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-hs2x-L6-D4096-E0_ 23%[===> ] 762.94M 46.2MB/s eta 54s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-hs2x-L6-D4096-E0_1 23%[===> ] 778.20M 45.7MB/s eta 54s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "hs2x-L6-D4096-E0_1- 24%[===> ] 793.46M 48.7MB/s eta 54s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "s2x-L6-D4096-E0_1-m 24%[===> ] 808.71M 49.6MB/s eta 54s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "2x-L6-D4096-E0_1-me 25%[====> ] 823.46M 52.4MB/s eta 54s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-L6-D4096-E0_1-mem 25%[====> ] 824.09M 49.9MB/s eta 52s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L6-D4096-E0_1-mem- 25%[====> ] 839.23M 51.2MB/s eta 52s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L6-D4096-E0_1-mem-c 26%[====> ] 854.49M 51.8MB/s eta 52s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D4096-E0_1-mem-ct 26%[====> ] 869.75M 49.8MB/s eta 52s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D4096-E0_1-mem-ctx 26%[====> ] 885.01M 50.8MB/s eta 51s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D4096-E0_1-mem-ctx- 27%[====> ] 900.27M 54.3MB/s eta 51s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4096-E0_1-mem-ctx-8 27%[====> ] 916.90M 54.9MB/s eta 51s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "096-E0_1-mem-ctx-8k 28%[====> ] 930.27M 57.2MB/s eta 51s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-E0_1-mem-ctx-8k. 28%[====> ] 944.48M 58.2MB/s eta 51s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-E0_1-mem-ctx-8k.p 28%[====> ] 946.04M 54.0MB/s eta 49s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 29%[====> ] 966.02M 58.1MB/s eta 49s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 29%[====> ] 976.55M 55.6MB/s eta 49s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 30%[=====> ] 991.82M 56.5MB/s eta 49s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 30%[=====> ] 1007M 56.6MB/s eta 49s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 31%[=====> ] 1022M 55.5MB/s eta 47s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 31%[=====> ] 1.01G 53.7MB/s eta 47s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 31%[=====> ] 1.02G 55.1MB/s eta 47s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 32%[=====> ] 1.03G 51.6MB/s eta 47s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 32%[=====> ] 1.03G 48.0MB/s eta 46s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 32%[=====> ] 1.04G 48.4MB/s eta 46s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 33%[=====> ] 1.06G 50.5MB/s eta 46s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 33%[=====> ] 1.07G 47.7MB/s eta 46s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 33%[=====> ] 1.09G 47.3MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 34%[=====> ] 1.09G 46.7MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 34%[=====> ] 1.10G 47.1MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 34%[=====> ] 1.10G 42.8MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 34%[=====> ] 1.12G 41.5MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 35%[======> ] 1.13G 43.4MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 35%[======> ] 1.13G 40.2MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 35%[======> ] 1.15G 40.3MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 36%[======> ] 1.16G 39.9MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 36%[======> ] 1.17G 42.0MB/s eta 45s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 36%[======> ] 1.18G 39.6MB/s eta 44s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 37%[======> ] 1.19G 39.2MB/s eta 44s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-h 37%[======> ] 1.21G 43.1MB/s eta 44s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs 38%[======> ] 1.22G 45.2MB/s eta 44s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2 38%[======> ] 1.24G 45.3MB/s eta 44s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x 38%[======> ] 1.24G 47.0MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x- 39%[======> ] 1.25G 45.6MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L 39%[======> ] 1.27G 45.3MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6 39%[======> ] 1.28G 45.0MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6- 40%[=======> ] 1.29G 46.3MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D 40%[=======> ] 1.31G 49.1MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4 40%[=======> ] 1.31G 46.1MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D40 41%[=======> ] 1.33G 46.7MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D409 41%[=======> ] 1.33G 46.8MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096 41%[=======> ] 1.34G 46.3MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096- 42%[=======> ] 1.36G 48.2MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096-E 42%[=======> ] 1.36G 45.5MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-hs2x-L6-D4096-E0 42%[=======> ] 1.37G 45.7MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-hs2x-L6-D4096-E0_ 43%[=======> ] 1.39G 46.5MB/s eta 41s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-hs2x-L6-D4096-E0_1 43%[=======> ] 1.40G 46.8MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "hs2x-L6-D4096-E0_1- 44%[=======> ] 1.42G 44.8MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "s2x-L6-D4096-E0_1-m 44%[=======> ] 1.43G 43.8MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "2x-L6-D4096-E0_1-me 44%[=======> ] 1.43G 43.7MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-L6-D4096-E0_1-mem 45%[========> ] 1.45G 43.1MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L6-D4096-E0_1-mem- 45%[========> ] 1.46G 43.9MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L6-D4096-E0_1-mem-c 46%[========> ] 1.47G 43.8MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D4096-E0_1-mem-ct 46%[========> ] 1.48G 42.6MB/s eta 38s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D4096-E0_1-mem-ctx 46%[========> ] 1.50G 47.1MB/s eta 38s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D4096-E0_1-mem-ctx- 46%[========> ] 1.50G 45.2MB/s eta 38s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4096-E0_1-mem-ctx-8 47%[========> ] 1.52G 46.9MB/s eta 38s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "096-E0_1-mem-ctx-8k 47%[========> ] 1.53G 47.9MB/s eta 38s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-E0_1-mem-ctx-8k. 47%[========> ] 1.54G 46.4MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-E0_1-mem-ctx-8k.p 48%[========> ] 1.55G 48.6MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 48%[========> ] 1.56G 47.8MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 49%[========> ] 1.58G 48.8MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 49%[========> ] 1.58G 46.7MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 49%[========> ] 1.59G 45.9MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 50%[=========> ] 1.61G 48.2MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 50%[=========> ] 1.62G 51.1MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 51%[=========> ] 1.64G 51.1MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 51%[=========> ] 1.65G 52.4MB/s eta 34s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 52%[=========> ] 1.67G 52.4MB/s eta 34s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 52%[=========> ] 1.68G 53.5MB/s eta 34s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 52%[=========> ] 1.68G 48.1MB/s eta 34s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 52%[=========> ] 1.70G 48.0MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 53%[=========> ] 1.71G 47.1MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 53%[=========> ] 1.72G 45.9MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 53%[=========> ] 1.73G 43.9MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 54%[=========> ] 1.74G 43.3MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 54%[=========> ] 1.76G 42.0MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 55%[==========> ] 1.78G 45.6MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 55%[==========> ] 1.79G 46.7MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 56%[==========> ] 1.80G 45.8MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 56%[==========> ] 1.82G 47.8MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 57%[==========> ] 1.83G 45.9MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 57%[==========> ] 1.85G 48.4MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 58%[==========> ] 1.86G 48.7MB/s eta 29s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-h 58%[==========> ] 1.88G 47.6MB/s eta 29s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs 58%[==========> ] 1.89G 47.8MB/s eta 29s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2 59%[==========> ] 1.91G 48.8MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x 59%[==========> ] 1.92G 50.3MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x- 60%[===========> ] 1.94G 51.6MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L 60%[===========> ] 1.94G 49.7MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6 60%[===========> ] 1.95G 50.3MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6- 61%[===========> ] 1.97G 53.8MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D 61%[===========> ] 1.98G 52.6MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4 62%[===========> ] 2.00G 52.5MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D40 62%[===========> ] 2.01G 52.3MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D409 63%[===========> ] 2.02G 52.2MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096 63%[===========> ] 2.04G 55.6MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096- 63%[===========> ] 2.04G 52.0MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096-E 64%[===========> ] 2.06G 51.6MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-hs2x-L6-D4096-E0 64%[===========> ] 2.07G 51.4MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-hs2x-L6-D4096-E0_ 65%[============> ] 2.09G 52.5MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-hs2x-L6-D4096-E0_1 65%[============> ] 2.10G 51.7MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "hs2x-L6-D4096-E0_1- 65%[============> ] 2.11G 50.5MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "s2x-L6-D4096-E0_1-m 66%[============> ] 2.12G 51.4MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "2x-L6-D4096-E0_1-me 66%[============> ] 2.13G 52.5MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-L6-D4096-E0_1-mem 66%[============> ] 2.14G 54.2MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L6-D4096-E0_1-mem- 67%[============> ] 2.16G 53.2MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L6-D4096-E0_1-mem-c 67%[============> ] 2.16G 49.3MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D4096-E0_1-mem-ct 67%[============> ] 2.17G 46.2MB/s eta 22s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D4096-E0_1-mem-ctx 68%[============> ] 2.19G 48.9MB/s eta 22s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D4096-E0_1-mem-ctx- 68%[============> ] 2.21G 48.4MB/s eta 22s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4096-E0_1-mem-ctx-8 69%[============> ] 2.23G 51.6MB/s eta 22s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "096-E0_1-mem-ctx-8k 70%[=============> ] 2.25G 52.0MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-E0_1-mem-ctx-8k. 70%[=============> ] 2.26G 50.6MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-E0_1-mem-ctx-8k.p 70%[=============> ] 2.27G 49.6MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 71%[=============> ] 2.28G 49.1MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 71%[=============> ] 2.29G 52.6MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 72%[=============> ] 2.31G 49.9MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 72%[=============> ] 2.32G 50.8MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 72%[=============> ] 2.34G 49.4MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 73%[=============> ] 2.35G 50.8MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 73%[=============> ] 2.37G 53.2MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 74%[=============> ] 2.38G 53.6MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 74%[=============> ] 2.38G 52.2MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 75%[==============> ] 2.41G 55.6MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 75%[==============> ] 2.41G 52.1MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 75%[==============> ] 2.43G 51.3MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 75%[==============> ] 2.43G 51.0MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 76%[==============> ] 2.44G 50.0MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 76%[==============> ] 2.46G 49.3MB/s eta 16s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 77%[==============> ] 2.47G 49.6MB/s eta 16s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 77%[==============> ] 2.49G 49.1MB/s eta 16s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 78%[==============> ] 2.50G 52.4MB/s eta 16s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 78%[==============> ] 2.50G 52.2MB/s eta 16s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 78%[==============> ] 2.52G 52.1MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 79%[==============> ] 2.53G 51.2MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 79%[==============> ] 2.55G 50.3MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 79%[==============> ] 2.56G 51.7MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 80%[===============> ] 2.58G 49.4MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-h 80%[===============> ] 2.59G 52.2MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs 81%[===============> ] 2.60G 48.0MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2 81%[===============> ] 2.62G 51.0MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x 81%[===============> ] 2.62G 48.0MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x- 82%[===============> ] 2.64G 49.9MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L 82%[===============> ] 2.65G 49.9MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6 82%[===============> ] 2.65G 49.1MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6- 83%[===============> ] 2.67G 48.6MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D 84%[===============> ] 2.70G 51.6MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4 84%[===============> ] 2.71G 49.3MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D40 84%[===============> ] 2.72G 50.9MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D409 85%[================> ] 2.73G 49.4MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096 85%[================> ] 2.75G 53.4MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096- 86%[================> ] 2.76G 50.7MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-hs2x-L6-D4096-E 86%[================> ] 2.77G 50.0MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-hs2x-L6-D4096-E0 86%[================> ] 2.78G 51.6MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-hs2x-L6-D4096-E0_ 86%[================> ] 2.79G 49.4MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-hs2x-L6-D4096-E0_1 87%[================> ] 2.80G 48.8MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "hs2x-L6-D4096-E0_1- 87%[================> ] 2.81G 48.5MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "s2x-L6-D4096-E0_1-m 87%[================> ] 2.82G 47.1MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "2x-L6-D4096-E0_1-me 88%[================> ] 2.82G 46.0MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-L6-D4096-E0_1-mem 88%[================> ] 2.84G 51.0MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L6-D4096-E0_1-mem- 89%[================> ] 2.86G 52.5MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L6-D4096-E0_1-mem-c 89%[================> ] 2.87G 49.9MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D4096-E0_1-mem-ct 89%[================> ] 2.88G 48.9MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D4096-E0_1-mem-ctx 90%[=================> ] 2.89G 48.9MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D4096-E0_1-mem-ctx- 90%[=================> ] 2.90G 49.9MB/s eta 6s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4096-E0_1-mem-ctx-8 90%[=================> ] 2.91G 50.5MB/s eta 6s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "096-E0_1-mem-ctx-8k 91%[=================> ] 2.92G 49.3MB/s eta 6s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-E0_1-mem-ctx-8k. 91%[=================> ] 2.93G 48.1MB/s eta 6s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-E0_1-mem-ctx-8k.p 91%[=================> ] 2.95G 46.8MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 92%[=================> ] 2.95G 46.9MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 92%[=================> ] 2.98G 50.2MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 93%[=================> ] 2.99G 51.3MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 93%[=================> ] 3.01G 52.1MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 94%[=================> ] 3.02G 52.2MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 94%[=================> ] 3.04G 55.1MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 94%[=================> ] 3.04G 53.9MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 95%[==================> ] 3.05G 54.2MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 95%[==================> ] 3.07G 56.2MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 96%[==================> ] 3.08G 55.3MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 96%[==================> ] 3.08G 53.1MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 96%[==================> ] 3.10G 51.6MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 97%[==================> ] 3.11G 51.2MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 97%[==================> ] 3.13G 54.8MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 97%[==================> ] 3.13G 52.5MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 98%[==================> ] 3.14G 54.3MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 98%[==================> ] 3.14G 51.0MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 98%[==================> ] 3.16G 52.4MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 99%[==================> ] 3.17G 51.3MB/s eta 1s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 99%[==================> ] 3.19G 50.0MB/s eta 1s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 99%[==================> ] 3.20G 50.3MB/s eta 1s \r", "v5-hs2x-L6-D4096-E0 100%[===================>] 3.21G 50.6MB/s in 67s \r\n", "\r\n", "2023-08-26 01:24:15 (48.8 MB/s) - ‘v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth’ saved [3441599109/3441599109]\r\n", "\r\n" ] } ], "source": [ "# Lets download the custom models\n", "!mkdir -p ../../../model/\n", "!cd ../../../model/ && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-headsize2x/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "7a2d2014", "metadata": { "execution": { "iopub.execute_input": "2023-08-26T01:24:15.329645Z", "iopub.status.busy": "2023-08-26T01:24:15.328840Z", "iopub.status.idle": "2023-08-26T01:24:15.584237Z", "shell.execute_reply": "2023-08-26T01:24:15.583291Z" }, "papermill": { "duration": 0.288951, "end_time": "2023-08-26T01:24:15.586329", "exception": false, "start_time": "2023-08-26T01:24:15.297378", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 3.3G\r\n", "-rw-r--r-- 1 root root 3.3G Aug 25 07:41 v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\r\n" ] } ], "source": [ "!ls -lh ../../../model/" ] }, { "cell_type": "code", "execution_count": 3, "id": "64a48fa1", "metadata": { "execution": { "iopub.execute_input": "2023-08-26T01:24:15.624728Z", "iopub.status.busy": "2023-08-26T01:24:15.623932Z", "iopub.status.idle": "2023-08-26T01:24:15.878118Z", "shell.execute_reply": "2023-08-26T01:24:15.877195Z" }, "papermill": { "duration": 0.276455, "end_time": "2023-08-26T01:24:15.880312", "exception": false, "start_time": "2023-08-26T01:24:15.603857", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/usr/bin/sh: 1: python: not found\r\n" ] } ], "source": [ "!python -m pip install aiocsv aiofiles" ] }, { "cell_type": "markdown", "id": "818142ee", "metadata": { "papermill": { "duration": 0.016531, "end_time": "2023-08-26T01:24:15.913644", "exception": false, "start_time": "2023-08-26T01:24:15.897113", "status": "completed" }, "tags": [] }, "source": [ "# 1B5 headsize 2x (128) - V5 benchmark" ] }, { "cell_type": "code", "execution_count": 4, "id": "65a14e4a", "metadata": { "execution": { "iopub.execute_input": "2023-08-26T01:24:15.949884Z", "iopub.status.busy": "2023-08-26T01:24:15.949229Z", "iopub.status.idle": "2023-08-26T01:25:32.703526Z", "shell.execute_reply": "2023-08-26T01:25:32.702547Z" }, "papermill": { "duration": 76.77519, "end_time": "2023-08-26T01:25:32.705679", "exception": false, "start_time": "2023-08-26T01:24:15.930489", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-08-26 01:24:20,440] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation start ###\r\n", "###\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5 tokens : 100.0% similarity, with 5 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10 tokens : 100.0% similarity, with 10 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15 tokens : 100.0% similarity, with 15 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 20 tokens : 100.0% similarity, with 20 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 25 tokens : 100.0% similarity, with 25 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 30 tokens : 100.0% similarity, with 30 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 35 tokens : 100.0% similarity, with 35 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 40 tokens : 100.0% similarity, with 40 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 45 tokens : 100.0% similarity, with 45 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 50 tokens : 100.0% similarity, with 50 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 55 tokens : 100.0% similarity, with 55 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 60 tokens : 98.33333333333333% similarity, with 59 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 65 tokens : 98.46153846153847% similarity, with 64 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 70 tokens : 98.57142857142858% similarity, with 69 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 75 tokens : 98.66666666666667% similarity, with 74 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 80 tokens : 96.25% similarity, with 77 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 85 tokens : 96.47058823529412% similarity, with 82 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 90 tokens : 96.66666666666667% similarity, with 87 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 95 tokens : 97.89473684210527% similarity, with 93 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 100 tokens : 99.0% similarity, with 99 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 105 tokens : 99.04761904761905% similarity, with 104 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 110 tokens : 98.18181818181819% similarity, with 108 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 115 tokens : 98.26086956521739% similarity, with 113 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 120 tokens : 98.33333333333333% similarity, with 118 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 125 tokens : 99.2% similarity, with 124 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 130 tokens : 99.23076923076923% similarity, with 129 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 135 tokens : 98.51851851851852% similarity, with 133 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 140 tokens : 98.57142857142858% similarity, with 138 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 145 tokens : 98.62068965517241% similarity, with 143 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 150 tokens : 98.66666666666667% similarity, with 148 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 160 tokens : 98.125% similarity, with 157 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 170 tokens : 97.6470588235294% similarity, with 166 matched token, and 4 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 180 tokens : 97.77777777777777% similarity, with 176 matched token, and 4 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 190 tokens : 98.42105263157895% similarity, with 187 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 200 tokens : 98.5% similarity, with 197 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 210 tokens : 99.04761904761905% similarity, with 208 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 220 tokens : 98.63636363636363% similarity, with 217 matched token, and 3 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 230 tokens : 99.1304347826087% similarity, with 228 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 240 tokens : 99.58333333333333% similarity, with 239 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 250 tokens : 99.6% similarity, with 249 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 260 tokens : 99.23076923076923% similarity, with 258 matched token, and 2 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 270 tokens : 98.14814814814815% similarity, with 265 matched token, and 5 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 280 tokens : 98.21428571428571% similarity, with 275 matched token, and 5 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 290 tokens : 97.93103448275862% similarity, with 284 matched token, and 6 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 300 tokens : 98.0% similarity, with 294 matched token, and 6 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 325 tokens : 98.15384615384616% similarity, with 319 matched token, and 6 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 350 tokens : 97.42857142857143% similarity, with 341 matched token, and 9 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 375 tokens : 96.0% similarity, with 360 matched token, and 15 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 400 tokens : 96.25% similarity, with 385 matched token, and 15 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 425 tokens : 95.52941176470588% similarity, with 406 matched token, and 19 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 450 tokens : 95.77777777777777% similarity, with 431 matched token, and 19 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 475 tokens : 95.36842105263158% similarity, with 453 matched token, and 22 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 500 tokens : 95.19999999999999% similarity, with 476 matched token, and 24 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 525 tokens : 94.47619047619048% similarity, with 496 matched token, and 29 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 550 tokens : 94.72727272727272% similarity, with 521 matched token, and 29 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 575 tokens : 94.78260869565217% similarity, with 545 matched token, and 30 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 600 tokens : 94.83333333333334% similarity, with 569 matched token, and 31 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 625 tokens : 94.88% similarity, with 593 matched token, and 32 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 650 tokens : 94.0% similarity, with 611 matched token, and 39 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 675 tokens : 94.37037037037037% similarity, with 637 matched token, and 38 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 700 tokens : 93.28571428571428% similarity, with 653 matched token, and 47 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 750 tokens : 93.2% similarity, with 699 matched token, and 51 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 800 tokens : 91.25% similarity, with 730 matched token, and 70 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 850 tokens : 90.23529411764706% similarity, with 767 matched token, and 83 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 900 tokens : 90.0% similarity, with 810 matched token, and 90 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 950 tokens : 89.57894736842105% similarity, with 851 matched token, and 99 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1000 tokens : 88.5% similarity, with 885 matched token, and 115 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Finished baseline model to eval output predictive matching (aka 0 memory?), for 1000 tokens\r\n", "###\r\n", "### Model validation end ###\r\n", "###\r\n" ] } ], "source": [ "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D4096-E0_1-1k.csv\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "7b65f1ce", "metadata": { "execution": { "iopub.execute_input": "2023-08-26T01:25:32.750445Z", "iopub.status.busy": "2023-08-26T01:25:32.749810Z", "iopub.status.idle": "2023-08-26T01:29:44.108998Z", "shell.execute_reply": "2023-08-26T01:29:44.108059Z" }, "papermill": { "duration": 251.385401, "end_time": "2023-08-26T01:29:44.112163", "exception": false, "start_time": "2023-08-26T01:25:32.726762", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-08-26 01:25:37,449] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation start ###\r\n", "###\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1100 tokens : 86.18181818181819% similarity, with 948 matched token, and 152 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1150 tokens : 85.04347826086956% similarity, with 978 matched token, and 172 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1200 tokens : 84.66666666666667% similarity, with 1016 matched token, and 184 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1250 tokens : 84.24000000000001% similarity, with 1053 matched token, and 197 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1300 tokens : 83.07692307692308% similarity, with 1080 matched token, and 220 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1350 tokens : 82.2962962962963% similarity, with 1111 matched token, and 239 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1400 tokens : 81.28571428571428% similarity, with 1138 matched token, and 262 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1450 tokens : 80.55172413793103% similarity, with 1168 matched token, and 282 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1500 tokens : 79.86666666666666% similarity, with 1198 matched token, and 302 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1550 tokens : 79.16129032258064% similarity, with 1227 matched token, and 323 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1600 tokens : 77.3125% similarity, with 1237 matched token, and 363 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1650 tokens : 75.63636363636364% similarity, with 1248 matched token, and 402 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1700 tokens : 74.41176470588235% similarity, with 1265 matched token, and 435 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1750 tokens : 72.39999999999999% similarity, with 1267 matched token, and 483 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1800 tokens : 70.22222222222221% similarity, with 1264 matched token, and 536 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1850 tokens : 69.62162162162161% similarity, with 1288 matched token, and 562 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1900 tokens : 68.84210526315789% similarity, with 1308 matched token, and 592 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1950 tokens : 67.02564102564102% similarity, with 1307 matched token, and 643 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2000 tokens : 66.3% similarity, with 1326 matched token, and 674 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2050 tokens : 64.04878048780488% similarity, with 1313 matched token, and 737 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2100 tokens : 63.66666666666667% similarity, with 1337 matched token, and 763 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2150 tokens : 62.83720930232558% similarity, with 1351 matched token, and 799 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2200 tokens : 61.68181818181818% similarity, with 1357 matched token, and 843 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2250 tokens : 60.84444444444445% similarity, with 1369 matched token, and 881 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2300 tokens : 58.434782608695656% similarity, with 1344 matched token, and 956 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2350 tokens : 57.48936170212766% similarity, with 1351 matched token, and 999 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2400 tokens : 56.458333333333336% similarity, with 1355 matched token, and 1045 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2450 tokens : 55.224489795918366% similarity, with 1353 matched token, and 1097 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2500 tokens : 54.400000000000006% similarity, with 1360 matched token, and 1140 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2550 tokens : 53.333333333333336% similarity, with 1360 matched token, and 1190 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2600 tokens : 51.76923076923077% similarity, with 1346 matched token, and 1254 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2650 tokens : 50.79245283018869% similarity, with 1346 matched token, and 1304 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2700 tokens : 49.81481481481482% similarity, with 1345 matched token, and 1355 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2750 tokens : 49.163636363636364% similarity, with 1352 matched token, and 1398 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2800 tokens : 48.285714285714285% similarity, with 1352 matched token, and 1448 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2850 tokens : 47.40350877192982% similarity, with 1351 matched token, and 1499 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2900 tokens : 46.310344827586206% similarity, with 1343 matched token, and 1557 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2950 tokens : 45.52542372881356% similarity, with 1343 matched token, and 1607 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3000 tokens : 44.56666666666666% similarity, with 1337 matched token, and 1663 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3050 tokens : 43.77049180327869% similarity, with 1335 matched token, and 1715 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3100 tokens : 42.45161290322581% similarity, with 1316 matched token, and 1784 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3150 tokens : 41.84126984126984% similarity, with 1318 matched token, and 1832 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3200 tokens : 41.1875% similarity, with 1318 matched token, and 1882 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3250 tokens : 40.43076923076923% similarity, with 1314 matched token, and 1936 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3300 tokens : 39.484848484848484% similarity, with 1303 matched token, and 1997 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3350 tokens : 38.776119402985074% similarity, with 1299 matched token, and 2051 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3400 tokens : 37.6764705882353% similarity, with 1281 matched token, and 2119 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3450 tokens : 36.608695652173914% similarity, with 1263 matched token, and 2187 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3500 tokens : 35.97142857142857% similarity, with 1259 matched token, and 2241 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3550 tokens : 35.267605633802816% similarity, with 1252 matched token, and 2298 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3600 tokens : 34.72222222222222% similarity, with 1250 matched token, and 2350 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3650 tokens : 33.91780821917808% similarity, with 1238 matched token, and 2412 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3700 tokens : 33.37837837837838% similarity, with 1235 matched token, and 2465 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3750 tokens : 32.4% similarity, with 1215 matched token, and 2535 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3800 tokens : 31.5% similarity, with 1197 matched token, and 2603 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3850 tokens : 30.80519480519481% similarity, with 1186 matched token, and 2664 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3900 tokens : 29.897435897435898% similarity, with 1166 matched token, and 2734 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3950 tokens : 29.012658227848103% similarity, with 1146 matched token, and 2804 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4000 tokens : 28.025% similarity, with 1121 matched token, and 2879 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation end ###\r\n", "###\r\n" ] } ], "source": [ "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D4096-E0_1-4k.csv\" 1100 4000" ] }, { "cell_type": "code", "execution_count": 6, "id": "4de9facd", "metadata": { "execution": { "iopub.execute_input": "2023-08-26T01:29:44.173983Z", "iopub.status.busy": "2023-08-26T01:29:44.173478Z", "iopub.status.idle": "2023-08-26T02:33:49.325748Z", "shell.execute_reply": "2023-08-26T02:33:49.324578Z" }, "papermill": { "duration": 3845.184098, "end_time": "2023-08-26T02:33:49.328432", "exception": false, "start_time": "2023-08-26T01:29:44.144334", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-08-26 01:29:48,903] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation start ###\r\n", "###\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4100 tokens : 26.195121951219512% similarity, with 1074 matched token, and 3026 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4200 tokens : 24.952380952380953% similarity, with 1048 matched token, and 3152 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4300 tokens : 23.13953488372093% similarity, with 995 matched token, and 3305 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4400 tokens : 21.386363636363637% similarity, with 941 matched token, and 3459 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4500 tokens : 19.2% similarity, with 864 matched token, and 3636 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4600 tokens : 16.97826086956522% similarity, with 781 matched token, and 3819 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4700 tokens : 15.319148936170212% similarity, with 720 matched token, and 3980 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4800 tokens : 13.645833333333332% similarity, with 655 matched token, and 4145 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4900 tokens : 12.306122448979592% similarity, with 603 matched token, and 4297 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5000 tokens : 10.92% similarity, with 546 matched token, and 4454 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5100 tokens : 9.450980392156863% similarity, with 482 matched token, and 4618 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5200 tokens : 7.98076923076923% similarity, with 415 matched token, and 4785 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5300 tokens : 7.283018867924529% similarity, with 386 matched token, and 4914 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5400 tokens : 6.12962962962963% similarity, with 331 matched token, and 5069 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5500 tokens : 5.50909090909091% similarity, with 303 matched token, and 5197 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5600 tokens : 5.017857142857142% similarity, with 281 matched token, and 5319 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5700 tokens : 4.385964912280701% similarity, with 250 matched token, and 5450 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5800 tokens : 4.068965517241379% similarity, with 236 matched token, and 5564 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5900 tokens : 3.8474576271186436% similarity, with 227 matched token, and 5673 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6000 tokens : 3.716666666666667% similarity, with 223 matched token, and 5777 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6100 tokens : 3.4918032786885247% similarity, with 213 matched token, and 5887 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6200 tokens : 3.3225806451612905% similarity, with 206 matched token, and 5994 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6300 tokens : 3.2222222222222223% similarity, with 203 matched token, and 6097 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6400 tokens : 3.2031249999999996% similarity, with 205 matched token, and 6195 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6500 tokens : 3.1846153846153844% similarity, with 207 matched token, and 6293 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6600 tokens : 3.1363636363636362% similarity, with 207 matched token, and 6393 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6700 tokens : 3.119402985074627% similarity, with 209 matched token, and 6491 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6800 tokens : 3.088235294117647% similarity, with 210 matched token, and 6590 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6900 tokens : 3.173913043478261% similarity, with 219 matched token, and 6681 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7000 tokens : 3.1285714285714286% similarity, with 219 matched token, and 6781 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7100 tokens : 3.140845070422535% similarity, with 223 matched token, and 6877 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7200 tokens : 3.111111111111111% similarity, with 224 matched token, and 6976 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7300 tokens : 3.1643835616438354% similarity, with 231 matched token, and 7069 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7400 tokens : 3.1621621621621623% similarity, with 234 matched token, and 7166 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7500 tokens : 3.1466666666666665% similarity, with 236 matched token, and 7264 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7600 tokens : 3.1578947368421053% similarity, with 240 matched token, and 7360 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7700 tokens : 3.1558441558441555% similarity, with 243 matched token, and 7457 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7800 tokens : 3.1538461538461537% similarity, with 246 matched token, and 7554 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7900 tokens : 3.151898734177215% similarity, with 249 matched token, and 7651 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8000 tokens : 3.1375% similarity, with 251 matched token, and 7749 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8100 tokens : 3.185185185185185% similarity, with 258 matched token, and 7842 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8200 tokens : 3.1707317073170733% similarity, with 260 matched token, and 7940 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8300 tokens : 3.156626506024096% similarity, with 262 matched token, and 8038 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8400 tokens : 3.130952380952381% similarity, with 263 matched token, and 8137 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8500 tokens : 3.2% similarity, with 272 matched token, and 8228 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8600 tokens : 3.1976744186046515% similarity, with 275 matched token, and 8325 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8700 tokens : 3.206896551724138% similarity, with 279 matched token, and 8421 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8800 tokens : 3.193181818181818% similarity, with 281 matched token, and 8519 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8900 tokens : 3.191011235955056% similarity, with 284 matched token, and 8616 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9000 tokens : 3.188888888888889% similarity, with 287 matched token, and 8713 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9100 tokens : 3.208791208791209% similarity, with 292 matched token, and 8808 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9200 tokens : 3.2282608695652173% similarity, with 297 matched token, and 8903 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9300 tokens : 3.21505376344086% similarity, with 299 matched token, and 9001 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9400 tokens : 3.1914893617021276% similarity, with 300 matched token, and 9100 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9500 tokens : 3.2105263157894735% similarity, with 305 matched token, and 9195 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9600 tokens : 3.21875% similarity, with 309 matched token, and 9291 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9700 tokens : 3.216494845360825% similarity, with 312 matched token, and 9388 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9800 tokens : 3.2040816326530615% similarity, with 314 matched token, and 9486 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9900 tokens : 3.2222222222222223% similarity, with 319 matched token, and 9581 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10000 tokens : 3.25% similarity, with 325 matched token, and 9675 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10100 tokens : 3.2574257425742577% similarity, with 329 matched token, and 9771 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10200 tokens : 3.2549019607843137% similarity, with 332 matched token, and 9868 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10300 tokens : 3.2427184466019416% similarity, with 334 matched token, and 9966 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10400 tokens : 3.240384615384615% similarity, with 337 matched token, and 10063 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10500 tokens : 3.257142857142857% similarity, with 342 matched token, and 10158 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10600 tokens : 3.2547169811320753% similarity, with 345 matched token, and 10255 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10700 tokens : 3.2710280373831773% similarity, with 350 matched token, and 10350 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10800 tokens : 3.2870370370370368% similarity, with 355 matched token, and 10445 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10900 tokens : 3.2660550458715596% similarity, with 356 matched token, and 10544 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11000 tokens : 3.2818181818181817% similarity, with 361 matched token, and 10639 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11100 tokens : 3.288288288288288% similarity, with 365 matched token, and 10735 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11200 tokens : 3.2857142857142856% similarity, with 368 matched token, and 10832 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11300 tokens : 3.2920353982300887% similarity, with 372 matched token, and 10928 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11400 tokens : 3.324561403508772% similarity, with 379 matched token, and 11021 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11500 tokens : 3.3391304347826085% similarity, with 384 matched token, and 11116 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11600 tokens : 3.3448275862068964% similarity, with 388 matched token, and 11212 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11700 tokens : 3.3247863247863245% similarity, with 389 matched token, and 11311 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11800 tokens : 3.347457627118644% similarity, with 395 matched token, and 11405 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11900 tokens : 3.3361344537815127% similarity, with 397 matched token, and 11503 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12000 tokens : 3.3416666666666663% similarity, with 401 matched token, and 11599 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12100 tokens : 3.3636363636363638% similarity, with 407 matched token, and 11693 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12200 tokens : 3.344262295081967% similarity, with 408 matched token, and 11792 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12300 tokens : 3.3577235772357725% similarity, with 413 matched token, and 11887 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12400 tokens : 3.362903225806452% similarity, with 417 matched token, and 11983 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12500 tokens : 3.3680000000000003% similarity, with 421 matched token, and 12079 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12600 tokens : 3.3650793650793656% similarity, with 424 matched token, and 12176 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12700 tokens : 3.362204724409449% similarity, with 427 matched token, and 12273 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12800 tokens : 3.359375% similarity, with 430 matched token, and 12370 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12900 tokens : 3.3643410852713176% similarity, with 434 matched token, and 12466 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13000 tokens : 3.3692307692307697% similarity, with 438 matched token, and 12562 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13100 tokens : 3.381679389312977% similarity, with 443 matched token, and 12657 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13200 tokens : 3.3939393939393945% similarity, with 448 matched token, and 12752 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13300 tokens : 3.37593984962406% similarity, with 449 matched token, and 12851 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13400 tokens : 3.395522388059701% similarity, with 455 matched token, and 12945 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13500 tokens : 3.3851851851851853% similarity, with 457 matched token, and 13043 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13600 tokens : 3.4044117647058822% similarity, with 463 matched token, and 13137 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13700 tokens : 3.386861313868613% similarity, with 464 matched token, and 13236 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13800 tokens : 3.398550724637681% similarity, with 469 matched token, and 13331 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13900 tokens : 3.3884892086330933% similarity, with 471 matched token, and 13429 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14000 tokens : 3.4357142857142855% similarity, with 481 matched token, and 13519 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14100 tokens : 3.4255319148936167% similarity, with 483 matched token, and 13617 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14200 tokens : 3.408450704225352% similarity, with 484 matched token, and 13716 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14300 tokens : 3.4055944055944054% similarity, with 487 matched token, and 13813 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14400 tokens : 3.4166666666666665% similarity, with 492 matched token, and 13908 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14500 tokens : 3.420689655172414% similarity, with 496 matched token, and 14004 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14600 tokens : 3.404109589041096% similarity, with 497 matched token, and 14103 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14700 tokens : 3.4421768707482996% similarity, with 506 matched token, and 14194 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14800 tokens : 3.4391891891891895% similarity, with 509 matched token, and 14291 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14900 tokens : 3.422818791946309% similarity, with 510 matched token, and 14390 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15000 tokens : 3.42% similarity, with 513 matched token, and 14487 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15100 tokens : 3.4172185430463573% similarity, with 516 matched token, and 14584 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15200 tokens : 3.4078947368421058% similarity, with 518 matched token, and 14682 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15300 tokens : 3.4117647058823533% similarity, with 522 matched token, and 14778 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15400 tokens : 3.4090909090909087% similarity, with 525 matched token, and 14875 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15500 tokens : 3.4000000000000004% similarity, with 527 matched token, and 14973 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15600 tokens : 3.4038461538461537% similarity, with 531 matched token, and 15069 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15700 tokens : 3.4012738853503186% similarity, with 534 matched token, and 15166 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15800 tokens : 3.3924050632911396% similarity, with 536 matched token, and 15264 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15900 tokens : 3.3962264150943398% similarity, with 540 matched token, and 15360 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 16000 tokens : 3.3875% similarity, with 542 matched token, and 15458 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation end ###\r\n", "###\r\n" ] } ], "source": [ "!python3 ./memory_script/eval_v5headsize2x_memory_guided.py \"../../../model/v5-hs2x-L6-D4096-E0_1-mem-ctx-8k.pth\" \"./logs/v5-hs2x-L6-D4096-E0_1-16k.csv\" 4100 16000" ] } ], "metadata": { "kernelspec": { "display_name": "rwkv-infctx", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "papermill": { "default_parameters": {}, "duration": 4243.976981, "end_time": "2023-08-26T02:33:49.795764", "environment_variables": {}, "exception": null, "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/memory-bench/Benchmark-V5headsize2x.ipynb", "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/memory-bench/Benchmark-V5headsize2x.ipynb", "parameters": {}, "start_time": "2023-08-26T01:23:05.818783", "version": "2.4.0" } }, "nbformat": 4, "nbformat_minor": 5 }