{ "cells": [ { "cell_type": "markdown", "id": "935eee73", "metadata": { "papermill": { "duration": 0.002228, "end_time": "2023-09-03T20:06:45.853194", "exception": false, "start_time": "2023-09-03T20:06:45.850966", "status": "completed" }, "tags": [] }, "source": [ "## Custom Experimental Memory benchmarking\n", "\n", "The following is meant to observe the memory performance in a more verbose CSV logging mode, for the baseline raven models" ] }, { "cell_type": "code", "execution_count": 1, "id": "b3e65aae", "metadata": { "execution": { "iopub.execute_input": "2023-09-03T20:06:45.858732Z", "iopub.status.busy": "2023-09-03T20:06:45.858219Z", "iopub.status.idle": "2023-09-03T20:07:31.745793Z", "shell.execute_reply": "2023-09-03T20:07:31.744962Z" }, "papermill": { "duration": 45.892824, "end_time": "2023-09-03T20:07:31.748027", "exception": false, "start_time": "2023-09-03T20:06:45.855203", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2023-09-03 20:06:46-- https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-memory/v5-L96-D1024-E0_1-mem-ctx-8k.pth\r\n", "Resolving huggingface.co (huggingface.co)... 18.154.227.87, 18.154.227.67, 18.154.227.7, ...\r\n", "Connecting to huggingface.co (huggingface.co)|18.154.227.87|:443... connected.\r\n", "HTTP request sent, awaiting response... 302 Found\r\n", "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/0d042262601b79b1635bdd82e73c9f26fb35b05d8cded92a03aad5df56944dde?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-L96-D1024-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-L96-D1024-E0_1-mem-ctx-8k.pth%22%3B&Expires=1694030806&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NDAzMDgwNn19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzBkMDQyMjYyNjAxYjc5YjE2MzViZGQ4MmU3M2M5ZjI2ZmIzNWIwNWQ4Y2RlZDkyYTAzYWFkNWRmNTY5NDRkZGU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=SDWEHufhsERLxOVC62L657VMlF9FQvyFnUfrwnY-g7h9EM07TNLrNgAr8AtlmzcnXA97w-JgfLHTabd9DcGDpfyB9veIW3e1AmVPaP5MooaFI9Unw%7EUxSgmQ25ZKgoikqx4THDNtjGMtkk6%7EsP6W4zM2x1nhZhxcku%7EGOCF3f66kE%7E8v4MqNkc0azV8yIotP3Vs5nSilEl-9I7n9d9JZtkykMCMLfSQz4auu1hQ5N6qrcZsS3X0aMVNmIZOmR2MEJ2t9dpKeZQA1KhNKIt1vdOaiWHeK4inCh3p24wBHHzYCaBVHLwto2nfSnET8SBkeh2ODRG%7EAVr-SNvA9mez%7E3w__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n", "--2023-09-03 20:06:46-- https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/0d042262601b79b1635bdd82e73c9f26fb35b05d8cded92a03aad5df56944dde?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-L96-D1024-E0_1-mem-ctx-8k.pth%3B+filename%3D%22v5-L96-D1024-E0_1-mem-ctx-8k.pth%22%3B&Expires=1694030806&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NDAzMDgwNn19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzBkMDQyMjYyNjAxYjc5YjE2MzViZGQ4MmU3M2M5ZjI2ZmIzNWIwNWQ4Y2RlZDkyYTAzYWFkNWRmNTY5NDRkZGU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=SDWEHufhsERLxOVC62L657VMlF9FQvyFnUfrwnY-g7h9EM07TNLrNgAr8AtlmzcnXA97w-JgfLHTabd9DcGDpfyB9veIW3e1AmVPaP5MooaFI9Unw%7EUxSgmQ25ZKgoikqx4THDNtjGMtkk6%7EsP6W4zM2x1nhZhxcku%7EGOCF3f66kE%7E8v4MqNkc0azV8yIotP3Vs5nSilEl-9I7n9d9JZtkykMCMLfSQz4auu1hQ5N6qrcZsS3X0aMVNmIZOmR2MEJ2t9dpKeZQA1KhNKIt1vdOaiWHeK4inCh3p24wBHHzYCaBVHLwto2nfSnET8SBkeh2ODRG%7EAVr-SNvA9mez%7E3w__&Key-Pair-Id=KVTP0A1DKRTAX\r\n", "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 18.165.98.12, 18.165.98.107, 18.165.98.51, ...\r\n", "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|18.165.98.12|:443... connected.\r\n", "HTTP request sent, awaiting response... " ] }, { "name": "stdout", "output_type": "stream", "text": [ "200 OK\r\n", "Length: 2825976699 (2.6G) [binary/octet-stream]\r\n", "Saving to: ‘v5-L96-D1024-E0_1-mem-ctx-8k.pth’\r\n", "\r\n", "\r", " v5-L96-D1 0%[ ] 0 --.-KB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D10 0%[ ] 15.26M 75.6MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D102 1%[ ] 30.52M 72.6MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024 2%[ ] 60.52M 83.5MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024- 2%[ ] 75.78M 78.0MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E 3%[ ] 85.23M 72.6MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0 3%[ ] 106.29M 72.7MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_ 4%[ ] 121.56M 70.0MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1 5%[> ] 137.33M 69.5MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1- 5%[> ] 152.59M 65.3MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-L96-D1024-E0_1-m 6%[> ] 167.33M 64.2MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-L96-D1024-E0_1-me 6%[> ] 183.10M 63.4MB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L96-D1024-E0_1-mem 7%[> ] 197.17M 63.9MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L96-D1024-E0_1-mem- 7%[> ] 198.49M 59.7MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-D1024-E0_1-mem-c 7%[> ] 213.62M 60.1MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D1024-E0_1-mem-ct 8%[> ] 228.68M 60.3MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D1024-E0_1-mem-ctx 8%[> ] 237.25M 59.4MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D1024-E0_1-mem-ctx- 9%[> ] 246.79M 58.0MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1024-E0_1-mem-ctx-8 9%[> ] 259.40M 57.5MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "024-E0_1-mem-ctx-8k 10%[=> ] 279.43M 56.9MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "24-E0_1-mem-ctx-8k. 10%[=> ] 289.92M 56.3MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4-E0_1-mem-ctx-8k.p 11%[=> ] 305.18M 57.6MB/s eta 42s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 11%[=> ] 320.43M 57.6MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 12%[=> ] 335.69M 58.3MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 13%[=> ] 350.44M 58.7MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 13%[=> ] 366.02M 58.6MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 14%[=> ] 380.96M 56.5MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 14%[=> ] 384.09M 56.8MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 14%[=> ] 400.91M 57.5MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 15%[==> ] 411.99M 56.2MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 16%[==> ] 432.22M 59.3MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 16%[==> ] 442.51M 60.0MB/s eta 39s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 17%[==> ] 459.95M 60.2MB/s eta 37s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 17%[==> ] 481.46M 65.0MB/s eta 37s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 18%[==> ] 488.28M 62.3MB/s eta 37s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 18%[==> ] 503.54M 60.5MB/s eta 37s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 19%[==> ] 518.80M 59.3MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 19%[==> ] 533.54M 59.4MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 19%[==> ] 534.18M 56.4MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 20%[===> ] 549.31M 55.9MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 21%[===> ] 578.99M 59.9MB/s eta 36s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 22%[===> ] 595.09M 60.9MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 23%[===> ] 623.63M 64.2MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 23%[===> ] 638.16M 67.6MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 23%[===> ] 640.87M 63.8MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 24%[===> ] 656.13M 62.9MB/s eta 35s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L 24%[===> ] 671.38M 63.8MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L9 25%[====> ] 686.64M 63.1MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96 26%[====> ] 701.90M 60.1MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96- 26%[====> ] 717.16M 58.5MB/s eta 33s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D 27%[====> ] 747.69M 62.3MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1 28%[====> ] 762.94M 62.0MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D10 29%[====> ] 791.63M 63.5MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D102 29%[====> ] 808.20M 66.8MB/s eta 32s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024 30%[=====> ] 823.46M 67.7MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024- 31%[=====> ] 838.71M 66.9MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E 31%[=====> ] 849.79M 63.1MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0 31%[=====> ] 854.49M 59.1MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_ 32%[=====> ] 869.75M 58.2MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1 32%[=====> ] 885.01M 57.6MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1- 33%[=====> ] 900.27M 59.6MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-L96-D1024-E0_1-m 34%[=====> ] 930.27M 61.4MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-L96-D1024-E0_1-me 35%[======> ] 944.21M 61.1MB/s eta 30s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L96-D1024-E0_1-mem 35%[======> ] 961.30M 61.7MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L96-D1024-E0_1-mem- 36%[======> ] 987.50M 66.6MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-D1024-E0_1-mem-c 37%[======> ] 1003M 65.9MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D1024-E0_1-mem-ct 37%[======> ] 1007M 62.2MB/s eta 28s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D1024-E0_1-mem-ctx 37%[======> ] 1022M 59.2MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D1024-E0_1-mem-ctx- 38%[======> ] 1.02G 63.5MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1024-E0_1-mem-ctx-8 39%[======> ] 1.03G 62.8MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "024-E0_1-mem-ctx-8k 39%[======> ] 1.04G 61.9MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "24-E0_1-mem-ctx-8k. 40%[=======> ] 1.06G 61.3MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4-E0_1-mem-ctx-8k.p 40%[=======> ] 1.06G 56.4MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 40%[=======> ] 1.07G 59.2MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 41%[=======> ] 1.09G 58.0MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 41%[=======> ] 1.10G 56.0MB/s eta 27s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 42%[=======> ] 1.12G 53.6MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 43%[=======> ] 1.13G 54.0MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 43%[=======> ] 1.15G 56.0MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 44%[=======> ] 1.16G 54.6MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 45%[========> ] 1.19G 58.3MB/s eta 26s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 45%[========> ] 1.19G 56.3MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 45%[========> ] 1.20G 52.6MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 45%[========> ] 1.21G 51.7MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 46%[========> ] 1.21G 48.9MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 46%[========> ] 1.22G 48.4MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 46%[========> ] 1.24G 50.7MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 47%[========> ] 1.25G 53.1MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 48%[========> ] 1.27G 53.6MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 48%[========> ] 1.28G 55.4MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 49%[========> ] 1.30G 54.9MB/s eta 24s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 49%[========> ] 1.31G 56.4MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 50%[=========> ] 1.33G 57.4MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 51%[=========> ] 1.35G 59.5MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 52%[=========> ] 1.37G 63.2MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 52%[=========> ] 1.39G 62.4MB/s eta 23s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 53%[=========> ] 1.42G 65.6MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L 54%[=========> ] 1.42G 61.6MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L9 54%[=========> ] 1.43G 61.4MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96 54%[=========> ] 1.45G 64.2MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96- 56%[==========> ] 1.47G 68.8MB/s eta 20s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D 56%[==========> ] 1.48G 69.0MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1 56%[==========> ] 1.49G 69.3MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D10 57%[==========> ] 1.51G 69.7MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D102 57%[==========> ] 1.52G 70.1MB/s eta 19s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024 58%[==========> ] 1.53G 68.6MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024- 58%[==========> ] 1.55G 68.1MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E 59%[==========> ] 1.56G 66.1MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0 59%[==========> ] 1.56G 63.2MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_ 60%[===========> ] 1.58G 61.6MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1 60%[===========> ] 1.60G 62.0MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1- 61%[===========> ] 1.61G 61.1MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-L96-D1024-E0_1-m 61%[===========> ] 1.62G 58.0MB/s eta 18s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-L96-D1024-E0_1-me 62%[===========> ] 1.64G 55.9MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L96-D1024-E0_1-mem 63%[===========> ] 1.66G 58.9MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L96-D1024-E0_1-mem- 63%[===========> ] 1.68G 60.2MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-D1024-E0_1-mem-c 63%[===========> ] 1.68G 58.3MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D1024-E0_1-mem-ct 64%[===========> ] 1.70G 56.5MB/s eta 17s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D1024-E0_1-mem-ctx 65%[============> ] 1.72G 57.9MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D1024-E0_1-mem-ctx- 65%[============> ] 1.73G 55.8MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1024-E0_1-mem-ctx-8 66%[============> ] 1.74G 57.3MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "024-E0_1-mem-ctx-8k 66%[============> ] 1.76G 58.9MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "24-E0_1-mem-ctx-8k. 67%[============> ] 1.77G 58.6MB/s eta 15s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4-E0_1-mem-ctx-8k.p 67%[============> ] 1.79G 57.9MB/s eta 14s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 68%[============> ] 1.80G 61.0MB/s eta 14s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 68%[============> ] 1.81G 62.0MB/s eta 14s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 69%[============> ] 1.82G 61.4MB/s eta 14s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 69%[============> ] 1.83G 60.9MB/s eta 14s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 70%[=============> ] 1.85G 60.6MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 70%[=============> ] 1.86G 61.8MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 70%[=============> ] 1.86G 57.3MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 71%[=============> ] 1.88G 59.0MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 71%[=============> ] 1.89G 57.4MB/s eta 13s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-ctx-8k.pth 72%[=============> ] 1.91G 56.1MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "ctx-8k.pth 72%[=============> ] 1.92G 57.0MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "tx-8k.pth 73%[=============> ] 1.94G 58.6MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "x-8k.pth 74%[=============> ] 1.95G 57.5MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-8k.pth 74%[=============> ] 1.97G 56.4MB/s eta 12s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "8k.pth 75%[==============> ] 1.98G 56.0MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "k.pth 75%[==============> ] 2.00G 56.1MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", ".pth 76%[==============> ] 2.01G 57.9MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "pth 76%[==============> ] 2.03G 56.8MB/s eta 11s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "th 77%[==============> ] 2.03G 54.3MB/s eta 10s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "h 77%[==============> ] 2.04G 54.7MB/s eta 10s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 78%[==============> ] 2.06G 55.1MB/s eta 10s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v 78%[==============> ] 2.07G 53.0MB/s eta 10s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5 79%[==============> ] 2.09G 55.6MB/s eta 10s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5- 79%[==============> ] 2.10G 58.3MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L 80%[===============> ] 2.13G 60.8MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L9 81%[===============> ] 2.14G 62.6MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96 81%[===============> ] 2.15G 58.8MB/s eta 9s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96- 81%[===============> ] 2.15G 56.6MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D 82%[===============> ] 2.16G 54.1MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1 82%[===============> ] 2.17G 51.9MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D10 83%[===============> ] 2.19G 55.9MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D102 83%[===============> ] 2.21G 55.4MB/s eta 8s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024 84%[===============> ] 2.22G 56.0MB/s eta 7s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024- 85%[================> ] 2.24G 59.5MB/s eta 7s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E 85%[================> ] 2.26G 61.4MB/s eta 7s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0 86%[================> ] 2.26G 60.3MB/s eta 7s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_ 86%[================> ] 2.29G 62.5MB/s eta 7s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1 87%[================> ] 2.31G 63.7MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " v5-L96-D1024-E0_1- 88%[================> ] 2.32G 63.8MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-L96-D1024-E0_1-m 88%[================> ] 2.34G 64.3MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "5-L96-D1024-E0_1-me 89%[================> ] 2.35G 59.2MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-L96-D1024-E0_1-mem 90%[=================> ] 2.37G 61.7MB/s eta 5s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "L96-D1024-E0_1-mem- 90%[=================> ] 2.38G 61.9MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "96-D1024-E0_1-mem-c 91%[=================> ] 2.40G 63.2MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "6-D1024-E0_1-mem-ct 91%[=================> ] 2.41G 64.0MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-D1024-E0_1-mem-ctx 91%[=================> ] 2.42G 60.8MB/s eta 4s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "D1024-E0_1-mem-ctx- 92%[=================> ] 2.43G 60.4MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1024-E0_1-mem-ctx-8 92%[=================> ] 2.44G 59.7MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "024-E0_1-mem-ctx-8k 93%[=================> ] 2.46G 58.9MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "24-E0_1-mem-ctx-8k. 93%[=================> ] 2.47G 57.9MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "4-E0_1-mem-ctx-8k.p 94%[=================> ] 2.48G 55.6MB/s eta 3s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-E0_1-mem-ctx-8k.pt 94%[=================> ] 2.49G 55.6MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "E0_1-mem-ctx-8k.pth 95%[==================> ] 2.50G 54.6MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "0_1-mem-ctx-8k.pth 95%[==================> ] 2.52G 53.4MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "_1-mem-ctx-8k.pth 96%[==================> ] 2.53G 51.0MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "1-mem-ctx-8k.pth 96%[==================> ] 2.55G 53.4MB/s eta 2s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "-mem-ctx-8k.pth 97%[==================> ] 2.56G 54.0MB/s eta 1s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "mem-ctx-8k.pth 97%[==================> ] 2.58G 54.1MB/s eta 1s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "em-ctx-8k.pth 98%[==================> ] 2.59G 55.6MB/s eta 1s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "m-ctx-8k.pth 99%[==================> ] 2.62G 58.9MB/s eta 1s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", "v5-L96-D1024-E0_1-m 100%[===================>] 2.63G 58.3MB/s in 45s \r\n", "\r\n", "2023-09-03 20:07:31 (59.8 MB/s) - ‘v5-L96-D1024-E0_1-mem-ctx-8k.pth’ saved [2825976699/2825976699]\r\n", "\r\n" ] } ], "source": [ "# Lets download the custom models\n", "!mkdir -p ../../../model/\n", "!cd ../../../model/ && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-memory/v5-L96-D1024-E0_1-mem-ctx-8k.pth\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "f464907a", "metadata": { "execution": { "iopub.execute_input": "2023-09-03T20:07:31.774310Z", "iopub.status.busy": "2023-09-03T20:07:31.773743Z", "iopub.status.idle": "2023-09-03T20:07:32.029470Z", "shell.execute_reply": "2023-09-03T20:07:32.028523Z" }, "papermill": { "duration": 0.27144, "end_time": "2023-09-03T20:07:32.031600", "exception": false, "start_time": "2023-09-03T20:07:31.760160", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 2.7G\r\n", "-rw-r--r-- 1 root root 2.7G Sep 3 06:19 v5-L96-D1024-E0_1-mem-ctx-8k.pth\r\n" ] } ], "source": [ "!ls -lh ../../../model/" ] }, { "cell_type": "code", "execution_count": 3, "id": "aacf07e4", "metadata": { "execution": { "iopub.execute_input": "2023-09-03T20:07:32.058135Z", "iopub.status.busy": "2023-09-03T20:07:32.057224Z", "iopub.status.idle": "2023-09-03T20:07:32.309265Z", "shell.execute_reply": "2023-09-03T20:07:32.308436Z" }, "papermill": { "duration": 0.267871, "end_time": "2023-09-03T20:07:32.311441", "exception": false, "start_time": "2023-09-03T20:07:32.043570", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/usr/bin/sh: 1: python: not found\r\n" ] } ], "source": [ "!python -m pip install aiocsv aiofiles" ] }, { "cell_type": "markdown", "id": "9252473f", "metadata": { "papermill": { "duration": 0.011632, "end_time": "2023-09-03T20:07:32.334991", "exception": false, "start_time": "2023-09-03T20:07:32.323359", "status": "completed" }, "tags": [] }, "source": [ "# 1B5 V5 benchmark" ] }, { "cell_type": "code", "execution_count": 4, "id": "deffae97", "metadata": { "execution": { "iopub.execute_input": "2023-09-03T20:07:32.361241Z", "iopub.status.busy": "2023-09-03T20:07:32.360502Z", "iopub.status.idle": "2023-09-03T20:09:22.587085Z", "shell.execute_reply": "2023-09-03T20:09:22.585615Z" }, "papermill": { "duration": 110.242708, "end_time": "2023-09-03T20:09:22.589584", "exception": false, "start_time": "2023-09-03T20:07:32.346876", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-09-03 20:07:36,900] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation start ###\r\n", "###\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5 tokens : 100.0% similarity, with 5 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10 tokens : 100.0% similarity, with 10 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15 tokens : 100.0% similarity, with 15 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 20 tokens : 100.0% similarity, with 20 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 25 tokens : 100.0% similarity, with 25 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 30 tokens : 100.0% similarity, with 30 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 35 tokens : 100.0% similarity, with 35 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 40 tokens : 100.0% similarity, with 40 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 45 tokens : 100.0% similarity, with 45 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 50 tokens : 100.0% similarity, with 50 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 55 tokens : 100.0% similarity, with 55 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 60 tokens : 100.0% similarity, with 60 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 65 tokens : 100.0% similarity, with 65 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 70 tokens : 100.0% similarity, with 70 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 75 tokens : 100.0% similarity, with 75 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 80 tokens : 100.0% similarity, with 80 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 85 tokens : 100.0% similarity, with 85 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 90 tokens : 100.0% similarity, with 90 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 95 tokens : 100.0% similarity, with 95 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 100 tokens : 100.0% similarity, with 100 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 105 tokens : 100.0% similarity, with 105 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 110 tokens : 100.0% similarity, with 110 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 115 tokens : 100.0% similarity, with 115 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 120 tokens : 100.0% similarity, with 120 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 125 tokens : 100.0% similarity, with 125 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 130 tokens : 100.0% similarity, with 130 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 135 tokens : 100.0% similarity, with 135 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 140 tokens : 100.0% similarity, with 140 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 145 tokens : 100.0% similarity, with 145 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 150 tokens : 100.0% similarity, with 150 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 160 tokens : 100.0% similarity, with 160 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 170 tokens : 100.0% similarity, with 170 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 180 tokens : 100.0% similarity, with 180 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 190 tokens : 100.0% similarity, with 190 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 200 tokens : 99.5% similarity, with 199 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 210 tokens : 99.52380952380952% similarity, with 209 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 220 tokens : 99.54545454545455% similarity, with 219 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 230 tokens : 100.0% similarity, with 230 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 240 tokens : 100.0% similarity, with 240 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 250 tokens : 99.6% similarity, with 249 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 260 tokens : 99.61538461538461% similarity, with 259 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 270 tokens : 99.62962962962963% similarity, with 269 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 280 tokens : 100.0% similarity, with 280 matched token, and 0 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 290 tokens : 99.6551724137931% similarity, with 289 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 300 tokens : 99.66666666666667% similarity, with 299 matched token, and 1 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 325 tokens : 98.76923076923076% similarity, with 321 matched token, and 4 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 350 tokens : 98.85714285714286% similarity, with 346 matched token, and 4 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 375 tokens : 98.66666666666667% similarity, with 370 matched token, and 5 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 400 tokens : 98.75% similarity, with 395 matched token, and 5 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 425 tokens : 98.82352941176471% similarity, with 420 matched token, and 5 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 450 tokens : 98.88888888888889% similarity, with 445 matched token, and 5 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 475 tokens : 98.52631578947368% similarity, with 468 matched token, and 7 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 500 tokens : 97.8% similarity, with 489 matched token, and 11 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 525 tokens : 97.90476190476191% similarity, with 514 matched token, and 11 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 550 tokens : 97.81818181818181% similarity, with 538 matched token, and 12 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 575 tokens : 97.3913043478261% similarity, with 560 matched token, and 15 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 600 tokens : 97.33333333333334% similarity, with 584 matched token, and 16 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 625 tokens : 97.6% similarity, with 610 matched token, and 15 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 650 tokens : 97.23076923076923% similarity, with 632 matched token, and 18 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 675 tokens : 97.48148148148148% similarity, with 658 matched token, and 17 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 700 tokens : 97.42857142857143% similarity, with 682 matched token, and 18 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 750 tokens : 97.73333333333333% similarity, with 733 matched token, and 17 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 800 tokens : 97.5% similarity, with 780 matched token, and 20 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 850 tokens : 97.29411764705883% similarity, with 827 matched token, and 23 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 900 tokens : 97.55555555555556% similarity, with 878 matched token, and 22 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 950 tokens : 97.36842105263158% similarity, with 925 matched token, and 25 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1000 tokens : 97.5% similarity, with 975 matched token, and 25 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Finished baseline model to eval output predictive matching (aka 0 memory?), for 1000 tokens\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation end ###\r\n", "###\r\n" ] } ], "source": [ "!python3 ./memory_script/eval_v5_memory_guided.py \"../../../model/v5-L96-D1024-E0_1-mem-ctx-8k.pth\" \"./logs/v5-L96-D1024-E0_1-mem-ctx-8k-1k.csv\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "3e0fa465", "metadata": { "execution": { "iopub.execute_input": "2023-09-03T20:09:22.623563Z", "iopub.status.busy": "2023-09-03T20:09:22.622346Z", "iopub.status.idle": "2023-09-03T20:15:28.464149Z", "shell.execute_reply": "2023-09-03T20:15:28.462982Z" }, "papermill": { "duration": 365.861939, "end_time": "2023-09-03T20:15:28.466647", "exception": false, "start_time": "2023-09-03T20:09:22.604708", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-09-03 20:09:27,210] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation start ###\r\n", "###\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1100 tokens : 97.0% similarity, with 1067 matched token, and 33 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1150 tokens : 97.21739130434783% similarity, with 1118 matched token, and 32 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1200 tokens : 97.25% similarity, with 1167 matched token, and 33 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1250 tokens : 97.04% similarity, with 1213 matched token, and 37 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1300 tokens : 96.76923076923077% similarity, with 1258 matched token, and 42 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1350 tokens : 96.22222222222221% similarity, with 1299 matched token, and 51 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1400 tokens : 96.14285714285714% similarity, with 1346 matched token, and 54 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1450 tokens : 96.13793103448276% similarity, with 1394 matched token, and 56 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1500 tokens : 95.53333333333333% similarity, with 1433 matched token, and 67 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1550 tokens : 95.22580645161291% similarity, with 1476 matched token, and 74 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1600 tokens : 95.1875% similarity, with 1523 matched token, and 77 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1650 tokens : 94.48484848484848% similarity, with 1559 matched token, and 91 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1700 tokens : 93.94117647058823% similarity, with 1597 matched token, and 103 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1750 tokens : 93.54285714285714% similarity, with 1637 matched token, and 113 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1800 tokens : 92.77777777777779% similarity, with 1670 matched token, and 130 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1850 tokens : 92.10810810810811% similarity, with 1704 matched token, and 146 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1900 tokens : 91.3157894736842% similarity, with 1735 matched token, and 165 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 1950 tokens : 90.71794871794872% similarity, with 1769 matched token, and 181 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2000 tokens : 89.4% similarity, with 1788 matched token, and 212 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2050 tokens : 86.63414634146342% similarity, with 1776 matched token, and 274 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2100 tokens : 84.80952380952381% similarity, with 1781 matched token, and 319 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2150 tokens : 83.16279069767441% similarity, with 1788 matched token, and 362 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2200 tokens : 80.77272727272728% similarity, with 1777 matched token, and 423 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2250 tokens : 78.26666666666667% similarity, with 1761 matched token, and 489 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2300 tokens : 75.8695652173913% similarity, with 1745 matched token, and 555 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2350 tokens : 74.34042553191489% similarity, with 1747 matched token, and 603 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2400 tokens : 72.29166666666667% similarity, with 1735 matched token, and 665 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2450 tokens : 70.61224489795919% similarity, with 1730 matched token, and 720 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2500 tokens : 68.84% similarity, with 1721 matched token, and 779 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2550 tokens : 66.94117647058823% similarity, with 1707 matched token, and 843 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2600 tokens : 65.1923076923077% similarity, with 1695 matched token, and 905 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2650 tokens : 63.77358490566037% similarity, with 1690 matched token, and 960 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2700 tokens : 62.11111111111111% similarity, with 1677 matched token, and 1023 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2750 tokens : 60.54545454545455% similarity, with 1665 matched token, and 1085 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2800 tokens : 58.92857142857143% similarity, with 1650 matched token, and 1150 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2850 tokens : 57.78947368421052% similarity, with 1647 matched token, and 1203 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2900 tokens : 56.55172413793104% similarity, with 1640 matched token, and 1260 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 2950 tokens : 55.1864406779661% similarity, with 1628 matched token, and 1322 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3000 tokens : 53.833333333333336% similarity, with 1615 matched token, and 1385 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3050 tokens : 52.49180327868852% similarity, with 1601 matched token, and 1449 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3100 tokens : 51.41935483870967% similarity, with 1594 matched token, and 1506 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3150 tokens : 50.06349206349206% similarity, with 1577 matched token, and 1573 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3200 tokens : 48.6875% similarity, with 1558 matched token, and 1642 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3250 tokens : 47.63076923076923% similarity, with 1548 matched token, and 1702 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3300 tokens : 46.484848484848484% similarity, with 1534 matched token, and 1766 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3350 tokens : 44.80597014925373% similarity, with 1501 matched token, and 1849 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3400 tokens : 43.29411764705882% similarity, with 1472 matched token, and 1928 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3450 tokens : 42.08695652173913% similarity, with 1452 matched token, and 1998 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3500 tokens : 40.74285714285714% similarity, with 1426 matched token, and 2074 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3550 tokens : 39.38028169014085% similarity, with 1398 matched token, and 2152 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3600 tokens : 37.94444444444444% similarity, with 1366 matched token, and 2234 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3650 tokens : 36.65753424657534% similarity, with 1338 matched token, and 2312 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3700 tokens : 35.43243243243243% similarity, with 1311 matched token, and 2389 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3750 tokens : 34.026666666666664% similarity, with 1276 matched token, and 2474 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3800 tokens : 33.1578947368421% similarity, with 1260 matched token, and 2540 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3850 tokens : 31.896103896103895% similarity, with 1228 matched token, and 2622 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3900 tokens : 30.538461538461537% similarity, with 1191 matched token, and 2709 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 3950 tokens : 29.443037974683545% similarity, with 1163 matched token, and 2787 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4000 tokens : 28.225% similarity, with 1129 matched token, and 2871 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation end ###\r\n", "###\r\n" ] } ], "source": [ "!python3 ./memory_script/eval_v5_memory_guided.py \"../../../model/v5-L96-D1024-E0_1-mem-ctx-8k.pth\" \"./logs/v5-L96-D1024-E0_1-mem-ctx-8k-4k.csv\" 1100 4000" ] }, { "cell_type": "code", "execution_count": 6, "id": "5fed7d73", "metadata": { "execution": { "iopub.execute_input": "2023-09-03T20:15:28.510591Z", "iopub.status.busy": "2023-09-03T20:15:28.509945Z", "iopub.status.idle": "2023-09-03T21:26:44.273967Z", "shell.execute_reply": "2023-09-03T21:26:44.272895Z" }, "papermill": { "duration": 4275.788679, "end_time": "2023-09-03T21:26:44.276323", "exception": false, "start_time": "2023-09-03T20:15:28.487644", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-09-03 20:15:33,087] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation start ###\r\n", "###\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4100 tokens : 25.658536585365855% similarity, with 1052 matched token, and 3048 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4200 tokens : 23.666666666666668% similarity, with 994 matched token, and 3206 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4300 tokens : 21.488372093023255% similarity, with 924 matched token, and 3376 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4400 tokens : 19.318181818181817% similarity, with 850 matched token, and 3550 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4500 tokens : 17.333333333333336% similarity, with 780 matched token, and 3720 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4600 tokens : 15.239130434782608% similarity, with 701 matched token, and 3899 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4700 tokens : 13.553191489361701% similarity, with 637 matched token, and 4063 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4800 tokens : 12.083333333333334% similarity, with 580 matched token, and 4220 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 4900 tokens : 10.061224489795919% similarity, with 493 matched token, and 4407 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5000 tokens : 8.74% similarity, with 437 matched token, and 4563 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5100 tokens : 7.705882352941177% similarity, with 393 matched token, and 4707 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5200 tokens : 6.865384615384615% similarity, with 357 matched token, and 4843 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5300 tokens : 6.0754716981132075% similarity, with 322 matched token, and 4978 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5400 tokens : 5.518518518518518% similarity, with 298 matched token, and 5102 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5500 tokens : 5.109090909090909% similarity, with 281 matched token, and 5219 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5600 tokens : 4.696428571428571% similarity, with 263 matched token, and 5337 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5700 tokens : 4.421052631578947% similarity, with 252 matched token, and 5448 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5800 tokens : 4.206896551724138% similarity, with 244 matched token, and 5556 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 5900 tokens : 4.101694915254237% similarity, with 242 matched token, and 5658 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6000 tokens : 3.75% similarity, with 225 matched token, and 5775 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6100 tokens : 3.6721311475409837% similarity, with 224 matched token, and 5876 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6200 tokens : 3.612903225806452% similarity, with 224 matched token, and 5976 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6300 tokens : 3.5238095238095237% similarity, with 222 matched token, and 6078 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6400 tokens : 3.421875% similarity, with 219 matched token, and 6181 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6500 tokens : 3.3846153846153846% similarity, with 220 matched token, and 6280 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6600 tokens : 3.318181818181818% similarity, with 219 matched token, and 6381 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6700 tokens : 3.2835820895522385% similarity, with 220 matched token, and 6480 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6800 tokens : 3.25% similarity, with 221 matched token, and 6579 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 6900 tokens : 3.27536231884058% similarity, with 226 matched token, and 6674 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7000 tokens : 3.2428571428571433% similarity, with 227 matched token, and 6773 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7100 tokens : 3.1971830985915495% similarity, with 227 matched token, and 6873 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7200 tokens : 3.138888888888889% similarity, with 226 matched token, and 6974 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7300 tokens : 3.1506849315068495% similarity, with 230 matched token, and 7070 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7400 tokens : 3.135135135135135% similarity, with 232 matched token, and 7168 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7500 tokens : 3.1466666666666665% similarity, with 236 matched token, and 7264 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7600 tokens : 3.1315789473684212% similarity, with 238 matched token, and 7362 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7700 tokens : 3.116883116883117% similarity, with 240 matched token, and 7460 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7800 tokens : 3.08974358974359% similarity, with 241 matched token, and 7559 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 7900 tokens : 3.113924050632911% similarity, with 246 matched token, and 7654 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8000 tokens : 3.1125% similarity, with 249 matched token, and 7751 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8100 tokens : 3.148148148148148% similarity, with 255 matched token, and 7845 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8200 tokens : 3.1341463414634148% similarity, with 257 matched token, and 7943 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8300 tokens : 3.132530120481928% similarity, with 260 matched token, and 8040 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8400 tokens : 3.0952380952380953% similarity, with 260 matched token, and 8140 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8500 tokens : 3.0941176470588236% similarity, with 263 matched token, and 8237 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8600 tokens : 3.1046511627906974% similarity, with 267 matched token, and 8333 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8700 tokens : 3.0919540229885056% similarity, with 269 matched token, and 8431 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8800 tokens : 3.0681818181818183% similarity, with 270 matched token, and 8530 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 8900 tokens : 3.0561797752808992% similarity, with 272 matched token, and 8628 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9000 tokens : 3.0666666666666664% similarity, with 276 matched token, and 8724 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9100 tokens : 3.087912087912088% similarity, with 281 matched token, and 8819 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9200 tokens : 3.0652173913043477% similarity, with 282 matched token, and 8918 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9300 tokens : 3.043010752688172% similarity, with 283 matched token, and 9017 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9400 tokens : 3.0106382978723407% similarity, with 283 matched token, and 9117 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9500 tokens : 3.0315789473684207% similarity, with 288 matched token, and 9212 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9600 tokens : 3.041666666666667% similarity, with 292 matched token, and 9308 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9700 tokens : 3.0309278350515463% similarity, with 294 matched token, and 9406 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9800 tokens : 3.0204081632653064% similarity, with 296 matched token, and 9504 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 9900 tokens : 3.0707070707070705% similarity, with 304 matched token, and 9596 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10000 tokens : 3.08% similarity, with 308 matched token, and 9692 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10100 tokens : 3.0792079207920793% similarity, with 311 matched token, and 9789 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10200 tokens : 3.0784313725490198% similarity, with 314 matched token, and 9886 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10300 tokens : 3.1067961165048543% similarity, with 320 matched token, and 9980 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10400 tokens : 3.0865384615384612% similarity, with 321 matched token, and 10079 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10500 tokens : 3.0952380952380953% similarity, with 325 matched token, and 10175 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10600 tokens : 3.0849056603773586% similarity, with 327 matched token, and 10273 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10700 tokens : 3.0934579439252334% similarity, with 331 matched token, and 10369 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10800 tokens : 3.0833333333333335% similarity, with 333 matched token, and 10467 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 10900 tokens : 3.0642201834862384% similarity, with 334 matched token, and 10566 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11000 tokens : 3.0636363636363635% similarity, with 337 matched token, and 10663 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11100 tokens : 3.063063063063063% similarity, with 340 matched token, and 10760 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11200 tokens : 3.0535714285714284% similarity, with 342 matched token, and 10858 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11300 tokens : 3.0707964601769913% similarity, with 347 matched token, and 10953 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11400 tokens : 3.1228070175438596% similarity, with 356 matched token, and 11044 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11500 tokens : 3.130434782608696% similarity, with 360 matched token, and 11140 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11600 tokens : 3.1379310344827585% similarity, with 364 matched token, and 11236 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11700 tokens : 3.128205128205128% similarity, with 366 matched token, and 11334 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11800 tokens : 3.1355932203389827% similarity, with 370 matched token, and 11430 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 11900 tokens : 3.1260504201680672% similarity, with 372 matched token, and 11528 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12000 tokens : 3.125% similarity, with 375 matched token, and 11625 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12100 tokens : 3.1404958677685952% similarity, with 380 matched token, and 11720 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12200 tokens : 3.114754098360656% similarity, with 380 matched token, and 11820 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12300 tokens : 3.1300813008130084% similarity, with 385 matched token, and 11915 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12400 tokens : 3.129032258064516% similarity, with 388 matched token, and 12012 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12500 tokens : 3.136% similarity, with 392 matched token, and 12108 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12600 tokens : 3.1428571428571432% similarity, with 396 matched token, and 12204 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12700 tokens : 3.1181102362204722% similarity, with 396 matched token, and 12304 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12800 tokens : 3.1171875% similarity, with 399 matched token, and 12401 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 12900 tokens : 3.1162790697674416% similarity, with 402 matched token, and 12498 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13000 tokens : 3.1307692307692307% similarity, with 407 matched token, and 12593 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13100 tokens : 3.1221374045801524% similarity, with 409 matched token, and 12691 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13200 tokens : 3.128787878787879% similarity, with 413 matched token, and 12787 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13300 tokens : 3.112781954887218% similarity, with 414 matched token, and 12886 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13400 tokens : 3.119402985074627% similarity, with 418 matched token, and 12982 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13500 tokens : 3.096296296296296% similarity, with 418 matched token, and 13082 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13600 tokens : 3.110294117647059% similarity, with 423 matched token, and 13177 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13700 tokens : 3.0875912408759127% similarity, with 423 matched token, and 13277 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13800 tokens : 3.0942028985507246% similarity, with 427 matched token, and 13373 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 13900 tokens : 3.093525179856115% similarity, with 430 matched token, and 13470 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14000 tokens : 3.1214285714285714% similarity, with 437 matched token, and 13563 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14100 tokens : 3.1063829787234045% similarity, with 438 matched token, and 13662 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14200 tokens : 3.091549295774648% similarity, with 439 matched token, and 13761 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14300 tokens : 3.090909090909091% similarity, with 442 matched token, and 13858 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14400 tokens : 3.0902777777777777% similarity, with 445 matched token, and 13955 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14500 tokens : 3.075862068965517% similarity, with 446 matched token, and 14054 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14600 tokens : 3.0684931506849313% similarity, with 448 matched token, and 14152 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14700 tokens : 3.108843537414966% similarity, with 457 matched token, and 14243 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14800 tokens : 3.0945945945945947% similarity, with 458 matched token, and 14342 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 14900 tokens : 3.0939597315436242% similarity, with 461 matched token, and 14439 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15000 tokens : 3.0733333333333333% similarity, with 461 matched token, and 14539 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15100 tokens : 3.066225165562914% similarity, with 463 matched token, and 14637 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15200 tokens : 3.0657894736842106% similarity, with 466 matched token, and 14734 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15300 tokens : 3.0457516339869284% similarity, with 466 matched token, and 14834 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15400 tokens : 3.0454545454545454% similarity, with 469 matched token, and 14931 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15500 tokens : 3.032258064516129% similarity, with 470 matched token, and 15030 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15600 tokens : 3.0448717948717947% similarity, with 475 matched token, and 15125 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15700 tokens : 3.0254777070063694% similarity, with 475 matched token, and 15225 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15800 tokens : 3.0253164556962022% similarity, with 478 matched token, and 15322 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 15900 tokens : 3.0125786163522013% similarity, with 479 matched token, and 15421 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "## Model validation for 16000 tokens : 2.9875% similarity, with 478 matched token, and 15522 token mismatch\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "###\r\n", "### Model validation end ###\r\n", "###\r\n" ] } ], "source": [ "!python3 ./memory_script/eval_v5_memory_guided.py \"../../../model/v5-L96-D1024-E0_1-mem-ctx-8k.pth\" \"./logs/v5-L96-D1024-E0_1-mem-ctx-8k-16k.csv\" 4100 16000" ] } ], "metadata": { "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "papermill": { "default_parameters": {}, "duration": 4800.015988, "end_time": "2023-09-03T21:26:44.638790", "environment_variables": {}, "exception": null, "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/memory-bench/Benchmark-V5-L96.ipynb", "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/memory-bench/Benchmark-V5-L96.ipynb", "parameters": {}, "start_time": "2023-09-03T20:06:44.622802", "version": "2.4.0" } }, "nbformat": 4, "nbformat_minor": 5 }