Upload 2 files

SinhalaMLM.ipynb

@@ -0,0 +1,710 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3564591f-25e4-4be6-9104-acec74b5a120",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting datasets\n",
+      "  Using cached datasets-2.16.1-py3-none-any.whl (507 kB)\n",
+      "Collecting pandas\n",
+      "  Downloading pandas-2.2.0-cp310-cp310-win_amd64.whl (11.6 MB)\n",
+      "     ---------------------------------------- 0.0/11.6 MB ? eta -:--:--\n",
+      "     --------------------------------------- 0.0/11.6 MB 640.0 kB/s eta 0:00:19\n",
+      "     --------------------------------------- 0.0/11.6 MB 640.0 kB/s eta 0:00:19\n",
+      "     --------------------------------------- 0.0/11.6 MB 195.7 kB/s eta 0:00:59\n",
+      "     --------------------------------------- 0.1/11.6 MB 381.3 kB/s eta 0:00:31\n",
+      "     --------------------------------------- 0.1/11.6 MB 435.7 kB/s eta 0:00:27\n",
+      "     --------------------------------------- 0.1/11.6 MB 399.4 kB/s eta 0:00:29\n",
+      "      -------------------------------------- 0.2/11.6 MB 591.4 kB/s eta 0:00:20\n",
+      "      -------------------------------------- 0.2/11.6 MB 602.4 kB/s eta 0:00:19\n",
+      "      -------------------------------------- 0.2/11.6 MB 602.4 kB/s eta 0:00:19\n",
+      "     - ------------------------------------- 0.3/11.6 MB 696.7 kB/s eta 0:00:17\n",
+      "     - ------------------------------------- 0.4/11.6 MB 673.7 kB/s eta 0:00:17\n",
+      "     - ------------------------------------- 0.4/11.6 MB 636.7 kB/s eta 0:00:18\n",
+      "     - ------------------------------------- 0.5/11.6 MB 702.9 kB/s eta 0:00:16\n",
+      "     -- ------------------------------------ 0.6/11.6 MB 893.0 kB/s eta 0:00:13\n",
+      "     -- ------------------------------------ 0.6/11.6 MB 863.8 kB/s eta 0:00:13\n",
+      "     -- ------------------------------------ 0.8/11.6 MB 955.9 kB/s eta 0:00:12\n",
+      "     --- ------------------------------------ 0.9/11.6 MB 1.1 MB/s eta 0:00:10\n",
+      "     ---- ----------------------------------- 1.2/11.6 MB 1.3 MB/s eta 0:00:09\n",
+      "     ---- ----------------------------------- 1.2/11.6 MB 1.3 MB/s eta 0:00:08\n",
+      "     ----- ---------------------------------- 1.5/11.6 MB 1.5 MB/s eta 0:00:07\n",
+      "     ----- ---------------------------------- 1.6/11.6 MB 1.6 MB/s eta 0:00:07\n",
+      "     ------ --------------------------------- 1.8/11.6 MB 1.6 MB/s eta 0:00:06\n",
+      "     ------ --------------------------------- 2.0/11.6 MB 1.7 MB/s eta 0:00:06\n",
+      "     ------- -------------------------------- 2.2/11.6 MB 1.9 MB/s eta 0:00:06\n",
+      "     -------- ------------------------------- 2.3/11.6 MB 1.9 MB/s eta 0:00:05\n",
+      "     -------- ------------------------------- 2.4/11.6 MB 1.9 MB/s eta 0:00:05\n",
+      "     --------- ------------------------------ 2.6/11.6 MB 1.9 MB/s eta 0:00:05\n",
+      "     --------- ------------------------------ 2.7/11.6 MB 1.9 MB/s eta 0:00:05\n",
+      "     --------- ------------------------------ 2.8/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 3.0/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 3.1/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 3.2/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 3.3/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 3.3/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 3.4/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ------------ --------------------------- 3.5/11.6 MB 1.9 MB/s eta 0:00:05\n",
+      "     ------------ --------------------------- 3.6/11.6 MB 2.0 MB/s eta 0:00:05\n",
+      "     ------------- -------------------------- 3.9/11.6 MB 2.0 MB/s eta 0:00:04\n",
+      "     ------------- -------------------------- 4.0/11.6 MB 2.1 MB/s eta 0:00:04\n",
+      "     -------------- ------------------------- 4.2/11.6 MB 2.1 MB/s eta 0:00:04\n",
+      "     -------------- ------------------------- 4.3/11.6 MB 2.1 MB/s eta 0:00:04\n",
+      "     --------------- ------------------------ 4.6/11.6 MB 2.2 MB/s eta 0:00:04\n",
+      "     ---------------- ----------------------- 4.8/11.6 MB 2.2 MB/s eta 0:00:04\n",
+      "     ---------------- ----------------------- 4.9/11.6 MB 2.2 MB/s eta 0:00:03\n",
+      "     ------------------ --------------------- 5.3/11.6 MB 2.4 MB/s eta 0:00:03\n",
+      "     ------------------- -------------------- 5.7/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     -------------------- ------------------- 5.8/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     -------------------- ------------------- 5.9/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     -------------------- ------------------- 6.1/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     --------------------- ------------------ 6.3/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     --------------------- ------------------ 6.3/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     ---------------------- ----------------- 6.5/11.6 MB 2.5 MB/s eta 0:00:03\n",
+      "     ----------------------- ---------------- 6.7/11.6 MB 2.5 MB/s eta 0:00:02\n",
+      "     ------------------------ --------------- 7.0/11.6 MB 2.6 MB/s eta 0:00:02\n",
+      "     ------------------------ --------------- 7.1/11.6 MB 2.6 MB/s eta 0:00:02\n",
+      "     ------------------------- -------------- 7.4/11.6 MB 2.6 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 7.6/11.6 MB 2.7 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 7.9/11.6 MB 2.7 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 8.1/11.6 MB 2.8 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 8.3/11.6 MB 2.8 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 8.6/11.6 MB 2.8 MB/s eta 0:00:02\n",
+      "     ------------------------------ --------- 8.8/11.6 MB 2.8 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 9.1/11.6 MB 2.9 MB/s eta 0:00:01\n",
+      "     -------------------------------- ------- 9.3/11.6 MB 2.9 MB/s eta 0:00:01\n",
+      "     -------------------------------- ------- 9.4/11.6 MB 2.9 MB/s eta 0:00:01\n",
+      "     -------------------------------- ------- 9.5/11.6 MB 2.9 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 9.6/11.6 MB 2.9 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 9.8/11.6 MB 2.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 10.1/11.6 MB 3.0 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 10.3/11.6 MB 3.1 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 10.5/11.6 MB 3.3 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 10.7/11.6 MB 3.5 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 10.9/11.6 MB 3.5 MB/s eta 0:00:01\n",
+      "     -------------------------------------- - 11.0/11.6 MB 3.6 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  11.3/11.6 MB 3.6 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  11.5/11.6 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  11.6/11.6 MB 3.6 MB/s eta 0:00:01\n",
+      "     ---------------------------------------- 11.6/11.6 MB 3.6 MB/s eta 0:00:00\n",
+      "Collecting filelock\n",
+      "  Using cached filelock-3.13.1-py3-none-any.whl (11 kB)\n",
+      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from datasets) (2.31.0)\n",
+      "Collecting multiprocess\n",
+      "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
+      "     ---------------------------------------- 0.0/134.8 kB ? eta -:--:--\n",
+      "     ---------------------------------- --- 122.9/134.8 kB 3.5 MB/s eta 0:00:01\n",
+      "     -------------------------------------- 134.8/134.8 kB 2.0 MB/s eta 0:00:00\n",
+      "Collecting huggingface-hub>=0.19.4\n",
+      "  Using cached huggingface_hub-0.20.3-py3-none-any.whl (330 kB)\n",
+      "Collecting pyarrow>=8.0.0\n",
+      "  Downloading pyarrow-15.0.0-cp310-cp310-win_amd64.whl (24.8 MB)\n",
+      "     ---------------------------------------- 0.0/24.8 MB ? eta -:--:--\n",
+      "     ---------------------------------------- 0.1/24.8 MB 3.3 MB/s eta 0:00:08\n",
+      "     ---------------------------------------- 0.2/24.8 MB 3.5 MB/s eta 0:00:08\n",
+      "      --------------------------------------- 0.5/24.8 MB 4.2 MB/s eta 0:00:06\n",
+      "     - -------------------------------------- 0.6/24.8 MB 4.1 MB/s eta 0:00:06\n",
+      "     - -------------------------------------- 0.8/24.8 MB 4.2 MB/s eta 0:00:06\n",
+      "     - -------------------------------------- 1.0/24.8 MB 4.2 MB/s eta 0:00:06\n",
+      "     - -------------------------------------- 1.2/24.8 MB 4.4 MB/s eta 0:00:06\n",
+      "     -- ------------------------------------- 1.4/24.8 MB 4.4 MB/s eta 0:00:06\n",
+      "     -- ------------------------------------- 1.6/24.8 MB 4.5 MB/s eta 0:00:06\n",
+      "     -- ------------------------------------- 1.8/24.8 MB 4.6 MB/s eta 0:00:05\n",
+      "     --- ------------------------------------ 2.0/24.8 MB 4.5 MB/s eta 0:00:06\n",
+      "     --- ------------------------------------ 2.2/24.8 MB 4.6 MB/s eta 0:00:05\n",
+      "     --- ------------------------------------ 2.3/24.8 MB 4.5 MB/s eta 0:00:06\n",
+      "     ---- ----------------------------------- 2.6/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     ---- ----------------------------------- 2.7/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     ---- ----------------------------------- 2.8/24.8 MB 4.4 MB/s eta 0:00:06\n",
+      "     ---- ----------------------------------- 3.0/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     ---- ----------------------------------- 3.1/24.8 MB 4.4 MB/s eta 0:00:05\n",
+      "     ----- ---------------------------------- 3.3/24.8 MB 4.4 MB/s eta 0:00:05\n",
+      "     ----- ---------------------------------- 3.4/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ----- ---------------------------------- 3.7/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ------ --------------------------------- 3.9/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     ------ --------------------------------- 4.2/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     ------- -------------------------------- 4.4/24.8 MB 4.7 MB/s eta 0:00:05\n",
+      "     ------- -------------------------------- 4.7/24.8 MB 4.7 MB/s eta 0:00:05\n",
+      "     ------- -------------------------------- 4.8/24.8 MB 4.6 MB/s eta 0:00:05\n",
+      "     ------- -------------------------------- 4.9/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     -------- ------------------------------- 5.0/24.8 MB 4.5 MB/s eta 0:00:05\n",
+      "     -------- ------------------------------- 5.1/24.8 MB 4.4 MB/s eta 0:00:05\n",
+      "     -------- ------------------------------- 5.2/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     -------- ------------------------------- 5.3/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     -------- ------------------------------- 5.5/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     --------- ------------------------------ 5.7/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     --------- ------------------------------ 5.9/24.8 MB 4.4 MB/s eta 0:00:05\n",
+      "     --------- ------------------------------ 6.1/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 6.3/24.8 MB 4.4 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 6.5/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 6.6/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ---------- ----------------------------- 6.8/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 7.0/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 7.1/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 7.2/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ----------- ---------------------------- 7.3/24.8 MB 4.2 MB/s eta 0:00:05\n",
+      "     ------------ --------------------------- 7.5/24.8 MB 4.3 MB/s eta 0:00:05\n",
+      "     ------------ --------------------------- 7.7/24.8 MB 4.2 MB/s eta 0:00:05\n",
+      "     ------------ --------------------------- 7.9/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     ------------- -------------------------- 8.1/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     ------------- -------------------------- 8.2/24.8 MB 4.2 MB/s eta 0:00:04\n",
+      "     ------------- -------------------------- 8.5/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     -------------- ------------------------- 8.7/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     -------------- ------------------------- 8.9/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     -------------- ------------------------- 9.1/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     -------------- ------------------------- 9.2/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     --------------- ------------------------ 9.5/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     --------------- ------------------------ 9.6/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     --------------- ------------------------ 9.9/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ---------------- ----------------------- 10.0/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ---------------- ----------------------- 10.3/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ---------------- ----------------------- 10.4/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ----------------- ---------------------- 10.6/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ----------------- ---------------------- 10.7/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ----------------- ---------------------- 10.9/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ----------------- ---------------------- 11.1/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ------------------ --------------------- 11.3/24.8 MB 4.4 MB/s eta 0:00:04\n",
+      "     ------------------ --------------------- 11.6/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ------------------ --------------------- 11.6/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ------------------ --------------------- 11.7/24.8 MB 4.3 MB/s eta 0:00:04\n",
+      "     ------------------- -------------------- 12.0/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ------------------- -------------------- 12.2/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     -------------------- ------------------- 12.5/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     -------------------- ------------------- 12.7/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     -------------------- ------------------- 12.9/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     --------------------- ------------------ 13.1/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     --------------------- ------------------ 13.3/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     --------------------- ------------------ 13.5/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ---------------------- ----------------- 13.7/24.8 MB 4.6 MB/s eta 0:00:03\n",
+      "     ---------------------- ----------------- 13.8/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ---------------------- ----------------- 14.0/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ---------------------- ----------------- 14.2/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ----------------------- ---------------- 14.3/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ----------------------- ---------------- 14.5/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ----------------------- ---------------- 14.7/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ------------------------ --------------- 15.0/24.8 MB 4.4 MB/s eta 0:00:03\n",
+      "     ------------------------ --------------- 15.2/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ------------------------ --------------- 15.4/24.8 MB 4.6 MB/s eta 0:00:03\n",
+      "     ------------------------- -------------- 15.5/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ------------------------- -------------- 15.7/24.8 MB 4.5 MB/s eta 0:00:03\n",
+      "     ------------------------- -------------- 15.9/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 16.1/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 16.3/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 16.4/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 16.6/24.8 MB 4.4 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 16.8/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 17.0/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 17.1/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 17.2/24.8 MB 4.4 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 17.4/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 17.6/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 17.8/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 18.0/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 18.2/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 18.4/24.8 MB 4.5 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 18.6/24.8 MB 4.4 MB/s eta 0:00:02\n",
+      "     ------------------------------ --------- 18.7/24.8 MB 4.4 MB/s eta 0:00:02\n",
+      "     ------------------------------ --------- 18.9/24.8 MB 4.4 MB/s eta 0:00:02\n",
+      "     ------------------------------ --------- 19.0/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     ------------------------------ --------- 19.2/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     ------------------------------- -------- 19.4/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     ------------------------------- -------- 19.6/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     ------------------------------- -------- 19.8/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     -------------------------------- ------- 20.0/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     -------------------------------- ------- 20.2/24.8 MB 4.3 MB/s eta 0:00:02\n",
+      "     -------------------------------- ------- 20.4/24.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     --------------------------------- ------ 20.5/24.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     --------------------------------- ------ 20.6/24.8 MB 4.2 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 20.7/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 20.8/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 21.0/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 21.1/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 21.3/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 21.4/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 21.4/24.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 21.6/24.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 21.8/24.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 22.0/24.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 22.1/24.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 22.3/24.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 22.5/24.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 22.6/24.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 22.7/24.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 22.8/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 22.9/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.0/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.0/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.0/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.0/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.0/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.0/24.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.1/24.8 MB 3.5 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.2/24.8 MB 3.5 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.4/24.8 MB 3.5 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.5/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 23.5/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     -------------------------------------- - 23.6/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     -------------------------------------- - 23.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     -------------------------------------- - 24.0/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.2/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.4/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.6/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.7/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  24.8/24.8 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------- 24.8/24.8 MB 3.1 MB/s eta 0:00:00\n",
+      "Collecting fsspec[http]<=2023.10.0,>=2023.1.0\n",
+      "  Using cached fsspec-2023.10.0-py3-none-any.whl (166 kB)\n",
+      "Collecting aiohttp\n",
+      "  Downloading aiohttp-3.9.1-cp310-cp310-win_amd64.whl (364 kB)\n",
+      "     ---------------------------------------- 0.0/364.6 kB ? eta -:--:--\n",
+      "     ----------- -------------------------- 112.6/364.6 kB 3.2 MB/s eta 0:00:01\n",
+      "     --------------------- ---------------- 204.8/364.6 kB 3.1 MB/s eta 0:00:01\n",
+      "     ----------------------------------- -- 337.9/364.6 kB 3.0 MB/s eta 0:00:01\n",
+      "     -------------------------------------- 364.6/364.6 kB 2.8 MB/s eta 0:00:00\n",
+      "Collecting numpy>=1.17\n",
+      "  Downloading numpy-1.26.3-cp310-cp310-win_amd64.whl (15.8 MB)\n",
+      "     ---------------------------------------- 0.0/15.8 MB ? eta -:--:--\n",
+      "      --------------------------------------- 0.2/15.8 MB 4.6 MB/s eta 0:00:04\n",
+      "      --------------------------------------- 0.3/15.8 MB 4.1 MB/s eta 0:00:04\n",
+      "     - -------------------------------------- 0.5/15.8 MB 3.9 MB/s eta 0:00:04\n",
+      "     - -------------------------------------- 0.7/15.8 MB 3.8 MB/s eta 0:00:04\n",
+      "     -- ------------------------------------- 0.8/15.8 MB 3.6 MB/s eta 0:00:05\n",
+      "     -- ------------------------------------- 1.0/15.8 MB 3.9 MB/s eta 0:00:04\n",
+      "     -- ------------------------------------- 1.1/15.8 MB 3.8 MB/s eta 0:00:04\n",
+      "     --- ------------------------------------ 1.3/15.8 MB 3.7 MB/s eta 0:00:04\n",
+      "     --- ------------------------------------ 1.4/15.8 MB 3.8 MB/s eta 0:00:04\n",
+      "     --- ------------------------------------ 1.6/15.8 MB 3.8 MB/s eta 0:00:04\n",
+      "     ---- ----------------------------------- 1.7/15.8 MB 3.8 MB/s eta 0:00:04\n",
+      "     ---- ----------------------------------- 1.9/15.8 MB 3.7 MB/s eta 0:00:04\n",
+      "     ----- ---------------------------------- 2.0/15.8 MB 3.7 MB/s eta 0:00:04\n",
+      "     ----- ---------------------------------- 2.1/15.8 MB 3.6 MB/s eta 0:00:04\n",
+      "     ----- ---------------------------------- 2.3/15.8 MB 3.6 MB/s eta 0:00:04\n",
+      "     ------ --------------------------------- 2.5/15.8 MB 3.7 MB/s eta 0:00:04\n",
+      "     ------ --------------------------------- 2.6/15.8 MB 3.7 MB/s eta 0:00:04\n",
+      "     ------- -------------------------------- 2.8/15.8 MB 3.7 MB/s eta 0:00:04\n",
+      "     ------- -------------------------------- 3.0/15.8 MB 3.8 MB/s eta 0:00:04\n",
+      "     -------- ------------------------------- 3.2/15.8 MB 3.9 MB/s eta 0:00:04\n",
+      "     -------- ------------------------------- 3.4/15.8 MB 3.9 MB/s eta 0:00:04\n",
+      "     --------- ------------------------------ 3.6/15.8 MB 3.9 MB/s eta 0:00:04\n",
+      "     --------- ------------------------------ 3.8/15.8 MB 4.0 MB/s eta 0:00:04\n",
+      "     ---------- ----------------------------- 4.0/15.8 MB 4.0 MB/s eta 0:00:03\n",
+      "     ---------- ----------------------------- 4.2/15.8 MB 4.0 MB/s eta 0:00:03\n",
+      "     ----------- ---------------------------- 4.4/15.8 MB 4.1 MB/s eta 0:00:03\n",
+      "     ----------- ---------------------------- 4.6/15.8 MB 4.1 MB/s eta 0:00:03\n",
+      "     ------------ --------------------------- 4.8/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     ------------- -------------------------- 5.1/15.8 MB 4.3 MB/s eta 0:00:03\n",
+      "     ------------- -------------------------- 5.2/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     ------------- -------------------------- 5.4/15.8 MB 4.1 MB/s eta 0:00:03\n",
+      "     -------------- ------------------------- 5.6/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     -------------- ------------------------- 5.8/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     --------------- ------------------------ 6.1/15.8 MB 4.3 MB/s eta 0:00:03\n",
+      "     --------------- ------------------------ 6.3/15.8 MB 4.3 MB/s eta 0:00:03\n",
+      "     ---------------- ----------------------- 6.5/15.8 MB 4.3 MB/s eta 0:00:03\n",
+      "     ---------------- ----------------------- 6.6/15.8 MB 4.3 MB/s eta 0:00:03\n",
+      "     ----------------- ---------------------- 6.7/15.8 MB 4.3 MB/s eta 0:00:03\n",
+      "     ----------------- ---------------------- 6.7/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     ----------------- ---------------------- 7.0/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     ------------------ --------------------- 7.1/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     ------------------ --------------------- 7.2/15.8 MB 4.2 MB/s eta 0:00:03\n",
+      "     ------------------ --------------------- 7.5/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     ------------------- -------------------- 7.7/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     -------------------- ------------------- 7.9/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     -------------------- ------------------- 8.0/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     -------------------- ------------------- 8.2/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     --------------------- ------------------ 8.3/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     --------------------- ------------------ 8.4/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     --------------------- ------------------ 8.6/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     ---------------------- ----------------- 8.7/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ---------------------- ----------------- 8.9/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ----------------------- ---------------- 9.1/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     ----------------------- ---------------- 9.3/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ----------------------- ---------------- 9.5/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ------------------------ --------------- 9.6/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ------------------------ --------------- 9.8/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ------------------------- -------------- 9.9/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ------------------------- -------------- 10.1/15.8 MB 4.2 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 10.3/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 10.4/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 10.5/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     -------------------------- ------------- 10.7/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 10.9/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     --------------------------- ------------ 10.9/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 11.1/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 11.3/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ---------------------------- ----------- 11.4/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 11.6/15.8 MB 4.1 MB/s eta 0:00:02\n",
+      "     ----------------------------- ---------- 11.8/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------ --------- 11.9/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------ --------- 12.1/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------ --------- 12.2/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 12.3/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 12.3/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 12.4/15.8 MB 4.1 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 12.5/15.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 12.6/15.8 MB 4.0 MB/s eta 0:00:01\n",
+      "     ------------------------------- -------- 12.6/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     -------------------------------- ------- 12.7/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     -------------------------------- ------- 12.9/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 13.0/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 13.1/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     --------------------------------- ------ 13.3/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 13.5/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 13.6/15.8 MB 3.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 13.8/15.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ---------------------------------- ----- 13.8/15.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 14.0/15.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 14.2/15.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 14.3/15.8 MB 3.8 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 14.5/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ------------------------------------ --- 14.6/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 14.8/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ------------------------------------- -- 15.0/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     -------------------------------------- - 15.1/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     -------------------------------------- - 15.3/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  15.4/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  15.6/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  15.7/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  15.8/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  15.8/15.8 MB 3.7 MB/s eta 0:00:01\n",
+      "     ---------------------------------------- 15.8/15.8 MB 3.5 MB/s eta 0:00:00\n",
+      "Collecting tqdm>=4.62.1\n",
+      "  Using cached tqdm-4.66.1-py3-none-any.whl (78 kB)\n",
+      "Collecting xxhash\n",
+      "  Downloading xxhash-3.4.1-cp310-cp310-win_amd64.whl (29 kB)\n",
+      "Collecting pyarrow-hotfix\n",
+      "  Using cached pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
+      "Collecting dill<0.3.8,>=0.3.0\n",
+      "  Using cached dill-0.3.7-py3-none-any.whl (115 kB)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from datasets) (6.0.1)\n",
+      "Requirement already satisfied: packaging in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from datasets) (23.2)\n",
+      "Collecting frozenlist>=1.1.1\n",
+      "  Downloading frozenlist-1.4.1-cp310-cp310-win_amd64.whl (50 kB)\n",
+      "     ---------------------------------------- 0.0/50.4 kB ? eta -:--:--\n",
+      "     ---------------------------------------- 50.4/50.4 kB 2.7 MB/s eta 0:00:00\n",
+      "Collecting aiosignal>=1.1.2\n",
+      "  Using cached aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
+      "Collecting multidict<7.0,>=4.5\n",
+      "  Downloading multidict-6.0.4-cp310-cp310-win_amd64.whl (28 kB)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from aiohttp->datasets) (23.2.0)\n",
+      "Collecting yarl<2.0,>=1.0\n",
+      "  Downloading yarl-1.9.4-cp310-cp310-win_amd64.whl (76 kB)\n",
+      "     ---------------------------------------- 0.0/76.4 kB ? eta -:--:--\n",
+      "     ---------------------------------------- 76.4/76.4 kB 2.1 MB/s eta 0:00:00\n",
+      "Collecting async-timeout<5.0,>=4.0\n",
+      "  Downloading async_timeout-4.0.3-py3-none-any.whl (5.7 kB)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface-hub>=0.19.4->datasets) (4.9.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests>=2.19.0->datasets) (3.6)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests>=2.19.0->datasets) (2.1.0)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
+      "Requirement already satisfied: colorama in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tqdm>=4.62.1->datasets) (0.4.6)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from pandas->datasets) (2.8.2)\n",
+      "Collecting pytz>=2020.1\n",
+      "  Using cached pytz-2023.3.post1-py2.py3-none-any.whl (502 kB)\n",
+      "Collecting tzdata>=2022.7\n",
+      "  Using cached tzdata-2023.4-py2.py3-none-any.whl (346 kB)\n",
+      "Requirement already satisfied: six>=1.5 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
+      "Installing collected packages: pytz, xxhash, tzdata, tqdm, pyarrow-hotfix, numpy, multidict, fsspec, frozenlist, filelock, dill, async-timeout, yarl, pyarrow, pandas, multiprocess, huggingface-hub, aiosignal, aiohttp, datasets\n",
+      "Successfully installed aiohttp-3.9.1 aiosignal-1.3.1 async-timeout-4.0.3 datasets-2.16.1 dill-0.3.7 filelock-3.13.1 frozenlist-1.4.1 fsspec-2023.10.0 huggingface-hub-0.20.3 multidict-6.0.4 multiprocess-0.70.15 numpy-1.26.3 pandas-2.2.0 pyarrow-15.0.0 pyarrow-hotfix-0.6 pytz-2023.3.post1 tqdm-4.66.1 tzdata-2023.4 xxhash-3.4.1 yarl-1.9.4\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "[notice] A new release of pip is available: 23.0.1 -> 23.3.2\n",
+      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "bd4b0343-d24f-4ca0-9fcb-94307eaaf732",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting ipywidgetsNote: you may need to restart the kernel to use updated packages.\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "[notice] A new release of pip is available: 23.0.1 -> 23.3.2\n",
+      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "  Downloading ipywidgets-8.1.1-py3-none-any.whl (139 kB)\n",
+      "     ---------------------------------------- 0.0/139.4 kB ? eta -:--:--\n",
+      "     -- ------------------------------------- 10.2/139.4 kB ? eta -:--:--\n",
+      "     -------- ---------------------------- 30.7/139.4 kB 445.2 kB/s eta 0:00:01\n",
+      "     -------- ---------------------------- 30.7/139.4 kB 445.2 kB/s eta 0:00:01\n",
+      "     ---------------- -------------------- 61.4/139.4 kB 299.4 kB/s eta 0:00:01\n",
+      "     ----------------------------- ------ 112.6/139.4 kB 469.7 kB/s eta 0:00:01\n",
+      "     ----------------------------- ------ 112.6/139.4 kB 469.7 kB/s eta 0:00:01\n",
+      "     ------------------------------------ 139.4/139.4 kB 414.0 kB/s eta 0:00:00\n",
+      "Requirement already satisfied: traitlets>=4.3.1 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipywidgets) (5.14.1)\n",
+      "Requirement already satisfied: comm>=0.1.3 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipywidgets) (0.2.1)\n",
+      "Collecting jupyterlab-widgets~=3.0.9\n",
+      "  Downloading jupyterlab_widgets-3.0.9-py3-none-any.whl (214 kB)\n",
+      "     ---------------------------------------- 0.0/214.9 kB ? eta -:--:--\n",
+      "     ----------- --------------------------- 61.4/214.9 kB 1.7 MB/s eta 0:00:01\n",
+      "     ------------------- ------------------ 112.6/214.9 kB 2.2 MB/s eta 0:00:01\n",
+      "     ------------------- ------------------ 112.6/214.9 kB 2.2 MB/s eta 0:00:01\n",
+      "     ----------------------------- ------ 174.1/214.9 kB 958.1 kB/s eta 0:00:01\n",
+      "     -------------------------------------- 214.9/214.9 kB 1.0 MB/s eta 0:00:00\n",
+      "Requirement already satisfied: ipython>=6.1.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipywidgets) (8.20.0)\n",
+      "Collecting widgetsnbextension~=4.0.9\n",
+      "  Downloading widgetsnbextension-4.0.9-py3-none-any.whl (2.3 MB)\n",
+      "     ---------------------------------------- 0.0/2.3 MB ? eta -:--:--\n",
+      "     - -------------------------------------- 0.1/2.3 MB 3.2 MB/s eta 0:00:01\n",
+      "     ---- ----------------------------------- 0.3/2.3 MB 3.1 MB/s eta 0:00:01\n",
+      "     ---- ----------------------------------- 0.3/2.3 MB 3.1 MB/s eta 0:00:01\n",
+      "     ----- ---------------------------------- 0.3/2.3 MB 1.9 MB/s eta 0:00:02\n",
+      "     -------- ------------------------------- 0.5/2.3 MB 2.0 MB/s eta 0:00:01\n",
+      "     ------------ --------------------------- 0.7/2.3 MB 2.7 MB/s eta 0:00:01\n",
+      "     --------------- ------------------------ 0.9/2.3 MB 2.8 MB/s eta 0:00:01\n",
+      "     ------------------- -------------------- 1.1/2.3 MB 3.0 MB/s eta 0:00:01\n",
+      "     ---------------------- ----------------- 1.3/2.3 MB 3.1 MB/s eta 0:00:01\n",
+      "     -------------------------- ------------- 1.5/2.3 MB 3.3 MB/s eta 0:00:01\n",
+      "     ------------------------------ --------- 1.7/2.3 MB 3.5 MB/s eta 0:00:01\n",
+      "     -------------------------------- ------- 1.8/2.3 MB 3.3 MB/s eta 0:00:01\n",
+      "     ----------------------------------- ---- 2.1/2.3 MB 3.4 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  2.2/2.3 MB 3.5 MB/s eta 0:00:01\n",
+      "     ---------------------------------------- 2.3/2.3 MB 3.4 MB/s eta 0:00:00\n",
+      "Requirement already satisfied: pygments>=2.4.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2)\n",
+      "Requirement already satisfied: stack-data in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n",
+      "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (3.0.43)\n",
+      "Requirement already satisfied: matplotlib-inline in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\n",
+      "Requirement already satisfied: jedi>=0.16 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\n",
+      "Requirement already satisfied: colorama in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (0.4.6)\n",
+      "Requirement already satisfied: decorator in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n",
+      "Requirement already satisfied: exceptiongroup in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from ipython>=6.1.0->ipywidgets) (1.2.0)\n",
+      "Requirement already satisfied: parso<0.9.0,>=0.8.3 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n",
+      "Requirement already satisfied: wcwidth in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.13)\n",
+      "Requirement already satisfied: pure-eval in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n",
+      "Requirement already satisfied: asttokens>=2.1.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)\n",
+      "Requirement already satisfied: executing>=1.2.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)\n",
+      "Requirement already satisfied: six>=1.12.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n",
+      "Installing collected packages: widgetsnbextension, jupyterlab-widgets, ipywidgets\n",
+      "Successfully installed ipywidgets-8.1.1 jupyterlab-widgets-3.0.9 widgetsnbextension-4.0.9\n"
+     ]
+    }
+   ],
+   "source": [
+    "pip install ipywidgets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "9995b754-58c9-4c1d-8e8f-9e55741186c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "87a513e0-f880-4219-9ffb-9a8c40707a69",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "100641"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_ds = datasets.list_datasets()\n",
+    "len(all_ds)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "f20dcb4d-bd17-4bf4-839f-3013b4f40c84",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['acronym_identification',\n",
+       " 'ade_corpus_v2',\n",
+       " 'adversarial_qa',\n",
+       " 'aeslc',\n",
+       " 'afrikaans_ner_corpus']"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_ds[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "63e676b3-0ce3-462a-b9e3-cc37c2f33703",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\Dell\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\datasets\\load.py:1429: FutureWarning: The repository for oscar contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/oscar\n",
+      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
+      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
+      "  warnings.warn(\n",
+      "Downloading builder script: 100%|█████████████████████████████████████████████████| 14.8k/14.8k [00:00<00:00, 14.8MB/s]\n",
+      "Downloading readme: 100%|████████████████████████████████████████████████████████████| 303k/303k [00:00<00:00, 412kB/s]\n",
+      "Downloading data: 100%|█████████████████████████████████████████████████████████████████████| 81.0/81.0 [00:00<?, ?B/s]\n",
+      "Downloading data: 100%|█████████████████████████████████████████████████████████████| 176M/176M [00:54<00:00, 3.21MB/s]\n",
+      "Generating train split: 100%|█████████████████████████████████████████| 120684/120684 [00:15<00:00, 7824.51 examples/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "dataset = datasets.load_dataset('oscar', 'unshuffled_deduplicated_si')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "b6593b70-5caa-4da2-92bb-594e67bd964c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'text'],\n",
+       "        num_rows: 120684\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "d80f383a-3928-48cb-9218-50b2e6a5577b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'id': 0,\n",
+       " 'text': 'ජනාධිපතිවරයා දුෂණයට ඔතරම් විරුද්ධ නම් ශ්\\u200dරී ලංකා ටෙලිකොම් – මොබිටෙල් සඳහා ජනාධිපති විමර්ශන කොමිසමක් පත්කළ යුතු බව එම ආයතනයේ සේවකයින් සඳහන් කරයි.\\nඒ මේ වන විට එම ආයතනය බරපතල සැක සහිත ගනුදෙනු වලින් පිරීගත් අන්ත දුෂිත මෙන්ම ඉතිහාසයේ කිසිදා නොවූවිරූ පාඩුවක් ලබන තැනට පත්ව ඇති නිසාය.\\nවාර්තා වන තොරතුරු අතුරින් ශ්\\u200dරී ලංකා ටෙලිකොම් – මොබිටෙල් හී සිදුව තිබෙන එවැනි අතිශය සැකසහිත ගනුදෙනුවක් වන්නේ Mobitel – stage 8 (upgrade) ටෙන්ඩරය Huawei සමාගමට සහ ZTE සමාගමට ලබාදී ඇති ආකාරයයි.\\nHuawei සමාගමට ලබාදී ඇති ටෙන්ඩරයේ වටිනාකම ඇමරිකානු ඩොලර් මිලියන 79 ක් වන අතර ZTE සමාගමට ලබාදී ඇති ටෙන්ඩරයේ වටිනාකම ඇමරිකානු ඩොලර් මිලියන 6 කි.\\nමෙහි ඇති ගැටළුව වන්නේ ඇමරිකානු ඩොලර් මිලියන 85ක් තරම් ඇති විශාල වටිනාකමකින් යුතු අදාළ ගනුදෙනුව සිදුකිරීමේදී කිසිඳු ටෙන්ඩර් පටිපාටියක් අනුගමනය කර නොතිබීමය. මෙවැනි අධික වැයක් දරන්නට සිදුවන ගනුදෙනුවකදී ඒ සඳහා කැබිනට් අනුමැතිය මෙන්ම විධිමත් ප්\\u200dරසම්පාදන ක්\\u200dරියාවලියක් තුළ සිදුකිරීම නෛතික අවශ්\\u200dයතාවක් වුවත් ඒ ගැන කිසිඳු තැකීමක් ටෙලිකොම්- මොබිටෙල් පරිපාලනය සිදුකර නොමැත.\\nසේවක පිරිස් සඳහන් කරන ආකාරයට මෙම ගනුදෙනුව මුළුමනින්ම පසුගිය වසරේ සිදුකොට ඇත්තේ ටෙලිකොම් සභාපති කුමාරසිංහ සිරිසේනගේ මැදිහත්වීම මතය.\\nඑසේම මෙම ගනුදෙනුව තවදුරටත් සැක උපදවන කාරණයක් වන්නේ පසුගිය සිංහල – හින්දු අවුරුදු සමයේ එනම් අප්\\u200dරේල් 16 සිට 22 දක්වා පී.ජී කුමාරසිංහගේ සමස්ථ පවුලටම චීන සංචාරයක් සඳහා Huawei සමාගම අනුග්\\u200dරහය දක්වා ඇති බවද සේවක පිරිස් එල්ල කරන චෝදනාවය.\\nඑපමණක් නොව මේ වන විට ශ්\\u200dරී ලංකා ටෙලිකොම් හී IT අංශය නංවාලීම සඳහා කියමින් Virtusa සමාගමට ලබාදීමේ සුදානමක් පවතින බවද වාර්තා වෙයි. ඒ පිළිබඳ සේවක පිරිස් අනාවරණය කරමින් සඳහන් කරන්නේ IT අංශයේ සමස්ථ ක්\\u200dරියාවලිය වසරකට රුපියල් බිලියන 1ක් ගෙවා අදාළ සමගම හරහා සිදුකර ගැනීමේ අපේක්ෂාවෙන් මෙම කටයුත්ත දියත්කර ඇති බවය.\\nඑහිදීද කිසිඳු ටෙන්ඩර් කැඳවීමක් හෝ විධිමත් ප්\\u200dරසම්පාදන ක්\\u200dරියාවලියක් සිදුව නොමැති බවද ඔවුන් සඳහන් කරයි. එපමණක් නොව ශ්\\u200dරී ලංකා ටෙලිකොම් හී හදවත බඳු IT අංශය පෞද්ගලික සමාගමක් හරහා කරවා ගැනීමට යාම තුළ සමස්ත ආයතන පද්ධතියම අවධානමේ වැටීමක් සිදුවන බවද ඔවුන් පෙන්වා දෙයි.\\nමේ පිළිබඳ තොරතුරු අනාවරණය කරන පාර්ශව සඳහන් කර සිටින්නේ මේ කිසිඳු සැකකටයුතු ගනුදෙනුවක් සම්බන්ධව මාධ්\\u200dය හෝ වගකිවයුතු පාර්ශව කතා නොකරන බවත් ශ්\\u200dරී ලංකා ටෙලිකොම් සභාපති පී.ජී.කුමාරසිංහ , ජනාධිපති මෛත්\\u200dරීපාල සිරිසේනගේ සහෝදරයාවීම ඊට හේතුවී ඇති බව තමන් විශ්වාස කරන බවය.\\nඑසේම වත්මන් සභාපතිවරයා යටතේ ශ්\\u200dරී ලංකා ටෙලිකොම් හී කඩාවැටීම පෙන්වා දෙමින් තොරතුරු මුලාශ්\\u200dර සඳහන් කරන්නේ මේ වන විට රුපියල් බිලියන 35 කට අධික ණයක් සහිතව ආයතනය මුල්\\u200dය අර්බුදයකට ගමන් කර ඇති බවත් ආයතනය පවත්වාගෙන යන්නේ මහජන බැංකුවෙන් ලබාගත් බැංකු අයිරාවකින් බවය.\\nඑහෙත් වත්මන් ආණ්ඩුව බලයට පත්වන විට ශ්\\u200dරී ටෙලිකොම් හී ණය ප්\\u200dරමාණය රුපියල් බිලියන 6.9 ක් පමණක් වූ බවත් ඒ ණයද සමාගම ගෙන ඇත්තේ ආයතනය වර්ධනය කිරීමට අවශ්\\u200dය ව්\\u200dයාපෘති සඳහා බවද වාර්තා වෙයි.\\nඒ අනුව ආයතනයේ ණය 5 ගුණයකින් පමණ වැඩිකිරීම හැර ආයතනයේ යහපත වෙනුවෙන් වත්මන් සභාපතිවරයා සිදුකළ කිසිඳු සේවයක් නොමැ��ි බවද සඳහන් වෙයි. එසේම ආයතනය රුපියල් මිලියන 500ක පමණ බරපතල පාඩුවක් ලබමින් සිටින බව ඒ හේතුවෙන් සේවකයින්ගේ බෝනස් ගෙවීමට පවා නොහැකි තත්ත්වයක් උද්ගතව පවතින බවත් සඳහන් කරන තොරතුරු මුලාශ්\\u200dර වැඩිදුරටත් සඳහන් කර සිටින්නේ මේ ආකාරයට තවත් කාලයක් ගියහොත් ශ්\\u200dරී ලංකා ටෙලිකොම් බංකොලොත්වීමේ අවධානමක්ද පවතින බවය.\\nමේ අතර ශ්\\u200dරී ලංකා ටෙලිකොම් හී විධායක නිලධාරී කිත්ති පෙරේරා මේ වන විටත් පුරප්පාඩුව පවතින ප්\\u200dරධාන විධායක නිලධාරී තනතුර සඳහා යොදා ගැනීමේ උත්සාහයකද සභාපතිවරයා නිරතව සිටින බව වාර්තා වෙයි.\\nටෙලිකොම් සභාපති කුමාරසිංහ මේ සම්බන්ධව “කිත්තිව CEO කරමු. මට ඒකට ඔයාලගේ උදව් ඕන” යනුවෙන් අධ්\\u200dයක්ෂ මණ්ඩලයටද පවසා ඇති බවත් වාර්තා වෙයි. අයතා ඉපයීම් සම්බන්ධව දුෂණ චෝදනා ලාභියකු වන කිත්ති පෙරේරා ප්\\u200dරධාන විධායක නිලධාරියා ලෙස පත්වුවහොත් ශ්\\u200dරී ලංකා ටෙලිකොම් පවතින තත්ත්වයද ඊට අහිමි වන බව මෙන්ම මේ වන විටත් ප්\\u200dරධාන විධායක නිලධාරී තනතුර සඳහා ඇබෑර්තු නිවේදනයේ දැක්වෙන සුදුසුකම් ද කිත්ති පෙරේරා සපුරා නොමැති බවද වැඩිදුරටත් වාර්තා වෙයි.\\nඑසේම කිත්ති පෙරේරා ඇතුළු රාජපක්ෂ හිතවාදී නඩයක්ම මේ වන විට සභාපතිවරයා සමඟ මෙම දුෂිත ගනුදෙනු රැසක නිරතව සිටින බවත් එය ජනාධිපතිවරයාගේ දේශපාලන කීර්ති නාමයට බරපතල හානිකර තත්ත්වයක් බවද ඔවුන් සඳහන් කරයි.\\nඒ අනුව තොරතුරු මුලාශ්\\u200dර සඳහන් කර සිටින්නේ මේවා ගැන කෝපනොවී එල්ලවන චෝදනා ඇත්තද නැත්තද යන්න ගැන විමර්ශනය කිරීමේ යුතුකමක් ජනාධිපතිවරයාට ඇති බවය. එසේම එවැනි විධිමත් විමර්ශනයක් සිදුකළ විට ශ්\\u200dරී ලංකා ටෙලිකොම්- මොබිටෙල් හී සිදුව තිබෙන දුෂණ සම්බන්ධවද ජනාධිපති විමර්ශන කොමිසමක් අවශ්\\u200dය බවට කරන යෝජනාවේ සාධාරණත්වයක් ඇති බව පෙනීයනු ඇති බවද ඔවුන් සඳහන් කරයි.\\n← තේරෙයි.., කල් යයි..! වැලිකඩ සමූහ ඝාතනයට සම්බන්ධ පොලිසියේ රංගජීව සහ බන්ධනාගාරයේ එමිල් රන්ජන්ට බඩු බනිස්..! ගෝඩ් ෆාදර්ට කවදා ද?\\nශ්\\u200dරී ලංකා ප්\\u200dරජාතන්ත්\\u200dරවාදී සමාජවාදී ජනරජයේ නිතියානුකුල අග්\\u200dරාමාත්\\u200dය ලෙස ඔබ පිළිගන්නේ? (මත විමසුම) October 28, 2018\\nලංකා ඊ නිව්ස් කර්තෘ සඳරුවන් සේනාධීර අත් අඩංගුවට ගන්න නැතිනම් පිටුවහල් කරන්න – ජනාධිපති ගමරාළ බ්\\u200dරිතාන්\\u200dයයෙන් ඉල්ලයි June 10, 2018\\nරටේ බිහි වූ දූෂිතම සහ භයංකරම රාජ්\\u200dය නිලධාරියා වූ ගෝඨා මැරී ඉපදිලා වගේ කතා කරමින් රට යලි විපත් මගකට ඇද දාන්න හදනවා – ෂැන්ග්\\u200dරිලා දේශණයට මුදල් ඇමැති මංගලගෙන් පිළිතුරක් May 20, 2018'}"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset['train'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "b6c82283-ce4c-491e-81c1-ba0e512f0704",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|████████████████████████████████████████████████████████████████████████| 120684/120684 [00:13<00:00, 8667.09it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "text_data = []\n",
+    "file_count = 0\n",
+    "\n",
+    "for sample in tqdm(dataset['train']):\n",
+    "    sample = sample['text'].replace('\\n', '')\n",
+    "    text_data.append(sample)\n",
+    "    if len(text_data) == 10_000:\n",
+    "        with open(f'sinhala_{file_count}.txt','w' , encoding='utf-8') as fp:\n",
+    "            fp.write('\\n'.join(text_data))\n",
+    "            text_data = []\n",
+    "            file_count += 1\n",
+    "with open(f'sinhala_{file_count}.txt','w' , encoding='utf-8') as fp:\n",
+    "            fp.write('\\n'.join(text_data))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Tokenizer.ipynb

@@ -0,0 +1,720 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "585f9800-984f-40fe-9b06-35cd40229d90",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "503f9c3c-e348-478f-b743-cff3ce5f4465",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['sinhala_0.txt',\n",
+       " 'sinhala_1.txt',\n",
+       " 'sinhala_10.txt',\n",
+       " 'sinhala_11.txt',\n",
+       " 'sinhala_12.txt']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "paths = [str(x) for x in Path('./').glob('*.txt')]\n",
+    "\n",
+    "paths[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "5e9baa0f-6c33-45b8-8487-202627067436",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: tokenizers in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (0.15.1)\n",
+      "Requirement already satisfied: huggingface_hub<1.0,>=0.16.4 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tokenizers) (0.20.3)\n",
+      "Requirement already satisfied: requests in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (2.31.0)\n",
+      "Requirement already satisfied: filelock in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (3.13.1)\n",
+      "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (4.66.1)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (2023.10.0)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (4.9.0)\n",
+      "Requirement already satisfied: packaging>=20.9 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (23.2)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers) (6.0.1)\n",
+      "Requirement already satisfied: colorama in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tqdm>=4.42.1->huggingface_hub<1.0,>=0.16.4->tokenizers) (0.4.6)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->huggingface_hub<1.0,>=0.16.4->tokenizers) (3.6)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->huggingface_hub<1.0,>=0.16.4->tokenizers) (2023.11.17)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->huggingface_hub<1.0,>=0.16.4->tokenizers) (3.3.2)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\dell\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->huggingface_hub<1.0,>=0.16.4->tokenizers) (2.1.0)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "[notice] A new release of pip is available: 23.0.1 -> 23.3.2\n",
+      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install tokenizers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "225c9c9a-0182-4bfe-92fc-2234e3515560",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tokenizers import ByteLevelBPETokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "501098a0-5df1-448b-99e6-52143cb6751f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = ByteLevelBPETokenizer()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "24ef02fe-5703-4b87-a92a-e6e936f7fd96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer.train(files=paths, vocab_size=30_522, min_frequency=2,\n",
+    "               special_tokens=['<s>', '<pad>', '</s>', '<mask>'\n",
+    "                              ])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "da5bfba4-6c97-4256-b669-f75026b93e09",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "FileExistsError",
+     "evalue": "[WinError 183] Cannot create a file when that file already exists: 'sinhalaMLM'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mFileExistsError\u001b[0m                           Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[8], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmkdir\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msinhalaMLM\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "\u001b[1;31mFileExistsError\u001b[0m: [WinError 183] Cannot create a file when that file already exists: 'sinhalaMLM'"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "os.mkdir('sinhalaMLM')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e25beeaa-e969-4c33-98e2-65523d827d95",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer.save_model('sinhalaMLM')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "222a9edf-15ed-44a8-aaba-2afee76b3cbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "864266b5-77d5-451e-9c01-096588ff62e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install torch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 --index-url https://download.pytorch.org/whl/cpu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "67d2fc2e-9cb0-4306-9769-0c34a2111c37",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import RobertaTokenizerFast"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "474f1e8c-e499-4205-96cc-44f4a4c9e4e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = RobertaTokenizerFast.from_pretrained('sinhalaMLM')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "3ea07602-9cd6-4c12-a860-9302e4db7607",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [0, 4689, 267, 300, 275, 469, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1]}"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer('ළමයා ගෙදර')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a9a1acf-fd7d-4d4b-9222-f2f25a39efff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lables == input_ids\n",
+    "\n",
+    "input_ids -> MLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "90cec429-ee77-4c34-814b-3fec3f5b035e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "def mlm(tensor):\n",
+    "    rand = torch.rand(tensor.shape) \n",
+    "    mask_arr = (rand < 0.15) * (tensor > 2)\n",
+    "    for i in range( tensor.shape[0]):\n",
+    "        selection = torch.flatten(mask_arr[i].nonzero()).tolist() #[[2,5,8]]\n",
+    "    tensor[i,  selection] = 3\n",
+    "    return tensor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "2c22dc9d-cf4f-4cf0-a7b4-364464c525d7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['sinhala_0.txt',\n",
+       " 'sinhala_1.txt',\n",
+       " 'sinhala_10.txt',\n",
+       " 'sinhala_11.txt',\n",
+       " 'sinhala_12.txt']"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "paths = [str(x) for x in Path('./').glob('*.txt')]\n",
+    "paths[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "d19907e4-e54d-4596-a8a8-099e6965bcef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9bb6ce3b55a644b4805cca4d442ada93",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/13 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "input_ids = []\n",
+    "mask = []\n",
+    "labels = []\n",
+    "\n",
+    "for path in tqdm(paths[:50]):\n",
+    "    with open(path, 'r', encoding='utf-8') as f:\n",
+    "        lines = f.read().split('\\n')\n",
+    "    sample = tokenizer(lines, max_length=512, padding='max_length',\n",
+    "truncation=True, return_tensors='pt')\n",
+    "    labels.append(sample.input_ids)\n",
+    "    mask.append(sample.attention_mask)\n",
+    "    input_ids.append(mlm(sample.input_ids.detach().clone()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 96,
+   "id": "ef40f10e-ca0b-416a-8481-6ff6e343372e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#input_ids = torch.cat(input_ids)\n",
+    "#mask = torch.cat(mask)\n",
+    "#labels = torch.cat(labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 89,
+   "id": "3b173e1a-96d0-40e6-9883-4f91772c47a1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([    0,   528,   267,   312,   264,   353,   264,   470,   267,   293,\n",
+       "          271,  1920, 20312,   263,   281,   264,   269,   271,   282,   263])"
+      ]
+     },
+     "execution_count": 89,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "input_ids[0][:20]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "id": "3915fa7d-ae56-4d86-b10d-4431a4739c0e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([  0, 528, 267, 312, 264, 353, 264, 470, 267, 293])"
+      ]
+     },
+     "execution_count": 90,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "labels[0][:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "id": "9202a316-001b-4602-9b8f-9ea93aa5083d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "encodings = {\n",
+    "    'input_ids': input_ids,\n",
+    "    'attention_mask': mask,\n",
+    "    'labels': labels\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 116,
+   "id": "1c5a01c5-14e2-4bdb-8bfa-12becbe02de6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#from torch.utils.data import Dataset\n",
+    "\n",
+    "class Dataset(torch.utils.data.Dataset):\n",
+    "    def __init__(self, encodings):\n",
+    "        self.encodings = encodings\n",
+    "    def __len__(self):\n",
+    "        return self.encodings['input_ids'].shape[0]\n",
+    "    def __getitem__(self, i):\n",
+    "        return {key: tensor[i] for key, tensor in self.encodings.items()}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 111,
+   "id": "e113560d-b382-49d1-afc0-9ad03da9b212",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "120684"
+      ]
+     },
+     "execution_count": 111,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "input_ids.shape[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 112,
+   "id": "ebc01aa1-6a6e-4e20-b88a-9d91deca0c0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_items([('input_ids', tensor([[  0, 528, 267,  ..., 283, 302,   2],\n",
+       "        [  0, 277, 560,  ...,   1,   1,   1],\n",
+       "        [  0, 278, 264,  ..., 296, 269,   2],\n",
+       "        ...,\n",
+       "        [  0, 272, 276,  ...,   1,   1,   1],\n",
+       "        [  0, 292, 296,  ..., 620, 271,   2],\n",
+       "        [  0, 307, 295,  ..., 269, 281,   2]])), ('attention_mask', tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
+       "        [1, 1, 1,  ..., 0, 0, 0],\n",
+       "        [1, 1, 1,  ..., 1, 1, 1],\n",
+       "        ...,\n",
+       "        [1, 1, 1,  ..., 0, 0, 0],\n",
+       "        [1, 1, 1,  ..., 1, 1, 1],\n",
+       "        [1, 1, 1,  ..., 1, 1, 1]])), ('labels', tensor([[  0, 528, 267,  ..., 283, 302,   2],\n",
+       "        [  0, 277, 560,  ...,   1,   1,   1],\n",
+       "        [  0, 278, 264,  ..., 296, 269,   2],\n",
+       "        ...,\n",
+       "        [  0, 272, 276,  ...,   1,   1,   1],\n",
+       "        [  0, 292, 296,  ..., 620, 271,   2],\n",
+       "        [  0, 307, 295,  ..., 269, 281,   2]]))])"
+      ]
+     },
+     "execution_count": 112,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "encodings.items()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 117,
+   "id": "9a329bc6-2286-4d9a-89f4-041ba2f78951",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = Dataset(encodings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 118,
+   "id": "1ad3e3f5-bb45-4d51-8ccb-62ab73a1788f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 120,
+   "id": "900571cd-5061-41f8-b361-e7889ba534cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import RobertaConfig"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 121,
+   "id": "d35e11be-3ee6-4e7c-b49c-6900434553cf",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "30522"
+      ]
+     },
+     "execution_count": 121,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.vocab_size\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 137,
+   "id": "69d081bc-a4a5-4970-a361-05b426d5afa2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = RobertaConfig(\n",
+    "    vocab_size = tokenizer.vocab_size,\n",
+    "    max_position_embeddings=514,\n",
+    "    hidden_size=768,\n",
+    "    num_attention_heads=12,\n",
+    "    num_hidden_layers=6,\n",
+    "    type_vocab_size=1\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 138,
+   "id": "4386905e-2647-4082-aa50-e884cded82b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import RobertaForMaskedLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 139,
+   "id": "8a6fec9d-31da-4657-906d-a5b356a32328",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = RobertaForMaskedLM(config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 140,
+   "id": "3ae60e59-6351-4cee-a780-8d753ba55c0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device ('cpu')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 141,
+   "id": "3558fcc7-64c5-46f3-be9a-3199aae9f59b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "RobertaForMaskedLM(\n",
+       "  (roberta): RobertaModel(\n",
+       "    (embeddings): RobertaEmbeddings(\n",
+       "      (word_embeddings): Embedding(30522, 768, padding_idx=1)\n",
+       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
+       "      (token_type_embeddings): Embedding(1, 768)\n",
+       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "      (dropout): Dropout(p=0.1, inplace=False)\n",
+       "    )\n",
+       "    (encoder): RobertaEncoder(\n",
+       "      (layer): ModuleList(\n",
+       "        (0-5): 6 x RobertaLayer(\n",
+       "          (attention): RobertaAttention(\n",
+       "            (self): RobertaSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): RobertaSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): RobertaIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (intermediate_act_fn): GELUActivation()\n",
+       "          )\n",
+       "          (output): RobertaOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (lm_head): RobertaLMHead(\n",
+       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "    (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "    (decoder): Linear(in_features=768, out_features=30522, bias=True)\n",
+       "  )\n",
+       ")"
+      ]
+     },
+     "execution_count": 141,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.to(device)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 142,
+   "id": "c9bfc830-d445-4203-97b9-a67f23f4b7f1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AdamW"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 150,
+   "id": "651b53ff-dbdb-4665-9ee8-4c15e96e4ba9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "optim = AdamW(model.parameters(), lr=1e-4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 151,
+   "id": "81d298ed-4bff-443b-97b7-8dd7cff5f2aa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm.auto import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 152,
+   "id": "7ec88289-b38d-4317-8797-b2b2192ff407",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "epochs = 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6fe6ccda-fc31-4785-b9f0-ec70e8bfb09a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "42dd8d1f843544e28d56e5d8b574c0a6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/120684 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "loop = tqdm(dataloader, leave=True)\n",
+    "for batch in loop:\n",
+    "    optim.zero_grad()\n",
+    "    input_ids = batch['input_ids'].to(device)\n",
+    "    mask = batch['attention_mask'].to(device)\n",
+    "    labels = batch['labels'].to(device)\n",
+    "    outputs = model(input_ids, attention_mask=mask, \n",
+    "                   labels=labels)\n",
+    "    loss = outputs.loss\n",
+    "    loss.backward()\n",
+    "    optim.step()\n",
+    "\n",
+    "    loop.set_description(f'Epoch: {epochs}')\n",
+    "    loop.set_postfix(loss=loss.item())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9c49ba7-a98d-47d3-8c7f-8807c591ca03",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.save_pretrained('./sinhalaMLM')  # and don't forget to save sinhalaMLM"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}