{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# installing tensorflow_text \n!pip install tensorflow_text  ","metadata":{"id":"DGFTkuRvzWqc","outputId":"5cd555ad-f28d-40fc-8931-effd87043ffd","execution":{"iopub.status.busy":"2023-03-16T11:42:55.387809Z","iopub.execute_input":"2023-03-16T11:42:55.388211Z","iopub.status.idle":"2023-03-16T11:43:17.345494Z","shell.execute_reply.started":"2023-03-16T11:42:55.388174Z","shell.execute_reply":"2023-03-16T11:43:17.344235Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Requirement already satisfied: tensorflow_text in /opt/conda/lib/python3.7/site-packages (2.11.0)\nRequirement already satisfied: tensorflow-hub>=0.8.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow_text) (0.12.0)\nRequirement already satisfied: tensorflow<2.12,>=2.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow_text) (2.11.0)\nRequirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.51.1)\nRequirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.16.0)\nRequirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (15.0.6.1)\nRequirement already satisfied: flatbuffers>=2.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (23.1.21)\nRequirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (59.8.0)\nRequirement already satisfied: keras<2.12,>=2.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.11.0)\nRequirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (0.29.0)\nRequirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (3.3.0)\nRequirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.4.0)\nRequirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (0.4.0)\nRequirement already satisfied: tensorflow-estimator<2.12,>=2.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.11.0)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (23.0)\nRequirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.14.1)\nRequirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.6.3)\nRequirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (3.8.0)\nRequirement already satisfied: tensorboard<2.12,>=2.11 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.11.2)\nCollecting protobuf<3.20,>=3.9.2\n  Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (4.4.0)\nRequirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.2.0)\nRequirement already satisfied: numpy>=1.20 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.21.6)\nRequirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (0.2.0)\nRequirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.7/site-packages (from astunparse>=1.6.0->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.38.4)\nRequirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.35.0)\nRequirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.6.1)\nRequirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.8.1)\nRequirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.4.1)\nRequirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.2.3)\nRequirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.28.2)\nRequirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.4.6)\nRequirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.2.8)\nRequirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (4.9)\nRequirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (4.2.4)\nRequirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.3.1)\nRequirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (4.11.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.26.14)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2022.12.7)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.4)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.1.1)\nRequirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.1.1)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.11.0)\nRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.4.8)\nRequirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.2.2)\nInstalling collected packages: protobuf\n  Attempting uninstall: protobuf\n    Found existing installation: protobuf 3.20.3\n    Uninstalling protobuf-3.20.3:\n      Successfully uninstalled protobuf-3.20.3\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ncudf 21.12.2 requires cupy-cuda115, which is not installed.\ntfx-bsl 1.12.0 requires google-api-python-client<2,>=1.7.11, but you have google-api-python-client 2.79.0 which is incompatible.\ntfx-bsl 1.12.0 requires pyarrow<7,>=6, but you have pyarrow 5.0.0 which is incompatible.\ntensorflow-transform 1.12.0 requires pyarrow<7,>=6, but you have pyarrow 5.0.0 which is incompatible.\nonnx 1.13.1 requires protobuf<4,>=3.20.2, but you have protobuf 3.19.6 which is incompatible.\napache-beam 2.44.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.6 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed protobuf-3.19.6\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"# importing required libraries\nimport numpy as np\n\nimport typing\nfrom typing import Any, Tuple\n\nimport tensorflow as tf\n\nimport tensorflow_text as tf_text\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker","metadata":{"id":"tnxXKDjq3jEL","execution":{"iopub.status.busy":"2023-03-16T11:43:17.349024Z","iopub.execute_input":"2023-03-16T11:43:17.349386Z","iopub.status.idle":"2023-03-16T11:43:24.129905Z","shell.execute_reply.started":"2023-03-16T11:43:17.349353Z","shell.execute_reply":"2023-03-16T11:43:24.128829Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"# using builtin implementations\nuse_builtins = True","metadata":{"id":"KPJ9J7iPUchc","execution":{"iopub.status.busy":"2023-03-16T11:43:24.131282Z","iopub.execute_input":"2023-03-16T11:43:24.132076Z","iopub.status.idle":"2023-03-16T11:43:24.137016Z","shell.execute_reply.started":"2023-03-16T11:43:24.132030Z","shell.execute_reply":"2023-03-16T11:43:24.135628Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"#@title Shape checker\nclass ShapeChecker():\n  def __init__(self):\n    # Keep a cache of every axis-name seen\n    self.shapes = {}\n\n  def __call__(self, tensor, names, broadcast=False):\n    if not tf.executing_eagerly():\n      return\n\n    if isinstance(names, str):\n      names = (names,)\n\n    shape = tf.shape(tensor)\n    rank = tf.rank(tensor)\n\n    if rank != len(names):\n      raise ValueError(f'Rank mismatch:\\n'\n                       f'    found {rank}: {shape.numpy()}\\n'\n                       f'    expected {len(names)}: {names}\\n')\n\n    for i, name in enumerate(names):\n      if isinstance(name, int):\n        old_dim = name\n      else:\n        old_dim = self.shapes.get(name, None)\n      new_dim = shape[i]\n\n      if (broadcast and new_dim == 1):\n        continue\n\n      if old_dim is None:\n        # If the axis name is new, add its length to the cache.\n        self.shapes[name] = new_dim\n        continue\n\n      if new_dim != old_dim:\n        raise ValueError(f\"Shape mismatch for dimension: '{name}'\\n\"\n                         f\"    found: {new_dim}\\n\"\n                         f\"    expected: {old_dim}\\n\")","metadata":{"id":"KqFqKi4fqN9X","execution":{"iopub.status.busy":"2023-03-16T11:43:24.141505Z","iopub.execute_input":"2023-03-16T11:43:24.142344Z","iopub.status.idle":"2023-03-16T11:43:24.155793Z","shell.execute_reply.started":"2023-03-16T11:43:24.142304Z","shell.execute_reply":"2023-03-16T11:43:24.154779Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"!pip install Metaphone\nfrom metaphone import doublemetaphone\nimport pandas as pd\nimport re","metadata":{"id":"rOduTbljY-4e","outputId":"b95ed0e1-fb62-464b-f0a4-738b6ffc11a2","execution":{"iopub.status.busy":"2023-03-16T11:43:24.157133Z","iopub.execute_input":"2023-03-16T11:43:24.158036Z","iopub.status.idle":"2023-03-16T11:43:35.424695Z","shell.execute_reply.started":"2023-03-16T11:43:24.157970Z","shell.execute_reply":"2023-03-16T11:43:35.423443Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Collecting Metaphone\n  Downloading Metaphone-0.6.tar.gz (14 kB)\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hBuilding wheels for collected packages: Metaphone\n  Building wheel for Metaphone (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for Metaphone: filename=Metaphone-0.6-py3-none-any.whl size=13919 sha256=0ea3a5d91487d74185eb933acaaff4b0ad7165cf87bdbc87c56a244527ff6eb8\n  Stored in directory: /root/.cache/pip/wheels/3a/2c/57/cb7d38aa6c80a870a0341ccf4265c2921fb55b1f7f4b76b46b\nSuccessfully built Metaphone\nInstalling collected packages: Metaphone\nSuccessfully installed Metaphone-0.6\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"data1=pd.read_csv(\"/kaggle/input/google-reasearch/train.tsv\",sep='\\t')\ndata1","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:35.428421Z","iopub.execute_input":"2023-03-16T11:43:35.428743Z","iopub.status.idle":"2023-03-16T11:43:36.846977Z","shell.execute_reply.started":"2023-03-16T11:43:35.428710Z","shell.execute_reply":"2023-03-16T11:43:36.845910Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"                                                 en_query  \\\n0       Who is the featured performer at The Grand Ole...   \n1                        Remind me of my meeting at 12 am   \n2                                   set an alarm for 5 am   \n3         What ’ s will weather be like at Sydney at 2 pm   \n4                         What is the weather in Canada ?   \n...                                                   ...   \n170078              How many miles from Humble to Brenham   \n170079            What is traffic so bad on Ronald Reagan   \n170080           Concerts by The Strokes for this weekend   \n170081    How many mm of rain do we expect for tomorrow ?   \n170082                                  cancel all alarms   \n\n                                                 cs_query  \\\n0       Is Saturday night The Grand Ole Opry me featur...   \n1       mujhe 12 am ko meri meeting ke baare me yaad d...   \n2                 subha 5 baje ke liye ek alarm set karen   \n3              dopaher 2 baje Sydney me mausam kaisa hoga   \n4                            Canada me mausam kaisa hai ?   \n...                                                   ...   \n170078              Humble se Brenham tak kitne miles hai   \n170079          Ronald Reagan par traffic kyu kharaab hai   \n170080       Is weekend The Strokes ke hone wale Concerts   \n170081               kal kitni mm baarish ki ummeed hai ?   \n170082                            sarey alarms cance kare   \n\n                                                en_parses  \\\n0       [IN:GET_EVENT Who is [SL:CATEGORY_EVENT the fe...   \n1       [IN:CREATE_REMINDER Remind [SL:PERSON_REMINDED...   \n2       [IN:CREATE_ALARM set an alarm [SL:DATE_TIME fo...   \n3       [IN:GET_WEATHER What ’s will weather be like a...   \n4       [IN:GET_WEATHER What is the weather in [SL:LOC...   \n...                                                   ...   \n170078  [IN:GET_DISTANCE How many [SL:UNIT_DISTANCE mi...   \n170079  [IN:GET_INFO_TRAFFIC What is traffic so bad on...   \n170080  [IN:GET_EVENT [SL:CATEGORY_EVENT Concerts ] by...   \n170081  [IN:UNSUPPORTED_WEATHER How many mm of [SL:WEA...   \n170082  [IN:DELETE_ALARM cancel [SL:AMOUNT all ] alarms ]   \n\n                                                cs_parses  \n0       [IN:GET_EVENT [SL:DATE_TIME Is Saturday night ...  \n1       [IN:CREATE_REMINDER [SL:PERSON_REMINDED mujhe ...  \n2       [IN:CREATE_ALARM [SL:DATE_TIME subha 5 baje ke...  \n3       [IN:GET_WEATHER [SL:DATE_TIME dopaher 2 baje ]...  \n4       [IN:GET_WEATHER [SL:LOCATION Canada ] me mausa...  \n...                                                   ...  \n170078  [IN:GET_DISTANCE [SL:SOURCE Humble ] se [SL:DE...  \n170079  [IN:GET_INFO_TRAFFIC [SL:LOCATION Ronald Reaga...  \n170080  [IN:GET_EVENT [SL:DATE_TIME Is weekend ] [SL:N...  \n170081  [IN:UNSUPPORTED_WEATHER kal kitni mm [SL:WEATH...  \n170082  [IN:DELETE_ALARM [SL:AMOUNT sarey ] alarms can...  \n\n[170083 rows x 4 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>en_query</th>\n      <th>cs_query</th>\n      <th>en_parses</th>\n      <th>cs_parses</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Who is the featured performer at The Grand Ole...</td>\n      <td>Is Saturday night The Grand Ole Opry me featur...</td>\n      <td>[IN:GET_EVENT Who is [SL:CATEGORY_EVENT the fe...</td>\n      <td>[IN:GET_EVENT [SL:DATE_TIME Is Saturday night ...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Remind me of my meeting at 12 am</td>\n      <td>mujhe 12 am ko meri meeting ke baare me yaad d...</td>\n      <td>[IN:CREATE_REMINDER Remind [SL:PERSON_REMINDED...</td>\n      <td>[IN:CREATE_REMINDER [SL:PERSON_REMINDED mujhe ...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>set an alarm for 5 am</td>\n      <td>subha 5 baje ke liye ek alarm set karen</td>\n      <td>[IN:CREATE_ALARM set an alarm [SL:DATE_TIME fo...</td>\n      <td>[IN:CREATE_ALARM [SL:DATE_TIME subha 5 baje ke...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>What ’ s will weather be like at Sydney at 2 pm</td>\n      <td>dopaher 2 baje Sydney me mausam kaisa hoga</td>\n      <td>[IN:GET_WEATHER What ’s will weather be like a...</td>\n      <td>[IN:GET_WEATHER [SL:DATE_TIME dopaher 2 baje ]...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>What is the weather in Canada ?</td>\n      <td>Canada me mausam kaisa hai ?</td>\n      <td>[IN:GET_WEATHER What is the weather in [SL:LOC...</td>\n      <td>[IN:GET_WEATHER [SL:LOCATION Canada ] me mausa...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>170078</th>\n      <td>How many miles from Humble to Brenham</td>\n      <td>Humble se Brenham tak kitne miles hai</td>\n      <td>[IN:GET_DISTANCE How many [SL:UNIT_DISTANCE mi...</td>\n      <td>[IN:GET_DISTANCE [SL:SOURCE Humble ] se [SL:DE...</td>\n    </tr>\n    <tr>\n      <th>170079</th>\n      <td>What is traffic so bad on Ronald Reagan</td>\n      <td>Ronald Reagan par traffic kyu kharaab hai</td>\n      <td>[IN:GET_INFO_TRAFFIC What is traffic so bad on...</td>\n      <td>[IN:GET_INFO_TRAFFIC [SL:LOCATION Ronald Reaga...</td>\n    </tr>\n    <tr>\n      <th>170080</th>\n      <td>Concerts by The Strokes for this weekend</td>\n      <td>Is weekend The Strokes ke hone wale Concerts</td>\n      <td>[IN:GET_EVENT [SL:CATEGORY_EVENT Concerts ] by...</td>\n      <td>[IN:GET_EVENT [SL:DATE_TIME Is weekend ] [SL:N...</td>\n    </tr>\n    <tr>\n      <th>170081</th>\n      <td>How many mm of rain do we expect for tomorrow ?</td>\n      <td>kal kitni mm baarish ki ummeed hai ?</td>\n      <td>[IN:UNSUPPORTED_WEATHER How many mm of [SL:WEA...</td>\n      <td>[IN:UNSUPPORTED_WEATHER kal kitni mm [SL:WEATH...</td>\n    </tr>\n    <tr>\n      <th>170082</th>\n      <td>cancel all alarms</td>\n      <td>sarey alarms cance kare</td>\n      <td>[IN:DELETE_ALARM cancel [SL:AMOUNT all ] alarms ]</td>\n      <td>[IN:DELETE_ALARM [SL:AMOUNT sarey ] alarms can...</td>\n    </tr>\n  </tbody>\n</table>\n<p>170083 rows × 4 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"data1['en_query']=data1['en_query'].apply(lambda x: str(x.strip()).lower())\ndata1['cs_query']=data1['cs_query'].apply(lambda x: str(x.strip()).lower())","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:36.848736Z","iopub.execute_input":"2023-03-16T11:43:36.849470Z","iopub.status.idle":"2023-03-16T11:43:37.029500Z","shell.execute_reply.started":"2023-03-16T11:43:36.849430Z","shell.execute_reply":"2023-03-16T11:43:37.028424Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"data1=data1[['cs_query','en_query']]\ndata1.rename(columns = {'cs_query':'Sentence','en_query':'trr'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.031152Z","iopub.execute_input":"2023-03-16T11:43:37.031552Z","iopub.status.idle":"2023-03-16T11:43:37.051279Z","shell.execute_reply.started":"2023-03-16T11:43:37.031512Z","shell.execute_reply":"2023-03-16T11:43:37.049916Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:5047: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame\n\nSee the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n  errors=errors,\n","output_type":"stream"}]},{"cell_type":"code","source":"data1","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.052792Z","iopub.execute_input":"2023-03-16T11:43:37.053596Z","iopub.status.idle":"2023-03-16T11:43:37.066177Z","shell.execute_reply.started":"2023-03-16T11:43:37.053557Z","shell.execute_reply":"2023-03-16T11:43:37.064925Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"                                                 Sentence  \\\n0       is saturday night the grand ole opry me featur...   \n1       mujhe 12 am ko meri meeting ke baare me yaad d...   \n2                 subha 5 baje ke liye ek alarm set karen   \n3              dopaher 2 baje sydney me mausam kaisa hoga   \n4                            canada me mausam kaisa hai ?   \n...                                                   ...   \n170078              humble se brenham tak kitne miles hai   \n170079          ronald reagan par traffic kyu kharaab hai   \n170080       is weekend the strokes ke hone wale concerts   \n170081               kal kitni mm baarish ki ummeed hai ?   \n170082                            sarey alarms cance kare   \n\n                                                      trr  \n0       who is the featured performer at the grand ole...  \n1                        remind me of my meeting at 12 am  \n2                                   set an alarm for 5 am  \n3         what ’ s will weather be like at sydney at 2 pm  \n4                         what is the weather in canada ?  \n...                                                   ...  \n170078              how many miles from humble to brenham  \n170079            what is traffic so bad on ronald reagan  \n170080           concerts by the strokes for this weekend  \n170081    how many mm of rain do we expect for tomorrow ?  \n170082                                  cancel all alarms  \n\n[170083 rows x 2 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Sentence</th>\n      <th>trr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>is saturday night the grand ole opry me featur...</td>\n      <td>who is the featured performer at the grand ole...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>mujhe 12 am ko meri meeting ke baare me yaad d...</td>\n      <td>remind me of my meeting at 12 am</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>subha 5 baje ke liye ek alarm set karen</td>\n      <td>set an alarm for 5 am</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>dopaher 2 baje sydney me mausam kaisa hoga</td>\n      <td>what ’ s will weather be like at sydney at 2 pm</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>canada me mausam kaisa hai ?</td>\n      <td>what is the weather in canada ?</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>170078</th>\n      <td>humble se brenham tak kitne miles hai</td>\n      <td>how many miles from humble to brenham</td>\n    </tr>\n    <tr>\n      <th>170079</th>\n      <td>ronald reagan par traffic kyu kharaab hai</td>\n      <td>what is traffic so bad on ronald reagan</td>\n    </tr>\n    <tr>\n      <th>170080</th>\n      <td>is weekend the strokes ke hone wale concerts</td>\n      <td>concerts by the strokes for this weekend</td>\n    </tr>\n    <tr>\n      <th>170081</th>\n      <td>kal kitni mm baarish ki ummeed hai ?</td>\n      <td>how many mm of rain do we expect for tomorrow ?</td>\n    </tr>\n    <tr>\n      <th>170082</th>\n      <td>sarey alarms cance kare</td>\n      <td>cancel all alarms</td>\n    </tr>\n  </tbody>\n</table>\n<p>170083 rows × 2 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"data=pd.read_csv(\"/kaggle/input/file-2/file2.csv\")\ndata","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.071478Z","iopub.execute_input":"2023-03-16T11:43:37.071828Z","iopub.status.idle":"2023-03-16T11:43:37.179204Z","shell.execute_reply.started":"2023-03-16T11:43:37.071794Z","shell.execute_reply":"2023-03-16T11:43:37.178042Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"                                                Sentence  \\\n0      congratulations on you celebrating british kid...   \n1      uske liye toh bahot kuch karna padega ye pappi...   \n2      yehi to hum semjhane ki koshish kar rahe hain....   \n3                                       cake kaha hai ??   \n4      jeet ka jashn aur shubah ki shuruat eating bre...   \n...                                                  ...   \n12143  dr kumar vishwas: \"koi deewana kehta hai.. koi...   \n12144  me: aaj kuch toofani karte hai.mom: pani ki bo...   \n12145  pyar mangi to jaan dengi,milk mango to kher de...   \n12146                    kaale kaale baal gaal gore gore   \n12147                          ye sab aunty'on ke saath?   \n\n                                     English_Translation  \\\n0      users congratulate you for celebrating British...   \n1      rat we should a lot more for that, by this evi...   \n2      ehi, this is what i'm expecting you to underst...   \n3                    is Dramebajakudi where is the cake?   \n4      the celebration of a victory and the start of ...   \n...                                                  ...   \n12143  Dr Kumar Vishwas: \"Some used to say lover.. So...   \n12144  Me: Let's do some stormy today.Mom: Fill the w...   \n12145  Ask for love we'll give life, ask for milk we'...   \n12146                        black hair cheeks fair fair   \n12147                             All this with aunties?   \n\n                                                     trr  \\\n0      congratulations on you celebrating british kid...   \n1      you will have to do a lot for that, it will no...   \n2      this is what we are trying to understand. peop...   \n3                                    where is the cake??   \n4      victory celebration and early morning eating b...   \n...                                                  ...   \n12143  dr. kumar vishwas: \"some say crazy..some think...   \n12144  me: let's do something stormy today. mother: f...   \n12145  if you ask for love, you will give your life, ...   \n12146                        kale kale bal gal gore gore   \n12147                         all this with the aunties?   \n\n                                                      tr  \n0      congratulations on you celebrating british kid...  \n1      you will have to do a lot for that, it will no...  \n2      this is what we are trying to understand. peop...  \n3                                    where is the cake??  \n4      victory celebration and early morning eating b...  \n...                                                  ...  \n12143  dr. kumar vishwas: \"some say crazy..some think...  \n12144  me: let's do something stormy today. mother: f...  \n12145  if you ask for love, you will give your life, ...  \n12146                        kale kale bal gal gore gore  \n12147                         all this with the aunties?  \n\n[12148 rows x 4 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Sentence</th>\n      <th>English_Translation</th>\n      <th>trr</th>\n      <th>tr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>congratulations on you celebrating british kid...</td>\n      <td>users congratulate you for celebrating British...</td>\n      <td>congratulations on you celebrating british kid...</td>\n      <td>congratulations on you celebrating british kid...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>uske liye toh bahot kuch karna padega ye pappi...</td>\n      <td>rat we should a lot more for that, by this evi...</td>\n      <td>you will have to do a lot for that, it will no...</td>\n      <td>you will have to do a lot for that, it will no...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>yehi to hum semjhane ki koshish kar rahe hain....</td>\n      <td>ehi, this is what i'm expecting you to underst...</td>\n      <td>this is what we are trying to understand. peop...</td>\n      <td>this is what we are trying to understand. peop...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>cake kaha hai ??</td>\n      <td>is Dramebajakudi where is the cake?</td>\n      <td>where is the cake??</td>\n      <td>where is the cake??</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>jeet ka jashn aur shubah ki shuruat eating bre...</td>\n      <td>the celebration of a victory and the start of ...</td>\n      <td>victory celebration and early morning eating b...</td>\n      <td>victory celebration and early morning eating b...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>12143</th>\n      <td>dr kumar vishwas: \"koi deewana kehta hai.. koi...</td>\n      <td>Dr Kumar Vishwas: \"Some used to say lover.. So...</td>\n      <td>dr. kumar vishwas: \"some say crazy..some think...</td>\n      <td>dr. kumar vishwas: \"some say crazy..some think...</td>\n    </tr>\n    <tr>\n      <th>12144</th>\n      <td>me: aaj kuch toofani karte hai.mom: pani ki bo...</td>\n      <td>Me: Let's do some stormy today.Mom: Fill the w...</td>\n      <td>me: let's do something stormy today. mother: f...</td>\n      <td>me: let's do something stormy today. mother: f...</td>\n    </tr>\n    <tr>\n      <th>12145</th>\n      <td>pyar mangi to jaan dengi,milk mango to kher de...</td>\n      <td>Ask for love we'll give life, ask for milk we'...</td>\n      <td>if you ask for love, you will give your life, ...</td>\n      <td>if you ask for love, you will give your life, ...</td>\n    </tr>\n    <tr>\n      <th>12146</th>\n      <td>kaale kaale baal gaal gore gore</td>\n      <td>black hair cheeks fair fair</td>\n      <td>kale kale bal gal gore gore</td>\n      <td>kale kale bal gal gore gore</td>\n    </tr>\n    <tr>\n      <th>12147</th>\n      <td>ye sab aunty'on ke saath?</td>\n      <td>All this with aunties?</td>\n      <td>all this with the aunties?</td>\n      <td>all this with the aunties?</td>\n    </tr>\n  </tbody>\n</table>\n<p>12148 rows × 4 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"data=data[['Sentence','trr']]\ndata","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.180595Z","iopub.execute_input":"2023-03-16T11:43:37.181056Z","iopub.status.idle":"2023-03-16T11:43:37.196234Z","shell.execute_reply.started":"2023-03-16T11:43:37.181017Z","shell.execute_reply":"2023-03-16T11:43:37.194971Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"                                                Sentence  \\\n0      congratulations on you celebrating british kid...   \n1      uske liye toh bahot kuch karna padega ye pappi...   \n2      yehi to hum semjhane ki koshish kar rahe hain....   \n3                                       cake kaha hai ??   \n4      jeet ka jashn aur shubah ki shuruat eating bre...   \n...                                                  ...   \n12143  dr kumar vishwas: \"koi deewana kehta hai.. koi...   \n12144  me: aaj kuch toofani karte hai.mom: pani ki bo...   \n12145  pyar mangi to jaan dengi,milk mango to kher de...   \n12146                    kaale kaale baal gaal gore gore   \n12147                          ye sab aunty'on ke saath?   \n\n                                                     trr  \n0      congratulations on you celebrating british kid...  \n1      you will have to do a lot for that, it will no...  \n2      this is what we are trying to understand. peop...  \n3                                    where is the cake??  \n4      victory celebration and early morning eating b...  \n...                                                  ...  \n12143  dr. kumar vishwas: \"some say crazy..some think...  \n12144  me: let's do something stormy today. mother: f...  \n12145  if you ask for love, you will give your life, ...  \n12146                        kale kale bal gal gore gore  \n12147                         all this with the aunties?  \n\n[12148 rows x 2 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Sentence</th>\n      <th>trr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>congratulations on you celebrating british kid...</td>\n      <td>congratulations on you celebrating british kid...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>uske liye toh bahot kuch karna padega ye pappi...</td>\n      <td>you will have to do a lot for that, it will no...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>yehi to hum semjhane ki koshish kar rahe hain....</td>\n      <td>this is what we are trying to understand. peop...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>cake kaha hai ??</td>\n      <td>where is the cake??</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>jeet ka jashn aur shubah ki shuruat eating bre...</td>\n      <td>victory celebration and early morning eating b...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>12143</th>\n      <td>dr kumar vishwas: \"koi deewana kehta hai.. koi...</td>\n      <td>dr. kumar vishwas: \"some say crazy..some think...</td>\n    </tr>\n    <tr>\n      <th>12144</th>\n      <td>me: aaj kuch toofani karte hai.mom: pani ki bo...</td>\n      <td>me: let's do something stormy today. mother: f...</td>\n    </tr>\n    <tr>\n      <th>12145</th>\n      <td>pyar mangi to jaan dengi,milk mango to kher de...</td>\n      <td>if you ask for love, you will give your life, ...</td>\n    </tr>\n    <tr>\n      <th>12146</th>\n      <td>kaale kaale baal gaal gore gore</td>\n      <td>kale kale bal gal gore gore</td>\n    </tr>\n    <tr>\n      <th>12147</th>\n      <td>ye sab aunty'on ke saath?</td>\n      <td>all this with the aunties?</td>\n    </tr>\n  </tbody>\n</table>\n<p>12148 rows × 2 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"file = open('/kaggle/input/anuvaad/anuvaad.txt','r')\n# print(file.read())\n# lst=targ, lst2=inp\nfileStrings = file.readlines()\ntarg = []\ninp = []\nfor line in fileStrings:\n    # print(line)\n    line = line.replace('\\n','')\n    lstO = line.split('\\t')\n    targ.append(lstO[0])\n    inp.append(lstO[1])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.197815Z","iopub.execute_input":"2023-03-16T11:43:37.198915Z","iopub.status.idle":"2023-03-16T11:43:37.212932Z","shell.execute_reply.started":"2023-03-16T11:43:37.198867Z","shell.execute_reply":"2023-03-16T11:43:37.211924Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"data_anu=pd.DataFrame(data={'Sentence':inp,'trr':targ})\ndata_anu","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.216354Z","iopub.execute_input":"2023-03-16T11:43:37.216636Z","iopub.status.idle":"2023-03-16T11:43:37.233188Z","shell.execute_reply.started":"2023-03-16T11:43:37.216609Z","shell.execute_reply":"2023-03-16T11:43:37.232043Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"                                               Sentence  \\\n0                 Come on boys, waqt hai shine karne ka   \n1                                Yeh heart Maange More!   \n2     She was bhunno-ing the masala-s jub phone ki g...   \n3                                     You know me na...   \n4                              Woh to mai sambhal lunga   \n...                                                 ...   \n1299                           muze vo bohot pasand hai   \n1300                                     mujhe psnd hai   \n1301         mujhe market se vegetables lekar aane hain   \n1302                                  sabse best friend   \n1303                                  best friend sabse   \n\n                                                  trr  \n0                  Come on boys, it is time to shine.  \n1                           This heart asks for more!  \n2     She was roasting the spices when the phone rang  \n3                                   You know me right  \n4                                  I will handle that  \n...                                               ...  \n1299                                  I like it a lot  \n1300                                  I like it a lot  \n1301       I have to bring vegetables from the market  \n1302                                 Most best friend  \n1303                                 most best friend  \n\n[1304 rows x 2 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Sentence</th>\n      <th>trr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Come on boys, waqt hai shine karne ka</td>\n      <td>Come on boys, it is time to shine.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Yeh heart Maange More!</td>\n      <td>This heart asks for more!</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>She was bhunno-ing the masala-s jub phone ki g...</td>\n      <td>She was roasting the spices when the phone rang</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>You know me na...</td>\n      <td>You know me right</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Woh to mai sambhal lunga</td>\n      <td>I will handle that</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1299</th>\n      <td>muze vo bohot pasand hai</td>\n      <td>I like it a lot</td>\n    </tr>\n    <tr>\n      <th>1300</th>\n      <td>mujhe psnd hai</td>\n      <td>I like it a lot</td>\n    </tr>\n    <tr>\n      <th>1301</th>\n      <td>mujhe market se vegetables lekar aane hain</td>\n      <td>I have to bring vegetables from the market</td>\n    </tr>\n    <tr>\n      <th>1302</th>\n      <td>sabse best friend</td>\n      <td>Most best friend</td>\n    </tr>\n    <tr>\n      <th>1303</th>\n      <td>best friend sabse</td>\n      <td>most best friend</td>\n    </tr>\n  </tbody>\n</table>\n<p>1304 rows × 2 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"data_anu['Sentence']=data_anu['Sentence'].apply(lambda x: str(x.strip()).lower())\ndata_anu['trr']=data_anu['trr'].apply(lambda x: str(x.strip()).lower())","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.236242Z","iopub.execute_input":"2023-03-16T11:43:37.236568Z","iopub.status.idle":"2023-03-16T11:43:37.245140Z","shell.execute_reply.started":"2023-03-16T11:43:37.236540Z","shell.execute_reply":"2023-03-16T11:43:37.244018Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"df=pd.concat([data,data1,data_anu],axis=0)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.246257Z","iopub.execute_input":"2023-03-16T11:43:37.246745Z","iopub.status.idle":"2023-03-16T11:43:37.272633Z","shell.execute_reply.started":"2023-03-16T11:43:37.246706Z","shell.execute_reply":"2023-03-16T11:43:37.271664Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"                                               Sentence  \\\n0     congratulations on you celebrating british kid...   \n1     uske liye toh bahot kuch karna padega ye pappi...   \n2     yehi to hum semjhane ki koshish kar rahe hain....   \n3                                      cake kaha hai ??   \n4     jeet ka jashn aur shubah ki shuruat eating bre...   \n...                                                 ...   \n1299                           muze vo bohot pasand hai   \n1300                                     mujhe psnd hai   \n1301         mujhe market se vegetables lekar aane hain   \n1302                                  sabse best friend   \n1303                                  best friend sabse   \n\n                                                    trr  \n0     congratulations on you celebrating british kid...  \n1     you will have to do a lot for that, it will no...  \n2     this is what we are trying to understand. peop...  \n3                                   where is the cake??  \n4     victory celebration and early morning eating b...  \n...                                                 ...  \n1299                                    i like it a lot  \n1300                                    i like it a lot  \n1301         i have to bring vegetables from the market  \n1302                                   most best friend  \n1303                                   most best friend  \n\n[183535 rows x 2 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Sentence</th>\n      <th>trr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>congratulations on you celebrating british kid...</td>\n      <td>congratulations on you celebrating british kid...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>uske liye toh bahot kuch karna padega ye pappi...</td>\n      <td>you will have to do a lot for that, it will no...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>yehi to hum semjhane ki koshish kar rahe hain....</td>\n      <td>this is what we are trying to understand. peop...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>cake kaha hai ??</td>\n      <td>where is the cake??</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>jeet ka jashn aur shubah ki shuruat eating bre...</td>\n      <td>victory celebration and early morning eating b...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1299</th>\n      <td>muze vo bohot pasand hai</td>\n      <td>i like it a lot</td>\n    </tr>\n    <tr>\n      <th>1300</th>\n      <td>mujhe psnd hai</td>\n      <td>i like it a lot</td>\n    </tr>\n    <tr>\n      <th>1301</th>\n      <td>mujhe market se vegetables lekar aane hain</td>\n      <td>i have to bring vegetables from the market</td>\n    </tr>\n    <tr>\n      <th>1302</th>\n      <td>sabse best friend</td>\n      <td>most best friend</td>\n    </tr>\n    <tr>\n      <th>1303</th>\n      <td>best friend sabse</td>\n      <td>most best friend</td>\n    </tr>\n  </tbody>\n</table>\n<p>183535 rows × 2 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"hin_vocab=dict()\nfor i in df['Sentence'].apply(lambda x: re.sub(\"[.!?\\\\-\\'\\\"]\", \"\", x)):\n    for j in i.split(' '):\n        hin_vocab[doublemetaphone(j)[0]+'*'+doublemetaphone(j[::-1])[0]+'*'+j[:2]+'*'+j[len(j)-1:]]=j\nfor i in hin_vocab:\n    a=set()\n    a.add(hin_vocab[i])\n    hin_vocab[i]=a\nfor i in df['Sentence'].apply(lambda x: re.sub(\"[.!?\\\\-\\'\\\"]\", \"\", x)):\n    for j in i.split(' '):\n        hin_vocab[doublemetaphone(j)[0]+'*'+doublemetaphone(j[::-1])[0]+'*'+j[:2]+'*'+j[len(j)-1:]].add(j)\nprint(len(hin_vocab))\nhin_vocab","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.274284Z","iopub.execute_input":"2023-03-16T11:43:37.274892Z","iopub.status.idle":"2023-03-16T11:45:07.573197Z","shell.execute_reply.started":"2023-03-16T11:43:37.274851Z","shell.execute_reply":"2023-03-16T11:45:07.572045Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"26683\n","output_type":"stream"},{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"{'KNKRTLXNS*SNTLTRNK*co*s': {'congratulations'},\n 'AN*N*on*n': {'on'},\n 'A*A*yo*u': {'you', 'youuuuu', 'youuuuuu'},\n 'SLPRTNK*NTRPLK*ce*g': {'celebrating'},\n 'PRTX*STRP*br*h': {'british'},\n 'KT*TK*ki*d': {'kid', 'kidd'},\n 'SNKRS*SRNS*si*s': {'singers'},\n 'SF*APS*so*a': {'sophia'},\n 'KRSS*SKRK*gr*s': {'graces', 'grocess'},\n 'ANT*TN*an*d': {'and'},\n 'RSS*SSR*ro*s': {'roses', 'rosies'},\n 'ST*TS*st*t': {'st'},\n 'ANFRSR*ARSRFN*an*y': {'anniversary'},\n 'AF*F*of*f': {'of', 'off'},\n 'A*A*a*a': {'a'},\n 'FST*TSF*vi*t': {'visit', 'vist'},\n 'AR*R*yo*r': {'yor', 'your'},\n 'X*AS*sh*w': {'shaw', 'show'},\n '***': {''},\n 'H*A*ho*w': {'how'},\n 'ASK*AKS*us*e': {'uske'},\n 'L*AL*li*e': {'lie', 'liye', 'liyee', 'liyeee', 'liyye'},\n 'T*HT*to*h': {'toh'},\n 'PHT*THP*ba*t': {'bahat', 'bahaut', 'bahot', 'bahuat', 'bahut'},\n 'KX*KK*ku*h': {'kucch', 'kucchh', 'kuch'},\n 'KRN*ANRK*ka*a': {'karana', 'kareena', 'karna', 'karwana'},\n 'PTK*AJTP*pa*a': {'padega', 'padhega', 'padhiyega'},\n 'A*A*ye*e': {'ye', 'yee', 'yeeeeee'},\n 'PPN*NPP*pa*n': {'paapon', 'pappiyan', 'pappiyon'},\n 'S*AS*se*e': {'se', 'see', 'sewee'},\n 'KM*MK*ka*m': {'kaam', 'kaayam', 'kam', 'kamm', 'kaum', 'kayam'},\n 'NH*AHN*na*i': {'naahi', 'nahi'},\n 'XLK*AJLK*ch*a': {'chalayega', 'chalega', 'chilayega'},\n 'AH*AH*ye*i': {'yehi'},\n 'T*AT*to*o': {'to', 'too', 'tooo'},\n 'HM*M*hu*m': {'hum'},\n 'SMJN*ANMS*se*e': {'semjhane'},\n 'K*AK*ki*i': {'ki', 'kii', 'kiwi'},\n 'KXX*SSK*ko*h': {'koshish', 'koshishh'},\n 'KR*RK*ka*r': {'kaar', 'kar', 'karr', 'kaur'},\n 'RH*AHR*ra*e': {'rahe', 'rahee', 'rahiye'},\n 'HN*N*ha*n': {'haan',\n  'haann',\n  'hain',\n  'han',\n  'hawaiian',\n  'hawaiyan',\n  'hawaon',\n  'hawayein',\n  'hawon'},\n 'LK*KL*lo*g': {'log', 'loog'},\n 'SP*PS*sa*b': {'saab', 'sab'},\n 'K*AK*ko*o': {'ko', 'koo'},\n 'AS*AS*is*i': {'isaai', 'isi', 'issi'},\n 'MN*NM*me*n': {'mean', 'mein', 'men', 'meun'},\n 'TL*LT*to*l': {'tol', 'toll', 'tool', 'towel'},\n 'TT*ATT*de*e': {'deate', 'dede', 'dete'},\n 'KNTKSTT*TKSTNK*co*:': {'context:'},\n 'KK*AKK*ca*e': {'cake'},\n 'KH*AHK*ka*a': {'kaha', 'kahaa'},\n 'H*A*ha*i': {'hai',\n  'haii',\n  'haiii',\n  'haiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii',\n  'hawai',\n  'hawaii'},\n 'JT*TJ*je*t': {'jeet', 'jet', 'jett'},\n 'K*AK*ka*a': {'ka', 'kaa', 'kauwa', 'kauwaa', 'kaya'},\n 'JXN*NSJ*ja*n': {'jashn'},\n 'AR*R*au*r': {'aur'},\n 'XP*HPS*sh*h': {'shubah'},\n 'XRT*TRS*sh*t': {'shart',\n  'shirt',\n  'shohrat',\n  'short',\n  'shroowat',\n  'shuruaat',\n  'shuruat'},\n 'ATNK*NT*ea*g': {'eating'},\n 'PRT*TRP*br*d': {'brad', 'bread', 'breed', 'brid', 'broad'},\n 'PKT*ATKP*pa*a': {'pakadta', 'pakata', 'pakda', 'pakoda'},\n 'AT*T*at*t': {'at', 'att'},\n 'TNTNS*SNTNT*ta*s': {'tandons'},\n 'KTJ*AKTK*co*e': {'cottage'},\n 'FXL*ALSF*va*i': {'vaishali'},\n 'TM*HMT*da*h': {'damoh'},\n 'SK*AKS*so*a': {'soca', 'sookha'},\n 'PR*ARP*pu*i': {'puri', 'puuri'},\n 'TN*ANT*de*a': {'deana', 'deanna', 'deewana', 'deewaniya', 'dena', 'denia'},\n 'MTLP*PLTM*ma*b': {'matlab', 'matlb'},\n 'TM*MT*tu*m': {'tum'},\n 'KRXM*AMSRK*ka*a': {'karishma'},\n 'NHN*NHN*na*n': {'nahin'},\n 'RFN*ANFR*ra*a': {'ravana', 'raveena', 'ravenna', 'ravinia'},\n 'H*A*ho*o': {'ho', 'hoo', 'hooo', 'hooooo'},\n 'A*H*wo*h': {'woh'},\n 'J*AJ*jo*o': {'jo', 'joao', 'joo'},\n '0*T*th*,': {'tha,', 'thaa,', 'the,', 'thi,'},\n 'AP*P*ab*b': {'ab', 'abb'},\n 'KHN*NHK*ka*n': {'kahaaaan',\n  'kahaan',\n  'kahaen',\n  'kahan',\n  'kaheen',\n  'kahein',\n  'kahen',\n  'kahin',\n  'kahun'},\n 'SK*AKS*sa*i': {'saakhi', 'sachai', 'sachhai', 'sachhi', 'sakai', 'saki'},\n 'HKNK*AKNK*ho*o': {'hogayi*aankho'},\n 'ANS*SN*aa**': {'aansoo*', 'aansu*'},\n 'SH*AHS*sa*i': {'sahi'},\n 'M*AM*me*e': {'me', 'mee'},\n 'AR*R*ya*r': {'yaaar', 'yaar', 'yar', 'yarr'},\n 'KN*NK*ka*n': {'kaan', 'kahn', 'kan', 'kaun', 'kawn', 'kayon'},\n 'SLL*LLS*za*l': {'zaleel'},\n 'ANSN*NSN*in*n': {'insaan', 'insan'},\n 'HNN*N*ha*,': {'haan,', 'hain,'},\n 'KTK*AKTK*kh*i': {'khidki'},\n 'KL*ALK*kh*i': {'khaali', 'khali', 'kheli', 'kholi', 'khuli'},\n '0*AT*th*i': {'thai', 'the/hai', 'thei', 'thi', 'thii'},\n 'JSMN*NMSJ*ji*n': {'jismein'},\n 'PPR*ARPP*ba*o': {'baburao'},\n 'JNK*NJ*jh*k': {'jhaank'},\n 'RH*AHR*ra*a': {'raha', 'rahaa'},\n '0*AT*th*a': {'tha', 'thaa', 'thha'},\n 'KNFS*SFNK*co*s': {'confess'},\n 'KRT*ATRK*ka*e': {'karate',\n  'karatee',\n  'karde',\n  'karode',\n  'karte',\n  'karwaate',\n  'karwate'},\n 'A*A*ya*a': {'ya', 'yaa'},\n 'TRTR*ARTRT*to*e': {'torture'},\n 'ATN*ANT*it*a': {'itana', 'itna', 'itnaa'},\n 'PT*ATP*ba*a': {'baadhaa',\n  'bada',\n  'badaaa',\n  'badaya',\n  'baddua',\n  'badha',\n  'badhaiya',\n  'badhaya',\n  'badhiya',\n  'badhya',\n  'badiya',\n  'badta',\n  'bata',\n  'bataa',\n  'bataya'},\n 'KNFSN*NSFNK*co*n': {'confession', 'conffesn', 'confusion'},\n 'TJ*AJT*tu*e': {'tuje', 'tujhe'},\n 'SN*NJS*si*n': {'sign'},\n 'AP*P*up*p': {'up'},\n 'K*AK*ke*e': {'ke', 'kee'},\n 'RS*SR*rs*s': {'rs', 'rss'},\n 'MR*ARM*me*e': {'mere', 'meree'},\n 'AKNT*TNK*ac*t': {'account'},\n 'TLN*ANLT*da*e': {'daalne', 'dalane', 'dalne', 'dalwane'},\n 'PTNJJ*NTP*pa*,': {'padenge,'},\n 'TP*PT*ta*b': {'tab'},\n 'H*A*hi*i': {'hi', 'hii', 'hiii'},\n 'RXRJ*AKRSR*re*e': {'recharge'},\n 'PK*AJP*pa*a': {'paaega', 'paayega', 'paega', 'payega'},\n 'MS*SM*mi*s': {'mis', 'miss'},\n 'A*A*u*u': {'u'},\n 'MST*TSM*ms*d': {'msd'},\n 'ATT*TT*aa*t': {'aadat'},\n 'K*AK*ga*i': {'gai', 'gaii', 'gayi', 'gayii'},\n 'APK*AKP*aa*o': {'aaapko', 'aapako', 'aapko'},\n 'KPTN*NTPK*ca*n': {'capetown', 'captain'},\n 'TKT*A0KT*de*e': {'dekhte'},\n 'H*A*hu*e': {'hue', 'huee', 'huwe', 'huye'},\n 'NT*ATN*no*e': {'note'},\n 'APLKXN*NTSLP*ap*n': {'application'},\n 'KKL*ALKK*go*e': {'google'},\n 'PLSTR*ARTSLP*pl*e': {'playstore'},\n 'TNLT*TLNT*do*d': {'donald', 'download'},\n 'HJ*AK*ho*i': {'hogi', 'hogyi'},\n '**,*,': {','},\n 'FK*AKF*fa*e': {'fake'},\n 'SKT*ATKS*sa*i': {'sakti'},\n 'PS*SP*ba*s': {'baas', 'bas', 'bass', 'baus'},\n 'KP*APK*ka*i': {'kabhi', 'kabi'},\n 'KMNT*TNMK*gh*d': {'ghamand'},\n 'K*AK*ki*a': {'kia', 'kiya', 'kiyaa', 'kiyaaa'},\n 'PX*ASP*bh*a': {'bhaasha', 'bhasha'},\n 'SK*AKS*si*a': {'sikha', 'sikhaya', 'sikka'},\n 'XNL*LNK*ch*l': {'chainal', 'chanel', 'channel'},\n 'PR*RP*pa*r': {'paar', 'pair', 'par', 'parr', 'pawar', 'payar'},\n 'KP*PK*ka*b': {'kab', 'kabb'},\n 'AT*AT*aa*a': {'aadha', 'aata', 'aatta'},\n 'PT*TP*ba*,': {'badho,', 'bata,', 'batao,'},\n 'SR*ARS*za*a': {'zaara', 'zara', 'zaraa', 'zariya'},\n 'P*AP*bh*i': {'bhaaaaiiiiii',\n  'bhaaaiiiii',\n  'bhaai',\n  'bhai',\n  'bhaii',\n  'bhaiiii',\n  'bhaui',\n  'bhi',\n  'bhii',\n  'bhoi'},\n 'TK*AKT*de*e': {'deke', 'dekhe', 'dekhiye', 'dekhye'},\n 'AKR*RK*ag*r': {'agar', 'ager', 'agr'},\n 'AS*AS*ay*i': {'ayesi'},\n 'H*A*he*e': {'he', 'hee', 'heeee'},\n 'PT*TP*ba*t': {'baaaat', 'baaat', 'baat', 'bat', 'baut'},\n 'PKL*ALSP*pe*y': {'peechley'},\n 'SL*ALS*sa*o': {'saalo', 'salo'},\n 'PJP*PP*bj*p': {'bjp'},\n 'AL*AL*wa*y': {'waaley', 'walay', 'waley', 'waly'},\n 'KNKRS*SRNK*co*s': {'congres', 'congress'},\n 'KLF*FLK*kh*f': {'khelaaf', 'khilaaf', 'khilaf'},\n 'APN*ANP*ap*a': {'apana', 'apanaa', 'apna', 'apnaya'},\n 'PRTX*ASTRP*pr*i': {'prattyashi'},\n 'K*AK*ky*u': {'kyu'},\n 'KR*ARK*kh*a': {'khara', 'kheera'},\n 'KRT*ATRK*ka*y': {'karrtey', 'kartay', 'kartey'},\n '0N*NT*th*n': {'than', 'thein', 'then', 'thin', 'thon'},\n 'TL*ALT*de*i': {'dehli', 'delhi', 'deli'},\n 'FTNSP*APSNTF*vi*a': {'vidhansabha'},\n 'ALKXN*NTSL*el*n': {'election'},\n 'K*AK*ky*a': {'kya', 'kyaa', 'kyaaa'},\n 'HK*AK*ho*a': {'hoga', 'hogaya'},\n 'AS*AS*ai*a': {'aisa'},\n 'ANN*ANN*un*e': {'unhone'},\n 'PL*ALP*pe*e': {'peele', 'pehle', 'pelee'},\n 'PK*SP*pi*c': {'pic'},\n 'N*AN*ni*i': {'ni', 'nii'},\n 'TK*AKT*dk*i': {'dkhi'},\n 'T*AT*ti*i': {'ti'},\n 'TMR*ARMT*tm*i': {'tmhari'},\n 'MLN*ANLM*mi*e': {'milane', 'milne'},\n 'K*K*k*k': {'k'},\n 'PT*TP*ba*d': {'baaad', 'baad', 'bad'},\n 'AKNR*ARNJ*ig*e': {'ignore'},\n 'TM*AMT*tm*e': {'tmhe'},\n 'M*AM*ma*i': {'mai', 'maui'},\n 'ATN*ANT*it*e': {'itane', 'itanee', 'itne', 'itnee'},\n 'KMS*SMK*ga*s': {'games'},\n 'P*P*b*b': {'b'},\n 'NNJ*ANN*ni*a': {'ninja'},\n 'TRTLS*SLTRT*tu*s': {'turtles'},\n 'STRT*TRTS*st*t': {'start', 'stewart', 'street', 'stuart'},\n 'FTR*R0JF*fi*r': {'fighter'},\n 'KR*RK*ca*r': {'caar', 'car', 'carr'},\n 'RS*AKR*ra*e': {'race'},\n 'MR*ARM*ma*o': {'maaro', 'mario', 'maro'},\n 'KL*LK*kh*l': {'khaal',\n  'khail',\n  'khal',\n  'khayaal',\n  'khayal',\n  'khel',\n  'khol',\n  'khool',\n  'khul',\n  'khyaal'},\n 'KX*SK*kh*h': {'khich', 'khush'},\n 'PK*AKP*ba*i': {'baaaki',\n  'baachhi',\n  'baakhi',\n  'baaki',\n  'bachai',\n  'bachhi',\n  'bakhi',\n  'baki',\n  'bakiii',\n  'baqi'},\n 'TN*NT*dh*n': {'dhan',\n  'dhawan',\n  'dhayan',\n  'dhayn',\n  'dhiyaan',\n  'dhuaan',\n  'dhuaon',\n  'dhyaan',\n  'dhyan'},\n 'TN*ANT*de*e': {'deewane', 'dene', 'dennie'},\n 'TM*MT*ty*m': {'tym'},\n 'KSK*AKSK*ki*e': {'kisake', 'kisike', 'kiske'},\n 'PS*SP*pa*s': {'paas', 'pas', 'pass', 'paws'},\n 'AS*S*is*s': {'is', 'iss'},\n 'RXTT*T0SR*ri*d': {'rishtewaad'},\n 'AST*ATS*az*i': {'azaadi', 'azadi'},\n 'APS*ASP*aa*e': {'aapse'},\n 'XT*A0K*ch*e': {'chahte'},\n 'HPLSSS*SSSLP*ha*z': {'haiplzzzzzz'},\n 'TNT*TNT*dn*t': {'dnt'},\n 'PRK*KRP*br*k': {'break', 'brk', 'brook'},\n '0R*RT*th*r': {'their', 'thor', 'thr', 'thur'},\n 'HRTS*STR*he*s': {'hearts'},\n 'SR*RS*si*r': {'sir'},\n 'PLSS*SSLP*pl*z': {'plzzz', 'plzzzz'},\n 'F*HF*vo*h': {'voh'},\n 'PL*ALP*bo*a': {'bola', 'boleya'},\n 'JSN*ANSJ*ji*e': {'jisne'},\n 'KS*ASK*ki*i': {'kisi', 'kisii', 'kissi'},\n 'PRT*ATRP*pa*y': {'parody', 'party'},\n 'SPRT*TRPS*su*t': {'suparhit', 'superhit', 'support'},\n 'KRN*ANRK*ka*e': {'karaane',\n  'karane',\n  'karene',\n  'karine',\n  'karne',\n  'karonye',\n  'karwaane',\n  'karwane'},\n 'MN*ANM*ma*a': {'maana', 'mana', 'mania', 'manna', 'manwa'},\n 'KRT*ATRK*ka*a': {'karadiya', 'karata', 'kardiya', 'karta', 'karwata'},\n 'FLP*PLF*fl*p': {'flap', 'flip', 'flop'},\n 'ANJ*AJN*an*i': {'annaji'},\n 'KN*ANK*ko*e': {'kone'},\n 'AL*AL*wa*e': {'waale', 'wale', 'walee', 'walle'},\n 'PLT*TLP*pl*t': {'plot'},\n 'JK*HKJ*ja*h': {'jagah'},\n 'ST*ATS*zy*a': {'zyaada', 'zyaadaa', 'zyada'},\n 'HT*AT*ho*i': {'hoti'},\n 'SL*ALS*sa*a': {'saaala', 'saala', 'sala', 'salaaa', 'salla'},\n 'KM*AMK*ga*e': {'game'},\n 'PR*ARP*bh*e': {'bhare'},\n 'RT*A0R*re*e': {'rehte'},\n '0*AT*th*e': {'the', 'thee', 'thhe'},\n 'ASM*AMS*is*e': {'isme'},\n 'TKPS*SPKT*dh*z': {'dhokebaaz'},\n '**>_*<': {'>_<'},\n 'X*AK*ch*a': {'cha', 'chaa', 'chaiyya', 'chiya', 'chya'},\n 'ST*ATS*se*a': {'seedha', 'seeta'},\n 'NSR*RSN*na*r': {'nasr', 'nazar'},\n 'AN*AN*aa*e': {'aaine', 'aane'},\n 'LK*AKL*la*a': {'laga', 'lagaa', 'lagaya', 'lagwa', 'laoga'},\n 'KSP*PSK*ka*b': {'kasab'},\n 'TRMP*PMRT*tr*p': {'trump'},\n 'TK*KT*ta*k': {'taak', 'tak'},\n 'TM*AMT*ti*e': {'time'},\n 'SN*ANS*su*a': {'suna', 'sunha', 'sunna'},\n 'AK*AK*ac*a': {'achha'},\n 'SMJ*AMS*sa*a': {'samja', 'samjha', 'samjhaa', 'samjhaya'},\n 'KP*PK*ka*,': {'kabhi,'},\n 'SR*ARS*sa*e': {'saaare', 'saare', 'sare', 'saree', 'sayre'},\n 'XSN*NSK*ch*n': {'cheezein',\n  'cheezen',\n  'cheezon',\n  'chizein',\n  'chizen',\n  'chosen'},\n 'TL*LT*de*l': {'deal', 'del', 'dell', 'deol'},\n 'XT*TS*sh*d': {'shayad',\n  'shayd',\n  'shed',\n  'shoud',\n  'showed',\n  'shud',\n  'shudd',\n  'shyad'},\n 'LF*AFL*li*e': {'life', 'live'},\n 'SKSSFL*LFSKS*su*l': {'succesful', 'succesfull', 'successfull'},\n 'HT*AT*ho*a': {'hota'},\n '0RK*KRT*th*k': {'tharak'},\n 'SL*ALS*sa*e': {'saale', 'sale'},\n 'HR*R*ha*r': {'haar', 'haaur', 'haiaur', 'hair', 'har', 'harr'},\n 'LTK*AKTL*la*i': {'ladki', 'latki'},\n 'MNXN*NTNM*me*n': {'mention'},\n 'AJ*J*aa*j': {'aaaj', 'aaj', 'aawaj'},\n 'MX*KTM*ma*h': {'match'},\n 'N*AN*na*a': {'na', 'naa', 'naaa', 'naiya', 'naiyya', 'naya', 'nayyiaa'},\n 'KL*LK*ka*l': {'kaal', 'kael', 'kal'},\n 'LTK*AKTL*la*e': {'ladakee', 'ladke', 'latke'},\n 'AN*AN*in*e': {'inhe', 'inne'},\n 'SNT*ATNS*se*i': {'sendai', 'senti'},\n 'KRN*ANRK*kr*e': {'krane', 'krenhe', 'krne'},\n 'KRT*ATRK*kr*e': {'krde', 'krtae', 'krte', 'krwate'},\n '**:(*(': {':('},\n 'TKST*TKST*te*t': {'text'},\n 'KR*ARK*ka*o': {'kaaro',\n  'karao',\n  'kariyo',\n  'karo',\n  'karoo',\n  'karooooo',\n  'karwao'},\n 'FR*RP*ph*r': {'pher', 'phir', 'phor', 'phr'},\n 'TKT*A0KT*de*y': {'dekhtay'},\n 'TT*ATT*di*i': {'didi'},\n 'AP*AP*ab*i': {'abhi', 'abhiii', 'abi'},\n 'APN*ANP*ap*i': {'apani', 'apni'},\n 'AP*AP*ab*e': {'abbe', 'abbie', 'abe'},\n 'KNKRSNN*NSRNK*co*,': {'congressiyon,'},\n 'NJRNS*SNRJN*ni*s': {'nigerians'},\n 'HT*AT*ho*e': {'hoodie', 'hootie', 'hote'},\n 'PNTR*RTNP*ba*r': {'bandar'},\n 'KN*NK*ky*n': {'kyon', 'kyoon', 'kyoun', 'kyun'},\n 'PR*ARP*bu*a': {'bura', 'buraiya'},\n 'MN*NM*ma*n': {'maan', 'maiin', 'main', 'man', 'mann', 'maun', 'mayan'},\n 'K*AK*ga*e': {'gaaye', 'gae', 'gaye', 'gayee'},\n 'KTN*ANTK*ki*e': {'kitane', 'kitne'},\n 'TK*KT*du*h': {'dukh'},\n 'TFRS*AKRFT*di*e': {'divorce'},\n 'ASM*AMS*us*e': {'usme', 'usshme'},\n 'K*AK*ko*i': {'koi', 'koii', 'koyi'},\n 'MTFT*ATFTM*ma*e': {'matvate'},\n 'RNTK*STNJRR*/r*c': {'/rajnetic'},\n 'HN*AN*ho*a': {'hona', 'honna'},\n 'XH*AHK*ch*e': {'chaahiye', 'chahe', 'chahie', 'chahiye', 'chahye', 'chuhe'},\n 'MHL*ALHM*ma*o': {'mahilao', 'mahilayo'},\n 'XRF*PRS*sh*h': {'sharaph'},\n 'SMN*NMS*sa*n': {'saamaan', 'samaan', 'saman', 'samman'},\n 'TRPL*ALPRT*tr*e': {'triple', 'trouble'},\n 'TLK*KLT*ta*q': {'talaq', 'taluq'},\n 'RK*AKR*ro*e': {'roke'},\n 'KRJ*AKRK*ka*e': {'karaoge', 'karge', 'karoge', 'karwaoge'},\n 'S0*TS*sa*h': {'saath', 'sath'},\n 'MN*ANM*ma*e': {'maane',\n  'maine',\n  'mainne',\n  'manaye',\n  'mane',\n  'manne',\n  'mayne'},\n 'TM*AMT*tu*e': {'tumhe'},\n 'ALT*ATL*ul*a': {'ulta'},\n 'HT*T*ha*h': {'hadh'},\n 'AS*AS*wi*e': {'wise'},\n 'T*AT*tu*u': {'tu'},\n 'PL*ALP*bo*i': {'boli'},\n 'N*AN*na*i': {'naayi', 'nai', 'nayi'},\n 'LNXR*RKNL*lu*r': {'lunchar', 'luncher'},\n 'PJ*AJP*pa*i': {'paaji', 'paegi', 'paji', 'payegi'},\n 'X*AK*ch*i': {'chaai', 'chaayi', 'chai', 'chaiwai', 'chi', 'chiii'},\n 'PSKT*TKSP*bi*t': {'biscuit', 'biskoot', 'bizkit'},\n 'K*AK*kh*a': {'kha', 'khaa', 'khaya', 'khoya'},\n '0K*KT*th*k': {'thaak', 'thak', 'thakk', 'theek', 'thik', 'thk', 'thok'},\n 'H**ha*,': {'haaye,', 'hai,', 'hai,,'},\n 'AS*AS*ai*e': {'aise'},\n 'TXRT*TRST*ts*t': {'tshirt'},\n 'KS*ASK*ka*e': {'kaise', 'kaisee', 'kassie'},\n 'TL*LT*da*l': {'daal', 'dal', 'dayal'},\n 'SKT*ATKS*sa*a': {'sakta'},\n 'TTR*RTT*tw*r': {'twiter', 'twitter', 'twtr'},\n 'ALN*NL*wa*n': {'waalon', 'walloon', 'walon', 'waylon'},\n 'KXNSP*APSNSK*kh*i': {'khushnaseebi'},\n 'HP*AP*ha*y': {'happy', 'happyyy'},\n 'PR0T*ATTRP*bi*y': {'birthday', 'birthdayyyy'},\n 'TN*ANT*dh*i': {'dhoni', 'dhunayi'},\n '**&*&': {'&'},\n 'HL*AL*he*o': {'hello', 'helo'},\n 'SKX*ASKS*sa*i': {'saakshi', 'sakshi'},\n 'PP*APP*bh*i': {'bhabhi', 'bhabi', 'bhaibhai'},\n 'KRT*TRK*kh*t': {'khairaat', 'khairiyat'},\n 'LLL*LL*lo*,': {'lol,'},\n 'FNK*NP*ph*k': {'phenk', 'phoonk'},\n 'TT*ATT*de*a': {'deta'},\n 'PXR*ARSP*be*e': {'bechare'},\n 'KT*TK*ka*t': {'kaat', 'kat', 'katt'},\n 'JT*ATJ*ja*a': {'jaada',\n  'jaaeta',\n  'jaata',\n  'jaataa',\n  'jada',\n  'jaida',\n  'jata',\n  'jataa',\n  'jataya',\n  'jayada',\n  'jayda'},\n 'PLL*LP*bi*,': {'bill,'},\n 'ASL*ALS*is*e': {'iseeliye', 'isiliye'},\n 'LT*ATL*la*a': {'ladaa', 'ladta', 'lata', 'lataa', 'lauta'},\n 'T*AT*di*a': {'dia', 'diya', 'diyaa'},\n 'XP*PK*ch*p': {'chaap', 'chap', 'cheap', 'chep', 'chip', 'chup', 'chupp'},\n 'TT*TT*di*d': {'did', 'died'},\n 'ARN*NR*ar*n': {'aran', 'aren', 'arun'},\n 'LL*LL*la*l': {'laal', 'lal'},\n 'S*S*sa*,': {'sa,', 'say,'},\n 'PNKLTX*ASTLNP*ba*i': {'bangladeshi'},\n 'PLR*RLP*pl*r': {'player'},\n 'KX*ASK*kh*i': {'khushi'},\n 'APN*ANP*ap*e': {'apne'},\n 'ANTKX*SKTN*an*h': {'andkosh'},\n 'TKR*ARKT*ta*a': {'takra'},\n 'XL*LK*ch*l': {'chaal', 'chal', 'chaul', 'chawal', 'chill'},\n 'AXNK*KNK*ac*k': {'achaanak', 'achanak'},\n 'SN*ANS*so*y': {'sonny', 'sony'},\n 'TF*FT*tv*v': {'tv'},\n 'TLL*ALLT*di*e': {'dilwale'},\n 'MF*AFM*mo*e': {'moive', 'moovee', 'move', 'movie', 'moviee'},\n 'TLKST*TSSLT*te*t': {'telecast'},\n 'FL*ALF*va*i': {'vaalii', 'vali'},\n 'RT*TR*ra*t': {'raat', 'raitt', 'rat', 'rawat'},\n 'A*A*wh*y': {'why'},\n 'HMX*ASM*ha*a': {'hamesha', 'hameshaa'},\n 'SN*ANS*su*y': {'sunny'},\n 'LN*ANL*le*e': {'leanne', 'lene', 'leone'},\n 'TK*AKT*di*u': {'dikhau'},\n 'PT*ATP*pa*a': {'pada',\n  'padha',\n  'padhaa',\n  'padhaya',\n  'padta',\n  'padwa',\n  'paida',\n  'pata',\n  'pataa',\n  'patta'},\n 'STTNTS*STNTTS*st*s': {'students'},\n 'MHNJ*AKNHM*me*e': {'mehenge'},\n 'TN*NT*di*n': {'dian', 'din', 'dion'},\n 'AK*AK*ag*e': {'agaye', 'agye'},\n 'SX*AKS*so*a': {'socha'},\n 'ANK*KN*aa*h': {'aankh'},\n 'PNT*TNP*ba*d': {'band', 'banned'},\n 'XRT*ATRS*sh*i': {'shruti'},\n 'HSN*NS*ha*n': {'hasan', 'haseen', 'hassan'},\n 'AT*AT*aa*i': {'aadhi', 'aati'},\n 'KT*TK*gu*d': {'gud'},\n 'MRNK*NRM*mr*g': {'mrng'},\n 'MHXFRTR*ARTRFSHM*ma*i': {'mahashivaratri', 'mahashivratri'},\n 'HRTK*KTR*ha*k': {'hardik'},\n 'MNKLKMN*ANMKLKNM*ma*e': {'mangalkamnaye'},\n 'SR*RS*so*,': {'sorry,'},\n 'MJ*AJM*mu*e': {'muje', 'mujhe'},\n 'PJ*AJP*ba*e': {'bajaaye', 'bajawe', 'bajaye', 'baje', 'bajee', 'bajhe'},\n 'FN*ANP*ph*e': {'phone'},\n 'ALPPR*RPPL*wa*r': {'wallpaper'},\n 'SNTJ*AKTNS*zi*i': {'zindagi'},\n 'RH*AHR*ra*i': {'raahi', 'rahi', 'rahii'},\n 'PTL*ALTP*ba*i': {'baatli', 'badhali'},\n 'TMPRR*ARRPMT*te*y': {'temporary'},\n 'PN*NP*ba*n': {'baan', 'baayen', 'ban', 'bann', 'bayaan', 'bayan'},\n 'LK*KL*la*g': {'laag', 'lag', 'lagg'},\n 'XK*AKK*ch*a': {'chacha', 'chakka', 'chauka', 'chokha', 'chuka', 'chukaya'},\n 'AK*AK*ac*i': {'achhai', 'achhhi', 'achhi'},\n 'NHH*HN*na*,': {'nahi,'},\n 'M0N*NTM*mi*n': {'mithun', 'mitthaiyan'},\n 'KL*ALK*kh*o': {'khayaalo', 'khelo', 'khilao', 'kholo'},\n 'AMK*KM*om*g': {'omg'},\n 'N*AN*ne*e': {'ne', 'nee'},\n 'RT*TR*rt*t': {'rt'},\n 'SPR*ARPS*sp*e': {'spare', 'spree'},\n '0NKS*SKNT*th*s': {'thanks', 'thinks', 'thnks'},\n 'KX*ASKK**k*i': {'*khushi'},\n 'AMT*TM*am*t': {'amit'},\n 'MHMT*TMHM*mu*d': {'muhammad'},\n 'AL*AL*al*i': {'ali'},\n 'PKL*ALKP*pa*i': {'pagli'},\n 'PLS*ASLP*pl*e': {'please'},\n 'FL*ALF*fo*w': {'follow', 'folow'},\n 'FR*ARF*fr*e': {'free'},\n 'TR*ART*da*u': {'daaru', 'daru', 'daruu'},\n 'PLT*ATLP*pi*e': {'pilaate'},\n 'HS*AS*hi*a': {'hisa', 'hissa'},\n 'AL*AL*wa*a': {'waala', 'wala', 'walaaaaa', 'walla'},\n 'TKLF*FLKT*ta*f': {'takleef', 'taklif', 'taqleef'},\n 'TK*AJT*de*a': {'dega', 'degaaa'},\n 'A*A*wo*o': {'wo'},\n 'KS*ASK*ki*a': {'kisa', 'kissa'},\n 'HMR*ARM*ha*i': {'hamaari', 'hamari'},\n 'KXT*A0SK*ka*i': {'kashti'},\n 'AHN*NH*wa*n': {'wahaan', 'wahan', 'wahin'},\n 'TP*APT*do*i': {'doobi'},\n 'JHN*NHJ*ja*n': {'jahaan', 'jahan'},\n 'PN*ANP*pa*i': {'paaani', 'paani', 'pani'},\n 'H*A*hu*u': {'hu', 'huuuu'},\n 'MR*ARM*me*a': {'mehra', 'meira', 'mera'},\n 'MTF*AFTM*mo*e': {'motive'},\n 'AR*R*er*r': {'er', 'err'},\n 'L*AL*le*e': {'le', 'lee', 'leye'},\n 'AK*K*ek*k': {'ek'},\n 'KR*ARK*gu*u': {'guru'},\n 'J*AJ*ji*i': {'ji', 'jii'},\n 'PXN*ANXP*pe*e': {'pehchaane', 'pehchane', 'pehchanne'},\n 'ANKR*RKN*in*r': {'inkaar', 'inkar'},\n 'LKN*NKL*lo*n': {'logan', 'logon', 'loogoun'},\n 'PKSTN*ANTSKP*pa*i': {'pakistani'},\n 'FS*ASF*vi*a': {'visa'},\n 'PSPRT*TRPSP*pa*t': {'passport'},\n 'MFT*TFM*mu*t': {'muft'},\n 'J*AJ*ja*e': {'jaae',\n  'jaaiye',\n  'jaaye',\n  'jaayiye',\n  'jae',\n  'jaie',\n  'jaiye',\n  'jaye'},\n 'PRNM*MNRP*pr*m': {'pranaam', 'pranam'},\n 'ARR*R*ya*,': {'yaar,'},\n 'T*AT*do*o': {'do', 'doo'},\n 'A*H*ye*h': {'yeah', 'yeh'},\n 'J*AJ*ja*a': {'ja', 'jaa', 'jaya'},\n '**_/*_': {'_/\\\\_'},\n 'RLJ*AKLR*ru*e': {'rulaoge'},\n 'ANT*ATN*in*a': {'india'},\n 'SR*ARS*sr*i': {'sri'},\n 'LNK*AKNL*la*a': {'laaunga', 'lanka', 'launga'},\n 'PL*ALP*bu*o': {'bulao'},\n 'FT*ATF*vo*e': {'vote'},\n 'RFS*SFR*re*s': {'reviews'},\n 'SRRT*TRRS*za*t': {'zaroorat', 'zarorat', 'zarurat'},\n 'KF*AFK*ka*i': {'kaafi', 'kafi', 'kaufii', 'kavi'},\n 'PRTSHN*NHSTRP*pr*n': {'protsahan'},\n 'MLT*ATLM*mi*a': {'milata', 'milta'},\n 'AN*N*in*n': {'in', 'inn'},\n 'PTN*NTP*ba*n': {'baatein',\n  'baaten',\n  'baaton',\n  'badhaiyaan',\n  'badhaiyan',\n  'badhen',\n  'badhun',\n  'badon',\n  'bataayein',\n  'bataen',\n  'batan',\n  'batayen',\n  'batein',\n  'baten',\n  'baton',\n  'batten',\n  'battiyan'},\n 'ATM*AMT*aa*i': {'aadami', 'aadamii', 'aadmi'},\n 'MLN*ANLM*mi*a': {'milania', 'milena', 'milna'},\n 'PHT*THP*bo*t': {'bohat', 'bohoot', 'bohot', 'bohut'},\n 'SLMNNN*NNNMLS*sa*n': {'salmannnnn', 'salmannnnnn'},\n 'A*A*aa*a': {'aa', 'aaa', 'aaaa', 'aaaaa', 'aaya', 'aayaa', 'aaye,aa'},\n 'ANTSR*RSTN*in*r': {'intazaar',\n  'intazar',\n  'intezaar',\n  'intezar',\n  'intzaar',\n  'intzar'},\n 'ATR*RT*id*r': {'idhar'},\n 'TR*ART*te*e': {'tere', 'terre'},\n 'AR*AR*ar*y': {'aray', 'arey', 'array', 'arrey', 'arry', 'ary'},\n 'AP*P*aa*p': {'aap'},\n 'TNK*NT*dh*g': {'dhang'},\n 'HHH*AHH*ha*a': {'hahaha',\n  'hahahaa',\n  'hahahaaa',\n  'hahahaaaa',\n  'hahahahha',\n  'hai_hahaha'},\n 'XF*AFS*sh*e': {'shave', 'shivaye'},\n 'LN*ANL*le*a': {'leeanna', 'leena', 'lena'},\n 'TR*ART*te*a': {'tera', 'teraa', 'terra'},\n 'SHR*ARHS*sa*a': {'sahaara', 'sahara'},\n 'HH*AH*ha*a': {'haaha',\n  'haahaa',\n  'haha',\n  'hahaa',\n  'hahahha',\n  'hahahhaa',\n  'hahahhahha'},\n 'TT*ATT*da*e': {'daate', 'date', 'daudte'},\n 'AT*T*ya*d': {'yaad', 'yad'},\n 'TLJ*AKLT*di*e': {'dialouge', 'dilaoge'},\n 'KS*ASK*kh*i': {'khaasi', 'khusi'},\n 'H*A*hu*i': {'huawei', 'hui', 'huiii', 'huwi', 'huyi'},\n 'TMR*ARMT*tu*a': {'tumara', 'tumhaaara', 'tumhaara', 'tumhara', 'tumhra'},\n 'JNMTN*NTMNJ*ja*n': {'janamdin', 'janmdin'},\n 'AS*AS*us*e': {'use', 'usse', 'ussee'},\n 'KT*ATK*ga*e': {'gaddhe', 'gadhe', 'gate'},\n 'PHR*RHP*ba*r': {'baahar', 'bahaar', 'bahar', 'baher'},\n 'ATRN*ANRT*ut*a': {'utarna'},\n 'KMPLSR*ARSLPMK*co*y': {'compulsory'},\n 'KR0*ATRK*ke*i': {'keerthi'},\n 'LTS*STL*le*s': {'leads', 'leeds', 'lets'},\n 'APN*NP*op*n': {'opan', 'open', 'oppn'},\n 'NK*AKN*ng*o': {'ngo'},\n 'TK0R*RTKT*to*r': {'together'},\n 'TT*ATT*to*e': {'toadie', 'tode', 'todte', 'toote'},\n 'TL*LT*di*l': {'dial', 'dil', 'dill'},\n 'MRSN*NSRM*ma*n': {'marezon'},\n 'L*AL*li*a': {'lia', 'liya', 'liyaa', 'liyea'},\n 'KP*APK*kb*i': {'kbhi', 'kbi'},\n 'XN*ANK*ch*a': {'chana', 'chanaa', 'china', 'chuna'},\n 'PX*ASP*pe*e': {'peche', 'peeche', 'peeeche', 'peshwe'},\n 'N*AN*nh*i': {'nhai', 'nhi', 'nhii'},\n 'KR*RK*kr*r': {'kr', 'krr'},\n '0T*TT*th*t': {'thaat', 'that'},\n 'SPN*NPS*sp*n': {'spain', 'spann', 'spin', 'spoon'},\n 'RF*AFR*ra*i': {'raffi', 'rafi', 'ravi'},\n 'AXN*NS*as*n': {'ashwin'},\n 'AFRK*ASRF*af*a': {'africa'},\n 'ARLT*TLR*wo*d': {'world'},\n 'KPS*SPK*cu*s': {'cubs', 'cups'},\n 'M*AM*me*i': {'mei'},\n 'T*T*t*t': {'t'},\n 'F*F*v*v': {'v'},\n 'KT*TK*gu*:': {'guddu:'},\n 'MM*AMM*mu*y': {'mummy'},\n 'KN*ANK*kh*e': {'khaane', 'khane', 'khanye'},\n 'XLK*KLK*ch*k': {'chalaak', 'chalk'},\n 'XRN*ANRK*ch*e': {'churane'},\n 'MS*ASM*ma*a': {'maaza', 'mausa', 'maza', 'mazaa', 'mazya', 'mazza'},\n 'PLKLL*LKLP*bi*,': {'bilkul,'},\n 'SKT*ATKS*sa*e': {'sakate', 'sakte'},\n 'SPK*AKPS*sa*o': {'sabhiko', 'sabko'},\n 'PK*AKP*ba*a': {'bacha', 'bachaa', 'bachha', 'baga', 'bakaya'},\n 'KHHRF*FRSSHK*ka*f': {'kaho,sirf'},\n 'AL*AL*il*a': {'ilawa', 'ilha'},\n 'AK*K*ok*k': {'ok', 'okk'},\n 'ML*ALM*mi*i': {'mili'},\n '**</*3': {'</3'},\n 'PK*KP*bh*h': {'bheekh', 'bhookh'},\n 'LJ*AKL*la*i': {'lagi'},\n 'ANT*ATN*un*a': {'undaa'},\n 'JT*TJ*ji*t': {'jit', 'jitt'},\n 'TN*ANT*tu*e': {'tune'},\n 'AFRN*ANRF*ev*e': {'everyone'},\n 'TL*LT*tl*l': {'tl'},\n '0NKNK*NKNT*th*g': {'thanking', 'thinking'},\n 'MMS*SMM*mo*s': {'mommys', 'momos', 'moms'},\n 'AX*AS*ai*i': {'aiyaashi'},\n 'PS*ASP*pa*e': {'paese', 'paise', 'pasie', 'pause'},\n 'PP*APP*pa*a': {'paapa', 'papa', 'papaa', 'pappa', 'pappiya', 'papua'},\n 'TNM*AMNT*ta*y': {'tanmay'},\n 'JL*LJ*ja*l': {'jaal', 'jail', 'jal'},\n 'XKT*ATKS*sh*i': {'shakti'},\n 'PXR*ARSP*be*a': {'bechaariya', 'bechara'},\n 'PLT*ATLP*bo*a': {'bolhata', 'bolta'},\n 'LK*AKL*li*e': {'like', 'likhaye', 'likhe', 'likhee', 'likhiye', 'liyeachhe'},\n 'HSPNT*TNPS*hu*d': {'husband'},\n 'AF*AF*ev*e': {'eve', 'evvie'},\n 'N*N*n*n': {'n'},\n 'PFR*ARFP*be*e': {'before'},\n 'MRJ*AKRM*ma*e': {'maaroge', 'marge', 'margie', 'maroge', 'marriage'},\n 'TKN*ANKT*de*e': {'deekhane', 'dekhane', 'dekhne', 'dekne'},\n 'APR*RP*up*r': {'upar', 'uppar', 'upper', 'upr'},\n 'P*AP*pe*e': {'pe', 'pee', 'peeye'},\n 'SMN*ANMS*za*a': {'zamaana', 'zamana'},\n 'KRP*PRK*kh*b': {'kharaab', 'kharab', 'khrab'},\n 'XN*ANS*sh*e': {'shaane', 'shane', 'shine'},\n 'LJP*PJL*la*b': {'laajawaab', 'lajawaab'},\n 'H**ha*:': {'hai:'},\n 'ST*ATS*sa*u': {'saadu', 'sadhu', 'satu'},\n 'SHP*PHS*sa*b': {'saahab', 'sahab', 'saheb', 'sahib'},\n 'SNR*RNSS*(s*r': {'(senior'},\n 'PR*ARP*br*o': {'brio', 'bro'},\n 'L*L*la*,': {'laaya,', 'law),'},\n 'HL*L*ha*l': {'haal', 'hail', 'hal', 'hall'},\n 'JPTST*TSTPJ*ja*t': {'jabatdast'},\n 'TST*TST*do*t': {'dost'},\n 'PJ*AKP*pa*e': {'paaoge', 'page', 'paoge'},\n 'RL*ALR*ru*a': {'rula', 'rulaya'},\n 'KML*LMK*ka*l': {'kamaaal', 'kamaal', 'kamal'},\n 'SPRP*PRPS*su*b': {'suburb', 'superb', 'supperb'},\n 'NT*TN*nd*d': {'nd'},\n 'HTS*ST*ha*s': {'hats', 'hatss'},\n 'AR*R*ur*r': {'ur'},\n 'KRRFT*ATFRRK*cr*y': {'crearivity'},\n 'PL*LP*bh*l': {'bhel', 'bhool', 'bhul'},\n 'HNJ*AKN*ho*e': {'hoayenge', 'honge'},\n 'SLMN*NMLS*sa*n': {'salmaan', 'salman', 'salmon'},\n 'KN*NK*kh*n': {'khaan', 'khaein', 'khan', 'khoon'},\n 'PKL*LKP*pa*l': {'paagal', 'pagal', 'pagl'},\n 'ART*TR*au*t': {'aurat'},\n 'ATLST*TSLT*at*t': {'atleast'},\n 'JS*ASJ*ji*e': {'jise', 'jisse'},\n '0NT*TNT*th*d': {'thand'},\n 'LKT*ATKL*la*i': {'lagati', 'lagti', 'lakdi'},\n 'T*AT*de*e': {'de', 'dee', 'deye'},\n 'SRF*FRS*si*f': {'siraf', 'sirf'},\n 'MK*AKM*ma*a': {'maaukaa', 'macha', 'machaa', 'machaya', 'mauka', 'mauqaaa'},\n 'XT*A0K*ch*a': {'chaahta', 'chaahtaa', 'chahta'},\n 'AK*AJ*aa*a': {'aaega', 'aayega'},\n 'RK*AKR*ru*e': {'ruke', 'rukiye'},\n '**:*:': {':'},\n 'ANS*AKN*on*e': {'once'},\n 'APN*NP*up*n': {'upon'},\n '0R*ART*th*e': {'thaare', 'thehre', 'there', 'theyre', 'three'},\n 'AS*S*wa*s': {'was', 'ways'},\n 'HXTK*K0S*ha*g': {'hashtag'},\n 'AKL*LK*aa*l': {'aajkal'},\n '**4*4': {'4'},\n 'SRS*SRS*se*s': {'sears', 'series', 'serious'},\n 'HLKPTR*RTPSL*he*r': {'helicopter'},\n 'JFT*TFK*gi*t': {'gift'},\n 'ANTRN*ANRTN*in*i': {'indrani'},\n 'N*AN*na*e': {'nae', 'naee', 'naye', 'nayee'},\n 'PT*ATP*pa*o': {'padhao', 'padho', 'pado', 'patio', 'patiyo'},\n 'XL*ALK*ch*a': {'chala',\n  'chalaa',\n  'chalaya',\n  'chalwa',\n  'chawla',\n  'chilla',\n  'cholaa',\n  'chula',\n  'chulha'},\n 'APTT*ATTP*up*e': {'update'},\n 'KRK*AKRK*ka*a': {'karega', 'karuga'},\n 'TT*TT*tw*t': {'tweet', 'twit'},\n 'PT*TP*pa*h': {'padh'},\n 'KS*ASK*gu*a': {'gussa'},\n 'K*AK*ga*a': {'ga',\n  'gaa',\n  'gaaa',\n  'gaaya',\n  'gaia',\n  'gaiyya',\n  'gawa',\n  'gaya',\n  'gayaa',\n  'gayya'},\n '**:/*/': {':/', '://'},\n 'XPST*TSPK*ch*t': {'cheapest'},\n 'PX*AKP*pu*a': {'pucha'},\n 'PST*TSP*be*t': {'beast', 'best'},\n 'TK*KT*de*h': {'dekh'},\n 'L*AL*lo*o': {'lo', 'loo'},\n 'TN*ANT*du*a': {'dunia', 'duniaa', 'duniya', 'duniyaa'},\n 'AL*AL*wa*o': {'waalo', 'wallo', 'walo'},\n 'PKT*ATKP*bi*a': {'bikta'},\n 'PL*LP*ba*l': {'baal', 'bail', 'bal', 'ball', 'bawaal', 'bawal'},\n 'TT*TT*to*d': {'toad', 'tod', 'todd'},\n 'TNJ*AKNT*de*e': {'denge'},\n 'KT*TK*ch*d': {'chhad', 'chhed', 'chhod'},\n 'TRT*TRT*de*t': {'dert'},\n 'A*A*i*i': {'i'},\n 'HP*AP*ho*e': {'hobe', 'hope'},\n 'ANS*SN*wi*s': {'wins'},\n 'KP*PK*cu*p': {'cup'},\n 'MSNK*NSM*mi*g': {'missing', 'missng'},\n 'TT*TT*da*t': {'daat', 'daawat', 'dat'},\n 'N*AN*ne*w': {'new'},\n 'AR*R*ye*r': {'year'},\n 'AL*L*al*l': {'al', 'all'},\n 'KNKRSS*SRNK*co*,': {'congress,'},\n 'TT*ATT*to*y': {'today'},\n 'A*A*we*e': {'we'},\n 'HF*AF*ha*e': {'haive', 'have'},\n 'RJF*FJR*ra*v': {'rajeev', 'rajiv'},\n 'KNT*ATNK*ga*i': {'gandhi', 'gandi'},\n 'JNM*AMNJ*ja*a': {'janma'},\n 'TS*ST*di*s': {'dies', 'dis', 'diss', 'diwas'},\n 'AJN*ANJ*yo*a': {'yojana', 'youjana'},\n 'RHL*LHR*ra*l': {'rahul'},\n 'L*L*le*,': {'le,'},\n 'PP*APP*pa*u': {'paapu', 'pappou', 'pappu', 'pappuuuu'},\n 'SMJNJ*AKNMS*sa*e': {'samjhenge'},\n 'AJ*J*aj*j': {'aj', 'ajj'},\n 'LRK*AKRL*la*a': {'larhka', 'larka', 'larkia'},\n 'LK*AKL*le*e': {'league', 'leke'},\n 'JJ*AJJ*ja*i': {'jaaegi',\n  'jaaigi',\n  'jaayegi',\n  'jaegi',\n  'jayegi',\n  'jayegii',\n  'jaygi'},\n 'PS*SP*bo*s': {'bos', 'boss', 'boys'},\n 'PLT*TLP*bo*d': {'boiled',\n  'bold',\n  'bolliwood',\n  'bollwood',\n  'bollywood',\n  'bowled'},\n 'SN*ANK*ci*e': {'cine'},\n 'STRS*SRTS*st*s': {'stairs', 'stars', 'stores', 'stories', 'stress'},\n 'KJ*JK*kh*j': {'khoj'},\n 'XN*NK*ch*n': {'chaan', 'chain', 'chan', 'cheeen', 'cheen', 'chin'},\n 'AK*K*ik*k': {'ik'},\n 'PL*LP*pa*l': {'paal', 'pal', 'paul', 'payal'},\n '**~*~': {'~'},\n 'A0R*RT*ot*r': {'other'},\n 'APRTRS*SRTRP*op*s': {'operators'},\n 'PTL*LTP*ba*l': {'baadal', 'badal', 'badl'},\n 'MLT*ATLM*mi*e': {'milate', 'milte'},\n 'KTN*ANTK*ki*a': {'kitna'},\n 'AT*X*wa*t': {'wait'},\n 'APK*AKP*aa*a': {'aapaka', 'aapakaa', 'aapka'},\n 'KR*RK*gh*r': {'ghar', 'ghor', 'ghr', 'ghur'},\n 'SNLL*LNS*su*,': {'sunil,'},\n 'JNT*ATNJ*ja*a': {'jaanta', 'janata', 'janta'},\n 'PNJ*AKNP*pi*e': {'piyenge'},\n 'NT*TN*no*t': {'not'},\n 'AR0*TR*wo*h': {'worth'},\n 'AT*T*it*t': {'it'},\n 'AF*AF*wi*i': {'wifi'},\n 'AH*AH*wa*i': {'wahi'},\n 'LL*LL*lo*l': {'lol', 'lool', 'loooooooll', 'lowell'},\n 'JK*AKJ*jo*a': {'jokeya'},\n 'AKS*AK*ac*a': {'accha', 'acchha'},\n 'KT*ATK*gu*u': {'guddu', 'guduu', 'guudu'},\n 'AMXNL*LNTM*em*l': {'emotional'},\n 'K*AK*gy*e': {'gye', 'gyee'},\n 'SRKSM*MSKRS*sa*m': {'sarcasm'},\n 'SMJT*A0JMS*sa*e': {'samajhte'},\n 'MR*ARM*me*i': {'meri'},\n 'KRNNM*MMNRK*ki*,': {'kiran~me,'},\n 'SNXN*ANSNS*su*e': {'sunshine'},\n 'MR*ARM*mh*i': {'mhaari', 'mhari'},\n 'KRN*NRK*ch*n': {'chhoriyan', 'chhoron'},\n 'PP*APP*ba*y': {'baby'},\n 'HLT*ATL*ho*y': {'hoilday', 'holday', 'holiday'},\n 'KPR*RPK*ga*r': {'gabbar', 'gayepar'},\n 'JS*ASJ*ja*i': {'jaiowaisi', 'jaisi'},\n 'ASM*AMS*aw*e': {'aweesomeeee', 'awesome', 'awsome'},\n 'MFS*SFM*mo*s': {'moves', 'movies', 'movis'},\n 'AK*AK*ak*i': {'akki'},\n 'RK*KR*rg*g': {'rg'},\n 'MM*MM*mo*m': {'mom'},\n 'PJT*TKTP*bu*t': {'budget'},\n 'KLN*ANLK*kh*a': {'khelna', 'khilona', 'kholna'},\n 'PT*ATP*be*a': {'beda', 'beta', 'bewda'},\n 'TMR*ARMT*tu*e': {'tumare', 'tumhaare', 'tumhare', 'tumharee', 'tumhre'},\n 'AMN*NM*wo*n': {'woman', 'women'},\n 'AMPRMNT*TNMRPM*em*t': {'empowerment'},\n 'KLNK*AKNLK*kh*e': {'khelneke'},\n 'A*A*ya*y': {'yaay',\n  'yaayy',\n  'yaayyy',\n  'yay',\n  'yayy',\n  'yayyy',\n  'yayyyyyy',\n  'yayyyyyyyy'},\n 'RNK*AKNR*ri*a': {'ringa'},\n 'AS*S*au*s': {'aus'},\n 'L*AL*la*e': {'laae', 'laaye', 'laiye', 'laye', 'layee'},\n 'TK*AKT*te*e': {'teeke', 'teekhe'},\n 'HMR*ARM*ha*a': {'hamaara', 'hamaaraa', 'hamara', 'hamra'},\n 'JN*NJ*ja*n': {'jaaan',\n  'jaaen',\n  'jaan',\n  'jaaon',\n  'jaaun',\n  'jaayein',\n  'jaayen',\n  'jaein',\n  'jain',\n  'jan',\n  'jaon',\n  'jaoon',\n  'jaun',\n  'jawaan',\n  'jawan',\n  'jayein',\n  'jayen',\n  'jayien'},\n 'KH*AHK*ka*i': {'kaahi', 'kahi'},\n 'TRLR*RLRT*tr*r': {'trailer', 'trailor'},\n 'TK*AKT*dh*e': {'dhoke'},\n 'R*HR*ra*h': {'raah', 'rah'},\n 'PHR*RHP*bi*r': {'bihaar', 'bihar'},\n 'AK*K*uc*h': {'uchh'},\n 'TRJ*AJRT*da*e': {'darje'},\n 'KTPN*NPTK*gh*n': {'ghatiyapan'},\n 'HT*T*ho*t': {'hot', 'hott', 'howhat'},\n 'ASK*AKS*is*a': {'iska'},\n 'KNKT*TSNK*co*t': {'connect'},\n 'ASK*AKS*is*e': {'iske'},\n 'S00*TS*sa*,': {'saath,'},\n 'MKR*RKM*ma*r': {'magar', 'maker'},\n 'LN*ANL*li*e': {'line'},\n 'HMN*NM*ha*n': {'haimain', 'haiman', 'hamain', 'hamein'},\n 'KNFST*TSFNK*co*d': {'confused'},\n 'TS*AST*de*i': {'desi'},\n 'FTS*ASTF*vi*i': {'videsi'},\n 'TTS*STT*tw*s': {'tweets'},\n 'RT*ATR*ro*e': {'roadie', 'rode', 'rote', 'route'},\n 'FK*AKP*ph*i': {'pheki'},\n 'A0T*ATT*ut*a': {'uthaata', 'uthata'},\n 'AMR*RM*um*r': {'umar', 'umr'},\n 'AKML*LMK*ak*l': {'akmal'},\n 'L*L*li*,': {'li,', 'liya,', 'liye,'},\n 'KT*ATK*ch*o': {'chhado', 'chhodo'},\n 'HT*AT*ha*e': {'hataiye',\n  'hataye',\n  'hatayiye',\n  'hate',\n  'hatte',\n  'hatyaye',\n  'haute'},\n 'A*A*aa*o': {'aaiyo', 'aao', 'aayo'},\n 'PR*RP*py*r': {'pyaar', 'pyar', 'pyr'},\n 'KRN*NRK*ka*n': {'kaaran',\n  'karan',\n  'karein',\n  'karen',\n  'karien',\n  'karin',\n  'karn',\n  'karon',\n  'karoon',\n  'karun',\n  'karwan',\n  'karwaun'},\n 'TM*MT*dm*m': {'dm'},\n 'AX*AS*ay*a': {'ayesha'},\n 'FN*NF*fa*n': {'fan'},\n 'JRN*NRJ*jo*:': {'journo:'},\n 'LLJ*JLL*la*,': {'laluji,'},\n 'APN*ANP*aa*e': {'aapane', 'aapne'},\n 'PX*AKP*ba*e': {'bacche', 'bache'},\n 'KLL*LLK*ki*:': {'kiyelalu:'},\n 'KNK*AKNK*ky*i': {'kyonhki', 'kyonki', 'kyunki'},\n '0*AT*th*y': {'thay', 'they', 'thy'},\n 'H*A*hu*a': {'hua', 'huaa', 'huwa'},\n 'MT*ATM*mo*i': {'modi', 'moti'},\n 'PM*MP*pm*m': {'pm'},\n 'PN*ANP*ba*a': {'bana',\n  'banaa',\n  'banaya',\n  'baniya',\n  'banna',\n  'banwa',\n  'banwaya'},\n 'XR*ARS*sh*i': {'shari', 'shayari', 'sheri', 'sherri', 'shri'},\n 'KJRL*LRJK*ke*l': {'kejariwal',\n  'kejiriwal',\n  'kejriwaaaaal',\n  'kejriwal',\n  'kejriwall',\n  'kejruwal'},\n 'AP*AP*ab*a': {'aba', 'abba'},\n 'JNJ*AKNJ*ja*e': {'jaanoge', 'jaayenge', 'jaenge', 'jayenge'},\n 'S*AS*so*o': {'so', 'soo', 'sooo', 'sooooo', 'soooooooo'},\n 'J*AJ*ja*o': {'jaao', 'jaiyo', 'jao', 'jaooooooooooooooo', 'jawo'},\n 'KN*ANK*kh*a': {'khaana', 'khana', 'khanna'},\n 'XL*ALK*ch*e': {'chaile',\n  'chalaiye',\n  'chalaye',\n  'chale',\n  'chaliye',\n  'chele',\n  'chile',\n  'chole'},\n 'MST*TSM*ma*t': {'maast', 'mast', 'mastt'},\n 'TK*AKT*de*a': {'dekha'},\n 'PT*ATP*pa*i': {'paati',\n  'padai',\n  'padhaai',\n  'padhai',\n  'padhi',\n  'padi',\n  'padti',\n  'pati',\n  'patti'},\n '0T*ATT*th*i': {'thodai', 'thodi'},\n 'S*AS*si*i': {'si', 'siwai'},\n 'L*AL*li*i': {'li'},\n 'KR*ARK*ch*i': {'chhori', 'chhuri', 'chori'},\n 'H*A*ha*a': {'ha',\n  'haa',\n  'haaa',\n  'hahha',\n  'hahhaa',\n  'haiya',\n  'hawa',\n  'hawaa',\n  'hayya'},\n 'HH**ha*,': {'ha,'},\n 'SNT*ATNS*sa*a': {'sandhya', 'sandia', 'sannaata', 'santa'},\n 'ANK*AKN*in*a': {'inka'},\n 'NM*MN*na*m': {'naaam', 'naam', 'nam'},\n 'LN*NL*le*n': {'lein', 'len', 'leon'},\n 'LK*AKL*lo*o': {'logo', 'loko'},\n 'RK*AKR*ru*a': {'ruka'},\n 'TX*ST*de*h': {'desh'},\n 'ATR*RT*ud*r': {'udaar', 'uddhar', 'udhaar', 'udhar', 'udher'},\n 'APK*AKP*ap*a': {'apka'},\n 'PK*KP*bi*g': {'big', 'bigg'},\n 'MJ*AJM*mu*y': {'mujhay', 'mujhey', 'mujhy', 'mujy'},\n 'APS*ASP*ap*e': {'apse'},\n 'KL*LK*ca*l': {'cal', 'call'},\n 'PT*TP*bt*t': {'bt'},\n 'KRN*ANRK*ka*i': {'karani', 'karni', 'karwani'},\n 'PR*RP*ba*r': {'baaar', 'baar', 'bahr', 'bair', 'bar'},\n 'KNTKT*TKTNK*co*t': {'contact'},\n 'NM*MN*nu*m': {'num'},\n 'SKP*APKS*sk*e': {'skype'},\n 'AT*T*id*d': {'id'},\n 'APTLRMNNNNN*NMRLTP*ab*1': {'abdulrehman6731'},\n 'MPTLRMNSHPT*ATPHSNMRLTPM*me*o': {'meabdulrehmansahabto'},\n 'RT*ATR*ra*e': {'raate', 'radhe', 'rate', 'rayte'},\n 'KT*TK*cu*t': {'cut'},\n 'FT*ATF*fa*a': {'faayda',\n  'faayeda',\n  'fadaa',\n  'faida',\n  'fatwa',\n  'fatwaa',\n  'fayda',\n  'fayeda'},\n 'KNSMR*RMSNK*co*r': {'consumer'},\n 'ML*LM*mi*l': {'mil', 'mill'},\n 'TRTR*RTRT*tr*r': {'trader'},\n 'PS*ASP*ba*e': {'base', 'basie'},\n 'PRS*ASRP*pr*e': {'praise', 'price', 'prize'},\n 'KRK*AKRK*ka*e': {'karake', 'karakoe', 'karaoke', 'karke', 'karoake'},\n 'AS*AS*us*i': {'usi', 'ussi'},\n 'PKT*A0SP*be*a': {'bechta'},\n 'ANTRTNMNT*TNMNTRTN*en*t': {'entertainment'},\n 'H*A*ha*e': {'haaye', 'haayee', 'hae', 'haiye', 'haye', 'hayee', 'hayeee'},\n 'SLM*MLS*za*m': {'zaaalim', 'zaalim'},\n 'PS*SP**w*s': {'*wipes'},\n 'TRR*RT*te**': {'tear*'},\n 'LTST*TSTL*la*t': {'latest'},\n 'SPKS*SPKS*xb*x': {'xbox'},\n 'S*SP*ps*s': {'ps', 'pss'},\n 'PKL*LKP*bh*l': {'bhokal'},\n 'AS*S*us*s': {'us', 'uss'},\n 'ATN*ANT*ud*e': {'udne'},\n 'AN*N*wo*n': {'won'},\n 'PLKL*LKLP*bi*l': {'bilakul', 'bilkool', 'bilkul'},\n 'SHPHR*RHPHS*sa*r': {'sahibihar'},\n 'M*AM*ma*y': {'may'},\n 'AK*AK*ac*e': {'achhe'},\n 'AN*AN*aa*y': {'aaney'},\n 'JR*RJ*jo*r': {'jor'},\n 'PTR*ARTP*po*y': {'poetry', 'pottery'},\n 'JN*NJ*ji*n': {'jin'},\n 'PN*ANP*bi*a': {'biana', 'bina'},\n 'NNT*TNN*ne*d': {'neend'},\n 'SRR*RRS*za*r': {'zaroor', 'zaror', 'zarur'},\n 'TKNN*NKT*de*,': {'dekhein,'},\n 'AS*AS*is*e': {'ise', 'isse', 'issue'},\n 'AN*AN*un*e': {'unhe'},\n 'AJJ*AJJ*aa*i': {'aajayegi'},\n 'NKTTKL*LKKTKN*na*l': {'naked,kal'},\n 'A*A*wh*i': {'whai', 'whi'},\n 'TM*MT*to*m': {'tom'},\n 'P*AP*bo*y': {'boy'},\n 'TP*PT*ty*p': {'typ'},\n 'JSK*AKSJ*ji*e': {'jiske'},\n 'PR*ARP*ba*e': {'baare', 'bare', 'barre', 'barrie'},\n 'PST*TSP*po*t': {'post'},\n 'X*AS*sh*e': {'she', 'shoe'},\n 'ANTS*STN*wa*s': {'wants'},\n 'M*AM*my*y': {'my', 'myy'},\n 'RXN*NTR*ra*n': {'ration'},\n 'KRT*TRK*ca*d': {'card'},\n 'PJRNJ*AKNRJP*ba*i': {'bajarangi', 'bajrangi', 'bajrngi'},\n 'PJN*NJP*bh*n': {'bhaaijaan',\n  'bhaijaaaaan',\n  'bhaijaaan',\n  'bhaijaan',\n  'bhaijan',\n  'bhaijann',\n  'bhajaan',\n  'bhajan',\n  'bhejen',\n  'bhijaan'},\n 'PJ*AJP*bh*a': {'bheeja', 'bheja', 'bhijwa'},\n 'ANKRS*ASRN*an*o': {'angrezo'},\n 'PRT*TRP*bh*t': {'bharat', 'bhart'},\n 'TLT*ATLT*de*e': {'delete', 'delwte'},\n 'TMR*ARMT*tu*i': {'tumhaari', 'tumhari'},\n 'PTNK*NTP*ba*g': {'batting'},\n 'ANS*ASN*aa*o': {'aansoo'},\n 'JLN*ANLJ*ja*e': {'jalaane', 'jalene', 'jalne', 'jaylene'},\n 'JLT*ATLJ*ja*e': {'jalate', 'jaldee', 'jalte'},\n 'RHMT*ATMHR*ra*i': {'rahomodi'},\n 'AJ*AK*aa*e': {'aage', 'aaoge'},\n 'PT*ATP*ba*o': {'baato',\n  'badao',\n  'badhao',\n  'badho',\n  'bado',\n  'bataao',\n  'batao',\n  'bato'},\n 'AMT*TM*um*d': {'umeed', 'umid', 'ummed', 'ummeed', 'ummid'},\n 'TMN*ANMT*tu*e': {'tumane', 'tumne'},\n 'PKN*NKP*bh*n': {'bhagawan', 'bhagwaan', 'bhagwan', 'bhaikyun'},\n 'TR*ART*da*o': {'darao', 'dariyo', 'daro', 'daroo'},\n 'AS*AS*wa*e': {'waise', 'waze'},\n 'ANST*ATSN*in*a': {'insta'},\n 'PKS*SP*pi*s': {'pics'},\n 'AKL*ALK*ak*e': {'akale', 'akele'},\n 'PL*ALP*pe*a': {'pehla', 'pela'},\n 'SX*AKS*so*o': {'socho'},\n 'JP*PJ*jb*b': {'jb'},\n 'KRR*ARRK*cr*e': {'crore'},\n 'PLNJ*AKNLP*pe*e': {'pelenge'},\n 'NTRK*KRTN*ne*k': {'network'},\n 'MT*TM*ma*t': {'mat', 'matt', 'maut'},\n 'PRR*RP*py*,': {'pyaar,', 'pyar,'},\n 'PRPT*TPRP*ba*d': {'barbaad', 'barbad'},\n 'HMR*ARM*ha*e': {'haimere', 'hamaare', 'hamare', 'hamre'},\n 'AT*AT*wh*e': {'whatye', 'white'},\n 'AX*S*wa*h': {'wash'},\n 'R*AR*rh*i': {'rhi'},\n 'AL*ALS*is*e': {'isle', 'islie', 'isliye'},\n 'KP*APK*co*y': {'coby', 'copay', 'copy', 'cowboy'},\n 'R*HR*re*h': {'reh'},\n 'KT*TK*kh*d': {'khod', 'khud'},\n 'HS*S*he*s': {'hes', 'hess'},\n 'TT*TT*da*d': {'dad', 'daud', 'dawood'},\n 'PT*TP*bu*t': {'but', 'butt'},\n 'PR*ARP*pu*a': {'pura', 'puraa'},\n 'ATS*ST*it*s': {'its'},\n 'PRT*ATRP*pr*y': {'preity', 'pretty'},\n 'FT*TF*fe*t': {'feat', 'feet'},\n 'ANKT*TKN*an*t': {'ankit'},\n 'ARNP*PNR*ar*b': {'arnab'},\n 'KT*ATK*ku*a': {'kutiya', 'kutta'},\n 'XM*MS*sh*m': {'shaam', 'sham'},\n 'KLT*TLK*ga*t': {'gahlot', 'galat', 'galt'},\n 'APN*NP*aa*n': {'aapn'},\n 'STJ*AKTS*st*e': {'stage'},\n '0NK*NT*th*k': {'thank', 'think', 'thnk'},\n 'APT*TP*ab*t': {'about', 'abouut', 'abt'},\n 'PRP*APRP*pr*u': {'prabhu'},\n 'K*AK*ke*y': {'keay', 'key'},\n 'M*AM*mi*e': {'mie'},\n 'TR*RT*de*r': {'dear', 'deer', 'deewar', 'der', 'derr', 'dewar'},\n 'PR*RP*pr*r': {'pr', 'prr'},\n 'ANTR*RTN*an*r': {'andar', 'ander', 'andhaer', 'andher', 'andr', 'anter'},\n 'ST*ATS*sa*a': {'saada',\n  'saadha',\n  'sada',\n  'sadaa',\n  'sata',\n  'satiya',\n  'satta',\n  'satya',\n  'sauda'},\n '**h*h': {'h'},\n 'SMR0N*NTRMS*sa*n': {'samarthan'},\n 'MRNT*ATNRM*mi*a': {'miranda'},\n 'HS*AS*ho*e': {'hoise', 'hose', 'hosue', 'house', 'houze'},\n 'P*AP*bi*i': {'bi', 'biwi'},\n 'MXR*ARSM*mi*a': {'mishra'},\n 'H0*AT*ha*i': {'haathi', 'hathi'},\n 'PL*LP*bo*l': {'boil', 'bol', 'bowel', 'bowl'},\n 'JS*ASJ*ja*e': {'jaise', 'jase'},\n 'AXX*AXK*ac*e': {'achche', 'achchhe', 'achchhee'},\n 'JF*AFJ*je*o': {'jeevo'},\n 'KL*ALK*ga*a': {'gaaliya', 'gala', 'galiya'},\n 'SK*KS*sa*h': {'saakh', 'sach', 'sachh'},\n 'MLRT*TRLM*mi*d': {'milord'},\n 'KM*MK*gh*m': {'gham', 'ghoom', 'ghum'},\n 'XL*ALK*ch*i': {'chalai', 'chali', 'chili', 'chilli', 'choli'},\n 'TT*TT*to*t': {'toot'},\n 'TPT*TPT*do*t': {'doubt'},\n 'NMPR*RPMN*nu*r': {'number'},\n 'TJ*AJT*di*e': {'dijie', 'dijiye', 'dijye'},\n 'TR*RT*do*r': {'door', 'dor', 'dour'},\n 'K*AK*ki*e': {'kie', 'kiye'},\n 'FR*ARF*ve*y': {'very'},\n 'NS*ASN*ni*e': {'nice', 'niece'},\n 'K*AK*ka*i': {'kai', 'kauai', 'kayi'},\n 'AK*K*ac*c': {'ac', 'acc'},\n 'AM*AM*em*u': {'emu'},\n 'SRX*SRS*su*h': {'suresh'},\n 'PRPS*SPRP*pr*s': {'prabhus'},\n 'TSPNTS*STNPST*di*s': {'disappoints'},\n 'XF*FS*sh*v': {'shiv'},\n 'SN*ANS*se*a': {'seena', 'sena'},\n 'AS*S*as*s': {'as', 'ass'},\n 'MHRXTR*AR0SRHM*ma*a': {'maharashtra'},\n 'TTNT*TNTT*di*t': {'didnt'},\n 'KT*TK*ge*t': {'get'},\n 'AN0*ATN*an*i': {'anythi'},\n 'KNTNTR*ARTNTNK*ga*a': {'gantantra'},\n 'TFS*SFT*di*s': {'divas'},\n 'LT*ATL*la*u': {'laddu'},\n 'A*H*wa*h': {'waaah', 'waah', 'wah'},\n 'PRN*ANRP*pu*i': {'puraani', 'purani'},\n 'RH*AHR*ra*o': {'raaho', 'rahiyo', 'raho'},\n 'TMK*KMT*dm*g': {'dmaag'},\n 'KL*ALK*kh*e': {'khele', 'khilaaye', 'khole', 'khule', 'khulle'},\n 'HN*AN*ho*e': {'hone'},\n 'MN*ANM*ma*o': {'maano', 'manao', 'mano'},\n 'HL*AL*ho*i': {'holi'},\n 'JNLKPL*LPKLNJ*ja*l': {'janlokpal'},\n 'FT*ATF*va*a': {'vaada', 'vada', 'vaidya'},\n 'PR*ARP*po*a': {'poora', 'pora'},\n 'SPNN*NNPS*sa*n': {'sapnon'},\n 'RN*ANR*ra*i': {'rani'},\n 'AJ*AJ*aa*i': {'aaegi', 'aayaigii', 'aayegi', 'aayegii'},\n 'SM*AMS*sa*e': {'same', 'sammie'},\n 'HR*AR*he*e': {'heere', 'here'},\n 'PSS*SSP*ba*s': {'basis', 'basss', 'bassss'},\n 'JJ*HKJ*ja*h': {'jageh'},\n 'M*AM*ma*a': {'ma', 'maa', 'maaya', 'maia', 'maiyya', 'maya'},\n 'PPL*LPP*bh*l': {'bhopal'},\n 'AKSM*MKS*ex*m': {'exam'},\n 'LKN*ANKL*li*e': {'likhne', 'likne'},\n 'SLPS*SPLS*sy*s': {'syllabus'},\n 'SL*LS*sa*l': {'saal', 'saall', 'sail', 'sal', 'saul', 'sawaal', 'sawal'},\n 'PNX*AKNP*po*u': {'poonchu'},\n 'ANS*SN*an*s': {'annies', 'anns', 'ans'},\n 'PL*ALP*bo*o': {'boliyo', 'bolo'},\n 'F*AF*vo*o': {'vo'},\n 'STR*RTS*st*r': {'star', 'steer', 'stir'},\n 'AX*KT*wa*h': {'watch'},\n 'SP*HPS*su*h': {'subah', 'subbah', 'subeh'},\n 'XS*SK*ch*z': {'cheez', 'chiz'},\n 'AS*AS*ai*i': {'aisi'},\n 'PSNT*TNSP*pa*d': {'pasand'},\n 'ALT*TL*ol*d': {'old'},\n 'KRT*ATRK*ka*i': {'karati', 'kardi', 'karti'},\n 'AR*R*yr*r': {'yr', 'yrr'},\n 'K*AK*gy*a': {'gya', 'gyaa'},\n 'AKX*SK*aa*h': {'aakaash'},\n 'A*H*oh*h': {'oh'},\n 'JSN*NSJ*ja*n': {'jason', 'jayson'},\n ...}"},"metadata":{}}]},{"cell_type":"code","source":"import pickle\nwith open('vocab_data.pkl', 'wb') as fp:\n    pickle.dump(hin_vocab, fp)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:49:09.356265Z","iopub.execute_input":"2023-03-16T17:49:09.357258Z","iopub.status.idle":"2023-03-16T17:49:09.427668Z","shell.execute_reply.started":"2023-03-16T17:49:09.357214Z","shell.execute_reply":"2023-03-16T17:49:09.426595Z"},"trusted":true},"execution_count":183,"outputs":[]},{"cell_type":"code","source":"df['Sentence']=df['Sentence'].apply(lambda x: re.sub(\"[.!?\\\\-\\'\\\"]\", \"\", x))\ndf['Sentence']","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:07.584668Z","iopub.execute_input":"2023-03-16T11:45:07.585757Z","iopub.status.idle":"2023-03-16T11:45:07.864119Z","shell.execute_reply.started":"2023-03-16T11:45:07.585718Z","shell.execute_reply":"2023-03-16T11:45:07.862968Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"0       congratulations on you celebrating british kid...\n1       uske liye toh bahot kuch karna padega ye pappi...\n2       yehi to hum semjhane ki koshish kar rahe hain ...\n3                                          cake kaha hai \n4       jeet ka jashn aur shubah ki shuruat eating bre...\n                              ...                        \n1299                             muze vo bohot pasand hai\n1300                                       mujhe psnd hai\n1301           mujhe market se vegetables lekar aane hain\n1302                                    sabse best friend\n1303                                    best friend sabse\nName: Sentence, Length: 183535, dtype: object"},"metadata":{}}]},{"cell_type":"code","source":"def change_vocab(line):\n    string=''\n    for j in line.split(' '):\n        string=string+list(hin_vocab[doublemetaphone(j)[0]+'*'+doublemetaphone(j[::-1])[0]+'*'+j[:2]+'*'+j[len(j)-1:]])[0]+' '\n    return string","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:07.865855Z","iopub.execute_input":"2023-03-16T11:45:07.866635Z","iopub.status.idle":"2023-03-16T11:45:07.873430Z","shell.execute_reply.started":"2023-03-16T11:45:07.866593Z","shell.execute_reply":"2023-03-16T11:45:07.872363Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"change_vocab('sarey alarms cance kare')","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:07.875354Z","iopub.execute_input":"2023-03-16T11:45:07.875799Z","iopub.status.idle":"2023-03-16T11:45:07.884744Z","shell.execute_reply.started":"2023-03-16T11:45:07.875763Z","shell.execute_reply":"2023-03-16T11:45:07.883474Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"'sarey alarms cance karwayee '"},"metadata":{}}]},{"cell_type":"code","source":"df['Sentence']=df['Sentence'].apply(lambda x: change_vocab(x).strip())","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:07.887050Z","iopub.execute_input":"2023-03-16T11:45:07.887418Z","iopub.status.idle":"2023-03-16T11:45:54.268676Z","shell.execute_reply.started":"2023-03-16T11:45:07.887381Z","shell.execute_reply":"2023-03-16T11:45:54.267619Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.270462Z","iopub.execute_input":"2023-03-16T11:45:54.270851Z","iopub.status.idle":"2023-03-16T11:45:54.282759Z","shell.execute_reply.started":"2023-03-16T11:45:54.270813Z","shell.execute_reply":"2023-03-16T11:45:54.281694Z"},"trusted":true},"execution_count":22,"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"                                               Sentence  \\\n0     congratulations on you celebrating british kid...   \n1     uske liye toh bahat kuch karwana padega yee pa...   \n2     yehi tooo hum semjhane ki koshish karr rahee h...   \n3                                         cake kaha hai   \n4     jett kauwa jashn aur shubah ki shroowat eating...   \n...                                                 ...   \n1299                           muse vo bohut pasand hai   \n1300                                      muje psnd hai   \n1301       muje maarkit see vegtables lekar aaine hawon   \n1302                                    sabse best frnd   \n1303                                    best frnd sabse   \n\n                                                    trr  \n0     congratulations on you celebrating british kid...  \n1     you will have to do a lot for that, it will no...  \n2     this is what we are trying to understand. peop...  \n3                                   where is the cake??  \n4     victory celebration and early morning eating b...  \n...                                                 ...  \n1299                                    i like it a lot  \n1300                                    i like it a lot  \n1301         i have to bring vegetables from the market  \n1302                                   most best friend  \n1303                                   most best friend  \n\n[183535 rows x 2 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Sentence</th>\n      <th>trr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>congratulations on you celebrating british kid...</td>\n      <td>congratulations on you celebrating british kid...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>uske liye toh bahat kuch karwana padega yee pa...</td>\n      <td>you will have to do a lot for that, it will no...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>yehi tooo hum semjhane ki koshish karr rahee h...</td>\n      <td>this is what we are trying to understand. peop...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>cake kaha hai</td>\n      <td>where is the cake??</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>jett kauwa jashn aur shubah ki shroowat eating...</td>\n      <td>victory celebration and early morning eating b...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1299</th>\n      <td>muse vo bohut pasand hai</td>\n      <td>i like it a lot</td>\n    </tr>\n    <tr>\n      <th>1300</th>\n      <td>muje psnd hai</td>\n      <td>i like it a lot</td>\n    </tr>\n    <tr>\n      <th>1301</th>\n      <td>muje maarkit see vegtables lekar aaine hawon</td>\n      <td>i have to bring vegetables from the market</td>\n    </tr>\n    <tr>\n      <th>1302</th>\n      <td>sabse best frnd</td>\n      <td>most best friend</td>\n    </tr>\n    <tr>\n      <th>1303</th>\n      <td>best frnd sabse</td>\n      <td>most best friend</td>\n    </tr>\n  </tbody>\n</table>\n<p>183535 rows × 2 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.284580Z","iopub.execute_input":"2023-03-16T11:45:54.285329Z","iopub.status.idle":"2023-03-16T11:45:54.321449Z","shell.execute_reply.started":"2023-03-16T11:45:54.285289Z","shell.execute_reply":"2023-03-16T11:45:54.320417Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"df['len_sen']=df['Sentence'].apply(lambda x: len(x.split(' ')))\ndf=df[df['len_sen']<=20]","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.323403Z","iopub.execute_input":"2023-03-16T11:45:54.323788Z","iopub.status.idle":"2023-03-16T11:45:54.763578Z","shell.execute_reply.started":"2023-03-16T11:45:54.323750Z","shell.execute_reply":"2023-03-16T11:45:54.762523Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"inp=list(df['Sentence'])\ntarg=list(df['trr'])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.765394Z","iopub.execute_input":"2023-03-16T11:45:54.765774Z","iopub.status.idle":"2023-03-16T11:45:54.833711Z","shell.execute_reply.started":"2023-03-16T11:45:54.765737Z","shell.execute_reply":"2023-03-16T11:45:54.832417Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"# printing last english sentence\nprint(targ[-1])","metadata":{"id":"lH_dPY8TRp3c","outputId":"a4486ac6-bd0f-4b6f-c685-5bdf1983e769","execution":{"iopub.status.busy":"2023-03-16T11:45:54.835218Z","iopub.execute_input":"2023-03-16T11:45:54.835825Z","iopub.status.idle":"2023-03-16T11:45:54.845721Z","shell.execute_reply.started":"2023-03-16T11:45:54.835785Z","shell.execute_reply":"2023-03-16T11:45:54.844518Z"},"trusted":true},"execution_count":26,"outputs":[{"name":"stdout","text":"remove all alarms and reminders\n","output_type":"stream"}]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.847682Z","iopub.execute_input":"2023-03-16T11:45:54.848100Z","iopub.status.idle":"2023-03-16T11:45:55.842937Z","shell.execute_reply.started":"2023-03-16T11:45:54.848038Z","shell.execute_reply":"2023-03-16T11:45:55.841702Z"},"trusted":true},"execution_count":27,"outputs":[{"name":"stdout","text":"Thu Mar 16 11:45:55 2023       \n+-----------------------------------------------------------------------------+\n| NVIDIA-SMI 470.82.01    Driver Version: 470.82.01    CUDA Version: 11.4     |\n|-------------------------------+----------------------+----------------------+\n| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n|                               |                      |               MIG M. |\n|===============================+======================+======================|\n|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n| N/A   35C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n|                               |                      |                  N/A |\n+-------------------------------+----------------------+----------------------+\n                                                                               \n+-----------------------------------------------------------------------------+\n| Processes:                                                                  |\n|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n|        ID   ID                                                   Usage      |\n|=============================================================================|\n|  No running processes found                                                 |\n+-----------------------------------------------------------------------------+\n","output_type":"stream"}]},{"cell_type":"code","source":"gpus = tf.config.experimental.list_physical_devices('GPU')\nfor gpu in gpus:\n    print(\"Name:\", gpu.name, \"  Type:\", gpu.device_type)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:55.866108Z","iopub.execute_input":"2023-03-16T11:45:55.866509Z","iopub.status.idle":"2023-03-16T11:45:56.134202Z","shell.execute_reply.started":"2023-03-16T11:45:55.866449Z","shell.execute_reply":"2023-03-16T11:45:56.133058Z"},"trusted":true},"execution_count":28,"outputs":[{"name":"stdout","text":"Name: /physical_device:GPU:0   Type: GPU\n","output_type":"stream"}]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\n\ndevice_lib.list_local_devices()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:56.135612Z","iopub.execute_input":"2023-03-16T11:45:56.136601Z","iopub.status.idle":"2023-03-16T11:45:58.548906Z","shell.execute_reply.started":"2023-03-16T11:45:56.136559Z","shell.execute_reply":"2023-03-16T11:45:58.547810Z"},"trusted":true},"execution_count":29,"outputs":[{"execution_count":29,"output_type":"execute_result","data":{"text/plain":"[name: \"/device:CPU:0\"\n device_type: \"CPU\"\n memory_limit: 268435456\n locality {\n }\n incarnation: 7469856977397221377\n xla_global_id: -1,\n name: \"/device:GPU:0\"\n device_type: \"GPU\"\n memory_limit: 16105865216\n locality {\n   bus_id: 1\n   links {\n   }\n }\n incarnation: 16859620279429889827\n physical_device_desc: \"device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\"\n xla_global_id: 416903419]"},"metadata":{}}]},{"cell_type":"code","source":"tf.test.is_gpu_available()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:58.552088Z","iopub.execute_input":"2023-03-16T11:45:58.552542Z","iopub.status.idle":"2023-03-16T11:45:58.567023Z","shell.execute_reply.started":"2023-03-16T11:45:58.552496Z","shell.execute_reply":"2023-03-16T11:45:58.566034Z"},"trusted":true},"execution_count":30,"outputs":[{"execution_count":30,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}]},{"cell_type":"code","source":"# printing first 100 english sentence\nprint(targ[:100])","metadata":{"id":"WtYLR0ZWXDld","outputId":"afcfed6b-ee0f-4007-9b3d-9cafc3eac537","execution":{"iopub.status.busy":"2023-03-16T11:45:58.570720Z","iopub.execute_input":"2023-03-16T11:45:58.571137Z","iopub.status.idle":"2023-03-16T11:45:58.579466Z","shell.execute_reply.started":"2023-03-16T11:45:58.571108Z","shell.execute_reply":"2023-03-16T11:45:58.578494Z"},"trusted":true},"execution_count":31,"outputs":[{"name":"stdout","text":"['set a new alarm', 'set an alarm for 9 am', 'resume my timer', 'set a recurring reminder every morning at 9 : 00 to take my vitamins', 'any parties in philly tonight', 'when is my alarm set for on monday ?', 'set up an alarm to go to sleep tonight .', 'how long will it take me to get to syracuse downtown if i leave here in 10 minutes ?', 'modiji has hand in this.', 'should i buy emergency supplies at the store tonight', 'how long is my drive home', 'are there any accidents on the 33', 'set up a 30 minute meditation timer', 'why is the turnpike backed up', 'vancouver night market next weekend', 'is it going to be hot today', 'set up a nightly sleep reminder for 10 pm', 'events in new york', \"remind me that i didn ' t finish my tax return .\", 'open a message to peter asking him to call me after the meeting .', 'set an alert for me to go and pick up the catering order at 11 : 45 pm', 'can you please remind me to pay the utility bill on friday at 2 pm ?', 'didi.. agreed.. both of them were seen as nirma super.', 'cancel reminder 4 pm tomorrow .', 'set a timer to go off every hour', 'i want to listen to classical music for 1 hour', 'set up a reminder to get my passport application started by the end of the week', 'play me some jazz', 'scrapbooking parties in atlanta next weekend', 'can you set a reminder for my eye appointment on august 15th at 2 pm ?', 'concerts by new edition', 'please set the alarm at 6 am for tomorrow .', 'play some rock music from the 60s .', 'run this timer twice', 'how far from cocoa beach to port canaveral', 'set an egg timer for 5 minutes ?', \"what ' s the average temp in virginia this time of year\", \"set up a recurring reminder about little matt ' s football practice monday thru friday at 5 : 30 pm\", 'should i bring a jacket with me ?', 'can i tour the princeton art museum today and park in front of the university ?', 'delete my daily 3 pm and 4 pm alarms', 'cancel tomorrows alarm .', 'how much of a delay for my trip home today', 'skip the next track', 'set timer for 20 minutes from now', 'how is the weather on saturday', 'is 75 faster than 45 right now', 'can i get from here to dover by 9 pm tonight ?', 'message tim asking if he has found my wallet at his house', 'set timer for the laundry', 'please create a weekly alarm for thursday at 5 pm', 'what temp will is be in washington dc this week', 'remind me to be at the meeting at 6 pm today', 'prateek johri dialogue is not in your control. ,', 'suspend my alarm for the weekend', 'restart all timers', 'text mom and tell her ill be at the show next thursday', 'any free xmas events i can go to this weekend', 'set the snooze setting on the alarm clock to 9 minutes .', 'set an alarm daily at 6 am for the following 2 weeks', 'is there a scenic backroad drive from jamesville to highlands , nc ?', 'turn off my saturday alarms .', 'start timer for 5 hours', 'replay this track', 'delete all reminders about the wedding .', 'i wanna text robert', 'how long will it take me to get to utsa by bicycle from the rim ?', 'cancel my reminder about the anniversary gift', 'what is the seven day forecast ?', 'when will the sun rise today ?', 'do i need a jacket today ?', 'change order of songs', 'remind me to make a salad dish for my social dinner tomrorow at 12 pm .', 'text steve , how are you doing ?', 'could you wake us up', 'set alarm for every hour', 'how cold is it for paris this weekend in degree c', 'what is the temperature today ? i am going to walk down to the casino .', 'set a reminder for a party tonight', 'remind me i have a dentist appointment on thursday', 'how is traffic from irvine to tustin', 'sound an alarm once timer is complete', 'restart timer', 'add another ten minutes to the timer .', 'cancel my alarm ( s ) for saturday', 'how hot will it be tomorrow afternoon ?', \"how long will it take me to get to john ' s house if i leave in 15 minutes ?\", 'please tell me what hours traffic is the heaviest in kansas city', 'set up an wake up alarm at midnight today .', 'looks like someone has made him an uncle in shares', 'start a new alarm for very tuesday at noon .', 'thumbs down the current song playing on pandora', 'what time is my doctor appointment reminder ?', 'look at my messages on whatsapp .', \"when will i arrive at bob ' s dinner party if i leave right now using the highway ?\", 'expressway is costlier or cheaper', 'set an alarm to wake up at 4 am .', 'how many miles are we from salt lake city', 'let me see meetup reminders for the history group .', 'what is the temperature today']\n","output_type":"stream"}]},{"cell_type":"code","source":"# printing first 100 hinglish sentences\nprint(inp[-1:])","metadata":{"id":"WdC25G6AZXr9","outputId":"26bc2c3e-c413-4715-f08a-65b5ad1ba001","execution":{"iopub.status.busy":"2023-03-16T11:45:58.580765Z","iopub.execute_input":"2023-03-16T11:45:58.582330Z","iopub.status.idle":"2023-03-16T11:45:58.591359Z","shell.execute_reply.started":"2023-03-16T11:45:58.582288Z","shell.execute_reply":"2023-03-16T11:45:58.589997Z"},"trusted":true},"execution_count":32,"outputs":[{"name":"stdout","text":"['sabhi alarms aur remnders koo hatya deon']\n","output_type":"stream"}]},{"cell_type":"code","source":"# arrays of strings are shuffled and batches are made\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 64\n\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)","metadata":{"id":"TqHsArVZ3jFS","execution":{"iopub.status.busy":"2023-03-16T11:45:58.593109Z","iopub.execute_input":"2023-03-16T11:45:58.593511Z","iopub.status.idle":"2023-03-16T11:46:00.512669Z","shell.execute_reply.started":"2023-03-16T11:45:58.593475Z","shell.execute_reply":"2023-03-16T11:46:00.511527Z"},"trusted":true},"execution_count":33,"outputs":[]},{"cell_type":"code","source":"for example_input_batch, example_target_batch in dataset.take(1):\n  print(example_input_batch[:5])\n  print()\n  print(example_target_batch[:5])\n  break","metadata":{"id":"qc6-NK1GtWQt","outputId":"4059f9ad-4d6e-4204-d40a-840261b82f90","execution":{"iopub.status.busy":"2023-03-16T11:46:00.515424Z","iopub.execute_input":"2023-03-16T11:46:00.516310Z","iopub.status.idle":"2023-03-16T11:46:00.828098Z","shell.execute_reply.started":"2023-03-16T11:46:00.516262Z","shell.execute_reply":"2023-03-16T11:46:00.826860Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"tf.Tensor(\n[b'taylor swift koo play karwayee' b'1 hour kee liye timer seat karein'\n b'kyaa aj koii live music hai'\n b'kyaa aap muje meree location kee liye traffic updates dee sakate heeee'\n b'9 a  m kee liye alaram koo cancel karein'], shape=(5,), dtype=string)\n\ntf.Tensor(\n[b'play taylor swift' b'set the timer for 1 hour'\n b'any live music tonight'\n b'can you give me traffic updates for my location'\n b'cancel alarm for 9 a . m .'], shape=(5,), dtype=string)\n","output_type":"stream"}]},{"cell_type":"code","source":"# unicode normalization operation - first step to standardize data\nexample_text = tf.constant('Yahi haal hai')\n\nprint(example_text.numpy())\nprint(tf_text.normalize_utf8(example_text, 'NFKD').numpy())","metadata":{"id":"mD0e-DWGQ2Vo","outputId":"52e95336-e962-4e89-da30-956c44458493","execution":{"iopub.status.busy":"2023-03-16T11:46:00.829577Z","iopub.execute_input":"2023-03-16T11:46:00.859628Z","iopub.status.idle":"2023-03-16T11:46:00.870845Z","shell.execute_reply.started":"2023-03-16T11:46:00.859578Z","shell.execute_reply":"2023-03-16T11:46:00.869271Z"},"trusted":true},"execution_count":35,"outputs":[{"name":"stdout","text":"b'Yahi haal hai'\nb'Yahi haal hai'\n","output_type":"stream"}]},{"cell_type":"code","source":"def tf_lower_and_split_punct(text):\n  # Split accecented characters.\n  text = tf_text.normalize_utf8(text, 'NFKD')\n  text = tf.strings.lower(text)\n  # Keep space, a to z, and select punctuation.\n  text = tf.strings.regex_replace(text, '[^ a-z.?!,¿]', '')\n  # Add spaces around punctuation.\n  text = tf.strings.regex_replace(text, '[.?!,¿]', r' \\0 ')\n  # Strip whitespace.\n  text = tf.strings.strip(text)\n\n  text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n  return text","metadata":{"id":"chTF5N885F0P","execution":{"iopub.status.busy":"2023-03-16T11:46:00.872665Z","iopub.execute_input":"2023-03-16T11:46:00.874045Z","iopub.status.idle":"2023-03-16T11:46:00.881931Z","shell.execute_reply.started":"2023-03-16T11:46:00.873999Z","shell.execute_reply":"2023-03-16T11:46:00.880784Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"code","source":"print(example_text.numpy().decode())\nprint(tf_lower_and_split_punct(example_text).numpy().decode())","metadata":{"id":"UREvDg3sEKYa","outputId":"c0abfd72-ff76-4661-809c-d1b40f6debb8","execution":{"iopub.status.busy":"2023-03-16T11:46:00.883659Z","iopub.execute_input":"2023-03-16T11:46:00.884080Z","iopub.status.idle":"2023-03-16T11:46:00.907442Z","shell.execute_reply.started":"2023-03-16T11:46:00.884041Z","shell.execute_reply":"2023-03-16T11:46:00.906256Z"},"trusted":true},"execution_count":37,"outputs":[{"name":"stdout","text":"Yahi haal hai\n[START] yahi haal hai [END]\n","output_type":"stream"}]},{"cell_type":"code","source":"max_vocab_size = 53000\n\ninput_text_processor = tf.keras.layers.TextVectorization(    # handles vocabulary extraction & conversion of i/p text to sequences of tokens\n    standardize=tf_lower_and_split_punct,\n    max_tokens=max_vocab_size)","metadata":{"id":"eAY9k49G3jE_","execution":{"iopub.status.busy":"2023-03-16T11:46:00.909139Z","iopub.execute_input":"2023-03-16T11:46:00.909823Z","iopub.status.idle":"2023-03-16T11:46:00.948831Z","shell.execute_reply.started":"2023-03-16T11:46:00.909781Z","shell.execute_reply":"2023-03-16T11:46:00.947635Z"},"trusted":true},"execution_count":38,"outputs":[]},{"cell_type":"code","source":"input_text_processor.adapt(inp)    # initializes the layer based on the data\n\n# Here are the first 10 words from the vocabulary:\ninput_text_processor.get_vocabulary()[:10]\n# Hinglish TextVectorization layer","metadata":{"id":"bmsI1Yql8FYe","outputId":"89009883-0834-449f-d408-0d947cd033b2","execution":{"iopub.status.busy":"2023-03-16T11:46:00.951091Z","iopub.execute_input":"2023-03-16T11:46:00.951727Z","iopub.status.idle":"2023-03-16T11:46:13.843270Z","shell.execute_reply.started":"2023-03-16T11:46:00.951684Z","shell.execute_reply":"2023-03-16T11:46:13.842283Z"},"trusted":true},"execution_count":39,"outputs":[{"execution_count":39,"output_type":"execute_result","data":{"text/plain":"['', '[UNK]', '[START]', '[END]', 'kee', 'liye', 'hai', 'koo', 'kyaa', 'mee']"},"metadata":{}}]},{"cell_type":"code","source":"output_text_processor = tf.keras.layers.TextVectorization(\n    standardize=tf_lower_and_split_punct,\n    max_tokens=max_vocab_size)\n\noutput_text_processor.adapt(targ)\noutput_text_processor.get_vocabulary()[:10]\n# English TextVectorization layer","metadata":{"id":"jlC4xuZnKLBS","outputId":"9b307cdd-52db-48bf-f9bc-2510dd84bd19","execution":{"iopub.status.busy":"2023-03-16T11:46:13.844725Z","iopub.execute_input":"2023-03-16T11:46:13.845119Z","iopub.status.idle":"2023-03-16T11:46:35.612054Z","shell.execute_reply.started":"2023-03-16T11:46:13.845081Z","shell.execute_reply":"2023-03-16T11:46:35.610936Z"},"trusted":true},"execution_count":40,"outputs":[{"execution_count":40,"output_type":"execute_result","data":{"text/plain":"['', '[UNK]', '[START]', '[END]', 'the', 'to', '.', 'for', 'is', '?']"},"metadata":{}}]},{"cell_type":"code","source":"# the above layers convert a batch of strings into a batch of token IDs\nexample_tokens = input_text_processor(example_input_batch)\nexample_tokens[:3, :10]","metadata":{"id":"9KZxj8IrNZ9S","outputId":"ed7ca51e-cd13-4c35-dd4e-fd706b1fb34a","execution":{"iopub.status.busy":"2023-03-16T11:46:35.613717Z","iopub.execute_input":"2023-03-16T11:46:35.614109Z","iopub.status.idle":"2023-03-16T11:46:35.701463Z","shell.execute_reply.started":"2023-03-16T11:46:35.614070Z","shell.execute_reply":"2023-03-16T11:46:35.700450Z"},"trusted":true},"execution_count":41,"outputs":[{"execution_count":41,"output_type":"execute_result","data":{"text/plain":"<tf.Tensor: shape=(3, 10), dtype=int64, numpy=\narray([[  2, 547, 626,   7,  79,  10,   3,   0,   0,   0],\n       [  2, 273,   4,   5,  19,  14,  41,   3,   0,   0],\n       [  2,   8,  63,  42, 510,  72,   6,   3,   0,   0]])>"},"metadata":{}}]},{"cell_type":"code","source":"# get_vocabulary - converts token IDs back to text\ninput_vocab = np.array(input_text_processor.get_vocabulary())\ntokens = input_vocab[example_tokens[0].numpy()]\n' '.join(tokens)","metadata":{"id":"98g9rcxGQY0I","outputId":"43515fd5-6811-41ec-d9e9-8e1c4d920d81","execution":{"iopub.status.busy":"2023-03-16T11:46:35.703028Z","iopub.execute_input":"2023-03-16T11:46:35.703389Z","iopub.status.idle":"2023-03-16T11:46:35.776070Z","shell.execute_reply.started":"2023-03-16T11:46:35.703352Z","shell.execute_reply":"2023-03-16T11:46:35.775051Z"},"trusted":true},"execution_count":42,"outputs":[{"execution_count":42,"output_type":"execute_result","data":{"text/plain":"'[START] taylor swift koo play karwayee [END]            '"},"metadata":{}}]},{"cell_type":"code","source":"# token IDs converted into a mask\nplt.subplot(1, 2, 1)\nplt.pcolormesh(example_tokens)\nplt.title('Token IDs')\n\nplt.subplot(1, 2, 2)\nplt.pcolormesh(example_tokens != 0)\nplt.title('Mask')","metadata":{"id":"_jx4Or_eFRSz","outputId":"fd68e833-9404-407b-a54b-333bb35ffd2b","execution":{"iopub.status.busy":"2023-03-16T11:46:35.778833Z","iopub.execute_input":"2023-03-16T11:46:35.779871Z","iopub.status.idle":"2023-03-16T11:46:36.103825Z","shell.execute_reply.started":"2023-03-16T11:46:35.779830Z","shell.execute_reply":"2023-03-16T11:46:36.101972Z"},"trusted":true},"execution_count":43,"outputs":[{"execution_count":43,"output_type":"execute_result","data":{"text/plain":"Text(0.5, 1.0, 'Mask')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 2 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAh8AAAGxCAYAAADCo9TSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvyklEQVR4nO3de3hU9Z3H8c/kwpA7csskCBhtvACKVTCC0qQqsWhZW+iKYhXUKhZ0m7JKC/iUYN0gWFncjcXV9YKtij7demltleyi0cpa4wVLsRWVcGkxRNOQBAgJSX77h81sxoDmB3N+c3u/nmeex5z5zpnvYfDLd34533N8xhgjAAAAR5IinQAAAEgsNB8AAMApmg8AAOAUzQcAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKdoPgAAgFM0HzHM5/P16fHSSy/1aV833nij90n3QUlJicaMGROy7bjjjgseT1JSknJycnTKKafoqquu0rp16yKUKYCHH374c2uNMUZf+tKX5PP5VFJSEvb337Ztm3w+n37yk5+Efd/wTkqkE8CR+9///d+Qn3/84x/rxRdf1Pr160O2jxo1ymVanjnnnHOCBWbv3r167733tHbtWl144YWaPn26Hn/8caWmpkY4SyAxZWVl6YEHHujVYFRXV+vDDz9UVlZWZBJDVKL5iGFnn312yM9DhgxRUlJSr+3xYsCAASHHdsEFF2jevHkqLy/X0qVLdeutt2r58uURzBBIXDNmzNCjjz6qe+65R9nZ2cHtDzzwgCZMmKDm5uYIZodow69d4tzf/vY3zZ07V8OGDVO/fv10/PHHa/HixWpra/vc1xljtGjRIqWmpur+++8Pbn/iiSc0YcIEZWRkKDMzUxdeeKHefvvtkNfOnj1bmZmZ+uCDD3TRRRcpMzNTw4cP1z//8z9/4fseifLyco0ePVqVlZU6cOBAcPvq1as1duxYZWZmKisrSyeffLIWLVoU9vcHIF1++eWSpMcffzy4rampSf/1X/+la665plf80qVLVVRUpIEDByo7O1tnnHGGHnjgAX32Xqfr169XSUmJBg0apLS0NI0YMULTp0/X/v37D5vLwYMHNWvWLGVmZurXv/51mI4Q4UTzEccOHDigr371q3rkkUc0f/58Pffcc/r2t7+tFStWaNq0aYd9XVtbm2bOnKnKykr96le/0nXXXSdJqqio0OWXX65Ro0bpySef1M9+9jO1tLRo0qRJevfdd0P2cfDgQf3DP/yDzj//fD3zzDO65ppr9K//+q+erUxMnTpV+/fv1xtvvCFJWrt2rebOnavi4mI99dRTevrpp/X9739f+/bt8+T9gUSXnZ2tb33rW3rwwQeD2x5//HElJSVpxowZveK3bdumOXPm6Mknn9Qvf/lLTZs2TTfddJN+/OMfh8RcfPHF6tevnx588EE9//zzuuOOO5SRkaH29vZD5rFnzx5deOGFWrdunaqrq/X1r389/AeLo2cQN2bNmmUyMjKCP997771GknnyySdD4pYvX24kmXXr1gW3STLz5s0zDQ0N5txzzzXDhg0zGzduDD6/Y8cOk5KSYm666aaQfbW0tJhAIGAuvfTSkDwO9b4XXXSROemkk77wOIqLi83o0aNDto0cOdJcfPHFh33N6tWrjSTzxBNPGGOMufHGG82AAQO+8L0AHJ2HHnrISDI1NTXmxRdfNJLMH//4R2OMMePHjzezZ882xhgzevRoU1xcfMh9dHZ2moMHD5rbbrvNDBo0yHR1dRljjPnFL35hJIXUos+qra01ksydd95pamtrzahRo8yoUaPMtm3bwnugCCtWPuLY+vXrlZGRoW9961sh22fPni1J+p//+Z+Q7bW1tcHfzb722msaO3Zs8LkXXnhBHR0duuqqq9TR0RF89O/fX8XFxb3Ocvf5fJo6dWrIttNOO03bt28P3wH2YD6zVHvWWWdpz549uvzyy/XMM8/ok08+8eR9Afy/4uJinXDCCXrwwQe1adMm1dTUHPJXLtKn9emCCy5QTk6OkpOTlZqaqh/96EdqaGhQfX29JOn0009Xv379dP3112vNmjXaunXrYd/7rbfe0tlnn63c3Fy9+uqrGjlypCfHiPCg+YhjDQ0NCgQC8vl8IduHDh2qlJQUNTQ0hGx//fXXtWXLFs2YMUPHHntsyHO7d++WJI0fP16pqakhjyeeeKLXP+7p6enq379/yDa/3x9yTkY4dTc1+fn5kqQrr7xSDz74oLZv367p06dr6NChKioqUlVVlSfvD+DTLx1XX321fv7zn+vee+/ViSeeqEmTJvWKe/3111VaWipJuv/++/Xqq6+qpqZGixcvliS1trZKkk444QT993//t4YOHap58+bphBNO0AknnKC777671z6rqqq0e/dufec739GAAQO8O0iEBdMucWzQoEH6/e9/L2NMSANSX1+vjo4ODR48OCR+xowZCgQCWrx4sbq6unTrrbcGn+uO/cUvfhF13yiMMfrVr36ljIwMjRs3Lrj96quv1tVXX619+/bp5Zdf1pIlS/T1r39dW7ZsibpjAOLF7Nmz9aMf/Uj33nuv/uVf/uWQMWvXrlVqaqp+/etfh3xJefrpp3vFTpo0SZMmTVJnZ6feeOMN/fu//7vKysqUm5uryy67LBh3yy236MMPPwyuzl511VVhPzaED81HHDv//PP15JNP6umnn9Y3v/nN4PZHHnkk+Pxn3XrrrcrKygqenLls2TJJ0oUXXqiUlBR9+OGHmj59upsD6KOlS5fq3Xff1aJFi3qttkhSRkaGpkyZovb2dn3jG9/Q5s2baT4AjwwbNky33HKL/vznP2vWrFmHjPH5fEpJSVFycnJwW2trq372s58ddr/JyckqKirSySefrEcffVRvvfVWSPORlJSk//iP/1BmZqZmz56tffv26bvf/W74DgxhRfMRx6666irdc889mjVrlrZt26ZTTz1Vv/vd71RRUaGLLrpIF1xwwSFf973vfU+ZmZm6/vrrtXfvXv3bv/2bjjvuON12221avHixtm7dqq997Ws65phjtHv3br3++uvKyMjQ0qVLPT2ePXv26LXXXpMk7du3L3iRsVdeeUWXXnppyPtfd911SktL0znnnKO8vDzV1dVp2bJlysnJ0fjx4z3NE0h0d9xxx+c+f/HFF2vlypWaOXOmrr/+ejU0NOgnP/mJ/H5/SNy9996r9evX6+KLL9aIESN04MCB4DTN4erXXXfdpaysLM2dO1d79+7VLbfcEp6DQljRfMSx/v3768UXX9TixYt155136uOPP9awYcN08803a8mSJZ/72muvvVYZGRm68sortW/fPv3nf/6nFi5cqFGjRunuu+/W448/rra2NgUCAY0fP1433HCD58fz6quvasKECfL5fMrIyNCwYcN01lln6dZbbw3+/rjbpEmT9PDDD+vJJ59UY2OjBg8erHPPPVePPPKIhgwZ4nmuAA7vvPPO04MPPqjly5dr6tSpGjZsmK677joNHTpU1157bTDu9NNP17p167RkyRLV1dUpMzNTY8aM0bPPPtvr//meysvLlZmZqVtuuUV79+71/IsR7PnMZ8cEAAAAPMS0CwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp2g+AACAU1F3nY+uri7t2rVLWVlZve5JAsANY4xaWlqUn5+vpKTY+I5C7QAiy6ZuRF3zsWvXLg0fPjzSaQCQtHPnzl43GYxW1A4gOvSlbkRd85GVlSVJOlcXKUWpfXpNcmaG1Xt07t1nFe9L6Vse3Uxnp1W8TJddvC+Kvona5o6Y0KGD+p1+E/z/MRZ057r9reOUnRlF/48g6n3zxFMjnUJcsKkbUdd8dC+XpihVKb4+Nh++fpbv0W4Zb9l8WDcHMdx82OaO2PD36x7H0q8vunPNzkxSdlbyF0QD/6+v/9bgC1jUjWj6VwwAACQAmg8AAOBU1P3a5Uh07t3r6f5Nx0FP92/N8jyLJH//Psd2tR2wzQYAos6F+WMjnQI+BysfAADAKZoPAADgFM0HAABwiuYDAAA4RfMBAACciotpF1vJmZlW8V5P09iyveIqEywAog3TKImNlQ8AAOAUzQcAAHCK5gMAADhF8wEAAJyi+QAAAE7FxbSL7fRHtE2v2Iq6e80AiDtMo8BLrHwAAACnrJuPv/71r/r2t7+tQYMGKT09XaeffrrefPPN4PPGGJWXlys/P19paWkqKSnR5s2bw5o0gNhD7QDQzar5aGxs1DnnnKPU1FT99re/1bvvvqu77rpLAwYMCMasWLFCK1euVGVlpWpqahQIBDR58mS1tLSEO3cAMYLaAaAnq3M+li9fruHDh+uhhx4KbjvuuOOC/22M0apVq7R48WJNmzZNkrRmzRrl5ubqscce05w5c8KTNYCYQu0A0JPVysezzz6rcePG6R//8R81dOhQffnLX9b9998ffL62tlZ1dXUqLS0NbvP7/SouLtaGDRsOuc+2tjY1NzeHPADEF2oHgJ6sVj62bt2q1atXa/78+Vq0aJFef/11/dM//ZP8fr+uuuoq1dXVSZJyc3NDXpebm6vt27cfcp/Lli3T0qVLjzD9T9lOf/iSk+3239lpt3/L6Rvb/ct0WYWn5A7tc2zH7nq7fZ9caBXf8ef3reKT0tOt4rv277eKTxk0yCq+o6HBKh6fitbagcN7Ydc7kU4BMaa5pVPHnNi3WKuVj66uLp1xxhmqqKjQl7/8Zc2ZM0fXXXedVq9eHRLn8/lCfjbG9NrWbeHChWpqago+du7caZMSgBhA7QDQk1XzkZeXp1GjRoVsO+WUU7Rjxw5JUiAQkKTgt5hu9fX1vb7RdPP7/crOzg55AIgv1A4APVk1H+ecc47ee++9kG1btmzRyJEjJUkFBQUKBAKqqqoKPt/e3q7q6mpNnDgxDOkCiEXUDgA9WZ3z8f3vf18TJ05URUWFLr30Ur3++uu67777dN9990n6dMm0rKxMFRUVKiwsVGFhoSoqKpSenq6ZM2d6cgAAoh+1A0BPVs3H+PHj9dRTT2nhwoW67bbbVFBQoFWrVumKK64IxixYsECtra2aO3euGhsbVVRUpHXr1ikrKyvsyQOIDdQOAD35jDEm0kn01NzcrJycHJXoEqX47KZG+iplxLFW8V0f2004dLW2WsXHMq8nhxAZHeagXtIzampqiplzKbprR+OW45WdZff3EomN+9iEh03d4N4uAADAKZoPAADgFM0HAABwiuYDAAA4ZTXtEq1sL8HdseMvHmXiiM+yZ7S4HHtSWprVrm1PrrX9rEy73aXzk4dYXi79o7ovDgIQ17y+lDwntPbGygcAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKfiYtqla/9+q/iUk75kFX9gxAC7/Ve9YRXvNZtLoJv2dg8zsf+sbDG9AsAW0yjusfIBAACcovkAAABO0XwAAACnaD4AAIBTNB8AAMCpuJh2sdXx3gdW8SlboqtHs5lekSTT0ff7oyRnZtqmY6Vz715P9w8Atry+t4utRJi+ia5/VQEAQNyj+QAAAE7RfAAAAKdoPgAAgFM0HwAAwKmEnHaRz7LnMl3e5NHNMp+kjHSr+M6mpr7HMo0CIAolwgRIImHlAwAAOEXzAQAAnKL5AAAATtF8AAAApxLzhFOP2V7+PHnoELs3SLX82CxOOI150XYyMYCwiLZLoEeTWDwZl5UPAADgFM0HAABwiuYDAAA4RfMBAACcovkAAABOJea0i+WEQ1JamlV814E2q/iOj+qs4q0nOhIJ0ytARMTixAUih3/FAACAUzQfAADAKZoPAADgFM0HAABwiuYDAAA4FRfTLil5Aat42+mSAf+dbhX/t3NareJt2d47xnRYTIBYTtL4knx2uXQZq3hrTLsAEcG9Vw6PSaDeWPkAAABO0XwAAACnaD4AAIBTNB8AAMApmg8AAOBUXEy7WN8bxdLfzmnwdP/JA4+xiu9qavYoE1lPi5hOj/Loxn1sAEQZpleOHpUdAAA4ZdV8lJeXy+fzhTwCgf+/xoYxRuXl5crPz1daWppKSkq0efPmsCcNILZQOwD0ZL3yMXr0aH300UfBx6ZNm4LPrVixQitXrlRlZaVqamoUCAQ0efJktbS0hDVpALGH2gGgm3XzkZKSokAgEHwMGTJE0qffXFatWqXFixdr2rRpGjNmjNasWaP9+/frscceC3viAGILtQNAN+vm4/3331d+fr4KCgp02WWXaevWrZKk2tpa1dXVqbS0NBjr9/tVXFysDRs2HHZ/bW1tam5uDnkAiD/UDgDdrKZdioqK9Mgjj+jEE0/U7t27dfvtt2vixInavHmz6uo+nTjJzc0NeU1ubq62b99+2H0uW7ZMS5cuPYLU40fn3xojnUKQLyXV7gW291IZN9pu96//0W7/ttMx3AvGCWoH4onX97FJhGkaq0o9ZcoUTZ8+XaeeeqouuOACPffcc5KkNWvWBGN8vtAbjRljem3raeHChWpqago+du7caZMSgBhA7QDQ01GN2mZkZOjUU0/V+++/HzxzvftbTLf6+vpe32h68vv9ys7ODnkAiG/UDiCxHVXz0dbWpj/96U/Ky8tTQUGBAoGAqqqqgs+3t7erurpaEydOPOpEAcQPageQ2KzO+bj55ps1depUjRgxQvX19br99tvV3NysWbNmyefzqaysTBUVFSosLFRhYaEqKiqUnp6umTNnepU/gBhA7QDQk1Xz8Ze//EWXX365PvnkEw0ZMkRnn322XnvtNY0cOVKStGDBArW2tmru3LlqbGxUUVGR1q1bp6ysLE+SP1Ipw/Kt4rs+/sQuvr3dKj6amI6D3r7B7//g7f4RleKldiSSRDjpEZHjM8aYSCfRU3Nzs3JyclSiS5Tis5y86COaD+DzdZiDeknPqKmpKWbOpeiuHY1bjld2VnKk04l5NB+wZVM3uLcLAABwiuYDAAA4RfMBAACcovkAAABOWU27RKvkMSdZxXdsft8q3pd0+KssJjpfst2Jfaaz06NMAIST15cQ9xonzEY3Vj4AAIBTNB8AAMApmg8AAOAUzQcAAHCK5gMAADgVF9MunX98z9P9GwY0DovpFQAuML0SX1j5AAAATtF8AAAAp2g+AACAUzQfAADAKZoPAADgVFxMu9hKzsqyiu9sabF7A59lT2e67OIt+VJS+xxrOg56mAkAAKx8AAAAx2g+AACAUzQfAADAKZoPAADgVEKecGp9Aqkt2xNIPT5BlZNIAQDRhJUPAADgFM0HAABwiuYDAAA4RfMBAACcovkAAABOxcW0S8rQIVbxZsgxVvGdm7dYxfuSk63iTWenVTwA2Lowf2ykUwCCWPkAAABO0XwAAACnaD4AAIBTNB8AAMApmg8AAOBUXEy7dNR/bPeCjxu8SeTvmF4BEG1e2PVOpFOIWkwCucfKBwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp+Ji2sWa6fJ0957f28Vn2TN6fLwAwMQIbLDyAQAAnKL5AAAATtF8AAAAp2g+AACAUzQfAADAqcScdvGY5/d2YXoFABDDjmrlY9myZfL5fCorKwtuM8aovLxc+fn5SktLU0lJiTZv3ny0eQKIE9QNAEfcfNTU1Oi+++7TaaedFrJ9xYoVWrlypSorK1VTU6NAIKDJkyerpaXlqJMFENuoGwCkI2w+9u7dqyuuuEL333+/jjnmmOB2Y4xWrVqlxYsXa9q0aRozZozWrFmj/fv367HHHgtb0gBiD3UDQLcjaj7mzZuniy++WBdccEHI9traWtXV1am0tDS4ze/3q7i4WBs2bDjkvtra2tTc3BzyABB/wlk3JGoHEMusTzhdu3at3nrrLdXU1PR6rq6uTpKUm5sbsj03N1fbt28/5P6WLVumpUuX2qaR0Gwv354cyP3ioL/r2FVnlwwnv6IPwl03JGpHtHlh1zuRTiEEl3uPblYrHzt37tT3vvc9/fznP1f//v0PG+fz+UJ+Nsb02tZt4cKFampqCj527txpkxKAKOdF3ZCoHUAss1r5ePPNN1VfX68zzzwzuK2zs1Mvv/yyKisr9d5770n69JtMXl5eMKa+vr7Xt5pufr9ffr//SHIHEAO8qBsStQOIZVYrH+eff742bdqkjRs3Bh/jxo3TFVdcoY0bN+r4449XIBBQVVVV8DXt7e2qrq7WxIkTw548gOhH3QDwWVYrH1lZWRozZkzItoyMDA0aNCi4vaysTBUVFSosLFRhYaEqKiqUnp6umTNnhi9rADGDugHgs8J+hdMFCxaotbVVc+fOVWNjo4qKirRu3TplZWWF+60AxAnqBpBYfMYYE+kkempublZOTo5KdIlSfKmevEeS//AnvR1KV9sBT/IAolWHOaiX9IyampqUnZ0d6XT6pLt2NG45XtlZdhNhcI9plPhjUze4sRwAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKfCPmobCSlDBlvFd3z8iUeZuOFLsZsCSh44oM+xHfUf2+VieZ8ZW8m5Q63iO3fXW8Wbzk6rePn63q8npdr975UUsDvWjh1/sYoHokm03QsGh+fFZBIrHwAAwCmaDwAA4BTNBwAAcIrmAwAAOEXzAQAAnIqLaZdEm15Jys60iu9s+JtVvA3raRFLHbs+8nT/1kxXn0O72tutdt3F9AoQl7iPTW+sfAAAAKdoPgAAgFM0HwAAwCmaDwAA4FRcnHDqNdtLiNuehGk6DlrFdzY2WcUDQLThJMzExsoHAABwiuYDAAA4RfMBAACcovkAAABO0XwAAACnmHbpA9vLn9tOu6QMGmQV39HQYLf/Y4f1fd9/+avVvuWz7F8tLk8OIH69sOudSKcQxOSNe6x8AAAAp2g+AACAUzQfAADAKZoPAADgFM0HAABwKi6mXby+90pX2wGreFu20yu20zc2Eyxe/1kCgAtMsEQ3Vj4AAIBTNB8AAMApmg8AAOAUzQcAAHCK5gMAADgVF9MuthMXyZmZVvFdra1W8V5PgJiOg97tm+kVIC4x/YFowsoHAABwiuYDAAA4RfMBAACcovkAAABOxcUJp7ZsT6q0jU/q188qvqu93SoeAGy9sOsdq3hOUIWXWPkAAABO0XwAAACnaD4AAIBTNB8AAMApmg8AAOBUQk672F4uPcnf327/bQes4uWz7AFNl93uk5P7vmsurw5A9tMxXmP6Jr6w8gEAAJyyaj5Wr16t0047TdnZ2crOztaECRP029/+Nvi8MUbl5eXKz89XWlqaSkpKtHnz5rAnDSC2UDsA9GTVfBx77LG644479MYbb+iNN97Qeeedp0suuSRYJFasWKGVK1eqsrJSNTU1CgQCmjx5slpaWjxJHkBsoHYA6MlnjDFHs4OBAwfqzjvv1DXXXKP8/HyVlZXpBz/4gSSpra1Nubm5Wr58uebMmdOn/TU3NysnJ0clukQpvtSjSS1sOOcDiabDHNRLekZNTU3Kzs725D28qh2NW45Xdlbf/84jNnDOR/SzqRtHfM5HZ2en1q5dq3379mnChAmqra1VXV2dSktLgzF+v1/FxcXasGHDYffT1tam5ubmkAeA+EXtAGA97bJp0yZNmDBBBw4cUGZmpp566imNGjUqWCRyc3ND4nNzc7V9+/bD7m/ZsmVaunSpbRohUk76klV8x3sfWMXb3nsl2u7tYrWa4fEqDBJXNNYOAJFhvfJx0kknaePGjXrttdf03e9+V7NmzdK7774bfN7n84XEG2N6betp4cKFampqCj527txpmxKAGEDtANDNeuWjX79++tKXPl1pGDdunGpqanT33XcHf1dbV1envLy8YHx9fX2vbzQ9+f1++f1+2zQAxBhqB4BuR32dD2OM2traVFBQoEAgoKqqquBz7e3tqq6u1sSJE4/2bQDEGWoHkLisVj4WLVqkKVOmaPjw4WppadHatWv10ksv6fnnn5fP51NZWZkqKipUWFiowsJCVVRUKD09XTNnzvQqfwAxgNoBoCer5mP37t268sor9dFHHyknJ0ennXaann/+eU2ePFmStGDBArW2tmru3LlqbGxUUVGR1q1bp6ysLE+SBxAbqB0Aejrq63yEWzRe5yNlWL5VfMdfd3mUyZGxus5Hl+VfB6Zd4pKL63yEG9f5QE9cF8Q9J9f5AAAAOBI0HwAAwCmaDwAA4BTNBwAAcIrmAwAAOGV9hdNoZDPNIdnfubWzbrdVfMqgQVbx+9bajRP6lwywik9qbu1zbOcf37Pbt+0dfz2+j03KcLvJpK6PG+ziW/v+Z5lsOSbqGzjAKl7tB63COz6qs9s/EMNe2PVOpFNIOM0tnTrmxL7FsvIBAACcovkAAABO0XwAAACnaD4AAIBTcXHCqe0JpPLZ9VzWJ6ju2WMV759sd9KjLZ/F5eFtT5LsbGmxTcdTHTv+EukUgqz/bKLszxJAZMTqpeE7zEFJW/sUy8oHAABwiuYDAAA4RfMBAACcovkAAABO0XwAAACn4mLaxZrp8nb3ltMxSWlpVvE2l/iWpI6/7rKKBwBbsTqhgchg5QMAADhF8wEAAJyi+QAAAE7RfAAAAKdoPgAAgFMJOe2S5O9vFd/VdsAqPnn0iVbx5v1tVvG+lFS7/XcctIoHAFsv7Hon0ik4w2TP0WPlAwAAOEXzAQAAnKL5AAAATtF8AAAAp2g+AACAUwk57WI7vZLUr59VfOfmLVbxvuRku3jLfJh2AeA1JkBgg5UPAADgFM0HAABwiuYDAAA4RfMBAACcSsgTTm11tbd7un/T2WkX39rqUSYAYgUneCKWsfIBAACcovkAAABO0XwAAACnaD4AAIBTNB8AAMCpxJx28Vn2XKbLmzz+zpeSahVvfbl0m+P1+Fij7c8eiFUv7Hon0inEDSaH3GPlAwAAOEXzAQAAnKL5AAAATtF8AAAAp2g+AACAU4k57RJlExTW0yvWbxBFxxtNuQA4LCZA4CVWPgAAgFNWzceyZcs0fvx4ZWVlaejQofrGN76h9957LyTGGKPy8nLl5+crLS1NJSUl2rx5c1iTBhBbqB0AerJqPqqrqzVv3jy99tprqqqqUkdHh0pLS7Vv375gzIoVK7Ry5UpVVlaqpqZGgUBAkydPVktLS9iTBxAbqB0AevIZY8yRvvjjjz/W0KFDVV1dra985Ssyxig/P19lZWX6wQ9+IElqa2tTbm6uli9frjlz5nzhPpubm5WTk6MSXaIUn92VPwGER4c5qJf0jJqampSdnR32/XtZOxq3HK/srOSw55xoOOcDtmzqxlGd89HU1CRJGjhwoCSptrZWdXV1Ki0tDcb4/X4VFxdrw4YNh9xHW1ubmpubQx4A4hu1A0hsR9x8GGM0f/58nXvuuRozZowkqa6uTpKUm5sbEpubmxt87rOWLVumnJyc4GP48OHWuST162f18Fq05QNEk2iqHQAi44ibjxtvvFF/+MMf9Pjjj/d6zufzhfxsjOm1rdvChQvV1NQUfOzcufNIUwIQA6gdAI7oOh833XSTnn32Wb388ss69thjg9sDgYCkT7/F5OXlBbfX19f3+kbTze/3y+/3H0kaAGIMtQOAZLnyYYzRjTfeqF/+8pdav369CgoKQp4vKChQIBBQVVVVcFt7e7uqq6s1ceLE8GQMIOZQOwD0ZLXyMW/ePD322GN65plnlJWVFfxdbE5OjtLS0uTz+VRWVqaKigoVFhaqsLBQFRUVSk9P18yZMz05AADRj9oBoCer5mP16tWSpJKSkpDtDz30kGbPni1JWrBggVpbWzV37lw1NjaqqKhI69atU1ZWVlgSBhB7qB0Aejqq63x4get8AJHn9XU+vMB1PiKL64LA2XU+AAAAbNF8AAAAp2g+AACAUzQfAADAqSO6yFi0Sc7MtIrv3LvXKt72Euhd7e1W8fJZ9oCmy273yX0/+c50dtrlAgCSXtj1jqf754TW+MLKBwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp+Ji2sV2esWW7fSKzXSJJJkub69wzwQLAFtMl8BLrHwAAACnaD4AAIBTNB8AAMApmg8AAOAUzQcAAHAqLqZdvL43ii3r6RLb/G3Z7N/yzyb5tFOs4jv/8CereFvWk0bRNAkUZX+Pkdi8vlcLYocXk0+sfAAAAKdoPgAAgFM0HwAAwCmaDwAA4BTNBwAAcCo+pl2ijC8l1SredBz0KJNPJaX2/WO2nf7wenrFVlRNr9hiegU4LO41E19Y+QAAAE7RfAAAAKdoPgAAgFM0HwAAwKn4OOHU9kQ9jy9j7fUJpLa62tsjnQIAHBUvL/fOyazusfIBAACcovkAAABO0XwAAACnaD4AAIBTNB8AAMCp+Jh2scVlrAEkGCY6EE1Y+QAAAE7RfAAAAKdoPgAAgFM0HwAAwCmaDwAA4FRCTruYSV+2ik/Z+IFVvC8n2yre7N9vFd/5t0ar+A/+7ew+x478dYfVvtc//IBVPGfcA5Hh5b1RohG1Jrqx8gEAAJyi+QAAAE7RfAAAAKdoPgAAgFM0HwAAwKmEnHbx/c7urO9O23vBtLTYxdvy2fWMX/qn1zxKhDPKARwZakdiY+UDAAA4Zd18vPzyy5o6dary8/Pl8/n09NNPhzxvjFF5ebny8/OVlpamkpISbd68OVz5AohB1A0APVk3H/v27dPYsWNVWVl5yOdXrFihlStXqrKyUjU1NQoEApo8ebJavP5VBICoRd0A0JP1OR9TpkzRlClTDvmcMUarVq3S4sWLNW3aNEnSmjVrlJubq8cee0xz5sw5umwBxCTqBoCewnrOR21trerq6lRaWhrc5vf7VVxcrA0bNhzyNW1tbWpubg55AEgcR1I3JGoHEMvCOu1SV1cnScrNzQ3Znpubq+3btx/yNcuWLdPSpUvDmcYXs5xeSfL3t4rvajsQVfu3kTxggFV85549nuSBxHEkdUOKUO1A2Njea4bpmPjiybSLz+cL+dkY02tbt4ULF6qpqSn42LlzpxcpAYhyNnVDonYAsSysKx+BQEDSp99k8vLygtvr6+t7favp5vf75ff7w5kGgBhyJHVDonYAsSysKx8FBQUKBAKqqqoKbmtvb1d1dbUmTpwYzrcCECeoG0DisV752Lt3rz744IPgz7W1tdq4caMGDhyoESNGqKysTBUVFSosLFRhYaEqKiqUnp6umTNnhjVxALGDugGgJ+vm44033tBXv/rV4M/z58+XJM2aNUsPP/ywFixYoNbWVs2dO1eNjY0qKirSunXrlJWVFb6sP8vycuO2TMdBT/fv5QmktjiBFF6IyrqBiOIE0sTmM8aYSCfRU3Nzs3JyclSiS5TiS+3bizxuPnxJhz/p7VBMZ6dHmQBudJiDeknPqKmpSdnZ2ZFOp0+6a0fjluOVnZUc6XTwBWg+4o9N3eDeLgAAwCmaDwAA4BTNBwAAcIrmAwAAOBXWi4zFDMvLqxvb80dtT4C1zMfz/duIsmNNGTrYKr5jd71VvA1fSh9PmP476xOVvfxcAY/ZXl4d0a+5pVPHnNi3WFY+AACAUzQfAADAKZoPAADgFM0HAABwiuYDAAA4FR/TLl5PUAweaBXf8fEnVvFJ6el28WlpVvEdDQ1W8Va8nriw3L+X0yu2vL4nEBBNuFw6OsxBSVv7FMvKBwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp+Jj2sXj+4vYTq/Y6tq/39N4APBatN2rhemb6MbKBwAAcIrmAwAAOEXzAQAAnKL5AAAATsXFCae+JJ/lK5Ktok1np+X+AQDhxAmk8YWVDwAA4BTNBwAAcIrmAwAAOEXzAQAAnKL5AAAATsXFtEusT6Mk9etnFd/V3u5RJvaSBwywiu9qabGKN13GKt720vkAYkO0Xb7dBpM6vbHyAQAAnKL5AAAATtF8AAAAp2g+AACAUzQfAADAqbiYdok2Sf7+VvFdbQc8ysR7nXv2RDoFAFGAiQ7YYOUDAAA4RfMBAACcovkAAABO0XwAAACnaD4AAIBTcTHt4ktJtYo3HQc9yuRTsTy9AgCA11j5AAAATtF8AAAAp2g+AACAUzQfAADAKZoPAADgVHxMu6TaHYZtfFdrq1U8AMQ67tUCL3m28vHTn/5UBQUF6t+/v84880y98sorXr0VgDhB3QASgyfNxxNPPKGysjItXrxYb7/9tiZNmqQpU6Zox44dXrwdgDhA3QAShyfNx8qVK3XttdfqO9/5jk455RStWrVKw4cP1+rVq714OwBxgLoBJI6wn/PR3t6uN998Uz/84Q9DtpeWlmrDhg294tva2tTW1hb8uampSZLUoYOS6dt7JpnkI0+4D7qMt1dEBaJNhz79O29MH/8nPEq2dUM6fO1o3tvlXaIJpIO6B0s2dSPszccnn3yizs5O5ebmhmzPzc1VXV1dr/hly5Zp6dKlvbb/Tr/p+5tyPijgiZaWFuXk5Hj+PrZ1Qzp87Rh5xjYvUkxAWyOdAGJUX+qGZ9MuPp8v5GdjTK9tkrRw4ULNnz8/+POePXs0cuRI7dixw0nRi7Tm5mYNHz5cO3fuVHZ2dqTT8RTHGjuMMWppaVF+fr7T9+1r3ZASu3bE+t8vW4l0vLF8rDZ1I+zNx+DBg5WcnNzr20p9fX2vbzWS5Pf75ff7e23PycmJuT/4o5GdnZ0wx8uxxgaX/4Db1g2J2iHF9t+vI5FIxxurx9rXuhH2E0779eunM888U1VVVSHbq6qqNHHixHC/HYA4QN0AEosnv3aZP3++rrzySo0bN04TJkzQfffdpx07duiGG27w4u0AxAHqBpA4PGk+ZsyYoYaGBt1222366KOPNGbMGP3mN7/RyJEjv/C1fr9fS5YsOeRyajxKpOPlWPF5jqZuSIn1Z55Ixyol1vEmyrH6jKtZOgAAAHFjOQAA4BjNBwAAcIrmAwAAOEXzAQAAnKL5AAAATkVd8/HTn/5UBQUF6t+/v84880y98sorkU4p7MrLy+Xz+UIegUAg0mmFzcsvv6ypU6cqPz9fPp9PTz/9dMjzxhiVl5crPz9faWlpKikp0ebNmyOT7FH6omOdPXt2r8/67LPPjkyycSwR6oYU37UjkeqGRO2IqubjiSeeUFlZmRYvXqy3335bkyZN0pQpU7Rjx45IpxZ2o0eP1kcffRR8bNq0KdIphc2+ffs0duxYVVZWHvL5FStWaOXKlaqsrFRNTY0CgYAmT56slpYWx5kevS86Vkn62te+FvJZ/+Y3FjdNxBdKpLohxW/tSKS6IVE7ZKLIWWedZW644YaQbSeffLL54Q9/GKGMvLFkyRIzduzYSKfhhCTz1FNPBX/u6uoygUDA3HHHHcFtBw4cMDk5Oebee++NQIbh89ljNcaYWbNmmUsuuSQi+SSKRKkbxiRO7UikumFMYtaOqFn5aG9v15tvvqnS0tKQ7aWlpdqwYUOEsvLO+++/r/z8fBUUFOiyyy7T1q2Jcfvq2tpa1dXVhXzOfr9fxcXFcfk5S9JLL72koUOH6sQTT9R1112n+vr6SKcUNxKtbkiJWTsSsW5I8V07oqb5+OSTT9TZ2dnrDpa5ubm97nQZ64qKivTII4/ohRde0P3336+6ujpNnDhRDQ0NkU7Nc92fZSJ8zpI0ZcoUPfroo1q/fr3uuusu1dTU6LzzzlNbW1ukU4sLiVQ3pMStHYlWN6T4rx2e3NvlaPh8vpCfjTG9tsW6KVOmBP/71FNP1YQJE3TCCSdozZo1mj9/fgQzcycRPmfp0/uVdBszZozGjRunkSNH6rnnntO0adMimFl8SZS/T4leOxLlc5biv3ZEzcrH4MGDlZyc3KuLra+v79XtxpuMjAydeuqpev/99yOdiue6z8xPxM9ZkvLy8jRy5MiE+KxdSOS6ISVO7Uj0uiHFX+2ImuajX79+OvPMM1VVVRWyvaqqShMnToxQVm60tbXpT3/6k/Ly8iKdiucKCgoUCARCPuf29nZVV1fH/ecsSQ0NDdq5c2dCfNYuJHLdkBKndiR63ZDir3ZE1a9d5s+fryuvvFLjxo3ThAkTdN9992nHjh264YYbIp1aWN18882aOnWqRowYofr6et1+++1qbm7WrFmzIp1aWOzdu1cffPBB8Ofa2lpt3LhRAwcO1IgRI1RWVqaKigoVFhaqsLBQFRUVSk9P18yZMyOY9ZH5vGMdOHCgysvLNX36dOXl5Wnbtm1atGiRBg8erG9+85sRzDq+JErdkOK7diRS3ZCoHVE1amuMMffcc48ZOXKk6devnznjjDNMdXV1pFMKuxkzZpi8vDyTmppq8vPzzbRp08zmzZsjnVbYvPjii0ZSr8esWbOMMZ+OzS1ZssQEAgHj9/vNV77yFbNp06bIJn2EPu9Y9+/fb0pLS82QIUNMamqqGTFihJk1a5bZsWNHpNOOO4lQN4yJ79qRSHXDGGqHzxhj3LY7AAAgkUXNOR8AACAx0HwAAACnaD4AAIBTNB8AAMApmg8AAOAUzQcAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKdoPgAAgFP/B2IplhRAsSS+AAAAAElFTkSuQmCC\n"},"metadata":{}}]},{"cell_type":"code","source":"# defining constants for the model\nembedding_dim = 256\nunits = 1024","metadata":{"id":"_a9uNz3-IrF-","execution":{"iopub.status.busy":"2023-03-16T11:46:36.105561Z","iopub.execute_input":"2023-03-16T11:46:36.105951Z","iopub.status.idle":"2023-03-16T11:46:36.111169Z","shell.execute_reply.started":"2023-03-16T11:46:36.105911Z","shell.execute_reply":"2023-03-16T11:46:36.109957Z"},"trusted":true},"execution_count":44,"outputs":[]},{"cell_type":"markdown","source":"Encoding","metadata":{"id":"6tQ8xnOH7mFc"}},{"cell_type":"code","source":"class Encoder(tf.keras.layers.Layer):\n  def __init__(self, input_vocab_size, embedding_dim, enc_units):\n    super(Encoder, self).__init__()\n    self.enc_units = enc_units\n    self.input_vocab_size = input_vocab_size\n\n    # The embedding layer converts tokens to vectors\n    self.embedding = tf.keras.layers.Embedding(self.input_vocab_size,\n                                               embedding_dim)\n\n    # The GRU RNN layer processes those vectors sequentially.\n    self.gru = tf.keras.layers.GRU(self.enc_units,\n                                   # Return the sequence and state\n                                   return_sequences=True,\n                                   return_state=True,\n                                   recurrent_initializer='glorot_uniform',\n                                  recurrent_dropout=0.1)\n\n  def call(self, tokens, state=None):\n    shape_checker = ShapeChecker()\n    shape_checker(tokens, ('batch', 's'))\n\n    # 2. The embedding layer looks up the embedding for each token.\n    vectors = self.embedding(tokens)\n    shape_checker(vectors, ('batch', 's', 'embed_dim'))\n\n    # 3. The GRU processes the embedding sequence.\n    #    output shape: (batch, s, enc_units)\n    #    state shape: (batch, enc_units)\n    output, state = self.gru(vectors, initial_state=state)\n    shape_checker(output, ('batch', 's', 'enc_units'))\n    shape_checker(state, ('batch', 'enc_units'))\n\n    # 4. Returns the new sequence and its state.\n    return output, state","metadata":{"id":"nZ2rI24i3jFg","execution":{"iopub.status.busy":"2023-03-16T11:46:36.113169Z","iopub.execute_input":"2023-03-16T11:46:36.113539Z","iopub.status.idle":"2023-03-16T11:46:36.124161Z","shell.execute_reply.started":"2023-03-16T11:46:36.113502Z","shell.execute_reply":"2023-03-16T11:46:36.122978Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"# Convert the input text to tokens.\nexample_tokens = input_text_processor(example_input_batch)\n\n# Encode the input sequence.\nencoder = Encoder(input_text_processor.vocabulary_size(),\n                  embedding_dim, units)\nexample_enc_output, example_enc_state = encoder(example_tokens)\n\nprint(f'Input batch, shape (batch): {example_input_batch.shape}')\nprint(f'Input batch tokens, shape (batch, s): {example_tokens.shape}')\nprint(f'Encoder output, shape (batch, s, units): {example_enc_output.shape}')\nprint(f'Encoder state, shape (batch, units): {example_enc_state.shape}')","metadata":{"id":"60gSVh05Jl6l","outputId":"99c5ac95-5e34-4591-e00a-16b2d068052d","execution":{"iopub.status.busy":"2023-03-16T11:46:36.125926Z","iopub.execute_input":"2023-03-16T11:46:36.126373Z","iopub.status.idle":"2023-03-16T11:46:38.342644Z","shell.execute_reply.started":"2023-03-16T11:46:36.126335Z","shell.execute_reply":"2023-03-16T11:46:38.341549Z"},"trusted":true},"execution_count":46,"outputs":[{"name":"stdout","text":"Input batch, shape (batch): (64,)\nInput batch tokens, shape (batch, s): (64, 19)\nEncoder output, shape (batch, s, units): (64, 19, 1024)\nEncoder state, shape (batch, units): (64, 1024)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Attention layer","metadata":{"id":"mI7Zgyrp89GK"}},{"cell_type":"code","source":"class BahdanauAttention(tf.keras.layers.Layer):\n  def __init__(self, units):\n    super().__init__()\n    # For Eqn. (4), the  Bahdanau attention\n    self.W1 = tf.keras.layers.Dense(units, use_bias=False)\n    self.W2 = tf.keras.layers.Dense(units, use_bias=False)\n\n    self.attention = tf.keras.layers.AdditiveAttention()\n\n  def call(self, query, value, mask):\n    shape_checker = ShapeChecker()\n    shape_checker(query, ('batch', 't', 'query_units'))\n    shape_checker(value, ('batch', 's', 'value_units'))\n    shape_checker(mask, ('batch', 's'))\n\n    # From Eqn. (4), `W1@ht`.\n    w1_query = self.W1(query)\n    shape_checker(w1_query, ('batch', 't', 'attn_units'))\n\n    # From Eqn. (4), `W2@hs`.\n    w2_key = self.W2(value)\n    shape_checker(w2_key, ('batch', 's', 'attn_units'))\n\n    query_mask = tf.ones(tf.shape(query)[:-1], dtype=bool)\n    value_mask = mask\n\n    context_vector, attention_weights = self.attention(\n        inputs = [w1_query, value, w2_key],\n        mask=[query_mask, value_mask],\n        return_attention_scores = True,\n    )\n    shape_checker(context_vector, ('batch', 't', 'value_units'))\n    shape_checker(attention_weights, ('batch', 't', 's'))\n\n    return context_vector, attention_weights","metadata":{"id":"momiE59lXo6U","execution":{"iopub.status.busy":"2023-03-16T11:46:38.344306Z","iopub.execute_input":"2023-03-16T11:46:38.344676Z","iopub.status.idle":"2023-03-16T11:46:38.354659Z","shell.execute_reply.started":"2023-03-16T11:46:38.344638Z","shell.execute_reply":"2023-03-16T11:46:38.353267Z"},"trusted":true},"execution_count":47,"outputs":[]},{"cell_type":"code","source":"# creating BahdanauAttention layer\nattention_layer = BahdanauAttention(units)","metadata":{"id":"t4QMlOp8Gidh","execution":{"iopub.status.busy":"2023-03-16T11:46:38.356518Z","iopub.execute_input":"2023-03-16T11:46:38.357309Z","iopub.status.idle":"2023-03-16T11:46:38.369756Z","shell.execute_reply.started":"2023-03-16T11:46:38.357220Z","shell.execute_reply":"2023-03-16T11:46:38.368696Z"},"trusted":true},"execution_count":48,"outputs":[]},{"cell_type":"code","source":"(example_tokens != 0).shape","metadata":{"id":"DYSHqmORgVFo","outputId":"8feaa6f2-f454-4fa6-bf69-29762aacebee","execution":{"iopub.status.busy":"2023-03-16T11:46:38.371579Z","iopub.execute_input":"2023-03-16T11:46:38.372037Z","iopub.status.idle":"2023-03-16T11:46:38.381523Z","shell.execute_reply.started":"2023-03-16T11:46:38.371917Z","shell.execute_reply":"2023-03-16T11:46:38.380268Z"},"trusted":true},"execution_count":49,"outputs":[{"execution_count":49,"output_type":"execute_result","data":{"text/plain":"TensorShape([64, 19])"},"metadata":{}}]},{"cell_type":"code","source":"# Later, the decoder will generate this attention query\nexample_attention_query = tf.random.normal(shape=[len(example_tokens), 2, 10])\n\n# Attend to the encoded tokens\n\ncontext_vector, attention_weights = attention_layer(\n    query=example_attention_query,\n    value=example_enc_output,\n    mask=(example_tokens != 0))\n\nprint(f'Attention result shape: (batch_size, query_seq_length, units):           {context_vector.shape}')\nprint(f'Attention weights shape: (batch_size, query_seq_length, value_seq_length): {attention_weights.shape}')","metadata":{"id":"7y7hjPkNMmHh","outputId":"600899f2-e9d6-467a-ba4a-04ab287d4f08","execution":{"iopub.status.busy":"2023-03-16T11:46:38.383542Z","iopub.execute_input":"2023-03-16T11:46:38.383972Z","iopub.status.idle":"2023-03-16T11:46:38.461258Z","shell.execute_reply.started":"2023-03-16T11:46:38.383936Z","shell.execute_reply":"2023-03-16T11:46:38.460196Z"},"trusted":true},"execution_count":50,"outputs":[{"name":"stdout","text":"Attention result shape: (batch_size, query_seq_length, units):           (64, 2, 1024)\nAttention weights shape: (batch_size, query_seq_length, value_seq_length): (64, 2, 19)\n","output_type":"stream"}]},{"cell_type":"code","source":"plt.subplot(1, 2, 1)\nplt.pcolormesh(attention_weights[:, 0, :])\nplt.title('Attention weights')\n\nplt.subplot(1, 2, 2)\nplt.pcolormesh(example_tokens != 0)\nplt.title('Mask')\n","metadata":{"id":"Rqr8XGsAJlf6","outputId":"a8f59ec0-d2a6-49a2-de95-aa80d7f7a7b7","execution":{"iopub.status.busy":"2023-03-16T11:46:38.462954Z","iopub.execute_input":"2023-03-16T11:46:38.463333Z","iopub.status.idle":"2023-03-16T11:46:38.765069Z","shell.execute_reply.started":"2023-03-16T11:46:38.463296Z","shell.execute_reply":"2023-03-16T11:46:38.763953Z"},"trusted":true},"execution_count":51,"outputs":[{"execution_count":51,"output_type":"execute_result","data":{"text/plain":"Text(0.5, 1.0, 'Mask')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 2 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAh8AAAGxCAYAAADCo9TSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1a0lEQVR4nO3de3hU1b3/8c8kJJOEXASUTCIQIg0qBW+ASChNqCQWKb9W1NbSQ0OvKtia0laLtBosBovKD4+0WCwqVlFP6/VYL8QL0V8DNWi1NPaxqBFiIUZpTMItIWH9/uBkTmKCZsHsNTOZ9+t55nnInpW9v3uGrHxnZX3X8hljjAAAAByJC3cAAAAgtpB8AAAAp0g+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcIvkAAABOkXyEwH/+53/K5/Np7NixvT7/xhtvqKysTO+++26P59avX6+VK1d6G2Af4pg3b55GjhzpJA6X3n33Xfl8Pt19991H9f0+n09XXHHFp7arqqpSWVmZPvroo6O6DhCt7r77bvl8Pvl8Pm3cuLHH88YYfeYzn5HP51NhYWHIr9/5M37zzTeH/NzwDslHCNx5552SpJqaGv3lL3/p8fwbb7yhJUuWRETycaQ4fvGLX+iRRx5xEodLWVlZ2rRpk2bOnOnpdaqqqrRkyRKSD8SstLQ0rV27tsfxyspKvf3220pLSwtDVIhUJB/HaMuWLXr99deDv9x6++GLBqNGjdKZZ54Z7jBCzu/365xzztEJJ5wQ7lCAfu1rX/uaHnroITU3N3c7vnbtWk2ePFkjRowIU2SIRCQfx6gz2bjxxhuVn5+vBx54QPv27Qs+f/fdd+viiy+WJE2bNi04PHn33XersLBQf/rTn7R9+/bgcZ/PF/zetrY2LV26VKeccor8fr9OOOEEfetb39IHH3zQLYaRI0fqS1/6kp5++mmdddZZSk5O1imnnBIckfm0OKTe/+xy4MABLVq0SLm5uUpMTNSJJ56oBQsW9Ph035frH8nEiRN7jEqMGzdOPp9P1dXVwWMPP/ywfD6ftm7dGjy2bds2zZkzR0OHDpXf79epp56qX//6193OdaQ/uzz22GM67bTT5Pf7ddJJJ+nWW29VWVlZt9e/q9///vc69dRTlZKSotNPP11PPPFE8LmysjL99Kc/lSTl5ub2GIJ+/vnnVVhYqCFDhig5OVkjRozQhRde2O3/CRDtvv71r0uS7r///uCxpqYmPfTQQ/r2t7/do/2SJUs0adIkDR48WOnp6TrrrLO0du1afXyv06P5+Tl48KBKSkqUmpra7WcVEcTgqO3bt89kZGSYiRMnGmOM+d3vfmckmbvvvjvYpqGhwZSXlxtJ5te//rXZtGmT2bRpk2loaDA1NTVmypQpJhAIBI9v2rTJGGNMR0eH+eIXv2gGDhxolixZYioqKszvfvc7c+KJJ5oxY8aYffv2Ba+Rk5Njhg0bZsaMGWPuuece88wzz5iLL77YSDKVlZWfGocxxpSUlJicnJzgOQ8dOmTOO+88M2DAAPOLX/zCbNiwwdx8881m4MCB5swzzzQHDhywuv6R/OxnPzOpqammra3NGGNMfX29kWSSk5PNDTfcEGx3+eWXm8zMzODXNTU1JiMjw4wbN87cc889ZsOGDebHP/6xiYuLM2VlZcF2tbW1RpK56667gseeeuopExcXZwoLC80jjzxi/vCHP5hJkyaZkSNHmo//SEgyI0eONGeffbb5r//6L/Pkk0+awsJCM2DAAPP2228bY4ypq6szP/jBD4wk8/DDDwdf26amJlNbW2uSkpJMUVGRefTRR83GjRvNfffdZ+bOnWsaGxs/8bUBosFdd91lJJnq6mozd+5cc/bZZwefW716tRk4cKBpbm42n/3sZ01BQUHwuXnz5pm1a9eaiooKU1FRYX75y1+a5ORks2TJkmCbvvz8dP6M33TTTcYYYxobG820adNMIBAwW7ZscfIawB7JxzG45557jCRz++23G2OMaWlpMampqWbq1Knd2v3hD38wkswLL7zQ4xwzZ87s9ku/0/33328kmYceeqjb8erqaiPJ/OY3vwkey8nJMUlJSWb79u3BY/v37zeDBw82l156aZ/i+Hjy8fTTTxtJZvny5d3aPfjgg0aSWbNmjfX1e/Pss88aSebFF180xhhz7733mrS0NDN//nwzbdq0YLu8vDwzZ86c4NfnnXeeGTZsmGlqaup2viuuuMIkJSWZf//738aY3pOPiRMnmuHDh5vW1tbgsZaWFjNkyJBek4/MzEzT3NwcPFZfX2/i4uLMsmXLgsduuukmI8nU1tZ2+/4//vGPRpJ57bXXPvF1AKJV1+TjhRdeMJLM3//+d2PM4Z+1efPmGWNMj+Sjq46ODnPw4EFz/fXXmyFDhphDhw4ZY/r289M1+aitrTVjxowxY8aMMe+++25obxQhxZ9djsHatWuVnJysSy65RJKUmpqqiy++WC+99JK2bdt2TOd+4okndNxxx2nWrFlqb28PPs444wwFAoEes8rPOOOMbn9TTUpK0ujRo7V9+/ajuv7zzz8v6fCfY7q6+OKLNXDgQD333HMhuf6UKVOUlJSkZ599VpJUUVGhwsJCffGLX1RVVZX27dunuro6bdu2TdOnT5d0+M9Bzz33nC644AKlpKR0e33OP/98HThwQJs3b+71env37tWWLVv0la98RYmJicHjqampmjVrVq/fM23atG6T5TIzMzV06NA+vbZnnHGGEhMT9f3vf1/r1q3TO++886nfA0SrgoICjRo1Snfeeae2bt2q6urqXv/kIh3uY6ZPn66MjAzFx8crISFB1157rXbv3q2GhgZJdj8/r776qs455xxlZmbqz3/+s3Jycjy5R4QGycdReuutt/Tiiy9q5syZMsboo48+0kcffaSLLrpIkvo03+GTvP/++/roo4+UmJiohISEbo/6+np9+OGH3doPGTKkxzn8fr/2799/VNffvXu3BgwY0GOips/nUyAQ0O7du0Ny/aSkJE2ZMiWYfDz33HMqKipSYWGhOjo69NJLL6miokKSgsnH7t271d7erttuu63Ha3P++edLUo/Xp1NjY6OMMcrMzOzxXG/HjuXepMMTeZ999lkNHTpUCxYs0KhRozRq1Cjdeuutn/q9QLTx+Xz61re+pXvvvVe33367Ro8eralTp/Zo9/LLL6u4uFiSdMcdd+jPf/6zqqurtXjxYkkK/mzZ/PxUVFTo/fff13e/+10dd9xx3t0kQmJAuAOIVnfeeaeMMfrjH/+oP/7xjz2eX7dunZYuXar4+PijOv/xxx+vIUOG6Omnn+71ea/L1oYMGaL29nZ98MEH3RIQY4zq6+s1ceLEkF3r3HPP1bXXXquXX35Z7733noqKipSWlqaJEyeqoqJCO3fu1OjRozV8+HBJ0qBBgxQfH6+5c+dqwYIFvZ4zNze31+ODBg2Sz+fT+++/3+O5+vr6kN1TV1OnTtXUqVPV0dGhLVu26LbbblNpaakyMzODo2ZAfzFv3jxde+21uv3223XDDTf02uaBBx5QQkKCnnjiCSUlJQWPP/rooz3a9vXn56c//anefvttffOb31R7e7u++c1vhvzeEDqMfByFjo4OrVu3TqNGjdILL7zQ4/HjH/9Yu3bt0lNPPSXp8KdkSb1+Uj7SJ+gvfelL2r17tzo6OjRhwoQej5NPPtk67k+K4+POPfdcSdK9997b7fhDDz2kvXv3Bp8PhenTp6u9vV2/+MUvNGzYMJ1yyinB488++2xweLZTSkqKpk2bpr/+9a867bTTen19ehutkKSBAwdqwoQJevTRR9XW1hY8vmfPnmOaFd+X1zY+Pl6TJk0KVuS8+uqrR309IFKdeOKJ+ulPf6pZs2appKSk1zY+n08DBgzo9uFs//79+v3vf3/E837az09cXJx++9vf6sorr9S8efO0evXqENwNvMLIx1F46qmntHPnTv3qV7/qdcW+sWPHatWqVVq7dq2+9KUvBVc+XbNmjdLS0pSUlKTc3FwNGTJE48aN08MPP6zVq1dr/PjxiouL04QJE3TJJZfovvvu0/nnn68rr7xSZ599thISEvTee+/phRde0Je//GVdcMEFVnF/UhwfV1RUpPPOO09XX321mpubNWXKFP3tb3/TddddpzPPPFNz5861f+GOYPz48Ro0aJA2bNigb33rW8Hj06dP1y9/+cvgv7u69dZb9bnPfU5Tp07V5ZdfrpEjR6qlpUVvvfWW/vu//zs4Z6U3119/vWbOnKnzzjtPV155pTo6OnTTTTcpNTVV//73v4/qHsaNGxeMq6SkRAkJCTr55JN133336fnnn9fMmTM1YsQIHThwIPgnuY/fE9Bf3HjjjZ/4/MyZM7VixQrNmTNH3//+97V7927dfPPNwSS+0+23327983PLLbcoLS1N8+fP1549e4Jl8Igw4Z3vGp2+8pWvmMTExGCZam8uueQSM2DAAFNfX2+MMWblypUmNzfXxMfHd6u++Pe//20uuugic9xxxxmfz9et2uLgwYPm5ptvNqeffrpJSkoyqamp5pRTTjGXXnqp2bZtW7BdTk6OmTlzZo8YCgoKeswuP1IcH692MeZwxcrVV19tcnJyTEJCgsnKyjKXX355jxJRm+sfyQUXXGAkmfvuuy94rK2tzQwcONDExcX1WpZaW1trvv3tb5sTTzzRJCQkmBNOOMHk5+ebpUuXdmujj1W7GGPMI488YsaNG2cSExPNiBEjzI033mh++MMfmkGDBnVrJ8ksWLCgx7VzcnJMSUlJt2OLFi0y2dnZJi4uLlhVtGnTJnPBBReYnJwc4/f7zZAhQ0xBQYF5/PHH+/S6AJGua7XLJ/l4tcudd95pTj75ZOP3+81JJ51kli1bZtauXdutaqwvPz8fL7Xt1FmBdu2114bsXhE6PmM+tqILEIMOHjyoM844QyeeeKI2bNgQ7nAAoF/jzy6ISd/5zndUVFSkrKws1dfX6/bbb9c//vEPqlAAwAGSD8SklpYW/eQnP9EHH3yghIQEnXXWWXryySeZhwEADvBnFwAA4BSltgAAwCmSDwAA4BTJBwAAcCriJpweOnRIO3fuVFpamnw+X7jDAWKSMUYtLS3Kzs5WXFx0fEah7wDCy6bfiLjkY+fOncE9PACEV11dnYYNGxbuMPqEvgOIDH3pNyIu+ejcMG3UFdcq3p/0Ka3/h8cfcozl+X2HLC9geX7beLzk87hWyvZeT7zpL94EEmPadVD/T096voFhKHXGuv3VkUpPjY7RGkSGC0aPC3cI/YJNvxFxyUfncGm8P4nk4whIPo5sgC/Bm0Bizf+8r9H054vOWNNT45SednS7SSM20W+EiEW/wccDAADgFMkHAABwKuL+7NKpPUkyffyri7VIG0m2jcf2Tx027b1+bSzPn/PzKm/iANCvnZd9erhDwCdg5AMAADhF8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMRW+3SFjiouOTIWCjIZ5mimQ7bVck8XqnLgu295n2n2ptAAPRrVKPENkY+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4FbHVLqNP2qUBA/19anvIct/1AXF2e95bn99neX7LDU9sz3+goN6qPQBQjQIvMfIBAACcsk4+/vWvf+k//uM/NGTIEKWkpOiMM87QK6+8EnzeGKOysjJlZ2crOTlZhYWFqqmpCWnQAKIPfQeATlbJR2Njo6ZMmaKEhAQ99dRTeuONN3TLLbfouOOOC7ZZvny5VqxYoVWrVqm6ulqBQEBFRUVqaWkJdewAogR9B4CurOZ8/OpXv9Lw4cN11113BY+NHDky+G9jjFauXKnFixdr9uzZkqR169YpMzNT69ev16WXXhqaqAFEFfoOAF1ZjXw8/vjjmjBhgi6++GINHTpUZ555pu64447g87W1taqvr1dxcXHwmN/vV0FBgaqqqno9Z2trq5qbm7s9APQv9B0AurIa+XjnnXe0evVqLVy4UNdcc41efvll/fCHP5Tf79c3v/lN1dcfrqrIzMzs9n2ZmZnavn17r+dctmyZlixZ0uP4njXDNCAhySY8HMmFIz07tWUhUMSJoG11rF9L29htzt9+8ID06GN2F/gELvsOhMYzO18PdwiIMs0tHRo0um9trUY+Dh06pLPOOkvl5eU688wzdemll+p73/ueVq9e3a2dz9e9lzPG9DjWadGiRWpqago+6urqbEICEAXoOwB0ZZV8ZGVlacyYMd2OnXrqqdqxY4ckKRAISFLwU0ynhoaGHp9oOvn9fqWnp3d7AOhf6DsAdGWVfEyZMkVvvvlmt2P//Oc/lZOTI0nKzc1VIBBQRUVF8Pm2tjZVVlYqPz8/BOECiEb0HQC6sprz8aMf/Uj5+fkqLy/XV7/6Vb388stas2aN1qxZI+nwkGlpaanKy8uVl5envLw8lZeXKyUlRXPmzPHkBgBEPvoOAF1ZJR8TJ07UI488okWLFun6669Xbm6uVq5cqW984xvBNldddZX279+v+fPnq7GxUZMmTdKGDRuUlpYW8uABRAf6DgBd+YwxETTfX2publZGRoZG/7hc8f4+VrtEecWF17ysSPG6WsQ29mE39F6WCTvt5qA26jE1NTVFzVyKzr6j8Z8nKT0tPtzhIIqwj01o2PQb7O0CAACcIvkAAABOkXwAAACnSD4AAIBTVtUuLiW2SPGtfWsb7Ut8e87i9fEd8i4MSTK26a7lhNaGKyzXhLD8vzP0Nia0Av2N10vJM6G1J0Y+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4FbHVLm3pUrzfo5N7vaC8ZQWFbbWO7ZLmNue3rnbxuNKI5dUBeI1qFPcY+QAAAE6RfAAAAKdIPgAAgFMkHwAAwCmSDwAA4FTEVrsk7ZbiE/vY2OvqFVuxtNeM1/dq+d7u/r7l3i62LO53yG+pvAGigdd7u9iKheobRj4AAIBTJB8AAMApkg8AAOAUyQcAAHCK5AMAADgVsdUuTXlGcUl9LHXwOoWKsL1grNnEb3uvlq/9qB9tsrwAAMRGBUgsYeQDAAA4RfIBAACcIvkAAABOkXwAAACnInbC6eAai+XVLZlYWv7clu1rYzlB9aN5k+1ObxmPzzKeSPq/MOguJuMCRxJpS6BHkmicjMvIBwAAcIrkAwAAOEXyAQAAnCL5AAAATpF8AAAApyK22qWpeK/iUzo8Obcvztv10j0uGPF0NfbhF2318OwA+qtorLhA+DDyAQAAnCL5AAAATpF8AAAAp0g+AACAUyQfAADAqYitdkn5c6riE5P61Nbr/Tmieb8QSVblMQ1X5HsXh+RtqY4DQ2+rCncIQERi75UjoxKoJ0Y+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4FbHVLh2Jkvx9bBxJm6McjQiK37ZSx7YSyGu28WfdQvUKADtUrxw7Rj4AAIBTVslHWVmZfD5ft0cgEAg+b4xRWVmZsrOzlZycrMLCQtXU1IQ8aADRhb4DQFfWIx+f/exntWvXruBj69b/3YJ9+fLlWrFihVatWqXq6moFAgEVFRWppaUlpEEDiD70HQA6WScfAwYMUCAQCD5OOOEESYc/uaxcuVKLFy/W7NmzNXbsWK1bt0779u3T+vXrQx44gOhC3wGgk3XysW3bNmVnZys3N1eXXHKJ3nnnHUlSbW2t6uvrVVxcHGzr9/tVUFCgqqojT+prbW1Vc3NztweA/oe+A0Anq2qXSZMm6Z577tHo0aP1/vvva+nSpcrPz1dNTY3q6+slSZmZmd2+JzMzU9u3bz/iOZctW6YlS5b0OJ6wV4o/2MfAvK4WsW3vdTxeVph4Xb0SYZVGu79vuZeNRfxDfkslTSeXfQfgNa/3sYmFahqrkY8ZM2bowgsv1Lhx4zR9+nT96U9/kiStW7cu2Mbn6947G2N6HOtq0aJFampqCj7q6upsQgIQBeg7AHR1TKW2AwcO1Lhx47Rt27bgzPXOTzGdGhoaenyi6crv9ys9Pb3bA0D/Rt8BxLZjSj5aW1v1j3/8Q1lZWcrNzVUgEFBFRUXw+ba2NlVWVio/3+Nt2gFEFfoOILZZzfn4yU9+olmzZmnEiBFqaGjQ0qVL1dzcrJKSEvl8PpWWlqq8vFx5eXnKy8tTeXm5UlJSNGfOHK/iBxAF6DsAdGWVfLz33nv6+te/rg8//FAnnHCCzjnnHG3evFk5OTmSpKuuukr79+/X/Pnz1djYqEmTJmnDhg1KS0uzDmzwhXUaMLBv66sPiDtkff5IcshyTfABPrv7bSvcZdUeCDWXfQdCIxYmPSJ8fMaYiNqdo7m5WRkZGSp84nKSjyMg+YDX2s1BbdRjampqipq5FJ19R+M/T1J6Wny4w4l6JB+wZdNvsLcLAABwiuQDAAA4RfIBAACcIvkAAABOWVW7uLRj03DFJyX1rbFtCuX1/FSvlxC3nSL8y1xPwjgqlq9Nzs9ZohwIB6+XEPcaE2YjGyMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithql46BRibJo5Xfva6OsT2/1wvce1l947MLflTpZo8CAdCfUb3SvzDyAQAAnCL5AAAATpF8AAAAp0g+AACAUyQfAADAqYitdvGUbXVJpFWv2PI0Hq83sgEA9DeMfAAAAKdIPgAAgFMkHwAAwCmSDwAA4BQTTvsi0uZURlI8lsurAwDAyAcAAHCK5AMAADhF8gEAAJwi+QAAAE6RfAAAAKcittrlu9OfU1Jq38I7ZOxyqDjfIav2B028VfsEX4en7Z8YM8iqPQCcl316uEMAghj5AAAATpF8AAAAp0g+AACAUyQfAADAKZIPAADgVMRWu/z+wemK9yf1rbHlXifGsr1lcYz38Vxj195qL5tI2jdG9q/NsBuqvAkEiHLP7Hw93CFELCqB3GPkAwAAOEXyAQAAnCL5AAAATpF8AAAAp0g+AACAUxFb7dJ++h6ZlHZvTm5T/SFZV4DE+WwvYMnD8w+78O+enRtA/0XFCGww8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMkHwAAwKmIrXaxYln84Yuz+wZju8GIJevaFct4fF5X3wAAYOGYRj6WLVsmn8+n0tLS4DFjjMrKypSdna3k5GQVFhaqpqbmWOME0E/QbwA46uSjurpaa9as0Wmnndbt+PLly7VixQqtWrVK1dXVCgQCKioqUktLyzEHCyC60W8AkI4y+dizZ4++8Y1v6I477tCgQYOCx40xWrlypRYvXqzZs2dr7NixWrdunfbt26f169eHLGgA0Yd+A0Cno0o+FixYoJkzZ2r69OndjtfW1qq+vl7FxcXBY36/XwUFBaqqqur1XK2trWpubu72AND/hLLfkOg7gGhmPeH0gQce0Kuvvqrq6uoez9XX10uSMjMzux3PzMzU9u3bez3fsmXLtGTJkp5PvJkqJSXZhtc3tvMvLVO0Q5ant43Hev6rxfm3L823PLkd29hHLj7yLx9Ej1D3G9In9B0Ii2d2vh7uELphuffIZvVrta6uTldeeaXuvfdeJX1CYuDzdf8NY4zpcazTokWL1NTUFHzU1dXZhAQgwnnRb0j0HUA0sxr5eOWVV9TQ0KDx48cHj3V0dOjFF1/UqlWr9Oabb0o6/EkmKysr2KahoaHHp5pOfr9ffr//aGIHEAW86Dck+g4gmlmNfJx77rnaunWrXnvtteBjwoQJ+sY3vqHXXntNJ510kgKBgCoqKoLf09bWpsrKSuXnezucDyAy0W8A+DirkY+0tDSNHTu227GBAwdqyJAhweOlpaUqLy9XXl6e8vLyVF5erpSUFM2ZMyd0UQOIGvQbAD4u5CucXnXVVdq/f7/mz5+vxsZGTZo0SRs2bFBaWlqoLwWgn6DfAGKLzxgTUWtvNzc3KyMjQ8NuW6K4ZK+qXSxLLiyXY7dfL92Oz7L6xuYd/oT5fb3K+07P6gVEv3ZzUBv1mJqampSenh7ucPqks+9o/OdJSk+LD3c4+BRUo/Q/Nv0GG8sBAACnSD4AAIBTJB8AAMApkg8AAOAUyQcAAHAq5KW2oRLYGK8BCX2csW6710mksa2Osbxfm+Ien2Usey45x+4bcETWe/ZYSrt/s7cXACxE2l4wODIvKpMY+QAAAE6RfAAAAKdIPgAAgFMkHwAAwCmSDwAA4FTEVrscGBKn+ERvciPrqgKvq2kiaXcd23u1jN22msaW1+/t0NuqLC8AINaxj01PjHwAAACnSD4AAIBTJB8AAMApkg8AAOBUxE44/WjcQcUl93F5dVu2sxIjbZZknF08PpvTW95r3re3WLUHAIlJmLGOkQ8AAOAUyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FbLVLyvYExfsT+tQ20opXbFkvCe4h29fmX4vyvQnkfxjL9HjYDSx/DkSDZ3a+Hu4Qgqi8cY+RDwAA4BTJBwAAcIrkAwAAOEXyAQAAnCL5AAAATkVstUu7XzJJfWzsdbWI7fltq2m8rr6x2tvF8ty2sVueP+fnVK8AsEcFS2Rj5AMAADhF8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMRW+1S8n+eV1KqN+HF+Q5ZtT9ku8GIJX/cQav2T4wZ5FEkAPorqj8QSRj5AAAATpF8AAAAp0g+AACAUyQfAADAqYidcPrS7lEacMDfp7aHjLfrq8f57NYQ9zqepMoOq/YHCuo9igRAtHhm5+tW7ZmgCi8x8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMkHwAAwKmIrXbZ9voIxSUleXNyu+IVybZ4Jc7yAoe8rY7Rity+t7UNxbISaFTpZssLAAgH2+oYr1F9078w8gEAAJyySj5Wr16t0047Tenp6UpPT9fkyZP11FNPBZ83xqisrEzZ2dlKTk5WYWGhampqQh40gOhC3wGgK6vkY9iwYbrxxhu1ZcsWbdmyRV/4whf05S9/OdhJLF++XCtWrNCqVatUXV2tQCCgoqIitbS0eBI8gOhA3wGgK6vkY9asWTr//PM1evRojR49WjfccINSU1O1efNmGWO0cuVKLV68WLNnz9bYsWO1bt067du3T+vXr/cqfgBRgL4DQFdHPeejo6NDDzzwgPbu3avJkyertrZW9fX1Ki4uDrbx+/0qKChQVVXVEc/T2tqq5ubmbg8A/Rd9BwDr5GPr1q1KTU2V3+/XZZddpkceeURjxoxRff3h/UMyMzO7tc/MzAw+15tly5YpIyMj+Bg+fLhtSIcrNGwecZYPWx0+u4fXbF4bY/vw2T0QsyKy7wAQFta/Wk8++WS99tpr2rx5sy6//HKVlJTojTfeCD7v83X/BWOM6XGsq0WLFqmpqSn4qKursw0JQBSg7wDQyXqdj8TERH3mM5+RJE2YMEHV1dW69dZbdfXVV0uS6uvrlZWVFWzf0NDQ4xNNV36/X35/33avBRC96DsAdDrmdT6MMWptbVVubq4CgYAqKiqCz7W1tamyslL5+fnHehkA/Qx9BxC7rEY+rrnmGs2YMUPDhw9XS0uLHnjgAW3cuFFPP/20fD6fSktLVV5erry8POXl5am8vFwpKSmaM2eOV/EDiAL0HQC6sko+3n//fc2dO1e7du1SRkaGTjvtND399NMqKiqSJF111VXav3+/5s+fr8bGRk2aNEkbNmxQWlqaJ8EDiA70HQC68hljbHc68VRzc7MyMjJ00uJyxXu1t4vXRRe25/d6r5lIeoctY8/5+ZFLLeGddnNQG/WYmpqalJ6eHu5w+qSz72j850lKT4sPdzgIM/aCcc+m32BvFwAA4BTJBwAAcIrkAwAAOEXyAQAAnCL5AAAATlmvcOrK8X8/pAEJh8IdhqTI25LEF0nVKx7be9E5Vu1j6bWxZfP/uP3gAenRx7wLBvDYMztfD3cIMae5pUODRvetLSMfAADAKZIPAADgFMkHAABwiuQDAAA4FbETTg8MjlN8Yh9zI6+XJ/ea18ul27SP9qXnvWYR/5DfRu/S8O3mYLhDAGJWtC4Nf7jfeKdPbRn5AAAATpF8AAAAp0g+AACAUyQfAADAKZIPAADgVMRWu+yZtkfxKe2enNvn8RrcxnI99vg4u2Xkbc9vc7/DLvy71bkBQIreCg2EByMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithqF/9fUhXvT/Lm5JG2t4sly2IX2RT37PpxvmfndsH2tcm6JXr3XwEiyTM7Xw93CM5Q2XPsGPkAAABOkXwAAACnSD4AAIBTJB8AAMApkg8AAOBUxFa7tJ3ToviUg31qa7vXiS3bvWBso/G6YMQmnuEXbfUsDgD9FxUgsMHIBwAAcIrkAwAAOEXyAQAAnCL5AAAATkXshNONE+9RelrfcqN4n90UT78vwap9nMc5GhO1ANii30A0Y+QDAAA4RfIBAACcIvkAAABOkXwAAACnSD4AAIBTEVvtcu7vFijen+TNyW3XM/d29XbpGrvmtqvJ26wO7+W5j+b8tobdUOXtBYAI8czO18MdQr9B5ZB7jHwAAACnSD4AAIBTJB8AAMApkg8AAOAUyQcAAHAqYqtdAgXvacBAf5/aJsZ3WJ07zrrcxU67scvp4ixLRmzjbyvcZdUeAKgAgZcY+QAAAE5ZJR/Lli3TxIkTlZaWpqFDh+orX/mK3nzzzW5tjDEqKytTdna2kpOTVVhYqJqampAGDSC60HcA6Moq+aisrNSCBQu0efNmVVRUqL29XcXFxdq7d2+wzfLly7VixQqtWrVK1dXVCgQCKioqUktLS8iDBxAd6DsAdOUzxhz1BIgPPvhAQ4cOVWVlpT7/+c/LGKPs7GyVlpbq6quvliS1trYqMzNTv/rVr3TppZd+6jmbm5uVkZGhz//3fOZ8HKk9cz7gsXZzUBv1mJqampSenh7y83vZdzT+8ySlp8WHPOZYw5wP2LLpN45pzkdTU5MkafDgwZKk2tpa1dfXq7i4ONjG7/eroKBAVVW9L3vd2tqq5ubmbg8A/Rt9BxDbjjr5MMZo4cKF+tznPqexY8dKkurr6yVJmZmZ3dpmZmYGn/u4ZcuWKSMjI/gYPnz44cB8ps8PW4fk8/Zh7B5ALPG67wAQ+Y46+bjiiiv0t7/9Tffff3+P53y+7r9QjTE9jnVatGiRmpqago+6urqjDQlAFKDvAHBU63z84Ac/0OOPP64XX3xRw4YNCx4PBAKSDn+KycrKCh5vaGjo8Ymmk9/vl9/ft7kdAKIbfQcAyXLkwxijK664Qg8//LCef/555ebmdns+NzdXgUBAFRUVwWNtbW2qrKxUfn5+aCIGEHXoOwB0ZTXysWDBAq1fv16PPfaY0tLSgn+LzcjIUHJysnw+n0pLS1VeXq68vDzl5eWpvLxcKSkpmjNnjic3ACDy0XcA6Moq+Vi9erUkqbCwsNvxu+66S/PmzZMkXXXVVdq/f7/mz5+vxsZGTZo0SRs2bFBaWlpIAgYQfeg7AHR1TOt8eKGzVv+ke65RfEqSJ9ewrZCxrUjxHUUFjtX5bdtbxDPswr9bnh39kdfrfHiBdT7Ci3VB4GydDwAAAFskHwAAwCmSDwAA4BTJBwAAcOqoFhlzoeNfA2WSvJlw6jnbGaFeT/m1iOftFZPtzh1nF/yo0s125wcQFZ7Z+bqn52dCa//CyAcAAHCK5AMAADhF8gEAAJwi+QAAAE6RfAAAAKcittpl1Gl1GjDQm+2yvV5e3fb8tuIsy2PaCnd5FAmA/orqEniJkQ8AAOAUyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FbLVL853DNCChj3u72O6lEmEsi2lkXUxzSY7lN3jH+l4PeRPH/17A4/N7yPa1TLuffXXQd17v1YLo4UXlEyMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithqlw8m+BSX1Mfp/LYVC7YVFLbnj/N2bxfrMgcvK0Ys09dRP9rkTRwA+jX2mulfGPkAAABOkXwAAACnSD4AAIBTJB8AAMCpiJ1w6v/Qp3i/RxNOPWZ8dgHZLpfu+XLsHnpvcb6n5x92Q5Wn5wcQHl4u985kVvcY+QAAAE6RfAAAAKdIPgAAgFMkHwAAwCmSDwAA4FTEVrtc8tXnlZSa0Ke28bIr54jzebneuPfxPDFmkFV7AKCiA5GEkQ8AAOAUyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FbLXLvY9/QfFJSX1rbLu3SwTtdSLJOn7fErv2NnvBmHjLWDx+LXN+zl4tQCh4uTdKJKK6J7Ix8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMkHwAAwKmIrXZpyzqouOQ+ll5EefWKNQ/v12eZjuZ9p9qbQAD0a1SjxDZGPgAAgFPWyceLL76oWbNmKTs7Wz6fT48++mi3540xKisrU3Z2tpKTk1VYWKiamppQxQsgCtFvAOjKOvnYu3evTj/9dK1atarX55cvX64VK1Zo1apVqq6uViAQUFFRkVpaWo45WADRiX4DQFfWcz5mzJihGTNm9PqcMUYrV67U4sWLNXv2bEnSunXrlJmZqfXr1+vSSy89tmgBRCX6DQBdhXTOR21trerr61VcXBw85vf7VVBQoKqq3pfJbm1tVXNzc7cHgNhxNP2GRN8BRLOQVrvU19dLkjIzM7sdz8zM1Pbt23v9nmXLlmnJkp6blcS3DFDcwT6G5/XeLpFWveLl/VrG8vb/nWzVftSPNtldAP3e0fQb0pH7DkQH271mqI7pXzypdvH5uv92NMb0ONZp0aJFampqCj7q6uq8CAlAhLPpNyT6DiCahXTkIxAISDr8SSYrKyt4vKGhocenmk5+v19+vz+UYQCIIkfTb0j0HUA0C+nIR25urgKBgCoqKoLH2traVFlZqfz8/FBeCkA/Qb8BxB7rkY89e/borbfeCn5dW1ur1157TYMHD9aIESNUWlqq8vJy5eXlKS8vT+Xl5UpJSdGcOXNCGjiA6EG/AaAr6+Rjy5YtmjZtWvDrhQsXSpJKSkp0991366qrrtL+/fs1f/58NTY2atKkSdqwYYPS0tKsrtMxsEMmuaNvjX3erq9uu+S4sQ0nkiacWp579HdZXh2fzlW/gejBBNLY5jPG+lelp5qbm5WRkaFh/3m94pKT+vZNJB+hOz/JByS1m4PaqMfU1NSk9PT0cIfTJ519R+M/T1J6Wh/3hULYkHz0Pzb9Bnu7AAAAp0g+AACAUyQfAADAKZIPAADgVEgXGQul7OfiNCAhOnMjYzlp0+P5sp7ae9E54Q6hG9vX0vq9OmRxgU9YnTMcbO61/eAB6dHHvAsGMc92eXVEvuaWDg0a3be20fnbHQAARC2SDwAA4BTJBwAAcIrkAwAAOEXyAQAAnIrYapcPL9yn+JRDnpzb53F5SZzl+Q/ZllxYsoln2IV/9zASRIt2czDcISDKsFw6Dvcb7/SpLSMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithqF72ZKiUl9a2tZbGIdXGJZXFMh20xjdfxWJx/+y/z7U5uy/Jec35e5U0cAEIq0vZqofomsjHyAQAAnCL5AAAATpF8AAAAp0g+AACAUxE74bRjoJFJ6uPMSm9XJ/d8QmhEsb1Xy6XkR5VutrwAADCBtL9h5AMAADhF8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMRW+2Sut2n+MS+lV5YFlzYL69uyfN4PKy+sY49zi6Yhis8Xr7dluVrOfQ2lnsHwiHSlm+3QaVOT4x8AAAAp0g+AACAUyQfAADAKZIPAADgFMkHAABwKmKrXfZP2aP4lHZPzh1nW9JhyeutXYxleYzN/Q6/aKttOABARQesMPIBAACcIvkAAABOkXwAAACnSD4AAIBTJB8AAMCpiK12sWJbXmK5n4fX1SvWW7tYVuvYtgcAwEuMfAAAAKdIPgAAgFMkHwAAwCmSDwAA4BTJBwAAcCpiq11GDG7UgIH+PrUdEHfI6tyHLPdGsWW7d0ycZT2N7fkPFNRbtQcA9mqBlzwb+fjNb36j3NxcJSUlafz48XrppZe8uhSAfoJ+A4gNniQfDz74oEpLS7V48WL99a9/1dSpUzVjxgzt2LHDi8sB6AfoN4DY4UnysWLFCn3nO9/Rd7/7XZ166qlauXKlhg8frtWrV3txOQD9AP0GEDtCPuejra1Nr7zyin72s591O15cXKyqqqoe7VtbW9Xa2hr8uqmpSZLUvq+t7xeN8jkfttHYnr/dHLS8AmJduw7/nzHGzeq4tv2GdOS+o3mPXX+A3tFvwJZNvxHy5OPDDz9UR0eHMjMzux3PzMxUfX3PiY/Lli3TkiVLehz/f19dG+rQAFhqaWlRRkaG59ex7TekI/cdOWe960WIMeidcAeAKNWXfsOzahefr/vneWNMj2OStGjRIi1cuDD49UcffaScnBzt2LHDSacXbs3NzRo+fLjq6uqUnp4e7nA8xb1GD2OMWlpalJ2d7fS6fe03pNjuO6L9/5etWLrfaL5Xm34j5MnH8ccfr/j4+B6fVhoaGnp8qpEkv98vv79nSW1GRkbUvfDHIj09PWbul3uNDi5/gdv2GxJ9hxTd/7+ORizdb7Tea1/7jZBPOE1MTNT48eNVUVHR7XhFRYXy8/NDfTkA/QD9BhBbPPmzy8KFCzV37lxNmDBBkydP1po1a7Rjxw5ddtllXlwOQD9AvwHEDk+Sj6997WvavXu3rr/+eu3atUtjx47Vk08+qZycnE/9Xr/fr+uuu67X4dT+KJbul3vFJzmWfkOKrdc8lu5Viq37jZV79RlXtXQAAABiYzkAAOAYyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FXPLxm9/8Rrm5uUpKStL48eP10ksvhTukkCsrK5PP5+v2CAQC4Q4rZF588UXNmjVL2dnZ8vl8evTRR7s9b4xRWVmZsrOzlZycrMLCQtXU1IQn2GP0afc6b968Hu/1OeecE55g+7FY6Dek/t13xFK/IdF3RFTy8eCDD6q0tFSLFy/WX//6V02dOlUzZszQjh07wh1ayH32s5/Vrl27go+tW7eGO6SQ2bt3r04//XStWrWq1+eXL1+uFStWaNWqVaqurlYgEFBRUZFaWlocR3rsPu1eJemLX/xit/f6ySefdBhh/xdL/YbUf/uOWOo3JPoOmQhy9tlnm8suu6zbsVNOOcX87Gc/C1NE3rjuuuvM6aefHu4wnJBkHnnkkeDXhw4dMoFAwNx4443BYwcOHDAZGRnm9ttvD0OEofPxezXGmJKSEvPlL385LPHEiljpN4yJnb4jlvoNY2Kz74iYkY+2tja98sorKi4u7na8uLhYVVVVYYrKO9u2bVN2drZyc3N1ySWX6J13YmP76traWtXX13d7n/1+vwoKCvrl+yxJGzdu1NChQzV69Gh973vfU0NDQ7hD6jdird+QYrPviMV+Q+rffUfEJB8ffvihOjo6euxgmZmZ2WOny2g3adIk3XPPPXrmmWd0xx13qL6+Xvn5+dq9e3e4Q/Nc53sZC++zJM2YMUP33Xefnn/+ed1yyy2qrq7WF77wBbW2toY7tH4hlvoNKXb7jljrN6T+33d4srfLsfD5fN2+Nsb0OBbtZsyYEfz3uHHjNHnyZI0aNUrr1q3TwoULwxiZO7HwPkuH9yvpNHbsWE2YMEE5OTn605/+pNmzZ4cxsv4lVv4/xXrfESvvs9T/+46IGfk4/vjjFR8f3yOLbWho6JHt9jcDBw7UuHHjtG3btnCH4rnOmfmx+D5LUlZWlnJycmLivXYhlvsNKXb6jljvN6T+13dETPKRmJio8ePHq6KiotvxiooK5efnhykqN1pbW/WPf/xDWVlZ4Q7Fc7m5uQoEAt3e57a2NlVWVvb791mSdu/erbq6uph4r12I5X5Dip2+I9b7Dan/9R0R9WeXhQsXau7cuZowYYImT56sNWvWaMeOHbrsssvCHVpI/eQnP9GsWbM0YsQINTQ0aOnSpWpublZJSUm4QwuJPXv26K233gp+XVtbq9dee02DBw/WiBEjVFpaqvLycuXl5SkvL0/l5eVKSUnRnDlzwhj10fmkex08eLDKysp04YUXKisrS++++66uueYaHX/88brgggvCGHX/Eiv9htS/+45Y6jck+o6IKrU1xphf//rXJicnxyQmJpqzzjrLVFZWhjukkPva175msrKyTEJCgsnOzjazZ882NTU14Q4rZF544QUjqcejpKTEGHO4bO66664zgUDA+P1+8/nPf95s3bo1vEEfpU+613379pni4mJzwgknmISEBDNixAhTUlJiduzYEe6w+51Y6DeM6d99Ryz1G8bQd/iMMcZtugMAAGJZxMz5AAAAsYHkAwAAOEXyAQAAnCL5AAAATpF8AAAAp0g+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACc+v/aa9Q0LfNgawAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"cell_type":"code","source":"attention_weights.shape","metadata":{"id":"ZuzrCdmYlTcJ","outputId":"8dfa3d00-cd0c-48e8-8718-af89ed82d106","execution":{"iopub.status.busy":"2023-03-16T11:46:38.766676Z","iopub.execute_input":"2023-03-16T11:46:38.767075Z","iopub.status.idle":"2023-03-16T11:46:38.773967Z","shell.execute_reply.started":"2023-03-16T11:46:38.767036Z","shell.execute_reply":"2023-03-16T11:46:38.772787Z"},"trusted":true},"execution_count":52,"outputs":[{"execution_count":52,"output_type":"execute_result","data":{"text/plain":"TensorShape([64, 2, 19])"},"metadata":{}}]},{"cell_type":"code","source":"attention_slice = attention_weights[0, 0].numpy()\nattention_slice = attention_slice[attention_slice != 0]","metadata":{"id":"qIMwC-f-ZC8N","execution":{"iopub.status.busy":"2023-03-16T11:46:38.775961Z","iopub.execute_input":"2023-03-16T11:46:38.776857Z","iopub.status.idle":"2023-03-16T11:46:38.786165Z","shell.execute_reply.started":"2023-03-16T11:46:38.776814Z","shell.execute_reply":"2023-03-16T11:46:38.785038Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"code","source":"#@title\nplt.suptitle('Attention weights for one sequence')\n\nplt.figure(figsize=(12, 6))\na1 = plt.subplot(1, 2, 1)\nplt.bar(range(len(attention_slice)), attention_slice)\n# freeze the xlim\nplt.xlim(plt.xlim())\nplt.xlabel('Attention weights')\n\na2 = plt.subplot(1, 2, 2)\nplt.bar(range(len(attention_slice)), attention_slice)\nplt.xlabel('Attention weights, zoomed')\n\n# zoom in\ntop = max(a1.get_ylim())\nzoom = 0.85*top\na2.set_ylim([0.90*top, top])\na1.plot(a1.get_xlim(), [zoom, zoom], color='k')","metadata":{"id":"ysWDPO6hOS8X","outputId":"9f941203-0962-4219-98cb-efdfe46a5faa","execution":{"iopub.status.busy":"2023-03-16T11:46:38.788275Z","iopub.execute_input":"2023-03-16T11:46:38.788715Z","iopub.status.idle":"2023-03-16T11:46:39.182116Z","shell.execute_reply.started":"2023-03-16T11:46:38.788670Z","shell.execute_reply":"2023-03-16T11:46:39.181038Z"},"trusted":true},"execution_count":54,"outputs":[{"execution_count":54,"output_type":"execute_result","data":{"text/plain":"[<matplotlib.lines.Line2D at 0x7fbd8d40d710>]"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 0 Axes>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 1200x600 with 2 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAA94AAAINCAYAAADIsKceAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSMElEQVR4nO3df1zUZb7//+cIwpgp/oBQCwGtVYxSd2gLjH5shcfKtK1dSsV2g1oWN0WOe5KwNekHW5mL/QA/kMa6pbK3Y51qY0vqpGm4WyJTntav9kPE4w43xC1R24WE6/uHN+c0O2AM8nZgeNxvt/ftMNf7el/v10WevXx6zbzHZowxAgAAAAAAlujn7wIAAAAAAAhkBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACwU7O8CuktbW5v+9re/adCgQbLZbP4uBwDQxxljdPToUY0aNUr9+vHv3N2BtR4A0NN0dr0PmOD9t7/9TVFRUf4uAwAADwcOHNAFF1zg7zICAms9AKCn+q71PmCC96BBgySdnPDgwYP9XA0AoK9rampSVFSUe33CmWOtBwD0NJ1d7wMmeJ96y9ngwYNZjAEAPQZvie4+rPUAgJ7qu9Z7PnQGAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFgv1dAAAgcMUsfsPfJfis9jc3+bsEAAAQYNjxBgAAAADAQux4A4AfsSMMAAAQ+NjxBgAAAADAQgRvAAAAAAAsxFvN+6je+PZWibe49jW98c8pf0YBAADwrwjeQC/VG0OpRDAFAABA30PwRsDqjcGUUAoAAAAEHoJ3BwhtAAAAAIDu0KWHqxUVFSk2NlZ2u10Oh0Nbt27tsK/L5dKsWbM0btw49evXT9nZ2acde8OGDbLZbJo5c2ZXSgMAAAAAoEfxOXiXl5crOztbeXl5qqmpUXJysqZNm6a6urp2+zc3NysiIkJ5eXmaOHHiacfev3+/Fi1apOTkZF/LAgAAAACgR/I5eK9YsULp6enKyMhQXFycCgsLFRUVpeLi4nb7x8TEaOXKlZo7d67CwsI6HLe1tVWzZ8/WsmXLNGbMGF/LAgAAAACgR/IpeLe0tKi6ulopKSke7SkpKaqqqjqjQvLz8xUREaH09PRO9W9ublZTU5PHAQAAAABAT+NT8G5sbFRra6siIyM92iMjI1VfX9/lIt5//32tXr1apaWlnb6moKBAYWFh7iMqKqrL9wcAAAAAwCpderiazWbzeG2M8WrrrKNHj2rOnDkqLS1VeHh4p6/Lzc3VkSNH3MeBAwe6dH8AAAAAAKzk09eJhYeHKygoyGt3u6GhwWsXvLM+//xz1dbWavr06e62tra2k8UFB2vPnj0aO3as13WhoaEKDQ3t0j0BAAAAADhbfNrxDgkJkcPhUGVlpUd7ZWWlkpKSulTA+PHjtWvXLjmdTvdxyy236Nprr5XT6eQt5AAAAACAXs2nHW9JysnJUVpamhISEpSYmKiSkhLV1dUpMzNT0sm3gB88eFBr1651X+N0OiVJx44d06FDh+R0OhUSEqIJEybIbrcrPj7e4x5DhgyRJK92AAAAAAB6G5+Dd2pqqg4fPqz8/Hy5XC7Fx8eroqJC0dHRkiSXy+X1nd6TJ092/1xdXa1169YpOjpatbW1Z1Y9AAAAAAA9nM/BW5KysrKUlZXV7rmysjKvNmOMT+O3NwYAAAAAAL1Rl55qDgAAAAAAOofgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AQB9TVFSk2NhY2e12ORwObd26tcO+LpdLs2bN0rhx49SvXz9lZ2d79SkrK5PNZvM6/vnPf3b5vgAABBKCNwAAfUh5ebmys7OVl5enmpoaJScna9q0aaqrq2u3f3NzsyIiIpSXl6eJEyd2OO7gwYPlcrk8Drvd3uX7AgAQSAjeAAD0IStWrFB6eroyMjIUFxenwsJCRUVFqbi4uN3+MTExWrlypebOnauwsLAOx7XZbBoxYoTHcSb3BQAgkBC8AQDoI1paWlRdXa2UlBSP9pSUFFVVVZ3R2MeOHVN0dLQuuOAC3XzzzaqpqTnj+zY3N6upqcnjAACgNyJ4AwDQRzQ2Nqq1tVWRkZEe7ZGRkaqvr+/yuOPHj1dZWZlee+01rV+/Xna7XVOmTNGnn356RvctKChQWFiY+4iKiupyjQAA+BPBGwCAPsZms3m8NsZ4tfniiiuu0Jw5czRx4kQlJyfrD3/4g773ve/pmWeeOaP75ubm6siRI+7jwIEDXa4RAAB/CvZ3AQAA4OwIDw9XUFCQ1y5zQ0OD1270mejXr58uu+wy9453V+8bGhqq0NDQbqsLAAB/YccbAIA+IiQkRA6HQ5WVlR7tlZWVSkpK6rb7GGPkdDo1cuTIs3pfAAB6Kna8AQDoQ3JycpSWlqaEhAQlJiaqpKREdXV1yszMlHTy7d0HDx7U2rVr3dc4nU5JJx+gdujQITmdToWEhGjChAmSpGXLlumKK67QRRddpKamJj399NNyOp167rnnOn1fAAACGcEbAIA+JDU1VYcPH1Z+fr5cLpfi4+NVUVGh6OhoSZLL5fL6bu3Jkye7f66urta6desUHR2t2tpaSdJXX32le++9V/X19QoLC9PkyZP13nvv6Qc/+EGn7wsAQCAjeAMA0MdkZWUpKyur3XNlZWVebcaY047329/+Vr/97W/P6L4AAAQyPuMNAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIWC/V1Adzt+/LiCgoLOeJy2ln92QzVn1/HjxzvdtzfOTwr8OQb6/KTAn6Mv85MCf46BPr+zMQ4AAOj9bMYY4+8iukNTU5PCwsL8XQYAAB6OHDmiwYMH+7uMgHBqred3CgDoKTq7NvFWcwAAAAAALBRwbzX/29/+1i3/Ch734JvdUM3Ztfvhf+t03944Pynw5xjo85MCf46+zE8K/DkG+vxOp6mpSaNGjeqWsQAAQO8WcMF74MCBGjhw4BmP0y/E3g3VnF2+zLs3zk8K/DkG+vykwJ+jr//7E+hzDPT5nU5ra2u3jAMAAHo/3moOAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFioS8G7qKhIsbGxstvtcjgc2rp1a4d9XS6XZs2apXHjxqlfv37Kzs726lNaWqrk5GQNHTpUQ4cO1fXXX68PPvigK6UBAAAAANCj+By8y8vLlZ2drby8PNXU1Cg5OVnTpk1TXV1du/2bm5sVERGhvLw8TZw4sd0+mzdv1p133ql3331X27dv1+jRo5WSkqKDBw/6Wh4AAAAAAD2Kz8F7xYoVSk9PV0ZGhuLi4lRYWKioqCgVFxe32z8mJkYrV67U3LlzFRYW1m6fl156SVlZWZo0aZLGjx+v0tJStbW16Z133vG1PAAAAAAAehSfgndLS4uqq6uVkpLi0Z6SkqKqqqpuK+rrr7/WN998o2HDhnXYp7m5WU1NTR4HAAAAAAA9jU/Bu7GxUa2trYqMjPRoj4yMVH19fbcVtXjxYp1//vm6/vrrO+xTUFCgsLAw9xEVFdVt9wcAAAAAoLt06eFqNpvN47Uxxqutq5544gmtX79eL7/8sux2e4f9cnNzdeTIEfdx4MCBbrk/AAAAAADdKdiXzuHh4QoKCvLa3W5oaPDaBe+K5cuX67HHHtPbb7+tSy+99LR9Q0NDFRoaesb3BAAAAADASj7teIeEhMjhcKiystKjvbKyUklJSWdUyJNPPqmHH35Yb775phISEs5oLAAAAAAAegqfdrwlKScnR2lpaUpISFBiYqJKSkpUV1enzMxMSSffAn7w4EGtXbvWfY3T6ZQkHTt2TIcOHZLT6VRISIgmTJgg6eTbyx988EGtW7dOMTEx7h31c889V+eee+6ZzhEAAAAAAL/xOXinpqbq8OHDys/Pl8vlUnx8vCoqKhQdHS1JcrlcXt/pPXnyZPfP1dXVWrdunaKjo1VbWytJKioqUktLi26//XaP65YuXaqHHnrI1xIBAAAAAOgxfA7ekpSVlaWsrKx2z5WVlXm1GWNOO96pAA4AAAAAQKDp0lPNAQAAAABA5xC8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEA6GOKiooUGxsru90uh8OhrVu3dtjX5XJp1qxZGjdunPr166fs7OzTjr1hwwbZbDbNnDnTo/3EiRNasmSJYmNjNWDAAI0ZM0b5+flqa2vrhhkBANCzEbwBAOhDysvLlZ2drby8PNXU1Cg5OVnTpk1TXV1du/2bm5sVERGhvLw8TZw48bRj79+/X4sWLVJycrLXuccff1yrVq3Ss88+q927d+uJJ57Qk08+qWeeeaZb5gUAQE9G8AYAoA9ZsWKF0tPTlZGRobi4OBUWFioqKkrFxcXt9o+JidHKlSs1d+5chYWFdThua2urZs+erWXLlmnMmDFe57dv364ZM2bopptuUkxMjG6//XalpKRox44d3TY3AAB6KoI3AAB9REtLi6qrq5WSkuLRnpKSoqqqqjMaOz8/XxEREUpPT2/3/JVXXql33nlHe/fulSR99NFH2rZtm2688cYOx2xublZTU5PHAQBAbxTs7wIAAMDZ0djYqNbWVkVGRnq0R0ZGqr6+vsvjvv/++1q9erWcTmeHfe6//34dOXJE48ePV1BQkFpbW/Xoo4/qzjvv7PCagoICLVu2rMt1AQDQU7DjDQBAH2Oz2TxeG2O82jrr6NGjmjNnjkpLSxUeHt5hv/Lycr344otat26ddu7cqd/97ndavny5fve733V4TW5uro4cOeI+Dhw40KUaAQDwN3a8AQDoI8LDwxUUFOS1u93Q0OC1C95Zn3/+uWprazV9+nR326knlQcHB2vPnj0aO3asfvWrX2nx4sW64447JEmXXHKJ9u/fr4KCAt11113tjh0aGqrQ0NAu1QUAQE/CjjcAAH1ESEiIHA6HKisrPdorKyuVlJTUpTHHjx+vXbt2yel0uo9bbrlF1157rZxOp6KioiRJX3/9tfr18/xrR1BQEF8nBgDoE9jxBgCgD8nJyVFaWpoSEhKUmJiokpIS1dXVKTMzU9LJt3cfPHhQa9eudV9z6rPbx44d06FDh+R0OhUSEqIJEybIbrcrPj7e4x5DhgyRJI/26dOn69FHH9Xo0aN18cUXq6amRitWrNDdd99t7YQBAOgBCN4AAPQhqampOnz4sPLz8+VyuRQfH6+KigpFR0dLklwul9d3ek+ePNn9c3V1tdatW6fo6GjV1tZ2+r7PPPOMHnzwQWVlZamhoUGjRo3Sz3/+c/3617/ulnkBANCTEbwBAOhjsrKylJWV1e65srIyrzZjjE/jtzfGoEGDVFhYqMLCQp/GAgAgEPAZbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCXQreRUVFio2Nld1ul8Ph0NatWzvs63K5NGvWLI0bN079+vVTdnZ2u/02btyoCRMmKDQ0VBMmTNArr7zSldIAAAAAAOhRfA7e5eXlys7OVl5enmpqapScnKxp06aprq6u3f7Nzc2KiIhQXl6eJk6c2G6f7du3KzU1VWlpafroo4+Ulpamn/zkJ/rLX/7ia3kAAAAAAPQoPgfvFStWKD09XRkZGYqLi1NhYaGioqJUXFzcbv+YmBitXLlSc+fOVVhYWLt9CgsLdcMNNyg3N1fjx49Xbm6urrvuOhUWFvpaHgAAAAAAPYpPwbulpUXV1dVKSUnxaE9JSVFVVVWXi9i+fbvXmFOnTj3tmM3NzWpqavI4AAAAAADoaXwK3o2NjWptbVVkZKRHe2RkpOrr67tcRH19vc9jFhQUKCwszH1ERUV1+f4AAAAAAFilSw9Xs9lsHq+NMV5tVo+Zm5urI0eOuI8DBw6c0f0BAAAAALBCsC+dw8PDFRQU5LUT3dDQ4LVj7YsRI0b4PGZoaKhCQ0O7fE8AAAAAAM4Gn3a8Q0JC5HA4VFlZ6dFeWVmppKSkLheRmJjoNeamTZvOaEwAAAAAAHoCn3a8JSknJ0dpaWlKSEhQYmKiSkpKVFdXp8zMTEkn3wJ+8OBBrV271n2N0+mUJB07dkyHDh2S0+lUSEiIJkyYIElasGCBrrrqKj3++OOaMWOGXn31Vb399tvatm1bN0wRAAAAAAD/8Tl4p6am6vDhw8rPz5fL5VJ8fLwqKioUHR0tSXK5XF7f6T158mT3z9XV1Vq3bp2io6NVW1srSUpKStKGDRu0ZMkSPfjggxo7dqzKy8t1+eWXn8HUAAAAAADwP5+DtyRlZWUpKyur3XNlZWVebcaY7xzz9ttv1+23396VcgAAAAAA6LG69FRzAAAAAADQOQRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAA+piioiLFxsbKbrfL4XBo69atHfZ1uVyaNWuWxo0bp379+ik7O/u0Y2/YsEE2m00zZ870Onfw4EHNmTNHw4cP1znnnKNJkyapurr6DGcDAEDPR/AGAKAPKS8vV3Z2tvLy8lRTU6Pk5GRNmzZNdXV17fZvbm5WRESE8vLyNHHixNOOvX//fi1atEjJycle57788ktNmTJF/fv315/+9Cf99a9/1VNPPaUhQ4Z0x7QAAOjRuvQ93gAAoHdasWKF0tPTlZGRIUkqLCzUW2+9peLiYhUUFHj1j4mJ0cqVKyVJa9as6XDc1tZWzZ49W8uWLdPWrVv11VdfeZx//PHHFRUVpRdeeMFjbAAA+gJ2vAEA6CNaWlpUXV2tlJQUj/aUlBRVVVWd0dj5+fmKiIhQenp6u+dfe+01JSQk6Mc//rHOO+88TZ48WaWlpacds7m5WU1NTR4HAAC9EcEbAIA+orGxUa2trYqMjPRoj4yMVH19fZfHff/997V69erTBukvvvhCxcXFuuiii/TWW28pMzNT8+fP19q1azu8pqCgQGFhYe4jKiqqyzUCAOBPBG8AAPoYm83m8doY49XWWUePHtWcOXNUWlqq8PDwDvu1tbXp+9//vh577DFNnjxZP//5z3XPPfeouLi4w2tyc3N15MgR93HgwIEu1QgAgL/xGW8AAPqI8PBwBQUFee1uNzQ0eO2Cd9bnn3+u2tpaTZ8+3d3W1tYmSQoODtaePXs0duxYjRw5UhMmTPC4Ni4uThs3buxw7NDQUIWGhnapLgAAehJ2vAEA6CNCQkLkcDhUWVnp0V5ZWamkpKQujTl+/Hjt2rVLTqfTfdxyyy269tpr5XQ63W8PnzJlivbs2eNx7d69exUdHd21yQAA0Iuw4w0AQB+Sk5OjtLQ0JSQkKDExUSUlJaqrq1NmZqakk2/vPnjwoMdnr51OpyTp2LFjOnTokJxOp0JCQjRhwgTZ7XbFx8d73OPUV4R9u33hwoVKSkrSY489pp/85Cf64IMPVFJSopKSEmsnDABAD0DwBgCgD0lNTdXhw4eVn58vl8ul+Ph4VVRUuHeeXS6X13d6T5482f1zdXW11q1bp+joaNXW1nb6vpdddpleeeUV5ebmKj8/X7GxsSosLNTs2bO7ZV4AAPRkBG8AAPqYrKwsZWVltXuurKzMq80Y49P47Y0hSTfffLNuvvlmn8YCACAQ8BlvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEJdCt5FRUWKjY2V3W6Xw+HQ1q1bT9t/y5YtcjgcstvtGjNmjFatWuXVp7CwUOPGjdOAAQMUFRWlhQsX6p///GdXygMAAAAAoMfwOXiXl5crOztbeXl5qqmpUXJysqZNm6a6urp2++/bt0833nijkpOTVVNTowceeEDz58/Xxo0b3X1eeuklLV68WEuXLtXu3bu1evVqlZeXKzc3t+szAwAAAACgBwj29YIVK1YoPT1dGRkZkk7uVL/11lsqLi5WQUGBV/9Vq1Zp9OjRKiwslCTFxcVpx44dWr58uW677TZJ0vbt2zVlyhTNmjVLkhQTE6M777xTH3zwQVfnBQAAAABAj+DTjndLS4uqq6uVkpLi0Z6SkqKqqqp2r9m+fbtX/6lTp2rHjh365ptvJElXXnmlqqur3UH7iy++UEVFhW666aYOa2lublZTU5PHAQAAAABAT+PTjndjY6NaW1sVGRnp0R4ZGan6+vp2r6mvr2+3/4kTJ9TY2KiRI0fqjjvu0KFDh3TllVfKGKMTJ07oF7/4hRYvXtxhLQUFBVq2bJkv5QMAAAAAcNZ16eFqNpvN47Uxxqvtu/p/u33z5s169NFHVVRUpJ07d+rll1/WH//4Rz388MMdjpmbm6sjR464jwMHDnRlKgAAAAAAWMqnHe/w8HAFBQV57W43NDR47WqfMmLEiHb7BwcHa/jw4ZKkBx98UGlpae7PjV9yySU6fvy47r33XuXl5alfP+9/HwgNDVVoaKgv5QMAAAAAulHM4jf8XUKX1P6m4481W8GnHe+QkBA5HA5VVlZ6tFdWViopKandaxITE736b9q0SQkJCerfv78k6euvv/YK10FBQTLGuHfHAQAAAADojXx+q3lOTo6ef/55rVmzRrt379bChQtVV1enzMxMSSffAj537lx3/8zMTO3fv185OTnavXu31qxZo9WrV2vRokXuPtOnT1dxcbE2bNigffv2qbKyUg8++KBuueUWBQUFdcM0AQAAAADwD5+/Tiw1NVWHDx9Wfn6+XC6X4uPjVVFRoejoaEmSy+Xy+E7v2NhYVVRUaOHChXruuec0atQoPf300+6vEpOkJUuWyGazacmSJTp48KAiIiI0ffp0Pfroo90wRQAAAAAA/Mfn4C1JWVlZysrKavdcWVmZV9vVV1+tnTt3dlxEcLCWLl2qpUuXdqUcAAAAAAB6rC491RwAAAAAAHQOwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQDoY4qKihQbGyu73S6Hw6GtW7d22NflcmnWrFkaN26c+vXrp+zs7NOOvWHDBtlsNs2cObPDPgUFBbLZbN85FgAAgYLgDQBAH1JeXq7s7Gzl5eWppqZGycnJmjZtmsdXgX5bc3OzIiIilJeXp4kTJ5527P3792vRokVKTk7usM+HH36okpISXXrppWc0DwAAehOCNwAAfciKFSuUnp6ujIwMxcXFqbCwUFFRUSouLm63f0xMjFauXKm5c+cqLCysw3FbW1s1e/ZsLVu2TGPGjGm3z7FjxzR79myVlpZq6NCh3TIfAAB6A4I3AAB9REtLi6qrq5WSkuLRnpKSoqqqqjMaOz8/XxEREUpPT++wz7x583TTTTfp+uuvP6N7AQDQ2wT7uwAAAHB2NDY2qrW1VZGRkR7tkZGRqq+v7/K477//vlavXi2n09lhnw0bNmjnzp368MMPOz1uc3Ozmpub3a+bmpq6XCMAAP7EjjcAAH2MzWbzeG2M8WrrrKNHj2rOnDkqLS1VeHh4u30OHDigBQsW6MUXX5Tdbu/02AUFBQoLC3MfUVFRXaoRAAB/Y8cbAIA+Ijw8XEFBQV672w0NDV674J31+eefq7a2VtOnT3e3tbW1SZKCg4O1Z88e7dq1Sw0NDXI4HO4+ra2teu+99/Tss8+qublZQUFBXmPn5uYqJyfH/bqpqYnwDQDolQjeAAD0ESEhIXI4HKqsrNStt97qbq+srNSMGTO6NOb48eO1a9cuj7YlS5bo6NGjWrlypaKionTeeed59fnZz36m8ePH6/777283dEtSaGioQkNDu1QXAAA9CcEbAIA+JCcnR2lpaUpISFBiYqJKSkpUV1enzMxMSSd3mQ8ePKi1a9e6rzn12e1jx47p0KFDcjqdCgkJ0YQJE2S32xUfH+9xjyFDhkiSuz0kJMSrz8CBAzV8+HCvdgAAAhHBGwCAPiQ1NVWHDx9Wfn6+XC6X4uPjVVFRoejoaEmSy+Xy+k7vyZMnu3+urq7WunXrFB0drdra2rNZOgAAvRbBGwCAPiYrK0tZWVntnisrK/NqM8b4NH57Y/yrzZs3+zQmAAC9GcEbAAAAACwSs/gNf5fgs9rf3OTvEgIOXycGAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhYL9XQAAAACsEbP4DX+X4LPa39zk7xJwFvFnFH0FO94AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhboUvIuKihQbGyu73S6Hw6GtW7eetv+WLVvkcDhkt9s1ZswYrVq1yqvPV199pXnz5mnkyJGy2+2Ki4tTRUVFV8oDAAAAAKDH8Dl4l5eXKzs7W3l5eaqpqVFycrKmTZumurq6dvvv27dPN954o5KTk1VTU6MHHnhA8+fP18aNG919WlpadMMNN6i2tlb/+Z//qT179qi0tFTnn39+12cGAAAAAEAP4PPXia1YsULp6enKyMiQJBUWFuqtt95ScXGxCgoKvPqvWrVKo0ePVmFhoSQpLi5OO3bs0PLly3XbbbdJktasWaO///3vqqqqUv/+/SVJ0dHRXZ0TAAAAAAA9hk873i0tLaqurlZKSopHe0pKiqqqqtq9Zvv27V79p06dqh07duibb76RJL322mtKTEzUvHnzFBkZqfj4eD322GNqbW3tsJbm5mY1NTV5HAAAAAAA9DQ+Be/Gxka1trYqMjLSoz0yMlL19fXtXlNfX99u/xMnTqixsVGS9MUXX+g///M/1draqoqKCi1ZskRPPfWUHn300Q5rKSgoUFhYmPuIioryZSoAAAAAAJwVXXq4ms1m83htjPFq+67+325va2vTeeedp5KSEjkcDt1xxx3Ky8tTcXFxh2Pm5ubqyJEj7uPAgQNdmQoAAAAAAJby6TPe4eHhCgoK8trdbmho8NrVPmXEiBHt9g8ODtbw4cMlSSNHjlT//v0VFBTk7hMXF6f6+nq1tLQoJCTEa9zQ0FCFhob6Uj4AAADQq8QsfsPfJfis9jc3+bsEoMfxacc7JCREDodDlZWVHu2VlZVKSkpq95rExESv/ps2bVJCQoL7QWpTpkzRZ599pra2NnefvXv3auTIke2GbgAAAAAAeguf32qek5Oj559/XmvWrNHu3bu1cOFC1dXVKTMzU9LJt4DPnTvX3T8zM1P79+9XTk6Odu/erTVr1mj16tVatGiRu88vfvELHT58WAsWLNDevXv1xhtv6LHHHtO8efO6YYoAAAAAAPiPz18nlpqaqsOHDys/P18ul0vx8fGqqKhwf/2Xy+Xy+E7v2NhYVVRUaOHChXruuec0atQoPf300+6vEpOkqKgobdq0SQsXLtSll16q888/XwsWLND999/fDVMEAAAAAMB/fA7ekpSVlaWsrKx2z5WVlXm1XX311dq5c+dpx0xMTNSf//znrpQDAAAAAECP1aWnmgMAAAAAgM4heAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAB9TFFRkWJjY2W32+VwOLR169YO+7pcLs2aNUvjxo1Tv379lJ2dfdqxN2zYIJvNppkzZ3q0FxQU6LLLLtOgQYN03nnnaebMmdqzZ083zAYAgJ6P4A0AQB9SXl6u7Oxs5eXlqaamRsnJyZo2bZrHV4F+W3NzsyIiIpSXl6eJEyeeduz9+/dr0aJFSk5O9jq3ZcsWzZs3T3/+859VWVmpEydOKCUlRcePH++WeQEA0JMRvAEA6ENWrFih9PR0ZWRkKC4uToWFhYqKilJxcXG7/WNiYrRy5UrNnTtXYWFhHY7b2tqq2bNna9myZRozZozX+TfffFM//elPdfHFF2vixIl64YUXVFdXp+rq6m6bGwAAPRXBGwCAPqKlpUXV1dVKSUnxaE9JSVFVVdUZjZ2fn6+IiAilp6d3qv+RI0ckScOGDTuj+wIA0BsE+7sAAABwdjQ2Nqq1tVWRkZEe7ZGRkaqvr+/yuO+//75Wr14tp9PZqf7GGOXk5OjKK69UfHx8h/2am5vV3Nzsft3U1NTlGgEA8Cd2vAEA6GNsNpvHa2OMV1tnHT16VHPmzFFpaanCw8M7dc0vf/lLffzxx1q/fv1p+xUUFCgsLMx9REVFdalGAAD8jR1vAAD6iPDwcAUFBXntbjc0NHjtgnfW559/rtraWk2fPt3d1tbWJkkKDg7Wnj17NHbsWPe5++67T6+99pree+89XXDBBacdOzc3Vzk5Oe7XTU1NhG94iFn8hr9L8Fntb27ydwkA/IDgDQBAHxESEiKHw6HKykrdeuut7vbKykrNmDGjS2OOHz9eu3bt8mhbsmSJjh49qpUrV7qDsjFG9913n1555RVt3rxZsbGx3zl2aGioQkNDu1QXAAA9CcEbAIA+JCcnR2lpaUpISFBiYqJKSkpUV1enzMxMSSd3mQ8ePKi1a9e6rzn12e1jx47p0KFDcjqdCgkJ0YQJE2S3270+pz1kyBBJ8mifN2+e1q1bp1dffVWDBg1y77qHhYVpwIABFs4YAAD/I3gDANCHpKam6vDhw8rPz5fL5VJ8fLwqKioUHR0tSXK5XF7f6T158mT3z9XV1Vq3bp2io6NVW1vb6fue+rqya665xqP9hRde0E9/+tMuzQUAgN6C4A0AQB+TlZWlrKysds+VlZV5tRljfBq/O8YAACCQ8FRzAAAAAAAsRPAGAAAAAMBCvNUcAAD0SXwVFQDgbGHHGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwUJeCd1FRkWJjY2W32+VwOLR169bT9t+yZYscDofsdrvGjBmjVatWddh3w4YNstlsmjlzZldKAwAAAACgR/E5eJeXlys7O1t5eXmqqalRcnKypk2bprq6unb779u3TzfeeKOSk5NVU1OjBx54QPPnz9fGjRu9+u7fv1+LFi1ScnKy7zMBAAAAAKAH8jl4r1ixQunp6crIyFBcXJwKCwsVFRWl4uLidvuvWrVKo0ePVmFhoeLi4pSRkaG7775by5cv9+jX2tqq2bNna9myZRozZkzXZgMAAAAAQA/jU/BuaWlRdXW1UlJSPNpTUlJUVVXV7jXbt2/36j916lTt2LFD33zzjbstPz9fERERSk9P71Qtzc3Nampq8jgAAAAAAOhpfArejY2Nam1tVWRkpEd7ZGSk6uvr272mvr6+3f4nTpxQY2OjJOn999/X6tWrVVpa2ulaCgoKFBYW5j6ioqJ8mQoAAAAAAGdFlx6uZrPZPF4bY7zavqv/qfajR49qzpw5Ki0tVXh4eKdryM3N1ZEjR9zHgQMHfJgBAAAAAABnR7AvncPDwxUUFOS1u93Q0OC1q33KiBEj2u0fHBys4cOH65NPPlFtba2mT5/uPt/W1nayuOBg7dmzR2PHjvUaNzQ0VKGhob6UDwAAAADAWefTjndISIgcDocqKys92isrK5WUlNTuNYmJiV79N23apISEBPXv31/jx4/Xrl275HQ63cctt9yia6+9Vk6nk7eQAwAAAAB6NZ92vCUpJydHaWlpSkhIUGJiokpKSlRXV6fMzExJJ98CfvDgQa1du1aSlJmZqWeffVY5OTm65557tH37dq1evVrr16+XJNntdsXHx3vcY8iQIZLk1Q4AAAAAQG/jc/BOTU3V4cOHlZ+fL5fLpfj4eFVUVCg6OlqS5HK5PL7TOzY2VhUVFVq4cKGee+45jRo1Sk8//bRuu+227psFAAAAAAA9lM/BW5KysrKUlZXV7rmysjKvtquvvlo7d+7s9PjtjQEAAAAAQG/UpaeaAwAAAACAziF4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAH1MUVGRYmNjZbfb5XA4tHXr1g77ulwuzZo1S+PGjVO/fv2UnZ192rE3bNggm82mmTNnntF9AQAIJARvAAD6kPLycmVnZysvL081NTVKTk7WtGnTPL4K9Nuam5sVERGhvLw8TZw48bRj79+/X4sWLVJycvIZ3xcAgEBC8AYAoA9ZsWKF0tPTlZGRobi4OBUWFioqKkrFxcXt9o+JidHKlSs1d+5chYWFdThua2urZs+erWXLlmnMmDFnfF8AAAIJwRsAgD6ipaVF1dXVSklJ8WhPSUlRVVXVGY2dn5+viIgIpaend9t9m5ub1dTU5HEAANAbEbwBAOgjGhsb1draqsjISI/2yMhI1dfXd3nc999/X6tXr1ZpaWm33regoEBhYWHuIyoqqss1AgDgTwRvAAD6GJvN5vHaGOPV1llHjx7VnDlzVFpaqvDw8G69b25uro4cOeI+Dhw40KUaAQDwt2B/FwAAAM6O8PBwBQUFee0yNzQ0eO1Gd9bnn3+u2tpaTZ8+3d3W1tYmSQoODtaePXsUFRXVpfuGhoYqNDS0S3UBANCTsOMNAEAfERISIofDocrKSo/2yspKJSUldWnM8ePHa9euXXI6ne7jlltu0bXXXiun06moqChL7gsAQG/CjjcAAH1ITk6O0tLSlJCQoMTERJWUlKiurk6ZmZmSTr69++DBg1q7dq37GqfTKUk6duyYDh06JKfTqZCQEE2YMEF2u13x8fEe9xgyZIgkebR/130BAAhkBG8AAPqQ1NRUHT58WPn5+XK5XIqPj1dFRYWio6MlSS6Xy+u7tSdPnuz+ubq6WuvWrVN0dLRqa2u77b4AAAQygjcAAH1MVlaWsrKy2j1XVlbm1WaM8Wn89sb4rvsCABDI+Iw3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFupS8C4qKlJsbKzsdrscDoe2bt162v5btmyRw+GQ3W7XmDFjtGrVKo/zpaWlSk5O1tChQzV06FBdf/31+uCDD7pSGgAAAAAAPYrPwbu8vFzZ2dnKy8tTTU2NkpOTNW3aNNXV1bXbf9++fbrxxhuVnJysmpoaPfDAA5o/f742btzo7rN582bdeeedevfdd7V9+3aNHj1aKSkpOnjwYNdnBgAAAABAD+Bz8F6xYoXS09OVkZGhuLg4FRYWKioqSsXFxe32X7VqlUaPHq3CwkLFxcUpIyNDd999t5YvX+7u89JLLykrK0uTJk3S+PHjVVpaqra2Nr3zzjtdnxkAAAAAAD2AT8G7paVF1dXVSklJ8WhPSUlRVVVVu9ds377dq//UqVO1Y8cOffPNN+1e8/XXX+ubb77RsGHDOqylublZTU1NHgcAAAAAAD2NT8G7sbFRra2tioyM9GiPjIxUfX19u9fU19e32//EiRNqbGxs95rFixfr/PPP1/XXX99hLQUFBQoLC3MfUVFRvkwFAAAAAICzoksPV7PZbB6vjTFebd/Vv712SXriiSe0fv16vfzyy7Lb7R2OmZubqyNHjriPAwcO+DIFAAAAAADOimBfOoeHhysoKMhrd7uhocFrV/uUESNGtNs/ODhYw4cP92hfvny5HnvsMb399tu69NJLT1tLaGioQkNDfSkfAAAAAICzzqcd75CQEDkcDlVWVnq0V1ZWKikpqd1rEhMTvfpv2rRJCQkJ6t+/v7vtySef1MMPP6w333xTCQkJvpQFAAAAAECP5fNbzXNycvT8889rzZo12r17txYuXKi6ujplZmZKOvkW8Llz57r7Z2Zmav/+/crJydHu3bu1Zs0arV69WosWLXL3eeKJJ7RkyRKtWbNGMTExqq+vV319vY4dO9YNUwQAAAAAwH98equ5JKWmpurw4cPKz8+Xy+VSfHy8KioqFB0dLUlyuVwe3+kdGxuriooKLVy4UM8995xGjRqlp59+Wrfddpu7T1FRkVpaWnT77bd73Gvp0qV66KGHujg1AAAAAAD8z+fgLUlZWVnKyspq91xZWZlX29VXX62dO3d2OF5tbW1XygAAAAAAoMfr0lPNAQAAAABA5xC8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAgD6mqKhIsbGxstvtcjgc2rp1a4d9XS6XZs2apXHjxqlfv37Kzs726vPyyy8rISFBQ4YM0cCBAzVp0iT9/ve/9+hz4sQJLVmyRLGxsRowYIDGjBmj/Px8tbW1dff0AADocQjeAAD0IeXl5crOzlZeXp5qamqUnJysadOmeXwV6Lc1NzcrIiJCeXl5mjhxYrt9hg0bpry8PG3fvl0ff/yxfvazn+lnP/uZ3nrrLXefxx9/XKtWrdKzzz6r3bt364knntCTTz6pZ555xpJ5AgDQkxC8AQDoQ1asWKH09HRlZGQoLi5OhYWFioqKUnFxcbv9Y2JitHLlSs2dO1dhYWHt9rnmmmt06623Ki4uTmPHjtWCBQt06aWXatu2be4+27dv14wZM3TTTTcpJiZGt99+u1JSUrRjxw5L5gkAQE9C8AYAoI9oaWlRdXW1UlJSPNpTUlJUVVXVLfcwxuidd97Rnj17dNVVV7nbr7zySr3zzjvau3evJOmjjz7Stm3bdOONN3Y4VnNzs5qamjwOAAB6o2B/FwAAAM6OxsZGtba2KjIy0qM9MjJS9fX1ZzT2kSNHdP7556u5uVlBQUEqKirSDTfc4D5///3368iRIxo/fryCgoLU2tqqRx99VHfeeWeHYxYUFGjZsmVnVBcAAD0BwRsAgD7GZrN5vDbGeLX5atCgQXI6nTp27Jjeeecd5eTkaMyYMbrmmmsknfxs+Ysvvqh169bp4osvltPpVHZ2tkaNGqW77rqr3TFzc3OVk5Pjft3U1KSoqKgzqhMAAH8geAMA0EeEh4crKCjIa3e7oaHBaxfcV/369dOFF14oSZo0aZJ2796tgoICd/D+1a9+pcWLF+uOO+6QJF1yySXav3+/CgoKOgzeoaGhCg0NPaO6AADoCfiMNwAAfURISIgcDocqKys92isrK5WUlNSt9zLGqLm52f3666+/Vr9+nn/tCAoK4uvEAAB9AjveAAD0ITk5OUpLS1NCQoISExNVUlKiuro6ZWZmSjr59u6DBw9q7dq17mucTqck6dixYzp06JCcTqdCQkI0YcIESSc/i52QkKCxY8eqpaVFFRUVWrt2rceT0qdPn65HH31Uo0eP1sUXX6yamhqtWLFCd99999mbPAAAfkLwBgCgD0lNTdXhw4eVn58vl8ul+Ph4VVRUKDo6WpLkcrm8vtN78uTJ7p+rq6u1bt06RUdHq7a2VpJ0/PhxZWVl6X//9381YMAAjR8/Xi+++KJSU1Pd1z3zzDN68MEHlZWVpYaGBo0aNUo///nP9etf/9r6SQMA4GcEbwAA+pisrCxlZWW1e66srMyrzRhz2vEeeeQRPfLII6ftM2jQIBUWFqqwsLCzZQIAEDD4jDcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAW6lLwLioqUmxsrOx2uxwOh7Zu3Xra/lu2bJHD4ZDdbteYMWO0atUqrz4bN27UhAkTFBoaqgkTJuiVV17pSmkAAAAAAPQoPgfv8vJyZWdnKy8vTzU1NUpOTta0adNUV1fXbv99+/bpxhtvVHJysmpqavTAAw9o/vz52rhxo7vP9u3blZqaqrS0NH300UdKS0vTT37yE/3lL3/p+swAAAAAAOgBfA7eK1asUHp6ujIyMhQXF6fCwkJFRUWpuLi43f6rVq3S6NGjVVhYqLi4OGVkZOjuu+/W8uXL3X0KCwt1ww03KDc3V+PHj1dubq6uu+46FRYWdnliAAAAAAD0BMG+dG5paVF1dbUWL17s0Z6SkqKqqqp2r9m+fbtSUlI82qZOnarVq1frm2++Uf/+/bV9+3YtXLjQq8/pgndzc7Oam5vdr48cOSJJampq8mVKHWpr/rpbxjmbfJl7b5yfFPhzDPT5SYE/R1//NyjQ5xjo8+vMOMaYbhkP//e7ZK3vvECfY6DPTwr8OQb6/KTAn2NvnJ909td7n4J3Y2OjWltbFRkZ6dEeGRmp+vr6dq+pr69vt/+JEyfU2NiokSNHdtinozElqaCgQMuWLfNqj4qK6ux0Ak5Yob8rsF6gzzHQ5ycF/hwDfX5S4M+xu+d39OhRhYWFde+gfdTRo0clsdYHukCfY6DPTwr8OQb6/CTm2BXftd77FLxPsdlsHq+NMV5t39X/X9t9HTM3N1c5OTnu121tbfr73/+u4cOHn/Y6f2tqalJUVJQOHDigwYMH+7ucbhfo85MCf46BPj8p8OcY6POTesccjTE6evSoRo0a5e9SAsaoUaN04MABDRo0iLXejwJ9flLgzzHQ5ycF/hwDfX5S75ljZ9d7n4J3eHi4goKCvHaiGxoavHasTxkxYkS7/YODgzV8+PDT9uloTEkKDQ1VaGioR9uQIUM6OxW/Gzx4cI/+A3SmAn1+UuDPMdDnJwX+HAN9flLPnyM73d2rX79+uuCCC/xdRqf19D+fZyrQ5ycF/hwDfX5S4M8x0Ocn9Y45dma99+nhaiEhIXI4HKqsrPRor6ysVFJSUrvXJCYmevXftGmTEhIS1L9//9P26WhMAAAAAAB6C5/fap6Tk6O0tDQlJCQoMTFRJSUlqqurU2ZmpqSTbwE/ePCg1q5dK0nKzMzUs88+q5ycHN1zzz3avn27Vq9erfXr17vHXLBgga666io9/vjjmjFjhl599VW9/fbb2rZtWzdNEwAAAAAA//A5eKempurw4cPKz8+Xy+VSfHy8KioqFB0dLUlyuVwe3+kdGxuriooKLVy4UM8995xGjRqlp59+Wrfddpu7T1JSkjZs2KAlS5bowQcf1NixY1VeXq7LL7+8G6bYs4SGhmrp0qVeb5MPFIE+Pynw5xjo85MCf46BPj+pb8wRvVeg//kM9PlJgT/HQJ+fFPhzDPT5SYE3R5vhe04AAAAAALCMT5/xBgAAAAAAviF4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgvdZVFRUpNjYWNntdjkcDm3dutXfJXWb9957T9OnT9eoUaNks9n0X//1X/4uqVsVFBTosssu06BBg3Teeedp5syZ2rNnj7/L6lbFxcW69NJLNXjwYA0ePFiJiYn605/+5O+yLFNQUCCbzabs7Gx/l9JtHnroIdlsNo9jxIgR/i6rWx08eFBz5szR8OHDdc4552jSpEmqrq72d1mAG2t978VaH3hY63unQF3rCd5nSXl5ubKzs5WXl6eamholJydr2rRpHl+91psdP35cEydO1LPPPuvvUiyxZcsWzZs3T3/+859VWVmpEydOKCUlRcePH/d3ad3mggsu0G9+8xvt2LFDO3bs0A9/+EPNmDFDn3zyib9L63YffvihSkpKdOmll/q7lG538cUXy+VyuY9du3b5u6Ru8+WXX2rKlCnq37+//vSnP+mvf/2rnnrqKQ0ZMsTfpQGSWOt7O9b6wMJa3zsF9FpvcFb84Ac/MJmZmR5t48ePN4sXL/ZTRdaRZF555RV/l2GphoYGI8ls2bLF36VYaujQoeb555/3dxnd6ujRo+aiiy4ylZWV5uqrrzYLFizwd0ndZunSpWbixIn+LsMy999/v7nyyiv9XQbQIdb6wMJa33ux1vdegbzWs+N9FrS0tKi6ulopKSke7SkpKaqqqvJTVTgTR44ckSQNGzbMz5VYo7W1VRs2bNDx48eVmJjo73K61bx583TTTTfp+uuv93cplvj00081atQoxcbG6o477tAXX3zh75K6zWuvvaaEhAT9+Mc/1nnnnafJkyertLTU32UBkljrAxFrfe/FWt97BfJaT/A+CxobG9Xa2qrIyEiP9sjISNXX1/upKnSVMUY5OTm68sorFR8f7+9yutWuXbt07rnnKjQ0VJmZmXrllVc0YcIEf5fVbTZs2KCdO3eqoKDA36VY4vLLL9fatWv11ltvqbS0VPX19UpKStLhw4f9XVq3+OKLL1RcXKyLLrpIb731ljIzMzV//nytXbvW36UBrPUBhrW+92Kt790Cea0P9ncBfYnNZvN4bYzxakPP98tf/lIff/yxtm3b5u9Sut24cePkdDr11VdfaePGjbrrrru0ZcuWgFiQDxw4oAULFmjTpk2y2+3+LscS06ZNc/98ySWXKDExUWPHjtXvfvc75eTk+LGy7tHW1qaEhAQ99thjkqTJkyfrk08+UXFxsebOnevn6oCTWOsDA2t978Raz1rfk7HjfRaEh4crKCjI61+8GxoavP5lHD3bfffdp9dee03vvvuuLrjgAn+X0+1CQkJ04YUXKiEhQQUFBZo4caJWrlzp77K6RXV1tRoaGuRwOBQcHKzg4GBt2bJFTz/9tIKDg9Xa2urvErvdwIEDdckll+jTTz/1dyndYuTIkV5/MYyLiwuYB1ehd2OtDxys9b0Xa33vF8hrPcH7LAgJCZHD4VBlZaVHe2VlpZKSkvxUFXxhjNEvf/lLvfzyy/rv//5vxcbG+ruks8IYo+bmZn+X0S2uu+467dq1S06n030kJCRo9uzZcjqdCgoK8neJ3a65uVm7d+/WyJEj/V1Kt5gyZYrXV/vs3btX0dHRfqoI+D+s9b0fa33vx1rf+wXyWs9bzc+SnJwcpaWlKSEhQYmJiSopKVFdXZ0yMzP9XVq3OHbsmD777DP363379snpdGrYsGEaPXq0HyvrHvPmzdO6dev06quvatCgQe4djbCwMA0YMMDP1XWPBx54QNOmTVNUVJSOHj2qDRs2aPPmzXrzzTf9XVq3GDRokNfn9AYOHKjhw4cHzOf3Fi1apOnTp2v06NFqaGjQI488oqamJt11113+Lq1bLFy4UElJSXrsscf0k5/8RB988IFKSkpUUlLi79IASaz1vR1rfe/HWt/7BfRa778Hqvc9zz33nImOjjYhISHm+9//fkB9PcW7775rJHkdd911l79L6xbtzU2SeeGFF/xdWre5++673X8+IyIizHXXXWc2bdrk77IsFWhfMZKammpGjhxp+vfvb0aNGmV+9KMfmU8++cTfZXWr119/3cTHx5vQ0FAzfvx4U1JS4u+SAA+s9b0Xa31gYq3vfQJ1rbcZY8zZDPoAAAAAAPQlfMYbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbyBAHPNNdcoOzvb32X4pKysTEOGDPHpmp/+9KeaOXOmJfUAAHA6rLXorNraWtlsNjmdTn+XAj8jeAM+qKqqUlBQkP7t3/7N69xDDz2kSZMmebXbbDb913/9V7fXsnnzZtlsNn311Vce7S+//LIefvjhbr+flVJTU7V3795uHzcmJkaFhYXdPi4AwDqstdZgrQX8i+AN+GDNmjW67777tG3bNtXV1fm7nHYNGzZMgwYN8ncZPhkwYIDOO+88f5cBAOgBWGutwVoL+BfBG+ik48eP6w9/+IN+8Ytf6Oabb1ZZWZn7XFlZmZYtW6aPPvpINptNNptNZWVliomJkSTdeuutstls7teS9Prrr8vhcMhut2vMmDFatmyZTpw44T5vs9n0/PPP69Zbb9U555yjiy66SK+99pqkk29buvbaayVJQ4cOlc1m009/+lNJ3m9/+/LLLzV37lwNHTpU55xzjqZNm6ZPP/3Uo/YhQ4borbfeUlxcnM4991z927/9m1wuV4e/C4fDoaeeesr9eubMmQoODlZTU5Mkqb6+XjabTXv27JEktbS06D/+4z90/vnna+DAgbr88su1efNmrxq+7ZFHHtF5552nQYMGKSMjQ4sXL253l2P58uUaOXKkhg8frnnz5umbb75x/x7279+vhQsXuv+bSNL+/fs1ffp0DR06VAMHDtTFF1+sioqKDucKADh7WGv/T19ea2NiYtzjffs4ZdeuXfrhD3+oAQMGaPjw4br33nt17Ngx9/m2tjbl5+frggsuUGhoqCZNmqQ333zTff7U27//8Ic/KDk5WQMGDNBll12mvXv36sMPP1RCQoL7v9GhQ4c8anvhhRcUFxcnu92u8ePHq6ioyOP8Bx98oMmTJ8tutyshIUE1NTWdnjcCnAHQKatXrzYJCQnGGGNef/11ExMTY9ra2owxxnz99dfm3//9383FF19sXC6Xcblc5uuvvzYNDQ1GknnhhReMy+UyDQ0Nxhhj3nzzTTN48GBTVlZmPv/8c7Np0yYTExNjHnroIff9JJkLLrjArFu3znz66adm/vz55txzzzWHDx82J06cMBs3bjSSzJ49e4zL5TJfffWVMcaYq6++2ixYsMA9zi233GLi4uLMe++9Z5xOp5k6daq58MILTUtLizHGmBdeeMH079/fXH/99ebDDz801dXVJi4uzsyaNavD30VOTo65+eabjTHGtLW1mWHDhpnw8HDzxhtvGGOMWbdunRkxYoS7/6xZs0xSUpJ57733zGeffWaefPJJExoaavbu3euuISwszN3/xRdfNHa73axZs8bs2bPHLFu2zAwePNhMnDjR3eeuu+4ygwcPNpmZmWb37t3m9ddfN+ecc44pKSkxxhhz+PBhc8EFF5j8/Hz3fxNjjLnpppvMDTfcYD7++GPz+eefm9dff91s2bKlk38KAABWYq39P315rW1oaHCP97//+7/miiuuMMnJycYYY44fP25GjRplfvSjH5ldu3aZd955x8TGxpq77rrLff2KFSvM4MGDzfr1683/9//9f+Y//uM/TP/+/d2/i3379hlJZvz48ebNN980f/3rX80VV1xhvv/975trrrnGbNu2zezcudNceOGFJjMz0z1uSUmJGTlypNm4caP54osvzMaNG82wYcNMWVmZMcaYY8eOmYiICJOammr+53/+x7z++utmzJgxRpKpqanp9PwRmAjeQCclJSWZwsJCY4wx33zzjQkPDzeVlZXu80uXLvVYrE6RZF555RWPtuTkZPPYY495tP3+9783I0eO9LhuyZIl7tfHjh0zNpvN/OlPfzLGGPPuu+8aSebLL7/0GOfbfxnYu3evkWTef/999/nGxkYzYMAA84c//MEYc3IhlmQ+++wzd5/nnnvOREZGdvi7eO2110xYWJhpbW01TqfTREREmIULF5pf/epXxhhj7r33XpOammqMMeazzz4zNpvNHDx40GOM6667zuTm5rpr+PZfBi6//HIzb948j/5Tpkzx+stAdHS0OXHihLvtxz/+sfu+xhgTHR1tfvvb33qMc8kll3j8pQsA0HOw1v4f1tqT5s+fb6Kjo93/oFJSUmKGDh1qjh075u7zxhtvmH79+pn6+npjjDGjRo0yjz76qMc4l112mcnKyjLG/F/wfv75593n169fbySZd955x91WUFBgxo0b534dFRVl1q1b5zHuww8/bBITE40xxvy///f/zLBhw8zx48fd54uLiwneMMYYw1vNgU7Ys2ePPvjgA91xxx2SpODgYKWmpmrNmjVdGq+6ulr5+fk699xz3cc999wjl8ulr7/+2t3v0ksvdf88cOBADRo0SA0NDZ2+z+7duxUcHKzLL7/c3TZ8+HCNGzdOu3fvdredc845Gjt2rPv1yJEjT3ufq666SkePHlVNTY22bNmiq6++Wtdee622bNki6eTDaK6++mpJ0s6dO2WM0fe+9z2P+W7ZskWff/55u+Pv2bNHP/jBDzza/vW1JF188cUKCgrqdN2SNH/+fD3yyCOaMmWKli5dqo8//vi0/QEAZwdrrSfWWqmkpESrV6/Wq6++qoiICEknf98TJ07UwIED3f2mTJmitrY27dmzR01NTfrb3/6mKVOmeIw1ZcoUj/8ekud/+8jISEnSJZdc4tF2aq6HDh3SgQMHlJ6e7vE7fuSRR9y/41O1nXPOOe4xEhMTuzR3BJ5gfxcA9AarV6/WiRMndP7557vbjDHq37+/vvzySw0dOtSn8dra2rRs2TL96Ec/8jpnt9vdP/fv39/jnM1mU1tbW6fvY4zpsP3bn5Vq7z4dXStJYWFhmjRpkjZv3qyqqir98Ic/VHJyspxOpz799FPt3btX11xzjaSTcw0KClJ1dbXHwi1J5557bof3+HZ9Hc2lK7+fjIwMTZ06VW+88YY2bdqkgoICPfXUU7rvvvtOex0AwFqstZ76+lq7efNm3XfffVq/fr0mTpzoUeO/1v3t2tr7uaPrvj23U+f+te3UXE/939LSUo9/ZJHk/p2f7r8nwI438B1OnDihtWvX6qmnnpLT6XQfH330kaKjo/XSSy9JkkJCQtTa2up1ff/+/b3av//972vPnj268MILvY5+/Tr3/5YhISGS1O49T5kwYYJOnDihv/zlL+62w4cPa+/evYqLi+vUfTpyzTXX6N1339V7772na665RkOGDNGECRPcD2o5Nf7kyZPV2tqqhoYGr7mOGDGi3bHHjRunDz74wKNtx44dPtfY0X+TqKgoZWZm6uWXX9a///u/q7S01OexAQDdh7W2fX11rf3ss89022236YEHHvD6h5MJEybI6XTq+PHj7rb3339f/fr10/e+9z0NHjxYo0aN0rZt2zyuq6qqOqP/HpGRkTr//PP1xRdfeP2OY2Nj3bV99NFH+sc//uG+7s9//nOX74nAQvAGvsMf//hHffnll0pPT1d8fLzHcfvtt2v16tWSTj6Bc9++fXI6nWpsbFRzc7O7/Z133lF9fb2+/PJLSdKvf/1rrV27Vg899JA++eQT7d69W+Xl5VqyZEmn64qOjpbNZtMf//hHHTp0yONpnqdcdNFFmjFjhu655x5t27ZNH330kebMmaPzzz9fM2bMOKPfyzXXXKM333xTNptNEyZMcLe99NJL7re+SdL3vvc9zZ49W3PnztXLL7+sffv26cMPP9Tjjz/e4RNO77vvPq1evVq/+93v9Omnn+qRRx7Rxx9/3OG/cHckJiZG7733ng4ePKjGxkZJUnZ2tt566y3t27dPO3fu1H//93+f8V+MAABnhrW2fX1xrf3HP/6h6dOna9KkSbr33ntVX1/vPiRp9uzZstvtuuuuu/Q///M/evfdd3XfffcpLS3N/XbxX/3qV3r88cdVXl6uPXv2aPHixXI6nVqwYIFPc/tXDz30kAoKCrRy5Urt3btXu3bt0gsvvKAVK1ZIkmbNmqV+/fopPT1df/3rX1VRUaHly5ef0T0RQPzwuXKgV7n55pvNjTfe2O656upqI8lUV1ebf/7zn+a2224zQ4YMcT9d1ZiTD0e58MILTXBwsImOjnZf++abb5qkpCQzYMAAM3jwYPODH/zA/ZRQY9p/UExYWJh7XGOMyc/PNyNGjDA2m839NM9/fdLq3//+d5OWlmbCwsLMgAEDzNSpU91P9TTG+2ErxhjzyiuvmO/6n4evvvrKBAUFmdtvv93rumeffdajb0tLi/n1r39tYmJiTP/+/c2IESPMrbfeaj7++OMOa8jPzzfh4eHm3HPPNXfffbeZP3++ueKKK9zn77rrLjNjxgyPaxYsWGCuvvpq9+vt27ebSy+91ISGhrrn88tf/tKMHTvWhIaGmoiICJOWlmYaGxtPO1cAgLVYa9sXqGttdHS0Wbp0abtzPvXgs/aOUz7++GNz7bXXGrvdboYNG2buuecec/ToUff51tZWs2zZMnP++eeb/v37m4kTJ7ofmPfte3z7gWftPUivvd/ZSy+9ZCZNmmRCQkLM0KFDzVVXXWVefvllj9/HxIkTTUhIiJk0aZL7yfg8XA02Y/gwAoCe74YbbtCIESP0+9//3t+lAAAQkM7GWvuPf/xDw4YNU0VFhft70oG+gIerAehxvv76a61atUpTp05VUFCQ1q9fr7fffluVlZX+Lg0AgIDgr7V2y5Yt+uEPf0joRp/DjjeAHufU57t27typ5uZmjRs3TkuWLGn3ybQAAMB3rLXA2UXwBgAAAADAQjzVHAAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAAL/f+lwFwXgyVx1gAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"cell_type":"markdown","source":"Decoder","metadata":{"id":"g68xf3c_-gBS"}},{"cell_type":"code","source":"class Decoder(tf.keras.layers.Layer):\n  def __init__(self, output_vocab_size, embedding_dim, dec_units):\n    super(Decoder, self).__init__()\n    self.dec_units = dec_units\n    self.output_vocab_size = output_vocab_size\n    self.embedding_dim = embedding_dim\n\n    # For Step 1. The embedding layer converts token IDs to vectors\n    self.embedding = tf.keras.layers.Embedding(self.output_vocab_size,\n                                               embedding_dim)\n\n    # For Step 2. The RNN keeps track of what's been generated so far.\n    self.gru = tf.keras.layers.GRU(self.dec_units,\n                                   return_sequences=True,\n                                   return_state=True,\n                                   recurrent_initializer='glorot_uniform',\n                                  recurrent_dropout=0.1)\n\n    # For step 3. The RNN output will be the query for the attention layer.\n    self.attention = BahdanauAttention(self.dec_units)\n\n    # For step 4. Eqn. (3): converting `ct` to `at`\n    self.Wc = tf.keras.layers.Dense(dec_units, activation=tf.math.tanh,\n                                    use_bias=False)\n\n    # For step 5. This fully connected layer produces the logits for each output token.\n    self.fc = tf.keras.layers.Dense(self.output_vocab_size)","metadata":{"id":"erYvHIgAl8kh","execution":{"iopub.status.busy":"2023-03-16T11:46:39.183861Z","iopub.execute_input":"2023-03-16T11:46:39.184272Z","iopub.status.idle":"2023-03-16T11:46:39.192243Z","shell.execute_reply.started":"2023-03-16T11:46:39.184233Z","shell.execute_reply":"2023-03-16T11:46:39.191028Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"class DecoderInput(typing.NamedTuple):\n  new_tokens: Any\n  enc_output: Any\n  mask: Any\n\nclass DecoderOutput(typing.NamedTuple):\n  logits: Any\n  attention_weights: Any","metadata":{"id":"7WfSIb2sArRT","execution":{"iopub.status.busy":"2023-03-16T11:46:39.193966Z","iopub.execute_input":"2023-03-16T11:46:39.194651Z","iopub.status.idle":"2023-03-16T11:46:39.206347Z","shell.execute_reply.started":"2023-03-16T11:46:39.194612Z","shell.execute_reply":"2023-03-16T11:46:39.205356Z"},"trusted":true},"execution_count":56,"outputs":[]},{"cell_type":"code","source":"def call(self,\n         inputs: DecoderInput,\n         state=None) -> Tuple[DecoderOutput, tf.Tensor]:\n  shape_checker = ShapeChecker()\n  shape_checker(inputs.new_tokens, ('batch', 't'))\n  shape_checker(inputs.enc_output, ('batch', 's', 'enc_units'))\n  shape_checker(inputs.mask, ('batch', 's'))\n\n  if state is not None:\n    shape_checker(state, ('batch', 'dec_units'))\n\n  # Step 1. Lookup the embeddings\n  vectors = self.embedding(inputs.new_tokens)\n  shape_checker(vectors, ('batch', 't', 'embedding_dim'))\n\n  # Step 2. Process one step with the RNN\n  rnn_output, state = self.gru(vectors, initial_state=state)\n\n  shape_checker(rnn_output, ('batch', 't', 'dec_units'))\n  shape_checker(state, ('batch', 'dec_units'))\n\n  # Step 3. Use the RNN output as the query for the attention over the\n  # encoder output.\n  context_vector, attention_weights = self.attention(\n      query=rnn_output, value=inputs.enc_output, mask=inputs.mask)\n  shape_checker(context_vector, ('batch', 't', 'dec_units'))\n  shape_checker(attention_weights, ('batch', 't', 's'))\n\n  # Step 4. Eqn. (3): Join the context_vector and rnn_output\n  #     [ct; ht] shape: (batch t, value_units + query_units)\n  context_and_rnn_output = tf.concat([context_vector, rnn_output], axis=-1)\n\n  # Step 4. Eqn. (3): `at = tanh(Wc@[ct; ht])`\n  attention_vector = self.Wc(context_and_rnn_output)\n  shape_checker(attention_vector, ('batch', 't', 'dec_units'))\n\n  # Step 5. Generate logit predictions:\n  logits = self.fc(attention_vector)\n  shape_checker(logits, ('batch', 't', 'output_vocab_size'))\n\n  return DecoderOutput(logits, attention_weights), state","metadata":{"id":"PJOi5btHAPNK","execution":{"iopub.status.busy":"2023-03-16T11:46:39.208199Z","iopub.execute_input":"2023-03-16T11:46:39.208844Z","iopub.status.idle":"2023-03-16T11:46:39.220786Z","shell.execute_reply.started":"2023-03-16T11:46:39.208582Z","shell.execute_reply":"2023-03-16T11:46:39.219581Z"},"trusted":true},"execution_count":57,"outputs":[]},{"cell_type":"code","source":"Decoder.call = call","metadata":{"id":"Ay_mTMPfnb2a","execution":{"iopub.status.busy":"2023-03-16T11:46:39.222435Z","iopub.execute_input":"2023-03-16T11:46:39.223014Z","iopub.status.idle":"2023-03-16T11:46:39.232325Z","shell.execute_reply.started":"2023-03-16T11:46:39.222949Z","shell.execute_reply":"2023-03-16T11:46:39.231281Z"},"trusted":true},"execution_count":58,"outputs":[]},{"cell_type":"code","source":"decoder = Decoder(output_text_processor.vocabulary_size(),\n                  embedding_dim, units)","metadata":{"id":"4ZUMbYXIEVeA","execution":{"iopub.status.busy":"2023-03-16T11:46:39.233853Z","iopub.execute_input":"2023-03-16T11:46:39.234238Z","iopub.status.idle":"2023-03-16T11:46:39.251412Z","shell.execute_reply.started":"2023-03-16T11:46:39.234201Z","shell.execute_reply":"2023-03-16T11:46:39.250373Z"},"trusted":true},"execution_count":59,"outputs":[]},{"cell_type":"code","source":"# Convert the target sequence, and collect the \"[START]\" tokens\nexample_output_tokens = output_text_processor(example_target_batch)\n\nstart_index = output_text_processor.get_vocabulary().index('[START]')\nfirst_token = tf.constant([[start_index]] * example_output_tokens.shape[0])","metadata":{"id":"4u6eJBU4GL40","execution":{"iopub.status.busy":"2023-03-16T11:46:39.252616Z","iopub.execute_input":"2023-03-16T11:46:39.253187Z","iopub.status.idle":"2023-03-16T11:46:39.336408Z","shell.execute_reply.started":"2023-03-16T11:46:39.253147Z","shell.execute_reply":"2023-03-16T11:46:39.335259Z"},"trusted":true},"execution_count":60,"outputs":[]},{"cell_type":"code","source":"# Run the decoder\ndec_result, dec_state = decoder(\n    inputs = DecoderInput(new_tokens=first_token,\n                          enc_output=example_enc_output,\n                          mask=(example_tokens != 0)),\n    state = example_enc_state\n)\n\nprint(f'logits shape: (batch_size, t, output_vocab_size) {dec_result.logits.shape}')\nprint(f'state shape: (batch_size, dec_units) {dec_state.shape}')","metadata":{"id":"E5hqvbR5FUCD","outputId":"ec292583-153d-4d60-d79e-082f2ed7a208","execution":{"iopub.status.busy":"2023-03-16T11:46:39.337807Z","iopub.execute_input":"2023-03-16T11:46:39.338547Z","iopub.status.idle":"2023-03-16T11:46:39.459382Z","shell.execute_reply.started":"2023-03-16T11:46:39.338507Z","shell.execute_reply":"2023-03-16T11:46:39.458221Z"},"trusted":true},"execution_count":61,"outputs":[{"name":"stdout","text":"logits shape: (batch_size, t, output_vocab_size) (64, 1, 23282)\nstate shape: (batch_size, dec_units) (64, 1024)\n","output_type":"stream"}]},{"cell_type":"code","source":"# sample token according to logits\nsampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)","metadata":{"id":"P5UY8wko3jFp","execution":{"iopub.status.busy":"2023-03-16T11:46:39.460768Z","iopub.execute_input":"2023-03-16T11:46:39.461652Z","iopub.status.idle":"2023-03-16T11:46:39.470290Z","shell.execute_reply.started":"2023-03-16T11:46:39.461610Z","shell.execute_reply":"2023-03-16T11:46:39.469239Z"},"trusted":true},"execution_count":62,"outputs":[]},{"cell_type":"code","source":"# decode the token as first word of o/p\nvocab = np.array(output_text_processor.get_vocabulary())\nfirst_word = vocab[sampled_token.numpy()]\nfirst_word[:5]","metadata":{"id":"lKXTLYu4IV7I","outputId":"4c243ce7-23fc-4781-98c6-9d64e07e13d0","execution":{"iopub.status.busy":"2023-03-16T11:46:39.471965Z","iopub.execute_input":"2023-03-16T11:46:39.472379Z","iopub.status.idle":"2023-03-16T11:46:39.554000Z","shell.execute_reply.started":"2023-03-16T11:46:39.472340Z","shell.execute_reply":"2023-03-16T11:46:39.553028Z"},"trusted":true},"execution_count":63,"outputs":[{"execution_count":63,"output_type":"execute_result","data":{"text/plain":"array([['slogan'],\n       ['foreigner'],\n       ['disney'],\n       ['ptag'],\n       ['annoying']], dtype='<U173')"},"metadata":{}}]},{"cell_type":"code","source":"# decoder generates second set of logits\ndec_result, dec_state = decoder(\n    DecoderInput(sampled_token,\n                 example_enc_output,\n                 mask=(example_tokens != 0)),\n    state=dec_state)","metadata":{"id":"pX1VF9XDJTOM","execution":{"iopub.status.busy":"2023-03-16T11:46:39.555471Z","iopub.execute_input":"2023-03-16T11:46:39.555961Z","iopub.status.idle":"2023-03-16T11:46:39.636012Z","shell.execute_reply.started":"2023-03-16T11:46:39.555917Z","shell.execute_reply":"2023-03-16T11:46:39.634954Z"},"trusted":true},"execution_count":64,"outputs":[]},{"cell_type":"code","source":"sampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)\nfirst_word = vocab[sampled_token.numpy()]\nfirst_word[:5]","metadata":{"id":"H1rs0XL7Y2aS","outputId":"44ae682c-c8c3-45e6-9f2b-e56c89f3a46a","execution":{"iopub.status.busy":"2023-03-16T11:46:39.637318Z","iopub.execute_input":"2023-03-16T11:46:39.637684Z","iopub.status.idle":"2023-03-16T11:46:39.651663Z","shell.execute_reply.started":"2023-03-16T11:46:39.637648Z","shell.execute_reply":"2023-03-16T11:46:39.650499Z"},"trusted":true},"execution_count":65,"outputs":[{"execution_count":65,"output_type":"execute_result","data":{"text/plain":"array([['foodie'],\n       ['callaway'],\n       ['devyani'],\n       ['polish'],\n       ['cow']], dtype='<U173')"},"metadata":{}}]},{"cell_type":"markdown","source":"Training","metadata":{"id":"hQpRXZdxBYAO"}},{"cell_type":"code","source":"# Loss function definition\nclass MaskedLoss(tf.keras.losses.Loss):\n  def __init__(self):\n    self.name = 'masked_loss'\n    self.loss = tf.keras.losses.SparseCategoricalCrossentropy(\n        from_logits=True, reduction='none')\n\n  def __call__(self, y_true, y_pred):\n    shape_checker = ShapeChecker()\n    shape_checker(y_true, ('batch', 't'))\n    shape_checker(y_pred, ('batch', 't', 'logits'))\n\n    # Calculate the loss for each item in the batch.\n    loss = self.loss(y_true, y_pred)\n    shape_checker(loss, ('batch', 't'))\n\n    # Mask off the losses on padding.\n    mask = tf.cast(y_true != 0, tf.float32)\n    shape_checker(mask, ('batch', 't'))\n    loss *= mask\n\n    # Return the total.\n    return tf.reduce_sum(loss)","metadata":{"id":"WmTHr5iV3jFr","execution":{"iopub.status.busy":"2023-03-16T11:46:39.653427Z","iopub.execute_input":"2023-03-16T11:46:39.654029Z","iopub.status.idle":"2023-03-16T11:46:39.661768Z","shell.execute_reply.started":"2023-03-16T11:46:39.653973Z","shell.execute_reply":"2023-03-16T11:46:39.660525Z"},"trusted":true},"execution_count":66,"outputs":[]},{"cell_type":"code","source":"# implementing training step\nclass TrainTranslator(tf.keras.Model):\n  def __init__(self, embedding_dim, units,\n               input_text_processor,\n               output_text_processor, \n               use_tf_function=True):\n    super().__init__()\n    # Build the encoder and decoder\n    encoder = Encoder(input_text_processor.vocabulary_size(),\n                      embedding_dim, units)\n    decoder = Decoder(output_text_processor.vocabulary_size(),\n                      embedding_dim, units)\n\n    self.encoder = encoder\n    self.decoder = decoder\n    self.input_text_processor = input_text_processor\n    self.output_text_processor = output_text_processor\n    self.use_tf_function = use_tf_function\n    self.shape_checker = ShapeChecker()\n\n  def train_step(self, inputs):\n    self.shape_checker = ShapeChecker()\n    if self.use_tf_function:\n      return self._tf_train_step(inputs)\n    else:\n      return self._train_step(inputs)","metadata":{"id":"WWIyuy71TkJT","execution":{"iopub.status.busy":"2023-03-16T11:46:39.663904Z","iopub.execute_input":"2023-03-16T11:46:39.664348Z","iopub.status.idle":"2023-03-16T11:46:39.674358Z","shell.execute_reply.started":"2023-03-16T11:46:39.664311Z","shell.execute_reply":"2023-03-16T11:46:39.673307Z"},"trusted":true},"execution_count":67,"outputs":[]},{"cell_type":"code","source":"# Receiving batches of english & hinglish data\n# Converting those raw text inputs to token-embeddings and masks.\ndef _preprocess(self, input_text, target_text):\n  self.shape_checker(input_text, ('batch',))\n  self.shape_checker(target_text, ('batch',))\n\n  # Convert the text to token IDs\n  input_tokens = self.input_text_processor(input_text)\n  target_tokens = self.output_text_processor(target_text)\n  self.shape_checker(input_tokens, ('batch', 's'))\n  self.shape_checker(target_tokens, ('batch', 't'))\n\n  # Convert IDs to masks.\n  input_mask = input_tokens != 0\n  self.shape_checker(input_mask, ('batch', 's'))\n\n  target_mask = target_tokens != 0\n  self.shape_checker(target_mask, ('batch', 't'))\n\n  return input_tokens, input_mask, target_tokens, target_mask","metadata":{"id":"ZlYE68wzXoA8","execution":{"iopub.status.busy":"2023-03-16T11:46:39.675938Z","iopub.execute_input":"2023-03-16T11:46:39.676821Z","iopub.status.idle":"2023-03-16T11:46:39.688202Z","shell.execute_reply.started":"2023-03-16T11:46:39.676780Z","shell.execute_reply":"2023-03-16T11:46:39.687116Z"},"trusted":true},"execution_count":68,"outputs":[]},{"cell_type":"code","source":"TrainTranslator._preprocess = _preprocess","metadata":{"id":"lHy6hzStrgjQ","execution":{"iopub.status.busy":"2023-03-16T11:46:39.689825Z","iopub.execute_input":"2023-03-16T11:46:39.690521Z","iopub.status.idle":"2023-03-16T11:46:39.698402Z","shell.execute_reply.started":"2023-03-16T11:46:39.690449Z","shell.execute_reply":"2023-03-16T11:46:39.697171Z"},"trusted":true},"execution_count":69,"outputs":[]},{"cell_type":"code","source":"# training\ndef _train_step(self, inputs):\n  input_text, target_text = inputs  \n\n  (input_tokens, input_mask,\n   target_tokens, target_mask) = self._preprocess(input_text, target_text)\n\n  max_target_length = tf.shape(target_tokens)[1]\n\n  with tf.GradientTape() as tape:\n    # Encode the input\n    enc_output, enc_state = self.encoder(input_tokens)\n    self.shape_checker(enc_output, ('batch', 's', 'enc_units'))\n    self.shape_checker(enc_state, ('batch', 'enc_units'))\n\n    # Initialize the decoder's state to the encoder's final state.\n    # This only works if the encoder and decoder have the same number of\n    # units.\n    dec_state = enc_state\n    loss = tf.constant(0.0)\n\n    for t in tf.range(max_target_length-1):\n      # Pass in two tokens from the target sequence:\n      # 1. The current input to the decoder.\n      # 2. The target for the decoder's next prediction.\n      new_tokens = target_tokens[:, t:t+2]\n      step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n                                             enc_output, dec_state)\n      loss = loss + step_loss\n\n    # Average the loss over all non padding tokens.\n    average_loss = loss / tf.reduce_sum(tf.cast(target_mask, tf.float32))\n\n  # Apply an optimization step\n  variables = self.trainable_variables \n  gradients = tape.gradient(average_loss, variables)\n  self.optimizer.apply_gradients(zip(gradients, variables))\n\n  # Return a dict mapping metric names to current value\n  return {'batch_loss': average_loss}","metadata":{"id":"Qs_gsISsYPpY","execution":{"iopub.status.busy":"2023-03-16T11:46:39.700067Z","iopub.execute_input":"2023-03-16T11:46:39.700442Z","iopub.status.idle":"2023-03-16T11:46:39.711601Z","shell.execute_reply.started":"2023-03-16T11:46:39.700403Z","shell.execute_reply":"2023-03-16T11:46:39.710543Z"},"trusted":true},"execution_count":70,"outputs":[]},{"cell_type":"code","source":"TrainTranslator._train_step = _train_step","metadata":{"id":"KGwWHIxLrjGR","execution":{"iopub.status.busy":"2023-03-16T11:46:39.712781Z","iopub.execute_input":"2023-03-16T11:46:39.713120Z","iopub.status.idle":"2023-03-16T11:46:39.724653Z","shell.execute_reply.started":"2023-03-16T11:46:39.713091Z","shell.execute_reply":"2023-03-16T11:46:39.723530Z"},"trusted":true},"execution_count":71,"outputs":[]},{"cell_type":"code","source":"# executes the decoder and calculates the incremental loss and new decoder state \ndef _loop_step(self, new_tokens, input_mask, enc_output, dec_state):\n  input_token, target_token = new_tokens[:, 0:1], new_tokens[:, 1:2]\n\n  # Run the decoder one step.\n  decoder_input = DecoderInput(new_tokens=input_token,\n                               enc_output=enc_output,\n                               mask=input_mask)\n\n  dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n  self.shape_checker(dec_result.logits, ('batch', 't1', 'logits'))\n  self.shape_checker(dec_result.attention_weights, ('batch', 't1', 's'))\n  self.shape_checker(dec_state, ('batch', 'dec_units'))\n\n  # `self.loss` returns the total for non-padded tokens\n  y = target_token\n  y_pred = dec_result.logits\n  step_loss = self.loss(y, y_pred)\n\n  return step_loss, dec_state","metadata":{"id":"9VrzgwztXzYJ","execution":{"iopub.status.busy":"2023-03-16T11:46:39.727129Z","iopub.execute_input":"2023-03-16T11:46:39.727439Z","iopub.status.idle":"2023-03-16T11:46:39.736613Z","shell.execute_reply.started":"2023-03-16T11:46:39.727406Z","shell.execute_reply":"2023-03-16T11:46:39.735354Z"},"trusted":true},"execution_count":72,"outputs":[]},{"cell_type":"code","source":"TrainTranslator._loop_step = _loop_step","metadata":{"id":"xj3I7VULrk1R","execution":{"iopub.status.busy":"2023-03-16T11:46:39.738814Z","iopub.execute_input":"2023-03-16T11:46:39.739618Z","iopub.status.idle":"2023-03-16T11:46:39.747289Z","shell.execute_reply.started":"2023-03-16T11:46:39.739581Z","shell.execute_reply":"2023-03-16T11:46:39.746497Z"},"trusted":true},"execution_count":73,"outputs":[]},{"cell_type":"code","source":"# testing the training step\ntranslator = TrainTranslator(\n    embedding_dim, units,\n    input_text_processor=input_text_processor,\n    output_text_processor=output_text_processor,\n    use_tf_function=False)\n\n# Configure the loss and optimizer\ntranslator.compile(\n    optimizer=tf.optimizers.Adam(),\n    loss=MaskedLoss(),\n)","metadata":{"id":"OA6bCske8TXm","execution":{"iopub.status.busy":"2023-03-16T11:46:39.748757Z","iopub.execute_input":"2023-03-16T11:46:39.749861Z","iopub.status.idle":"2023-03-16T11:46:39.787947Z","shell.execute_reply.started":"2023-03-16T11:46:39.749822Z","shell.execute_reply":"2023-03-16T11:46:39.786905Z"},"trusted":true},"execution_count":74,"outputs":[]},{"cell_type":"code","source":"np.log(output_text_processor.vocabulary_size())","metadata":{"id":"zHe-OudqCFGK","outputId":"f2fbae89-05d6-4eae-850b-804a97879845","execution":{"iopub.status.busy":"2023-03-16T11:46:39.789442Z","iopub.execute_input":"2023-03-16T11:46:39.789885Z","iopub.status.idle":"2023-03-16T11:46:39.798258Z","shell.execute_reply.started":"2023-03-16T11:46:39.789847Z","shell.execute_reply":"2023-03-16T11:46:39.796914Z"},"trusted":true},"execution_count":75,"outputs":[{"execution_count":75,"output_type":"execute_result","data":{"text/plain":"10.055435808808188"},"metadata":{}}]},{"cell_type":"code","source":"%%time\nfor n in range(10):\n  print(translator.train_step([example_input_batch, example_target_batch]))\nprint()","metadata":{"id":"VwMU9cFEfjha","outputId":"55585974-6d03-477b-fa1b-c040e9bd7a30","execution":{"iopub.status.busy":"2023-03-16T11:46:39.800310Z","iopub.execute_input":"2023-03-16T11:46:39.801231Z","iopub.status.idle":"2023-03-16T11:47:08.879695Z","shell.execute_reply.started":"2023-03-16T11:46:39.801193Z","shell.execute_reply":"2023-03-16T11:47:08.878578Z"},"trusted":true},"execution_count":76,"outputs":[{"name":"stdout","text":"{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=9.057506>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=9.044932>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=9.021192>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=8.952689>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=8.709905>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=7.702799>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=5.35024>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.6029773>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.344391>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.2793393>}\n\nCPU times: user 28.7 s, sys: 1.24 s, total: 30 s\nWall time: 29.1 s\n","output_type":"stream"}]},{"cell_type":"code","source":"# maximizing performance while training\n@tf.function(input_signature=[[tf.TensorSpec(dtype=tf.string, shape=[None]),\n                               tf.TensorSpec(dtype=tf.string, shape=[None])]])\ndef _tf_train_step(self, inputs):\n  return self._train_step(inputs)","metadata":{"id":"UFUsTKQx0jaH","execution":{"iopub.status.busy":"2023-03-16T11:47:08.881371Z","iopub.execute_input":"2023-03-16T11:47:08.881738Z","iopub.status.idle":"2023-03-16T11:47:08.889074Z","shell.execute_reply.started":"2023-03-16T11:47:08.881701Z","shell.execute_reply":"2023-03-16T11:47:08.886914Z"},"trusted":true},"execution_count":77,"outputs":[]},{"cell_type":"code","source":"TrainTranslator._tf_train_step = _tf_train_step","metadata":{"id":"2-bgU59jrztQ","execution":{"iopub.status.busy":"2023-03-16T11:47:08.891048Z","iopub.execute_input":"2023-03-16T11:47:08.891867Z","iopub.status.idle":"2023-03-16T11:47:08.901275Z","shell.execute_reply.started":"2023-03-16T11:47:08.891829Z","shell.execute_reply":"2023-03-16T11:47:08.900192Z"},"trusted":true},"execution_count":78,"outputs":[]},{"cell_type":"code","source":"translator.use_tf_function = True","metadata":{"id":"KC8bRv_Gr3H9","execution":{"iopub.status.busy":"2023-03-16T11:47:08.902845Z","iopub.execute_input":"2023-03-16T11:47:08.904047Z","iopub.status.idle":"2023-03-16T11:47:08.910837Z","shell.execute_reply.started":"2023-03-16T11:47:08.904005Z","shell.execute_reply":"2023-03-16T11:47:08.909469Z"},"trusted":true},"execution_count":79,"outputs":[]},{"cell_type":"code","source":"translator.train_step([example_input_batch, example_target_batch])","metadata":{"id":"pLQZsX2dp1QK","outputId":"db5ec937-a3e4-40b2-ec54-9dea9c4f3519","execution":{"iopub.status.busy":"2023-03-16T11:47:08.912266Z","iopub.execute_input":"2023-03-16T11:47:08.912796Z","iopub.status.idle":"2023-03-16T11:47:15.710605Z","shell.execute_reply.started":"2023-03-16T11:47:08.912760Z","shell.execute_reply":"2023-03-16T11:47:15.709612Z"},"trusted":true},"execution_count":80,"outputs":[{"execution_count":80,"output_type":"execute_result","data":{"text/plain":"{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.291374>}"},"metadata":{}}]},{"cell_type":"code","source":"%%time\nfor n in range(10):\n  print(translator.train_step([example_input_batch, example_target_batch]))\nprint()","metadata":{"id":"UzXXMwjXCqqh","outputId":"c536483a-a0b5-4c1b-b99b-3cc35a2caf48","execution":{"iopub.status.busy":"2023-03-16T11:47:15.712754Z","iopub.execute_input":"2023-03-16T11:47:15.713156Z","iopub.status.idle":"2023-03-16T11:47:19.076763Z","shell.execute_reply.started":"2023-03-16T11:47:15.713109Z","shell.execute_reply":"2023-03-16T11:47:19.075669Z"},"trusted":true},"execution_count":81,"outputs":[{"name":"stdout","text":"{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.2973747>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.2701573>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.210058>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.1579113>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.11467>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.070429>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.0463653>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.046787>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.040867>}\n{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.0127487>}\n\nCPU times: user 4.32 s, sys: 389 ms, total: 4.71 s\nWall time: 3.36 s\n","output_type":"stream"}]},{"cell_type":"code","source":"losses = []\nfor n in range(100):\n  print('.', end='')\n  logs = translator.train_step([example_input_batch, example_target_batch])\n  losses.append(logs['batch_loss'].numpy())\n\nprint()\nplt.plot(losses)","metadata":{"id":"U-dIWMIBqK7b","outputId":"6df08356-994b-48fa-b64c-ebcdae2c6291","execution":{"iopub.status.busy":"2023-03-16T11:47:19.087055Z","iopub.execute_input":"2023-03-16T11:47:19.087394Z","iopub.status.idle":"2023-03-16T11:47:51.493218Z","shell.execute_reply.started":"2023-03-16T11:47:19.087365Z","shell.execute_reply":"2023-03-16T11:47:51.491996Z"},"trusted":true},"execution_count":82,"outputs":[{"name":"stdout","text":"....................................................................................................\n","output_type":"stream"},{"execution_count":82,"output_type":"execute_result","data":{"text/plain":"[<matplotlib.lines.Line2D at 0x7fbd835563d0>]"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABF3UlEQVR4nO3deVhU9f4H8PeZhRm2GVlEQBYFxQ1wARdcc+2mmV7LzL3FjLK09Vba79ZdTMvqWplmlmWblqaV5Z77ggqiIqiooOwgIgzrwMyc3x8ISaKyDJyZ4f16nnme6znfmfnM9/F23p7zXQRRFEUQERERSUQmdQFERETUsjGMEBERkaQYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREklJIXUBdmEwmZGRkwNnZGYIgSF0OERER1YEoiigsLIS3tzdkstvf/7CKMJKRkQFfX1+pyyAiIqIGSE1NhY+Pz23PW0UYcXZ2BlD5YzQajcTVEBERUV3odDr4+vpWX8dvxyrCSNWjGY1GwzBCRERkZe42xIIDWImIiEhSDCNEREQkqUaFkUWLFkEQBDz//PN3bLdv3z6EhYVBrVYjICAAn376aWO+loiIiGxIg8PI8ePH8dlnnyE0NPSO7ZKTkzF69GgMGjQIsbGxmD9/PubOnYuffvqpoV9NRERENqRBYaSoqAhTp07FqlWr4OLicse2n376Kfz8/LB06VJ06dIFs2bNwuOPP4733nuvQQUTERGRbWlQGJkzZw7GjBmDESNG3LXtkSNHMGrUqBrH7r33XkRHR6OioqLW9+j1euh0uhovIiIisk31DiPr1q3DiRMnsGjRojq1z8rKQps2bWoca9OmDQwGA3Jzc2t9z6JFi6DVaqtfXPCMiIjIdtUrjKSmpmLevHn49ttvoVar6/y+v84vFkWx1uNVXn/9dRQUFFS/UlNT61MmERERWZF6LXoWExODnJwchIWFVR8zGo3Yv38/li1bBr1eD7lcXuM9np6eyMrKqnEsJycHCoUCbm5utX6PSqWCSqWqT2lERERkpeoVRoYPH464uLgaxx577DF07twZr7766i1BBAAiIiKwefPmGsd27NiB8PBwKJXKBpRMREREtqReYcTZ2RnBwcE1jjk6OsLNza36+Ouvv4709HR8/fXXAIDIyEgsW7YML774Ip588kkcOXIEX3zxBdauXWumn0BERETWzOwrsGZmZiIlJaX6z+3bt8eWLVuwd+9e9OjRA//5z3/w0Ucf4cEHHzT3VxMREZEVEsSq0aQWTKfTQavVoqCgwKwb5R1LzsPHuy/g/Ynd4aGp+4BcIiIiuru6Xr9b7N40oijinW3ncOBCLj7Zc1HqcoiIiFqsFhtGBEHAS6OCAADfH0tB2vUSiSsiIiJqmVpsGAGA/oHu6B/ohgqjiI//4N0RIiIiKbToMAIAL43qBADYcCINybnFEldDRETU8rT4MBLm74JhnT1gNIn4cFei1OUQERG1OC0+jADAiyMrx478cioDidmFEldDRETUsjCMAAhuq8V9wZ4QReCDHbw7QkRE1JwYRm54cWQQBAHYFp+FuLQCqcshIiJqMRhGbujYxhnje7QFACzZcR5WsBYcERGRTWAYucm84R2hkAnYn3gVXx66LHU5RERELQLDyE3auTtiwZguAIC3t5zF0aRrEldERERk+xhG/uLR/u0wroc3DCYRc76PRbauTOqSiIiIbBrDyF8IgoBFE0LQ2dMZuUV6PP1tDMoNJqnLIiIislkMI7VwsFNg5fQwaNQKnEjJx39+S5C6JCIiIpvFMHIb/m6OWPpIDwDAN1FX8NKPp3A+iwuiERERmZsgWsEcVp1OB61Wi4KCAmg0mmb97mW7L+C9mxZCGxzUGk8Oao8Bge7IKylHZn4ZMgtKUVBagRFd2sDF0a5Z6yMiIrJUdb1+M4zUQcyV6/j8QBK2x2fBdKO35DIBRlPNrhsT6oVPpvRq9vqIiIgsUV2v34pmrMlqhfm7IMw/DCnXSrD6UDJ+jE5FSbkRggC0dlKhtbMK8Rk67IzPRkFpBbT2SqlLJiIishq8M9IAxXoD8ksr4OGsglIugyiKuHfpfiRmF+GdB0Mwqbef1CUSERFJrq7Xbw5gbQBHlQJtW9lDKa/sPkEQMO7GUvI/x2ZIWRoREZHVYRgxk3E9vAEAUcnXkFlQKnE1RERE1oNhxEx8XBzQu50LRBH49STvjhAREdUVw4gZje9541ENwwgREVGdMYyY0ZgQLyjlAs5m6pCYzQXSiIiI6oJhxIxaOdhhSJAHAODn2HSJqyEiIrIODCNmNr5n5UDWX05mwGSy+FnTREREkmMYMbMRXdrASaVAen4poq9cl7ocIiIii8cwYmZqpRx/C/YEAPx8ko9qiIiI7oZhpAmMv7EA2u+nM7H7XDbKKowSV0RERGS5uDdNE4gIdIOnRo0sXRke/yoaaqUM/QPdMbSzBx7s1RYOdux2IiKiKrwz0gTkMgGrH+2Naf384K1Vo6zChN3ncvB/P5/BU9/ESF0eERGRReFGeU1MFEWczy7EH2dz8OGuCyg3mrA+MgK927lKXRoREVGT4kZ5FkIQBHT21GDO0A54MMwHAPDJnosSV0VERGQ5GEaaUeSQAMgEYO/5qziTXiB1OURERBaBYaQZ+bs5Ymz3ykXRVuy9JHE1REREloFhpJk9fU8gAGDLmUxculokcTVERETSYxhpZp09NRjRpQ1EEfiUd0eIiIgYRqTwzNDKuyObYtORnl8qcTVERETSYhiRQC8/F/QPdIPBJGLV/iSpyyEiIpIUw4hE5gztAABYeywFaddLJK6GiIhIOgwjEukf6Iaefq2gN5gwaWUULucWS10SERGRJBhGJCIIApZN6YV2bg5Izy/FxJVHcD6rUOqyiIiImh3DiITatrLHj5ER6OzpjKuFekz67AhOpeZLXRYREVGzYhiRmIezGutm90N331bIL6nA1M+P4mjSNanLIiIiajYMIxaglYMdvpvVFxEBbijSGzBrTTQXRCMiohaDYcRCOKkU+PKx3ujTzhWFegOeXBONgtIKqcsiIiJqcgwjFkStlGP5tF7w1qqRlFuMeetiYTSJUpdFRETUpBhGLIy7kwqfzQiHWinD3vNX8e72c1KXRERE1KQYRixQcFstljzUHQCwcl8SfjmZLnFFRERETYdhxEKN7e6NZ27s8PuPDadx+GKuxBURERE1DYYRC/bSqE4Y3tkDeoMJM1Yfw/dHU6QuiYiIyOwYRiyYXCbgk6m98EB3bxhMIuZvisO/NydwUCsREdkUhhELp1bK8eEjPfDSyCAAwOpDyXhizXEUlnHaLxER2QaGESsgCAKeG94Ry6f2qp5lM/HTI8gvKZe6NCIiokZjGLEio0O8sP6p/mjtrMK5rEI8+uVxFOsNUpdFRETUKAwjVibER4vvZvVFKwclTqbmY/Y30dAbjFKXRURE1GAMI1YoqI0zvnqsDxzt5Dh08Rrmro2FwWiSuiwiIqIGYRixUj18W2HVjHDYKWTYHp+N1zbGwcRZNkREZIUYRqxY/w7uWDa5J+QyARti0vDaxtO8Q0JERFaHYcTKjermiSUPhUImAD9Gp+Gpb2JQWs4xJEREZD0YRmzAhF4++HRaGFQKGf44l4Mpn0chr5jTfomIyDowjNiIUd088d2svtDaKxGbko+HPj2M1LwSqcsiIiK6K4YRGxLezhUbIiPgrVUj6WoxJn56hHdIiIjI4tUrjKxYsQKhoaHQaDTQaDSIiIjA1q1b7/ie7777Dt27d4eDgwO8vLzw2GOP4dq1a40qmm6vYxtn/PRMfwS4OyJLV4Z//nJG6pKIiIjuqF5hxMfHB4sXL0Z0dDSio6MxbNgwjBs3DvHx8bW2P3jwIGbMmIEnnngC8fHxWL9+PY4fP45Zs2aZpXiqnZfWHksf6QG5TMBvpzPx++lMqUsiIiK6rXqFkbFjx2L06NEICgpCUFAQFi5cCCcnJ0RFRdXaPioqCu3atcPcuXPRvn17DBw4EE899RSio6PNUjzdXqhPKzxzTyAA4P9+OYPcIr3EFREREdWuwWNGjEYj1q1bh+LiYkRERNTapn///khLS8OWLVsgiiKys7OxYcMGjBkz5o6frdfrodPparyo/p4b1hGdPZ2RV1yO//v5DESRi6IREZHlqXcYiYuLg5OTE1QqFSIjI7Fp0yZ07dq11rb9+/fHd999h0mTJsHOzg6enp5o1aoVPv744zt+x6JFi6DVaqtfvr6+9S2TANgpZHhvYncoZAK2nsnCb3xcQ0REFqjeYaRTp044efIkoqKi8PTTT2PmzJlISEiotW1CQgLmzp2Lf/7zn4iJicG2bduQnJyMyMjIO37H66+/joKCgupXampqfcukG4LbavHssA4AKh/X5BSWSVwRERFRTYLYyHv3I0aMQGBgIFauXHnLuenTp6OsrAzr16+vPnbw4EEMGjQIGRkZ8PLyqtN36HQ6aLVaFBQUQKPRNKbcFqnCaMK4ZYeQkKlDn3au+PqJPlAr5VKXRURENq6u1+9GrzMiiiL0+toHR5aUlEAmq/kVcrm8+n3UPJRyGZY+0gPOKgWOXc7DnO9OoIJ72BARkYWoVxiZP38+Dhw4gMuXLyMuLg4LFizA3r17MXXqVACVj1dmzJhR3X7s2LHYuHEjVqxYgaSkJBw6dAhz585Fnz594O3tbd5fQncU1MYZXzzau3rJ+H9sOM1dfomIyCIo6tM4Ozsb06dPR2ZmJrRaLUJDQ7Ft2zaMHDkSAJCZmYmUlJTq9o8++igKCwuxbNkyvPTSS2jVqhWGDRuGd955x7y/guqkT3tXLJ/aC7O/icGm2HRo7ZV4c2xXCIIgdWlERNSCNXrMSHPgmBHz+jk2Hc//cBIAMHtwACb0aot2bo4cR0JERGZV1+s3w0gLtebwZbz5658r5woC4OvigA4eTnh2WAf08nORsDoiIrIFdb1+1+sxDdmOmf3bwU4hw4/RqbiYU4TCMgNS8kqQkleCU6n52PPKPdColVKXSURELQDDSAs2uY8fJvfxgyiKyC0qx6WrRZi/KQ5JV4uxfM8lvHZfZ6lLJCKiFqDRU3vJ+gmCgNbOKvQLcMP8+7oAAFYfSkZqXonElRERUUvAMEI1DO/igYgAN5QbTFiy/bzU5RARUQvAMEI1CIKABWO6QBCAX09l4GRqvtQlERGRjWMYoVsEt9XiwV4+AID//pbA1XKJiKhJMYxQrV4e1Qn2Sjmir1zHtjNZUpdDREQ2jGGEauWpVePJwQEAgEVbz6GgtELiioiIyFYxjNBtPTU4AK2dVUjJK8E9S/ZgzeHL3GCPiIjMjmGEbstRpcDK6WHo4OGE6yUVePPXeNy7dD92JmRzHAkREZkNl4OnuzIYTVh7PBVLdybiWnE5AGBMiBc+fKQHFHLmWSIiql1dr9+8ktBdKeQyTO/njz2v3IOn7wmEnVyG3+My8dbmeN4hISKiRmMYoTrTqJV49W+d8eEjPSAIwLdRKVix75LUZRERkZVjGKF6uy/EC/83pisA4N1t5/HLyXSJKyIiImvGMEIN8vjA9nhiYHsAwMvrT+HwpVyJKyIiImvFMEINtmB0F4wO8USFUcRT38TgYk6R1CUREZEVYhihBpPJBHzwcA+E+7ugsMyAuWtjoTcYpS6LiIisDMMINYpaKcfyqb3g4qBEQqYO7+9IlLokIiKyMgwj1GgeGjXeeTAUAPDZ/iQcvMDxI0REVHcMI2QWo7p5YkpfPwDAS+tP4vqNxdGIiIjuhmGEzOaNMV0Q0NoR2To9Xtt4mguiERFRnTCMkNk42Cnw0SM9oZQL2B6fje+OpkhdEhERWQGGETKr4LZavDSqEwDgjZ/PYOrnUTiadE3iqoiIyJIppC6AbM/sQQHIKijDt1FXcOjiNRy6eA1927ti3vCOiAh0gyAIUpdIREQWhLv2UpNJu16CFXsv4cfoVFQYK/+aDe/sgbce6AZfVweJqyMioqZW1+s3wwg1uYz8Uny67xLWHktBhVGEWinD3OEdMWtgAOwUfFJIRGSrGEbI4lzMKcQbP59BVFIeAKCjhxMWTQhBeDtXiSsjIqKmUNfrN/9ZSs2mg4cz1j7ZDx883B1ujna4kFOERz7jAFciopaOYYSalSAImNDLB7tfugcju7aBwSTi6e9OIDWvROrSiIhIIgwjJAmtgxIfPdITIW21yCsux5NfR6NYb5C6LCIikgDDCEnG3k6Oz2aEobWzCueyCvHCDydhMln8ECYiIjIzhhGSlJfWHiunh8FOLsOOhGz8bxd3/SUiamkYRkhyvfxcsGhCCADg490XsSk2TeKKiIioOTGMkEV4MMwHswcHAABe+vEUAwkRUQvCMEIW47W/dcbkPr4wicCLP57Cj9GpUpdERETNgGGELIZMJmDh+BBM6+cHUQT+seE0vufOv0RENo8b5ZFFkckE/GdcMBQyGb46fBnzN8Uht0iPHr6t4KiSw1GlgNZeCS+tvdSlEhGRmTCMkMURBAFvju0KpVzAqgPJ+GDnrTNsHunti0UTQrgDMBGRDWAYIYskCALmj+4CT609tp3JRJHeiGK9AcV6A/JKyrHueCr6d3DHA929pS6ViIgaiRvlkdVZuisRS3ddgNZeiZ0vDIaHRi11SUREVAtulEc2a87QDghuq0FBaQVe3xgHK8jTRER0BwwjZHWUchnen9gDdnIZ/jiXgw0xXJOEiMiaMYyQVerk6YznR3YEAPx7cwIy8kslroiIiBqKYYSs1uxBAejh2wqFegNe/ek0H9cQEVkphhGyWgq5DO8/3B0qhQwHLuTin7/Ec9dfIiIrxDBCVi2wtRMW/j0EggB8E3UFr208DSMDCRGRVWEYIav3UJgP3p/YHTIB+DE6DS/+eBIGo0nqsoiIqI4YRsgmTOjlg48m94RCJuCXkxl4bm0syg0MJERE1oBhhGzG/aHeWD61F+zkMmw9k4VZX0ejoLRC6rKIiOguGEbIpozq5olVM8OhVsqwP/Eqxn9yCBdzCqUui4iI7oBhhGzOkKDW2BDZH95aNZJzizH+k8PYlZAtdVlERHQbDCNkk4LbavHrcwPRp50rivQGPPlNND7+4wKn/hIRWSCGEbJZ7k4qfDurL6b384coAu/vTMSYjw9iX+JVLpBGRGRBGEbIptkpZPjP+GC882AInNUKnM3UYebqY5j2xVHEpRVIXR4REQEQRCv4J2JdtyAmupPrxeX4ZM9FfH3kCspvrEMyOKg1BnZwQ0SAO7p6ayCXCRJXSURkO+p6/WYYoRYnNa8EH+xMxM8n03Hz336NWoF+AW6YN6IjunlrpSuQiMhGMIwQ3cWlq0XYcy4HRy5dw7HkPBTqDQAqH+28ObYrpvTxgyDwTgkRUUMxjBDVg8FoQnyGDh/9cQF/nMsBAIzt7o1FE0LgpFJIXB0RkXWq6/WbA1iJULkDcHffVlg1IxzzR3eGXCZg86kMjP34IBIydFKXR0Rk0xhGiG4ikwmYPTgQPz7V789F05YfwuqDyVyjhIioiTCMENUizN8Vv88dhOGdPVBuMOHfvyVg5pfHkKMrk7o0IiKbwzBCdBsujnb4fGY4/jM+GCqFDAcu5OLepfuxPT5L6tKIiGwKwwjRHQiCgOn9/PH73IHo5q3B9ZIKPPVNDD4/kCR1aURENqNeYWTFihUIDQ2FRqOBRqNBREQEtm7desf36PV6LFiwAP7+/lCpVAgMDMTq1asbVTRRc+vg4YxNzwzArIHtAQBLtp9HyrUSiasiIrIN9Zqz6OPjg8WLF6NDhw4AgDVr1mDcuHGIjY1Ft27dan3Pww8/jOzsbHzxxRfo0KEDcnJyYDAYGl85UTOzU8iwYEwXnM3S4dDFa3jz1zNY/WhvrkVCRNRIjV5nxNXVFUuWLMETTzxxy7lt27bhkUceQVJSElxdXRv8HVxnhCzJpatF+NvS/agwivh0Whj+FuwpdUlERBapydcZMRqNWLduHYqLixEREVFrm19//RXh4eF499130bZtWwQFBeHll19GaWnpHT9br9dDp9PVeBFZisDWTnhqcCAA4N+b41Gs550+IqLGqHcYiYuLg5OTE1QqFSIjI7Fp0yZ07dq11rZJSUk4ePAgzpw5g02bNmHp0qXYsGED5syZc8fvWLRoEbRabfXL19e3vmUSNalnh3WAr6s9MgrK8NEfF2qcyy3S48fjqThw4SoqbmzIR0REt1fvxzTl5eVISUlBfn4+fvrpJ3z++efYt29frYFk1KhROHDgALKysqDVVm48tnHjRjz00EMoLi6Gvb19rd+h1+uh1+ur/6zT6eDr68vHNGRRdp/LxuNfRUMhE/D73EEo0hvwzZHL2BKXVb0rsEatwPAubXBvtzYYHNQaDnZcWp6IWo5m25tmxIgRCAwMxMqVK285N3PmTBw6dAgXL16sPnb27Fl07doViYmJ6NixY52+g2NGyFI99U00tsdnw14pR2mFsfp4Vy8NcgrLkFtUXn3M3ckO6yP7o727oxSlEhE1u2bbm0YUxRp3MW42YMAAZGRkoKioqPpYYmIiZDIZfHx8GvvVRJL759hu1UHETiHDQ2E++PXZAdgybxCOzh+B9ZERmDWwPTw1auQWlWPZ7ot3/1AiohamXndG5s+fj/vuuw++vr4oLCzEunXrsHjxYmzbtg0jR47E66+/jvT0dHz99dcAgKKiInTp0gX9+vXDv/71L+Tm5mLWrFkYMmQIVq1aVecieWeELFnMlTyczSzEmBAvuDja1drmZGo+xn9yCAqZgH3/GIq2rWp/RElEZEua5M5IdnY2pk+fjk6dOmH48OE4evRodRABgMzMTKSkpFS3d3Jyws6dO5Gfn4/w8HBMnToVY8eOxUcffdTAn0VkecL8XTGtn/9tgwgA9PBthYgANxhMIr44kNyM1RERWb5GjxlpDrwzQrZgX+JVzFx9DPZKOQ6/NuyO4YWIyBY025gRIqqbwR3d0dVLg9IKI74+ckXqcoiILAbDCFEzEQQBkfdULpb21eFklJRzsTQiIoBhhKhZjQ72hK+rPa6XVODH46lSl0NEZBEYRoiakUIuw+wbS8mvOpBcvUJrWYURBy/kYn10KvKKy+/0EURENofLQRI1s4lhPvhwVyLS80vxyvpTyNKV4cSV/OpVW+2Vckzq7YsnBwdwCjARtQi8M0LUzNRKOR7t3w4A8PPJDEQl5aHcaIKnRo2gNk4orTDiq8OXMeTdPXjxh5NIulp05w8kIrJyvDNCJIGZ/dshPqNyN+r+HdwxINCtepn4gxdzsWLvJRy+dA0bY9OxIyEb62b3Q3BbrZQlExE1Ga4zQmShTqXm463N8YhNyYe7kx02RPZHO+5rQ0RWhOuMEFm57r6tsObxPujipUFuUTlmrD6GHF2Z1GUREZkdwwiRBdOolVjzeG/4uTogJa8EM788Dl1ZhdRlERGZFcMIkYXzcFbjmyf6wN1JhbOZOsxaE42yCqPUZRERmQ3DCJEV8HdzxJrHe8NZpcCx5Dz8a3O81CUREZkNwwiRlejmrcWKaWEQBGDtsVTsiM+SuiQiIrNgGCGyIgM7umP2oAAAwGsb45BTyAGtRGT9GEaIrMyLo4LQ1UuDvOJyvLL+NKxgdj4R0R0xjBBZGZVCjg8f6QGVQoZ9iVfx9ZEr1efyS8rx6b5LmLTyCA5cuCphlU2jpNyAzIJSqcsgIjPjCqxEVqhjG2e8fl9nvLU5AW9vOYs2GjX2JeZgU2w6yioq97i59MMp7Hl5CJzVSomrNZ+X15/CzoRsbJk7CB3bOEtdDhGZCe+MEFmpmf3bYUhQa+gNJkR+G4O1x1JRVmFCVy8NfFzskVukx9JdF6Qu02xMJhF7z19FhVHE/gu5UpdDRGbEMEJkpQRBwJKHQuHhrIJcJmBMiBfWR0bg97kDsfDvIQCArw5fRmJ2ocSVmseVvBKUlFeur3ImvUDiaojInPiYhsiKeWjU2PniEBhNIlwd7aqPDwlqjXu7tcH2+Gz885czWPtkPwiCIGGljXc2U1f9v+MYRohsCu+MEFk5rb2yRhCp8saYrlApZIhKysPm05kSVGZeCRl/hpFLV4tQpDdIWA0RmRPDCJGN8nV1wJyhHQAAC39PQLGVX7xvvjMiijXDCRFZN4YRIhs2e3AA/FwdkK3T46Pd1j2YNeFGGHF3UgHgoxoiW8IwQmTD1Eo53hzbFQDw2f4kzFpzHPsTr8Jksq6F0vJLypFZULna7IRebQEAcWn5ElZERObEMEJk44Z3aYPp/fwhisCuszmYsfoYRvxvH746lGw14y6q7or4uTogIsANAO+MENkShhGiFuA/44Ox+6UheLR/OzipFEi6Woy3Nidg0Du7sWLvJYsfT1I1PqSLlzOC22oBAEm5xVYTpojozhhGiFqIgNZOeOuBboiaPxz/GR+MAHdHXC+pwDvbzmHQu3vw2f5LKL2xjoelOZtZuVZKVy8tWjur4KVVQxSBeN4dIbIJDCNELYyTSoHp/fyx44XB+ODh7vB3c0BecTne3nIOg5fsweGLlre6adVjmi5elUvAV90d4aMaItvAMELUQinkMkzo5YM/XhyCJQ+FwtfVHlcL9Zj2xVEs33vRYga5lhtMuJhz486ItwYAEMowQmRTGEaIWjiFXIaJ4b7Y+cIQPBzuA5MIvLvtPGZ/E4OC0gqpy8PFnCJUGEVo1Aq0bWUPAAj2YRghsiUMI0QEoHIa8LsPdcfiCSGwU8iw62w2Hlh2EHFp0l7wqxY76+ylqV7SPuTGnZHk3GIUlt09MOkNRhgt5E4PEd2KYYSIanikjx9+iuyPtq3sceVaCcZ9chCvb4xDbpFeknqqxot09dJUH3N3UsG7ahDrXVZizS8px+B392Dq51FNWicRNRzDCBHdIsRHi9+eG4j7Q71gEoG1x1IwdMlerNx3CXpD8864OVtLGAH+HMR6tx189yVeRbZOj6ikPIt47EREt2IYIaJauTjaYdmUXlgfGYGQtloU6g1YtPUc7v3ffpzLap59YURR/PPOiHfNMBJSx0GsBy/8OTuoaiAsEVkWhhEiuqPe7Vzxy5wBeG9id3g4q3D5WgmmrDraLIEkS1eG/JIKyGUCOng41TgXUjWI9Q5jWkRRxMGbpiqfzypqmkKJqFEYRojormQyAQ+F+WDnC0MQ6qNFXnE5pqw6WmMn3aZQtfJqh9ZOUCvlNc6F3LQS6+0GsV66Wly9pw0AJGbzzgiRJWIYIaI60zoo8c0TfasDydTPmzaQnP3LYmc3c7sxiBW4/SDWgxeuAgBklZNwGEaILBTDCBHVi9a+ZiCZsiqqyab/Vi0D3+Uvg1er3O1RTdUjmlFdPQEAidl8TENkiRhGiKjebg4k10sqMHbZQYz56ACW7b5g1kGitxu8WuVOg1grjCZEJeUBAB4d0A4AkFukR15xudnqIyLzYBghogapCiQjunhAJlQ+KnlvRyJGfLAfIz7Yh6NJ1xr1+cV6Ay5fKwZw+zsjoT6tAACHLubessnfqdR8FOkNcHFQok87V/i4VK7eykc1RJaHYYSIGkxrr8TnM3vj+IIReOfBEAzt1Bp2chku5hRhxupj2H0uu8GffS6rEKIIeDir4O6kqrVNRKAb2rayx7Xicqw9llLj3IEbU3r7d3CHTCagU5vKcScXbhNGCssqoKvDaq5EZH4MI0TUaG5OKkzq7YcvH+uD6P8bgRFd2kBvMGH21zHYfCqj3p8niiI+3XcJABB6Y1xIbZRyGZ4ZGggAWLn/Esoq/rw7UjVeZFAHdwBAxxth5HwtYURvMOK+Dw/gvqUHbrnDQkRNj2GEiMxKo1ZixbReGNfDGwaTiLnrYrHuL3ct7uaH46nYmZANpVzACyOD7tj2oTAfeGnVyNbpsT4mDQCgK6vAydR8AMDAjpVhpJNn5ToltQ1ijbl8HWnXS5GeX4r9N2bgEFHzYRghIrNTymX44OEemNLXD6IIvLYxDkt3JdZpU7vk3GL8a3MCAODlUZ3Qzfv2d0YAQKWQI3JI5d2RFXsuotxgQtSlazCaRLR3d4SPiwMAoKNH5Z2RxOxCiGLNTfMO3LQw2vb4rLr/UCIyC4YRImoScpmAheOD8dSQAADA0l0X0GfhH3hl/SnEXMm7JRAAlTNgnv/hJEorjOgX4IpZgwLq9F2TevuitbMKGQVl2HgirfoRzcAbj2gAoIOHE2QCkF9Sgat/2fTv5iXj/zibgwqjqd6/l4gajmGEiJqMIAh47W+d8c6DIejg4YTSCiPWx6ThwRVHMPJ/+/HJnou4nFtc3X7Z7os4lZoPZ7UCHzzcA/Kq1cruQq2U46nBlcHlk70XsT+x8lFL1SOaqjb+bo4AgMSbloW/XlyOMxmVU4OdVQoUlFbgWHJe4344EdULwwgRNSlBEDCptx92vjAYPz0dgYlhPrBXynExpwhLtp/HPe/txf0fH8CirWexbM9FAMDCv4fAu5V9vb5nSl8/uDnaITWvFJevlUAuExAR6FajTVCbqnEjfw5iPXQpF6IIdGrjjNEhXgD4qIaouTGMEFGzEAQBYf6uWDKxO44tGI53HwzFoI7ukMsEnEnXYeW+JBhNIsb38MYD3b3r/fkOdooaj3W6+2ihUStrtAlq8+e4kSpVj2gGdnTHvcFtAFSGEZPp1sdIRNQ0FFIXQEQtj7NaiYd7++Lh3r64VqTH9vhsbInLhCAA/xoX3ODPnR7hj5X7LyG/pAIDO7a+5fxfw4goitXrkQzs6I6IADc42smRrdPjVFo+evq5NLgWIqo7hhEikpSbkwpT+vphSl+/Rn+Wk0qBheND8PWRy5jcx/eW80HVC58VQRRFJOcWIz2/FHZyGfq2d4VaKcfQzh747XQmtsdnM4wQNRM+piEimzIm1As/PBUBL+2tY07auztCIRNQqDcgs6CsetZNmL8LHOwq/212b7fKTfW2x2fVOuOHiMyPYYSIWgw7hQzt3Stn1JzPLqzxiKbKPTeWtE/OLcaFnLvv8nsqNb9O66cQ0e0xjBBRixLkWfmo5mymDlGXKjfzG3RTGHFWKzGgQ+UsnO1n7jyrZuOJNIz75BDe/CW+iaolahkYRoioRQm6sRLrhpg0FOoNaOWgvGWV178F33hUk3D7MGIwmvDRHxcAAHvO5/CRDlEjMIwQUYtStUdN0tXKxdYGBLrfsrjaiC5tIBOAM+k6pF0vqfVzfo/LxOVrleeul1Tg0tXiWtsR0d0xjBBRi1K1e2+Vm8eLVHFzUiG8nSsA4PfTmbecN5lEfHJjgTbhRo6JvsxVW4kaimGEiFoUf1cH2Cn+/E/fzfvX3Kxq4bUP/7iAizmFNc7tPJuNxOwiOKsUmN7PHwBw/PL1JqqYyPYxjBBRi6KQyxDYuvJRTXt3R/i6OtTabnIfP0QEuKGk3Iinvz2BknIDgMqF0pbtrrwrMqO/P4Z19gAARF/hnRGihmIYIaIWp/ONGTW3uysCVO46/OHkHvBwVuFCThHe2HQGoihi/4VcxKUXwF4px+MD2qOXvwsEAbhyrQQ5hWXN9ROIbArDCBG1OE/fE4gJvdrimaGBd2zn4azGx5N7Qi4TsDE2HeuOp2LZ7soZNFP6+sHNSQWNWonOnhoAQAwf1RA1CMMIEbU4QW2c8cHDPWpdpfWv+ga44eVRnQAA//fzGRy/fB12chlmD/5zU75w/8pl4zluhKhhGEaIiO7iqcEBGNHFA4YbO/lODPdBG426+nx4u8owwnEjRA3DMEJEdBcymYD3J/ZAOzcHOKkUiBxS8/FO7xvTgOMzdCjWG6Qokciq1SuMrFixAqGhodBoNNBoNIiIiMDWrVvr9N5Dhw5BoVCgR48eDamTiEhSWgclfp87CPv/MfSWGTjerezRtpU9jCYRJ1PzpSmQyIrVK4z4+Phg8eLFiI6ORnR0NIYNG4Zx48YhPv7O+zIUFBRgxowZGD58eKOKJSKSkqNKAVdHu1rPVT2qOc7Fz4jqrV5hZOzYsRg9ejSCgoIQFBSEhQsXwsnJCVFRUXd831NPPYUpU6YgIiKiUcUSEVmqqhVbY67cOojVaOK+NUR30uAxI0ajEevWrUNxcfEdQ8aXX36JS5cu4c0336zzZ+v1euh0uhovIiJLVjWj5sSV6zAYTdXHY65cR9+3d2HWmmiUVRilKo/IotU7jMTFxcHJyQkqlQqRkZHYtGkTunbtWmvbCxcu4LXXXsN3330HhUJR5+9YtGgRtFpt9cvX17e+ZRIRNaugNs5wVitQXG7EuazK5eOTc4sxa81x5BaVY9fZbMz57gQqbgoqRFSp3mGkU6dOOHnyJKKiovD0009j5syZSEhIuKWd0WjElClT8K9//QtBQUH1+o7XX38dBQUF1a/U1NT6lklE1KzkMgFh/n+OG8krLsdjXx7D9ZIKdPRwgkohwx/ncvD8Dyf52IboLwRRFBv1/4oRI0YgMDAQK1eurHE8Pz8fLi4ukMvl1cdMJhNEUYRcLseOHTswbNiwOn2HTqeDVqtFQUEBNBpNY8olImoyn+y5iCXbz2N4Zw9cLynHiZR8+LjYY9MzA3AmowCzv45GhVHExDAfvPNgKGQyQeqSiZpUXa/fdX92chuiKEKv199yXKPRIC4ursax5cuXY/fu3diwYQPat2/f2K8mIrIoVeNG/jiXAwDQqBX46rHeaO2swtBOHvh4ck/M+T4W62PS4KhS4M2xXSEIDCRE9Qoj8+fPx3333QdfX18UFhZi3bp12Lt3L7Zt2wag8vFKeno6vv76a8hkMgQHB9d4v4eHB9Rq9S3HiYhsQXffVlDKBVQYRdjJZfhsRjg6eDhXn/9bsBfem2jEiz+ewleHLyOwtSOmR7STrmAiC1GvMSPZ2dmYPn06OnXqhOHDh+Po0aPYtm0bRo4cCQDIzMxESkpKkxRKRGTp1Eo5BnVsDZkALJkYin4Bbre0+XtPH8y/rwsA4O0t53DlWnFzl0lkcRo9ZqQ5cMwIEVmL0nIj8krK0bbV7TfhM5lETP38KI4kXUPvdi5YNzsCco4fIRtU1+s396YhIjIjezv5HYMIULnXzbsPhcLRTo7jl6/jy0PJt7RJulqErw4lc68bahEYRoiIJODr6oD/u79yjaZ3t5/HxZzKtUkMRhOW772Iv314AG9tTsA/f7nzdhtEtoBhhIhIIpN6++KeTq1RbjDhpR9PIS6tAOOXH8K7286j3FC5ONrG2DQkZHAVarJtDCNERBIRBAGLJ4RCo1bgVFoBxi47iDPpOmjtlXh/YnfcH+oFUQQWbT0rdalETYphhIhIQp5aNf41rlv1n+8L9sTOFwfjwTAf/OPezlDKBRy4kIv9iVclrJKoaTV60TMiImqc8T3aQiGTwVmtwD2dPKqP+7k5YEZEO3xxMBlvbzmLAR3cOeuGbBLvjBARSUwQBIzt7l0jiFR5dmgHOKsVOJdViI0n0iSojqjpMYwQEVkwF0c7PDu0AwDg/R2JKKswSlwRkfkxjBARWbiZ/duhbSt7ZOnK8MXBW9ckIbJ2DCNERBZOrZTj5XuDAABLtp/HpJVHsD46lQuikc3gcvBERFbAZBLxyobT2Bibhqr/ajvYyfG3YE/08G0FX1cH+Lk6oG0re6iVcmmLJbqhrtdvhhEiIiuSkV+KTbHp2BCThuTc2jfZ6x/ohpXTw+CsVjZzdUQ1MYwQEdkwURRxIuU6tsdn43JuMVLySpCaV4Li8soBrrMHB2D+6C4SV0ktXV2v31xnhIjICgmCgDB/V4T5u1YfE0URu87m4Mmvo7H6YDIeDvdFBw8nCaskqhsOYCUishGCIGBk1zYY0cUDBpOIf22OhxXc/CZiGCEisjVvjOkKO7kMBy7kYkdCttTlEN0VwwgRkY1p5+6IJwe3BwD857eEGgulVRhNWHcsBR/sOM8F1MhicMwIEZENmjO0AzaeSEfa9VKs3JeEucM7YEtcFt7bcb56Fk6h3oA3x3a7yycRNT3OpiEislGbT2XgubWxUClk6OTpjNNpBQCAVg5K5JdUAAC+frwPBge1lrJMsmF1vX7zMQ0RkY26P9QL/QJcoTeYcDqtAI52cjw/oiMOvjoM0/v5AwBeXn8K14vLJa6UWjqGESIiGyUIAt7+ewjC/V3waP922PePoXh+RBCcVArMH90FAa0dkVOox/xNcZx1Q5LiYxoiohYqLq0Af19+CAaTiPcmdsdDYT5Sl0Q2ho9piIjojkJ8tHhhZOUGfG/+cgYp10okrohaKoYRIqIWLHJIIML9XVBcbsTkVVHYGpfJRzbU7BhGiIhaMLlMwP8m9UDbVvZIzy/F09+dwORVUTibqZO6NGpBOGaEiIhQUm7Ap/uSsHLfJegNJsgEYFJvXzzYywc9fFtBIee/Xan+uGsvERHVW9r1Eizaeg6/n86sPqa1V2JQR3cM7eSBUd3awFmtlLBCsiYMI0RE1GBHk67h26Mp2J94FQWlFdXH/d0c8MucAWjlYCdhdWQt6nr95nLwRER0i74Bbugb4AaD0YRTafnYc+4q1sek4sq1Erz04ymsmhEOmUyQukyyEXwISEREt6WQyxDm74qX7+2E1Y/2hp1Chj/O5eDT/ZekLo1sCMMIERHVSTdvLf79QOXGeu9tP48jl65JXBHZCoYRIiKqs6oZNiYReG5tLHJ0ZVKXRDaAY0aIiKjOBEHAf8cHIz6jAOeyCvHs97GYHuGP/JJyXC+pQH5JBbp4OWNiuK/UpZIVYRghIqJ6sbeTY/nUXnhg2SEcu5yHY5fzbmljEkVM6u0nQXVkjRhGiIio3gJaO+HjyT2xdFci1Eo5XBzs0MpBiYLSCmw9k4U3fj6Ddm6O6BvgJnWpZAW4zggREZmNySTiubWx+D0uE66OdvhlzgD4ujpIXRZJhLv2EhFRs5PJBLw3sTtC2mqRV1yOJ9YcR2FZxd3fSC0awwgREZmVvZ0cq2aEw8NZhcTsIsxbdxJGk8XfhCcJ8TENERE1iVOp+Xh45RHoDSa4O9mhb3s39A1wRb8AN3T0cIIgcAVXW8e9aYiISHJb4zLx8vpTKC431jge4O6IVTPDEdjaSaLKqDkwjBARkUXQG4w4lVqAqKRrOJp8DTFXrqOswgQ3RzusebwPgttqpS6RmgjDCBERWaRrRXrMWH0M8Rk6OKsU+OLR3ujT3lXqsqgJcDYNERFZJDcnFdbO7oc+7VxRqDdgxuqj2HM+R+qySEIMI0RE1Ow0aiXWPN4HQzu1RlmFCU+uicYXB5NRbjBJXRpJgGGEiIgkYW8nx2czwvFAd28YTCL+81sCRv5vHzafyoCJU4FbFIYRIiKSjFIuw9JJPfDf8cFwd1LhyrUSPLc2FuM+OYTDF3OlLo+aCQewEhGRRSjWG/DFwWSs3Hepeirwk4Pa49W/dYZCzn87WyMOYCUiIqviqFJg7vCO2PePoZjWr3LH31UHkjH186PIKSyTuDpqSgwjRERkUdydVPjv+BCsmNoLTioFjibn4f6PDiL6cp7UpVETYRghIiKLdF+IF355dgA6ejghp1CPRz6LwrLdF1BWYbz7m8mqMIwQEZHFCmzthJ/nDMD9oV4wmES8tyMRw9+vnHFjBUMeqY4YRoiIyKI5qhT4eHJPfPBwd3hq1EjPL8Vza2MxYcVhxFy5LnV5ZAacTUNERFajtNyIVQeS8Om+Syi5MePmqcEBePneTlByxo3F4WwaIiKyOfZ2cswd3hF7X74HE8N8AAAr9ydhyqooZBVwxo21YhghIiKr46FRY8nE7lh+Y8bN8cvXMfqjAzhw4arUpVEDMIwQEZHVGh3ihd+eG4iuXhrkFZdjxupjWLT1LIr0BqlLo3pgGCEiIqvWzt0RG5/pj8l9/CCKwMp9SbhnyR58d/QKDEZuvGcNOICViIhsxo74LLy95SwuXysBAHT0cML80V0wtLOHxJW1THW9fjOMEBGRTSk3mPD90Sv48I8LuF5SAQCYFO6Lf4/vBpVCLnF1LQtn0xARUYtkp5Dh0QHtsfeVoZg9OAAyAfghOhWTP4tCjo4zbiwRwwgREdkkrb0S80d3wZeP9YFGrcCJlHyMXXYQsSlcKM3SMIwQEZFNGxLUGr8+OxAdPZyQrdNj0soorI9OlbosugnDCBER2bx27o7YNGcARnZtg3KjCa9sOI2FvyfAaLL4YZMtQr3CyIoVKxAaGgqNRgONRoOIiAhs3br1tu03btyIkSNHonXr1tXtt2/f3uiiiYiI6stJpcDKaWGYO6wDAGDVgWTMWnMchWUVEldG9QojPj4+WLx4MaKjoxEdHY1hw4Zh3LhxiI+Pr7X9/v37MXLkSGzZsgUxMTEYOnQoxo4di9jYWLMUT0REVB8ymYAXR3XCx5N7QqWQYc/5q5iw/DBSbkwFJmk0emqvq6srlixZgieeeKJO7bt164ZJkybhn//8Z52/g1N7iYjI3E6n5ePJr6ORrdPDxUGJT6eFoW+Am9Rl2ZQmn9prNBqxbt06FBcXIyIiok7vMZlMKCwshKura0O/loiIyCxCfVrhlzkDEeqjxfWSCkz74ig2xaZJXVaLVO8wEhcXBycnJ6hUKkRGRmLTpk3o2rVrnd77/vvvo7i4GA8//PAd2+n1euh0uhovIiIic/PUqvHjUxEYHeKJCqOIF344haW7EmEF64HalHqHkU6dOuHkyZOIiorC008/jZkzZyIhIeGu71u7di3eeust/PDDD/DwuPOyvIsWLYJWq61++fr61rdMIiKiOlEr5Vg2uRcihwQCAJbuuoCXfjwFvcEocWUtR6PHjIwYMQKBgYFYuXLlbdv88MMPeOyxx7B+/XqMGTPmrp+p1+uh1+ur/6zT6eDr68sxI0RE1KTWHkvBGz+fgdEkItzfBQv/HoJOns5Sl2W1mm05eFEUawSHv1q7di0effRRfP/993UKIgCgUqmqpw9XvYiIiJra5D5++Oqx3nBWKRB95Tru+3A/FmyKQ27R7a9z1Hj1CiPz58/HgQMHcPnyZcTFxWHBggXYu3cvpk6dCgB4/fXXMWPGjOr2a9euxYwZM/D++++jX79+yMrKQlZWFgoKCsz7K4iIiMxkUMfW+H3uINwX7AmTCHx3NAX3LNmLFXsvoayCj26aQr3CSHZ2NqZPn45OnTph+PDhOHr0KLZt24aRI0cCADIzM5GSklLdfuXKlTAYDJgzZw68vLyqX/PmzTPvryAiIjIjPzcHrJgWhh9m90NIWy2K9Aa8s+0cxn58EGfS+Q9qc2v0mJHmwHVGiIhIKiaTiJ9PpuPtLeeQW6SHQibg+REdETkkEAo5d1W5k2YbM0JERGTLZDIBE3r5YMcLg3FfsCcMJhHv7UjEwyuP4HJusdTl2QSGESIiojpwdbTD8qm98MHD3eGsUuBESj5G/m8fZq2Jxq+nMlBSbmjQ567Yewmhb23H4Yu5Zq7YevAxDRERUT2l55fi1Q2ncfCmAGGvlGNE1zYYE+KFezq1hlopv+vnbIhJw8vrTwEAwv1dsOHp/k1WsxTqev1mGCEiImqg81mF2HwqA7+eykBK3p+b7TnYyTG0swdGB1cGE0eV4pb3HrqYi5mrj8Fg+vMyvOmZ/ujp59IstTcHhhEiIqJmIooiTqcVYPOpDGw9k4X0/NLqc/ZKOab188PswYFo7awCACRmF+LBFYdRWGbA2O7eUMoEbIxNx/2hXlg2pZdUP8PsGEaIiIgkUBVMtp7JwtYzmbhyrfKOiVopw9S+/pjQqy1mfx2D9PxS9G7ngm+e6Iukq8UY/dEByGUC9r1yD3xcHCT+FebBMEJERCQxURSxN/EqPtx1ASdT82uca+/uiI1P94eLox0AYOrnUTh08RqeHNQeC8bUbQNaS8epvURERBITBAFDO3lg0zP9sebxPujl1woA4OKgxJeP9q4OIgAwa2AAAGDdsVQUllVIUa5kbh1RQ0RERGYlCAKGBLXG4I7uOJ1WgDYaNTy16hpthgS1RmBrR1y6Wowfo9PwxMD2ElXb/HhnhIiIqJkIgoDuvq1uCSJA5eJqT9y4O/LloWQYjKbmLk8yDCNEREQWYkKvtnBxUCLteil2JGRLXU6zYRghIiKyEGqlHNP7+QMAVh1Ikria5sMwQkREZEGmRfjDTi5DbEo+TqRcl7qcZsEwQkREZEE8nNUY290bALD6YLLE1TQPhhEiIiIL8/jAdgCArWeykHHTaq62imGEiIjIwnTz1qJfgCuMJhFrjlyWupwmxzBCRERkgaqm+a49moKScoPE1TQthhEiIiILNLyzB/zdHKArM+CnmDSpy2lSDCNEREQWSCYT8Fj/dgCA1Ycuw2Sy+K3kGoxhhIiIyEJNDPeFs1qB5Nxi7E3MkbqcJsMwQkREZKEcVQo80tsXAPCFDU/zZRghIiKyYDP7t4NMAA5dvIbTaflSl9MkGEaIiIgsmI+LA+4L9gIATFh+GHPXxuJUar60RZmZQuoCiIiI6M7euL8LrhXrEZWUh19PZeDXUxkI93fBuB7ecFYroVbKYW8nh4OdHO5OKnhq1LC3k0tddp0Joiha/PBcnU4HrVaLgoICaDQaqcshIiKSxJn0Aqw+mIzNpzNQYbzz5Vtrr4SXVo2u3hq8/fcQqJXNH07qev1mGCEiIrIyOboyfHs0BQkZBSirMKG0wojSciNKyg3IKdSjpNxYo/3Cvwdjal//Zq+zrtdvPqYhIiKyMh4aNV4cGVTrOVEUUag3IKugDD/FpGHl/iR8c+QKpvTxgyAIzVxp3XAAKxERkQ0RBAEatRJBbZzxzD0doFbKcC6rENFXrktd2m0xjBAREdkorYMS47q3BQB8c+SKxNXcHsMIERGRDZseUTlWZOuZTFwt1EtcTe0YRoiIiGxYcFstevq1QoVRxA/HU6Qup1YMI0RERDZuer/KuyPfH02BwWiSuJpbMYwQERHZuNEhXnB1tENGQRn+OGd5G+4xjBAREdk4tVKOh8MrN9z7NsryBrIyjBAREbUAU/v6QRCAAxdykXS1SOpyamAYISIiagF8XR0wrJMHAODj3RctauwIwwgREVEL8diA9gCATbHpeOSzKKTmlUhcUSWGESIiohZiYEd3/G9SdzipFIi+ch33fXgAP8WkQept6hhGiIiIWpC/9/TB1nmDEO7vgiK9AS+tP4Vn18aioKRCspoYRoiIiFoYX1cHrJvdDy+PCoJCJuD305n44mCSZPVw114iIqIWSCGX4dlhHTGoY2ss33sRzwztIF0tkn0zERERSa67byusnB4uaQ18TENERESSYhghIiIiSTGMEBERkaQYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREkmIYISIiIkkxjBAREZGkGEaIiIhIUgwjREREJCmr2LVXFEUAgE6nk7gSIiIiqquq63bVdfx2rCKMFBYWAgB8fX0lroSIiIjqq7CwEFqt9rbnBfFuccUCmEwmZGRkwNnZGYIgmO1zdTodfH19kZqaCo1GY7bPpVuxr5sX+7v5sK+bD/u6+Zirr0VRRGFhIby9vSGT3X5kiFXcGZHJZPDx8Wmyz9doNPyL3UzY182L/d182NfNh33dfMzR13e6I1KFA1iJiIhIUgwjREREJKkWHUZUKhXefPNNqFQqqUuxeezr5sX+bj7s6+bDvm4+zd3XVjGAlYiIiGxXi74zQkRERNJjGCEiIiJJMYwQERGRpBhGiIiISFItOowsX74c7du3h1qtRlhYGA4cOCB1SVZv0aJF6N27N5ydneHh4YHx48fj/PnzNdqIooi33noL3t7esLe3xz333IP4+HiJKrYNixYtgiAIeP7556uPsZ/NKz09HdOmTYObmxscHBzQo0cPxMTEVJ9nf5uHwWDAG2+8gfbt28Pe3h4BAQH497//DZPJVN2Gfd0w+/fvx9ixY+Ht7Q1BEPDzzz/XOF+XftXr9Xjuuefg7u4OR0dHPPDAA0hLS2t8cWILtW7dOlGpVIqrVq0SExISxHnz5omOjo7ilStXpC7Nqt17773il19+KZ45c0Y8efKkOGbMGNHPz08sKiqqbrN48WLR2dlZ/Omnn8S4uDhx0qRJopeXl6jT6SSs3HodO3ZMbNeunRgaGirOmzev+jj72Xzy8vJEf39/8dFHHxWPHj0qJicni7t27RIvXrxY3Yb9bR7//e9/RTc3N/G3334Tk5OTxfXr14tOTk7i0qVLq9uwrxtmy5Yt4oIFC8SffvpJBCBu2rSpxvm69GtkZKTYtm1bcefOneKJEyfEoUOHit27dxcNBkOjamuxYaRPnz5iZGRkjWOdO3cWX3vtNYkqsk05OTkiAHHfvn2iKIqiyWQSPT09xcWLF1e3KSsrE7Varfjpp59KVabVKiwsFDt27Cju3LlTHDJkSHUYYT+b16uvvioOHDjwtufZ3+YzZswY8fHHH69xbMKECeK0adNEUWRfm8tfw0hd+jU/P19UKpXiunXrqtukp6eLMplM3LZtW6PqaZGPacrLyxETE4NRo0bVOD5q1CgcPnxYoqpsU0FBAQDA1dUVAJCcnIysrKwafa9SqTBkyBD2fQPMmTMHY8aMwYgRI2ocZz+b16+//orw8HBMnDgRHh4e6NmzJ1atWlV9nv1tPgMHDsQff/yBxMREAMCpU6dw8OBBjB49GgD7uqnUpV9jYmJQUVFRo423tzeCg4Mb3fdWsVGeueXm5sJoNKJNmzY1jrdp0wZZWVkSVWV7RFHEiy++iIEDByI4OBgAqvu3tr6/cuVKs9dozdatW4cTJ07g+PHjt5xjP5tXUlISVqxYgRdffBHz58/HsWPHMHfuXKhUKsyYMYP9bUavvvoqCgoK0LlzZ8jlchiNRixcuBCTJ08GwL/bTaUu/ZqVlQU7Ozu4uLjc0qax184WGUaqCIJQ48+iKN5yjBru2WefxenTp3Hw4MFbzrHvGyc1NRXz5s3Djh07oFarb9uO/WweJpMJ4eHhePvttwEAPXv2RHx8PFasWIEZM2ZUt2N/N94PP/yAb7/9Ft9//z26deuGkydP4vnnn4e3tzdmzpxZ3Y593TQa0q/m6PsW+ZjG3d0dcrn8liSXk5NzSyqkhnnuuefw66+/Ys+ePfDx8ak+7unpCQDs+0aKiYlBTk4OwsLCoFAooFAosG/fPnz00UdQKBTVfcl+Ng8vLy907dq1xrEuXbogJSUFAP9em9Mrr7yC1157DY888ghCQkIwffp0vPDCC1i0aBEA9nVTqUu/enp6ory8HNevX79tm4ZqkWHEzs4OYWFh2LlzZ43jO3fuRP/+/SWqyjaIoohnn30WGzduxO7du9G+ffsa59u3bw9PT88afV9eXo59+/ax7+th+PDhiIuLw8mTJ6tf4eHhmDp1Kk6ePImAgAD2sxkNGDDglinqiYmJ8Pf3B8C/1+ZUUlICmazmpUkul1dP7WVfN4269GtYWBiUSmWNNpmZmThz5kzj+75Rw1+tWNXU3i+++EJMSEgQn3/+edHR0VG8fPmy1KVZtaefflrUarXi3r17xczMzOpXSUlJdZvFixeLWq1W3LhxoxgXFydOnjyZ0/LM4ObZNKLIfjanY8eOiQqFQly4cKF44cIF8bvvvhMdHBzEb7/9troN+9s8Zs6cKbZt27Z6au/GjRtFd3d38R//+Ed1G/Z1wxQWFoqxsbFibGysCED84IMPxNjY2OolLerSr5GRkaKPj4+4a9cu8cSJE+KwYcM4tbexPvnkE9Hf31+0s7MTe/XqVT39lBoOQK2vL7/8srqNyWQS33zzTdHT01NUqVTi4MGDxbi4OOmKthF/DSPsZ/PavHmzGBwcLKpUKrFz587iZ599VuM8+9s8dDqdOG/ePNHPz09Uq9ViQECAuGDBAlGv11e3YV83zJ49e2r97/PMmTNFUaxbv5aWlorPPvus6OrqKtrb24v333+/mJKS0ujaBFEUxcbdWyEiIiJquBY5ZoSIiIgsB8MIERERSYphhIiIiCTFMEJERESSYhghIiIiSTGMEBERkaQYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREkvp/xdr/sdiC+N8AAAAASUVORK5CYII=\n"},"metadata":{}}]},{"cell_type":"code","source":"# building a fresh copy of the model to train from scratch since the training step is working\ntrain_translator = TrainTranslator(\n    embedding_dim, units,\n    input_text_processor=input_text_processor,\n    output_text_processor=output_text_processor)\n\n# Configure the loss and optimizer\ntrain_translator.compile(\n    optimizer=tf.optimizers.Adam(),\n    loss=MaskedLoss(),\n)","metadata":{"id":"Emgfgh4tAmJt","execution":{"iopub.status.busy":"2023-03-16T11:47:51.494792Z","iopub.execute_input":"2023-03-16T11:47:51.495166Z","iopub.status.idle":"2023-03-16T11:47:51.524712Z","shell.execute_reply.started":"2023-03-16T11:47:51.495127Z","shell.execute_reply":"2023-03-16T11:47:51.523701Z"},"trusted":true},"execution_count":83,"outputs":[]},{"cell_type":"markdown","source":"Training the model","metadata":{"id":"1h7Dmiy1De8S"}},{"cell_type":"code","source":"class BatchLogs(tf.keras.callbacks.Callback):\n  def __init__(self, key):\n    self.key = key\n    self.logs = []\n\n  def on_train_batch_end(self, n, logs):\n    self.logs.append(logs[self.key])\n\nbatch_loss = BatchLogs('batch_loss')","metadata":{"id":"J7m4mtnj80sq","execution":{"iopub.status.busy":"2023-03-16T11:47:51.526321Z","iopub.execute_input":"2023-03-16T11:47:51.526708Z","iopub.status.idle":"2023-03-16T11:47:51.532735Z","shell.execute_reply.started":"2023-03-16T11:47:51.526670Z","shell.execute_reply":"2023-03-16T11:47:51.531558Z"},"trusted":true},"execution_count":84,"outputs":[]},{"cell_type":"code","source":"train_translator.fit(dataset, epochs=14,\n                   callbacks=[batch_loss])","metadata":{"id":"BQd_esVVoSf3","outputId":"06c29d9e-195b-46a6-a401-92eb33ccdc94","execution":{"iopub.status.busy":"2023-03-16T11:47:51.534342Z","iopub.execute_input":"2023-03-16T11:47:51.535032Z","iopub.status.idle":"2023-03-16T15:34:16.598152Z","shell.execute_reply.started":"2023-03-16T11:47:51.534976Z","shell.execute_reply":"2023-03-16T15:34:16.597041Z"},"trusted":true},"execution_count":85,"outputs":[{"name":"stdout","text":"Epoch 1/14\n2820/2820 [==============================] - 989s 348ms/step - batch_loss: 3.3863\nEpoch 2/14\n2820/2820 [==============================] - 961s 341ms/step - batch_loss: 1.3731\nEpoch 3/14\n2820/2820 [==============================] - 958s 340ms/step - batch_loss: 0.9362\nEpoch 4/14\n2820/2820 [==============================] - 969s 343ms/step - batch_loss: 0.7388\nEpoch 5/14\n2820/2820 [==============================] - 959s 340ms/step - batch_loss: 0.6408\nEpoch 6/14\n2820/2820 [==============================] - 957s 339ms/step - batch_loss: 0.5801\nEpoch 7/14\n2820/2820 [==============================] - 956s 339ms/step - batch_loss: 0.5346\nEpoch 8/14\n2820/2820 [==============================] - 961s 341ms/step - batch_loss: 0.5017\nEpoch 9/14\n2820/2820 [==============================] - 956s 339ms/step - batch_loss: 0.4785\nEpoch 10/14\n2820/2820 [==============================] - 957s 339ms/step - batch_loss: 0.4618\nEpoch 11/14\n2820/2820 [==============================] - 953s 338ms/step - batch_loss: 0.4497\nEpoch 12/14\n2820/2820 [==============================] - 952s 337ms/step - batch_loss: 0.4415\nEpoch 13/14\n2820/2820 [==============================] - 962s 341ms/step - batch_loss: 0.4363\nEpoch 14/14\n2820/2820 [==============================] - 955s 339ms/step - batch_loss: 0.4364\n","output_type":"stream"},{"execution_count":85,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7fbd8cc13510>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 128\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T15:40:53.225716Z","iopub.execute_input":"2023-03-16T15:40:53.226111Z","iopub.status.idle":"2023-03-16T16:10:40.299151Z","shell.execute_reply.started":"2023-03-16T15:40:53.226076Z","shell.execute_reply":"2023-03-16T16:10:40.298062Z"},"trusted":true},"execution_count":101,"outputs":[{"name":"stdout","text":"Epoch 1/3\n1410/1410 [==============================] - 592s 420ms/step - batch_loss: 0.3202\nEpoch 2/3\n1410/1410 [==============================] - 575s 408ms/step - batch_loss: 0.2631\nEpoch 3/3\n1410/1410 [==============================] - 571s 405ms/step - batch_loss: 0.2652\n","output_type":"stream"},{"execution_count":101,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7fbd93ec3d50>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 256\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T16:42:20.557205Z","iopub.execute_input":"2023-03-16T16:42:20.557606Z","iopub.status.idle":"2023-03-16T17:03:02.329620Z","shell.execute_reply.started":"2023-03-16T16:42:20.557572Z","shell.execute_reply":"2023-03-16T17:03:02.328512Z"},"trusted":true},"execution_count":118,"outputs":[{"name":"stdout","text":"Epoch 1/3\n705/705 [==============================] - 424s 600ms/step - batch_loss: 0.1966\nEpoch 2/3\n705/705 [==============================] - 409s 580ms/step - batch_loss: 0.1409\nEpoch 3/3\n705/705 [==============================] - 408s 579ms/step - batch_loss: 0.1234\n","output_type":"stream"},{"execution_count":118,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7fbd82a36490>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 512\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:04:42.238316Z","iopub.execute_input":"2023-03-16T17:04:42.239043Z","iopub.status.idle":"2023-03-16T17:22:34.668307Z","shell.execute_reply.started":"2023-03-16T17:04:42.238994Z","shell.execute_reply":"2023-03-16T17:22:34.667065Z"},"trusted":true},"execution_count":131,"outputs":[{"name":"stdout","text":"Epoch 1/3\n353/353 [==============================] - 351s 993ms/step - batch_loss: 0.0902\nEpoch 2/3\n353/353 [==============================] - 339s 959ms/step - batch_loss: 0.0665\nEpoch 3/3\n353/353 [==============================] - 337s 953ms/step - batch_loss: 0.0593\n","output_type":"stream"},{"execution_count":131,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7fbd82a36890>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 1024\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:23:44.856932Z","iopub.execute_input":"2023-03-16T17:23:44.857689Z","iopub.status.idle":"2023-03-16T17:39:40.716749Z","shell.execute_reply.started":"2023-03-16T17:23:44.857648Z","shell.execute_reply":"2023-03-16T17:39:40.715750Z"},"trusted":true},"execution_count":147,"outputs":[{"name":"stdout","text":"Epoch 1/3\n177/177 [==============================] - 326s 2s/step - batch_loss: 0.0480\nEpoch 2/3\n177/177 [==============================] - 314s 2s/step - batch_loss: 0.0421\nEpoch 3/3\n177/177 [==============================] - 314s 2s/step - batch_loss: 0.0405\n","output_type":"stream"},{"execution_count":147,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7fbd8cd3d890>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 1536\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:43:00.227000Z","iopub.execute_input":"2023-03-16T17:43:00.227861Z","iopub.status.idle":"2023-03-16T17:43:28.536846Z","shell.execute_reply.started":"2023-03-16T17:43:00.227811Z","shell.execute_reply":"2023-03-16T17:43:28.534001Z"},"trusted":true},"execution_count":163,"outputs":[{"name":"stdout","text":"Epoch 1/3\n  6/118 [>.............................] - ETA: 4:50 - batch_loss: 0.0351","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_23/188685183.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_tensor_slices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUFFER_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mtrain_translator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_loss\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0;31m# To get the full stack trace, call:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m             \u001b[0;31m# `tf.debugging.disable_traceback_filtering()`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiltered_tb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mfiltered_tb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 53\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     54\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mResourceExhaustedError\u001b[0m: Graph execution error:\n\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n      \"__main__\", mod_spec)\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n      exec(code, run_globals)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n      app.start()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n      self._run_once()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n      handle._run()\n    File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n      await self.process_one()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n      await dispatch(*args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n      await result\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n      reply_content = await reply_content\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n      cell_id=cell_id,\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n      raw_cell, store_history, silent, shell_futures, cell_id\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n      return runner(coro)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n      interactivity=interactivity, compiler=compiler, result=result)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n      if (await self.run_code(code, result,  async_=asy)):\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_23/2555411180.py\", line 2, in <module>\n      callbacks=[batch_loss])\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n      return step_function(self, iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n      outputs = model.train_step(data)\n    File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n      return self._tf_train_step(inputs)\n    File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n      return self._train_step(inputs)\n    File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n      for t in tf.range(max_target_length-1):\n    File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n      step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n    File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n      dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n      context_vector, attention_weights = self.attention(\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n      w1_query = self.W1(query)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n      outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n      \"__main__\", mod_spec)\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n      exec(code, run_globals)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n      app.start()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n      self._run_once()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n      handle._run()\n    File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n      await self.process_one()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n      await dispatch(*args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n      await result\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n      reply_content = await reply_content\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n      cell_id=cell_id,\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n      raw_cell, store_history, silent, shell_futures, cell_id\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n      return runner(coro)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n      interactivity=interactivity, compiler=compiler, result=result)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n      if (await self.run_code(code, result,  async_=asy)):\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_23/2555411180.py\", line 2, in <module>\n      callbacks=[batch_loss])\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n      return step_function(self, iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n      outputs = model.train_step(data)\n    File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n      return self._tf_train_step(inputs)\n    File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n      return self._train_step(inputs)\n    File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n      for t in tf.range(max_target_length-1):\n    File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n      step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n    File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n      dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n      context_vector, attention_weights = self.attention(\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n      w1_query = self.W1(query)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n      outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\n2 root error(s) found.\n  (0) RESOURCE_EXHAUSTED:  OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n\t [[StatefulPartitionedCall/while/loop_body_control/_609/_53]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n  (1) RESOURCE_EXHAUSTED:  OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n0 successful operations.\n0 derived errors ignored. [Op:__inference_train_function_274694]"],"ename":"ResourceExhaustedError","evalue":"Graph execution error:\n\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n      \"__main__\", mod_spec)\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n      exec(code, run_globals)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n      app.start()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n      self._run_once()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n      handle._run()\n    File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n      await self.process_one()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n      await dispatch(*args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n      await result\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n      reply_content = await reply_content\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n      cell_id=cell_id,\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n      raw_cell, store_history, silent, shell_futures, cell_id\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n      return runner(coro)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n      interactivity=interactivity, compiler=compiler, result=result)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n      if (await self.run_code(code, result,  async_=asy)):\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_23/2555411180.py\", line 2, in <module>\n      callbacks=[batch_loss])\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n      return step_function(self, iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n      outputs = model.train_step(data)\n    File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n      return self._tf_train_step(inputs)\n    File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n      return self._train_step(inputs)\n    File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n      for t in tf.range(max_target_length-1):\n    File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n      step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n    File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n      dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n      context_vector, attention_weights = self.attention(\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n      w1_query = self.W1(query)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n      outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n      \"__main__\", mod_spec)\n    File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n      exec(code, run_globals)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n      app.start()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n      self._run_once()\n    File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n      handle._run()\n    File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n      await self.process_one()\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n      await dispatch(*args)\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n      await result\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n      reply_content = await reply_content\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n      cell_id=cell_id,\n    File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n      raw_cell, store_history, silent, shell_futures, cell_id\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n      return runner(coro)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n      interactivity=interactivity, compiler=compiler, result=result)\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n      if (await self.run_code(code, result,  async_=asy)):\n    File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_23/2555411180.py\", line 2, in <module>\n      callbacks=[batch_loss])\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n      return step_function(self, iterator)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n      outputs = model.train_step(data)\n    File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n      return self._tf_train_step(inputs)\n    File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n      return self._train_step(inputs)\n    File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n      for t in tf.range(max_target_length-1):\n    File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n      step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n    File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n      dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n      context_vector, attention_weights = self.attention(\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n      w1_query = self.W1(query)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n      outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\n2 root error(s) found.\n  (0) RESOURCE_EXHAUSTED:  OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n\t [[StatefulPartitionedCall/while/loop_body_control/_609/_53]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n  (1) RESOURCE_EXHAUSTED:  OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n0 successful operations.\n0 derived errors ignored. [Op:__inference_train_function_274694]","output_type":"error"}]},{"cell_type":"code","source":"plt.plot(batch_loss.logs)\nplt.ylim([0, 5])\nplt.xlabel('Batch #')\nplt.ylabel('CE/token')","metadata":{"id":"38rLdlmtQHCm","outputId":"2c24e8f8-3ef3-4477-b845-34da28c8f96b","execution":{"iopub.status.busy":"2023-03-16T17:41:37.110859Z","iopub.execute_input":"2023-03-16T17:41:37.111920Z","iopub.status.idle":"2023-03-16T17:41:37.396141Z","shell.execute_reply.started":"2023-03-16T17:41:37.111881Z","shell.execute_reply":"2023-03-16T17:41:37.394978Z"},"trusted":true},"execution_count":162,"outputs":[{"execution_count":162,"output_type":"execute_result","data":{"text/plain":"Text(0, 0.5, 'CE/token')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAioAAAG2CAYAAABRfK0WAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLa0lEQVR4nO3dd1hT9/4H8HfYyIggsgRBxY0TF+5Vt62tHXqtP21v7dLeWturVeusLd7bW2uXtnbYoVXbOuve4B4oAg5coKAiInsFSM7vD2okECAJSc4Jeb+eh+eBk2/O+XhszZvvOjJBEAQQERERSZCN2AUQERERVYVBhYiIiCSLQYWIiIgki0GFiIiIJItBhYiIiCSLQYWIiIgki0GFiIiIJItBhYiIiCSLQYWIiIgki0GFiIiIJEvUoLJw4ULIZDKNL19fXzFLIiIiIgmxE7uAtm3bYv/+/eqfbW1tRayGiIiIpET0oGJnZ8deFCIiItJK9KBy7do1+Pv7w9HREd27d8fHH3+Mpk2bam2rUCigUCjUP6tUKmRkZKBBgwaQyWTmKpmIiIhqQRAE5Obmwt/fHzY21c9CkQmCIJiprkp27dqFgoICtGjRAvfv38eSJUtw5coVXLx4EQ0aNKjUfuHChVi0aJEIlRIREZGxJScnIyAgoNo2ogaVivLz89GsWTPMnDkTM2bMqPR6xR6V7OxsNG7cGMnJyXB3dzdqLaEL9mg9PndkK4zvFmTUaxEREVmTnJwcBAYGIisrC3K5vNq2og/9lOfi4oJ27drh2rVrWl93dHSEo6NjpePu7u5GDyo2jvW0Hnd2cTP6tYiIiKyRLtM2JLWPikKhwOXLl+Hn5yd2KURERCQBogaV9957D5GRkUhMTMSpU6fw7LPPIicnB5MmTRKzLCIiIpIIUYd+UlJSMH78eKSnp6Nhw4bo0aMHTp48iaAg6c4BuZNVCEEQuMqIiIjIDEQNKuvXrxfz8gb5NvIm4lKy8duUHmKXQkREVOdJao6KpTh+4yFuPywQuwwiIqI6j0HFQIkP88UugYiIqM5jUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAxEHdRISIiMj0GFSIiIpIsBhUiIiKSLAYVAwliF0BERGQFGFSIiIhIshhUqlC/nr3YJRAREVk9BpUqONnZVvu6IHDwh4iIyNQYVIiIiEiyGFSIiIhIshhUiIiISLIYVAzEGSpERESmx6BioB+PJqK4VCV2GURERHUag0oVAj2dq339yLV0fBt5w0zVEBERWScGlSqM69q4xjZfHrpuhkqIiIisF4NKFexsa34+skrFmSpERESmxKBSC4wpREREpsWgUgtK9qgQERGZFIMKERERSRaDShUCPKpf9UNERESmx6BShbAgT7FLICIisnoMKkRERCRZDCrVaO7tKnYJREREVo1BpRrPhgWIXQIREZFVY1Cpho2s5k3fiIiIyHQYVKrBnEJERCQuBpVqyJhUiIiIRMWgUg3GFCIiInExqFSDHSpERETiYlCphi45JU9RavI6iIiIrBWDSjVsbGqOKqnZRWaohIiIyDoxqFRDlx6VwcsikZFfbPJaiIiIrBGDSnV0nKRy8W62iQshIiKyTgwq1dB1Lq0gmLQMIiIiq8WgUg2u+iEiIhIXg0o1HGx5e4iIiMTET+JqjO7gL3YJREREVo1BpRpO9rY6teMUFSIiItNgUDECgbNpiYiITIJBhYiIiCSLQYWIiIgki0GFiIiIJItBxQj2XrqPtFw+84eIiMjYGFSM4LdTtzH0syixyyAiIqpzGFSMJLOgROwSiIiI6hwGlRqENnIXuwQiIiKrxaBSgx8mddW57fdHbmLB1njuq0JERGQkdmIXIHU+7k46t12y4zIA4OnOAegYWN9EFREREVkP9qiYQEFxqdglEBER1QkMKkRERCRZDCo66BAgF7sEIiIiq8SgoguZTOwKiIiIrBKDChEREUkWg4ouuNyYiIhIFAwqREREJFkMKkRERCRZDCo64MAPERGROBhUdCDTc9WPDFwlREREZAySCSoRERGQyWSYPn262KVUMrV/M7FLICIiskqSCCpnzpzBqlWr0L59e7FL0WpIW1+xSyAiIrJKogeVvLw8TJgwAd999x08PDzELoeIiIgkRPSgMnXqVIwcORKDBw+usa1CoUBOTo7GFxEREdVddmJefP369Th37hzOnDmjU/uIiAgsWrTIxFURERGRVIjWo5KcnIy3334ba9asgZOTk07vmT17NrKzs9VfycnJJq6SiIiIxCRaUImOjkZaWhrCwsJgZ2cHOzs7REZG4osvvoCdnR2USmWl9zg6OsLd3V3jy1xe7dvUbNciIiKiMqIN/QwaNAhxcXEax1566SW0atUKs2bNgq2trUiVaTd7eCusiropdhlERERWRbSg4ubmhtDQUI1jLi4uaNCgQaXjUqDPpm967g9HREREVRB91Q8RERFRVURd9VPR4cOHxS6BiIiIJIQ9KnpY/kJHsUsgIiKyKgwqehjTqZHYJRAREVkVBhUiIiKSLAYVE0hIzcX4VScRfStD7FKIiIgsGoOKCSzYdhEnbj7E2JUnxC6FiIjIojGoEBERkWQxqBAREZFkMagQERGRZDGoEBERkWQxqOhpyRjpPYeIiIiormJQ0ZOdDZ84SEREZC4MKnrik5GJiIjMh0FFTzIwqRAREZkLg4qePF0cxC6BiIjIajCo6KlLsIfYJRAREVkNBhU9ceiHiIjIfBhU9MWcQkREZDYMKkRERCRZDCp64vJkIiIi82FQISIiIsliUNETO1SIiIjMh0GFiIiIJItBhYiIiCSLQUVPMj1n02YVFOPXk7eQmV9sooqIiIjqLgYVE3tr3XnM2xKP19dEi10KERGRxWFQ0ZO+k2mPXEsHAJxKzDB+MURERHUcgwoRERFJFoMKERERSRaDip64My0REZH5MKjoyc6Gt4yIiMhc+KmrJwc7G3wxvpPYZRAREVkFBhUDjGznJ3YJREREVoFBxQA2nKdCRERkFgwqBtB3d1oiIiIyDIOKgV7t21TsEoiIiOo8BhUDBXo4i10CERFRncegYiBB7AKIiIisAIMKERERSRaDChEREUkWgwoRERFJFoMKERERSRaDioEEzqYlIiIyOQYVIiIikiwGFSIiIpIsBhUiIiKSLAYVIiIikiwGFQMJnE1LRERkcgwqREREJFkMKkRERCRZDCpmVlSihErFYSMiIiJdMKiYUXZhCVrN242nVx4XuxQiIiKLwKBiRkeuPQAAXEjOErcQIiIiC8GgYiAO3hAREZkegwoRERFJFoOKGckgE7sEIiIii8KgQkRERJLFoEJERESSxaBiIEN20L+XXWj8QoiIiOowBhUzWrLjstglEBERWRQGFSIiIpIsBhUiIiKSLAYVIiIikiwGFQO18XdXf9+5cX3xCiEiIqrDRA0qK1euRPv27eHu7g53d3eEh4dj165dYpaksx5NG2DlhM7Y+05f/PrP7mKXQ0REVCfZiXnxgIAALF26FCEhIQCAn3/+GU899RTOnz+Ptm3bilmaToa38xO7BCIiojpN1KAyevRojZ8/+ugjrFy5EidPnrSIoEJERESmJWpQKU+pVOKPP/5Afn4+wsPDtbZRKBRQKBTqn3NycsxVHhEREYlA9Mm0cXFxcHV1haOjI15//XVs3rwZbdq00do2IiICcrlc/RUYGGjmaomIiMicRA8qLVu2RExMDE6ePIk33ngDkyZNwqVLl7S2nT17NrKzs9VfycnJZq6WiIiIzEn0oR8HBwf1ZNouXbrgzJkz+Pzzz/Htt99Wauvo6AhHR0dzl0hEREQiEb1HpSJBEDTmoViKQE9nsUsgIiKqc0QNKnPmzMGRI0eQlJSEuLg4zJ07F4cPH8aECRPELMsgB9/tL3YJREREdY6oQz/379/HxIkTce/ePcjlcrRv3x67d+/GE088IWZZBrG3lVznFBERkcUTNaj88MMPYl6eiIiIJI7dAERERCRZDCpEREQkWQYN/eTn52Pp0qU4cOAA0tLSoFKpNF6/efOmUYojIiIi62ZQUHnllVcQGRmJiRMnws/PDzKZzNh1ERERERkWVHbt2oUdO3agV69exq6HiIiISM2gOSoeHh7w9PQ0di1WZWN0itglEBERSZ5BQeXDDz/E/PnzUVBQYOx6rMaCbRfFLoGIiEjyDBr6+fTTT3Hjxg34+PggODgY9vb2Gq+fO3fOKMURERGRdTMoqIwZM8bIZVifPEWp2CUQERFJnkFBZcGCBcauwyolpOaipa+b2GUQERFJlsEbvmVlZeH777/H7NmzkZGRAaBsyOfOnTtGK66uS87gHB8iIqLqGNSjEhsbi8GDB0MulyMpKQlTpkyBp6cnNm/ejFu3buGXX34xdp1ERERkhQzqUZkxYwYmT56Ma9euwcnJSX18+PDhiIqKMlpxREREZN0MCipnzpzBa6+9Vul4o0aNkJqaWuuiiIiIiAADg4qTkxNycnIqHU9ISEDDhg1rXRQRERERYGBQeeqpp7B48WKUlJQAAGQyGW7fvo33338fY8eONWqBREREZL0MCir/+9//8ODBA3h7e6OwsBD9+vVDSEgI3Nzc8NFHHxm7RiIiIrJSBq36cXd3x9GjR3Hw4EGcO3cOKpUKnTt3xuDBgyEIgrFrJCIiIitlUFCJiIjA7NmzMXDgQAwcOFB9XKlU4sUXX8S6deuMVmBdpmSoIyIiqpZBQz/Lly/HqlWrNI4plUqMGzcOMTExxqjLIgU3qKdX+9OJGSaqhIiIqG4wKKjs3LkTs2bNwu+//w4AKCkpwXPPPYeLFy/i0KFDRi3Qknw/qYte7VXsUSEiIqqWQUElLCwMmzdvxpQpU7B161aMHTsWCQkJOHToEHx9fY1do8UI8XbDnBGtdG6/+lgS3vvjggkrIiIismwGP+unf//++PXXX/Hss88iKSkJkZGR8PHxMWZtFun5LoF6tf8zOgVFJUoTVUNERGTZdJ5M+8wzz2g93rBhQ9SvXx+vvvqq+timTZtqXxkRERFZPZ2Dilwu13p86NChRiuGiIiIqDydg8rq1atNWUedYcj8WM6pJSIi0s6gfVQeefDgARISEiCTydCiRQs+5wcAMwcREZHxGDSZNj8/Hy+//DL8/PzQt29f9OnTB/7+/vjnP/+JgoICY9dIREREVsqgoDJjxgxERkbir7/+QlZWFrKysrB161ZERkbi3XffNXaNFsWQRwgI7IchIiLSyqChn40bN+LPP/9E//791cdGjBgBZ2dnPP/881i5cqWx6iMiIiIrZlCPSkFBgdY9U7y9vTn0Q0REREZjUFAJDw/HggULUFRUpD5WWFiIRYsWITw83GjFWSJDBnG46oeIiEg7g4Z+li9fjuHDhyMgIAAdOnSATCZDTEwMnJycsGfPHmPXSERERFbKoKDSrl07XLt2DWvWrMGVK1cgCALGjRuHCRMmwNnZ2dg1EhERkZUyKKhERUWhZ8+emDJlisbx0tJSREVFoW/fvkYpzhLJDHgPR36IiIi0M2iOyoABA5CRkVHpeHZ2NgYMGFDroiyZp4sDBrf2wRNt+IBGIiKi2jKoR0UQBMhklfsOHj58CBcXl1oXZclkMhm+n9QFABD8/g6d3mPI3itERETWQK+g8ugJyjKZDJMnT4ajo6P6NaVSidjYWPTs2dO4FRIREZHV0iuoPHqCsiAIcHNz05g46+DggB49elSat0JERERkKL2CypdffglXV1cEBwfjvffes/phHmNZefgG/j20pdbhNCIiImum12RaLy8vDB8+HN7e3sjOzjZVTVZnxeEb2BWfKnYZREREkqNXUElISMCIESOwceNGNGnSBF27dsWHH36I2NhYU9Vn0U7NGaRz29sZfPQAERFRRXoFlaCgILz11lvYv38/0tLSMGPGDFy8eBF9+/ZFkyZN8Pbbb+PgwYNQKpWmqtei+Lg76dyWgz5ERESVGbSPClA2sXb8+PFYv3490tPT8e2330KlUuGll15Cw4YNsXbtWmPWSURERFbIoH1UKp3Ezg5DhgzBkCFD8OWXX+L8+fMoLS01xqmtBufREhERVaZXj8p///tfFBYWqn+OioqCQqFQ/5ybm4s333wTnTp1QteuXY1XpRWQcfCHiIioEr2CyuzZs5Gbm6v+edSoUbhz547654KCAnz77bfGq86KsEeFiIioMr2CSsWt3rn1OxEREZmSwZNpybiSHuaLXQIREZHkMKhIxJqTt9Xff7TjEp766iiKSrjMm4iIrJveq36+//57uLq6AgBKS0vx008/wcvLCwA05q+Q/oZ+FoUtU3vhuyOJAICN51Lg4mCHfi0awsPFQeTqiIiIzE8m6DHRJDg4WKfn0SQmJtaqKF3l5ORALpcjOzsb7u7uZrmmvoLf36FX+4hn2mH2pjgAgL2tDCVKAaGN3LH9rT6mKI+IiMjs9Pn81qtHJSkpqTZ1kQ6Uqse5sURZ9n38nRyxyiEiIhKVXnNUDh48iDZt2iAnp/IHZ3Z2Ntq2bYsjR44Yrbi6wJNDNkRERAbTK6gsX74cU6ZM0dpNI5fL8dprr2HZsmVGK64uWDUxTOwSiIiILJZeQeXChQsYNmxYla8PGTIE0dHRtS6qLtF3I7cfj5pnfg8REZEl0Cuo3L9/H/b29lW+bmdnhwcPHtS6KGt2M537qRARET2iV1Bp1KgR4uLiqnw9NjYWfn5+tS6KiIiICNAzqIwYMQLz589HUVFRpdcKCwuxYMECjBo1ymjFERERkXXTa3nyBx98gE2bNqFFixaYNm0aWrZsCZlMhsuXL+Prr7+GUqnE3LlzTVUrERERWRm9goqPjw+OHz+ON954A7Nnz1Y/lFAmk2Ho0KFYsWIFfHx8TFIolcnIL8ath/no1NhD7FKIiIhMTu8t9IOCgrBz505kZmbi+vXrEAQBzZs3h4cHPzi18ZM7G/V8vZYeRGGJEr+90h09Q7yMem4iIiKpMfihhB4eHujatSu6devGkFIN//rOGNjK22jnK/z7QYWHr3J1FRER1X2iPj05IiICXbt2hZubG7y9vTFmzBgkJCSIWZJJ9GLPBxERkUFEDSqRkZGYOnUqTp48iX379qG0tBRDhgxBfn7d2kukY2B9sUsgIiKySHrPUTGm3bt3a/y8evVqeHt7Izo6Gn379hWpKuMLaeha63OMW3UCX/2js/pnPR56TUREZLFEDSoVZWdnAwA8PT21vq5QKKBQKNQ/a3s4ohQ52NW+4+rkzQwM/SzKCNUQERFZDlGHfsoTBAEzZsxA7969ERoaqrVNREQE5HK5+iswMNDMVRrG2cHWKOd5mF9slPMQERFZCskElWnTpiE2Nhbr1q2rss3s2bORnZ2t/kpOTjZjhdLCkR8iIrIGkhj6eeutt7Bt2zZERUUhICCgynaOjo5wdHQ0Y2VEREQkJlF7VARBwLRp07Bp0yYcPHgQTZo0EbMckwpuUE/sEoiIiCyOqD0qU6dOxW+//YatW7fCzc0NqampAAC5XA5nZ+Pu6FrXcOSHiIisgag9KitXrkR2djb69+8PPz8/9deGDRvELMskZDKZUc9X1RyV76JuYvLq01CUKo16PSIiIjGIPvSj7Wvy5MlilmURDiekaT3+0c7LOJzwAFvO31EfU5Qqse70bSRnFJirPCIiIqOQzKqfum7OiNZGPV9mgeZS5XxFKdJyitQ/FxY/7lH5+tANzN4Uh0GfRhq1BiIiIlOTxKofa/BEGx+jnq/8yE9KZgF6/+cQ6tez19r2+PV0AECxUmXUGoiIiEyNPSp1wJ/RKQCArIISra/HJGeZsRoiIiLjYVCxAqUqrhEiIiLLxKBiocr3nshg3BVFREREUsGgYsF2xt2r8rWcolIAQCnnpRARkQXjZFoL9ubac4hfNBTatmhZtu8qTtx4iBM3H5q/MCIiIiNhj4qFO3Gj6iDCkEJERJaOPSoWbsovZ7X2qFRFqRJga8M5LUREZBnYo1IHVLWdvjYdF+3FvexC0xVDRERkRAwqViZXUYrvohLFLoOIiEgnDCpm5GDL201ERKQPfnKaUcQz7cQugYiIyKIwqJhRv5YNxS6BiIjIojComJGXqyP+2buJ2GVAALfUJyIiy8CgYmYtfFzFLoGIiMhiMKiYmT5LiYmIiKwdgwoRERFJFoMKERERSRaDipnZ6LPfvYlw+ImIiCwFg4qZje7gj5Y+bqLWcCU1R9TrExER6YpBxcycHWyx552+otZQomSXChERWQYGFSIiIpIsBhUrJHCSChERWQgGFStUm5iiUgkMOkREZDYMKlbI0JxRqlRhyPIoPPfNCeMWREREVAU7sQsgy3EzPR/X0/K0vnYoIQ0QgAGtvM1cFRER1WUMKlbI0IGbqnpiCouVeGn1GQBA/KKhcHXkf1ZERGQcHPqxRkaeY6IoVaq/LyguNeq5iYjIujGoWCGDe1RqNQ2XiIhIfwwqVkhlylU7zDJERGREDCpWSFtOKSpRVj5YwfYL99Tfp+UUIfpWJgBABvGfX0RERHUTgwrhm8gbaDVvNw5dSauyTWxKFr46dF39c7ePD2DsyuM4dzvTHCUSEZGVYlCxQqoKPSpLd10BAMzcGFvle24+yNd6/JkVxzV+5sgPEREZE4OKFSq/s+zG6BSN1wqKSxF9KwP5Cs3VO9VNpL1c7mnM59nDQkRERsQNL6xQ+Tkqyw9cVX+fV1SKNvP3AABa+bph9/S+Wt9TkaJUpf5+zuZ4DAv1M16xRERk1dijIpJmDV1Eu3b53pHyAaSw3ITaK6m5SM9T6H3ujPxiLNl+qVb1ladUCbibVWi08xERkWVhUBHJb1N6iF0CgOp7SiJ2XtGpXUXfH0002oMLX/s1Gj2XHsS+S/eNcj4iIrIsDCoi8XF3Eu3aRSWqmhsBuHwvByXKmts+moxb3tMrjkNVcdauAfZfLgso3x25WetzERGR5WFQsUK3Mwp0anfpXg6eXnEM19NykVNUUmW7y/dyKh2LSc7C1bRcg2skIiICOJnW6tU0RBN/JweDl0UZdO7U7CK08nU36L1EREQAe1Ss3t3sIpOde/LfT1QmIiIyFIMKERERSRaDiohe7dtU1Osfv5Eu6vWJiIhqwqAiojkjWot6/X98d0rU6+uDjz0kIrJODCpW6mxShtglEBER1YhBRWQ+7o6iXPc/uyvvfUJERCQ1DCoiOzproCjX1XXTt9pfR1lzIyIioiowqIjM3lacv4K4O9lmuY7KSFvpExGRdWJQIZNiTiEiotpgUCGLIOOyHyIiq8SgQibFDhUiIqoNBhUiIiKSLD6UkCQn6uoDnEp8iOAGLupjJ29m4HpaLkK83TTaKkqVeGdDDHqHNMQ/ujc2d6lERGRiDCpkUjfS8tAhsD4A4PsjN/HziSRseDUc/vWdq3zP//14WuvxwcuikLR0pPrntNwidPvoAABgZ1wqgwoRUR3EoR8yqZyiEvX3S3ZcRnJGYa03m8suLMGPRxPx4fbLtS2PiIgkjj0qEuLj7oj7OQqxyzAqbcuTS1W1m2I7689Y7L6YWqtzEBGRZWCPioR0CKiP3iFeYpdhVFojSS2XAh24cr92JyAiIovBoCIBW6b2wriugfj4mXZilyJ5sSlZKFFy0TMRkbXg0I8EdAysj45/TzitawQtYz9CLbpUnvzqWG3KISIiC8OgIjF1bQdWfSJJdmEJFKV8iCERET3GoEKmVU1SyS4owcnEh2jr7w5BAPr895D56iIiIovAoEImVd0wz/jvTuLSvRwzVlOzguJSrD6WhKFtfSptLkdEROYn6mTaqKgojB49Gv7+/pDJZNiyZYuY5UhC/5beAAA3RzvMHNZS5GpqT9vy5J1xqbieliuJkFJUotSYR7Ns71V8sicBg5dFAQCW77+KWX/Gap1ro6ulu67gya+OoqiEw1pERPoSNajk5+ejQ4cO+Oqrr8QsQ1ImhQfh6390xv53++HN/iFil1Nrx64/xPnbmZWOPwoCYsrIL0arebvxj+9OqY+dq1Dr8v3XsOFsMi7ezUFaTpFB1/km8gZiU7KxNeZOreolIrJGog79DB8+HMOHDxezBMmxs7XByPZ+YpdhND8eS8SPxxI1tr6Xipl/xgIATtx8qD72IO/xhnvp5b7/394EHE54gFnDWuGN/s0Mup5SpVs7QRCw99J9FJUoMbKdH+xsuYsAEVkvi5qjolAooFA8/vDIyRF/6MDUnO1tUcghA5PYf1lz47i/LtxFckah+uecwsfb/x9OeAAA+M/uKwYHFV1XdG2NuYvpG2IAAHezigy+HhFRXWBRv6pFRERALpervwIDA8UuyeTOfDBY7BIshqqWW/N/sidB42dZFcni5oM8g86v68rz3fGPHw+w6VyKQdciIqorLCqozJ49G9nZ2eqv5ORksUsyOVdHi+r0kiylSkCeorTK16+n5eF2RoFO58ou19OiD117VMq3u5ZmWCgisjSluo6NktWxqKDi6OgId3d3jS+iR6rrT3n2m+MIXbAH97LLhnYqrsAZvCyy0nuqyhW74g17IKKshj6VUzcfYsHWeA71kdVZvv8qWs3bjUt36/5wPunPooKKtZoUHiR2CbWWlmvYihljOX87CwDw2q/RmPD9SbSat7vG91TVA7Iq6qZhRdTQo/LCqpP4+cQt9XwYImuxfP81lKoEfLzzstilkASJOq6Ql5eH69evq39OTExETEwMPD090bhxYxErk5ZFT4Xi5xO3xC6jVrp9dMDk1yjb66T6NBCbkq3z+VYfS6pdQRXUsacjEBGZhag9KmfPnkWnTp3QqVMnAMCMGTPQqVMnzJ8/X8yyyEIZ+5nKPx1PMur57ucUYemuK7iTVVhzYyIiAiByj0r//v1rteMnUU0OJaTB3cne6OctLlXBwU6/nP+/vVcBAHsvpuLge/0BAEevpSMxPQ8Tw4ONXKH0lCpVOJ2UgY6B9VHPgZPEqbLaPFmd6i7+a0F1RsXMezerEC+tPmOSa/18PAlT+jY16L030/NRWKzE5weu4ZvIGwCANv5yY5an4dj1dMzbEo+IZ9qhe9MGJrtOTb4+dAOf7b+KHk09sf7VcNHqICLLwsm0VGcIECAIAnbHp+L2wwKkGrjlvS6upeXW6v1fHHwcUgDj7ZcSfycbXx+6juLSx0s9J3x/CjfT8/HCqpNGuYahfjtdNs/q5M0MUesgIsvCoGIhfnm5G/o098ITbXyqbHNhwRAzViRNey6m4vU10ej7ySFJT16Nv6M5qXftqdtGOe+oL4/ikz0J+OFootbXFaVKbI+9i+tpuXjyq6NYe8p8k7Q5yktEhuDQj4Xo26Ih+rZoiIXbLlbZRu5sD1sbGZS13KHVUgmC5m/rVe0sa2ylShWSHuajWUNXna955Fq6SWu6kqp9P4ple6/i23LLq2NTsjGhu2mXv9/PKdJ5Mz1LU6JUwZ7PYjIahlnShv+H1TF7pvfBlD5NxC5DEszVo/LWuvMYvCzKaL0ixlDVP/jbY++ZtxAA3T8+gOe+OYG0XEXNjS3I+duZaD53F5btuyp2KUR1GoOKhSm/Sur6R8PR3NtV4/UQbzfMHdnG3GVJ0vEbD2tuZASPdqr9YEs85m+Nx/T15/HlgWsYtjzK4O32yxv06WGkZNbcI5Ga/XhOzuGENNzNKsShK2kabbQtjb5vwrk8plTdIxHMYfH2SwCALw5cM+p5U7OL8Py3J7BDhFBJJEUMKhbMztYGHQPri12GZCzYehEl5Z4X8p/dV0x2rRsP8rUe/+XELWyJuYtP913FldRcrIq6obWdvtdasr3mHTvLb72fU1SKnksP4qWfal711P3jA5UeKSB1n+5NQOiCPdh36X6l1x7mKZCRX2zWes4mGW+C8MJtF3E6MQNTfzuncVylEnD+diYKiy3r76oiQRBE36maLAuDCtUZG84mm234JfpWJgAg8mr1291/faj2QQUomwRblRm/x+Cpr4/Vam7S4YQHFrWn0ZcHy3a0XrjtItafvo11p8v+3otLVQhbsh+dP9ynsfLJ1J795oTRzpVRoD1krTl1C0+vOI5JP5422rXEMGdzHLp9dADbY+9Wes2C/hMkM2JQsTDPdA4AALTx0++BjD7ujqYox6oVlSjN9qFRXQbZdO4OLiRn4Uwtfqt/fU00ftHjMQ3JGQV45ecz6PvfQzhyrSysCYKArHIfsopSpcnDT35xKd7fFIfZm+KQXViiMdSWW1T7YTdRaLllgiBg/tayifSnjdh7I4Z1p8uees+5PaQrBhUL0yGwPk7MHoit03oB0H3b+HARN/qqqxRm/I1dl79nVS1DwYJqVpRVNG7VSey/nIbbGQWY+ENZWJu1MRYdF+9D1NUHSM4oQNv5ezDOxHu3lJT7O1CUKjXm6fxy4hZ+NvJjEMxB2+6sO+MMe2J3db6JvIFxq06IN+zH3hPSEZcnWyA/ubPe73m6cwC2xFTuaiXDLfpL9w/22qqqZ+Li3cf7sczdHG+WWtacvFVpUu7ZpAz8frZs07r/K9fLdCrR+L/955ebRKuxHFwARn91VP3j539Pch3TqRHkzsZ7jEJabhFe+zVa/URuY9P2V70jzvj/7y7dVTaHa8OZZEzqGWz089dE23/RZtpRgCwMe1TqqD9e19yi3NWRmdTYNp27I3YJGPnF0ZobGdkHWyoHImPO0agoX1GKjov3Ivj9Hfj+yE1M/OGU+rXyn2tV/YJefoK1Mfx3d4LeIeVQQho+2BKnU++Ftp4xU46gFVaoyVxzlRLT86GqMKZpykufuPEQhxPSam5IksOgUkd1DfbEvnf6qn92sudftSV79OFVVKJEWk4RBEFAQbG4y3Nro+IHVHV+OJqIrIKy+SZLdlzGufIhoVxSyS3Sfj9q+iU9M78Yi/66iEt3tW+SV1FeFdepbjLzS6vPYM3J2/jxmPYdgzXOL+Ky63vZhej+8QF8ZqT5I7EpWej730PYHa99qfWGs8l6nW/F4ev4ZI/+q/mUKgHjvzuJyavP4O3157HhjHT2PKKa8dPLwo3u4A8ACGpQr9JrzX3c8O+hLfFij8Z6T74laXn0m2anxfvQ7eMDaDN/D9rM3yNuUbWwI073PUIehRRtyoeQmX9e0N6mhvGED7bEY/WxJIz44ki17dLzFMhXlFY5PLE1puYetrta9rGp6Or9vErHjLEfT1XK92J8ceAa0nIV6mGz2nr1l2jczijA62vOaX39wOU0rD+tW2hQqQT8d3cCvj50Q6d9hcorHyK3xtzFrI1xer2fxMXxAAvXr0VD7J7eB409KwcVAJg6IMTMFZEpCAJw40Geupu+Yne9MR29lg5nB1uEBXmoj20+n4I/zqZgYCtvo1wjPU/3XWq1TS59JKdc78a5KoZjaupRKT/P55HtsXexdNcVrJwQhnYBcmTkF6PLkv0AgOGhvlrPY4qdd/dcTMXQtr5G37ywfI9W+Z658qFl7uY4vDWwOXzlTgZfp6iaZfUAsP/yfey//HgvnOr+rsu/outE9pWHbyBPUYK3B7XQqT1JE3tU6oBWvu6o58DMWZepBAGDPo00+XXS8xR48YdTGLvyuMbxdzZcwPEbD7FkR80bz+lCnzmTtZ23UNMEzfKnf7QvzrTfziMlsxCvr4kGAOwqN3TxaCfiitacrHl5d/k/S25RCT7Zc0XjuUwVh49e+zUa3T/eX+k8mbXc0K780uAvD17HmpO3UFyqwvozj4di1p66jR4RBwy+RtlydfGWiKtUAv6z+wq+PnRD647MZDkYVIgsgLlWcurT01Eb5npgJADIaohF5cPDpB9PqzfzA8oeOZBbVKLTiqqUTN0+DO9mFeJ6Wh4+3nkZXx+6gWHLHw85ffj3tvzl3c+p/Hfy5lrtQylVmfF7DN774/HQ2FeHrmu8/sGWeDz3rXEnRG+7YNpVhtmFJRj6WZTW+TSCICCr3HDZlvPiT3wnwzGoEFmA0yZY5luTRxu5mYKNluxw40Ee+vz3oMZEx6irD/BTLfdCkVXzr1xxqarSk50r9iZ1/nBfra5fngCg59KDGLwsElFXNZ+g/Wd0is5/1hM3tQ8FbYxOwaZzKRrH0vMU2HTuDv6MTql2rsuF5Cytx+PvVB4aq0mpUoW318doHDt45T5iU7RfQ1+CAPx8PAkJ93MrzacRBAEvrDqp8fdmrDk3JA6OFxCRVhN/OI05I1rhld5NjX7urw/dwITuQbCxkeHmgzxEXn2AbRfuIjmjELM2xqFpQ1e0D5Br7MliqOr6U3QZ2ihRGt6fdSghDZEJjwNf+d6b8sMRvf9zUOcemarkFJXg3b97TYaF+qqHg8sPJ526+VBj7pEu9lxMRWgjeZWvf33oOtJyirDwybbqnjJtw2Av/3S2xmtVN8xXcdm0tkckCIKARX9dEiXYk+kwqBCRWkyFCakf77yCvRcrP/ivtlJzirDm1C38X3gwBmqZe/OcEfdlqWqY6UJylskfXvjS6pofCgnoPmxUnfJ7tBSXqlDPoez7/+1JUB9/9ddovc9b0xyhT/4+/7hujdH679WFhk7+rdi7VW1dFQZEswtKEHWt9j1wJD0c+iEitfc3VV62ebbcnA1j2mimDfOq6lF59pvjVbxiSqaZbVRaYVO7uHLDNX9Ep1RsrpfoKv7+72QVajzhefjnR5Cep8DhhDTs1fJUa13cy676qcqac14EjQd+pucp0GHxXry17rzO17Kkh3BaOwYVIhKHmT4oqpq3W5shHUOZIpz9fDwJrebtxpnEx4Fi4g+nkVVQjF9PJNX6/FXNh5m+/jx2xGruh9NlyX5M1rEXSV8zfn88GbiwWDOYPVo6rg9t+9WQNDGokIZxXQMrHVs3pYcIlVBddyElW+fdYGuj/KqfwmIlNkanIPj9HSa5VmJ6Pnr/52CVS5W1zauorQXbLqJUJeDdP2I0jo9bdRLzthrneVQPchUY+cUR/Fruz5X0UL9N16TGlHsRkXFxjgpp+PjpdnjniRZIySzA2JVl8wQauDrodY5/9m6CH47WvFU4UU27wRpDWm4Rghq4IKeoBO0X7jXptaZviEFKZiE+2BKPF3sEmfRaFVXcg+VKaq7Rzt31o7Iei3lb4hHS0BXX0ox3bkP8a73uQzxVeWCCDfrINBhUSG315K6wsZHBx90JPu5OaOHjChdHO7TwccO+d/ritTXRuPkgv9pzyGTAByNbM6iQZPT75DDcnew0drE1lfJLfE3Va/NIUYkSNx48Hr4w11DW+O9OmvT87/1xAYpSFb4Y1xF/RKfg8r0czB/VRqNNYnr1/w7pQlHDrrkkHQwqpBbs5aLx8953+qm/b+7jhoPv9seluznq34I/fKqtumu5Q2B9LH2mHYIa1DPrZl5EujBHSDG3catOIqaKvU8s2Z9/T/6dM6IVZv4ZCwBoH1D18mhDVfVwSZIezlGxcmtf6a5X+zb+7tg9vQ+OzhqAieHB6uMNXR3R2o9b+ROZS10MKeWFRxxUf//OBu0PnKwNDv1YDn6qWLnwpg3U33vpOBellS+fxExERObBoGLFJvYo2xk06t8DUKJSwc3J3uBz9WzWoOZGtTR9cHNsu3AXSpWAWxa+4oCIiHTDoGKlpg0IwXtDWwIAGjeoZ/B5js4agDNJGRjd3t9YpQEAXu/XDOdvZ+JUua2wpw9ugemDW2D/pft45Zeat+MmIqqKivu9WQwGFSvUp7mXOqTUVoBHPQR4GB50qtLK1w3ZhSXqoLJ7eh/1awNaeePJDv5wtLPB+O6N8cwKMXYYJSJLpuLOtBaDk2lJVJve7Ik2fpXnvDRwddAYTio/L8bWRoYvxnfCJ891QOfG+j1gjYgIePwwgy3n72DprivcUl/C2KNCJvGvQc3xxYFr6BLkUe2zYgI8nLFtWi9cS8vDgm0XUd/ZHs28XdE7xAsA4GBnU+2TW4mIDLHl/B3MeKIFpm+IAVDW09zr7393Kvr9TDJWHL6OHyd3RdOGrmaskgAGFTKRGU+0wNQBzeBoZ4vvj9zE14euI7OgpFI7GWSws7VBaz93/P5aeKXXh7b1NUe5OhnbOQAbz9XuAW9EJA0Vn9ScVe7fp0GfHsaNB/nYMrUXOgbWx8yNZfu5zN0cj3Wv8pEi5sahHzIZRztbAMArfZri3Lwn0Ny78m8iUt0bLrRR5eEoqdZKRIZJy3n8tGabcv9/3/h7B+4xXx/TaF9QzE3ixMAeFTILmUyGDa+F48Dl+2ji5YJnvyl7jpBUP/u3v9UHey+m4tVfo9XHpForERmm28cH1N+/sfYcWvi44skOmisYn1nxOKxcSMk2W230GHtUyGw8XRzwXJdAja36jbHdfvlN66rzybPtEVRhKXbHwPro26KhxrHgv9sMaeuLpKUj1cft7fi/C1FddvV+Hv6396rGsXO3s0x+3aISJX46logZv8fg8j3TP1Hc0vBfXjK7Bi4O6N7EEz2aesKjnuGbzD3y6fMdMKyGuSzLX+iI57oEIvLfAzSOb5naC4Nbe6t//nJ8J2x+s5fWcwxo6a31uK4GtNQMRD+91LVW5yMi6Ym+lYF5W+KRU1R5Tl5VPtt3FQv/uoRN5+5g+OdHkJpdVPObrAiDCpmdTCbD+ld7YN2UHkbpUfGv74xvJoZpHDs1ZxCOvz9Q/fOgcmGkd4WZ/f/o1hj/HtoSW6b2wugO/vBw0f4oAXcnO0zo3tjgOle/1E39/WcvdEB/LcFnYo8g3Px4hMHXICLxRN/KwNiVJ/DryVtov3Avgt/fgQOX71fZ/mGeAjHJWfg26qbG8R4RB1BUwqc7P8KgYoWksF2ATCYz+lOW1/zz8QMWfdyd4Cd3woh2vniqo7/G4wHaVpgoa2drg6kDQtAxsH6NNS8ZE1qrGj98qi1GtvertJOvt5sjYuY/gcVPtYWNDWfDEFkCQRBw7X4uFKVKHLuejrErT1Rq88+fK++iLQgCZm+KRdiS/ZUm7D5y9Fq60eu1VJxMS3VGr5AGWPRkW/XqIplMhhUTwiq1mz6oBZztbTGkjW5Ln4eH+iLpYQE6N66vNVzNHdEaH+28rPW9U/o0wXdHEtHv73kwE8ODNZ46/ciknsGoX0+3h0LW5PScQRqTBG9+PAL/2X2l0m9txvLByNY4lJCGfi0a4uOdV0xyDSIpidh5GcVKFTo39sBb687X2F4QBI1/O66n5WHd6eRq3/PKL2c15shZMwYVqjNkMhkm9QyusZ2zgy2mD26h83lXvhhW6R+a8qb0baoOKq183fDj5K5Ycfg6JvdsguAG9TCotQ86BNTX+t5AT2ckZxRiaFsfnevR5vLiYWg9fzcAwMvVUeM1GxsZZo9ojTf6N0PHxftqdR1twoI88EqfpgCAiT2CAUBdyyMh3q64npZnlOt9Pq4jikqUmLUxDsNDfTGinZ9OHxZExnA3q1Ad+lcfS9LpPVkFJRpDyk98FmWK0uosBhUiHVQVUj59rgMAIOKZdvh071Use74j/Os7Y8mYduo2PapZlbTvnX7IKiiBr9xJpzqae7vimpYPfGcHW5yeOwh2NjawsZHh24lhWPzXJXwxvpO6Tf16Dmjt527QqoJnOjXCrvhUFJYbN5/YIwgFxUqNITNnh7K9c4Ib1EPSwwLELhwCN0c79f0Lfn+H3teuqH1AfTTxcsELXcvmCylVQqWgIpOZZ4hzeKgvnurYCK+via65MYmqjZ87Fj/VVr01gqF6Lj2o93s6fbgP3Zt44oORbTD6q6O1ur414hwVIj399kp3tPFzx9apvTA2LAAAML5bY5yZOwht/CtvFFcdJ3tbnUMKAOz4Vx8kRmhOtl3/906Z3m5O8Pz7t7ahbX1x7P2BCAvyqNT2x8ldcPaDwZg6oBn2z+hb4zUn9wzG0rHtMXdka/UxX3cnLHyyLT59voPWEHf43wOQtHQk3J3sDZqLNLmKnrE+zb3QpNzydqDs2U/fvKg5xHf9I9NMSJ4zohXOfjAYcmd7DGnjg5UvhsHe1nRzil7u1cRk564o0NPZpOd3trc16flrsviptkYbXjXEqcQMhhQDsUfFijz6LbNHU0+xS7FoPUO8sPPtPpWOG3tycEWN6jvDocJeLh0C5NX22FQkd7bHwFZlw0z/HtpKp/fMG9UGtjYyjO/WGF6ujmgfIEdDN0fYmnDS76OemfI6BMjxc7mVU+UNC30832jOiFZGr+3qkuHIV5Squ+/PzXtCvZNpKy0P1TSWqQOaYfaIVmg+d5dRz9uovjPuZBWqf57Spwnmjmyj7vF6sUdjrDl522jX+2FSFwxs5Y3/+/E0HuYVQ1GqVO/+agqDW3vj42faYd+l+3iqYyM42NrAwc4G19NyTXZNMh0GFSsS9e8BOHY9Hc90DhC7FKqBl6sj0vMUGscOvddf/f32t3rjh6OJeHeI7nNtqlLVcFLM/Cc0fgO1tZFpBAJT++P1cDxXrpt+y9ReOoXBsCD9g/iX4zuhtZ8bBi8rmzuw9pXumPD9KQDAv4e2hIOdDRzsNO/FI43qG94T8eX4Tjhx8yF+O3UbS8aE4sUeQQCAB7kK2NvK1Pd/x79647VfozG4tQ9+Op5k8PW++kcn9G/pDRnKtofv07whnu7USN0TGPFMO1y7n4d5o1rXKqi4ONgiv/jxMOGg1mXh+Ne/V+bdzSrE8v1X8fvZ2j07y9neFmc+GIzQBXsAlD2NvfwT1Sd0D9Jor5LAikfSH4OKFQn0rIdx3QzfB4TMp2KHQLcmnhq9KaGN5PjshY5Guda6V3vg0JU0jGrvjye/OopraXl4soO/qN3kgR710DXYE2tf6Y53NsRg6dh2OvdYGdKxNfrvbdNj5j8BNyd72NrIsOnNnjh4OQ2v9DHe8EsrXzek5ykQ6FkPbw9qjv4tvTGqvR/e6NcMgZ6Pd01u6KY5IbqtvxxHZ5XtC1SboDKq3LL4fTP6VXp9vIH/PjRwccDD/GIAZfO2BrX2xuivjiI5o1Bre//6zvjvsx0gd7aHna0NVh6+odf1ghvUwyfPdUDX4LJQuunNnrj1MF8jpGgjha0ZSH8MKkQSVP7D1tHOBqsmVl5mbSxero54rksgAGDbtN44dj29ysfdG8vIdn7oGuyBhX9dAgC4OdphzsjWWH8mGb1DGuD5LmW9fr1CvHB67mC9zv3o1r3Rv5nWD0BfdycsHdsOzRq6Ys7mOHQLftwDUz6cdW7sUeMH3yMznmiBZfuuan3N2d4WHQPrI8TbFR9q2YdHJpNphJTa6BrsgTNJmeqfl7/QEetO38apxAy9z1VxeKg680a1wdHr6bC3tVHP23o+LBCfVnFPHpk7sg0A4GxShkbd2pT/+zzwbn+NXi1d/64aG+k+m8uei6kY2tYXKpWAYzfS0dZfrp6HZk0YVIgkLmHJcLNdy9nBFoPb1G6ptC4aujlicq8m6qDSKcgD47s1Nvg3+vJC/t5HZ9awVnhvSEvY2sg0Vhvtm9FXvQHgr+U2CayNfw1qjl4hDTB25QlMHdAMT3ZohJa+btUuazeW3dP7YNjyIwAAdyd7dG/iiVOJGephkLvZhTiVmIF6Wub9VOfge/3Q8oPHy8y3v9UbzX1cce5WFi7ezcaSHWVL8je/2RMdA+tjTKdGGu/XZ+PCpWPbY9CnkZWON6rvjJ9f7oYAD2c42dtiVHs/+MmdDZ6D5Oxgiwvzh6DD4r0Gvd/cXvv7oahO9jYoKlEBgFXurcKgQiRBsjr+rOZHn93rX+2Bn48nYcHotrU+Z/yioSguVWnsQvzoA+35LgH4/WwKzswdrPG6MYUFeSIxYoRGMDFFSDkycwD2XEzFgctpaNLQBa18H0/mbeHrhveGtMTDfAW83cpWk03p0xT+cmd013MSvaOdLb77vy7YEXsXHz3dDi6OZR8X4c0aoGuwB7bE3EFTL1d0qqIn48XuQdhwJhnD29U8r6lZQ1f191MHNMM/ezdFckYBmjZ00dxV2l+u159BG3k9e3i5OiA9r7j6ds72yC7U/Xk9pvQopABlS/wr/ndW18kEwXJH7XJyciCXy5GdnQ13d9PNvCcytxGfH8Glv/c7qUu/QT3q2fhn7yaYN6qNyNXUHbEpWdhzMRXTBjTXumLKFHTpLdKnR+mvC3cRfStTvcrMlFIyCzDj9ws4XWFIrIGLA+T17DFrWCsMbeuL7MIS/HXhLj7YEm/SevS16+0+aG3C1WbmoM/nN3tUiCToy390wsw/YzFtYIjYpZhEK183sUuoU9oH1Ef7KnY/NhVdAog+v/WP7uCvntRsagEe9fD7a+EoKlFiV/w9FBarsPl8Cj5+uh2a+zz+b1PubI8XewQZFFT2z+irXkVWk/HdGuNBbhEauDhiw9nqt9YHgIJizQcW/nIiCfO3XsSF+UMgN8IT6aWGQYVIgpo1dMXGN3qKXYbR/TWtN84kZWAsl8iTBDjZ2+LpTmX/Lf6jFk9GL2/VxDD0bdFQ516hCwuGQO78OFzoElQEQcDNB3nYEnMXJ26kqycid1i8V6MH9kGuAkM+i8T80W3Uf05LxJ1pichs2gXI8XLvJnxCNFkUXXsA3Z3sMKStL5zsbaudZWZvK0NTLxesfqmrRkgBgP+MbVfFux5LySzEwE8j8cWBa1pXSwmCgK0xd9D1o/3ILCjBOxsuQGnBm8iwR4WIiKgaz3cJxOLtZSvUxnUNRG5RKS7ezcaWqb3w9aHr+O5IIgAget4T6vdoG/baP6MvQrzdoFIJVYb157sEYtbGuGrrmb4hpsrXikqUmL81vtJmes3m7LTY+W6cTEtERFQNpUrAtgt30NzbDaGNNFceqVQCbqbnoVlD10rh5O3155FZUIKfX+qq13ydM0kZePH7U1CUqmpurAcpBRV9Pr8ZVIiIiCTqX+vOIyWzAL9N6YFW83bX/IZqWGpQ4dAPERGRRH0xvpPRztV+4R58+nxHPGGGTR2NiZNpiYiIrEBOUSmm/HIWI784InYpemFQISIiskD1HGzR0scNoY30m/pw8W4Ogt/fgeD3d+CPv5dDlypVyMyvfrdesXCOChERkYXIV5SinoMtZDKZesnxiRsP8eIPpyq1jfx3f/T75LDe1zDHXBZOpiUiIrIi3x+5iZvp+egW7Ak7WxlGtS/b5VepEtBszk69zhXetAGm9G2C5IxCDG/nq35ulDExqBAREZHa2lO3MHez4c8sMnYviz6f36LPUVmxYgWaNGkCJycnhIWF4cgRy5rkQ0REJHUTugfh6pLhYpdhEFGDyoYNGzB9+nTMnTsX58+fR58+fTB8+HDcvn1bzLKIiIjqHAc7GyRGjMCFBUOQsGQYvnkxTP3a3nf6ilhZ9UQd+unevTs6d+6MlStXqo+1bt0aY8aMQURERI3v59APERGR8RQWK7H21C3kFJbgi4PX8a+BIegc5IH+Lb2Neh2L2PCtuLgY0dHReP/99zWODxkyBMePH9f6HoVCAYVCof45OzsbQNkfmIiIiGrv+Q5eAIBXevipjxn7c/bR+XTpKxEtqKSnp0OpVMLHR3OHPB8fH6Smpmp9T0REBBYtWlTpeGBgoElqJCIiItPJzc2FXC6vto3oW+hXfFCTIAhVPrxp9uzZmDFjhvpnlUqFjIwMNGjQQK8HPukiJycHgYGBSE5O5rCSmfHei4f3Xjy89+LhvTc/QRCQm5sLf3//GtuKFlS8vLxga2tbqfckLS2tUi/LI46OjnB0dNQ4Vr9+fVOVCABwd3fnf7gi4b0XD++9eHjvxcN7b1419aQ8ItqqHwcHB4SFhWHfvn0ax/ft24eePXuKVBURERFJiahDPzNmzMDEiRPRpUsXhIeHY9WqVbh9+zZef/11McsiIiIiiRA1qLzwwgt4+PAhFi9ejHv37iE0NBQ7d+5EUFCQmGUBKBtmWrBgQaWhJjI93nvx8N6Lh/dePLz30mbRW+gTERFR3Sb6FvpEREREVWFQISIiIsliUCEiIiLJYlAhIiIiyWJQ0WLFihVo0qQJnJycEBYWhiNHjohdkqRFRUVh9OjR8Pf3h0wmw5YtWzReFwQBCxcuhL+/P5ydndG/f39cvHhRo41CocBbb70FLy8vuLi44Mknn0RKSopGm8zMTEycOBFyuRxyuRwTJ05EVlaWRpvbt29j9OjRcHFxgZeXF/71r3+huLjYFH9sSYiIiEDXrl3h5uYGb29vjBkzBgkJCRpteP9NY+XKlWjfvr16k7Dw8HDs2rVL/Trvu/lERERAJpNh+vTp6mO8/3WIQBrWr18v2NvbC999951w6dIl4e233xZcXFyEW7duiV2aZO3cuVOYO3eusHHjRgGAsHnzZo3Xly5dKri5uQkbN24U4uLihBdeeEHw8/MTcnJy1G1ef/11oVGjRsK+ffuEc+fOCQMGDBA6dOgglJaWqtsMGzZMCA0NFY4fPy4cP35cCA0NFUaNGqV+vbS0VAgNDRUGDBggnDt3Tti3b5/g7+8vTJs2zeT3QCxDhw4VVq9eLcTHxwsxMTHCyJEjhcaNGwt5eXnqNrz/prFt2zZhx44dQkJCgpCQkCDMmTNHsLe3F+Lj4wVB4H03l9OnTwvBwcFC+/bthbffflt9nPe/7mBQqaBbt27C66+/rnGsVatWwvvvvy9SRZalYlBRqVSCr6+vsHTpUvWxoqIiQS6XC998840gCIKQlZUl2NvbC+vXr1e3uXPnjmBjYyPs3r1bEARBuHTpkgBAOHnypLrNiRMnBADClStXBEEoC0w2NjbCnTt31G3WrVsnODo6CtnZ2Sb580pNWlqaAECIjIwUBIH339w8PDyE77//nvfdTHJzc4XmzZsL+/btE/r166cOKrz/dQuHfsopLi5GdHQ0hgwZonF8yJAhOH78uEhVWbbExESkpqZq3FNHR0f069dPfU+jo6NRUlKi0cbf3x+hoaHqNidOnIBcLkf37t3VbXr06AG5XK7RJjQ0VOMhV0OHDoVCoUB0dLRJ/5xSkZ2dDQDw9PQEwPtvLkqlEuvXr0d+fj7Cw8N5381k6tSpGDlyJAYPHqxxnPe/bhH96clSkp6eDqVSWemhiD4+PpUenki6eXTftN3TW7duqds4ODjAw8OjUptH709NTYW3t3el83t7e2u0qXgdDw8PODg4WMXfnyAImDFjBnr37o3Q0FAAvP+mFhcXh/DwcBQVFcHV1RWbN29GmzZt1B9ivO+ms379epw7dw5nzpyp9Br/u69bGFS0kMlkGj8LglDpGOnHkHtasY229oa0qaumTZuG2NhYHD16tNJrvP+m0bJlS8TExCArKwsbN27EpEmTEBkZqX6d9900kpOT8fbbb2Pv3r1wcnKqsh3vf93AoZ9yvLy8YGtrWykFp6WlVUrMpBtfX18AqPae+vr6ori4GJmZmdW2uX//fqXzP3jwQKNNxetkZmaipKSkzv/9vfXWW9i2bRsOHTqEgIAA9XHef9NycHBASEgIunTpgoiICHTo0AGff/4577uJRUdHIy0tDWFhYbCzs4OdnR0iIyPxxRdfwM7OTv3n5v2vGxhUynFwcEBYWBj27duncXzfvn3o2bOnSFVZtiZNmsDX11fjnhYXFyMyMlJ9T8PCwmBvb6/R5t69e4iPj1e3CQ8PR3Z2Nk6fPq1uc+rUKWRnZ2u0iY+Px71799Rt9u7dC0dHR4SFhZn0zykWQRAwbdo0bNq0CQcPHkSTJk00Xuf9Ny9BEKBQKHjfTWzQoEGIi4tDTEyM+qtLly6YMGECYmJi0LRpU97/usS8c3el79Hy5B9++EG4dOmSMH36dMHFxUVISkoSuzTJys3NFc6fPy+cP39eACAsW7ZMOH/+vHpJ99KlSwW5XC5s2rRJiIuLE8aPH691mWBAQICwf/9+4dy5c8LAgQO1LhNs3769cOLECeHEiRNCu3bttC4THDRokHDu3Dlh//79QkBAQJ1eJvjGG28IcrlcOHz4sHDv3j31V0FBgboN779pzJ49W4iKihISExOF2NhYYc6cOYKNjY2wd+9eQRB4382t/KofQeD9r0sYVLT4+uuvhaCgIMHBwUHo3LmzeqknaXfo0CEBQKWvSZMmCYJQtlRwwYIFgq+vr+Do6Cj07dtXiIuL0zhHYWGhMG3aNMHT01NwdnYWRo0aJdy+fVujzcOHD4UJEyYIbm5ugpubmzBhwgQhMzNTo82tW7eEkSNHCs7OzoKnp6cwbdo0oaioyJR/fFFpu+8AhNWrV6vb8P6bxssvv6z+d6Jhw4bCoEGD1CFFEHjfza1iUOH9rztkgiAI4vTlEBEREVWPc1SIiIhIshhUiIiISLIYVIiIiEiyGFSIiIhIshhUiIiISLIYVIiIiEiyGFSIiIhIshhUiMhi/PTTT6hfv77YZRCRGTGoEJFeJk+eDJlMpv5q0KABhg0bhtjYWL3Os3DhQnTs2NE0RVZj6tSpmDNnDgDgo48+wssvv2z2GohIdwwqRKS3YcOG4d69e7h37x4OHDgAOzs7jBo1SuyydHLixAn06tULAHD06FH190QkTQwqRKQ3R0dH+Pr6wtfXFx07dsSsWbOQnJyMBw8eqNvMmjULLVq0QL169dC0aVPMmzcPJSUlAMqGcBYtWoQLFy6oe2Z++uknAEBWVhZeffVV+Pj4wMnJCaGhodi+fbvG9ffs2YPWrVvD1dVVHZp0kZ+fj/j4eISHh0OlUmmEFiKSJjuxCyAiy5aXl4e1a9ciJCQEDRo0UB93c3PDTz/9BH9/f8TFxWHKlClwc3PDzJkz8cILLyA+Ph67d+/G/v37AQByuRwqlQrDhw9Hbm4u1qxZg2bNmuHSpUuwtbVVn7egoAD/+9//8Ouvv8LGxgYvvvgi3nvvPaxdu7bKGt9880389ttvUKlUKCkpQdOmTSEIAnJyctCjRw8AQGxsLBo3bmyiu0REhmJQISK9bd++Ha6urgDKein8/Pywfft22Ng87qT94IMP1N8HBwfj3XffxYYNGzBz5kw4OzvD1dUVdnZ28PX1Vbfbu3cvTp8+jcuXL6NFixYAgKZNm2pcu6SkBN988w2aNWsGAJg2bRoWL15cbb2LFy/GzJkzsWTJEnVtq1atwpUrV7Bs2TIAgL+/v6G3g4hMiEGFiPQ2YMAArFy5EgCQkZGBFStWYPjw4Th9+jSCgoIAAH/++SeWL1+O69evIy8vD6WlpXB3d6/2vDExMQgICFCHFG3q1aunDikA4Ofnh7S0tGrP6+XlBS8vLxw/fhyff/45goODcebMGUyaNAnBwcE6/qmJSAyco0JEenNxcUFISAhCQkLQrVs3/PDDD8jPz8d3330HADh58iTGjRuH4cOHY/v27Th//jzmzp2L4uLias/r7Oxc47Xt7e01fpbJZBAEocr2a9euhaurK1xdXXH58mWMGTMGrq6uOHDgAF599VW4urpWO2xEROJijwoR1ZpMJoONjQ0KCwsBAMeOHUNQUBDmzp2rbnPr1i2N9zg4OECpVGoca9++PVJSUnD16tVqe1X08eSTT6J79+7YunUrNm3ahJ9//hnHjx/HkiVLsHPnTgCAj4+PUa5FRMbHoEJEelMoFEhNTQUAZGZm4quvvkJeXh5Gjx4NAAgJCcHt27exfv16dO3aFTt27MDmzZs1zhEcHIzExET1cI+bmxv69euHvn37YuzYsVi2bBlCQkJw5coVyGQyDBs2zKBa3dzc4ObmhmvXrmHw4MEICQnBb7/9hgEDBiAkJKR2N4KITI5DP0Skt927d8PPzw9+fn7o3r07zpw5gz/++AP9+/cHADz11FN45513MG3aNHTs2BHHjx/HvHnzNM4xduxYDBs2DAMGDEDDhg2xbt06AMDGjRvRtWtXjB8/Hm3atMHMmTMr9bwY4vDhw+jbty8AIDIyUv09EUmbTKhucJeIiIhIROxRISIiIsliUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAhIiIiyfp/VK70p39kYHoAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"cell_type":"markdown","source":"Translation","metadata":{"id":"ygvrNyB-DvUF"}},{"cell_type":"code","source":"class Translator(tf.Module):\n\n  def __init__(self, encoder, decoder, input_text_processor,\n               output_text_processor):\n    self.encoder = encoder\n    self.decoder = decoder\n    self.input_text_processor = input_text_processor\n    self.output_text_processor = output_text_processor\n\n    self.output_token_string_from_index = (\n        tf.keras.layers.StringLookup(\n            vocabulary=output_text_processor.get_vocabulary(),\n            mask_token='',\n            invert=True))\n\n    # The output should never generate padding, unknown, or start.\n    index_from_string = tf.keras.layers.StringLookup(\n        vocabulary=output_text_processor.get_vocabulary(), mask_token='')\n    token_mask_ids = index_from_string(['', '[UNK]', '[START]']).numpy()\n\n    token_mask = np.zeros([index_from_string.vocabulary_size()], dtype=np.bool)\n    token_mask[np.array(token_mask_ids)] = True\n    self.token_mask = token_mask\n\n    self.start_token = index_from_string(tf.constant('[START]'))\n    self.end_token = index_from_string(tf.constant('[END]'))","metadata":{"id":"PO-CLL1LVBbM","execution":{"iopub.status.busy":"2023-03-16T17:39:56.628844Z","iopub.execute_input":"2023-03-16T17:39:56.629611Z","iopub.status.idle":"2023-03-16T17:39:56.637695Z","shell.execute_reply.started":"2023-03-16T17:39:56.629566Z","shell.execute_reply":"2023-03-16T17:39:56.636484Z"},"trusted":true},"execution_count":148,"outputs":[]},{"cell_type":"code","source":"translator = Translator(\n    encoder=train_translator.encoder,\n    decoder=train_translator.decoder,\n    input_text_processor=input_text_processor,\n    output_text_processor=output_text_processor,\n)","metadata":{"id":"iBQzFZ9uWU79","execution":{"iopub.status.busy":"2023-03-16T17:39:56.793148Z","iopub.execute_input":"2023-03-16T17:39:56.794470Z","iopub.status.idle":"2023-03-16T17:39:57.546904Z","shell.execute_reply.started":"2023-03-16T17:39:56.794408Z","shell.execute_reply":"2023-03-16T17:39:57.544201Z"},"trusted":true},"execution_count":149,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here.\nDeprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n","output_type":"stream"}]},{"cell_type":"code","source":"# converting token IDs to human readable text\ndef tokens_to_text(self, result_tokens):\n  shape_checker = ShapeChecker()\n  shape_checker(result_tokens, ('batch', 't'))\n  result_text_tokens = self.output_token_string_from_index(result_tokens)\n  shape_checker(result_text_tokens, ('batch', 't'))\n\n  result_text = tf.strings.reduce_join(result_text_tokens,\n                                       axis=1, separator=' ')\n  shape_checker(result_text, ('batch'))\n\n  result_text = tf.strings.strip(result_text)\n  shape_checker(result_text, ('batch',))\n  return result_text","metadata":{"id":"8IjwKTwtmdFf","execution":{"iopub.status.busy":"2023-03-16T17:39:57.549319Z","iopub.execute_input":"2023-03-16T17:39:57.549740Z","iopub.status.idle":"2023-03-16T17:39:57.557163Z","shell.execute_reply.started":"2023-03-16T17:39:57.549696Z","shell.execute_reply":"2023-03-16T17:39:57.555921Z"},"trusted":true},"execution_count":150,"outputs":[]},{"cell_type":"code","source":"Translator.tokens_to_text = tokens_to_text","metadata":{"id":"912aV0K7r90w","execution":{"iopub.status.busy":"2023-03-16T17:39:57.558702Z","iopub.execute_input":"2023-03-16T17:39:57.559697Z","iopub.status.idle":"2023-03-16T17:39:57.566470Z","shell.execute_reply.started":"2023-03-16T17:39:57.559651Z","shell.execute_reply":"2023-03-16T17:39:57.565321Z"},"trusted":true},"execution_count":151,"outputs":[]},{"cell_type":"code","source":"# inputting random token IDs to see what it generates\nexample_output_tokens = tf.random.uniform(\n    shape=[5, 2], minval=0, dtype=tf.int64,\n    maxval=output_text_processor.vocabulary_size())\ntranslator.tokens_to_text(example_output_tokens).numpy()","metadata":{"id":"cWCMHdoS32QN","outputId":"b49a9c58-b11b-4aa3-e127-3a7b4f5b98d8","execution":{"iopub.status.busy":"2023-03-16T17:39:57.568863Z","iopub.execute_input":"2023-03-16T17:39:57.569707Z","iopub.status.idle":"2023-03-16T17:39:57.619101Z","shell.execute_reply.started":"2023-03-16T17:39:57.569665Z","shell.execute_reply":"2023-03-16T17:39:57.618059Z"},"trusted":true},"execution_count":152,"outputs":[{"execution_count":152,"output_type":"execute_result","data":{"text/plain":"array([b'suncream priest', b'imupro heber', b'milles cgs', b'dudley br',\n       b'handsome corvallis'], dtype=object)"},"metadata":{}}]},{"cell_type":"code","source":"# takes the decoder's logit outputs and samples token IDs from that distribution\ndef sample(self, logits, temperature):\n  shape_checker = ShapeChecker()\n  # 't' is usually 1 here.\n  shape_checker(logits, ('batch', 't', 'vocab'))\n  shape_checker(self.token_mask, ('vocab',))\n\n  token_mask = self.token_mask[tf.newaxis, tf.newaxis, :]\n  shape_checker(token_mask, ('batch', 't', 'vocab'), broadcast=True)\n\n  # Set the logits for all masked tokens to -inf, so they are never chosen.\n  logits = tf.where(self.token_mask, -np.inf, logits)\n\n  if temperature == 0.0:\n    new_tokens = tf.argmax(logits, axis=-1)\n  else: \n    logits = tf.squeeze(logits, axis=1)\n    new_tokens = tf.random.categorical(logits/temperature,\n                                        num_samples=1)\n  \n  shape_checker(new_tokens, ('batch', 't'))\n\n  return new_tokens","metadata":{"id":"8lfuj3GcdD6e","execution":{"iopub.status.busy":"2023-03-16T17:39:57.621086Z","iopub.execute_input":"2023-03-16T17:39:57.621913Z","iopub.status.idle":"2023-03-16T17:39:57.630581Z","shell.execute_reply.started":"2023-03-16T17:39:57.621874Z","shell.execute_reply":"2023-03-16T17:39:57.629751Z"},"trusted":true},"execution_count":153,"outputs":[]},{"cell_type":"code","source":"Translator.sample = sample","metadata":{"id":"4DpDnBdBdL9_","execution":{"iopub.status.busy":"2023-03-16T17:39:57.789040Z","iopub.execute_input":"2023-03-16T17:39:57.789406Z","iopub.status.idle":"2023-03-16T17:39:57.796297Z","shell.execute_reply.started":"2023-03-16T17:39:57.789371Z","shell.execute_reply":"2023-03-16T17:39:57.794960Z"},"trusted":true},"execution_count":154,"outputs":[]},{"cell_type":"code","source":"# testing the function on random i/p\nexample_logits = tf.random.normal([5, 1, output_text_processor.vocabulary_size()])\nexample_output_tokens = translator.sample(example_logits, temperature=1.0)\nexample_output_tokens","metadata":{"id":"rwLT0nxXym80","outputId":"492baa1e-7a3f-4ee3-a0cb-4df725ece439","execution":{"iopub.status.busy":"2023-03-16T17:39:58.065949Z","iopub.execute_input":"2023-03-16T17:39:58.067146Z","iopub.status.idle":"2023-03-16T17:39:58.091672Z","shell.execute_reply.started":"2023-03-16T17:39:58.067091Z","shell.execute_reply":"2023-03-16T17:39:58.090659Z"},"trusted":true},"execution_count":155,"outputs":[{"execution_count":155,"output_type":"execute_result","data":{"text/plain":"<tf.Tensor: shape=(5, 1), dtype=int64, numpy=\narray([[12756],\n       [17940],\n       [ 5991],\n       [16676],\n       [13349]])>"},"metadata":{}}]},{"cell_type":"code","source":"# implementing text to text translation loop\ndef translate_unrolled(self,\n                       input_text, *,\n                       max_length=50,\n                       return_attention=True,\n                       temperature=1.0):\n  batch_size = tf.shape(input_text)[0]\n  input_tokens = self.input_text_processor(input_text)\n  enc_output, enc_state = self.encoder(input_tokens)\n\n  dec_state = enc_state\n  new_tokens = tf.fill([batch_size, 1], self.start_token)\n\n  result_tokens = []\n  attention = []\n  done = tf.zeros([batch_size, 1], dtype=tf.bool)\n\n  for _ in range(max_length):\n    dec_input = DecoderInput(new_tokens=new_tokens,\n                             enc_output=enc_output,\n                             mask=(input_tokens!=0))\n    \n    dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n\n    attention.append(dec_result.attention_weights)\n\n    new_tokens = self.sample(dec_result.logits, temperature)\n\n    # If a sequence produces an `end_token`, set it `done`\n    done = done | (new_tokens == self.end_token)\n    # Once a sequence is done it only produces 0-padding.\n    new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n\n    # Collect the generated tokens\n    result_tokens.append(new_tokens)\n\n    if tf.executing_eagerly() and tf.reduce_all(done):\n      break\n\n  # Convert the list of generates token ids to a list of strings.\n  result_tokens = tf.concat(result_tokens, axis=-1)\n  result_text = self.tokens_to_text(result_tokens)\n\n  if return_attention:\n    attention_stack = tf.concat(attention, axis=1)\n    return {'text': result_text, 'attention': attention_stack}\n  else:\n    return {'text': result_text}\n","metadata":{"id":"ZmOvVrZmwAxg","execution":{"iopub.status.busy":"2023-03-16T17:39:58.289827Z","iopub.execute_input":"2023-03-16T17:39:58.291300Z","iopub.status.idle":"2023-03-16T17:39:58.320405Z","shell.execute_reply.started":"2023-03-16T17:39:58.291246Z","shell.execute_reply":"2023-03-16T17:39:58.319212Z"},"trusted":true},"execution_count":156,"outputs":[]},{"cell_type":"code","source":"Translator.translate = translate_unrolled","metadata":{"id":"JOmd8Y269MG3","execution":{"iopub.status.busy":"2023-03-16T17:39:58.570219Z","iopub.execute_input":"2023-03-16T17:39:58.571429Z","iopub.status.idle":"2023-03-16T17:39:58.577001Z","shell.execute_reply.started":"2023-03-16T17:39:58.571373Z","shell.execute_reply":"2023-03-16T17:39:58.575644Z"},"trusted":true},"execution_count":157,"outputs":[]},{"cell_type":"code","source":"change_vocab('tum kaise ho')","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:39:58.882755Z","iopub.execute_input":"2023-03-16T17:39:58.883159Z","iopub.status.idle":"2023-03-16T17:39:58.892384Z","shell.execute_reply.started":"2023-03-16T17:39:58.883123Z","shell.execute_reply":"2023-03-16T17:39:58.891112Z"},"trusted":true},"execution_count":158,"outputs":[{"execution_count":158,"output_type":"execute_result","data":{"text/plain":"'tum kassie hoo '"},"metadata":{}}]},{"cell_type":"code","source":"# TESTING\ninput_text = tf.constant([\n    change_vocab('mera naam rohan hai'),\n    change_vocab('kaha jana hai'),\n    change_vocab('tumhare papa kya kaam karte hai '),\n    change_vocab('tumhare papa ka naam kya hai'),\n    change_vocab('maine khana kha liya hai')\n])\n\nresult = translator.translate(\n    input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint(result['text'][2].numpy().decode())\nprint(result['text'][3].numpy().decode())\nprint(result['text'][4].numpy().decode())\nprint()","metadata":{"id":"hd2rgyHwVVrv","outputId":"72a81edc-d510-4038-fc8b-527a5b824127","execution":{"iopub.status.busy":"2023-03-16T17:40:15.051171Z","iopub.execute_input":"2023-03-16T17:40:15.052400Z","iopub.status.idle":"2023-03-16T17:40:16.039222Z","shell.execute_reply.started":"2023-03-16T17:40:15.052348Z","shell.execute_reply":"2023-03-16T17:40:16.038061Z"},"trusted":true},"execution_count":161,"outputs":[{"name":"stdout","text":"my name is loud\nwhere is it ?\ndo you do your papa . ?\nis there a name of your father ?\ni have to take\n\n","output_type":"stream"}]},{"cell_type":"code","source":"@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\ndef tf_translate(self, input_text):\n  return self.translate(input_text)\n\nTranslator.tf_translate = tf_translate","metadata":{"id":"_JhTZ5hOptO-","execution":{"iopub.status.busy":"2023-03-16T17:43:54.731553Z","iopub.execute_input":"2023-03-16T17:43:54.733482Z","iopub.status.idle":"2023-03-16T17:43:54.739930Z","shell.execute_reply.started":"2023-03-16T17:43:54.733439Z","shell.execute_reply":"2023-03-16T17:43:54.738640Z"},"trusted":true},"execution_count":164,"outputs":[]},{"cell_type":"code","source":"tf.saved_model.save(translator, 'translator1',\n                    signatures={'serving_default': translator.tf_translate})","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:43:55.442867Z","iopub.execute_input":"2023-03-16T17:43:55.443608Z","iopub.status.idle":"2023-03-16T17:44:17.215533Z","shell.execute_reply.started":"2023-03-16T17:43:55.443567Z","shell.execute_reply":"2023-03-16T17:44:17.214346Z"},"trusted":true},"execution_count":165,"outputs":[]},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n    input_text = input_text)","metadata":{"id":"_NzrixLvVBjQ","outputId":"469e8d6c-c011-43b1-ead8-0f442e9c602f","execution":{"iopub.status.busy":"2023-03-16T11:27:34.224701Z","iopub.execute_input":"2023-03-16T11:27:34.225096Z","iopub.status.idle":"2023-03-16T11:27:54.830320Z","shell.execute_reply.started":"2023-03-16T11:27:34.225060Z","shell.execute_reply":"2023-03-16T11:27:54.829264Z"},"trusted":true},"execution_count":125,"outputs":[{"name":"stdout","text":"CPU times: user 20.5 s, sys: 90.8 ms, total: 20.6 s\nWall time: 20.6 s\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n    input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint()","metadata":{"id":"USJdu00tVFbd","outputId":"9c40b1e8-a2a9-4ad1-9992-430dd580d8fb","execution":{"iopub.status.busy":"2023-03-16T17:46:33.012957Z","iopub.execute_input":"2023-03-16T17:46:33.013943Z","iopub.status.idle":"2023-03-16T17:46:53.566845Z","shell.execute_reply.started":"2023-03-16T17:46:33.013903Z","shell.execute_reply":"2023-03-16T17:46:53.565732Z"},"trusted":true},"execution_count":170,"outputs":[{"name":"stdout","text":"everyone is in my name ?\nwhere is it ?\n\nCPU times: user 20.3 s, sys: 243 ms, total: 20.5 s\nWall time: 20.5 s\n","output_type":"stream"}]},{"cell_type":"code","source":"#@title [Optional] Use a symbolic loop\ndef translate_symbolic(self,\n                       input_text,\n                       *,\n                       max_length=50,\n                       return_attention=True,\n                       temperature=1.0):\n  shape_checker = ShapeChecker()\n  shape_checker(input_text, ('batch',))\n\n  batch_size = tf.shape(input_text)[0]\n\n  # Encode the input\n  input_tokens = self.input_text_processor(input_text)\n  shape_checker(input_tokens, ('batch', 's'))\n\n  enc_output, enc_state = self.encoder(input_tokens)\n  shape_checker(enc_output, ('batch', 's', 'enc_units'))\n  shape_checker(enc_state, ('batch', 'enc_units'))\n\n  # Initialize the decoder\n  dec_state = enc_state\n  new_tokens = tf.fill([batch_size, 1], self.start_token)\n  shape_checker(new_tokens, ('batch', 't1'))\n\n  # Initialize the accumulators\n  result_tokens = tf.TensorArray(tf.int64, size=1, dynamic_size=True)\n  attention = tf.TensorArray(tf.float32, size=1, dynamic_size=True)\n  done = tf.zeros([batch_size, 1], dtype=tf.bool)\n  shape_checker(done, ('batch', 't1'))\n\n  for t in tf.range(max_length):\n    dec_input = DecoderInput(\n        new_tokens=new_tokens, enc_output=enc_output, mask=(input_tokens != 0))\n\n    dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n\n    shape_checker(dec_result.attention_weights, ('batch', 't1', 's'))\n    attention = attention.write(t, dec_result.attention_weights)\n\n    new_tokens = self.sample(dec_result.logits, temperature)\n    shape_checker(dec_result.logits, ('batch', 't1', 'vocab'))\n    shape_checker(new_tokens, ('batch', 't1'))\n\n    # If a sequence produces an `end_token`, set it `done`\n    done = done | (new_tokens == self.end_token)\n    # Once a sequence is done it only produces 0-padding.\n    new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n\n    # Collect the generated tokens\n    result_tokens = result_tokens.write(t, new_tokens)\n\n    if tf.reduce_all(done):\n      break\n\n  # Convert the list of generated token ids to a list of strings.\n  result_tokens = result_tokens.stack()\n  shape_checker(result_tokens, ('t', 'batch', 't0'))\n  result_tokens = tf.squeeze(result_tokens, -1)\n  result_tokens = tf.transpose(result_tokens, [1, 0])\n  shape_checker(result_tokens, ('batch', 't'))\n\n  result_text = self.tokens_to_text(result_tokens)\n  shape_checker(result_text, ('batch',))\n\n  if return_attention:\n    attention_stack = attention.stack()\n    shape_checker(attention_stack, ('t', 'batch', 't1', 's'))\n\n    attention_stack = tf.squeeze(attention_stack, 2)\n    shape_checker(attention_stack, ('t', 'batch', 's'))\n\n    attention_stack = tf.transpose(attention_stack, [1, 0, 2])\n    shape_checker(attention_stack, ('batch', 't', 's'))\n\n    return {'text': result_text, 'attention': attention_stack}\n  else:\n    return {'text': result_text}","metadata":{"id":"EbQpyYs13jF_","execution":{"iopub.status.busy":"2023-03-16T17:46:53.569825Z","iopub.execute_input":"2023-03-16T17:46:53.570556Z","iopub.status.idle":"2023-03-16T17:46:53.586068Z","shell.execute_reply.started":"2023-03-16T17:46:53.570509Z","shell.execute_reply":"2023-03-16T17:46:53.584852Z"},"trusted":true},"execution_count":171,"outputs":[]},{"cell_type":"code","source":"Translator.translate = translate_symbolic","metadata":{"id":"ngywxv1WYO_O","execution":{"iopub.status.busy":"2023-03-16T17:46:53.587484Z","iopub.execute_input":"2023-03-16T17:46:53.588398Z","iopub.status.idle":"2023-03-16T17:46:53.605490Z","shell.execute_reply.started":"2023-03-16T17:46:53.588358Z","shell.execute_reply":"2023-03-16T17:46:53.604434Z"},"trusted":true},"execution_count":172,"outputs":[]},{"cell_type":"markdown","source":"The initial implementation used python lists to collect the outputs. This uses `tf.range` as the loop iterator, allowing `tf.autograph` to convert the loop. The biggest change in this implementation is the use of `tf.TensorArray` instead of python `list` to accumulate tensors. `tf.TensorArray` is required to collect a variable number of tensors in graph mode. ","metadata":{"id":"lItV7qjEGsYc"}},{"cell_type":"markdown","source":"With eager execution this implementation performs on par with the original:","metadata":{"id":"AJ_NznOgZTxC"}},{"cell_type":"code","source":"# RUN THIS\nresult = translator.translate(\n    input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint()","metadata":{"id":"JRh66y-YYeBw","outputId":"8992e4bf-e4c7-47cb-84a4-0465f2153d83","execution":{"iopub.status.busy":"2023-03-16T17:46:53.608455Z","iopub.execute_input":"2023-03-16T17:46:53.608896Z","iopub.status.idle":"2023-03-16T17:46:54.641135Z","shell.execute_reply.started":"2023-03-16T17:46:53.608858Z","shell.execute_reply":"2023-03-16T17:46:54.640057Z"},"trusted":true},"execution_count":173,"outputs":[{"name":"stdout","text":"peace is as loud\nwhere is it ?\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"But when you wrap it in a `tf.function` you'll notice two differences.","metadata":{"id":"l6B8W4_MZdX0"}},{"cell_type":"code","source":"@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\ndef tf_translate(self, input_text):\n  return self.translate(input_text)\n\nTranslator.tf_translate = tf_translate","metadata":{"id":"WX6EF8KtYh20","execution":{"iopub.status.busy":"2023-03-16T17:46:54.644456Z","iopub.execute_input":"2023-03-16T17:46:54.644768Z","iopub.status.idle":"2023-03-16T17:46:54.651847Z","shell.execute_reply.started":"2023-03-16T17:46:54.644737Z","shell.execute_reply":"2023-03-16T17:46:54.650594Z"},"trusted":true},"execution_count":174,"outputs":[]},{"cell_type":"markdown","source":"First: Graph creation is much faster (~10x), since it doesn't create `max_iterations` copies of the model.","metadata":{"id":"9S0kQ-bBZswZ"}},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n    input_text = input_text)","metadata":{"id":"Eq8d40RKYoJa","outputId":"e8bfbaaa-9b1c-4eaa-d08e-78b76749add4","execution":{"iopub.status.busy":"2023-03-16T17:46:54.653377Z","iopub.execute_input":"2023-03-16T17:46:54.654207Z","iopub.status.idle":"2023-03-16T17:46:57.327429Z","shell.execute_reply.started":"2023-03-16T17:46:54.654122Z","shell.execute_reply":"2023-03-16T17:46:57.326288Z"},"trusted":true},"execution_count":175,"outputs":[{"name":"stdout","text":"CPU times: user 2.58 s, sys: 102 ms, total: 2.68 s\nWall time: 2.66 s\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Second: The compiled function is much faster on small inputs (5x on this example), because it can break out of the loop.","metadata":{"id":"2ABEwtKIZ6eE"}},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n    input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint()","metadata":{"id":"d5VdCLxPYrpz","outputId":"b0e36859-576d-4d40-c6df-efb0b3f7a4c6","execution":{"iopub.status.busy":"2023-03-16T17:46:57.329281Z","iopub.execute_input":"2023-03-16T17:46:57.329764Z","iopub.status.idle":"2023-03-16T17:46:57.390001Z","shell.execute_reply.started":"2023-03-16T17:46:57.329722Z","shell.execute_reply":"2023-03-16T17:46:57.389026Z"},"trusted":true},"execution_count":176,"outputs":[{"name":"stdout","text":"my name is loud\nwhere is it ?\n\nCPU times: user 57 ms, sys: 16.7 ms, total: 73.8 ms\nWall time: 53.4 ms\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Visualize the process","metadata":{"id":"eo5sf4jZaO2l"}},{"cell_type":"markdown","source":"The attention weights returned by the `translate` method show where the model was \"looking\" when it generated each output token.\n\nSo the sum of the attention over the input should return all ones:","metadata":{"id":"FzZzC2cJacTv"}},{"cell_type":"code","source":"a = result['attention'][0]\n\nprint(np.sum(a, axis=-1))","metadata":{"id":"UEd2GljgqQ-0","outputId":"79eecf74-f06e-4d9a-ec44-5338c4df2c8c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is the attention distribution for the first output step of the first example. Note how the attention is now much more focused than it was for the untrained model:","metadata":{"id":"k_HWQHcI2_h5"}},{"cell_type":"code","source":"_ = plt.bar(range(len(a[0, :])), a[0, :])","metadata":{"id":"M8BHdqQujALu","outputId":"31b6618c-07e0-4821-c953-efe35ddc95e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since there is some rough alignment between the input and output words, you expect the attention to be focused near the diagonal:","metadata":{"id":"qB13OG472Z3V"}},{"cell_type":"code","source":"plt.imshow(np.array(a), vmin=0.0)","metadata":{"id":"xyeXuEYHd0kQ","outputId":"8e85df2f-bfc4-43db-abd2-412001afb29e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is some code to make a better attention plot:","metadata":{"id":"mXECcNTn2mxN"}},{"cell_type":"code","source":"#@title Labeled attention plots\ndef plot_attention(attention, sentence, predicted_sentence):\n  sentence = tf_lower_and_split_punct(sentence).numpy().decode().split()\n  predicted_sentence = predicted_sentence.numpy().decode().split() + ['[END]']\n  fig = plt.figure(figsize=(10, 10))\n  ax = fig.add_subplot(1, 1, 1)\n\n  attention = attention[:len(predicted_sentence), :len(sentence)]\n\n  ax.matshow(attention, cmap='viridis', vmin=0.0)\n\n  fontdict = {'fontsize': 14}\n\n  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n\n  ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n  ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n\n  ax.set_xlabel('Input text')\n  ax.set_ylabel('Output text')\n  plt.suptitle('Attention weights')","metadata":{"id":"s5hQWlbN3jGF","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=0\nplot_attention(result['attention'][i], input_text[i], result['text'][i])","metadata":{"id":"rrGawQv2eiA4","outputId":"a8ce33b3-cfaf-49f9-c04f-2f6e89d90eeb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nthree_input_text = tf.constant([#'sabse best friend', \n                                #'That toh I know',\n                                #change_vocab('She was bhunnoing the masalas jub phone ki ghuntee bugee'),\n                                change_vocab('tum batao'),\n                                change_vocab('kya karna hai'),\n])\n\nresult = translator.tf_translate(three_input_text)\nprint(result['text'])\nfor tr in result['text']:\n  print(tr.numpy().decode())\n\nprint()","metadata":{"id":"WrAM0FDomq3E","outputId":"8641124a-e81a-4aea-ff61-51425a2e0b42","execution":{"iopub.status.busy":"2023-03-16T17:46:57.391628Z","iopub.execute_input":"2023-03-16T17:46:57.392024Z","iopub.status.idle":"2023-03-16T17:46:57.421652Z","shell.execute_reply.started":"2023-03-16T17:46:57.391965Z","shell.execute_reply":"2023-03-16T17:46:57.420668Z"},"trusted":true},"execution_count":177,"outputs":[{"name":"stdout","text":"tf.Tensor([b'you know' b'is there any ?'], shape=(2,), dtype=string)\nyou know\nis there any ?\n\nCPU times: user 26.1 ms, sys: 4.04 ms, total: 30.1 ms\nWall time: 23.2 ms\n","output_type":"stream"}]},{"cell_type":"code","source":"result['text']","metadata":{"id":"-LjFp0AljOaZ","outputId":"8f3de323-4809-4d52-8092-bac9d1e1f8fc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nplot_attention(result['attention'][i], three_input_text[i], result['text'][i])","metadata":{"id":"v7QwIMrG-id2","outputId":"02b9f2a4-4ea0-4b45-a294-283b21e1cec9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nplot_attention(result['attention'][i], three_input_text[i], result['text'][i])","metadata":{"id":"zYVoVf8P-lr-","outputId":"59fde2f6-58e3-4cfa-e9b1-f2bba1a872a4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 2\nplot_attention(result['attention'][i], three_input_text[i], result['text'][i])","metadata":{"id":"9sFvlZBk-me4","outputId":"7e25b713-c576-4c03-8614-c55bb3c7d83e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"long_input_text = tf.constant([inp[-1]])\n\nimport textwrap\nprint('Expected output:\\n', '\\n'.join(textwrap.wrap(targ[-1])))","metadata":{"id":"-FUHFLEvSMbG","outputId":"4ea7bcc6-e870-4959-dc40-dd0e6b1ff60a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = translator.tf_translate(long_input_text)\n\ni = 0\nplot_attention(result['attention'][i], long_input_text[i], result['text'][i])\n_ = plt.suptitle('Graph')","metadata":{"id":"lDa_8NaN_RUy","outputId":"e8e9580b-e815-465d-dc0b-e42875d3399c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.saved_model.save(translator, 'translator1',\n                    signatures={'serving_default': translator.tf_translate})","metadata":{"id":"OyvxT5V0_X5B","outputId":"ced85551-7dd8-48b7-cf12-b37055831235","execution":{"iopub.status.busy":"2023-03-16T17:47:01.584098Z","iopub.execute_input":"2023-03-16T17:47:01.584497Z","iopub.status.idle":"2023-03-16T17:47:06.183442Z","shell.execute_reply.started":"2023-03-16T17:47:01.584463Z","shell.execute_reply":"2023-03-16T17:47:06.182299Z"},"trusted":true},"execution_count":178,"outputs":[]},{"cell_type":"code","source":"!cd /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:06.190128Z","iopub.execute_input":"2023-03-16T17:47:06.190452Z","iopub.status.idle":"2023-03-16T17:47:07.219232Z","shell.execute_reply.started":"2023-03-16T17:47:06.190421Z","shell.execute_reply":"2023-03-16T17:47:07.217784Z"},"trusted":true},"execution_count":179,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:07.222283Z","iopub.execute_input":"2023-03-16T17:47:07.223150Z","iopub.status.idle":"2023-03-16T17:47:08.261595Z","shell.execute_reply.started":"2023-03-16T17:47:07.223102Z","shell.execute_reply":"2023-03-16T17:47:08.260165Z"},"trusted":true},"execution_count":180,"outputs":[{"name":"stdout","text":"__notebook_source__.ipynb  translator.zip  translator1\n","output_type":"stream"}]},{"cell_type":"code","source":"!zip -r translator.zip translator1","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:08.265384Z","iopub.execute_input":"2023-03-16T17:47:08.265863Z","iopub.status.idle":"2023-03-16T17:47:19.369936Z","shell.execute_reply.started":"2023-03-16T17:47:08.265818Z","shell.execute_reply":"2023-03-16T17:47:19.368688Z"},"trusted":true},"execution_count":181,"outputs":[{"name":"stdout","text":"updating: translator1/ (stored 0%)\nupdating: translator1/variables/ (stored 0%)\nupdating: translator1/variables/variables.index (deflated 56%)\nupdating: translator1/variables/variables.data-00000-of-00001 (deflated 7%)\nupdating: translator1/saved_model.pb (deflated 74%)\nupdating: translator1/assets/ (stored 0%)\nupdating: translator1/fingerprint.pb (stored 0%)\n","output_type":"stream"}]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'translator.zip')","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:19.372132Z","iopub.execute_input":"2023-03-16T17:47:19.372458Z","iopub.status.idle":"2023-03-16T17:47:19.381320Z","shell.execute_reply.started":"2023-03-16T17:47:19.372424Z","shell.execute_reply":"2023-03-16T17:47:19.380160Z"},"trusted":true},"execution_count":182,"outputs":[{"execution_count":182,"output_type":"execute_result","data":{"text/plain":"/kaggle/working/translator.zip","text/html":"<a href='translator.zip' target='_blank'>translator.zip</a><br>"},"metadata":{}}]},{"cell_type":"code","source":"reloaded = tf.saved_model.load('translator1')\nresult = reloaded.tf_translate(input_text)","metadata":{"id":"-I0j3i3ekOba","execution":{"iopub.status.busy":"2023-03-16T11:27:57.379083Z","iopub.execute_input":"2023-03-16T11:27:57.379373Z","iopub.status.idle":"2023-03-16T11:28:12.032478Z","shell.execute_reply.started":"2023-03-16T11:27:57.379346Z","shell.execute_reply":"2023-03-16T11:28:12.031392Z"},"trusted":true},"execution_count":133,"outputs":[]},{"cell_type":"code","source":"%%time\nresult = reloaded.tf_translate(input_text)\n\nfor tr in result['text']:\n  print(tr.numpy().decode())\n\nprint()","metadata":{"id":"GXZF__FZXJCm","outputId":"2473fa25-b3e1-460d-a647-76c1350f2c81","execution":{"iopub.status.busy":"2023-03-16T11:28:12.034087Z","iopub.execute_input":"2023-03-16T11:28:12.034457Z","iopub.status.idle":"2023-03-16T11:28:12.143606Z","shell.execute_reply.started":"2023-03-16T11:28:12.034419Z","shell.execute_reply":"2023-03-16T11:28:12.142646Z"},"trusted":true},"execution_count":134,"outputs":[{"name":"stdout","text":"where to take ?\nwhat are you doing this , whats love\ndoes your name have his sister\nhave food lost\n\nCPU times: user 127 ms, sys: 11 ms, total: 138 ms\nWall time: 104 ms\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}