{ "cells": [ { "cell_type": "raw", "id": "42dcf697", "metadata": {}, "source": [ "---\n", "title: 18 Neural Machine Translation using Transformer\n", "description: An implementation of Transformer to translate human readabke dates in any format to YYYY-MM-DD format.\n", "---" ] }, { "cell_type": "markdown", "id": "df9402ac", "metadata": {}, "source": [ "\"Colab\"" ] }, { "cell_type": "markdown", "id": "-EKH_E-Z2bBp", "metadata": { "id": "-EKH_E-Z2bBp" }, "source": [ "\"Kaggle\"" ] }, { "cell_type": "markdown", "id": "eea58ad9", "metadata": { "id": "eea58ad9", "papermill": { "duration": 0.01953, "end_time": "2022-04-19T17:09:39.123240", "exception": false, "start_time": "2022-04-19T17:09:39.103710", "status": "completed" }, "tags": [] }, "source": [ "# Neural Machine Translation\n", "\n", "In this notebook we will implement a small transformer model for machine translation task. Our model would be able to translate human readable dates in any format to YYYY-MM-DD format.\n", "\n", "We will be using `faker` module to generate our dataset" ] }, { "cell_type": "code", "execution_count": null, "id": "4f15b8f3", "metadata": { "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_kg_hide-output": true, "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "execution": { "iopub.execute_input": "2022-04-19T17:09:39.165791Z", "iopub.status.busy": "2022-04-19T17:09:39.164311Z", "iopub.status.idle": "2022-04-19T17:09:51.020422Z", "shell.execute_reply": "2022-04-19T17:09:51.019834Z", "shell.execute_reply.started": "2022-04-19T16:26:16.961398Z" }, "id": "4f15b8f3", "papermill": { "duration": 11.87833, "end_time": "2022-04-19T17:09:51.020579", "exception": false, "start_time": "2022-04-19T17:09:39.142249", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "%%capture\n", "!pip install -q faker" ] }, { "cell_type": "code", "execution_count": null, "id": "72747262", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:09:51.064603Z", "iopub.status.busy": "2022-04-19T17:09:51.063890Z", "iopub.status.idle": "2022-04-19T17:09:51.939467Z", "shell.execute_reply": "2022-04-19T17:09:51.939924Z", "shell.execute_reply.started": "2022-04-19T16:26:27.654284Z" }, "id": "72747262", "papermill": { "duration": 0.899865, "end_time": "2022-04-19T17:09:51.940100", "exception": false, "start_time": "2022-04-19T17:09:51.040235", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from tqdm.auto import tqdm\n", "import re, random\n", "\n", "from faker import Faker\n", "from babel.dates import format_date\n", "\n", "pd.options.display.max_colwidth = None\n", "sns.set_style('darkgrid')" ] }, { "cell_type": "code", "execution_count": null, "id": "aaa08590", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:09:51.986753Z", "iopub.status.busy": "2022-04-19T17:09:51.986001Z", "iopub.status.idle": "2022-04-19T17:09:56.373183Z", "shell.execute_reply": "2022-04-19T17:09:56.372645Z", "shell.execute_reply.started": "2022-04-19T16:26:28.610487Z" }, "id": "aaa08590", "papermill": { "duration": 4.41297, "end_time": "2022-04-19T17:09:56.373332", "exception": false, "start_time": "2022-04-19T17:09:51.960362", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "import tensorflow as tf\n", "from tensorflow import keras\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "from tensorflow.keras.models import Model, Sequential\n", "from tensorflow.keras import losses, callbacks, utils, models, Input\n", "from tensorflow.keras import layers as L" ] }, { "cell_type": "markdown", "id": "5aa3910c", "metadata": { "id": "5aa3910c", "papermill": { "duration": 0.019209, "end_time": "2022-04-19T17:09:56.411691", "exception": false, "start_time": "2022-04-19T17:09:56.392482", "status": "completed" }, "tags": [] }, "source": [ "# Data Generation" ] }, { "cell_type": "code", "execution_count": null, "id": "86eeb03e", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:09:56.454573Z", "iopub.status.busy": "2022-04-19T17:09:56.453776Z", "iopub.status.idle": "2022-04-19T17:09:56.456276Z", "shell.execute_reply": "2022-04-19T17:09:56.455821Z", "shell.execute_reply.started": "2022-04-19T16:26:33.512965Z" }, "id": "86eeb03e", "papermill": { "duration": 0.026236, "end_time": "2022-04-19T17:09:56.456385", "exception": false, "start_time": "2022-04-19T17:09:56.430149", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# Constants\n", "class config(): \n", " SAMPLE_SIZE = 10_00_000 \n", " DATE_FORMATS = [\n", " 'short', 'medium', 'long', 'full',\n", " 'd MMM YYY', 'd MMMM YYY', 'dd/MM/YYY',\n", " 'EE d, MMM YYY', 'EEEE d, MMMM YYY', 'd of MMMM YYY',\n", " ]\n", " VALIDATION_SIZE = 0.1\n", " BATCH_SIZE = 32\n", " MAX_EPOCHS = 25\n", " EMBED_DIM = 16\n", " DENSE_DIM = 16\n", " NUM_HEADS = 2\n", " X_LEN = 30\n", " Y_LEN = 14\n", " NUM_ENCODER_TOKENS = 35\n", " NUM_DECODER_TOKENS = 14" ] }, { "cell_type": "code", "execution_count": null, "id": "b0ce1618", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:09:56.497409Z", "iopub.status.busy": "2022-04-19T17:09:56.496869Z", "iopub.status.idle": "2022-04-19T17:09:56.550468Z", "shell.execute_reply": "2022-04-19T17:09:56.551032Z", "shell.execute_reply.started": "2022-04-19T16:26:33.526883Z" }, "id": "b0ce1618", "outputId": "5aba41ad-61a3-4126-f7bc-ed7f5c6e3050", "papermill": { "duration": 0.076264, "end_time": "2022-04-19T17:09:56.551205", "exception": false, "start_time": "2022-04-19T17:09:56.474941", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Sample dates for each format\n", "\n", "d MMMM YYY => 24 October 2007\n", "short => 11/19/75\n", "dd/MM/YYY => 22/12/1982\n", "long => June 11, 2013\n", "d of MMMM YYY => 16 of June 1971\n", "d MMM YYY => 23 Aug 1976\n", "EEEE d, MMMM YYY => Saturday 19, October 2013\n", "EE d, MMM YYY => Sun 12, Nov 2017\n", "full => Friday, May 24, 1996\n", "medium => Jul 24, 2012\n" ] } ], "source": [ "faker = Faker()\n", "print('Sample dates for each format\\n')\n", "\n", "for fmt in set(config.DATE_FORMATS):\n", " print(f'{fmt:20} => {format_date(faker.date_object(), format=fmt, locale=\"en\")}')" ] }, { "cell_type": "code", "execution_count": null, "id": "60475f85", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:09:56.597660Z", "iopub.status.busy": "2022-04-19T17:09:56.596963Z", "iopub.status.idle": "2022-04-19T17:09:56.599245Z", "shell.execute_reply": "2022-04-19T17:09:56.599632Z", "shell.execute_reply.started": "2022-04-19T16:31:12.688221Z" }, "id": "60475f85", "papermill": { "duration": 0.028292, "end_time": "2022-04-19T17:09:56.599753", "exception": false, "start_time": "2022-04-19T17:09:56.571461", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# a utility data cleaning function\n", "def clean_date(raw_date):\n", " return raw_date.lower().replace(',', '')\n", "\n", "# this function will generate our data in a data frame\n", "def create_dataset(num_rows):\n", " dataset = []\n", " \n", " for i in tqdm(range(num_rows)):\n", " dt = faker.date_object()\n", " for fmt in config.DATE_FORMATS:\n", " try:\n", " date = format_date(dt, format=fmt, locale='en')\n", " human_readable = clean_date(date)\n", " machine_readable = f\"@{dt.isoformat()}#\" # adding a start token '@' and end token '#'\n", " except AttributeError as e:\n", " date = None\n", " human_readable = None\n", " machine_readable = None\n", " if human_readable is not None and machine_readable is not None:\n", " dataset.append((human_readable, machine_readable))\n", " \n", " return pd.DataFrame(dataset, columns=['human_readable', 'machine_readable'])" ] }, { "cell_type": "code", "execution_count": null, "id": "6def5add", "metadata": { "colab": { "referenced_widgets": [ "0ac32bd68ce545b59c43e0b6b6e5614d" ] }, "execution": { "iopub.execute_input": "2022-04-19T17:09:56.642243Z", "iopub.status.busy": "2022-04-19T17:09:56.641504Z", "iopub.status.idle": "2022-04-19T17:14:50.497206Z", "shell.execute_reply": "2022-04-19T17:14:50.497826Z", "shell.execute_reply.started": "2022-04-19T16:31:13.00423Z" }, "id": "6def5add", "outputId": "052d7e71-674c-4358-e65c-6e5a43e37b8f", "papermill": { "duration": 293.879505, "end_time": "2022-04-19T17:14:50.498045", "exception": false, "start_time": "2022-04-19T17:09:56.618540", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0ac32bd68ce545b59c43e0b6b6e5614d", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/1000000 [00:00\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
human_readablemachine_readable
006/10/2001@2001-10-06#
1sun 14 aug 2005@2005-08-14#
2thu 5 oct 1972@1972-10-05#
3tuesday 27 october 1970@1970-10-27#
4saturday 5 april 2014@2014-04-05#
\n", "" ], "text/plain": [ " human_readable machine_readable\n", "0 06/10/2001 @2001-10-06#\n", "1 sun 14 aug 2005 @2005-08-14#\n", "2 thu 5 oct 1972 @1972-10-05#\n", "3 tuesday 27 october 1970 @1970-10-27#\n", "4 saturday 5 april 2014 @2014-04-05#" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# generate the dataset using the function defined above\n", "dataset = create_dataset(config.SAMPLE_SIZE)\n", "dataset = dataset.drop_duplicates(subset=['human_readable']).sample(frac=1.0).reset_index(drop=True)\n", "print(dataset.shape)\n", "dataset.head()" ] }, { "cell_type": "markdown", "id": "a05b6bea", "metadata": { "id": "a05b6bea", "papermill": { "duration": 0.021212, "end_time": "2022-04-19T17:14:50.541382", "exception": false, "start_time": "2022-04-19T17:14:50.520170", "status": "completed" }, "tags": [] }, "source": [ "**Define tokenizers for both the languages (human readable and machine readable dates)**" ] }, { "cell_type": "code", "execution_count": null, "id": "c9ecad19", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:14:50.640648Z", "iopub.status.busy": "2022-04-19T17:14:50.625463Z", "iopub.status.idle": "2022-04-19T17:14:52.799618Z", "shell.execute_reply": "2022-04-19T17:14:52.799144Z", "shell.execute_reply.started": "2022-04-19T16:36:12.52806Z" }, "id": "c9ecad19", "outputId": "b948c056-5fef-4a5a-8516-af3dedfd6e0f", "papermill": { "duration": 2.237894, "end_time": "2022-04-19T17:14:52.799745", "exception": false, "start_time": "2022-04-19T17:14:50.561851", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{' ': 1, '1': 2, '2': 3, '9': 4, '0': 5, 'a': 6, 'e': 7, 'r': 8, 'u': 9, '/': 10, '8': 11, '7': 12, 'y': 13, 'o': 14, 'n': 15, 'd': 16, 'm': 17, 't': 18, 's': 19, '3': 20, 'b': 21, '5': 22, '4': 23, '6': 24, 'f': 25, 'j': 26, 'c': 27, 'p': 28, 'l': 29, 'h': 30, 'i': 31, 'g': 32, 'v': 33, 'w': 34}\n", "{'-': 1, '0': 2, '1': 3, '2': 4, '9': 5, '@': 6, '#': 7, '8': 8, '7': 9, '3': 10, '4': 11, '6': 12, '5': 13}\n" ] } ], "source": [ "human_tokenizer = Tokenizer(char_level=True)\n", "machine_tokenizer = Tokenizer(char_level=True)\n", "\n", "human_tokenizer.fit_on_texts(dataset['human_readable'].values)\n", "machine_tokenizer.fit_on_texts(dataset['machine_readable'].values)\n", "\n", "print(human_tokenizer.word_index)\n", "print(machine_tokenizer.word_index)" ] }, { "cell_type": "code", "execution_count": null, "id": "a2446345", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:14:52.847036Z", "iopub.status.busy": "2022-04-19T17:14:52.845469Z", "iopub.status.idle": "2022-04-19T17:14:52.847655Z", "shell.execute_reply": "2022-04-19T17:14:52.848052Z", "shell.execute_reply.started": "2022-04-19T16:36:14.614322Z" }, "id": "a2446345", "papermill": { "duration": 0.02753, "end_time": "2022-04-19T17:14:52.848199", "exception": false, "start_time": "2022-04-19T17:14:52.820669", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# A utility function to clean and tokenize the text and then pad the sequence\n", "def preprocess_input(date, tokenizer, max_len):\n", " seq = [i[0] for i in tokenizer.texts_to_sequences(date.lower().replace(',', ''))]\n", " seq = pad_sequences([seq], padding='post', maxlen=max_len)[0]\n", " return seq" ] }, { "cell_type": "markdown", "id": "a82b63eb", "metadata": { "id": "a82b63eb", "papermill": { "duration": 0.019917, "end_time": "2022-04-19T17:14:52.888246", "exception": false, "start_time": "2022-04-19T17:14:52.868329", "status": "completed" }, "tags": [] }, "source": [ "**Preprocessing the data**" ] }, { "cell_type": "code", "execution_count": null, "id": "c22eef4b", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:14:53.534066Z", "iopub.status.busy": "2022-04-19T17:14:53.044005Z", "iopub.status.idle": "2022-04-19T17:15:07.244362Z", "shell.execute_reply": "2022-04-19T17:15:07.244775Z", "shell.execute_reply.started": "2022-04-19T16:36:14.62205Z" }, "id": "c22eef4b", "outputId": "92c02543-0b91-4219-d5b0-babf4e694409", "papermill": { "duration": 14.336399, "end_time": "2022-04-19T17:15:07.244929", "exception": false, "start_time": "2022-04-19T17:14:52.908530", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 14.3 s, sys: 143 ms, total: 14.4 s\n", "Wall time: 14.3 s\n" ] }, { "data": { "text/plain": [ "((187584, 30), (187584, 14))" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%time\n", "\n", "X = np.array(list(map(lambda x: preprocess_input(x, human_tokenizer, config.X_LEN), dataset['human_readable'])))\n", "y = np.array(list(map(lambda x: preprocess_input(x, machine_tokenizer, config.Y_LEN), dataset['machine_readable'])))\n", "\n", "X.shape, y.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "b196d522", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:15:07.290991Z", "iopub.status.busy": "2022-04-19T17:15:07.290336Z", "iopub.status.idle": "2022-04-19T17:15:07.444257Z", "shell.execute_reply": "2022-04-19T17:15:07.444685Z", "shell.execute_reply.started": "2022-04-19T16:36:28.203578Z" }, "id": "b196d522", "papermill": { "duration": 0.178932, "end_time": "2022-04-19T17:15:07.444844", "exception": false, "start_time": "2022-04-19T17:15:07.265912", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# Cross Validation\n", "from sklearn.model_selection import train_test_split\n", "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=config.VALIDATION_SIZE, random_state=19)" ] }, { "cell_type": "markdown", "id": "aefc8974", "metadata": { "id": "aefc8974", "papermill": { "duration": 0.020711, "end_time": "2022-04-19T17:15:07.486561", "exception": false, "start_time": "2022-04-19T17:15:07.465850", "status": "completed" }, "tags": [] }, "source": [ "A utility function to generate batches of data\n", "* encoder input data would be source language text\n", "* decoder input data would be target language text\n", "* decoder output data would be target language text shifted one time stamp forward (one hot encoded)" ] }, { "cell_type": "code", "execution_count": null, "id": "78ae0af8", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:15:07.534541Z", "iopub.status.busy": "2022-04-19T17:15:07.533826Z", "iopub.status.idle": "2022-04-19T17:15:07.535804Z", "shell.execute_reply": "2022-04-19T17:15:07.536201Z", "shell.execute_reply.started": "2022-04-19T16:36:28.380373Z" }, "id": "78ae0af8", "papermill": { "duration": 0.02915, "end_time": "2022-04-19T17:15:07.536332", "exception": false, "start_time": "2022-04-19T17:15:07.507182", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def generate_batch(X, y, batch_size=config.BATCH_SIZE):\n", " ''' Generate a batch of data '''\n", " while True:\n", " for j in range(0, len(X), batch_size):\n", " encoder_input_data = X[j:j+batch_size]\n", " decoder_input_data = y[j:j+batch_size]\n", " output = y[j:j+batch_size]\n", " decoder_output_data = np.zeros_like(output)\n", " decoder_output_data[:,:-1] = output[:, 1:]\n", " decoder_target_data = utils.to_categorical(decoder_output_data, num_classes=config.NUM_DECODER_TOKENS)\n", " yield([encoder_input_data, decoder_input_data], decoder_target_data)" ] }, { "cell_type": "markdown", "id": "387ce58f", "metadata": { "id": "387ce58f", "papermill": { "duration": 0.020439, "end_time": "2022-04-19T17:15:07.577265", "exception": false, "start_time": "2022-04-19T17:15:07.556826", "status": "completed" }, "tags": [] }, "source": [ "# Model" ] }, { "cell_type": "markdown", "id": "34ca7f45", "metadata": { "id": "34ca7f45", "papermill": { "duration": 0.020627, "end_time": "2022-04-19T17:15:07.618444", "exception": false, "start_time": "2022-04-19T17:15:07.597817", "status": "completed" }, "tags": [] }, "source": [ "Let's define the key components for our model\n", "* Positional Embedding Layer\n", "* Encoder Block\n", "* Decoder Block" ] }, { "cell_type": "code", "execution_count": null, "id": "0a78e9a7", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:15:07.681954Z", "iopub.status.busy": "2022-04-19T17:15:07.681151Z", "iopub.status.idle": "2022-04-19T17:15:07.683582Z", "shell.execute_reply": "2022-04-19T17:15:07.683101Z", "shell.execute_reply.started": "2022-04-19T16:41:41.27963Z" }, "id": "0a78e9a7", "papermill": { "duration": 0.044262, "end_time": "2022-04-19T17:15:07.683694", "exception": false, "start_time": "2022-04-19T17:15:07.639432", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "class PositionalEmbedding(L.Layer):\n", " def __init__(self, sequence_length, input_dim, output_dim, **kwargs):\n", " super().__init__(**kwargs)\n", " self.token_embeddings = L.Embedding(input_dim, output_dim)\n", " self.position_embeddings = L.Embedding(sequence_length, output_dim)\n", " self.sequence_length = sequence_length\n", " self.input_dim = input_dim\n", " self.output_dim = output_dim\n", " \n", " def call(self, inputs):\n", " length = tf.shape(inputs)[-1]\n", " positions = tf.range(start=0, limit=length, delta=1)\n", " embedded_tokens = self.token_embeddings(inputs)\n", " embedded_positions = self.position_embeddings(positions)\n", " return embedded_tokens + embedded_positions\n", " \n", " def get_config(self):\n", " config = super().get_config()\n", " config.update({\n", " \"output_dim\": self.output_dim,\n", " \"sequence_length\": self.sequence_length,\n", " \"input_dim\": self.input_dim,\n", " })\n", " return config\n", " \n", "class TransformerEncoder(L.Layer):\n", " def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n", " super().__init__(**kwargs)\n", " self.embed_dim = embed_dim\n", " self.dense_dim = dense_dim\n", " self.num_heads = num_heads\n", " self.attention = L.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n", " self.dense_proj = keras.Sequential([L.Dense(dense_dim, activation='relu'), L.Dense(embed_dim)])\n", " self.layernorm1 = L.LayerNormalization()\n", " self.layernorm2 = L.LayerNormalization()\n", " \n", " def call(self, inputs, mask=None):\n", " if mask is not None:\n", " mask = mask[: tf.newaxis, :]\n", " attention_output = self.attention(inputs, inputs, attention_mask=mask)\n", " proj_input = self.layernorm1(inputs + attention_output)\n", " proj_output = self.dense_proj(proj_input)\n", " return self.layernorm2(proj_input + proj_output)\n", " \n", " def get_config(self):\n", " config = super().get_confog()\n", " config.update({\n", " \"embed_dim\": self.embed_dim,\n", " \"num_heads\": self.num_heads,\n", " \"dense_dim\": self.dense_dim\n", " })\n", " return config \n", " \n", "class TransformerDecoder(L.Layer):\n", " def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n", " super().__init__(**kwargs)\n", " self.embed_dim = embed_dim\n", " self.dense_dim = dense_dim\n", " self.num_heads = num_heads\n", " self.attention_1 = L.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n", " self.attention_2 = L.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n", " self.dense_proj = keras.Sequential([\n", " L.Dense(dense_dim, activation='relu'), L.Dense(embed_dim)\n", " ])\n", " self.layernorm_1 = L.LayerNormalization()\n", " self.layernorm_2 = L.LayerNormalization()\n", " self.layernorm_3 = L.LayerNormalization()\n", " self.support_masking = False\n", " \n", " def get_config(self):\n", " config = super().get_config()\n", " config.update({\n", " 'embed_dim': self.embed_dim,\n", " \"num_heads\": self.num_heads,\n", " \"dense_dim\": self.dense_dim,\n", " })\n", " return config\n", " \n", " def call(self, inputs, encoder_outputs, mask=None):\n", " if mask is not None:\n", " padding_mask = tf.cast(mask[:, tf.newaxis, :], dtype=\"int32\")\n", " padding_mask = tf.minimum(padding_mask, causal_mask)\n", " attention_output_1 = self.attention_1(\n", " query=inputs, value=inputs, key=inputs, attention_mask=None\n", " )\n", " attention_output_1 = self.layernorm_1(inputs + attention_output_1)\n", " attention_output_2 = self.attention_2(\n", " query=attention_output_1, value=encoder_outputs, key=encoder_outputs, attention_mask=None\n", " )\n", " attention_output_2 = self.layernorm_2(\n", " attention_output_1 + attention_output_2\n", " )\n", " proj_output = self.dense_proj(attention_output_2)\n", " return self.layernorm_3(attention_output_2 + proj_output)" ] }, { "cell_type": "markdown", "id": "59041ce3", "metadata": { "id": "59041ce3", "papermill": { "duration": 0.02078, "end_time": "2022-04-19T17:15:07.725525", "exception": false, "start_time": "2022-04-19T17:15:07.704745", "status": "completed" }, "tags": [] }, "source": [ "Lets Define our model\n", "\n", "* Positional Embedding is applied to Encoder Inputs\n", "* Embedded Encoder Inputs are fed to the Transformer Encoder\n", "* Positional Embedding is applied to Decoder Inputs \n", "* Encoder Outputs and along with the Embedded Decoder Inputs are fed to the transformer Decoder\n", "* We optionally apply dropout to the outputs from the decoder block before feeding it to the output layer" ] }, { "cell_type": "code", "execution_count": null, "id": "6b6b27af", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:15:07.769789Z", "iopub.status.busy": "2022-04-19T17:15:07.769002Z", "iopub.status.idle": "2022-04-19T17:15:11.474914Z", "shell.execute_reply": "2022-04-19T17:15:11.475446Z", "shell.execute_reply.started": "2022-04-19T16:41:42.112793Z" }, "id": "6b6b27af", "outputId": "467a54ba-b7d2-48d9-cf7f-bcc5e6e639f2", "papermill": { "duration": 3.729544, "end_time": "2022-04-19T17:15:11.475593", "exception": false, "start_time": "2022-04-19T17:15:07.746049", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2022-04-19 17:15:07.919085: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:08.014929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:08.015752: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:08.017094: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "2022-04-19 17:15:08.018224: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:08.018906: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:08.019575: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:09.689143: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:09.690007: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:09.690787: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "2022-04-19 17:15:09.691425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15403 MB memory: -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Model: \"model\"\n", "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", "human (InputLayer) [(None, None)] 0 \n", "__________________________________________________________________________________________________\n", "machine (InputLayer) [(None, None)] 0 \n", "__________________________________________________________________________________________________\n", "positional_embedding (Positiona (None, None, 16) 1040 human[0][0] \n", "__________________________________________________________________________________________________\n", "positional_embedding_1 (Positio (None, None, 16) 448 machine[0][0] \n", "__________________________________________________________________________________________________\n", "transformer_encoder (Transforme (None, None, 16) 2768 positional_embedding[0][0] \n", "__________________________________________________________________________________________________\n", "transformer_decoder (Transforme (None, None, 16) 4960 positional_embedding_1[0][0] \n", " transformer_encoder[0][0] \n", "__________________________________________________________________________________________________\n", "dropout (Dropout) (None, None, 16) 0 transformer_decoder[0][0] \n", "__________________________________________________________________________________________________\n", "dense_4 (Dense) (None, None, 14) 238 dropout[0][0] \n", "==================================================================================================\n", "Total params: 9,454\n", "Trainable params: 9,454\n", "Non-trainable params: 0\n", "__________________________________________________________________________________________________\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABdUAAAIECAIAAABwiFA1AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdf1xUdb748c8AM8MwwIDIb0mJyu6SjYqukrKktrBef5AEUonpdu267aOIzHtTs9pHaa7GVu4j20h3t7vlbTX36r1o1nW52l11vAutYFZGF7NCQAFj+BEoyPn+ce6e79nh1zAwcwZ4Pf9iPuczn/OeM8c5b99z5vPRSZIkAAAAAAAA4DY+WgcAAAAAAAAwwlF/AQAAAAAAcC/qLwAAAAAAAO5F/QUAAAAAAMC9/LQOAD146aWXbDab1lEAwAj07rvvah0CMDKRvQCA5pKTk9esWaN1FOgV9794I5vNdurUKa2jwEh26tQpzjG1ffv2VVVVaR0F3Kuqqmrfvn1aRwGMWGQv8Cpc2dW4Ao4Sp06dog7u5bj/xUvNnDmTL2nhPtnZ2YIbAVR0Ot3jjz++dOlSrQOBG+3duzcnJ0frKICRjOwF3oMru5p8BeSf54gnZ/jwZtz/AgAAAAAA4F7UXwAAAAAAANyL+gsAAAAAAIB7UX8BAAAAAABwL+ovAOCUt99+W/dXgYGBDlu/+uqrxYsXNzU11dfXK92mTJnS3t6u7qbeqtPppk2b5sFX0A+vjXzdunV79uzp3qgEM3PmTI8FAwDASEWqQ6oDd6P+AmAAWlpabr755oULF2odiGZ+9atfSZLU0tKibiwrK5s2bVpaWlpwcPDYsWMlSSopKZHb8/Pz1T3lrTabLSwsTJKk0tJSj0bfJ6+N/KGHHlq/fv3TTz+tbvz5z38uSZIkSb6+vh6LBAAwspHnCFIdUh24E/UXAAMgSVJXV1dXV5dWAQQGBs6ePVurvfeoqalp0aJF99xzzyOPPKJuNxqNYWFhhYWF77zzjlaxucbbIk9ISNi/f//mzZv37t2rdSwAgJGMPKdHpDruRqozelB/ATAAQUFBlZWV7733ntaBeJFt27bV1tY+88wzDu3+/v67d+/28fFZvXp1RUWFJrG5xgsjt1qtWVlZTzzxRGdnp9axAABGLPKcHpHqeACpzihB/QUAXCdJ0q5du2bMmBETE9N9a3p6+saNG5ubm7Ozsx1+Y+zlvDDyJUuWVFVVHTp0SOtAAAAYRUh1PIZUZzSg/gLAWQcOHFCmAZMvVOqWCxcu5OTkhISEhIWFLVy4sLKyUn5WQUGB3GHcuHElJSXz5s0LCgoKCAiYM2fOiRMn5D6bNm2S+yj33L7//vtyy9ixY9XjtLa2njhxQt7k5+fn8WPgqLy8/NKlS1artbcOzz77bFpa2pkzZx599NE+xmloaFizZk1CQoLBYAgNDZ0/f/7Ro0flTc4cZFldXV1eXt6ECRMMBkN4eHhmZmZZWZnLL83bIp88ebIQ4oMPPnD5FQEA0AfynB6R6pDqYChJ8D5ZWVlZWVlaR4GRbDDnWEZGhhCira3NoSUjI+PkyZMtLS1HjhwxmUzTp09XP8tqtZrN5uTkZLlPSUnJ7bffbjAYjh07pvQxm82zZs1SPyspKUmeAq2PPrI5c+aMGTPGZrO59qKEEHv27Om7z1tvvSX+OimdQ+MLL7zg0LmkpMRisch/19XVxcXFCSHefvttuUWZ2k1WU1MTHx8fGRlZVFRkt9s///zzzMxMnU63c+dOpU+/B7m6unr8+PGRkZGHDh1qbm4+e/Zsamqqv7//yZMnB3QovDZyu90uhEhJSXEI2NfXd8aMGc68NHllgYEcDAADQPYCr+LMlb1HIzLPcfIKSKqjbeSDT3X4HPZ+ZKLeiH85cDd31F+KiorU4wsh6urqlBb5a5PTp08rLWfOnBFCWK1WpWUweUlqampoaOhAL8AKl+sv27ZtE0Ls2LHDobP60i5Jks1m0+v1ZrP5s88+k7pd2leuXCmEeOedd5SW9vb2mJgYk8lUW1srt/R7kFesWCGE2L17t9KhpqbGaDQmJSU5cQCGR+Q6ne6mm25yaKT+AngJshd4lSGvvwzrPGcw9RdSnWGU6vA57P34/RGAoTF9+nTlb/mbhOrqanUHs9ks31cpmzRpUkxMTHl5eU1NzeD3fuzYsStXriQnJw9+qAGR70/W6/V9d5s5c2ZBQUFra2t2dnZbW5vD1v379wshFixYoLQYjcZ58+a1tbU53IPax0E+cOCAj4+PesnMqKioxMTEjz76qKqqyoWX5oWR+/n5dY8BAAAPGJ15jiDVIdXBkKL+AmBoWCwW5W+DwSCEcFi+MSQkxOEpERERQojLly+7Pzp38ff3F0J0dHT02zMvLy8nJ+fs2bMOazdevXrVbrf7+/sHBQWp2yMjI4UQtbW16sbeDrI8SFdXl8Vi0an85S9/EUJ88cUXLr9Ar4q8s7PTZDIN5rUAAOCa0ZnnCFIdUh0MKa+Y1QnAaNDQ0CBJkk6nU1rkjETOToQQPj4+165dUz+lsbHRYRD1071BdHS0EEL+vW6/du3aVVZW9pvf/EZOZWRGo9Fisdjt9ubmZvXV/dKlS0KIqKgoZ0Y2Go0hISEtLS1tbW3umK7PGyJvamqSJEk+4AAAeJsRmecIUh1SHQwp7n8B4CHt7e0lJSXKw48//ri6utpqtSqXmejo6IsXLyodamtrv/76a4dBAgIClNxl4sSJb7zxhpuj7sdtt90mhHDyrtfAwMA//OEPZrP5tddeU7cvWbJECKFebvDq1avFxcUmkyk9Pd3JSDIzMzs7O5WlFmRbt2694YYbOjs7nRzEmyOXzw35gAMA4G1GZJ4jSHVIdTCkqL8A8BCLxbJhwwabzdba2lpaWpqbm2swGLZv3650SEtLq66ufvXVV1taWiorKx977DHlKyPF1KlTKyoqvvnmG5vNdv78+ZSUFLl97ty5YWFhp06d8tzrEUIIYbVaIyIiysvLneyfmJhYWFjo0Lhly5b4+Pj8/PyDBw82NzdXVFTcf//9NTU127dvl29wdcaWLVsSEhIefPDBw4cP2+32K1euFBYWPvfccwUFBcqXLbm5uTqd7ssvv3RyTO+JXAghL9OYlpbmQvAAALjbiMxzBKkOqQ6GlrbT/6JHzFwNd3PtHJNnIFMsW7bMZrOpW5566ilJktQtCxYskJ9rtVpjY2M//fTT9PT0oKAgk8mUmpp6/Phx9fiNjY2rVq2Kjo42mUyzZ88uKSlJSkqSx3nyySflPufOnUtJSTGbzXFxceqp+FNSUjRZ/0iSpA0bNvj5+V28eFF+WFdXpz4CPc7J//DDDzusd1BfX5+fnx8fH6/X6y0WS3p6enFxsbzJ+YPc0NCwZs2aG2+8Ua/Xh4eHp6WlHTlyRL2XuXPnBgYGdnZ29vgCvTlySZKys7NjY2OvXbvm0M76R4CXIHuBV3Hmyu5gBOc5g1n/SCLVGT6pDp/D3o9M1BvxLwfu5vlzTM5LPLnHAXEmS+stKWlsbIyNjV29erXbohsa3377rclkWrVqldaBuKKsrEyn06lXf1RQfwG8BNkLvIoL9ZfB8PI8Z5D1F1IdDxiSVIfPYe/H748AYFAsFktRUdG+fft27NihdSy9kiQpLy8vODj4+eef1zqWATt//nxmZub69evvvfderWMBAGDUIdVxN1Kd0YP6y3B14MABZfWy9vZ2rcNxo8DAQPVSbQUFBVpH9H+8NjC41cMPP6zT6QIDA9WNU6ZMKS0tPXz4cFNTk1aB9e3SpUvnz58vLi52cq5+r1JYWLh58+bNmzerG9etWyf/07t+/bpWgQEYFrw/ZfLajMJrA4Nbkep4HqnO6EH9Zbi6++67JUnKyMjQOhC3a2lpOX36tBAiIyNDkqS1a9dqHdH/8drAvE1BQYFOpysvL7948aJOp9u4caPWEbkoNzdXuXWwpaXFYeuECRMOHjwYHBysSWz9ioqKOn78eGJiotaBuGLr1q3dvw76+c9/rrwdmsxHCGC4cDJlamlpufnmmxcuXOiZqBx27Z0ZhdcG5lVGTJ4jSHW0Q6ozelB/AfoXGBg4e/ZsraMYrtauXav+0eOmTZu0jggAAEeSJHV1dXV1dWkdiDZIdVxGngPAeX79dwEAAABGtKCgoMrKSq2jAACMZNz/AgAAAAAA4F7UX0aC2tranJyckJCQsLCwhQsXKt/ebNq0SZ60Sbmh9P3335dbxo4dK7eoJ6X76quvcnJygoKCwsLCli9f/u233164cGHRokVBQUHR0dEPPfRQc3OzstPOzs49e/b88Ic/jIqKMplMkyZN2r59u3LXrnrYCxcu9Biey5wZXP4trk6nGzduXElJybx584KCggICAubMmXPixAnnj488Tmtr64kTJ+RNfn4DuGusj6PU2NiontNOvlu1s7NTacnKypIHqaury8vLmzBhgsFgCA8Pz8zMLCsr634oPv/886VLl4aFhckP6+vrB3mcAQAYWm7KOmQNDQ1r1qxJSEgwGo3jxo2766673nzzzba2NnWf3lKm7nP0Op/J9HGZHpIDRapDqgNg5BiylawxdJxfuV2eTC4jI+PkyZMtLS3FxcXBwcHTp09X9zGbzbNmzVK3JCUlhYWFdR8nMzOztLS0paXld7/7nRBi/vz5GRkZp0+fbm5ufv3114UQjz/+uPKUoqIiIcQLL7xw5cqVurq6X/7ylz4+Pg6/gHUI78iRIyaTySG8OXPmjBkzxmaz9fEy1XO/DWhwq9VqNpuTk5PlPiUlJbfffrvBYDh27NiAjk/3Pn0EptbvUUpPT/fx8fnf//1f9bOSk5N3794t/11dXT1+/PjIyMhDhw41NzefPXs2NTXV39//5MmTDociNTX16NGjra2tp06d8vX1raur6y0qaSDn2CghhNizZ4/WUcC99uzZw1UPcJ+BZi9Dm3XU1NTEx8dHRUUVFRU1NTXV1tbKa9C+/PLL6p32nTLJfdra2hxa+kg2nLlMk+pokupwZVfjCjhKkOF7P/4deqOBZjBFRUVKy/333y+EUF+QnK+/HDp0SGmRJw//8MMPlZb4+PiJEycqD4uKiu688071ILm5uXq93m639xGe/EWHOrzU1NTQ0FD1Jba7PpKSvge3Wq1CiNOnTystZ86cEUJYrValxd1JSd9H6YMPPhBC/PSnP1U6HD9+PDY29tq1a/LDFStWCCGUHEWSpJqaGqPRmJSU5HAo3nvvvd7C6I5PZwdkaaMB2SfgVgPNXoY261i5cmX3T/If/ehHDvWXvlOm3uovfSQbzlymSXU0SXW4sqtxBRwlyPC9H78/GgmmT5+u/B0bGyuEqK6udmGcadOmKX/HxMQ4tMTGxqqHXbhw4dGjR9VPt1qtHR0dn3zySR/hxcXFOYR37NixK1euJCcnuxBwv4MLIcxm8+TJk5WHkyZNiomJKS8vr6mpcW2PA9LvUUpLS5s0adKbb77Z0NAgt7z44ouPPvqoXq+XHx44cMDHx0e9HGZUVFRiYuJHH31UVVWlHvn73//+gGLbt2+fDn8lhMjJydE6CrhXTk7OAP8FA3Cjoc069u/fL4SYP3++us/hw4fz8/PVLa6lTH0kG85cpkl1tEp1uLIr5Cug1lHA7fbt2zfAf8HwNNY/GgksFovyt4+PjxDCtdUTg4OD1eP4+voGBAQoLb6+vuph7Xb7L37xi/3791dVVTU2Nirt3333XR/hGQwGl8PrUb+Dh4SEODwlIiKiurr68uXL0dHRQxVGb5w5Svn5+f/wD//w2muvPf300xUVFf/1X//129/+Vt509epVu90u/vZlKr744otx48YpD81m84Bimzlz5uOPPz6gp4xgOTk5+fn5LifHGBZsNtsrr7yidRQA/s8QZh3y5dLf3z8oKKjvnbqWMvWWbAzoMu0yUh3l4UBTHa7sCvkKKN8FgxHs5Zdf1joE9IP6y8jn4+Nz7do1dYv66uiyRYsW/elPf9q+fft99903duxYnU73yiuvyD/VHvzgQ6ihoUGSJJ1Op7RcvnxZCBERESE/dOb4qJ8+IM4cpWXLlm3YsOHVV1/953/+51/84hcrVqwIDQ2VNxmNxpCQkJaWlra2tgFNhueMcePGLV26dGjHHL5ycnKSk5M5ICMe9RdgmOr7emo0Gi0Wi91ub25u7rcEM4Tcepl2HqlOb7iyq73yyiscjRHv3Xff1ToE9IPfH4180dHRFy9eVB7W1tZ+/fXXgxzz+vXrJ06ciIqKysvLCw8Pl6/ZDksMeIn29vaSkhLl4ccff1xdXW21WpVvhJw5PgEBAUriMnHixDfeeKPf/fr5+X3yySfOHCWj0fjTn/708uXLv/jFL3bv3v3YY4+pt2ZmZnZ2dioLGci2bt16ww03dHZ29hsGAADDnTNZx5IlS4QQ7733nrpxypQp7r7T0xsu06Q6ADBcUH8Z+dLS0qqrq1999dWWlpbKysrHHntM+T7EZb6+vnfeeWdtbe2LL75YX1/f1tZ29OhRebWCgZo7d25YWNipU6cGGVJvLBbLhg0bbDZba2traWlpbm6uwWDYvn270sGZ4zN16tSKiopvvvnGZrOdP38+JSXFmV07f5R++tOfmkymjRs33nXXXTfddJN605YtWxISEh588MHDhw/b7fYrV64UFhY+99xzBQUFGn7VBgCAxzhzPd2yZUt8fPzjjz8ur6FTVVX105/+tKamxt31F2cu06Q6glQHAGRaTv6LXjgzc7XNZlO/j0899ZT0tz/8WbBggdyzsbFx1apV0dHRJpNp9uzZJSUlSUlJcp8nn3yy+zjqr1CEEFu2bPnTn/6kbnn22WclSaqrq1u9enVcXJxer4+MjFy5cuW6devkDklJSc6Hl5KS0veiAA6/9X3xxRedH9xqtcbGxn766afp6elBQUEmkyk1NfX48ePq8fs+PnKfc+fOpaSkmM3muLi4HTt29BhYd5999lnfR0kdxkMPPST+dukHRUNDw5o1a2688Ua9Xh8eHp6WlnbkyJEeTwPh9L9oZkd3IFglYRRg9QfArVzLXoYk65AHr6+vz8/Pj4+P1+v10dHR9957b0VFRY87lbqlDfL0vYply5Y5n2z0cZmWkerIPJzqCK7sKlwBRwkyfO+nk7xstg4IIbKzswW/3xsKkydPrq+vd5g83zv99re/3bFjR2lpqWd2xznmQKfT7dmzh99Fj2x79+7Nycnhqge4CVcWTZDq9IYruxpXwFGCz2Hvx++PAK/w+uuvr1mzRuso0Je3335bWd4vMDDQYetXX321ePHipqam+vp6pduUKVPa29vV3dRbdTqderFVzXlt5OvWreu+ZMO6deuUYGbOnOmxYAAAriHV8X6kOqQ6cDfqL4Bmdu3atWTJkpaWltdff/3bb7/lK5ph4Ve/+pUkSS0tLerGsrKyadOmpaWlBQcHjx07VpIk+Y76srKy/Px8dU95q81mCwsLkyTJY18DOsNrI3/ooYfWr1//9NNPqxt//vOfy7dx+vr6eiwSAMCAkOoMR6Q6pDpwH+ovGJkKCgp0Ol15efnFixd1Ot3GjRu1jqhnBw4cCA0N/dWvfvX73/9+BE8yFxgYOHv27OE7ft+ampoWLVp0zz33PPLII+p2o9EYFhZWWFj4zjvvaBWba7wt8oSEhP3792/evHnv3r1axwIA3oJUx3uM7DxHkOq4H6nO6EH9BSPT2rVr1RMdbdq0SeuIerBq1SpJkjo6OsrLy6dOnap1OHDRtm3bamtrn3nmGYd2f3//3bt3+/j4rF69uqKiQpPYXOOFkVut1qysrCeeeIK1SAFARqoDjyHV8QBSnVGC+gsAuE6SpF27ds2YMSMmJqb71vT09I0bNzY3N2dnZzv8xtjLeWHkS5YsqaqqOnTokNaBAAAwipDqeAypzmhA/QVAX+QlIRMSEgwGQ2ho6Pz5848ePSpv2rRpkzwlmHJP7Pvvvy+3jB07Vm6R745ubW09ceKEvEm+91hu1+l048aNKykpmTdvXlBQUEBAwJw5c06cODH48T2mvLz80qVLVqu1tw7PPvtsWlramTNnHn300T7G6eM4HzhwQJl97cKFCzk5OSEhIWFhYQsXLqysrFQPUldXl5eXN2HCBIPBEB4enpmZWVZW5vJL87bIJ0+eLIT44IMPXH5FAAA4IM/pF6kOqQ6G0pCvaI3BY+V2uJuT51hNTU18fHxkZGRRUZHdbv/8888zMzN1Ot3OnTuVPmazedasWepnJSUlyfOW9dFHZrVazWZzcnLyyZMnW1paSkpKbr/9doPBcOzYsSEZf86cOWPGjLHZbP2+UiHEnj17+u7z1ltvib9OSufQ+MILLzh0LikpsVgs8t91dXVxcXFCiLfffltuUaZ2kzlznDMyMoQQGRkZ8rE6cuSIyWSaPn260qG6unr8+PGRkZGHDh1qbm4+e/Zsamqqv7//yZMn+335wyJyu90uhEhJSXEI2NfXd8aMGc68NHllgYEcDAADQPYCr+LMlX305DlOXgFJdbSNfPCpDp/D3o9M1BvxLwfu5uQ5tnLlSiHEO++8o7S0t7fHxMSYTKba2lq5ZZB5iRDi9OnTSsuZM2eEEFartY/nOj9+ampqaGioM1dll+sv27ZtE0Ls2LHDobP60i5Jks1m0+v1ZrP5s88+k7pd2p05zvKlvaioSOmTlZUlhKirq5MfrlixQgixe/dupUNNTY3RaExKSur35Q+XyHU63U033eTQSP0F8BJkL/AqzlzZR0+eM5j6C6nOMEp1+Bz2fvz+CECv9u/fL4RYsGCB0mI0GufNm9fW1jZU90aazWb5ZkvZpEmTYmJiysvLa2pqBj/4sWPHrly5kpycPPiheiP/YFiv1/fdbebMmQUFBa2trdnZ2W1tbQ5bnT/O06dPV/6Wv66prq6WHx44cMDHx2fhwoVKh6ioqMTExI8++qiqqsqFl+aFkfv5+XWPAQAA15DnOINUx5ORk+qMeNRfAPTs6tWrdrvd398/KChI3R4ZGSmEqK2tHZK9hISEOLREREQIIS5fvjwk47ubv7+/EKKjo6Pfnnl5eTk5OWfPnnVYu3FAx9lisSh/GwwGIURXV5cySFdXl8Vi0an85S9/EUJ88cUXLr9Ar4q8s7PTZDIN5rUAACAjz3ESqQ6pDoaQpydwAjBcGI1Gi8Vit9ubm5vVV51Lly4JIaKiouSHPj4+165dUz+xsbHRYSidTtfbXhoaGiRJUneQMxI5Oxn8+O4WHR0thJB/r9uvXbt2lZWV/eY3v5FTGZmTx7lvRqMxJCSkpaWlra3NHTPzeUPkTU1NkiTJBxwAgEEiz3ESqQ6pDoYQ978A6NWSJUuEEOpl8K5evVpcXGwymdLT0+WW6OjoixcvKh1qa2u//vprh3ECAgKU3GLixIlvvPGGsqm9vb2kpER5+PHHH1dXV1utVuXaM8jx3e22224TQjh512tgYOAf/vAHs9n82muvqdudOc79yszM7OzsVFZVkG3duvWGG27o7Ox0chBvjlw+DeQDDgDA4JHnOINUx2ORk+qMBtRfAPRqy5Yt8fHx+fn5Bw8ebG5urqiouP/++2tqarZv3y7feCmESEtLq66ufvXVV1taWiorKx977DHlKx3F1KlTKyoqvvnmG5vNdv78+ZSUFGWTxWLZsGGDzWZrbW0tLS3Nzc01GAzbt29XOgxm/Llz54aFhZ06dWroD81fWa3WiIiI8vJyJ/snJiYWFhY6NDpznPu1ZcuWhISEBx988PDhw3a7/cqVK4WFhc8991xBQYHyZUtubq5Op/vyyy+dHNN7IhdCyMs0pqWluRA8AADdkec4g1SHVAdDSdvpf9EjZq6Guzl/jtXX1+fn58fHx+v1eovFkp6eXlxcrO7Q2Ni4atWq6Ohok8k0e/bskpKSpKQk+ePlySeflPucO3cuJSXFbDbHxcWp58+3Wq2xsbGffvppenp6UFCQyWRKTU09fvz4UI2fkpLi7vWPJEnasGGDn5/fxYsX5Yd1dXXqz9ge5+R/+OGHHZY26OM422w29YBPPfWUHLBiwYIFcs+GhoY1a9bceOONer0+PDw8LS3tyJEj6r3MnTs3MDCws7OzxxfozZFLkpSdnR0bG3vt2jWHdtY/ArwE2Qu8ijNXdmnU5DmDWf9IItUZPqkOn8Pej0zUG/EvB+7mJeeYnJdoHYUkDa7+0tjYGBsbu3r1ardFNzS+/fZbk8m0atUqrQNxRVlZmU6nU6/+qKD+AngJL7myADIn6y9u5T15ziDrL6Q6HjAkqQ6fw96P3x8BwKBYLJaioqJ9+/bt2LFD61h6JUlSXl5ecHDw888/r3UsA3b+/PnMzMz169ffe++9WscCAMCoQ6rjbqQ6owf1FwAYgIcfflin0wUGBqobp0yZUlpaevjw4aamJq0C69ulS5fOnz9fXFzs5Fz9XqWwsHDz5s2bN29WN65bt05evvH69etaBQYAwMhDquN5pDqjB/UXABooKCjQ6XTl5eUXL17U6XQbN27UOqL+5ebmKrcOtrS0OGydMGHCwYMHg4ODNYmtX1FRUcePH09MTNQ6EFds3bq1+9dBP//5z5W3w91TDwIAMCDDMc8RpDraIdUZPYZ+7XQA6NfatWvXrl2rdRQAAABDjzwHQI+4/wUAAAAAAMC9qL8AAAAAAAC4F/UXAAAAAAAA96L+AgAAAAAA4F7Mv+ulqqqq9u7dq3UUGLGqqqqEEJxjajabTesQ4F68xYC7kb3Aq/Cxr5APBf88R7yqqqpx48ZpHQX6opMkSesY4Cg7O3vfvn1aRwEAIxBXPcBNyF4AQHNZWVnvvvuu1lGgV9RfAAyKTqfbs2fP0qVLtQ4EAADAdXIyw00iANyH+V8AAAAAAADci/oLAAAAAACAe1F/AQAAAAAAcC/qLwAAAAAAAO5F/QUAAAAAAMC9qL8AAAAAAAC4F/UXAAAAAAAA96L+AgAAAAAA4F7UXwAAAAAAANyL+gsAAAAAAIB7UX8BAAAAAABwL+ovAAAAAAAA7kX9BQAAAAAAwL2ovwAAAAAAALgX9RcAAAAAAAD3ov4CAAAAAADgXtRfAAAAAAAA3Iv6CwAAAAAAgHtRf3dhPDQAACAASURBVAEAAAAAAHAv6i8AAAAAAADuRf0FAAAAAADAvai/AAAAAAAAuBf1FwAAAAAAAPei/gIAAAAAAOBe1F8AAAAAAADci/oLAAAAAACAe1F/AQAAAAAAcC/qLwAAAAAAAO5F/QUAAAAAAMC9qL8AAAAAAAC4F/UXAAAAAAAA96L+AgAAAAAA4F7UXwAAAAAAANzLT+sAAAwzO3fuvHLlirrl3//937/88kvl4Y9//OOIiAiPxwUAADAA//3f/22z2ZSH586dE0Js3bpVaUlOTv7BD36gQWQARiidJElaxwBgOPnJT35SWFhoNBq7b+ro6AgNDa2trfXzo7YLAAC8WnFx8V133aXX6318HH8T0NXV1dHR8cc//nHevHmaxAZgRKL+AmBgPvzwwzvvvLPHTXq9/ic/+ckvf/lLz0YEAAAwYF1dXVFRUXV1dT1uHTt2bG1tra+vr4ejAjCCMf8LgIH5wQ9+EB0d3eOmjo6O++67z8PxAAAAuMDHx2fZsmUGg6H7JoPBkJubS/EFwNCi/gJgYHQ6XW/JSkxMzMyZMz0fEgAAgAvuu+++a9eudW+/du0aXykBGHLUXwAMWI/JisFgWLFihU6n0yQkAACAgfr+978/fvz47u1xcXHTp0/3fDwARjbqLwAGbOrUqTfddJNDI98UAQCAYWf58uV6vV7dotfrV65cyVdKAIYc9RcArsjNzXVIVm666aZJkyZpFQ8AAIALli1b1tHRoW7p6OjIycnRKh4AIxj1FwCuyM3N7ezsVB7q9fof//jHGsYDAADggltvvTUxMVF9t8v3vve9xMREDUMCMFJRfwHgioSEhNtvv11JVjo7O/nxEQAAGI4eeOABZakjvV6/YsUKbeMBMFJRfwHgIiVZ0el0SUlJ8fHxWkcEAAAwYPfdd9/169flvzs7O5cuXaptPABGKuovAFx03333dXV1CSF8fX0feOABrcMBAABwRVxc3IwZM3x8fHx8fGbMmDFhwgStIwIwMlF/AeCi6OjoWbNm6XS6rq6u7OxsrcMBAABw0fLly3U6nY+Pz/Lly7WOBcCIRf0FgOuWL18uSdKdd94ZFRWldSwAAAAukn9zJElSVlaW1rEAGLkklT179mgdDgAAGOGysrKkQeP/SAAAwMs55Dx+3XtQhQHgvJdffvkf//EfzWaz1oFowGazvfLKK3xmKl5++WUhxOOPP651IPBq8nkyJGbOnMn5BmCo/Pd//7dOp0tJSdE6EG9EzuOAnAfO6J7z9FB/YcZvAM6bPXt2TEyM1lFo5pVXXuEzU/Huu+8KLiLoj3yeDIlx48ZxvgEYKvPnzxdCBAUFaR2IlyLnUSPngTO65zw91F8AwHmjufgCAABGDCovANyN+XcBAAAAAADci/oLAAAAAACAe1F/AQAAAAAAcC/qLwCA4eSrr75avHhxU1NTfX297q+mTJnS3t6u7qbeqtPppk2bplXA3Xlt5OvWrWNtCwAAvAQ5j/tolfNQfwEAT2tpabn55psXLlyodSDDT1lZ2bRp09LS0oKDg8eOHStJUklJidyen5+v7ilvtdlsYWFhkiSVlpZqFHIPvDbyhx56aP369U8//bTH9ggAGNnIeVxGzuNWWuU81F8AwNMkSerq6urq6tIqgMDAwNmzZ2u1d5c1NTUtWrTonnvueeSRR9TtRqMxLCyssLDwnXfe0So213hb5AkJCfv379+8efPevXu1jgUAMBKQ87iGnMfdtMp5qL8AgKcFBQVVVla+9957WgcyzGzbtq22tvaZZ55xaPf399+9e7ePj8/q1asrKio0ic01Xhi51WrNysp64oknOjs7tY4FADDskfO4hpzHAzTJeai/AACGAUmSdu3aNWPGjJiYmO5b09PTN27c2NzcnJ2d7fDTYi/nhZEvWbKkqqrq0KFDWgcCAMBoRM7jMZ7Peai/AIBHHThwQJlmTL72qFsuXLiQk5MTEhISFha2cOHCyspK+VkFBQVyh3HjxpWUlMybNy8oKCggIGDOnDknTpyQ+2zatEnuo9xn+/7778stY8eOVY/T2tp64sQJeZOfn5/Hj4ErysvLL126ZLVae+vw7LPPpqWlnTlz5tFHH+1jnIaGhjVr1iQkJBgMhtDQ0Pnz5x89elTe5MwbIaurq8vLy5swYYLBYAgPD8/MzCwrK3P5pXlb5JMnTxZCfPDBBy6/IgAABDmPq8h5RnLOI6nIMwBLAAAnDOYzMyMjQwjR1tbm0JKRkXHy5MmWlpYjR46YTKbp06ern2W1Ws1mc3JystynpKTk9ttvNxgMx44dU/qYzeZZs2apn5WUlCTPatZHH9mcOXPGjBljs9lce1FZWVlZWVmuPbdfb731lhDihRdecGgvKSmxWCzy33V1dXFxcUKIt99+W25RZnST1dTUxMfHR0ZGFhUV2e32zz//PDMzU6fT7dy5U+nT7xtRXV09fvz4yMjIQ4cONTc3nz17NjU11d/f/+TJkwN6RV4bud1uF0KkpKQM6OU4b6jOE7eebwAANXIeB+Q8A3pFXhu553Me6i8A4CJ35CJFRUVKS1ZWlhCirq5OaZG/CTl9+rTScubMGSGE1WpVWgaTi6SmpoaGhg70mqoO2H25yLZt24QQO3bscGhXX9ElSbLZbHq93mw2f/bZZ1K3K/rKlSuFEO+8847S0t7eHhMTYzKZamtr5ZZ+34gVK1YIIXbv3q10qKmpMRqNSUlJA3pF3hy5Tqe76aabBvRynEf9BQCGHXIeB+Q8A3pF3hy5h3Mefn8EAF5k+vTpyt/ylwPV1dXqDmazWb5VUjZp0qSYmJjy8vKamprB7/3YsWNXrlxJTk4e/FBDTr5vWa/X991t5syZBQUFra2t2dnZbW1tDlv3798vhFiwYIHSYjQa582b19bW5nDraR9vxIEDB3x8fNRLaUZFRSUmJn700UdVVVUuvDQvjNzPz697DAAADCFynt6Q83gycg/nPNRfAMCLWCwW5W+DwSCEcFiyMSQkxOEpERERQojLly+7Pzot+fv7CyE6Ojr67ZmXl5eTk3P27FmHJRuvXr1qt9v9/f2DgoLU7ZGRkUKI2tpadWNvb4Q8SFdXl8Vi0an85S9/EUJ88cUXLr9Ar4q8s7PTZDIN5rUAANA3cp7ekPOM4JxneExBBACQNTQ0yLdKKi1yFiJnJEIIHx+fa9euqZ/S2NjoMIj66cNFdHS0EEL+mW6/du3aVVZW9pvf/EbOYGRGo9Fisdjt9ubmZvVF/dKlS0KIqKgoZ0Y2Go0hISEtLS1tbW3umMbPGyJvamqSJEk+4AAAaIWcx5nO3pA5uMYbIvd8zsP9LwAwnLS3t5eUlCgPP/744+rqaqvVqlw5oqOjL168qHSora39+uuvHQYJCAhQ8pWJEye+8cYbbo56CNx2221CCCdvdg0MDPzDH/5gNptfe+01dfuSJUuEEOpVBq9evVpcXGwymdLT052MJDMzs7OzU1mCQbZ169Ybbrihs7PTyUG8OXL5/JEPOAAAWiHncaazN2QOrvGGyD2f81B/AYDhxGKxbNiwwWaztba2lpaW5ubmGgyG7du3Kx3S0tKqq6tfffXVlpaWysrKxx57TPmaSDF16tSKiopvvvnGZrOdP38+JSVFbp87d25YWNipU6c893qcZrVaIyIiysvLneyfmJhYWFjo0Lhly5b4+Pj8/PyDBw82NzdXVFTcf//9NTU127dvl+9rdcaWLVsSEhIefPDBw4cP2+32K1euFBYWPvfccwUFBcp3LLm5uTqd7ssvv3RyTO+JXAghr86YlpbmQvAAAAwVch4n+5PzuBa50CTnUU/Gy/pHAOA81z4z5UnFFMuWLbPZbOqWp556SpIkdcuCBQvk51qt1tjY2E8//TQ9PT0oKMhkMqWmph4/flw9fmNj46pVq6Kjo00m0+zZs0tKSpKSkuRxnnzySbnPuXPnUlJSzGZzXFycenb9lJQUr10LQJKkDRs2+Pn5Xbx4UX5YV1enPko9TsX/8MMPO6yDUF9fn5+fHx8fr9frLRZLenp6cXGxvMn5N6KhoWHNmjU33nijXq8PDw9PS0s7cuSIei9z584NDAzs7Ozs8YV4c+SSJGVnZ8fGxl67dq3H4AeP9Y8AYNgh53FAzqMg5+lD9/NEpw537969OTk5Di8AANAjz39mTp48ub6+fjATzrtVdna2EOLdd9910/h2uz0xMXHhwoWvv/66m3YxJBobG2NiYpYtW7Zz506tYxmw8vLyKVOm/Ou//uu9997rpl0M1Xni7vMNAKAg53FAziMj5+lb9/OE3x+5oqCgQJ5Cedy4cSMsht///vfysOppkFzo42GBgYG6v+Xj4xMeHn733XerfzU6gvV4PnQ/LA5KS0uHdo+DN0zPQHiGxWIpKirat2/fjh07tI6lV5Ik5eXlBQcHP//881rHMmDnz5/PzMxcv369+xIReB5Ji7ddMt57771bbrnF5cksyXnIeTAakPO4m1Y5j5fWX1paWm6++Wb1et2D7zmE1q5dK0mS1Wr15E49E8O9994rSdK8efMG2keTN0K999OnTwshMjIy5Du7vv322zfeeMNms82aNeuPf/zjkOxi2J2T3Q+LmnrBtqHa4+AN0zMQHjNlypTS0tLDhw83NTVpHUvPLl26dP78+eLiYien6PcqhYWFmzdv3rx5s9aBDDPD8QLhYVwyZJWVlYsXL16/fr28kIdryHnIech5RglyHrfSKufx0vqLJEldXV0OK8AHBgbOnj3bmZ7wPG97IywWy5IlS1566aWOjo78/PzBD8g56eVG/GGXv3wrLy+/ePGiTqfbuHGj1hFpZsKECQcPHgwODtY6kJ5FRUUdP348MTFR60BcsXXrVu58cQEXiGFHqzfi6aefvuOOOz766CP1oqqDR84z2oz4w07OoyDncR+tcp6hX8d7SAQFBVVWVg5tT7iVd74Rc+bMEUJ88sknjY2NISEhgxlq5J2TjY2NWocwlIbLYXfZ2rVr165dq3UUAHow8i4QI55Wb8Svf/1rk8nkpsHJefpAzjO8kPNgBPPS+1+AIaFMEqbT6bSNxKvMnj37zTff1DoKAABGF/cVXwQ5Ty/IeQB4lQHXX9QTUJWUlMybNy8oKCggIGDOnDknTpxQ95QXfEpISDAYDKGhofPnzz969Kiy9erVq88888ytt94aEBAwZsyYRYsW/cd//Mf169eFEAcOHFDmympvb1d22traeuLECbldnrSse89+d61+yoULF3JyckJCQsLCwhYuXKguJHd2du7Zs+eHP/xhVFSUyWSaNGnS9u3bB3ObX11dXV5e3oQJEwwGQ3h4eGZmprzYuENIX331VU5OTlBQUFhY2PLly7/99tsLFy4sWrQoKCgoOjr6oYceam5u7j74uXPnFixYYLFYenwj+ti18vS7777bYrGYzeaUlJTjx4/3uIs++nR/I5w8zuqRAwICvv/97x88ePCuu+6Sn7hq1SqXDvb/d+zYMSFEYmKi/LtfzskecQa67wwEoC2SFteQtAzHSwY5jzM4gb32BAZGC/XcVM6v6261Ws1mc3Jy8smTJ1taWkpKSm6//XaDwXDs2DG5Q01NTXx8fGRkZFFRkd1u//zzzzMzM3U63c6dO+UOq1atslgs//mf//ndd9/V1tbK95gdPXpU2UVGRoYQoq2tTWkxm82zZs3qHoxDz353rTwlIyNDjv/IkSMmk2n69OlKh6KiIiHECy+8cOXKlbq6ul/+8pc+Pj7y5FvqgxAbG+vM4aqurh4/fnxkZOShQ4eam5vPnj2bmprq7++vXnBeDikzM7O0tLSlpeV3v/udEGL+/PkZGRmnT59ubm6W1x57/PHHHWKwWCxz5sw5fvx4c3Nz9zei311/8cUXISEhsbGx//mf/9nc3HzmzJm0tLQJEyYYjUZlL8706fEt6/c4O4x89uzZu+66Kzw83GHkOXPmjBkzxmaz9XGQHSZds9vt//Zv/xYREaHX6+WV3kfnOSkflu5++9vf9hgzZ2CPZ2BvnP/MHCWysrKysrK0jgLebqjOE+fHIWmRSFq84JKhFhsb6+vr2+Mmch6JnMcrT2ByHgfkPHBG9/PE9fqLEOL06dNKy5kzZ4QQVqtVfrhy5UohxDvvvKN0aG9vj4mJMZlMtbW1kiTFx8ffcccd6jFvueWWIfnc73fXylOKioqUPllZWUKIuro6+WFRUdGdd96p3ktubq5er7fb7eqD4GQqs2LFCiHE7t27lZaamhqj0ZiUlOTwKg4dOqS0yFMZffjhh0pLfHz8xIkT1SPLb4T6Iu3wRvS7a3lN8n379ikdLl68aDQa1Z/FzvSRer8S9HGcu498+fLlgIAAh5FTU1NDQ0PVmV93DhddnU4XFha2ePHiP//5z3KH0XlO9rgWwKxZs3rLRTgDezwDe0Mu4oBcBM7QpP5C0kLSovklQ62P+gs5j0TO45UnMDmPA3IeOKP7eeL6/Ltms3ny5MnKw0mTJsXExJSXl9fU1ERHR+/fv18IsWDBAqWD0WicN2/eW2+99cEHHzzwwAM/+tGPfvWrX/3jP/7jgw8+OH36dF9f388//9zlYNT63bXSPn36dOXvuLg4IUR1dfXYsWOFEAsXLnRY1M1qtb799tuffPJJcnLyQEM6cOCAj4+PesCoqKjExMSPPvqoqqpq3LhxSvu0adOUv2NiYj755BN1S2xsbHl5ucPg/v7+M2bMUB46vBH97vr9998XQqSnp6v3e8stt1RUVCgtzvTpQx/HufvI4eHht9566yeffKIeQb6l1hkZGRkHDhzocRPnpDM4A3s8A/u2d+9e5zuPbFVVVYIDgv44XPg8g6TFeSQtyt/uuGT0i5xHkPN48QnMJV5BzgNndM95XK+/dJ9ZPSIiorq6+vLly2PGjLHb7f7+/g6r60VGRgohamtrhRA7duxITk7+l3/5F3lB+5SUlNWrVy9ZssTleGRXr17td9cK+fexMoPBIIRQfllqt9t/8Ytf7N+/v6qqSj1l+nfffedaSA67U3zxxRfqt0S9upiPj4+vr29AQIDS4uvr2/23r2FhYbq/nWjN4Y3oY9fh4eHNzc3+/v6BgYEOIyif8levXu23T996O869jRwaGurMsAPizIkxes7JHn9sLOMMFAM/A3NycgbUf8TjgKBf8reynkTSMqCQBEmLEEK7pGWQyHnUyHnIedyKA4J+OeQ8rq9/1NDQIP11onXZ5cuXhRARERFGo9FisbS3tztMW3Xp0iUhRFRUlBBCp9MtX778j3/8Y2Nj44EDByRJyszMfOmll/rYo86J6dyd2bUzFi1a9Pzzzz/00EMVFRVdXV2SJL388stCNbe884xGY0hIiJ+fX0dHR/dbkuTFAgdD/qxXU78Rfe/aaDQGBQW1t7e3tLSoR7hy5Yo6/n77uKa3keX4hxbnpPtwBg76zsSRg3tx4QzPF18ESYvTSFp648mkZZA4pd2HE3gIrwXDHTkPnNE953G9/tLe3l5SUqI8/Pjjj6urq61Wa3R0tBBCrqAfOnRI6XD16tXi4mKTySTf9hYSEnLu3DkhhF6v/+EPfyhP3K3u311AQMC1a9fkvydOnPjGG2/02K3fXffr+vXrJ06ciIqKysvLCw8Pl683bW1tzjy3R5mZmZ2dnQ4TpG/duvWGG27o7Ox0eVhZS0uL+vZIhzei313Pnz9f/PWORFl9fb3DPajO9HFN95Fra2udLPAPFOekg2nTpv3+978fzAgyzkAA3o+kxXkkLb0ZRpcMTmkH5Dw9juy1JzAwgrlef7FYLBs2bLDZbK2traWlpbm5uQaDYfv27fLWLVu2xMfH5+fnHzx4sLm5uaKi4v7776+pqdm+fbt8C6IQ4ic/+cmZM2euXr16+fLlbdu2SZI0d+7cPvY4derUioqKb775xmaznT9/PiUlpcduzuy6b76+vnfeeWdtbe2LL75YX1/f1tZ29OhReSJ012zZsiUhIeHBBx88fPiw3W6/cuVKYWHhc889V1BQIC/gNxhms/mRRx75n//5n97eiL53/cILL4wZMyY/P//IkSMtLS2ffvppbm6uw62JzvRxjcPIZ8+e/fGPf9z965G5c+eGhYWdOnVqMPvinHST0XAGAhjuSFqcR9LSG89cMsh5yHmG9QkMoB/q22MGtP5RbGzsp59+mp6eHhQUZDKZUlNTjx8/ru5TX1+fn58fHx+v1+stFkt6enpxcbGytaysbPXq1X/3d38XEBAwZsyYmTNn7ty5U77DUJ64S7Fs2TL5KefOnUtJSTGbzXFxcTt27OijZx+7ttls6qc89dRT0t/eyrhgwQJJkurq6lavXh0XF6fX6yMjI1euXLlu3Tq5Q1JS0osvvth9kL41NDSsWbPmxhtv1Ov14eHhaWlp8gKBPYak/o5OCLFly5Y//elP6pZnn31WiSE2NvbPf/7znDlzAgMDe3wj+ti17PPPP7/77ruDg4PlZeoOHjwo/xhYCPEP//APzvTp/kY4eZzVIwcEBNxxxx0ffvjhnXfeGRAQoI4wJSWl77UAzGazenCH+eoVo+2cdDgs3cnLE3AG9nsG9oa1ABxwLy6cocn6RyQtDoP0jaSlx+MsDe6SIf11WWUH6oWZJXIech5vPYHJeRyQ88AZ3c8TnaT697l3796cnBzJiZ9WTp48ub6+Xp72GRhat956a1tb21dffaV1IBilnD8Dnf/MHCXktS3fffddrQOBVxuq88T5cUha4D4kLRjWyHlcRs4DZ3Q/T1z//REweLW1tWPGjOno6FBaLly4UFlZ2fcdsMBQ4QwEADiJSwaGNU5gwBtQf4HGvv3229WrV3/zzTfffffdn//855ycnODg4KefflrruDBacAZq6Kuvvlq8eHFTU1N9fb3ur6ZMmdLe3q7upt6q0+mmTZumVcDdDd/I1d57771bbrmlj6k9Ojo6Xn755aSkpKCgoIiIiPnz5xcVFSnfgq5bt06+Lx0Y8bhkYFjjBNYQOY+X0DznGXD9paCgQKfTlZeXX7x4UafTbdy4cTC7HzF0vfvZz36mdXTeKyoqSl778Ac/+EFoaOjixYtvvvnmP//5zzfeeKPWoWFU4AzUUFlZ2bRp09LS0oKDg8eOHStJkvwj/LKysvz8fHVPeavNZgsLC5MkqbS0VKOQezB8I5dVVlYuXrx4/fr18uqwPWptbZ07d+6bb7758ssvX758ubS0NDAwcPHixZ988onc4aGHHlq/fr13ZvAkLT0iaXFNv5cMDiy8GTmPhsh5vIGX5DwDnsd+7dq1a9eudXl/IxU/hnTZvHnzlEnFAM8bRmdgYGDg5MmTjx8/PkzHV2tqalq0aNE999zzyCOPqNuNRmNgYGBhYWFqaup9993ngUiGyjCN/Omnn77jjjv+7d/+bcKECa2trT32+ad/+qczZ85UVFTIa5fccMMNb775pnoa0YSEhP3790+ZMmXSpElLly71UOjOIWnpEUmLy/q+ZHBg4eXIeTw2vho5j5fwkpyH3x8BADxt27ZttbW1zzzzjEO7v7//7t27fXx8Vq9eXVFRoUlsrhmmkf/6179et25dH3fhXrp06Y033li2bJl64Viz2dze3n7bbbcpLVarNSsr64knnujs7HRvxAAADCvkPF7CS3Ie6i8AAI+SJGnXrl0zZsyIiYnpvjU9PX3jxo3Nzc3Z2dkOPy32csMxcpPJ1HeH//iP/7h+/frs2bP7HWrJkiVVVVWHDh0aotAAABj2yHm8h5fkPNRfAMDtGhoa1qxZk5CQYDAYQkND58+ff/ToUXnTpk2b5NkBlI/7999/X24ZO3as3CLPYdHa2nrixAl5k1y8l9t1Ot24ceNKSkrmzZsXFBQUEBAwZ86cEydODH58NykvL7906ZLVau2tw7PPPpuWlnbmzJlHH320j3H6OKoHDhxQpl24cOFCTk5OSEhIWFjYwoULKysr1YPU1dXl5eVNmDDBYDCEh4dnZmaWlZW5/NKGb+S9+ctf/iKECA0NfeKJJ+Li4gwGw/jx4/Py8q5cueLQc/LkyUKIDz74YMhjAAAMI+Q8auQ8Xhh5bzyU80gq8ly+EgDACU5+ZtbU1MTHx0dGRhYVFdnt9s8//zwzM1On0+3cuVPpYzabZ82apX5WUlKSPIFZH31kVqvVbDYnJyefPHmypaWlpKTk9ttvNxgMx44dG5Lx58yZM2bMGJvN1u8rzcrKysrK6rfbW2+9JYR44YUXHNpLSkosFov8d11dXVxcnBDi7bfflluUGd1kzhzVjIwMIURGRoZ8ZI4cOWIymaZPn650qK6uHj9+fGRk5KFDh5qbm8+ePZuamurv73/y5Ml+X8XIiFwRGxvr6+vbvV2OJCoqatmyZZWVld9+++2//Mu/mM3mW265pbGxUd3TbrcLIVJSUvrdl5PnicfGAQD0i5zHATnPsItcoW3OQ/0FAFzk5GfmypUrhRDvvPOO0tLe3h4TE2MymWpra+WWQeYiQojTp08rLWfOnBFCWK3WPp7r/PipqamhoaHOXOSczEW2bdsmhNixY4dDu/qKLkmSzWbT6/Vms/mzzz6Tul3RnTmq8nVUXjVQiVAIUVdXJz9csWKFEGL37t1Kh5qaGqPRmJSU1O+rGBmRK3rLRdLT04UQ8fHxHR0dSuOmTZuEEE8//bRDZ51Od9NNN/W7L+ovADDskPM4IOcZdpErtM15+P0RALjX/v37hRALFixQWoxG47x589ra2obqxxpms1m+E1I2adKkmJiY8vLympqawQ9+7NixK1euJCcnD34omfw7Yb1e33e3mTNnFhQUtLa2Zmdnt7W1OWx1/qhOnz5d+Vv+lqa6ulp+eODAAR8fn4ULFyodoqKiEhMTP/roo6qqKhde2nCPvDuz2SyEuOuuu9S3Zy9atEj0dNutn59f99cLABg9yHkckPN4beTdeSbnof4CAG509epVu93u7+8f/hZcuQAAIABJREFUFBSkbpdnVq+trR2SvYSEhDi0RERECCEuX748JOMPLX9/fyFER0dHvz3z8vJycnLOnj3rsGTjgI6qxWJR/jYYDEKIrq4uZZCuri6LxaJTkX/9+8UXX7j8Aod15A4mTJgghAgLC1M3ymdXXV2dQ+fOzs5+J7cDAIxU5DzdkfN4c+QOPJPzuHG2IQCA0Wi0WCx2u725uVl9+bl06ZIQIioqSn7o4+Nz7do19RMbGxsdhtLpdL3tpaGhQb4TUmmRsxD5mjH48YdWdHS0EEL+6Wy/du3aVVZW9pvf/EbOYGROHtW+GY3GkJCQlpaWtrY2d8y9N3wjV5s9e/ZLL73k8KWifHapV2cUQjQ1NUmSJL+5AIBRiJynO3IeL49czTM5D/e/AIB7LVmyRAihXqPu6tWrxcXFJpNJ/qGpECI6OvrixYtKh9ra2q+//tphnICAACWfmDhx4htvvKFsam9vLykpUR5+/PHH1dXVVqtVuTAMcvyhddtttwkhnLxlNDAw8A9/+IPZbH7ttdfU7c4c1X5lZmZ2dnYq6ybItm7desMNN3R2djo5yMiLXO3v//7vY2Nj33//ffXqkkVFRUKIu+++W91TPsHkNxcAMDqR8zgg5/HyyNU8k/NQfwEA99qyZUt8fHx+fv7Bgwebm5srKiruv//+mpqa7du3K9X0tLS06urqV199taWlpbKy8rHHHlO+xlFMnTq1oqLim2++sdls58+fT0lJUTZZLJYNGzbYbLbW1tbS0tLc3FyDwbB9+3alw2DGnzt3blhY2KlTp4bqgFit1oiIiPLycif7JyYmFhYWOjQ6c1T7tWXLloSEhAcffPDw4cN2u/3KlSuFhYXPPfdcQUGB8h1Lbm6uTqf78ssvnRxzZESuMBqNu3btamhouPfee7/44ovGxsa33npry5YtM2bMyMvLU/eUV4JMS0sbzO4AAMMaOY8Dch7vj1zhoZxHPRkv6x8BgPOc/8ysr6/Pz8+Pj4/X6/UWiyU9Pb24uFjdobGxcdWqVdHR0SaTafbs2SUlJUlJSfKn9JNPPin3OXfuXEpKitlsjouLU0+kb7VaY2NjP/300/T09KCgIJPJlJqaevz48aEaPyUlZWjXApAkacOGDX5+fhcvXpQfOvyqtscJ7R9++GGHxQv6OKo2m0094FNPPSVJkrplwYIFcs+GhoY1a9bceOONer0+PDw8LS3tyJEj6r3MnTs3MDCws7OzxxcyfCOXyd/qOFCvCik7efJkenq6xWIxGAy33nrrz372s++++86hT3Z2dmxs7LVr1/rYnYz1jwBg2CHncUDOM4wil3lJzqNTv8i9e/fm5OQ4vGwAQI+85DNz8uTJ9fX1QzsDvGuys7OFEO+++26/Pe12e2Ji4sKFC19//XX3x+W6xsbGmJiYZcuW7dy5U+tYBsbDkZeXl0+ZMuVf//Vf77333n47O3+eeGYcAEC/yHkckPN4j+GV8/D7IwCAp1kslqKion379u3YsUPrWHolSVJeXl5wcPDzzz+vdSwD4+HIz58/n5mZuX79emcSEQAARhVyHrcadjkP9RcAgAamTJlSWlp6+PDhpqYmrWPp2aVLl86fP19cXOzkFP3ew8ORFxYWbt68efPmzR7YFwAAww45j/sMu5yH9acBYLgqKCj4p3/6J/lvnU731FNPbdq0SduQBmTChAkHDx7UOopeRUVFHT9+XOsoXOHhyLdu3eqxfQEARidyHrci53HS4HMe6i8AMFytXbt27dq1WkcBAADgXuQ8GBn4/REAAAAAAIB7UX8BAAAAAABwL+ovAAAAAAAA7kX9BQAAAAAAwL16mH83Ozvb83EAwLBTVVUl+MxUOXXqlOCAoD+nTp2aOXPmUA3F+QYAHkDO44CcB87onvP4/uxnP1MeNDU12e12TwcFYDgrLi42m82BgYFaB6KB4ODg733ve1pH4UXGjRs3btw4raOAtxs3blxycnJycvIgx5H/MwAAQ+Xjjz++fPlyZGSk1oF4I3IeB+Q8cEb3nEcnSZKGAQEY7nQ63Z49e5YuXap1IAAAAK6Tk5m9e/dqHQiAEYv5XwAAAAAAANyL+gsAAAAAAIB7UX8BAAAAAABwL+ovAAAAAAAA7kX9BQAAAAAAwL2ovwAAAAAAALgX9RcAAAAAAAD3ov4CAAAAAADgXtRfAAAAAAAA3Iv6CwAAAAAAgHtRfwEAAAAAAHAv6i8AAAAAAADuRf0FAAAAAADAvai/AAAAAAAAuBf1FwAAAAAAAPei/gIAAAAAAOBe1F8AAAAAAADci/oLAAAAAACAe1F/AQAAAAAAcC/qLwAAAAAAAO5F/QUAAAAAAMC9qL8AAAAAAAC4F/UXAAAAAAAA96L+AgAAAAAA4F7UXwAAAAAAANyL+gsAAAAAAIB7UX8BAAAAAABwL+ovAAAAAAAA7kX9BQAAAAAAwL2ovwAAAAAAALgX9RcAAAAAAAD3ov4CAAAAAADgXtRfAAAAAAAA3EsnSZLWMQAYTh544IHTp08rD7/55puwsLCAgAD5oV6vP3jwYExMjEbRAQAAOOV3v/vdSy+9dP36dflhfX29EGLs2LHyQ19f3zVr1jzwwAOaxQdgxPHTOgAAw8zEiRPfeustdYvdblf+/t73vkfxBQAAeL+ZM2eWl5c7NNbW1qo7eDYiACMcvz8CMDC5ubk6na7HTXq9fuXKlZ4NBwAAwBW33HKL1WrtMavR6XRWq/WWW27xfFQARjDqLwAGZvz48VOnTu0xWens7MzOzvZ8SAAAAC544IEHfH19u7f7+fmtWLHC8/EAGNmovwAYsB6TFR8fn5kzZ06YMEGLiAAAAAbs3nvv7erq6t7e2dm5dOlSz8cDYGSj/gJgwHpMVnx8fJijDgAADCMxMTF33HGHj8/f/J/Ix8dn1qxZsbGxWkUFYKSi/gJgwCIiIlJTUx1ugZEkKTMzU6uQAAAAXLB8+XKHFp1Ox1dKANyB+gsAVyxfvly9er2vr+9dd90VERGhYUgAAAADlZ2d7fCVkk6n4yslAO5A/QWAK+655x4/v/+/gL0kSbm5uRrGAwAA4ILQ0NAf/vCHSgnG19c3PT09LCxM26gAjEjUXwC4Ijg4eP78+UoJxs/Pb/HixdqGBAAA4ILc3FxlYju+UgLgPtRfALgoNzf3+vXrQgg/P7+MjIzg4GCtIwIAABiwjIwMo9Eo/20wGBYtWqRtPABGKuovAFy0cOHCgIAAIcT169eXLVumdTgAAACuCAgIWLJkiV6v1+v1S5YsMZvNWkcEYGSi/gLARf7+/vfcc48Qwmw2/+hHP9I6HAAAABfdf//9HR0dHR0d999/v9axABix/PreXFVVdfLkSc+EAmDYGTdunBBi+vTp//7v/651LAC8VFxcXHJystZRCCHE3r17tQ4BgJe6fv16QECAJElNTU18VgDozdKlSwf1fKlPe/bsGaI4AQDAaJSVldV3suExWh8JAAAwvA0yFenn/hdlH+5+GQCGqc2bN69bt05ZtXEE27t3b05ODp+HiuzsbCHEu+++q3Ug8GryeeI99uzZM9hvrgCMUB9++KFOp/vBD36gdSCeoNPp+DxUkOPBGfJ5MshBnKq/AEBvnnzyydFQfAEAACNbSkqK1iEAGOGovwAYFD8/PkYAAMCw5+PDyiQA3ItPGQAAAAAAAPei/gIAAAAAAOBe1F8A/D/27j+uqipf/P86yDmHwwEOCPJTFOSOda/a0UHHKIirNDAmShKIpk1NY9dHM0WkdpPK6tFojj6YHO8du2nW2DUrzHnoDX+Vl9GZUfAxBwvMGsTBMpUf8iN+Cgpyvn/s7+zPvgc5HvX85vX8C9Zee+33Xm7PXufN3msBAAAAAByL/AsAwBnOnTs3d+7cjo6O5uZm1T9MmTKlt7dXWU25VaVSTZ061VUBD+a5kSvt379//PjxVmZu6uvr27BhQ2JiYmBgYHh4+KxZs0pKSuRVIVauXFlcXOysYAEAcDsMadyEJw5pyL8AgGN1dXX94Ac/yMzMdHUgrlRZWTl16tT09PSgoKCwsDCz2WwymaTygoICZU1pa3l5eWhoqNlsrqiocFHI1+G5kUtqa2vnzp1bWFjY2Ng4VJ3u7u6ZM2du27Ztw4YNly5dqqioCAgImDt37ldffSVVeOKJJwoLC1etWuWsqAEA7oIhjWBI4x48d0hD/gUAHMtsNg8MDAwMDLgqgICAgOTkZFcdXQjR0dExZ86chx566KmnnlKWa7Xa0NDQzZs3f/jhh66K7dZ4aOSrVq265557Tpw4ERgYOFSd55577uTJk5999tl9992n0+nGjBmzbds2rVYrV0hISNi9e/eaNWt27tzplKgBAO6CIQ1DGjfhuUMa8i8A4FiBgYG1tbX79+93dSAus379+oaGhpdfftmi3M/Pb8eOHT4+PkuXLq2pqXFJbLfGQyN/5513Vq5caeUx3cbGxi1btixatCgiIkIu1Ov1vb29EydOlEuMRmNOTs7y5cv7+/sdGzEAwJ0wpGFI4yY8d0hD/gUA4EBms3nr1q3Tp0+Pjo4evDUjI+Oll17q7OzMzc21ePfYzXli5DqdznqFTz755Nq1a7b8aXHevHkXLlzYt2+fnUIDAMDdMaRxH547pCH/AgAOtGfPHnn2MumWpiz59ttv8/LygoODQ0NDMzMza2trpb2KioqkCqNHjzaZTGlpaYGBgf7+/jNmzDh27JhUZ/Xq1VId+dZy8OBBqSQsLEzZTnd397Fjx6RNVv5Q4CBVVVWNjY1Go3GoCq+88kp6evrJkyeffvppK+20tLQsW7YsISFBo9GEhITMmjXr8OHD0iZbulTS1NSUn58fFxen0WhGjRqVnZ1dWVl5y6fmuZEP5fPPPxdChISELF++PDY2VqPRjB07Nj8/v7W11aLm5MmThRCffvqp3WMAALgnhjQMadww8qG475DGbJU0IbD1OgAwHNzO52FWVpYQoqenx6IkKyurrKysq6vr0KFDOp1u2rRpyr2MRqNer09KSpLqmEymu+66S6PRHDlyRK6j1+vvvfde5V6JiYnSZGlW6khmzJgxcuTI8vLyWzupnJycnJycG1bbvn27EOL111+3KDeZTAaDQfq5qakpNjZWCPH+++9LJfKUb5L6+vr4+PiIiIiSkpL29vbTp09nZ2erVKq3335brnPDLq2rqxs7dmxERMS+ffs6OztPnTqVmprq5+dXVlZ2UyfuuZHLYmJiRowYMbhciiQyMnLRokW1tbXff//9e++9p9frx48f39bWpqzZ3t4uhEhJSbnhsWy8TpxDCFFcXOzqKADA9W7589ArhzQ2jvEY0rhP5DJnDmnskhsh/wIANnFE/kVaA0+Sk5MjhGhqapJLpD+wfPHFF3LJyZMnhRBGo1EuuZ3BSmpqakhIyC3f8Gz8Xr1+/XohxKZNmyzKlbd8s9lcXl6uVqv1ev3f/vY386Bb/mOPPSaE+PDDD+WS3t7e6OhonU7X0NAgldywSx999FEhxI4dO+QK9fX1Wq02MTHR9rP26MhlQw1WMjIyhBDx8fF9fX1y4erVq4UQq1atsqisUqn+6Z/+6YbHIv8CAG7I7vkXjx7S2DjGY0jjPpHLnDmksUtuhPePAMBlpk2bJv8s/c2hrq5OWUGv10tPRUomTZoUHR1dVVVVX19/+0c/cuRIa2trUlLS7TdlhfSIslqttl7t7rvvLioq6u7uzs3N7enpsdi6e/duIcTs2bPlEq1Wm5aW1tPTY/G8qJUu3bNnj4+Pj3LVzMjIyAkTJpw4ceLChQu3cGqeHvlger1eCHH//fcrH+qeM2eOuN5zub6+voPPFwAwPDGkkXnuwMBzIx/MbYc05F8AwGUMBoP8s0ajEUJYrOkYHBxssUt4eLgQ4tKlS46Pzj78/PyEEH19fTesmZ+fn5eXd+rUKYs1Ha9cudLe3u7n52exxKA0oX1DQ4OycKgulRoZGBgwGAwqBen14DNnztzyCXp05Bbi4uKEEKGhocpC6ZJramqyqNzf33/D2e8AAMMEQxolzx0YeG7kFtx2SOPsWYsAALZraWmRnoqUS6RhinT/EEL4+PhcvXpVuUtbW5tFI8rdnS8qKkoIIb1be0Nbt26trKx89913pSGORKvVGgyG9vb2zs5O5V2/sbFRCBEZGWlLy1qtNjg4uKurq6enxxEz9nlu5ErJyclvvPGGxZ8ipUtOuXyjEKKjo8NsNkv/uAAA3BBDGuEhAwPPjVzJbYc0PP8CAO6rt7fXZDLJv3755Zd1dXVGo1G+SURFRV28eFGu0NDQ8N1331k04u/vLw9o7rjjji1btjg46v9j4sSJQggbnykNCAj4wx/+oNfr33zzTWX5vHnzhBDKpQGvXLlSWlqq0+mk93ttkZ2d3d/fL6+2IFm3bt2YMWP6+/ttbMT7Ild64IEHYmJiDh48qFx+sqSkRAjx4IMPKmtKV530jwsAwA0xpJG4/8DAcyNXctshDfkXAHBfBoPhhRdeKC8v7+7urqioWLx4sUaj2bhxo1whPT29rq7ud7/7XVdXV21t7TPPPCP/HUn2wx/+sKam5vz58+Xl5WfPnk1JSZHKZ86cGRoaevz4cYeegtFoDA8Pr6qqsrH+hAkTNm/ebFG4du3a+Pj4goKCvXv3dnZ21tTUPPzww/X19Rs3brT4I4YVa9euTUhIePzxxw8cONDe3t7a2rp58+bXXnutqKhI/iPM4sWLVSrVN998Y2Ob3hG5TKvVbt26taWlZcGCBWfOnGlra9u+ffvatWunT5+en5+vrCktFZmenn47hwMADB8MaSQeMTDw3Mhl7jukccIcvwDgBW7t81Caq0y2aNGi8vJyZcmLL75oNpuVJbNnz5b2NRqNMTExX3/9dUZGRmBgoE6nS01NPXr0qLL9tra2JUuWREVF6XS65ORkk8mUmJgotfP8889Ldaqrq1NSUvR6fWxsrHLS/pSUFCesf2Q2m1944QVfX9+LFy9Kv1q8dnvdGe+ffPJJiyUPmpubCwoK4uPj1Wq1wWDIyMgoLS2VNtnepS0tLcuWLRs3bpxarR41alR6evqhQ4eUR5k5c2ZAQEB/f/91T8RzI5dIf/axoFw2UlJWVpaRkWEwGDQazZ133vnqq69evnzZok5ubm5MTMzVq1etHE7C+kcA4IZu4fPQi4c0to/xGNK4PHKJS4Y0rD8NAM7j/M9DabDizCPeFNu/V7e1tcXExCxdutTRId2m77//XqfTLVmyxNWB3DQnR15ZWalSqZQrUFpB/gUA3JCTPw/dfEhj+xiPIY2jufOQhvWnbVJcXDx58mSdTifNrnzq1ClXR+S1PvroI6mTlXM1OdTkyZNVNyIt8+5yHn0dBgQEWO/kiooKV8Xm/KsOt8BgMJSUlOzatWvTpk2ujmVIZrM5Pz8/KCjoV7/6latjuTlOjvzs2bPZ2dmFhYULFixwwuGg5NG3Etu5wwc7QxorPPo6tBjS+Pj4hISEGI3GX/ziFydOnHBtbO5w5eOGGNI41HAY0nh5/uXYsWMLFy5MT09vamr6+9//Pnr0aFdH5M0WLFhgNpvT0tKcedCPP/5YziYuXbpUCHHgwAG5JC8vz5nBDMXTr8Ourq4vvvhCCJGVlTU4iatcYc75XHLV4RZMmTKloqLiwIEDHR0dro7l+hobG8+ePVtaWmrjHP7uw8mRb968ec2aNWvWrHHCsaDk6bcS27nDBztDmqF4+nVoMaTp6+urrq5+7bXXqqurp06d+rOf/ezy5cuuis0drnzYgiGN4wyHIY3dln0KCAiYPHny0aNH7dWgXUh3smeeeSYgICAgIOD8+fOujgjDEdchblZRUdFzzz0n/axSqV588UU3+bPn7YiLi9u7d6+roxhSZGSku93CbOTkyNetW+e0Y7kKQxpgKF52HY4YMSIiIiIrKysrK+v5559fv359a2vrnj17VC5d5NjLMKRxPoY0NnLJkMaxy267nHRXCA0NdXUgcAhptmorPvroI+dEYp13X4dtbW2uDsELrVixYsWKFa6OAoAb8e5bCRjSuNyvf/3rP/3pT5988slHH320cOFCV4fjPRjSAEpe/v7RtWvXXB0C4LXXYXJy8rZt21wdBQAMC956K4Fn8eLrUKVSPfXUU0KIN99809WxAPBadsi/FBUVqVSq7u7uY8eOSbNGSWt3Sw/vSU6fPj1//vzQ0FDp1+bm5v7+/uLi4h//+MeRkZE6nW7SpEkbN24cGBiQ2lTu++233+bl5QUHB4eGhmZmZtbW1sqHvnLlyssvv3znnXf6+/uPHDlyzpw5n3zyiXRjkFr4n//5HyGENEPY3XffLe0lLXmVkJCg0WhCQkJmzZp1+PDhwce1iHnr1q3ypnPnzuXl5QUGBoaGhj7yyCPff//9t99+O2fOnMDAwKioqCeeeKKzs1PZRU1NTfn5+XFxcRqNZtSoUdnZ2fJfOaz3kvWet7FZKx2o7A2tVjt69Oj7779/27ZtPT09N+wrSXV19YMPPmgwGPR6fUpKynUfGHPQ6duI69Bp/xBcdQA8HUMaV91KbvOD3aI3vPXmwnXo6H+I5ORkIcTx48f7+vpueCCLc/fWqw6AnVlfHsn2NZb0ev299947uDwrK0sIkZqaevjw4e7u7uPHj48YMaKpqUlasvv1119vbW1tamr6j//4Dx8fnxUrVgzeNysrq6ysrKur69ChQzqdbtq0aXKFJUuWGAyGzz777PLlyw0NDdKzbYcPH7ZooaenRy6pr6+Pj4+PiIgoKSlpb28/ffp0dna2SqVSrhY+VMzypuzs7IqKiq6urv/+7/8WQsyaNSsrK+uLL77o7Ox86623hBDPPvus3FpdXd3YsWMjIiL27dvX2dl56tSp1NRUPz8/5QL1Vo44FNubtdKBUm9ERkaWlJR0dHQ0NDRIc01v2LDBlr46c+ZMcHBwTEzMZ5991tnZefLkyfT09Li4OK1Wa6/TnzFjxsiRI8vLy610hWzwZHU3PATXoY3/ENJkdYP9/ve/v25Xe/RVNxTnrz/t5txqXWG4Lbe6ToRt660ypHHykMYuH+zuf3NhSOMm16GVJQXkvEldXZ0tB3L/q84KGz8PhwnGeLCFXa4TJ+Vf9u/fb1FeUlLyr//6r8qSxYsXq9Xq9vZ2i31LSkrkkpycHCGE/IESHx9/zz33KBsZP3689ZvEY489JoRQLvHd29sbHR2t0+kaGhqsxyxv2rdvn1wyYcIEIcSf/vQnuSQ+Pv6OO+6Qf3300UeFEDt27JBL6uvrtVptYmLiDXvJCtubtdKBUm9YfPj+5Cc/kW4bN+yr3NxcIcSuXbvkChcvXtRqtcrbxm2efmpqakhIiPIeY8UNBytch7f8D3Hdwcq99947VP7Fo6+6oXBvtuBW36vhttzqOrFL/oVbid0/Xe3ywe7+NxeGNG5yHVrJv8iLH0n5Fy+46qwg/6LEGA+2sMt14qT5d3/0ox9ZlGRmZmZmZipLjEbj+++//9VXXyUlJSnLp02bJv8cGxsrhKirqwsLCxNC/OQnP/mv//qvf/u3f3v88cenTZs2YsSI06dPW49k9+7dQojZs2fLJVqtNi0tbfv27Z9++ulPf/pTKzHLpk6dKv8cHR391VdfKUtiYmKqqqrkX/fs2ePj46M82cjIyAkTJpw4ceLChQvKdfusHHEw25u10oFSb8yaNUvZ8oEDB6QfbthXBw8eFEJkZGQoe2P8+PE1NTX2Ov0jR47Y1h824Tq0+3U4FI++6qyTRksQQhw/flzQIbiR48ePyy8peAduJXb/dLXLB7v731wY0rj5dSiEqK+vF0Ko1WqpN7zgqrNuw4YNH3/88c3u5ZUuXLggGNLgRqTr5DY5Kf+i1+stStrb23/zm9/s3r37woULyvVT5MSzzGAwyD9rNBohhPwu66ZNm5KSkt577720tDQhREpKytKlS+fNmzdUGFeuXGlvb/fz8wsMDFSWR0RECCEaGhqsxywLCgqSf/bx8RkxYoS/v79cMmLECDlC6YgWZyE7c+aM8nPTyhGveyI2NjtUBw7VG8pDWOmrK1eudHZ2+vn5BQQEKCuEh4fLtw0Hnf4t4zq07z+ElfXhuOoAeCtuJXYf0tz+B/uoUaOG282F69AR/xDSwCYpKUmtVnPVAXAEu+VfVCrVTdWfM2fOX/7yl40bNy5cuDAsLEylUv32t7+V3u28qYM+8sgjjzzySF9f35EjR4qKirKzs3/zm98sW7bsuvW1Wq3BYGhvb+/s7FR+GjY2NgohIiMjb+oUbKHVaoODg7u6unp6eqQ5/Nyn2aF6w/pWua+0Wm1gYGBnZ2dXV5fyztHa2mrfOB2K69DJvOCq4y9FMunPRHQIrPPEPycypBnqiA4a0tjlg93Tby63j+vwNg0MDGzatEkI8ctf/tLGA3n6Vffss8/Onz/f7s16op07d+bl5TGkgXXSdXKbjdht/Wl/f/+rV69KP99xxx1btmyxUvnatWvHjh2LjIzMz88fNWqUNNCRp7yyXXBwcHV1tRBCrVb/+Mc/luYA37dvn5VdpBS+ss6VK1dKS0t1Op3y8T87ys7O7u/vP3bsmLJw3bp1Y8aM6e/vd22zUm/s379fWThlypRnn31W2NBX0vOW0sOTkubmZovnVB10+nbBdWiXf4ipU6d+9NFHttcf5lcdAPfHkGYoDvp0tcsH+zC/uXAd3v4/RGFh4V//+td58+bJWWOuOgD2Z685Zn7yk58YDIbvvvuurKzM19f366+/lsoHT9MlmTlzphBi/fr1TU1Nly9f/uMf/zhmzBghxKFDh+Q6g/d9/vnnhRBffPGF9KvBYEhNTa2qqurt7W1sbHz11VeFEKtXr7bSgnIq8o6ODnkq8i1btljZy8qmjIyMESNGKOukpqbq9Xr518bGxoSEhHHjxu3fv7+tra2lpeWtt97y9/dXTnll5YhDubVmLTpQ6o3sfYLpAAAgAElEQVSoqKi9e/d2dHScP3/+ySefjIiIOHfunC199fe//33kyJHytO1fffVVRkZGeHi4ctqw2zx9+y4WwHV4y/8QViarS0xMVM4t5wVX3VCYm82CW82rCrflVteJsG2+SYY0MucMaezywe7+NxeGNG5yHSqHNNeuXWtsbNyzZ4/Ue48//vjly5dtP5D7X3VW2Ph5OEwwxoMt3Gv9o+rq6pSUFL1eHxsbu2nTJrPZXF5ebiXX09TUtHTp0tjYWLVaHRER8dhjj61cuVKqlpiYaLHviy++aP6/D0/Onj3bbDZXVlYuXbr0n//5n/39/UeOHHn33Xe//fbbAwMDZrNZmvJKSb7hNTc3FxQUxMfHq9Vqg8GQkZFRWloqbbIS8+CQTCaTsmTt2rV/+ctflCWvvPKKtG9LS8uyZcvGjRunVqtHjRqVnp4u3wut95J1tjc7VAda9EZUVNSCBQtqamrkQ1jpK8np06cffPDBoKAgaf3CvXv3Si8MCyF+/vOf3/7pp6Sk2LJYwO9//3uLdjo7O205BNehLb10wzeKpfyL11x1Q+HebMGtvlfDbbnVdSJs+77BkMb5Q5rb/GAf3BtueHNhSGMRszsMaVQqlcFgmDRp0pNPPnnixInB/xyeftVZIci/KDDGgy3scp2ozFZfCpXecbJeBwCGAz4PLTD/C2zhVteJSqUqLi5mvgMA4PNQiTEebGGX68Ru878AAGDFuXPn5s6d29HR0dzcrPqHKVOm9Pb2Kqspt6pUKuUSpC7nuZEr7d+/f/z48Vamcuzr69uwYUNiYmJgYGB4ePisWbNKSkrk0cbKlSulv/8AADA8MaRxE544pCH/AgBwuMrKyqlTp6anpwcFBYWFhZnNZuk588rKyoKCAmVNaWt5eXloaKjZbK6oqHBRyNfhuZFLamtr586dW1hYKC3AcV3d3d0zZ87ctm3bhg0bLl26VFFRERAQMHfu3K+++kqq8MQTTxQWFq5atcpZUQMA4EYY0rgDzx3SkH9xX6qhSdOhAfBiAQEBycnJntu+UkdHx5w5cx566KGnnnpKWa7VakNDQzdv3vzhhx86JxJ78dDIV61adc8995w4cWLwUqmy55577uTJk5999tl9992n0+nGjBmzbds2rVYrV0hISNi9e/eaNWt27tzplKjhDRjSAMMZQxp35qGRe+6QhvyL+7IybQ+DFQAeZP369Q0NDS+//LJFuZ+f344dO3x8fJYuXVpTU+OS2G6Nh0b+zjvvrFy50spjuo2NjVu2bFm0aFFERIRcqNfre3t7J06cKJcYjcacnJzly5ez+ilsxJAGgHdgSOMmPHdIQ/4FAOBAZrN569at06dPj46OHrw1IyPjpZde6uzszM3NtXj32M15YuQ6nc56hU8++eTatWu2/BVx3rx5Fy5c2Ldvn51CAwDA3TGkcR+eO6Qh/wIAdiYtFZmQkKDRaEJCQmbNmnX48GFp0+rVq6VH7uX7wcGDB6WSsLAwqaSoqEilUnV3dx87dkzaJGX3pXKVSjV69GiTyZSWlhYYGOjv7z9jxoxjx47dfvsOUlVV1djYaDQah6rwyiuvpKennzx58umnn7bSjpVe3bNnj/wuw7fffpuXlxccHBwaGpqZmVlbW6tspKmpKT8/Py4uTqPRjBo1Kjs7u7Ky8pZPzXMjH8rnn38uhAgJCVm+fHlsbKxGoxk7dmx+fn5ra6tFzcmTJwshPv30U7vHAABwHwxplBjSuGHkQ3HfIY315alZCx0AJDZ+HtbX18fHx0dERJSUlLS3t58+fTo7O1ulUr399ttyHb1ef++99yr3SkxMlGY4s1JHYjQa9Xp9UlJSWVlZV1eXyWS66667NBrNkSNH7NL+jBkzRo4cWV5efsMzzcnJycnJuWG17du3CyFef/11i3KTyWQwGKSfm5qaYmNjhRDvv/++VCJP+SaxpVezsrKEEFlZWVLPHDp0SKfTTZs2Ta5QV1c3duzYiIiIffv2dXZ2njp1KjU11c/Pr6ys7IZn4R2Ry2JiYkaMGDG4XIokMjJy0aJFtbW133///XvvvafX68ePH9/W1qas2d7eLoRISUm54bFsvE6cQwhRXFzs6igAwPVs+TwcPkMaG8d4DGncJ3KZM4c0dsmNkH8BAJvY+Hn42GOPCSE+/PBDuaS3tzc6Olqn0zU0NEgltzlYEUJ88cUXcsnJkyeFEEaj0cq+trefmpoaEhJiy13Qxu/V69evF0Js2rTJolx5yzebzeXl5Wq1Wq/X/+1vfzMPuuXb0qvSjVZaVlCOUAjR1NQk/froo48KIXbs2CFXqK+v12q1iYmJNzwL74hcNtRgJSMjQwgRHx/f19cnF65evVoIsWrVKovKKpXqn/7pn254LPIvAOCGbPk8HD5DGhvHeAxp3CdymTOHNHbJjfD+EQDY0+7du4UQs2fPlku0Wm1aWlpPT4+9nmzU6/XSo5KSSZMmRUdHV1VV1dfX337jR44caW1tTUpKuv2mJNKLxGq12nq1u+++u6ioqLu7Ozc3t6enx2Kr7b06bdo0+Wfpzzh1dXXSr3v27PHx8cnMzJQrREZGTpgw4cSJExcuXLiFU/P0yAfT6/VCiPvvv1/5/PacOXPE9Z7L9fX1HXy+AACvwZDGAkMat418MLcd0pB/AQC7uXLlSnt7u5+fn8VieNLU6w0NDXY5SnBwsEVJeHi4EOLSpUt2ad++/Pz8hBB9fX03rJmfn5+Xl3fq1CmLNR1vqlcNBoP8s0ajEUIMDAzIjQwMDBgMBuXat9LrwWfOnLnlE/ToyC3ExcUJIUJDQ5WF0tXV1NRkUbm/v/+Gs98BADwUQ5rBGNK4c+QW3HZI48AJigBguNFqtQaDob29vbOzU3l/amxsFEJERkZKv/r4+Fy9elW5Y1tbm0VTKpVqqKO0tLRIj0rKJdIwRbqp3H779hUVFSWEkN6tvaGtW7dWVla+++670hBHYmOvWqfVaoODg7u6unp6ehwxOZ/nRq6UnJz8xhtvWPzVUbq6lMs3CiE6OjrMZrP0jwsA8D4MaQZjSOPmkSu57ZCG518AwJ7mzZsnhFAuYnflypXS0lKdTie9iSqEiIqKunjxolyhoaHhu+++s2jH399fHnDccccdW7ZskTf19vaaTCb51y+//LKurs5oNMp3jtts374mTpwohLDxmdKAgIA//OEPer3+zTffVJbb0qs3lJ2d3d/fLy+sIFm3bt2YMWP6+/ttbMT7Ild64IEHYmJiDh48qFx+sqSkRAjx4IMPKmtKF5j0jwsA8EoMaSwwpHHzyJXcd0jjhDlmAMAL3ML6Rx0dHfLk8Fu2bJHrSI90/ud//mdnZ+ff//73+fPnx8TEWEwm95Of/MRgMHz33XdlZWW+vr5ff/21VG40Gg0GQ1pampXFAm6nfbuvfzQwMBAeHj54YjyLKd+U3n//fSHEUFPuD9Wr0pRvPT09csnzzz8vFBP7NTY2JiQkjBs3bv/+/W1tbS0tLW+99Za/v79yAsJFixYJIc6ePWvljDw3ctlQk9WZzeYDBw74+vpmZWXV1NR8//33//3f/63X66dPn3758mVltQ8++EAIsXv37hsei/l3AcAN2fJ5OHyGNDaO8RjSuE/kMmcOaVj/CACcx/bPw+bm5oKCgvj4eLVabTAYMjIySktLlRXa2tqWLFkSFRWl0+mSk5NNJlNiYqKUE3/++eelOtXV1SkpKXq9PjY2VjnTvtFojImJ+frrrzMyMgIDA3U6XWpq6tGjR+3VfkpKin3XPzKbzS+88IKvr+/FixelXy1eu73ujPdPPvmkxejKSq+Wl5crG3zxxRfNZrOyZPbs2VLNlpaWZcuWjRs3Tq1Wjxo1Kj09/dChQ8qjzJw5MyAgoL+//7on4rmRS6Q/+1hQLhspKSsry8jIMBgMGo3mzjvvfPXVVy1GKmazOTc3NyYm5urVq1YOJyH/AgBuyMbPw2EypLF9jMeQxuWRS1wypCH/AgDO4yafh9JgxdVRmM038726ra0tJiZm6dKljg7pNn3//fc6nW7JkiWuDuSmOTnyyspKlUqlXIHSCvIvAOCG3OHz0H2GNLaP8RjSOJo7D2lYfxoA4AEMBkNJScmuXbs2bdrk6liGZDab8/Pzg4KCfvWrX7k6lpvj5MjPnj2bnZ1dWFi4YMECJxwOAAD3wZDGoYbDkIb8CwDA4aZMmVJRUXHgwIGOjg5Xx3J9jY2NZ8+eLS0ttXEOf/fh5Mg3b968Zs2aNWvWOOFYAAC4G4Y0jjMchjSsPw0AnqGoqOi5556TflapVC+++OLq1atdG9JNiYuL27t3r6ujGFJkZOTRo0ddHcWtcHLk69atc9qxAABeiSGNQzGksZFLhjTkXwDAM6xYsWLFihWujgIAAOC2MKTBsMX7RwAAAAAAAI5F/gUAAAAAAMCxyL8AAAAAAAA4FvkXAAAAAAAAxyL/AgAAAAAA4GBmq4qLi10dIAAA8GA5OTnWBxtO4+qeAAAAnu02hyI3WH/6nnvuIQUDAABuWWxsrKtD+P8xpAFgxYYNG4QQzz77rKsDAeC1VPw5CAAAAMAwN3/+fCHEzp07XR0IAK/F/C8AAAAAAACORf4FAAAAAADAsci/AAAAAAAAOBb5FwAAAAAAAMci/wIAAAAAAOBY5F8AAAAAAAAci/wLAAAAAACAY5F/AQAAAAAAcCzyLwAAAAAAAI5F/gUAAAAAAMCxyL8AAAAAAAA4FvkXAAAAAAAAxyL/AgAAAAAA4FjkXwAAAAAAAByL/AsAAAAAAIBjkX8BAAAAAABwLPIvAAAAAAAAjkX+BQAAAAAAwLHIvwAAAAAAADgW+RcAAAAAAADHIv8CAAAAAADgWORfAAAAAAAAHIv8CwAAAAAAgGORfwEAAAAAAHAs8i8AAAAAAACORf4FAAAAAADAsci/AAAAAAAAOBb5FwAAAAAAAMci/wIAAAAAAOBY5F8AAAAAAAAci/wLAAAAAACAY5F/AQAAAAAAcCxfVwcAAAAAAM7W3Nzc0dEh/9rd3S2EOHv2rFwSFBQUFhbmgsgAeCmV2Wx2dQwAAAAA4FS///3vH3/8cSsV3n333Z/97GdOiweA1yP/AgAAAGDYaW9vHzVqVF9f33W3qtXqpqYmg8Hg5KgAeDHmfwEAAAAw7BgMhgceeMDX9zoTMvj6+s6ePZvkCwD7Iv8CAAAAYDhavHjxtWvXBpcPDAwsXrzY+fEA8G68fwQAAABgOOrt7Q0LC5Nm3lXy9/dvbm7W6XQuiQqAt+L5FwAAAADDkZ+fX3Z2tlqtVhaq1eqcnBySLwDsjvwLAAAAgGHq4YcftpiCt6+v7+GHH3ZVPAC8GO8fAQAAABim+vv7IyIiWltb5ZLg4OCmpqbrzssLALeD518AAAAADFO+vr4LFy6UX0FSq9WLFy8m+QLAEci/AAAAABi+Fi5cKL+C1NfXt3DhQtfGA8Bb8f4RAAAAgOHLbDbHxsZevHhRCBEVFXXx4kWVSuXqoAB4IZ5/AQAAADB8qVSqRx55RKPRaDSaRx99lOQLAAfh+RcAAAAAw9rJkyeNRqP0w6RJk1wdDgDvxMxSAAAAsLPy8vI33njD1VEANyEgIEAI8dprr7k6EOAmLFu2LCkpydVRwFa8fwQAAAA7O3/+/K5du1wdBTzG8ePHjx8/7toYxo4dGxcX59oYZLt27bpw4YKro4C727Vr1/nz510dBW4Cz78AAADAIT7++GNXhwDPkJubK1x9wZw9e1YIMW7cOBfGIFOpVM8+++z8+fNdHQjcGnMVeRzyLwAAAACGOzfJvADwYrx/BAAAAAAA4FjkXwAAAAAAAByL/AsAAAAAAIBjkX8BAAAAAE917ty5uXPndnR0NDc3q/5hypQpvb29ymrKrSqVaurUqa4KeDDPjVxp//7948eP9/UdcorVvr6+DRs2JCYmBgYGhoeHz5o1q6SkxGw2S1tXrlxZXFzsrGDhGuRfAAAAAHikrq6uH/zgB5mZma4OxGUqKyunTp2anp4eFBQUFhZmNptNJpNUXlBQoKwpbS0vLw8NDTWbzRUVFS4K+To8N3JJbW3t3LlzCwsLGxsbh6rT3d09c+bMbdu2bdiw4dKlSxUVFQEBAXPnzv3qq6+kCk888URhYeGqVaucFTVcgPwLAAAAAI9kNpsHBgYGBgZcFUBAQEBycrKrjt7R0TFnzpyHHnroqaeeUpZrtdrQ0NDNmzd/+OGHrort1nho5KtWrbrnnntOnDgRGBg4VJ3nnnvu5MmTn3322X333afT6caMGbNt2zatVitXSEhI2L1795o1a3bu3OmUqOEC5F8AAAAAeKTAwMDa2tr9+/e7OhDXWL9+fUNDw8svv2xR7ufnt2PHDh8fn6VLl9bU1LgktlvjoZG/8847K1eutPLmUWNj45YtWxYtWhQRESEX6vX63t7eiRMnyiVGozEnJ2f58uX9/f2OjRguQv4FAAAAADyM2WzeunXr9OnTo6OjB2/NyMh46aWXOjs7c3NzLaZTcXOeGLlOp7Ne4ZNPPrl27Zotj0rNmzfvwoUL+/bts1NocC/kXwAAAAB4nj179shzskpf1JUl3377bV5eXnBwcGhoaGZmZm1trbRXUVGRVGH06NEmkyktLS0wMNDf33/GjBnHjh2T6qxevVqqI39hPnjwoFQSFhambKe7u/vYsWPSJiuPPzhCVVVVY2Oj0WgcqsIrr7ySnp5+8uTJp59+2ko7LS0ty5YtS0hI0Gg0ISEhs2bNOnz4sLTJlv6UNDU15efnx8XFaTSaUaNGZWdnV1ZW3vKpeW7kQ/n888+FECEhIcuXL4+NjdVoNGPHjs3Pz29tbbWoOXnyZCHEp59+avcY4BbMAAAAgF1Jq3i4Ogp4jJycnJycnFvbNysrSwjR09NjUZKVlVVWVtbV1XXo0CGdTjdt2jTlXkajUa/XJyUlSXVMJtNdd92l0WiOHDki19Hr9ffee69yr8TERGkKWCt1JDNmzBg5cmR5efmtnZQQori42Hqd7du3CyFef/11i3KTyWQwGKSfm5qaYmNjhRDvv/++VCLPYiupr6+Pj4+PiIgoKSlpb28/ffp0dna2SqV6++235To37M+6urqxY8dGRETs27evs7Pz1KlTqampfn5+ZWVlN3XWnhu5LCYmZsSIEYPLpUgiIyMXLVpUW1v7/fffv/fee3q9fvz48W1tbcqa7e3tQoiUlBRbDmfLdQK3wvMvAAAAALzNkiVLkpKS9Hr9/fffP3v2bJPJ1NzcrKzQ3d395ptvSnWmTp36/vvvX7169ZlnnrHL0QcGBqSvW3Zp7brq6+uFEAaDwUqdsLCwnTt3qtXqpUuXVldXD65QWFj4zTff/Pa3v83MzAwKCho/fvwHH3wQFRWVn59vsZSPlf4sLCw8d+7cG2+88cADDwQEBEyYMOGjjz4ym83Wn16xznMjvy7p+SydTrdt27Zx48YFBwf/9Kc/LSwsrKmp+c1vfqOsGRQUpFKppH9ceB/yLwAAAAC8zbRp0+SfpScp6urqlBX0er30rodk0qRJ0dHRVVVVdvnqe+TIkdbW1qSkpNtvaijSV3q1Wm292t13311UVNTd3Z2bm9vT02Oxdffu3UKI2bNnyyVarTYtLa2np8fiFRgr/blnzx4fHx/lKuCRkZETJkw4ceLEhQsXbuHUPD3ywfR6vRDi/vvvV76kNmfOHHG9V418fX0Hny+8A/kXAAAAAN5G+WCIRqMRQlgsUx0cHGyxS3h4uBDi0qVLjo/ODvz8/IQQfX19N6yZn5+fl5d36tQpi2Wqr1y50t7e7ufnZ7FqsrRGT0NDg7JwqP6UGhkYGDAYDCoFacaTM2fO3PIJenTkFuLi4oQQoaGhykLpemtqarKo3N/ff8MJfeGhnDpHFAAAAAC4g5aWFrPZrFKp5BIp8yJ9KxZC+Pj4XL16VblLW1ubRSPK3Z0sKipKCCFNF3JDW7duraysfPfdd6WsjUSr1RoMhvb29s7OTmUiQ3p/JzIy0paWtVptcHBwV1dXT0+PI2Yg9tzIlZKTk9944w2LR6uk6025IrUQoqOjw2w2S/+48D48/wIAAABg2Ont7TWZTPKvX375ZV1dndFolL/6RkVFXbx4Ua7Q0NDw3XffWTTi7+8v52juuOOOLVu2ODjq/2fixIlCCBtfkwkICPjDH/6g1+vffPNNZfm8efOEEMrVjq9cuVJaWqrT6TIyMmyMJDs7u7+/X149SrJu3boxY8b09/fb2Ij3Ra70wAMPxMTEHDx4ULmidklJiRDiwQcfVNaULjnpHxfeh/wLAAAAgGHHYDC88MIL5eXl3d3dFRUVixcv1mg0GzdulCukp6fX1dX97ne/6+rqqq2tfeaZZ+RHY2Q//OEPa2pqzp8/X15efvbs2ZSUFKl85syZoaGhx48fd1z8RqMxPDy8qqrKxvoTJkzYvHmzReHatWvj4+MLCgr27t3b2dlZU1Pz8MMP19fXb9y40eK5DCvWrl2bkJDw+OOPHzhwoL29vbW1dfPmza+99lpRUZH8XMnixYtVKtU333xjY5veEblMq9Vu3bq1paVlwYIFZ86caWtr2759+9q1a6dPn56fn6+sKa1+nZ6efjuHg/ty3dJLAAAA8E6sP42bcmvrT0szsMoWLVpUXl6uLHnxxRfN/3cFotmzZ0v7Go3GmJiYr7/+OiMjIzAwUKfTpaamHj16VNl+W1vbkiVLoqKidDpdcnKyyWRKTEyU2nn++eelOtXV1SkpKXq9PjY2dtOmTfK+KSkpISEht7yMsbBtXeEXXnjB19f34sWL0q8WM4kkJiYO3uXJJ5+0WEK7ubm5oKAgPj5erVYbDIaMjIzS0lJpk+392dLSsmzZsnHjxqnV6lGjRqWnpx86dEh5lJkzZwYEBPT391/3RDw3con0JIsF5UrYkrKysoyMDIPBoNFo7rzzzldfffXy5csWdXJzc2NiYq5evWrlcDIbrxO4D5XZkYuiAQAAYBjauXNnXl4e40zYKDc3Vwjx8ccfO+2IkydPbm5utu8aN3akUqmKi4vnz59vvVp7e/uECRMyMzPfeust5wR2a9ra2qKjoxctWvT222+7Opab4+TIq6qqpkyZ8sEHHyxYsMCW+jZeJ3AfvH8EAAAAAJ7HYDCUlJTs2rVr06ZNro5lSGazOT8/Pygo6Fe/+pWrY7k5To787Nmz2dnZhYWFNiZf4InIvwAAAMAtfPTRR9L6r8qFTrxeQECAcu1bHx+fkJAQo9H4i1/84sSJE66ODu5uypQpFRUVBw4c6OjocHUs19fY2Hj27NnS0lIblyVyH06OfPPmzWvWrFmzZo0TjgVXIf8CAAAAt7BgwQKz2ZyWlubqQJyqq6vriy++EEJkZWWZzea+vr7q6urXXnuturp66tSpP/vZzy5fvuzqGL1KUVGRSqWqqqq6ePGiSqV66aWXXB3R7YqLi9u7d29QUJCrA7m+yMjIo0ePTpgwwdWB3DQnR75u3TqefPF65F8AAACA2xUQEJCcnHz77YwYMSIiIiIrK+uPf/zjv//7v2/btm3hwoUeN5OOvXrDEVasWKGcDnP16tWujgjAcEH+BQAAAHBHv/71r6dPn/7JJ5989NFHro4FAHC7yL8AAAAA7kilUj311FNCiDfffNPVsQAAbhf5FwAAALhMdXX1gw8+aDAY9Hp9SkrK0aNHlVv37NkjT0x7+vTp+fPnh4aGSr82NzcLIVpaWpYtW5aQkKDRaEJCQmbNmnX48GFpX2maD5VKNXr0aJPJlJaWFhgY6O/vP2PGjGPHjimPYqWR1atXS43Ib9McPHhQKgkLC1MeqLu7+9ixY9ImX19fe/WPdNzjx4/39fXRGwDg2cwAAACAXRUXF9syzjxz5kxwcHBMTMxnn33W2dl58uTJ9PT0uLg4rVarrJaVlSWESE1NPXz4cHd39/Hjx0eMGNHU1FRfXx8fHx8REVFSUtLe3n769Ons7GyVSvX222/L+xqNRr1en5SUVFZW1tXVZTKZ7rrrLo1Gc+TIEamCLY3o9fp7771XGVJiYmJoaKiyZHAdyYwZM0aOHFleXm6lH5Tz71ro6emRBu11dXVe0BtDycnJycnJsb2+1xNCFBcXuzoKuDuuE49D/gUAAAB2ZmP+JTc3Vwixa9cuueTixYtarfa6+Zf9+/db7P7YY48JIT788EO5pLe3Nzo6WqfTNTQ0SCVGo1EI8cUXX8h1Tp48KYQwGo22N3I7GYfU1NSQkJCysjIr/WAl/yIvfmSRf/HQ3hgK+RcLfK+GLbhOPA7vHwEAAMA1Dh48KITIyMiQS6Kjo8ePH3/dyj/60Y8sSnbv3i2EmD17tlyi1WrT0tJ6eno+/fRTuVCv10+ePFn+ddKkSdHR0VVVVfX19bY3csuOHDnS2tqalJR0a7tLQarVavkFH4mH9oYVu3btUuEfhBB5eXmujgLuzqH/JeEIvI0JAAAAF7hy5UpnZ6efn19AQICyPDw8vKamZnB9vV5vsXt7e7ufn19gYKCyPCIiQgjR0NAglwQHB1s0FR4eXldXd+nSpZEjR9rYiKtIE+IkJSWp1Wpluff1xt133/3ss8869BAeJC8vr6Cg4JbTdhgm8vLyXB0Cbg75FwAAALiAVqsNDAzs7Ozs6upSpmBaW1tt3N1gMLS3t3d2dirzBY2NjUKIyMhIuaSlpcVsNiv/Vnzp0iUhRHh4uI2N+Pj4XL16VXn0trY2i3gc8bfogYGBTZs2CSF++ctfWq/pBb0xevTo+fPn3+xe3iovLy8pKYkOgXXkXzwO7x8BAADANWbNmiX+8RaSpLm5+fTp0zbuPm/ePCHEvn375JIrV66UlpbqdDrlO029vb0mk0n+9csvv6yrqzMajVFRUTY2EhUVdfHiRblCQ0PDd999ZxGMv7+/nJW44447tmzZYuNZWFFYWPjXv/513rx50kQ51nl9bwCAp99bAdcAACAASURBVCP/AgAAANd4/fXXR44cWVBQcOjQoa6urq+//nrx4sUWryNZsXbt2vj4+IKCgr1793Z2dtbU1Dz88MP19fUbN26UXpmRGAyGF154oby8vLu7u6KiYvHixRqNZuPGjbY3kp6eXldX97vf/a6rq6u2tvaZZ54JDw+3COaHP/xhTU3N+fPny8vLz549m5KSIpXPnDkzNDT0+PHjNp7UwMDApUuX/ud//ictLW39+vWPP/74jh07bHmcxCN6AwCGNVdPAAwAAABvY+P6R2az+fTp0w8++GBQUJBOp5s2bdrevXvT0tKkYerPf/7z8vJy62PX5ubmgoKC+Ph4tVptMBgyMjJKS0uVFYxGY0xMzNdff52RkREYGKjT6VJTU48ePXpTjbS1tS1ZsiQqKkqn0yUnJ5tMpsTERCme559/XqpTXV2dkpKi1+tjY2M3bdok75uSkmJ9/SOLmVxUKpXBYJg0adKTTz554sQJZU0v6I2hsP6RBcG6NrAB14nHUZnNZjtndAAAADC87dy5My8vzx3GmZMnT25ubr5w4YKrA3ELbtsb0gtWH3/8sasDcRcqlaq4uJj5X2Ad14nH4f0jAAAAAPBU586dmzt3bkdHR3Nzs7wy8ZQpU3p7e5XVlFtVKtXUqVNdFfBgnhu50v79+8ePH+/rO+QSN319fRs2bEhMTAwMDAwPD581a1ZJSYmcp165cqX05CC8GPkXAAAAAPBIlZWVU6dOTU9PDwoKCgsLM5vN0vzKlZWVBQUFyprS1vLy8tDQULPZXFFR4aKQr8NzI5fU1tbOnTu3sLBQWizsurq7u2fOnLlt27YNGzZcunSpoqIiICBg7ty5X331lVThiSeeKCwsXLVqlbOihguQfwEAAIAXKioqUqlUVVVVFy9eVKlUL730kqsjciV6QykgICA5Odlz25d1dHTMmTPnoYceeuqpp5TlWq02NDR08+bNH374oRPCsCMPjXzVqlX33HPPiRMnlAu3W3juuedOnjz52Wef3XfffTqdbsyYMdu2bdNqtXKFhISE3bt3r1mzZufOnU6JGi5A/gUAAABeaMWKFcpZD1evXu3qiFyJ3vBK69evb2hoePnlly3K/fz8duzY4ePjs3Tp0pqaGpfEdms8NPJ33nln5cqVVt48amxs3LJly6JFi5SLken1+t7e3okTJ8olRqMxJydn+fLl/f39jo0YLkL+BQAAAAA8jNls3rp16/Tp06OjowdvzcjIeOmllzo7O3Nzcy2mU3Fznhi5TqezXuGTTz65du2aLU9FzZs378KFC/v27bNTaHAv5F8AAAAAeIaWlpZly5YlJCRoNJqQkJBZs2YdPnxY2rR69Wppflb5W+7BgwelkrCwMKlEeg+ru7v72LFj0ibpmQWpXKVSjR492mQypaWlBQYG+vv7z5gx49ixY7ffviNUVVU1NjYajcahKrzyyivp6eknT558+umnrbRjpUv37Nkjz3r77bff5uXlBQcHh4aGZmZm1tbWKhtpamrKz8+Pi4vTaDSjRo3Kzs6urKy85VPz3MiH8vnnnwshQkJCli9fHhsbq9Foxo4dm5+f39raalFz8uTJQohPP/3U7jHALThjkWsAAAAMJ9IqHq6OAh4jJycnJyfnhtXq6+vj4+MjIiJKSkra29tPnz6dnZ2tUqnefvttuY5er7/33nuVeyUmJkrztlqpIzEajXq9PikpqaysrKury2Qy3XXXXRqN5siRI3Zpf8aMGSNHjiwvL7/hmQohiouLrdfZvn27EOL111+3KDeZTAaDQfq5qakpNjZWCPH+++9LJfIsthJbujQrK0sIkZWVJXXLoUOHdDrdtGnT5Ap1dXVjx46NiIjYt29fZ2fnqVOnUlNT/fz8ysrKbnim3hG5LCYmZsSIEYPLpUgiIyMXLVpUW1v7/fffv/fee3q9fvz48W1tbcqa7e3tQoiUlBRbDmfLdQK3wvMvAAAAADxAYWHhN99889vf/jYzMzMoKGj8+PEffPBBVFRUfn6+lXVnbkp3d/ebb76ZlJSk1+unTp36/vvvX7169ZlnnrFL4wMDA9J3MLu0Vl9fL4QwGAxW6oSFhe3cuVOtVi9durS6unpwBdu7dMmSJVK33H///bNnzzaZTM3NzXIj586de+ONNx544IGAgIAJEyZ89NFHZrPZ+tMr1nlu5NclvUil0+m2bds2bty44ODgn/70p4WFhTU1Nb/5zW+UNYOCglQqlfSPC+9D/gUAAACAB9i9e7cQYvbs2XKJVqtNS0vr6emx1/saer1eegFEMmnSpOjo6KqqKrt8Hz5y5Ehra2tSUtLtNyX+8ZVerVZbr3b33XcXFRV1d3fn5ub29PRYbLW9S6dNmyb/LD2ZUldXJ/26Z88eHx+fzMxMuUJkZOSECRNOnDhx4cKFWzg1T498ML1eL4S4//77le+jzZkzR1zvVSNfX9/B5wvvQP4FAAAAgLu7cuVKe3u7n5+fxRK/0oIyDQ0NdjlKcHCwRUl4eLgQ4tKlS3Zp3478/PyEEH19fTesmZ+fn5eXd+rUKYtlqm+qS5UP2mg0GiHEwMCA3MjAwIDBYFApSDOenDlz5pZP0KMjtxAXFyeECA0NVRZKl1ZTU5NF5f7+/htO6AsP5ajpoAAAAADAXrRarcFgaG9v7+zsVH7rll42iYyMlH718fG5evWqcse2tjaLplQq1VBHaWlpMZvNygpS5kX6qnz77dtRVFSUEEKaLuSGtm7dWllZ+e6770pZG4mNXWqdVqsNDg7u6urq6elxxGTDnhu5UnJy8htvvGHxFJV0aSlXpBZCdHR0mM1m6R8X3ofnXwAAAAB4gHnz5gkhlEvzXrlypbS0VKfTZWRkSCVRUVEXL16UKzQ0NHz33XcW7fj7+8s5lDvuuGPLli3ypt7eXpPJJP/65Zdf1tXVGY1G+fvwbbZvRxMnThRC2PiaTEBAwB/+8Ae9Xv/mm28qy23p0hvKzs7u7++XF4qSrFu3bsyYMf39/TY24n2RKz3wwAMxMTEHDx5UrqhdUlIihHjwwQeVNaWrS/rHhfch/wIAAADAA6xduzY+Pr6goGDv3r2dnZ01NTUPP/xwfX39xo0b5YcI0tPT6+rqfve733V1ddXW1j7zzDPyoyuyH/7whzU1NefPny8vLz979mxKSoq8yWAwvPDCC+Xl5d3d3RUVFYsXL9ZoNBs3bpQr3E77M2fODA0NPX78uF16w2g0hoeHV1VV2Vh/woQJmzdvtii0pUtvaO3atQkJCY8//viBAwfa29tbW1s3b9782muvFRUVyc+VLF68WKVSffPNNza26R2Ry7Ra7datW1taWhYsWHDmzJm2trbt27evXbt2+vTp+fn5yprS6tfp6em3czi4L9ctvQQAAADvxPrTuCk2rj9tNpubm5sLCgri4+PVarXBYMjIyCgtLVVWaGtrW7JkSVRUlE6nS05ONplMiYmJ0hef559/XqpTXV2dkpKi1+tjY2M3bdok72s0GmNiYr7++uuMjIzAwECdTpeamnr06FF7tZ+SkhISEmLL2sbCtnWFX3jhBV9f34sXL0q/WswkkpiYOHiXJ5980mK1bCtdWl5ermzwxRdfNP/fxZtmz54t1WxpaVm2bNm4cePUavWoUaPS09MPHTqkPMrMmTMDAgL6+/uveyKeG7lEepLFgnIlbElZWVlGRobBYNBoNHfeeeerr756+fJlizq5ubkxMTFXr161cjiZjdcJ3IfKbKf1zwAAAADJzp078/LyGGfCRrm5uUKIjz/+2LVhTJ48ubm52b4L39walUpVXFw8f/5869Xa29snTJiQmZn51ltvOSewW9PW1hYdHb1o0aK3337b1bHcHCdHXlVVNWXKlA8++GDBggW21LfxOoH74P0jAAAAAPA8BoOhpKRk165dmzZtcnUsQzKbzfn5+UFBQb/61a9cHcvNcXLkZ8+ezc7OLiwstDH5Ak9E/gUAAAAAPNKUKVMqKioOHDjQ0dHh6liur7Gx8ezZs6WlpTYuS+Q+nBz55s2b16xZs2bNGiccC67C+tMAAAAAhrWioqLnnntO+lmlUr344ourV692bUi2i4uL27t3r6ujGFJkZOTRo0ddHcWtcHLk69atc9qx4CrkXwAAAAAMaytWrFixYoWrowDg5Xj/CAAAAAAAwLHIvwAAAAAAADgW+RcAAAAAAADHIv8CAAAAAADgWMy/CwAAAIfYuXOnq0OAZ7hw4YLggvm/ysvLXR0CADtTmc1mV8cAAAAAr7Jz5868vDxXRwEAXq64uHj+/PmujgK2Iv8CAAAAYLiTvsTyDA4Ax2H+FwAAAAAAAMci/wIAAAAAAOBY5F8AAAAAAAAci/wLAAAAAACAY5F/AQAAAAAAcCzyLwAAAAAAAI5F/gUAAAAAAMCxyL8AAAAAAAA4FvkXAAAAAAAAxyL/AgAAAAAA4FjkXwAAAAAAAByL/AsAAAAAAIBjkX8BAAAAAABwLPIvAAAAAAAAjkX+BQAAAAAAwLHIvwAAAAAAADgW+RcAAAAAAADHIv8CAAAAAADgWORfAAAAAAAAHIv8CwAAAAAAgGORfwEAAAAAAHAs8i8AAAAAAACORf4FAAAAAADAsci/AAAAAAAAOBb5FwAAAAAAAMci/wIAAAAAAOBY5F8AAAAAAAAci/wLAAAAAACAY5F/AQAAAAAAcCzyLwAAAAAAAI5F/gUAAAAAAMCxyL8AAAAAAAA4FvkXAAAAAAAAx/J1dQAAAAAA4Gx//vOfy8vL5V+rq6uFEOvWrZNLkpKS7rvvPhdEBsBLqcxms6tjAAAAAACnKi0tvf/++9VqtY+P5TsBAwMDfX19//u//5uWluaS2AB4JfIvAAAAAIadgYGByMjIpqam624NCwtraGgYMWKEk6MC4MWY/wUAAADAsOPj47No0SKNRjN4k0ajWbx4MckXAPZF/gUAAADAcLRw4cKrV68OLr969erChQudHw8A78b7RwAAAACGqbi4uHPnzlkUxsbGnjt3TqVSuSQkAN6K518AAAAADFOPPPKIWq1WlqjV6scee4zkCwC74/kXAAAAAMNUdXX1P//zP1sUnjp1asKECS6JB4AX4/kXAAAAAMPUnXfeOWHCBOXTLv/yL/9C8gWAI5B/AQAAADB8/fSnP5WXOlKr1Y8++qhr4wHgrXj/CAAAAMDwdf78+bFjx0pfi1Qq1dmzZ+Pi4lwdFAAvxPMvAAAAAIav2NjY6dOn+/j4+Pj4TJ8+neQLAAch/wIAAABgWHvkkUdUKpWPj88jjzzi6lgAeC3ePwIAAAAwrDU3N0dGRgoh6urqwsPDXR0OAO9E/gUAAABOsnPnzry8PFdHAQAepri4eP78+a6OArfL19UBAAAAYHgpLi52dQhwjQ0bNgghnn32WVcHch1//vOfVSpVSkqKMw+al5dXUFCQlJTkzIPC45C29hrkXwAAAOBU/BV32Pr444+Fu14As2bNEkIEBgY686B5eXlJSUnu2SFwH+RfvAb5FwAAAADDnZMzLwCGIdY/AgAAAAAAcCzyLwAAAAAAAI5F/gUAAAAAAMCxyL8AAAAAgLs7d+7c3LlzOzo6mpubVf8wZcqU3t5eZTXlVpVKNXXqVFcFPJjnRq60f//+8ePH+/oOOZVqX1/fhg0bEhMTAwMDw8PDZ82aVVJSYjabpa0rV65kDbhhi/wLAAAAALfW1dX1gx/8IDMz09WBuExlZeXUqVPT09ODgoLCwsLMZrPJZJLKCwoKlDWlreXl5aGhoWazuaKiwkUhX4fnRi6pra2dO3duYWFhY2PjUHW6u7tnzpy5bdu2DRs2XLp0qaKiIiAgYO7cuV999ZVU4YknnigsLFy1apWzooYbIf8CAAAAwK2ZzeaBgYGBgQFXBRAQEJCcnOyqo3d0dMyZM+ehhx566qmnlOVarTY0NHTz5s0ffvihq2K7NR4a+apVq+65554TJ05YWS3rueeeO3ny5GeffXbffffpdLoxY8Zs27ZNq9XKFRISEnbv3r1mzZqdO3c6JWq4EfIvAAAAANxaYGBgbW3t/v37XR2Ia6xfv76hoeHll1+2KPfz89uxY4ePj8/SpUtrampcEtut8dDI33nnnZUrV1p586ixsXHLli2LFi2KiIiQC/V6fW9v78SJE+USo9GYk5OzfPny/v5+x0YMN0P+BQAAAADclNls3rp16/Tp06OjowdvzcjIeOmllzo7O3Nzcy2mU3Fznhi5TqezXuGTTz65du2aLY9KzZs378KFC/v27bNTaPAM5F8AAAAAuK89e/bIc7JKX9SVJd9++21eXl5wcHBoaGhmZmZtba20V1FRkVRh9OjRJpMpLS0tMDDQ399/xowZx44dk+qsXr1aqiN/YT548KBUEhYWpmynu7v72LFj0iYrjz84QlVVVWNjo9FoHKrCK6+8kp6efvLkyaefftpKOy0tLcuWLUtISNBoNCEhIbNmzTp8+LC0yZb+lDQ1NeXn58fFxWk0mlGjRmVnZ1dWVt7yqXlu5EP5/PPPhRAhISHLly+PjY3VaDRjx47Nz89vbW21qDl58mQhxKeffmr3GODWzAAAAIBTSKt+uDoKuExOTk5OTs6t7ZuVlSWE6OnpsSjJysoqKyvr6uo6dOiQTqebNm2aci+j0ajX65OSkqQ6JpPprrvu0mg0R44ckevo9fp7771XuVdiYqI0BayVOpIZM2aMHDmyvLz81k5KCFFcXGy9zvbt24UQr7/+ukW56f9r7+6DorrOP4Cfiyxvu7Ag7yAWJIN2Ng4SYZQWxgi6mwjKS1nxBTtTiqU2BnZQmyxKmhEohVIqTSAhUJpEpYBmJAGLHYu1LQot0EJiLELAaHiVl+7yEt623N8fd3p/N4uuCy7sAt/PX+65z57z3LM7DvvMPec0NAiFQubfAwMDbm5uhJALFy4wLewutoze3l4PDw9HR8fKykqlUnnv3r3IyEiKogoLC9mYp85nT0/Pt771LUdHx6tXr46Ojt65c2fHjh1mZma3b9+e110v38xZrq6ua9asmdvOZOLk5HT48OGOjo7//Oc/H3zwAZ/P9/LyUigU3EilUkkICQwM1GY4bb4nsCzg+RcAAAAAAFiu4uLi/P39+Xz+rl27QkJCGhoaBgcHuQHj4+P5+flMjK+v74ULF6anpxMTE3Uy+uzsLPOzSie9PVZvby8hRCgUaoixs7MrLy/n8Xjx8fGtra1zA+Ry+f3798+dOxcaGmplZeXl5VVSUuLs7JyQkKB2lI+G+ZTL5Q8ePMjJydmzZ49AIBCJRKWlpTRNa356RbPlm/ljMc9nmZubv//++xs2bLC2tv7+978vl8vb2tp+9atfcSOtrKwoimI+XFg9UH8BAAAAAIDlys/Pj/038yRFT08PN4DP5zNrPRibN292cXFpaWnRyU/fmzdvDg8P+/v7P3tXT8L8pOfxeJrDtm/fnp2dPT4+LpVKJyYm1K5euXKFEBISEsK2mJqaBgcHT0xMqC2B0TCfFRUVRkZG3FPAnZycRCJRU1NTV1fXAm5tuWc+F5/PJ4Ts2rWLu0ht79695HFLjYyNjefeL6xsqL8AAAAAAMByxX0wxMTEhBCidky1tbW12lscHBwIIY8ePVr87HTAzMyMEDIzM/PUyISEhOjo6Dt37qgdUz01NaVUKs3MzNROTWbO6Onr6+M2Pmk+mU5mZ2eFQiHFwex40t7evuAbXNaZq3F3dyeE2NrachuZ79vAwIBasEqleuqGvrDCLOneUQAAAAAAAEtpaGiIpmmKotgWpvLC/ComhBgZGU1PT3PfolAo1Drhvn2JOTs7E0KY7UKeqqioqLm5ubi4mKnaMExNTYVCoVKpHB0d5RYymPU7Tk5O2vRsampqbW09NjY2MTGxGDsQL9/MuQICAnJyctQerWK+b9wTqQkhIyMjNE0zHy6sHnj+BQAAAAAAVqzJycmGhgb25WeffdbT0+Pt7c3+9HV2du7u7mYD+vr6Hj58qNaJhYUFW6PZuHHje++9t8hZ/7/nn3+eEKLlMhmBQPDRRx/x+fz8/Hxue0REBCGEe9rx1NRUTU2Nubm5RCLRMpPIyEiVSsWeHsXIzMxcv369SqXSspOVlznXnj17XF1dr127xj1Ru7KykhASHh7OjWS+csyHC6sH6i8AAAAAALBiCYXC5OTkurq68fHxxsbGmJgYExOT3NxcNkAsFvf09Lz99ttjY2MdHR2JiYnsozGsF154oa2t7auvvqqrq+vs7AwMDGTag4KCbG1t6+vrFy9/b29vBweHlpYWLeNFIlFBQYFaY0ZGhoeHh0wmq6qqGh0dbWtrO3ToUG9vb25urtpzGRpkZGR4enrGxsZWV1crlcrh4eGCgoKzZ89mZ2ezz5XExMRQFHX//n0t+1wZmbNMTU2LioqGhoYOHDjQ3t6uUCjOnz+fkZGxbdu2hIQEbiRz+rVYLH6W4WD50d/RSwAAAACwuuD86VVuYedPMzuwsg4fPlxXV8dtOX36NP3NE4hCQkKY93p7e7u6ut69e1cikVhaWpqbm+/YsaO2tpbbv0KhiIuLc3Z2Njc3DwgIaGho2Lp1K9PPa6+9xsS0trYGBgby+Xw3N7e8vDz2vYGBgTY2Ngs+xphod65wcnKysbFxd3c381JtJ5GtW7fOfcuxY8fUjtAeHByUyWQeHh48Hk8oFEokkpqaGuaS9vM5NDSUlJS0YcMGHo9nb28vFouvX7/OHSUoKEggEKhUqsfeyPLNnME8yaKGexI24/bt2xKJRCgUmpiYbNq06c033/z666/VYqRSqaur6/T0tIbhWFp+T8DwUfRiHpYGAAAAAMAqLy+Pjo7G35+rllQqJYRcunRpyUbcsmXL4OCgbs+40SGKosrKyvbv3685TKlUikSi0NDQd999d2kSWxiFQuHi4nL48OHCwkJ95zI/S5x5S0uLj49PSUnJgQMHtInX8nsChg/rjwAAAADAoJWWljLnlXA35lxe9u3bR1FUWlrafN8oEAi4J7YYGRnZ2Nh4e3v/5Cc/aWpqWoxUwQAJhcLKysrLly/n5eXpO5cnomk6ISHBysoqNTVV37nMzxJn3tnZGRkZKZfLtSy+wEqC+gsAAAAAGLQDBw7QNB0cHKzvRBboww8/fOyyBW2MjY3961//IoSEhYXRND0zM9Pa2nr27NnW1lZfX98f/OAHX3/9tU6TBQPl4+PT2NhYXV09MjKi71wer7+/v7Ozs6amRstjiQzHEmdeUFCQnp6enp6+BGOBoUH9BQAAAABgsfT09MhksiNHjuiktzVr1jg6OoaFhd24ceOnP/3p+++/f/DgQazneqzs7GyKolpaWrq7uymKOnPmjL4zelbu7u5VVVVWVlb6TuTxnJycamtrRSKRvhOZtyXOPDMzE0++rFqovwAAAAAALJajR49KpdLFOOXkF7/4xbZt2z755JPS0lKdd74CnDx5krvt5QIWfwEA6BbqLwAAAAAAi6K4uPjzzz/Pzs5ejM4pijp+/DghJD8/fzH6BwAA3UL9BQAAAAAMTmtra3h4uFAo5PP5gYGBtbW1c2MGBgYSEhLc3d1NTEzs7e0jIyObm5uZSxUVFeyetV9++WV0dLS1tbWtrW1oaGhHRwfbw9TU1BtvvLFp0yYLC4u1a9fu3bv3k08++e9//6vNEE/V1dV14sSJ4uJiS0vLZ5gJTQICAggh9fX1MzMzT03YEOYEAGA1Q/0FAAAAAAzLF1984e/v39jYePny5f7+/vz8/NTUVG6NgBDS29vr5+dXXl6en58/PDx88+bN4eFhf3//uro6Qkh4eDhN02FhYYQQmUwmk8m6u7vLyspu3Lhx8OBBtpPjx4//5je/eeutt4aGhv79739v2rQpLCzsb3/7mzZDPFVcXNyhQ4eCgoI0xAQFBdna2tbX1893ihjMdqEqlWpwcHBZzAkAwGqG+gsAAAAAGJbk5GSFQpGbm7t7926BQLB58+bf/e53vb293Bi5XP7gwYOcnJw9e/YIBAKRSFRaWkrT9KuvvqrWW1xcnL+/P5/P37VrV0hISENDA1OtIITU1NSIRKLdu3ebm5s7Ojr+8pe/9PLyWsAQcxUWFra3t2dlZWkOm52dZXYn0Wpe5lB7o4HPCQDAKmes7wQAAAAAAL7h2rVrhBCJRMK2uLi4eHl5tbW1sS0VFRVGRkahoaFsi5OTk0gkampq6urqWrduHdvu5+fH/tvNzY0Q0tPTY2dnRwh56aWX3nnnnR/96EexsbF+fn5r1qy5d+/eAoZQ8/Dhw1OnTn388cd8Pl/znd68eVNzgGZMTYrH4zG3Y8hzwurq6iovL3+Wu15h8OgQwOqB+gsAAAAAGJCpqanR0VEzMzOBQMBtd3BwYOsvU1NTSqWSECIUCuf20N7ezi0EcGNMTEwIIbOzs8zLvLw8f3//Dz74IDg4mBASGBgYHx8fEREx3yHUVFZWKpXKF198Ua09JSUlJSWFeftzzz33pLdrj9kWx9/fn8fjGficsOrr66Ojo+dxkyvduXPnzp07p+8sAGApYP0RAAAAABgQU1NTS0vLycnJsbExbvvw8DA3xtra2tjYeGZmhp5j586dWo5FUdSRI0f+9Kc/KRSKiooKmqYjIyNzcnKecYhXXnlFLf78+fOEkNTUVOalToovs7OzeXl5zHDPmDDXIs0JKyoqau57Vy1CSFlZmb6zAEO3wP8jwPCg/gIAAAAAhuXll18m/1uFxBgcHOSugiGEREZGqlSqW7ducRszMzPXr1+vUqm0HMja2rq1tZUQwuPxdu/ezZwQdPXqVR0OsXjkcvk//vGPiIgIqVTKtGBOAAAMGeovAAAAAGBYfv7zn69du1Ymk12/fn1sbOzu3bsxMTFqy5EyMjI8PT1jY2Orq6uVSuXw8HBBQcHZs2ezs7ONjeexxP7HP/7xp59+OjU19ejRo6ysLJqm2ROLdDWEBvM9a7UgQAAAC0JJREFU/2h2dvbRo0cff/xxcHBwVlZWbGzsxYsXKYrSbcL6nRMAgBVL389SAQAAAMBqUVZWpuXfn/fu3QsPD7eysjI3N/fz86uqqmK2IyGE/PCHP2RihoaGkpKSNmzYwOPx7O3txWLx9evXmUtqe5qePn2a/uYz/CEhITRNNzc3x8fHf/vb37awsFi7du327dsLCwvZM4k0D6Gl+Ph4tT+/JRIJezUwMNDGxub27dtPervaDr4URQmFws2bNx87dqypqWluvIHPSVRUFNYfcRGsPwIt4HuyYlA0lpMBAAAAwJIoLy+Pjo7G35+rFrNU6tKlS/pOxFBQFFVWVrZ//359JwIGDd+TFQPrjwAAAAAAAAAAFhfqLwAAAAAAAIbuwYMH+/btGxkZGRwcpP7Hx8dncnKSG8a9SlGUr6+vvhKea/lmzvWHP/zBy8tLmw2P9u3bR1FUWloat/H1119nVmLCKoT6CwAAAADAvFFP9uabb+o7O1hpmpubfX19xWKxlZWVnZ0dTdMNDQ1Mu0wm40YyV+vq6mxtbWmabmxs1FPKj7F8M2d0dHTs27dPLpf39/c/NfjDDz+srKyc23706FG5XJ6SkrIICYKhQ/0FAAAAAGDeNOywiPqLgRAIBAEBAcu3f9bIyMjevXu/973vHT9+nNtuampqa2tbUFDw+9//fgnS0KFlmnlKSsp3vvOdpqYmS0tLzZE9PT0ymezIkSNzL3l6el65ciU9Pb28vHxx0gTDhfoLAAAAAACA4crKyurr63vjjTfU2s3MzC5evGhkZBQfH9/W1qaX3BZmmWb+29/+9vXXX9dm5dHRo0elUqlYLH7sVW9v76ioqBMnTqhUKl3nCAYN9RcAAAAAAAADRdN0UVHRtm3bXFxc5l6VSCRnzpwZHR2VSqVq26kYuOWYubm5uTZhxcXFn3/+eXZ2toaYiIiIrq6uq1ev6ig1WB5QfwEAAAAAAMMyNDSUlJTk6elpYmJiY2Pz8ssv//nPf2YupaWlMfvssGt/rl27xrTY2dkxLdnZ2RRFjY+P37p1i7nEPLPAtFMUtW7duoaGhuDgYEtLSwsLi507d966devZ+18MLS0t/f393t7eTwr42c9+JhaLP/3001dffVVDPxqmtKKigt296Msvv4yOjra2tra1tQ0NDe3o6OB2MjAwkJCQ4O7ubmJiYm9vHxkZ2dzcvOBbW76Za9DV1XXixIni4mLNa5S2bNlCCPnjH/+4GDmAwUL9BQAAAAAADEhfX5+fn19JSUlubu7g4ODf//53CwuL4ODgoqIiQsiZM2domubz+Wz8Sy+9RNP01q1b2ZaTJ08yMd/97neZTXmYhR5Mu7e3t0KhSExMTEtL6+vr++tf/zo8PBwUFPSXv/zlGftnBAUF2dra1tfX62Q27ty5QwhZt27dkwKMjIwuXrzo5uZWVFR08eLFx8ZontLw8HCapsPCwgghMplMJpN1d3eXlZXduHHj4MGDbCe9vb1+fn7l5eX5+fnDw8M3b94cHh729/evq6tb2K0t38w1iIuLO3ToUFBQkOYwV1dX8r8PF1YP1F8AAAAAAMCAyOXy+/fvnzt3LjQ01MrKysvLq6SkxNnZOSEhQZtzZ7QxPj6en5/v7+/P5/N9fX0vXLgwPT2dmJiok85nZ2eZooxOeuvt7SWECIVCDTF2dnbl5eU8Hi8+Pr61tXVugPZTGhcXx0zLrl27QkJCGhoaBgcH2U4ePHiQk5OzZ88egUAgEolKS0tpmtb89IpmyzfzxyosLGxvb8/KynpqpJWVFUVRzIcLqwfqLwAAAAAAYECuXLlCCAkJCWFbTE1Ng4ODJyYmdLVeg8/nMwtAGJs3b3ZxcWlpadHJ72H28Ypn74oQwuyNwuPxNIdt3749Ozt7fHxcKpVOTEyoXdV+Sv38/Nh/u7m5EUJ6enqYlxUVFUZGRqGhoWyAk5OTSCRqamrq6upawK0t98zVPHz48NSpU8XFxdyHpzQwNjaee7+wsqH+AgAAAAAAhmJqakqpVJqZmaltn+Ho6EgI6evr08ko1tbWai0ODg6EkEePHumkfx0yMzMjhMzMzDw1MiEhITo6+s6dO2rHVM9rSrkP2piYmBBCZmdn2U5mZ2eFQiHF8c9//pMQ0t7evuAbXNaZc1VWViqVyhdffJEdgjl/OiUlhXn5xRdfcONVKpWWG/rCioH6CwAAAAAAGApTU1OhUDg5OTk6OsptZxabODk5MS+NjIymp6e5AQqFQq0riqKeNMrQ0JDa+iCm8sJUYZ69fx1ydnYmhCiVSm2Ci4qKNm7cWFxcfP78ebZRyynVzNTU1Nra2tjYeGZmhp5j586d87illZU565VXXlHrnLmX1NRU5uVzzz3HBo+MjNA0zXy4sHqg/gIAAAAAAAYkIiKCEMI9mndqaqqmpsbc3FwikTAtzs7O3d3dbEBfX9/Dhw/V+rGwsGBrKBs3bnzvvffYS5OTkw0NDezLzz77rKenx9vbm/09/Iz969Dzzz9PCNFymYxAIPjoo4/4fH5+fj63XZspfarIyEiVSsUeFMXIzMxcv349d/vhhVm+mS8M8+1iPlxYPVB/AQAAAAAAA5KRkeHh4SGTyaqqqkZHR9va2g4dOtTb25ubm8ssPCGEiMXinp6et99+e2xsrKOjIzExkX10hfXCCy+0tbV99dVXdXV1nZ2dgYGB7CWhUJicnFxXVzc+Pt7Y2BgTE2NiYpKbm8sGPEv/uj3/yNvb28HBoaWlRct4kUhUUFCg1qjNlD5VRkaGp6dnbGxsdXW1UqkcHh4uKCg4e/ZsdnY2e/x2TEwMRVH379/Xss+VkfkCMKdfi8XipRkODMXcR7AAAAAAABZDWVkZ/v5czaKioqKiorSJHBwclMlkHh4ePB5PKBRKJJKamhpugEKhiIuLc3Z2Njc3DwgIaGhoYM+Hfu2115iY1tbWwMBAPp/v5uaWl5fHvtfb29vV1fXu3bsSicTS0tLc3HzHjh21tbW66j8wMNDGxub27dtPvU1CSFlZ2VPDkpOTjY2Nu7u7mZcDAwPcH3Rbt26d+5Zjx47Z2tpyWzRMqdoxzKdPn6a/uTgrJCSEiRwaGkpKStqwYQOPx7O3txeLxdevX+eOEhQUJBAIVCrVY29k+WbOqKysnPuDurCwcG5kfHy8WphEIuEGSKVSV1fX6elpDcOxtPyegOGjaB2diwYAAAAAoFl5eXl0dDT+/ly1pFIpIeTSpUv6TWPLli2Dg4M6PPhmwSiKKisr279/v+YwpVIpEolCQ0PffffdpUlsYRQKhYuLy+HDhwsLC/Wdy/wsceYtLS0+Pj4lJSUHDhzQJl7L7wkYPqw/AgAAAAAAMFxCobCysvLy5ct5eXn6zuWJaJpOSEiwsrJKTU3Vdy7zs8SZd3Z2RkZGyuVyLYsvsJKg/gIAAAAAAGDQfHx8Ghsbq6urR0ZG9J3L4/X393d2dtbU1Gh5LJHhWOLMCwoK0tPT09PTl2AsMDTG+k4AAAAAAABgKWRnZ586dYr5N0VRp0+fTktL029K2nN3d6+qqtJ3Fk/k5ORUW1ur7ywWYokzz8zMXLKxwNCg/gIAAAAAAKvCyZMnT548qe8sAGCVwvojAAAAAAAAAIDFhfoLAAAAAAAAAMDiQv0FAAAAAAAAAGBxof4CAAAAAAAAALC4sP8uAAAAACwpqVSq7xRAP+rr6wm+AN/061//+tKlS/rOAgCWAkXTtL5zAAAAAIBVoa6uLicnR99ZAAAsM0lJSf7+/vrOAp4V6i8AAAAAAAAAAIsL+78AAAAAAAAAACwu1F8AAAAAAAAAABYX6i8AAAAAAAAAAIsL9RcAAAAAAAAAgMX1f/j38Yvh4Gn9AAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "encoder_inputs = keras.Input(shape=(None, ), dtype=\"int64\", name=\"human\")\n", "x = PositionalEmbedding(config.X_LEN, config.NUM_ENCODER_TOKENS, config.EMBED_DIM)(encoder_inputs)\n", "encoder_outputs = TransformerEncoder(config.EMBED_DIM, config.DENSE_DIM, config.NUM_HEADS)(x)\n", "\n", "decoder_inputs = keras.Input(shape=(None, ), dtype=\"int64\", name=\"machine\" )\n", "x = PositionalEmbedding(config.Y_LEN, config.NUM_DECODER_TOKENS, config.EMBED_DIM)(decoder_inputs)\n", "x = TransformerDecoder(config.EMBED_DIM, config.DENSE_DIM, config.NUM_HEADS)(x, encoder_outputs)\n", "x = L.Dropout(0.5)(x)\n", "decoder_outputs = L.Dense(config.NUM_DECODER_TOKENS, activation=\"softmax\")(x)\n", "\n", "transformer = keras.Model([encoder_inputs, decoder_inputs], decoder_outputs)\n", "transformer.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=\"accuracy\")\n", "\n", "transformer.summary()\n", "utils.plot_model(transformer, show_shapes=True, expand_nested=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "8664f4c1", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:15:11.530241Z", "iopub.status.busy": "2022-04-19T17:15:11.528519Z", "iopub.status.idle": "2022-04-19T17:15:11.530831Z", "shell.execute_reply": "2022-04-19T17:15:11.531287Z", "shell.execute_reply.started": "2022-04-19T16:41:46.074568Z" }, "id": "8664f4c1", "papermill": { "duration": 0.0324, "end_time": "2022-04-19T17:15:11.531437", "exception": false, "start_time": "2022-04-19T17:15:11.499037", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "es = callbacks.EarlyStopping(monitor=\"val_loss\", patience=3, verbose=1, restore_best_weights=True, min_delta=1e-4)\n", "rlp = callbacks.ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1)" ] }, { "cell_type": "code", "execution_count": null, "id": "986826c9", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:15:11.589487Z", "iopub.status.busy": "2022-04-19T17:15:11.588613Z", "iopub.status.idle": "2022-04-19T17:19:29.560469Z", "shell.execute_reply": "2022-04-19T17:19:29.561005Z", "shell.execute_reply.started": "2022-04-19T16:41:46.082314Z" }, "id": "986826c9", "outputId": "a431935d-8eb1-4d1d-fc4a-9588134bd9c8", "papermill": { "duration": 258.005138, "end_time": "2022-04-19T17:19:29.561211", "exception": false, "start_time": "2022-04-19T17:15:11.556073", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2022-04-19 17:15:11.668989: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n", "2022-04-19 17:15:15.266490: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8005\n" ] } ], "source": [ "%%capture training_log\n", "\n", "history = transformer.fit(\n", " generate_batch(X_train, y_train), steps_per_epoch = np.ceil(len(X_train)/config.BATCH_SIZE),\n", " validation_data=generate_batch(X_valid, y_valid), validation_steps=np.ceil(len(X_valid)/config.BATCH_SIZE),\n", " epochs=config.MAX_EPOCHS, callbacks=[es, rlp], \n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "d8fc94d6", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:19:29.614789Z", "iopub.status.busy": "2022-04-19T17:19:29.613915Z", "iopub.status.idle": "2022-04-19T17:19:29.616847Z", "shell.execute_reply": "2022-04-19T17:19:29.617319Z", "shell.execute_reply.started": "2022-04-19T16:41:46.082314Z" }, "id": "d8fc94d6", "papermill": { "duration": 0.03217, "end_time": "2022-04-19T17:19:29.617449", "exception": false, "start_time": "2022-04-19T17:19:29.585279", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "with open('training.log', 'w') as f: f.write(training_log.stdout)" ] }, { "cell_type": "code", "execution_count": null, "id": "036a2d44", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:19:29.669598Z", "iopub.status.busy": "2022-04-19T17:19:29.668798Z", "iopub.status.idle": "2022-04-19T17:19:30.028795Z", "shell.execute_reply": "2022-04-19T17:19:30.029237Z", "shell.execute_reply.started": "2022-04-19T16:42:22.93183Z" }, "id": "036a2d44", "outputId": "bfc25d5f-b98a-4251-fdd7-0d0261a5e062", "papermill": { "duration": 0.388763, "end_time": "2022-04-19T17:19:30.029386", "exception": false, "start_time": "2022-04-19T17:19:29.640623", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABIEAAAFlCAYAAAB82/jyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/z0lEQVR4nO3deXxU9b3/8feZmezbZGMCikEIaisUl9pa61KCgWpE7QVabYtURb39ubW22tqFVqvirVWLXbTUXbvc61YXtGiBSm/dqqCp2xXQIAoZEknMBklm5vz+SDKZNTOBJJPM9/XsQ5M553u+3++ZTyadvD3fM5Zt27YAAAAAAACQ1hypngAAAAAAAABGHiEQAAAAAACAAQiBAAAAAAAADEAIBAAAAAAAYABCIAAAAAAAAAMQAgEAAAAAABjAlaqBA4GA/P70+HR6p9NKm3PB0FB7M1F3c1F7c1F7c1F7c1F7c1F7M6VT3TMynHH3pSwE8vtttbR0pmr4YeV256bNuWBoqL2ZqLu5qL25qL25qL25qL25qL2Z0qnu5eUFcfexHAwAAAAAAMAAhEAAAAAAAAAGIAQCAAAAAAAwACEQAAAAAACAAQiBAAAAAAAADEAIBAAAAAAAYABCIAAAAAAAAAMQAgEAAAAAABiAEAgAAAAAAMAACUOgK6+8Up/73Od0yimnxNxv27auueYa1dTUaP78+XrjjTeGfZIAAAAAAADYNwlDoP/4j//Q7bffHnf/+vXrVV9fr6efflo/+9nP9NOf/nQ45wcAAAAAAIBh4ErU4KijjtIHH3wQd/+aNWt0+umny7IsHXbYYWptbdXOnTs1YcKEYZ3oWBVo3qU9Wzepp33P0A607b0bcC8P26cx93bQvT7H0X1u7H04v878bHWPWu33pfijPF6an2NHXpa6O7qGeTIYD6i9uai9uai9uai9uai9mbqmVUrl+6V6GiMuYQiUiNfrVUVFRfBxRUWFvF5vwhDI6bTkdufu6/Ap9/45X9OuTZtSPQ2kSGuqJ4CUoO7movbmovbmovbmovbmovZmastwadqGV1M9jRG3zyHQ3vL7bbW0dKZq+GGTd83PVbrLq47RTIota/SP3ZcxR3O8UT6uID9b7e17Ufu9fTpHvX6p+Fkb5fGGOqAdUEGO1N62W5IdfTWRbfdu7//H7h/FDm9vB/8VcYyC7azQq9RshRwf2jZ0e2hfEfOxQ/qOuT/GHGO1CzmnsHlGXVUV2leMufW1t+yQtmHnH9pH6LbI9qFtFXFc9LhWrHOy440bfUxmplPdXT0xxo2YvyQrclvYfCKPCR930NpH1Sd2XWJ+H/H8938fXXvFOC76nIPzDKtjvDkMzNEKm0ecn8nQ70PGteIeE/lc2LH7D+s34vmPGM8aODlZ0sCVozF/3qO/jbMhxrzjtLXj7wv7GYkaY7B+hjKHWM9b3I7CHlmDDgoAwNjjLMpPi4xCksrLC+Lu2+cQyOPxqKGhIfi4oaFBHo9nX7sdN5yT9lPeJ6erJ01+WDA0Oe5cdVH7sc+2Jd8eWd1tcvS0y+pul9Xd1vu1p01Wd0fvvu52WVH7+x/3fu/o6ZAkxf+1CpPZsvoCyZCv/UGjFfq19x87GF5GHqc434ceZwV3RY3niBy3L86Imlv4fMLnKEWNFzXHWHPv78PRe6ysGHOMc5w10OeQnsvQcxzKc9n31Q72G+O57Ps+M8ul7m5/+Hkr3sPwfXZUWytu2+hAeyhtk+xn0DHCH9tD6mfAQHSW5JhJPyeD9xM933j9RB4bf8ycnEzt3t2TXD8J/4PE3j63e/mcRE1nuJ7bodRvsDkM4ed/CPXb+5+D8P15edkx/kNvMq/Dobyu44jR7+C/S5IdO9b4Mdok+lkeprGjzinmcXsxv33styA/W23te/a6n+RqtRfzG6mfgSTml9qfv1ht9n3syHMqKi+XDPjTbp9DoOrqat1///2qra3Va6+9poKCAmPuBwRghPm7ZfV0BAMZR1gwMxDehD529LQNhDahbe1AwuFsyyk7s0B2Zn7fPwWys4vlLzyg93FG7/bsgiLt7goo/A9LSXH/WLaC+2P+kakYf5z3t4v8Qz1mgBB7f+wAobetPUi/GqRfO2a/EWPE2GbH6zfec9XXNnYQEDle9FxiHpdEcGHH7bd3v9udp5aPdw/SL9KV252rdkJ/I2W5c7Wb2hsp152rbmpvJNudKx+1N09mrtSZ/nVPGAJddtlleumll9Tc3Kzjjz9eF198sXw+nyTpzDPP1AknnKBnn31WNTU1ysnJ0XXXXTfikwYwhgX8fcFNaFjT97WnIyLIib7iJuxqHH9yS+0CGfkDwU1Gb3gTyJ0gOzO/b1+B7My83q8Rj0OPlTM7qT/oM/mDwFzODMnRk7gdAAAAMAYlDIFuuummQfdblqWf/OQnwzYhACkQtVwq5GqaqOVSIVfgRCyXcnS3yfIlF47YzqzeEKb/ipuMPAXyJ8kfcsVNcH/IYzsjL+RqnQLZGbnqXXYCAAAAABhMym4MDWAY+LsjrqKJXC4VsSRqxJdLhVxxExLeBCIe2xn5vVdUAAAAAABGDSEQMNpCl0sF72eTXsulAAAAAABjDyEQkAzblny7e0OY0KtsvD3KamkKWQ7FcikAAAAAwNhECIT0NsTlUo6I0CaZ5VKFId+zXAoAAAAAMFYRAmHsGWS5lCNymVTIx4f3BzaO0H0jvFwqr6RcrV0ulksBAAAAAMY8QiAMj3jLpYLLoyKXS0VeaTNOl0u5cxXgo8IBAAAAAOMAIZDp+pdLxbwpcVvUFTfRy6VCrsbh06UAAAAAABizCIHGo6Esl4oMbUKXS3W3ywp0Jzckny4FAAAAAMC4Rgg0WvqWSzli3scmmeVSvZ84NaTlUq5s2Rn5sZdLhSyLilouFbV8ik+XAgAAAABgvCME2kcZ7/9djtdeVV5by8CVOPFuXpzMcimHKySA6bviJrtEdtzlUr2PWS4FAAAAAAAGQwi0j3Jf+ZWc219UdowbD0cvlwpfThW+XKr3MculAAAAAADASCAE2kcfn/6A3O5ctXy8J9VTAQAAAAAAiIsbvewry8H9cgAAAAAAwJhHegEAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABkgqBFq/fr3mzZunmpoarVy5Mmr/9u3btXjxYp1++umaP3++nn322WGfKAAAAAAAAPaeK1EDv9+vq6++WnfddZc8Ho8WLlyo6upqVVVVBdvceuutOumkk/TVr35Vmzdv1vnnn6+1a9eO6MQBAAAAAACQvIRXAtXV1amyslKTJ09WZmamamtrtWbNmrA2lmWpvb1dktTW1qYJEyaMzGwBAAAAAACwVxJeCeT1elVRURF87PF4VFdXF9bmoosu0rnnnqv7779fu3fv1l133TX8MwUAAAAAAMBeSxgCJWPVqlX60pe+pHPOOUcbN27UFVdcoSeeeEIOR/wLjZxOS2537nAMn3JOpyNtzgVDQ+3NRN3NRe3NRe3NRe3NRe3NRe3NZErdE4ZAHo9HDQ0Nwcder1cejyeszYMPPqjbb79dknT44Yerq6tLzc3NKi0tjduv32+rpaVzb+c9prjduWlzLhgaam8m6m4uam8uam8uam8uam8uam+mdKp7eXlB3H0J7wk0c+ZM1dfXa9u2beru7taqVatUXV0d1mbixIl6/vnnJUlbtmxRV1eXSkpK9nHaAAAAAAAAGC4JrwRyuVxatmyZli5dKr/frwULFmj69OlasWKFZsyYoTlz5uj73/++fvSjH+nuu++WZVm6/vrrZVnWaMwfAAAAAAAASbBs27ZTMXBPjz9tLrVKp8vGMDTU3kzU3VzU3lzU3lzU3lzU3lzU3kzpVPd9Wg4GAAAAAACA8Y8QCAAAAAAAwACEQAAAAAAAAAYgBAIAAAAAADAAIRAAAAAAAIABCIEAAAAAAAAMQAgEAAAAAABgAEIgAAAAAAAAAxACAQAAAAAAGIAQCAAAAAAAwACEQAAAAAAAAAYgBAIAAAAAADAAIRAAAAAAAIABCIEAAAAAAAAMQAgEAAAAAABgAEIgAAAAAAAAAxACAQAAAAAAGIAQCAAAAAAAwACEQAAAAAAAAAYgBAIAAAAAADAAIRAAAAAAAIABCIEAAAAAAAAMQAgEAAAAAABgAEIgAAAAAAAAAxACAQAAAAAAGIAQCAAAAAAAwACEQAAAAAAAAAYgBAIAAAAAADAAIRAAAAAAAIABCIEAAAAAAAAMQAgEAAAAAABgAEIgAAAAAAAAAxACAQAAAAAAGIAQCAAAAAAAwACEQAAAAAAAAAYgBAIAAAAAADAAIRAAAAAAAIABCIEAAAAAAAAMQAgEAAAAAABgAEIgAAAAAAAAAxACAQAAAAAAGIAQCAAAAAAAwACEQAAAAAAAAAYgBAIAAAAAADCAK9UTAAAAAAAAY5/f71Nzc6N8vu5UT2XYeb2WbNtO9TSGxOXKVHFxuZzO5KMdQiAAAAAAAJBQc3OjsrNzlZdXIcuyUj2dYeV0OuT3B1I9jaTZtq2OjlY1NzeqrGxi0sexHAwAAAAAACTk83UrL68w7QKg8ciyLOXlFQ75qixCIAAAAAAAkBQCoLFjb2pBCAQAAAAAAMaFmprjUj2FcY0QCAAAAAAAwADcGBoAAAAAAIwrtm3rt7+9RS+88E9ZlqUlS87VnDlz1dTUpJ/85Ep1dHTI7/fpu9+9UjNmfErXX/8zvf32m7IsS7W1p+orX/laqk8hJZIKgdavX69rr71WgUBAixYt0vnnnx/V5sknn9Svf/1rWZalQw45RDfeeOOwTxYAAAAAAKTeqje8euz1hmHt89QZFao91JNU22efXatNm/5Pd9/9J338cYuWLj1Ls2YdoWee+as+85mjtWTJufL7/erq2qNNm95RY+NO3Xff/0iS2trahnXe40nCEMjv9+vqq6/WXXfdJY/Ho4ULF6q6ulpVVVXBNvX19Vq5cqX+9Kc/qaioSB999NGIThoAAAAAAJirru5VnXjiPDmdTpWUlOrww4/Q22+/oU984pNavvxq+Xw+HX/8FzR9+sGaNGk/bd/+oW6++ef63OeO1Wc+c3Sqp58yCUOguro6VVZWavLkyZKk2tparVmzJiwE+p//+R997WtfU1FRkSSptLR0hKYLAAAAAABSrfZQT9JX7Yymww47Qr/5ze/13HP/q2uvvUpf+cpXddJJp+juu/+kl156Xo8++pDWrn1GP/jBT1I91ZRIGAJ5vV5VVFQEH3s8HtXV1YW1qa+vlySdccYZCgQCuuiii3T88ccP2q/Tacntzt2LKY89Tqcjbc4FQ0PtzUTdzUXtzUXtzUXtzUXtzUXt4/N6LTmdqf98KafTocMPP0J/+ctDOuWUU9Xa2qrXXtuoiy/+tnbubNCECR596UsL5PP1aNOm/9Oxxx6njIwMzZlToylTDtRVV/0o5nmMhXMbKssaWrYyLDeG9vv92rp1q+677z41NDTo61//uh5//HEVFhYOcoytlpbO4Rg+5dzu3LQ5FwwNtTcTdTcXtTcXtTcXtTcXtTcXtY/Ptm35/YFUT0N+f0DHHvsF1dXVafHir8iyLH3zm5fI7S7RU089oT/+8V65XC7l5OTqRz+6Sg0NXi1ffpUCAVuSdMEFF0adh9PpGBPnNlS2HZ2tlJcXxG2fMATyeDxqaBi42ZPX65XH44lqM2vWLGVkZGjy5MmaMmWK6uvr9alPfWqo8wcAAAAAAIjpmWf+Ian3CpgLL7xUF154adj+k046RSeddErUcXfe+YdRmd9Yl/Bap5kzZ6q+vl7btm1Td3e3Vq1aperq6rA2J554ol566SVJ0q5du1RfXx+8hxAAAAAAAABSL+GVQC6XS8uWLdPSpUvl9/u1YMECTZ8+XStWrNCMGTM0Z84cHXfccfrnP/+pk08+WU6nU1dccYWKi4tHY/4AAAAAAABIgmXbtp2KgXt6/GmzzpI1o+ai9mai7uai9uai9uai9uai9uai9vE1NGxVRUVlqqcxIsbrPYFi1WSwewKNv1tfAwAAAAAAYMgIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAACE8Pl8qZ7CiEj4EfEAAAAAAABjxZVXfkder1fd3d1atOgMnXbaf+iFF57TypW/kd8fkNvt1ooVt6qzs1O//OUNevvtN2VZls4++zx94QtzVFNznJ555h+SpHXr/qbnnvtfLVt2ta699qfKzMzUO+/8nz71qVmaM2euVqy4Ud3dXcrKytYPfrBMBxwwRX6/X7fe+iu9+OJzcjgcmj//dB144DQ9+OCftXz5jZKkf/3rBT388INavvwXqXyqohACAQAAAACAIcl6+0Flv/XnYe1zzyfOUNchCxO2u/LKZSosLFJX1x4tXXqWjjvuBP3859fq179eqUmT9lNr68eSpLvvvl15efm6997/liS1trYm7Luxcaduu+1OOZ1OdXS06ze/+b1cLpf+9a8X9bvf/UbXXnuDHnvsETU0bNddd/1RLpdLra0fq6CgUDfeeL2am5tVXFysVaseV23tqfv2hIwAQiAAAAAAADBuPPDAn7V+/d8lSTt3evXYY49o1qzDNWnSfpKkwsIiSdLLL7+kq666LnhcYWFhwr5nzz5RTqdTktTe3q5rrvmpPvjgfVmWFVwi9vLLL+r00xfI5XKFjTdv3sl6+ukndfLJp+qNN/6tH/3oquE54WFECAQAAAAAAIak65CFSV21M9w2bHhZL7/8kn73u7uUnZ2tiy46X1VVB2nr1voh9GIFv+vu7g7bk52dHfz+9ttv0xFHfFrLl/9CO3Zs18UXXzBor7W1p+qKK76tzMwszZ49JxgSjSXcGBoAAAAAAIwLHR3tKigoVHZ2trZurdebb76u7u5uvfbaRm3f/qEkBZeDHXXUZ/Xwww8Ej+1fDlZSUqL6+vcUCAS0fv26uGO1t7ervLxckvTkk48Htx911Gf16KMPB68M6h+vrKxcZWXluueeO3TyyWNvKZhECAQAAAAAAMaJz372GPn9fn3tawt1222/0ic/OUNut1uXX/4D/fCHl2vJkjO1bNmVkqQlS85VW1urFi/+spYsOVMbN74sSfrP/7xIV1zxLf3nf56j0tKyuGN97Wtn6bbbfqOzz/6q/H5/cPspp5wuj6dC3/jGmVqy5Ew988xfg/vmzv2iJkzwaMqUA0foGdg3lm3bdioG7unxq6WlMxVDDzu3OzdtzgVDQ+3NRN3NRe3NRe3NRe3NRe3NRe3ja2jYqoqKylRPY0Q4nQ75/YF97uemm/5LBx10sE455fR9n1QSYtWkvLwgbnuuBAIAAAAAANhH55zzdW3Zsllz556c6qnENfbuUgQAAAAAADDO3Hnn/ameQkJcCQQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAaaem5ri4+3bs2K7Fi788irMZGwiBAAAAAAAADOBK9QQAAAAAAMD48vQHT+mpD54Y1j5P2v8Uzd3/pLj7b731V5owwaMFC3qv4Lnjjt/J6XRq48ZX1NbWKp/Pp/PO+6aOO+4LQxq3q6tLN910vd566005nU5dfPFlOuKIT+vdd7do+fKr1NPjk20HdM01P1dZWbmWLfu+du7cqUDAr298Y6nmzJm7L6c9qgiBAAAAAADAmDdnTo1uueWmYAi0bt3fdOONv9KiRWcoLy9fLS0tuuCCb+jYY0+QZVlJ9/vwww9IsnTvvf+trVvr9e1vX6g//elhPfroQ1q06EzNnXuSenp6FAj49fzz/1RZWbluuGGFJKm9vX0kTnXEEAIBAAAAAIAhmbv/SYNetTMSDjroEDU371JTU6Oam5tVUFCg0tIy3XLLjXrttY2yLIcaGxu1a9dHKi0tS7rfurpX9eUvnyFJqqycooqKidq27X0deuindO+9d2rnTq9OOKFakycfoKlTq/TrX/9Sv/3tLfr854/TrFmHj9TpjgjuCQQAAAAAAMaF2bNP1Lp1a7R27TOqrp6rp59+Si0tLbrjjvt1991/VElJibq7u4dlrLlzv6j/+q+blJWVrcsvv1SvvPIvHXBApe68835Nm1al3//+Vt111++HZazRQggEAAAAAADGherqGq1Z87TWrVuj2bNPVHt7u4qLi+VyubRhw8tqaNgx5D5nzTpMq1c/JUl6//2t8nobdMABlfrwww80adJ+WrToDB177AnasmWTmpoalZWVrXnzTtaZZy7WO++8PdynOKJYDgYAAAAAAMaFqVOnqbOzQ+Xl5SorK9PcuSfpe9/7ts466ys65JBPqrJyypD7/NKXFummm67XWWd9RU6nUz/84U+VmZmptWv/ptWrn5TL5VJJSanOOutsvfXWm/rtb1fIshxyuVz67ne/P/wnOYIs27btVAzc0+NXS0tnKoYedm53btqcC4aG2puJupuL2puL2puL2puL2puL2sfX0LBVFRWVqZ7GiHA6HfL7A6mexpDFqkl5eUHc9iwHAwAAAAAAMADLwQAAAAAAQFrasmWzfvazZWHbMjIy9Pvf35OiGaUWIRAAAAAAAEhL06ZV6e67/5jqaYwZLAcDAAAAAAAwACEQAAAAAACAAQiBAAAAAAAADEAIBAAAAAAAYABCIAAAAAAAkHZqao5L9RTGHEIgAAAAAACAEeLz+VI9hSA+Ih4AAAAAAAzJnr+u0p5Vjw9rn9m185X9xdq4+2+99VeaMMGjBQu+LEm6447fyel0auPGV9TW1iqfz6fzzvumjjvuCwnH6uzs1JVXfid43AUXXKjPf/54SdJTTz2hP//5fkmWqqqq9OMf/0y7dn2kG25Yru3bP5Qkffe731dZWbmuuOJbuu++/5Ek/fGP92n37k6de+4Fuuii8zV9+sGqq3tVJ544T5MnH6B77rlDPl+PCgvd+slPfqaSklJ1dnbql7+8QW+//aYsy9LZZ5+n9vZ2bdmyWZde+h1J0mOPPaL6+nd1ySXf2YdntxchEAAAAAAAGPPmzKnRLbfcFAyB1q37m2688VdatOgM5eXlq6WlRRdc8A0de+wJsixr0L4yMzN13XU3hB13zDHH6b333tU999yp2267U263W62tH0uSfvnLX+jww4/Q8uW/kN/v1+7du9XW1jroGD09PbrjjvskSa2trVq58m5ZlqXHH/+L/vCHe3Xxxd/W3Xffrry8fN17738H27lcLt1775268MJL5XK59OSTj+vyy3+wr0+fJEIgAAAAAAAwRNlfrB30qp2RcNBBh6i5eZeamhrV3NysgoIClZaW6ZZbbtRrr22UZTnU2NioXbs+UmlpWcL+fve730Qdt2HDvzR79hy53W5JUmFhkSRpw4Z/6Uc/ukqS5HQ6lZ+fnzAEmjOnJvh9Y+NO/eQnV+qjj5rU09OjiRP3kyS9/PJLuuqq64LtCgsLJUlHHnmU/vnPf2jKlAPl8/k0bVpV8k/UIAiBAAAAAADAuDB79olat26Ndu36SNXVc/X000+ppaVFd9xxv1wulxYunK/u7u6E/eztcaGcTqds2w4+7u7uCtufk5MT/P7mm3+uM874mo499gRt2PCy7rxz5aB9n3LK6brvvjt1wAFTdPLJ84c0r8FwY2gAAAAAADAuVFfXaM2ap7Vu3RrNnn2i2tvbVVxcLJfLpQ0bXlZDw46k+ol33BFHHKV169bo449bJCm4HOzII4/SX/7yoCTJ7/ervb1dJSWlam7epY8/blF3d7eee+5/447X0dGusrIJkqS//nVVcPtRR31WDz/8QPBxa2vv1UWHHjpDO3d69be/rdaJJ85L8tlJjBAIAAAAAACMC1OnTlNnZ4fKy8tVVlamuXNP0ttvv6WzzvqK/vrXVaqsnJJUP/GOmzp1mpYsOUcXXXS+liw5U7/61c2SpEsv/a42bHhZZ531FZ177mLV178rl8ulb3zjPJ133hJ9+9sXDjr2Oeecrx//+Ps655yvq6jIHdy+ZMm5amtr1eLFX9aSJWdq48aXg/tmz67RzJmfCi4RGw6WHXrt0ijq6fGrpaUzFUMPO7c7N23OBUND7c1E3c1F7c1F7c1F7c1F7c1F7eNraNiqiorKVE9jRDidDvn9gVRPI8wVV3xLX/7yV/XpT38mbptYNSkvL4jbniuBAAAAAAAAxoi2tjadccZ/KDMza9AAaG9wY2gAAAAAAJCWtmzZrJ/9bFnYtoyMDP3+9/ekaEaJFRQU6M9/fnhE+iYEAgAAAAAAaWnatCrdffcfUz2NMYPlYAAAAAAAAAYgBAIAAAAAADAAIRAAAAAAAIABkgqB1q9fr3nz5qmmpkYrV66M22716tU6+OCD9e9//3vYJggAAAAAACBJNTXHpXoK41rCEMjv9+vqq6/W7bffrlWrVumJJ57Q5s2bo9q1t7fr3nvv1axZs0ZkogAAAAAAAJF8Pl+qpzBuJAyB6urqVFlZqcmTJyszM1O1tbVas2ZNVLsVK1bovPPOU1ZW1ohMFAAAAAAAQJI2bHhZ/+//LdX3vvdtff3rX071dMaNhB8R7/V6VVFREXzs8XhUV1cX1uaNN95QQ0ODvvCFL+iOO+5IamCn05LbnTvE6Y5NTqcjbc4FQ0PtzUTdzUXtzUXtzUXtzUXtzUXt4/N6LTmdvdeSvLehUe++0jis/U89slwHHlGesJ3T6ZDT6dA777ytP/zhAU2atN+wjN9/buOJZQ0tW0kYAiUSCAR0/fXXa/ny5UM6zu+31dLSua/Djwlud27anAuGhtqbibqbi9qbi9qbi9qbi9qbi9rHZ9u2/P6AJCkQsGXbw9t/IDDQ/2D8/oD8/oA+8YlD5fFMTOqYRJxOx7D0M9psOzpbKS8viNs+YQjk8XjU0NAQfOz1euXxeIKPOzo69M477+iss86SJDU2Nuqb3/ymbr31Vs2cOXPIJwAAAAAAAMa2KYeXacrhZSmdQ05OTkrHH48ShkAzZ85UfX29tm3bJo/Ho1WrVunGG28M7i8oKNCLL74YfLx48WJdccUVBEAAAAAAAABjSMIQyOVyadmyZVq6dKn8fr8WLFig6dOna8WKFZoxY4bmzJkzGvMEAAAAAADAPrBse7hX8SWnp8efNussWTNqLmpvJupuLmpvLmpvLmpvLmpvLmofX0PDVlVUVKZ6GiNivN4TKFZNBrsn0Pi79TUAAAAAAACGjBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAASUnRbYURw97UghAIAAAAAAAk5HJlqqOjlSBoDLBtWx0drXK5Mod0XMKPiAcAAAAAACguLldzc6Pa21tSPZVhZ1nWuAu3XK5MFReXD+2YEZoLAAAAAABII06nS2VlE1M9jRHhdueqpaUz1dMYcSwHAwAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAQiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAyQVAi0fv16zZs3TzU1NVq5cmXU/rvuuksnn3yy5s+fryVLlujDDz8c9okCAAAAAABg7yUMgfx+v66++mrdfvvtWrVqlZ544glt3rw5rM0nPvEJPfTQQ3r88cc1b9483XDDDSM2YQAAAAAAAAxdwhCorq5OlZWVmjx5sjIzM1VbW6s1a9aEtTn66KOVk5MjSTrssMPU0NAwMrMFAAAAAADAXkkYAnm9XlVUVAQfezweeb3euO0ffPBBHX/88cMzOwAAAAAAAAwL13B29uijj+r111/X/fffn7Ct02nJ7c4dzuFTxul0pM25YGiovZmou7movbmovbmovbmovbmovZlMqXvCEMjj8YQt7/J6vfJ4PFHtnnvuOd122226//77lZmZmXBgv99WS0vnEKc7NrnduWlzLhgaam8m6m4uam8uam8uam8uam8uam+mdKp7eXlB3H0Jl4PNnDlT9fX12rZtm7q7u7Vq1SpVV1eHtXnzzTe1bNky3XrrrSotLd33GQMAAAAAAGBYJbwSyOVyadmyZVq6dKn8fr8WLFig6dOna8WKFZoxY4bmzJmjn//85+rs7NSll14qSZo4caJuu+22EZ88AAAAAAAAkmPZtm2nYuCeHn/aXGqVTpeNYWiovZmou7movbmovbmovbmovbmovZnSqe77tBwMAAAAAAAA4x8hEAAAAAAAgAGG9SPiTXTr/74nb0ePDijKVlV5nqrK8jSxMEuWZaV6agAAAAAAAEGEQPvIlvTahx9r1esNwW15mU5NK8vT9PK83q9lvV8Lsnm6AQAAAABAapBK7KP/d+yB+sEph+oDb6ve/ahTmxvbtbmp9+vTbzeqrWtHsK2nICs8GCrP05TiHLmcrMoDAAAAAAAjixBomORnufSpSYX61KTC4DbbtuVt69KWpk5tamzX5qYObWnq1PP1zfIHej+UzeWwdGBprqrKepeS9S8pK8/PZEkZAAAAAAAYNoRAI8iyLFUUZquiMFufn1oS3N7jD2jrrt3a1NSuzY2d2tzUrle2teipt3YG2xRmu6KCoWllecrNdKbiVAAAAAAAwDhHCJQCGU5Hb7BTnid9YmB7654ebW7qCAZDmxs79cQbXnX2+INt9ivKDi4p6w+IJrtz5HRw1RAAAAAAAIiPEGgMKczO0BH7u3XE/u7gtoBta0frHm1u7AgLiNZv+Uh9K8qU5XJoamlu+M2oy/NUkpuZmhMBAAAAAABjDiHQGOewLO1XlKP9inJ0QlVZcPueHr/qd3Vqc1OHNjV2aEtTh557b5eeeMMbbFOSmxEVDB1YkqvsDJaUAQAAAABgGkKgcSo7w6lDPAU6xFMQtn1XZ3fwqqEtfQHRQ6/tUJcvIElyWNJkd07wPkP9S8omFWXLwY2oAQAAAABIW4RAaaYkN1OfqczUZyqLg9v8AVsftOwOhkKbmzr0fzvbtfadJvWtKFNOhmPgPkMhN6MuyslIzYkAAAAAAIBhRQhkAKfDUmVJripLclV9UHlw++4ev94NCYY2N3Vo3aYm/eXfDcE2E/Izw5aUVZXlaUpJrjJdjlScCgAAAAAA2EuEQAbLyXDq0ImFOnRiYXCbbdtq6ujuuwl1R/Dry9ta1OPvvW7I6bBUWZwTdq+hqrI8eQqyZLGkDAAAAACAMYkQCGEsy1J5fpbK87P0uSklwe0+f0Dvt+wOBkObGjtUt71Vq99uDLbJz3Kqqiw8GJpWlqf8LH7MAAAAAABINf46R1JcToemluZpamme5oZsb+/yhd1raEtTh/761k499Jo/2GZiYVbYfYaqyvN0QHGuXA6uGgIAAAAAYLQQAmGf5Ge5NGu/Is3aryi4zbZtedu6Bu411Pf1ufd2qW9FmTKclg4syQ0LhqaX5ak0L5MlZQAAAAAAjABCIAw7y7JUUZitisJsHTetNLi92xdQ/a7OsGDoX++36Mk3dwbbFGW7ou41NLUsTzkZzlScCgAAAAAAaYMQCKMm0+XQQRPyddCE/LDtLbt7tKUvGNrUt6TssdcbtLsnIEmyJO3vzo6619D+7hw5WVIGAAAAAEBSCIGQcu6cDB052a0jJ7uD2wK2re0f7wkLhjY1dmj9lo8U6FtSluVyaGppbtSVQ8W5mak5EQAAAAAAxjBCIIxJDsvS/u4c7e/O0RemlwW37+nx692PwpeU/WPLLj32ujfYpjQvU1Vluaoqy1dVea6qyvJ0YGmeslyOVJwKAAAAAABjAiEQxpXsDKc+WVGgT1YUhG3/qKM7LBja3NihB1/bri5f75IypyVNLs4JCYZ6v04szJaDG1EDAAAAAAxACIS0UJqXqdK8TH22sji4zRew9UHz7t5QqC8Yesvbpr+90xhsk5fp1NTSvLBgqKosT4XZGak4DQAAAAAARgwhENKWy2FpSmmuppTm6sSDy4PbO7p9ereps/deQ333HFrzTpMeqWsItpmQn6np5flh9xqqLMlRhpMlZQAAAACA8YkQCMbJy3Rp5qRCzZxUGNxm27Ya27vDgqEtTR16cWuzfH13onY5LE0pydW0slxNL89XVVmejpjmULZty2JJGQAAAABgjCMEAiRZlqUJBVmaUJClzx9YEtze4w9oa/PusHsNvfphq1a/PbCkrCDLpaq+q4WqynJVVZ6vaWW5ysvk5QUAAAAAGDv4KxUYRIbT0Rfu5IVtb93Toy1Nndre0aO695u1ualDT77pVUe3P9hmUlF277F9AdH0sjztX5wjl4OrhgAAAAAAo48QCNgLhdkZOnz/Is1256r24N6PsLdtWztau7SpsXcpWf+VQ/989yP5e1eUKdNpaWppnqaFBEPTyvNUmpvBkjIAAAAAwIgiBAKGiWVZmlSUrUlF2TqhqjS4vcsXUP1Hndrc1BEMiF6ob9aqN7zBNsU5GZpW3hsK9V89NLU0V9kZzlScCgAAAAAgDRECASMsy+XQwZ58HezJD9ve0tnTGwyF3Iz6kbod2uMLSJIsSZOLc4LL0fqXle3nzpaDq4YAAAAAAENECASkiDs3Q58+wK1PH+AObvMHbH348Z6+pWTt2tzUewXRuk1N6ltRppwMh6aW5oXcjLr3e3dORkrOAwAAAAAwPhACAWOI02HpgOIcHVCco+rpZcHtu3v8evejzoFgqLFdf9/UpEf/3RBsU5aXOXAT6vI8TSvL04Elucp0OVJxKgAAAACAMYYQCBgHcjKcOrSiQIdWFAS32batjzq6w+41tLmpU/+98UP19N2J2mlJB5Tk9t5rqC8Yml6ep4qCLG5EDQAAAACGIQQCxinLslSWn6Wy/CwdPaUkuN0XsLWtebc2NbZrS19A9PqOVj39f43BNnmZzrD7DPV/n5/FrwQAAAAASFf8xQekGZfD0oGluTqwNDdse3uXT1uaOoLB0JamDq1+e6ce6vIH21QUZEUFQ5XFOXI5WVIGAAAAAOMdIRBgiPwsl2btV6RZ+xUFt9m2LW9bV9+NqDt6vzZ16Pn6ZvkDvUvKMpyWppTkht1rqKosT+X5mSwpAwAAAIBxhBAIMJhlWaoozFZFYbaOnVoa3N7jD6h+V2dYOPTKthY99dbOYJuibFfwHkP9X6eW5ik305mKUwEAAAAAJEAIBCBKhtOh6eX5ml6eL31iYPvHu3u05aOQq4YaO/T461519gwsKduvKDssGJpWlqfJ7hw5HVw1BAAAAACpRAgEIGlFORk6Yn+3jtjfHdwWsG1t/3hP2L2GNjd1aP2Wj9S3okxZLoemluYG7zPUHxCV5Gam5kQAAAAAwECEQAD2icOytL87R/u7c3RCVVlw+54ev+p3dWpTyFVD/3xvlx5/wxtsU5KbEf4pZeV5OrAkV9kZLCkDAAAAgOFGCARgRGRnOHWIp0CHeArCtu/q7A5bTra5qUMPvbZDXb6AJMlhSZPdOWE3oa4qz9Okomw5uBE1AAAAAOw1QiAAo6okN1OfqczUZyqLg9v8AVsftOwOC4be3tmuv73TFGyTm+HUtLLcsHsNVZXlqSgnIxWnAQAAAADjDiEQgJRzOixVluSqsiRXcw4qD27v7Pbr3Y/CP75+3aYm/eXfDcE2E/Izoz6lbEpJrjKcjlScCgAAAACMWYRAAMas3EynZkws1IyJhcFttm2rqaM7eBPq/nsOvbytRT3+3jtROx2WppTk9C4lC7nnkKcgSxZLygAAAAAYihAIwLhiWZbK87NUnp+lYw4sCW73+QPa2rw7LBh67cNWrX67MdgmP8sZFQxNK8tTfha/CgEAAACkP/7yAZAWXE6HpvWFOnMPGdjetscX/Nj6/nsOPfXWTnW85g+2mVSY1XuPoZBPKTugOFcuB1cNAQAAAEgfhEAA0lpBtkuH7V+kw/YvCm6zbVsNbV1hn1K2qalDz723S30rypTptDSlJDfsXkNVZXkqzctM0ZkAAAAAwL4hBAJgHMuyNLEwWxMLs3XctNLg9m5fQO/t6uy9cqgvGHrp/RatenNnsE1RtkvTPQVy2LYcliXL6r0HkSXJYVlyOCw5rL7vrd6xnH1fB7YPfD9wfN82R/Rxzr52UcdZ1kC/DksOKWL8WHMI2e+QHAqZg9X7OPY5xJlD37n3Hh9+XOS5OkKeB+7NBAAAAIw+QiAA6JPpcujgCfk6eEJ+2PaW3T1h9xra0dalPT0BBWxbAbv3yiJ/wJYt9W4L9H61pd7tti1/X7uArfDjbFu2rYjj+/an4kkYJb2hWX9oFRokRYRbicI0hyKOj3/c4EGWJacjsv/o43KyM9TT7Qvf33+cYodwjrBx+7fFOIdBzz10f7wQL3YIF3Vc6FeH1Rf8JZ4DAAAAxj9CIABIwJ2ToSMnu3XkZHfvY3euWlo6R3xcOyQsCg2Per/GCJRiBFH9bWxbfYFTgiDKlgKKdXysIKvv+4CtgPrnEN5nMnMYCL3sgTnEOS5umBbz3Hv3+wN27HEjziEyhIv1HPcHe7H2BdI4tesP7XpDr9Ar36JDuOiwLn4IF7zyLU4IFz/I6u93sHEHCeH6r5xLIkjs/5qbm6Xdu7vVH4dZffPuf4JibQ/NzsK2R+2L3m6FdGApvP/+FqFtB74fmExkf6M534E+Es0r/vaQUw3p0wrrP+Z8Q9oPOt++xonmq8xute7pGZhXxLiR2yPHjtV/3HEJXAEAI4wQCADGqP4/bMP/pEAqDRYA9odEsUKkuAFYXxAVP4QLDevihHCRYVqsfYOFaWEhXnQIN/gcwkO48ONjBY6Jg8RAwJYviXOIPvdYcxj83IGxbG/Do6gwbR9Cq3jbI/9faV/CrqRDwdD5JDHf6D5C/h0VhMaeb4bLKZ/PHx1whh0b/kRGzidmyBpjnoOdQ3h/4b1EnkuoQfsY5JxC5x0ZGsc7p/4+BjunyDlFHx/eb3gfg+2LU8NBzilyf+TYOdkZ2r2nJ2atkn2eY+6LahO/jqHnFNlHMufUvz96vuHH7tXPX4L/SBHrmOReLyFzG8prKolz6j+vwc5pquVQjtJfUiHQ+vXrde211yoQCGjRokU6//zzw/Z3d3friiuu0BtvvCG3262bb75Z+++//4hMGACAsaj/jYXDaSVsi7Eh2SDLtm3lF+bo4493S+p93M+W1P/Qlq2QXSHbB46xg//q2973wA5dAmrH3x4cJ2SMgf2xt4fNN3R73PnaYXNPPG74fCOfF0X0N6rzjTNu5BwHm292ToZ27+6JP69BtodmjUOd78D+oc03dJyRnm9UH4PON3zcmOcxTPONbBvvNTXofAO2fH3/RD7HA4fZMbbFn3usfaHnMDC/6NdT6DcxzzFOv9F92EnNt397wjZxahx7DuHnNdi+8NomPqd48wWGItPl0D8vPTbV0xhxCUMgv9+vq6++WnfddZc8Ho8WLlyo6upqVVVVBds88MADKiws1DPPPKNVq1bpF7/4hX75y1+O5LzHjL9ufVJbX39PXd0+9V+63JucWmFJ5ECa2p9cDuwdSERDtlmheweOCdtmhR0dsS20z4hMNGTbwH+JsUL+p9Aew+Y6MPeIlqHnbMXoMXKbFfps9M89vNfw5zE08e3vI2xWMbbFmKsV60xDegh7PuJsC0mx8zty1NHeFTavgXH7+w5/3gbOLbKWitg2MIfw5yM0RR94jsIT8Mht4RUJ2xbxvCv0OeOy9Jj2ZPSoq9OX6mkgBah9+ur/be2I3NL3pcjplOV0pmh2SKWiooEAEGaJrP2gb4sirziI3hD/0ERvtyIaWHEfJDp08DlFdRV2pcX4fk84aJAUY19RyJW/kcF/+PHRwVb/46TDvJht4s9338K2/scRIWRo/3sTQsY4p6g+EoR5g4amI/Y8h7epmuSWCRKGQHV1daqsrNTkyZMlSbW1tVqzZk1YCLR27VpddNFFkqR58+bp6quvlm3b4/6XRTIaHnaqeNdnUj2NIbNlSkI+0mfaOoJ9AwAAAGOfHfV+O+SxFXdP3C0Dx8YOPJJpGz3KYOMMPqewRwnGGXTcBONE7Q1rn2jciP1W3D2Dth303GMZ5PmIes4HjQcSjDuE590etG38fZsKfTr7P+cnPc54lTAE8nq9qqioCD72eDyqq6uLajNx4sTeDl0uFRQUqLm5WSUlJXH7dTotud25ezvvMePLX5+t5h2dsgOBER0nLAUNSTzD08vQfw8cGHY5cEgoEn25cEhLO6JtaKprR45hR7QamGdErwN9hibHij4+bPxYKW7Y+dsRYwQ7iNgTp0875JjIM7ZDj4we36GBT4EKO94OP6NYl2YH+xzsuYg5fmQSbod/H/ULtH9/9HNkhw8eNkZon73jRc8yej4xxo94LsLGtyOejxi1Hnz8yMuzY4xlD7Ttn2fsGUUsMwidRUSNYDBLke/IYAjLUtR/gYQJ7L7XfarngdFmy5YlK+Q95yBX40QdPIT/nxhKvwn7HqSv6L+oE4wT2jL5tgn7jmg7aN9D6Df6b/6hzDnW8xbnhT/U9wAh7Ye1tgl/Jw2aAEW0TL7tsNU2qv3gba0htB3s5yZRBpeZnx4ZRSIpuzG032+PyqfrjLTc8kxNmu5Oi3PB0I3Wp0RhbKHu5qL25qL25qL25qL25qL2ZkqnupeXF8Td54i7p4/H41FDQ0PwsdfrlcfjiWqzY8cOSZLP51NbW5uKi4v3dr4AAAAAAAAYZglDoJkzZ6q+vl7btm1Td3e3Vq1aperq6rA21dXVeuSRRyRJq1ev1tFHH23E/YAAAAAAAADGi4TLwVwul5YtW6alS5fK7/drwYIFmj59ulasWKEZM2Zozpw5WrhwoS6//HLV1NSoqKhIN99882jMHQAAAAAAAEmy7MjPmBslPT3+tFlvl05rBzE01N5M1N1c1N5c1N5c1N5c1N5c1N5M6VT3fbonEAAAAAAAAMY/QiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgQAAAAAAAAxACAQAAAAAAGAAy7ZtO9WTAAAAAAAAwMjiSiAAAAAAAAADEAIBAAAAAAAYgBAIAAAAAADAAIRAAAAAAAAABiAEAgAAAAAAMAAhEAAAAAAAgAEIgYZg/fr1mjdvnmpqarRy5cqo/d3d3frWt76lmpoaLVq0SB988EEKZonhlqjuDz/8sI4++middtppOu200/TAAw+kYJYYCVdeeaU+97nP6ZRTTom537ZtXXPNNaqpqdH8+fP1xhtvjPIMMRIS1f3FF1/UkUceGXzN//rXvx7lGWKk7NixQ4sXL9bJJ5+s2tpa3XPPPVFteN2np2Rqz2s/PXV1dWnhwoU69dRTVVtbq1tuuSWqDe/x008ydec9fnrz+/06/fTTdcEFF0TtS/vXvI2k+Hw+e86cOfb7779vd3V12fPnz7c3bdoU1ub++++3f/zjH9u2bdtPPPGEfemll6ZgphhOydT9oYcesq+66qoUzRAj6aWXXrJff/11u7a2Nub+v//97/a5555rBwIBe+PGjfbChQtHeYYYCYnq/sILL9jnn3/+KM8Ko8Hr9dqvv/66bdu23dbWZs+dOzfqdz6v+/SUTO157aenQCBgt7e327Zt293d3fbChQvtjRs3hrXhPX76SabuvMdPb3feead92WWXxfy9nu6vea4ESlJdXZ0qKys1efJkZWZmqra2VmvWrAlrs3btWn3pS1+SJM2bN0/PP/+8bNtOxXQxTJKpO9LXUUcdpaKiorj716xZo9NPP12WZemwww5Ta2urdu7cOYozxEhIVHekrwkTJujQQw+VJOXn52vq1Knyer1hbXjdp6dkao/0ZFmW8vLyJEk+n08+n0+WZYW14T1++kmm7khfDQ0N+vvf/66FCxfG3J/ur3lCoCR5vV5VVFQEH3s8nqg3B16vVxMnTpQkuVwuFRQUqLm5eVTnieGVTN0l6emnn9b8+fN1ySWXaMeOHaM5RaRQ5M9HRUUFfzQY4tVXX9Wpp56qpUuXatOmTameDkbABx98oLfeekuzZs0K287rPv3Fq73Eaz9d+f1+nXbaaTrmmGN0zDHHxHzd8x4//SSqu8R7/HR13XXX6fLLL5fDETsOSffXPCEQsI9mz56ttWvX6vHHH9cxxxyj733ve6meEoARdOihh2rt2rV67LHHtHjxYl144YWpnhKGWUdHhy655BL94Ac/UH5+fqqng1E0WO157acvp9OpRx99VM8++6zq6ur0zjvvpHpKGAWJ6s57/PS0bt06lZSUaMaMGameSsoQAiXJ4/GooaEh+Njr9crj8US16U+IfT6f2traVFxcPKrzxPBKpu7FxcXKzMyUJC1atIibhBok8uejoaEh6ucD6Sc/Pz94CfkJJ5wgn8+nXbt2pXhWGC49PT265JJLNH/+fM2dOzdqP6/79JWo9rz2019hYaE++9nP6h//+EfYdt7jp7d4dec9fnrasGGD1q5dq+rqal122WV64YUX9N3vfjesTbq/5gmBkjRz5kzV19dr27Zt6u7u1qpVq1RdXR3Wprq6Wo888ogkafXq1Tr66KNZWzrOJVP30HtBrF27VtOmTRvtaSJFqqur9Ze//EW2bevVV19VQUGBJkyYkOppYYQ1NjYG14XX1dUpEAik1RsDk9m2rR/+8IeaOnWqzj777JhteN2np2Rqz2s/Pe3atUutra2SpD179ui5557T1KlTw9rwHj/9JFN33uOnp+985ztav3691q5dq5tuuklHH320fvGLX4S1SffXvCvVExgvXC6Xli1bpqVLl8rv92vBggWaPn26VqxYoRkzZmjOnDlauHChLr/8ctXU1KioqEg333xzqqeNfZRM3e+77z6tXbtWTqdTRUVFWr58eaqnjWFy2WWX6aWXXlJzc7OOP/54XXzxxfL5fJKkM888UyeccIKeffZZ1dTUKCcnR9ddd12KZ4zhkKjuq1ev1p/+9Cc5nU5lZ2frpptuSqs3BiZ75ZVX9Oijj+qggw7SaaedJqn352H79u2SeN2ns2Rqz2s/Pe3cuVPf//735ff7Zdu2vvjFL2r27Nm8x09zydSd9/hmMek1b9npdJtrAAAAAAAAxMRyMAAAAAAAAAMQAgEAAAAAABiAEAgAAAAAAMAAhEAAAAAAAAAGIAQCAAAAAAAwACEQAAAAAACAAQiBAAAAAAAADEAIBAAAAAAAYID/D2tLjmCya3YfAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "fig, ax = plt.subplots(figsize=(20, 6))\n", "pd.DataFrame(history.history).plot(ax=ax)\n", "del history" ] }, { "cell_type": "markdown", "id": "546982b5", "metadata": { "id": "546982b5", "papermill": { "duration": 0.023925, "end_time": "2022-04-19T17:19:30.077703", "exception": false, "start_time": "2022-04-19T17:19:30.053778", "status": "completed" }, "tags": [] }, "source": [ "# Evaluation" ] }, { "cell_type": "markdown", "id": "a8492d32", "metadata": { "id": "a8492d32", "papermill": { "duration": 0.023713, "end_time": "2022-04-19T17:19:30.125341", "exception": false, "start_time": "2022-04-19T17:19:30.101628", "status": "completed" }, "tags": [] }, "source": [ "**Lets see our model in action**" ] }, { "cell_type": "code", "execution_count": null, "id": "c2b8eaa5", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:19:30.181573Z", "iopub.status.busy": "2022-04-19T17:19:30.180896Z", "iopub.status.idle": "2022-04-19T17:19:30.183571Z", "shell.execute_reply": "2022-04-19T17:19:30.183144Z", "shell.execute_reply.started": "2022-04-19T16:42:23.208867Z" }, "id": "c2b8eaa5", "papermill": { "duration": 0.034088, "end_time": "2022-04-19T17:19:30.183677", "exception": false, "start_time": "2022-04-19T17:19:30.149589", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# this function converts the token ids to token text\n", "def word_for_id(integer, tokenizer):\n", " for word, index in tokenizer.word_index.items():\n", " if index == integer:\n", " return word\n", " return \"\"\n", "\n", "# this function is used to generate predictions\n", "# target sequence is intialised as the start token\n", "# then predictions are taken from the model iteratively \n", "def predict_sequence(source):\n", " target_seq = \"@\"\n", " for i in range(14):\n", " tok_target_seq = preprocess_input(target_seq, machine_tokenizer, config.Y_LEN).reshape(1, -1)\n", " output_tokens = transformer.predict([source, tok_target_seq])\n", " token_index = np.argmax(output_tokens[0, i, :])\n", " token = word_for_id(token_index, machine_tokenizer)\n", " target_seq += token\n", " \n", " return target_seq\n", "\n", "# this function decodes the complete sequence of token ids into text\n", "def decode_sequence(tokenizer, source):\n", " target = list()\n", " for i in source:\n", " word = word_for_id(i, tokenizer)\n", " if word is None:\n", " break\n", " target.append(word)\n", " return ''.join(target)" ] }, { "cell_type": "code", "execution_count": null, "id": "321eea53", "metadata": { "execution": { "iopub.execute_input": "2022-04-19T17:19:30.236734Z", "iopub.status.busy": "2022-04-19T17:19:30.235849Z", "iopub.status.idle": "2022-04-19T17:19:31.166237Z", "shell.execute_reply": "2022-04-19T17:19:31.167189Z", "shell.execute_reply.started": "2022-04-19T16:42:23.210716Z" }, "id": "321eea53", "outputId": "a9d774bf-8f97-45c7-aa37-71218514def2", "papermill": { "duration": 0.95973, "end_time": "2022-04-19T17:19:31.167348", "exception": false, "start_time": "2022-04-19T17:19:30.207618", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "SOURCE : saturday 19 september 1998\n", "PREDICTION : @1989-09-19#\n" ] } ], "source": [ "query_text = 'saturday 19 september 1998'\n", "query = np.array(list(map(lambda x: preprocess_input(x, human_tokenizer, config.X_LEN), [query_text])))\n", "print('SOURCE :', query_text)\n", "print('PREDICTION :', predict_sequence(query))" ] }, { "cell_type": "code", "execution_count": null, "id": "e0d15d13", "metadata": { "id": "e0d15d13", "papermill": { "duration": 0.024883, "end_time": "2022-04-19T17:19:31.216970", "exception": false, "start_time": "2022-04-19T17:19:31.192087", "status": "completed" }, "tags": [] }, "outputs": [], "source": [] } ], "metadata": { "colab": { "name": "Attention-Is-All-You-Need.ipynb", "provenance": [] }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" }, "papermill": { "default_parameters": {}, "duration": 603.501486, "end_time": "2022-04-19T17:19:34.167951", "environment_variables": {}, "exception": null, "input_path": "__notebook__.ipynb", "output_path": "__notebook__.ipynb", "parameters": {}, "start_time": "2022-04-19T17:09:30.666465", "version": "2.3.3" } }, "nbformat": 4, "nbformat_minor": 5 }