{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "VhQSb7PdZznG" }, "source": [ "# Sequence-to-sequence activity recognition" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "n2y0GYTdc-nY", "outputId": "8a4c97ee-752b-4ef3-a83d-e6da46c5f019" }, "outputs": [], "source": [ "!pip3 install wandb" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "gSxOaWIFSBM-", "outputId": "475bc447-6414-46af-c5ec-49526e2808f8" }, "outputs": [], "source": [ "!pip3 install git+https://github.com/tensorflow/addons.git" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VSncNSHtZznI" }, "outputs": [], "source": [ "from tensorflow.keras.layers import Add, Dense, Dropout, MultiHeadAttention, LayerNormalization, Layer, Normalization\n", "from tensorflow.keras.optimizers import Adam\n", "from tensorflow.keras import Model\n", "from tensorflow.keras.initializers import TruncatedNormal\n", "from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, Callback\n", "from tensorflow_addons.optimizers import AdamW\n", "from wandb.keras import WandbCallback\n", "from sklearn.model_selection import train_test_split \n", "\n", "import math\n", "import wandb\n", "import numpy as np\n", "import pandas as pd\n", "import tensorflow as tf\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n" ] }, { "cell_type": "markdown", "metadata": { "id": "_kdFgpMxZznJ" }, "source": [ "## Init logger" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Z6DnpqLPZznK", "outputId": "078f5861-e753-4525-fe92-0516ac23f007" }, "outputs": [], "source": [ "wandb.login()\n", "\n", "sweep_config = {\n", " 'method': 'grid',\n", " 'metric': {\n", " 'goal': 'maximize',\n", " 'name': 'val_accuracy'\n", " },\n", " 'parameters': {\n", " 'epochs': {\n", " 'value': 50\n", " },\n", " 'num_layers': {\n", " 'value': 3\n", " },\n", " 'embed_layer_size': {\n", " 'value': 128\n", " },\n", " 'fc_layer_size': {\n", " 'value': 256\n", " },\n", " 'num_heads': {\n", " 'value': 6\n", " },\n", " 'dropout': {\n", " 'value': 0.1\n", " },\n", " 'attention_dropout': {\n", " 'value': 0.1\n", " },\n", " 'optimizer': {\n", " 'value': 'adam'\n", " },\n", " 'amsgrad': {\n", " 'value': False\n", " },\n", " 'label_smoothing': {\n", " 'value': 0.1\n", " },\n", " 'learning_rate': {\n", " 'value': 1e-3\n", " },\n", " #'weight_decay': {\n", " # 'values': [2.5e-4, 1e-4, 5e-5, 1e-5]\n", " #},\n", " 'warmup_steps': {\n", " 'value': 10\n", " },\n", " 'batch_size': {\n", " 'value': 64\n", " },\n", " 'global_clipnorm': {\n", " 'value': 3.0\n", " },\n", " }\n", "}\n", "\n", "sweep_id = wandb.sweep(sweep_config, project=\"HAR-Transformer\")\n" ] }, { "cell_type": "markdown", "metadata": { "id": "-mGp0L3_ZznL" }, "source": [ "## Layer" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0lFGhNtyZznL" }, "outputs": [], "source": [ "class PositionalEmbedding(Layer):\n", " def __init__(self, units, dropout_rate, **kwargs):\n", " super(PositionalEmbedding, self).__init__(**kwargs)\n", "\n", " self.units = units\n", "\n", " self.projection = Dense(units, kernel_initializer=TruncatedNormal(stddev=0.02))\n", "\n", " self.dropout = Dropout(rate=dropout_rate)\n", "\n", " def build(self, input_shape):\n", " super(PositionalEmbedding, self).build(input_shape)\n", "\n", " self.position = self.add_weight(\n", " name=\"position\",\n", " shape=(1, input_shape[1], self.units),\n", " initializer=TruncatedNormal(stddev=0.02),\n", " trainable=True,\n", " )\n", "\n", " def call(self, inputs, training):\n", " x = self.projection(inputs)\n", " x = x + self.position\n", "\n", " return self.dropout(x, training=training)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PIwd6GlIZznM" }, "outputs": [], "source": [ "class Encoder(Layer):\n", " def __init__(\n", " self, embed_dim, mlp_dim, num_heads, dropout_rate, attention_dropout_rate, **kwargs\n", " ):\n", " super(Encoder, self).__init__(**kwargs)\n", "\n", " self.mha = MultiHeadAttention(\n", " num_heads=num_heads,\n", " key_dim=embed_dim,\n", " dropout=attention_dropout_rate,\n", " kernel_initializer=TruncatedNormal(stddev=0.02),\n", " )\n", "\n", " self.dense_0 = Dense(\n", " units=mlp_dim,\n", " activation=\"gelu\",\n", " kernel_initializer=TruncatedNormal(stddev=0.02),\n", " )\n", " self.dense_1 = Dense(\n", " units=embed_dim, kernel_initializer=TruncatedNormal(stddev=0.02)\n", " )\n", "\n", " self.dropout_0 = Dropout(rate=dropout_rate)\n", " self.dropout_1 = Dropout(rate=dropout_rate)\n", "\n", " self.norm_0 = LayerNormalization(epsilon=1e-5)\n", " self.norm_1 = LayerNormalization(epsilon=1e-5)\n", "\n", " self.add_0 = Add()\n", " self.add_1 = Add()\n", "\n", " def call(self, inputs, training):\n", " # Attention block\n", " x = self.norm_0(inputs)\n", " x = self.mha(\n", " query=x,\n", " value=x,\n", " key=x,\n", " training=training,\n", " )\n", " x = self.dropout_0(x, training=training)\n", " x = self.add_0([x, inputs])\n", "\n", " # MLP block\n", " y = self.norm_1(x)\n", " y = self.dense_0(y)\n", " y = self.dense_1(y)\n", " y = self.dropout_1(y, training=training)\n", "\n", " return self.add_1([x, y])\n" ] }, { "cell_type": "markdown", "metadata": { "id": "YRQTRP60ZznN" }, "source": [ "## Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "UYEKK7pYZznN" }, "outputs": [], "source": [ "class Transformer(Model):\n", " def __init__(\n", " self,\n", " num_layers,\n", " embed_dim,\n", " mlp_dim,\n", " num_heads,\n", " num_classes,\n", " dropout_rate,\n", " attention_dropout_rate,\n", " **kwargs\n", " ):\n", " super(Transformer, self).__init__(**kwargs)\n", "\n", " # Input (normalization of RAW measurements)\n", " self.input_norm = Normalization()\n", "\n", " # Input\n", " self.pos_embs = PositionalEmbedding(embed_dim, dropout_rate)\n", "\n", " # Encoder\n", " self.e_layers = [\n", " Encoder(embed_dim, mlp_dim, num_heads, dropout_rate, attention_dropout_rate)\n", " for _ in range(num_layers)\n", " ]\n", "\n", " # Output\n", " self.norm = LayerNormalization(epsilon=1e-5)\n", " self.final_layer = Dense(num_classes, kernel_initializer=\"zeros\")\n", "\n", " def call(self, inputs, training):\n", " x = self.input_norm(inputs)\n", " x = self.pos_embs(x, training=training)\n", "\n", " for layer in self.e_layers:\n", " x = layer(x, training=training)\n", "\n", " x = self.norm(x)\n", " x = self.final_layer(x)\n", "\n", " return x\n" ] }, { "cell_type": "markdown", "metadata": { "id": "j42cze_qiAIb" }, "source": [ "## Loss" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "NK6QapYViAIb" }, "outputs": [], "source": [ "def smoothed_sparse_categorical_crossentropy(label_smoothing: float = 0.0):\n", " def loss_fn(y_true, y_pred):\n", " num_classes = tf.shape(y_pred)[-1]\n", " y_true = tf.one_hot(y_true, num_classes)\n", "\n", " loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits=True, label_smoothing=label_smoothing)\n", " return tf.reduce_mean(loss)\n", "\n", " return loss_fn" ] }, { "cell_type": "markdown", "metadata": { "id": "PxmZ1ZWBAgLX" }, "source": [ "## LR scheduler" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "GEtbF3TdAjDU" }, "outputs": [], "source": [ "def cosine_schedule(base_lr, total_steps, warmup_steps):\n", " def step_fn(epoch):\n", " lr = base_lr\n", " epoch += 1\n", "\n", " progress = (epoch - warmup_steps) / float(total_steps - warmup_steps)\n", " progress = tf.clip_by_value(progress, 0.0, 1.0)\n", " \n", " lr = lr * 0.5 * (1.0 + tf.cos(math.pi * progress))\n", "\n", " if warmup_steps:\n", " lr = lr * tf.minimum(1.0, epoch / warmup_steps)\n", "\n", " return lr\n", "\n", " return step_fn\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "MBlu9AxBHG09" }, "outputs": [], "source": [ "class PrintLR(Callback):\n", " def on_epoch_end(self, epoch, logs=None):\n", " wandb.log({\"lr\": self.model.optimizer.lr.numpy()}, commit=False)" ] }, { "cell_type": "markdown", "metadata": { "id": "7dIynjZAZznP" }, "source": [ "## Dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "4GkimkgOZznP", "outputId": "c43e079b-f8b2-4d51-f9b0-a5339a3c9b77" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(60060, 300, 6) (60060, 300)\n", "(12470, 300, 6) (12470, 300)\n", "(10599, 300, 6) (10599, 300)\n" ] } ], "source": [ "CLASS_LABELS = np.array(\n", " [\n", " \"Stand\",\n", " \"Sit\",\n", " \"Talk-sit\",\n", " \"Talk-stand\",\n", " \"Stand-sit\",\n", " \"Lay\",\n", " \"Lay-stand\",\n", " \"Pick\",\n", " \"Jump\",\n", " \"Push-up\",\n", " \"Sit-up\",\n", " \"Walk\",\n", " \"Walk-backward\",\n", " \"Walk-circle\",\n", " \"Run\",\n", " \"Stair-up\",\n", " \"Stair-down\",\n", " \"Table-tennis\"\n", " ]\n", ")\n", "\n", "# load dataset\n", "f = np.load('./new_dataset.npz')\n", "signals = f['signals']\n", "labels = f['labels']\n", "\n", "# split to train-test\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " signals, labels, test_size=0.15, random_state=9, stratify=labels\n", ")\n", "X_train, X_val, y_train, y_val = train_test_split(\n", " X_train, y_train, test_size=0.15, random_state=9, stratify=y_train\n", ")\n", "print(X_train.shape, y_train.shape)\n", "print(X_test.shape, y_test.shape)\n", "print(X_val.shape, y_val.shape)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 739 }, "id": "RSXjG7qHZznQ", "outputId": "c83fae89-3e09-4f05-eb7f-6c4b6ab76db4" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAApUAAALSCAYAAACMDCpzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdfbxtVV0v/s+XA3q8HqAEFfSIBz2CCMhBCPMBFZ8FJfMhExMrLMwwCyswTf2Z1yiNuMY1SEDMRMGHS3hBSa+QWYIiED4gSHqEgxAKQpAiT+P3x1wbN3jg7MNYa+192O/367Vfe6+15l5jrLnWmvMzxxhzzGqtBQAAemw03xUAAGDDJ1QCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAui2oUFlV76mq1VXVqmrVHP/n56vqQ1V1cVV9vaoOm3Q9AQC4owUVKpN8LMmTk3x3Pf7nuCTntda2a63tmOSIidQMAIC7tKBCZWvt8621NXe+v6p+oao+V1XnVNV5VfXS0f0rk+ye5PBZz3Hl9GoMAECSbDzfFViXqvq5JH+XZO/W2hVVtWWSc6vq35I8JsmaJH9bVbsnuTrJIa218+avxgAAi8+CD5VJnpjkEUk+VVWz798+Q/33SPInrbUDq+p5Sf5vVa1ord08/aoCACxOG0KorCRfb6098WceGFonL2+tnZEkrbVPVdV9kjw8ySXTrSYAwOK1oMZU3oV/S7JtVT1z5o6qWjUKj19J8l9V9djR/XtkCKGXzUtNAQAWqWqtzXcdbldVRyfZJ8lWGcZHXt9aW1lVj0vy7iRbJNkkyaVJXthau7Gqdkvy3iT3S/KTJH/YWvvneXkBAACL1IIKlQAAbJg2hO5vAAAWOKESAIBuC+bs7/ve977tgQ984HxXAwCAu3D55Zff1Fq779oeWzCh8oEPfGDWrPmZi+kAALBAVNX37+ox3d8AAHQTKgEA6CZUAgDQbcGMqQQAWJvbbrst5tWejqrKRhvdszZHoRIAWJBuuummXHrppbn55pvnuyqLyiabbJJtttkm97nPfdbr/4RKAGBBuvTSS7Pppptmiy22SFXNd3UWhdZarr766lx66aVZuXLlev2vUAkALDi33XZbbr755myxxRbZeGNxZZq22GKLXHPNNbntttvWqyvciToAwIIzM4ZSC+X0zazz9R3HOufoX1X/lGSrJLcluT7J77XWzlvLcgckOTRDYP1ckte21gyGAAC6rDj01Ik87+rD9pnzstdff3223nrrvOxlL8uxxx6bJDn++ONz8skn5+STT86ZZ56Z3//938/5558/kbouZOvTUvkrrbXHttZWJTk8yfF3XqCqtk3yZ0n2TLIyyYOT/PYY6gkAMO9OPPHE7LbbbvnEJz6RG264Yb6rs6DMOVS21q6ddXPzJGtrE31JklNaa1e2oc30qCQv76siAMDCcOyxx+aQQw7JU57ylJx44onzXZ0FZb1GvlbV3yfZa3Rz77Ussk2S7866vXp0HwDABu0b3/hGLrvssjznOc/JLbfcksMOOywHHHDAfFdrwVivE3Vaa/u31h6W5M1J/qKn4Ko6uKrWzPxoQgYAFrJjjz02+++/f5YsWZK999473/nOd3LhhRfOd7UWjHt09ndr7QNJ9qqqLe700KVJHj7r9orRfWt7jsNba8tnfpYtW3ZPqgIAMHE333xzPvjBD+YDH/hAVqxYkZUrV+ZHP/rR7SfrMMdQWVU/V1UPmXX7hUmuTnLNnRb9eJJ9q2qrGs5Hf02Sj4yrsgAA8+GUU07JIx7xiFx++eVZvXp1Vq9enbPOOisf/OAHXfFnZK5jKjdP8tGqul+GKYW+n+T5rbVWVcdkODnnlNbat6vqrUn+dfR/ZyY5etyVBgCYpmOPPTaveMUr7nDfDjvskIc+9KG5/vrr56lWC0stlAu0L1++vK1Zs2a+qwEALAC33nprLr744my33XZZsmTJfFdnUbm7dV9Vl7fWlq/t/1xRBwCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoNtcJz8HAJhfb9t8Qs973ToXWbVqVZLkpptuykUXXZSdd945SbL99tvnxBNPnFMxp5xySs4444z89V//9T2v6wImVAIArMP555+fJFm9enVWrVp1++3Zbrnllmy88V1Hq3333Tf77rvvxOo433R/AwDcQytWrMghhxySPfbYI6961aty5ZVXZq+99spuu+2WHXfcMQcddFBuu+22JMnxxx+fF77whUmSM888MzvttFNe+9rXZpdddsmOO+6Yc845Zz5fSrdF21K54tBTJ17G6sP2mXgZAMD8uvrqq3P22WenqnLjjTfmk5/8ZJYtW5Zbb701v/RLv5STTjopv/qrv/oz//fNb34zxx57bN773vfmqKOOypve9Kacfvrp8/AKxmPRhkrmj0APwL3Jr//6r6eqkiS33XZbDjnkkHzhC19Iay1XXXVVdtppp7WGypUrV+bxj398kuQJT3hC3v3ud0+13uMmVAIAdFi2bNntfx9++OG56qqrcvbZZ2fp0qU5+OCDc+ONN671/5YuXXr730uWLMktt9wy8bpOkjGVAABj8sMf/jBbbbVVli5dmiuvvDIf/ehH57tKU6OlEgBgTF7/+tfnJS95SXbcccc85CEPyTOf+cz5rtLUVGttvuuQJFm+fHlbs2bN1Mozrm/+WPcArMutt96aiy++ONttt12WLFky39VZVO5u3VfV5a215Wv7P93fAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG7mqQQANgg7f2DniTzvV1/11XUus2rVqiTJTTfdlIsuuig77zzUZfvtt8+JJ54457LOPPPM3HjjjXnuc597zyq7gAmVAADrcP755ydJVq9enVWrVt1+e32deeaZufbaa++VoVL3NwDAPXT66afnyU9+cnbbbbfsscceOeOMM5Ik3/rWt/KkJz0pu+yyS3beeee8+c1vzvnnn5+jjjoqH/rQh7Jq1aq8/e1vn+faj5eWSgCAe+Db3/523va2t+X000/PZpttlksuuSR77rlnVq9enSOPPDLPf/7z88Y3vjFJcs011+QBD3hAXvOa1+Taa6/NEUccMc+1Hz+hEgDgHvj0pz+dSy65JE95ylNuv2+jjTbKpZdemqc85Sn5oz/6o9xwww156lOfuiiuAS5UAgDcA621POtZz8oJJ5zwM4896lGPyhOf+MR85jOfyZFHHpkjjjgip5122jzUcnqMqQQAuAee85zn5LOf/WwuuOCC2+/70pe+lGQYU/ngBz84+++/f/7yL/8yZ511VpJks802y3XXXTcv9Z00oRIA4B5YuXJlTjjhhBx44IHZZZddssMOO9w+VvJjH/tYdt555+y666552ctelqOOOipJ8su//Ms5//zz75Un6lRrbb7rkCRZvnx5W7NmzdTKW3HoqRMvY/Vh+0y8jA2RdQ/Autx66625+OKLs91222XJkiXzXZ1F5e7WfVVd3lpbvrb/01IJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQzWUaAYANwoWP3mEiz7vDNy9c5zJ777139t577xx00EF3uH+XXXbJW9/61rzoRS/6mf85/vjjc/LJJ+fkk0/OOeeck3e961058cQTf2a5G264IZtuumnWNXf4tddem6OOOiqHHnro7fe9+tWvzite8Yrstdde63wNk6alEgBgHQ444IC8//3vv8N955xzTq644oq84AUvWOf/77777msNlOvj2muvzWGHHXaH+4455pgFESgToRIAYJ323XffXHbZZXe4zvdxxx2XfffdN89+9rOz2267Zccdd8xBBx2U22677Wf+/8wzz8yqVatuv3300UfnUY96VHbdddf89V//9R2WfcUrXpHdd989j33sY7PPPvvkyiuvTJK85jWvyfXXX59Vq1Zl9913T5I87WlPy8knn5wkueqqq/KiF70oO++8c3baaaccffTRtz/nihUr8pa3vCVPeMITsu222+Yd73jH+FbOiFAJALAOm2yySV75ylfmuOOOS5LceOON+fCHP5w3vOEN+eQnP5mvfOUrueCCC7J69eqcdNJJd/tcX/va1/LWt741n//853Peeeflxz/+8R0eP+KII3LOOefkggsuyJ577pm3ve1tSZKjjjoqm266ac4///ycc845P/O8r3vd67L99tvnq1/9aj73uc/lHe94R84666zbH7/22mvzxS9+MV/+8pfzrne9K5dffnnnWrkjoRIAYA4OOOCAfOhDH8pNN92UT3ziE9lhhx3y8Ic/PIccckh22WWX7LrrrjnnnHNy/vnn3+3zfO5zn8vznve8bL311kmS3/md37nD4yeccEJ233337LTTTjnmmGPW+XwzPvvZz+bAAw9MkjzoQQ/Ki170onz2s5+9/fH99tsvSbLlllvmEY94RL7zne/M+bXPhVAJADAHj3nMY7Jy5cp88pOfzHHHHZcDDjgghx9+eK666qqcffbZueCCC7LffvvlxhtvXK/nrarb//7CF76Q97znPTnttNPyta99LYcffvh6P9/anjdJli5devvfS5YsyS233HKPnveuCJUAAHN0wAEH5J3vfGe+9KUv5WUve1l++MMfZquttsrSpUtz5ZVX5qMf/eg6n+PpT396Pv3pT98+VvKoo466/bEf/vCH2XTTTbPFFlvkpptuusO4yM022yw//vGPc9NNN631eZ/5zGfmfe97X5Lk+9//fj7xiU/kWc96Vs/LXS9CJQDAHL3sZS/LRRddlJe+9KVZtmxZXv/61+fss8/OjjvumFe+8pV55jOfuc7n2GmnnfK2t70te+65Z3bdddfc9773vf2x5z73udl+++2z/fbbZ88997zDyT0PeMADsv/+++exj33s7SfqzPae97wnF154YXbeeefstddeedOb3pTHP/7x43nhc1DrmhNpWpYvX97WrFkztfJWHHrqxMtYfdg+Ey9jQ2TdA7Aut956ay6++OJst912WbJkyXxXZ1G5u3VfVZe31pav7f+0VAIA0E2oBACgm1AJAEA3oRIAWHBmpsNZKOd+LCYz6/zOUxKty8aTqAwAQI+NNtoom2yySa6++upsscUW6x1wuGdaa7n66quzySabZKON1q/tUagEABakbbbZJpdeemmuueaa+a7KorLJJptkm222We//EyoBgAXpPve5T1auXJnbbrtNN/iUVNV6t1DOECoBgAXtnoYcpsu7BABAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdNp7vCgCLw4pDT51KOasP22cq5QBwR1oqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3eYUKqtqaVWdXFUXV9W/V9VnqmrlWpZbUVW3VtX5s34eOf5qAwCwkGy8Hsv+XZJPtdZaVR2U5JgkT1vLcte31laNo3IAAGwY5tRS2Vq7sbV2Wmutje46K8mKidUKAIANyj0dU/n6JP94F4/dv6q+XFXnVtVbqmrJPSwDAIANxHqHyqr6kyQrk7xxLQ9fkeShrbVfSPLMJHsmecNdPM/BVbVm5ueGG25Y36oAALBArFeorKo/TPKiJM9rrf3ozo+31n7SWrtq9Pc1SY7LECx/Rmvt8Nba8pmfZcuWrX/tAQBYEOYcKqvq4CQvT/Ks1tq1d7HMg6pqk9Hf980QQM8bR0UBAFi45jql0PIkf5Xk55KcMZoq6OzRY2+vqteMFn1ykvOq6t+TnJvkyiT/c/zVBgBgIZnTlEKttTVJ6i4ee8usvz+R5BPjqRoAABsKV9QBAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQbeP5rsB8Wb10vymUct0UytjwWPcAcO+jpRIAgG5CJQAA3RZt9zcA07Hi0FMnXsbqw/aZeBnA3dNSCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6DanUFlVS6vq5Kq6uKr+vao+U1Ur72LZ51fVN6vqW1X1iarabLxVBgBgoVmflsq/S7J9a22XJP+Y5Jg7L1BVy5Icm+SFrbVHJflekj8dR0UBAFi45hQqW2s3ttZOa6210V1nJVmxlkWfl+S81to3R7ffm+Tl3bUEAGBBu6djKl+fobXyzrZJ8t1Zt1cn2bqqNr6H5QAAsAFY77BXVX+SZGWSZ/QUXFUHJzl45vbmm2/e83QAAMyj9WqprKo/TPKiJM9rrf1oLYtcmuThs26vSHJFa+2WOy/YWju8tbZ85mfZsmXrUxUAABaQOYfKUcviy5M8q7V27V0s9ukkj6uqR49uvzbJR/qqCADAQjen7u+qWp7kr5J8O8kZVZUkP2mtPb6q3p7ke621o1pr11fVq5OcPBpH+bUkr5pQ3QEAWCDmFCpba2uS1F089pY73T4lySn9VQMAYEPhijoAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0G3j+a4AsDisXrrflEq6bkrlADCblkoAALoJlQAAdNP9DSwKKw49deJlrD5sn4mXARsS37vFRUslAADdhEoAALoJlQAAdBMqAQDo5kQdAOBeZxonCSVOFJpNSyUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQbU6hsqreU1Wrq6pV1aq7WOZpVfXjqjp/1s/9xltdAAAWoo3nuNzHkvxlki+sY7mLWmtrDZ0ALE6rl+43hVKum0IZwN2ZU6hsrX0+SapqsrUBAGCDNO4xlY+sqnOr6stV9doxPzcAAAvUXLu/5+LcJMtba9dV1fIkp1XVD1prJ61t4ao6OMnBM7c333zzMVYFAIBpGltLZWvtv1pr143+XpPkw0n2vJvlD2+tLZ/5WbZs2biqAgDAlI0tVFbV1lW10ejvTZM8P8l543p+AAAWrrlOKXR0Va1JsjzJ6VV1yej+Y6pq39FiL07y1ar69yRnJflMkvdPoM4AACwwcz37+8C7uP/Vs/4+MsmRY6oXAAAbEFfUAQCgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdNp7vCgAAk7Hi0FMnXsbqw/aZeBlsGLRUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN02nu8KAEzD6qX7TaGU66ZQBmw4fO8WFy2VAAB0EyoBAOgmVAIA0M2YSgDgXmc64zkTYzp/SkslAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOi28XxXAADg3mbFoadOvIzVh+0z8TLWh5ZKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbs7/nwTTOCEsW3llhAMC9l5ZKAAC6CZUAAHQTKgEA6CZUAgDQzYk6i9BivHQUADBZWioBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQzWUaAeBeavXS/aZQynVTKIMNwZxaKqvqPVW1uqpaVa26m+UOqKpvVdV/VNX7qmqT8VUVAICFaq7d3x9L8uQk372rBapq2yR/lmTPJCuTPDjJb/dWEACAhW9OobK19vnW2pp1LPaSJKe01q5srbUkRyV5eW8FAQBY+MZ5os42uWNL5urRfWtVVQdX1ZqZnxtuuGGMVQEAYJrm7ezv1trhrbXlMz/Lli2br6oAANBpnKHy0iQPn3V7xeg+AADu5cYZKj+eZN+q2qqqKslrknxkjM8PAMACNdcphY6uqjVJlic5vaouGd1/TFXtmySttW8neWuSf01ySZLvJzl6IrUGAGBBmdPk5621A+/i/lff6fb7krxvDPUCAGAD4jKNAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3YRKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB023i+KwCLyYpDT514GasP22fiZQDAnWmpBACgm1AJAEA3oRIAgG5CJQAA3ZyoAwAwZquX7jeFUq6bQhlzp6USAIBuQiUAAN2ESgAAugmVAAB0c6LOPJjO4N1koQ3gBQDuvbRUAgDQTagEAKCb7u9FaDHOnQUwH1YceupUyll92D5TKQfujpZKAAC6CZUAAHQTKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2ESgAAugmVAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3Tae7wrANK049NSplLP6sH2mUg4ALBRaKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdhEoAALqZpxKmaPXS/aZQynVTKAMA7khLJQAA3bRUAtzLuZIUMA1aKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbs7+BuBebRpnvzvzHbRUAgAwBloqAWBCpnMVrcSVtFgItFQCANBNqAQAoJtQCQBAN6ESAIBuQiUAAN2c/Q2LiPn6AJgULZUAAHSbc6isqkdV1b9V1cVV9eWq2nEtyzytqn5cVefP+rnfeKsMAMBCsz7d30cn+bvW2vFV9ZIkxyf5hbUsd1FrbdU4KgcAwIZhTi2VVfWgJLsn+YfRXR9P8rCqWjmpigEAsOGYa/f3w5Jc0Vq7JUlaay3JpUm2Wcuyj6yqc0dd5K8dUz0BAFjAxn3297lJlrfWrquq5UlOq6oftNZOuvOCVXVwkoNnbm+++eZjrgoAANMy15bKy5JsXVUbJ0lVVYZWyktnL9Ra+6/W2nWjv9ck+XCSPdf2hK21w1try2d+li1bdk9fAwAA82xOobK1dlWGVshfG9314iRrWmuXzF6uqrauqo1Gf2+a5PlJzhtfdQEAWIjWZ57KA5McWFUXJzk0yW8kSVUdU1X7jpZ5cZKvVtW/JzkryWeSvH+M9QUAYAGa85jK1tpFSZ6wlvtfPevvI5McOZ6qAQCwoXCZRhaV1Uv3m1JJ102pHABYGFymEQCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBt0U4ptPO220y8jK9OvIQNk3UP02UqLWAatFQCANBNqAQAoJtQCQBAN6ESAIBui/ZEHQAWh+mcqOQkJdBSCQBAN6ESAIBuQiUAAN2MqQSmYhqT3icmvgeYL1oqAQDopqUSFhFnwQIwKVoqAQDotmhbKk/681smX8irJl/Ehsi6B4B7Hy2VAAB0EyoBAOgmVAIA0E2oBACgm1AJAEA3oRIAgG5CJQAA3RbtPJXAdE1lftLEHKUA80RLJQAA3YRKAAC66f4GYKJ23nabiZfx1YmXAKyLlkoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAN1MKAYuCaW1g+nzvFhehEgC415lGoE2E2tl0fwMA0E1LJQATddKf3zL5Ql41+SKAu6elEgCAbkIlAADdhEoAALoJlQAAdHOiDgAwEU7SWlyESmBRsHODxWUq3/nE934W3d8AAHQTKgEA6CZUAgDQzZhKALiXmsb1r137mhlaKgEA6CZUAgDQTagEAKCbUAkAQDehEgCAbkIlAADdTCkEAPdSLk/KNGmpBACgm5ZKAIAxW4wTzwuV82AaH7Tkrj9si/GDDgBMlu5vAAC6CZUAAHQTKgEA6GZMJcAUrDj01FhtiXQAACAASURBVImXsfqwfSZeBjA3i3E6J6FyHkzlg5bc5YdtMX7QAYDJ0v0NAEA3LZUwRaZzAuDeSkslAADdtFQCwITM98UuYJq0VAIA0E1LJUyRM+8BuLcSKgGmYPXS/aZQynVTKANg7YRKFhXjm4Bpmu95iWGahEqAezkHU8A0OFEHAIBuWipZVHRFweLjogMwHUIlLCJ2rouTgylgGoRKAO7VTOUF0yFUwiJi5wrApDhRBwCAbkIlAADdhEoAALoJlQAAdBMqAQDoJlQCANBNqAQAoJtQCQBAtzlPfl5Vj0rygSRbJrkuya+31r6+luUOSHJohsD6uSSvba3dPJ7qAmyYXCITuLdbnyvqHJ3k71prx1fVS5Icn+QXZi9QVdsm+bMkj0vyn0n+MclvJ/nfY6ktwAbK1YyAe7s5dX9X1YOS7J7kH0Z3fTzJw6pq5Z0WfUmSU1prV7bWWpKjkrx8XJUFAGBhmuuYyocluaK1dkuSjALjpUnu3J+zTZLvzrq9ei3LAABwL7M+3d9jVVUHJzl41l23VtWV81WfOViW5Ib1+o+qxVv+Yn7t813+Yn7t813+Yn7t813+Yn7t813+Yn7t813+eMueqwfe1QNzDZWXJdm6qjZurd1SVZWhBfLSOy13aZJHzrq9Yi3LJElaa4cnOXyO5c+7qlrTWluu/MVV9mIvfzG/9vkufzG/9vkufzG/9vkufzG/9oVQfq85dX+31q5Kcm6SXxvd9eIka1prl9xp0Y8n2beqthoFz9ck+ci4KgsAwMK0PvNUHpjkwKq6OMOUQb+RJFV1TFXtmySttW8neWuSf01ySZLvZzhrHACAe7E5j6lsrV2U5Alruf/Vd7r9viTv66/agjPfXfWLufzF/Nrnu/zF/Nrnu/zF/Nrnu/zF/Nrnu/zF/NoXQvldajiRGwAA7jmXaQQAoJtQCQBAN6ESYB1Gs1nAHVSVfSjM4gvBnMzsVKtqp9E13uezLo+tqudU1X3msx4sHs3gc8F6LVprtyXDurF+JmPWvmejxb6ON4SDmAVfwYWiqpbOY9k/P19lz7LF6Pd7kmya3OHLft8p1+WpSQ5K8t6qOrCqHjPl8lNVD5tyeQvuu7oQ6zQOsz7X96mqbatq36p6aVU9aPbj82k+drAzwXpa73tVbTKNcu6JqnpcVX2uqn67qrZqI6PHpvLeLITP4TTMrNfW2m0O7vKrVbV5klTVkvmuzNrcK3cK4zYKdR+oqj+uqj2mVOaS0e9tkrx2GmXeTV22TPKqqjomydZJrq6q/zHrC75/VW02xSr9nyRHJrkxyZuSHFZVf1tVL55kK+rMTm60Pg4d/T2R79Cs93/r5KctIvOtql5QVbsl02mlmWmNnvIOdOY9/b0kJ2S4iMNzk7y9qrZdCDu2Se9gZwXrpVX11Ko6qaoOGb3+iX0WZ5W7fZLjJlXOGPwoyelJ9kpyclX979H2536TeG9mf/6raoeqetg0P4fTDrCztn+7VNVBVXVqVR03s+2Zr0BdVZtPu4ds1rr41ST7t9auq6pHJnljVb1hmnWZC6FybjZP8pUMgeqtVXXU6Aj1URMsc7uq2jXDZPJbzH6gqn5uyi1lP0zyb0l2TnL/JIcl+b2q+sWq2i/J61pr/zWNilTVRq21NUnuk2G9HJrkQ0luTfLuJH8xwRaOx1TVb2SY0P+W5A7Bauuqety4Cmqt3Tr688TRxnSLUTnzfXT67CT/q6o+VlX7V9VDZrfSjNMosL81uUMr2cRf/6x1/5okv5LkVUnem2TLJAdPu9di1k7liVX11qq6sKo+VVVPm2CxM/uGNyR5Y5JvZXjvz62qt0yq0Fmfo5uSbFpVW84cuFXVQ+ajV2JtWmvfbK39RYbLEN+QYR+xf5JPVtWRVfXwMZc38/n/qySvT/L1qnpMVW1SVXOeb/ruzFrP9x+t6ydU1YrZ5U/LrO/gkUl2SfLRJNcnObKqnjSt+sxaJ4+uqqMyzCH5J1X1q1W13TTqkGTmtb4yyeFV9eAkhyR5UpIdq+rxU6rHnAiVc9BaW91a+8sMwWXjJA9NsluGN/i9VfWkCRS7bZJ3Jdkvw4HZk0ZHJ0nyxxk+UBM3+lI9oLX2xQw7l+cm+WyS3ZP8RYZLdh41jbrcyW8mOaG1dkJr7cQkf5rkC0lOb63dPKEyK8n2SV6Q5CFV9ftV9czRY7+VYZ2M2ysyBNj9kztsbKduFG7ekuRtSX6S5M8yhN7jq+p5VXX/MRf5mCS7VdUDZ+5ord1aVTuNuZzbzdqJ7JDknNbaZa2177fWvpLhPXhShgOaqZn1nv+vDNuf1yb5epJjq+qXJ1zmE5K8pbX2ptbaM5Lsk+SxVTX2axOPGrwfXlVLW2vfSfKfSZ7QWrutql6X5O+T/NK4y11fsz4je2ao3zOT/H6SdybZJMkjM/SiTKK8HZJ8IMlFrbVvJFme8R/o/FWSTyb5oyR/WFUHV9U+NaXeqFmt1Tsl+Ulr7bcyvOY/THJKkt+qqvtNoy6z/HGG7/2FSZYmeVaSN1fVr0+64NHnf2abszrJnyc5r7X2vCRbJVkIw+NuN5YjnHuzqqrWWquqByR5dmvtsVW1aZLtMrSi7JDk2nGX21o7raq+l+GL9IAMrYOXVtUFGVpOfubqRhPyggw7kaOTLEvy7dHG7ANVtTJD0L50SnWZ+YJtnOSaJI+vqjOS/Li19sNRqFkziXKrarPW2vlVdWOSJUm+k+EIeveqemWGjczY35PW2mVV9b4MBzBPT3JAa+2qmc/luMtbh9tG6/nZGTZun8jwPrw6ybFJ/inJr4+rsNba16rq6iTPS/L3VfXE0fPfkOTgcZVz52JHv1+S5Beq6p1JjsnwOp+W5Puttf+qqiXTCPiztj+PTnJFa+1PRw+dUVVnJ3llVX2mtXbDBMpcmeS2JFfMPNZa+7eq+ocM24Jxe26GIQenVdW/JTkzyVuq6tAk387wnl84gXLXy6zu/72S/Pvovh8k+UFVHZFkj9baf46zyNHvl2c4sHhIki+N7tspydNHjR59hQzb1i2TPDnD9uzRSR6X5BFJ9kzyH0mm0SNVGV7zbkmur2FIwY+T3FxVn0uyz+j2xI3WSSV5+OigKqNA+9gkz0hy0STLr6r7ttZ+0lq7qapOTXJehlbb91XVVkke3Fr79CTrsL6EynWYteN+RoYj57TWrk/ylap6bZL3tNa+Ps4yq+rNGbrbvpvkt1prPx61Uv5SkgclObC1NvEgN9q5/GNV/VOSvTOMcbpkFOQ+nuRL89Fy1lq7par+JkN39x8k+VpV/UKSbVtrp0+o2N+oqg9n2Ngd1Vr7j1E3xG5Jfi7J8a21746zwKr6+dbaD5NcnuRXM5wk9Y6qesPoMzhVs74LeyfZvbV24yjgr8lovYzqfY8D16hL6eQMgfUjSf4kySGj93fnJKdl2KhOxKzX+I0kn07ylAwHkMuTXJLhPUiGsDUNMzvYpyW5fw0nhVw5euzSJA8ZZ6BM7rAOds0QLs6sqsOSnJFhXfxza+2bVbVxa+2WMRZ9fpJ/TvL0DAezN49+PpHk6NbaDTOtWAvER5IcU8MQoH9N8r0MPTdfutv/ugdGr/vbSVZkaFQ4aPTQyzKG70MNw4puS7JHkjNaa1dkOJg4o4Zx3bu01r7ZW85czArtOyd5fJKPVNVpGbaDz0ly/KjOEz2wm7VOdk5yY1X9VpKPt9auSXL26GfSfrOq/i7Jlq2191bVsUk2Gu0D/yjJOVOow3pxmcY5qqqHJvmbDDu1kzIMHfj9DEcKvzPGcjZK8muttb8ftUR8ZVTev06wW3fOqmrfJL+WYYdzS5KXtta+Nk91+cUMXYFbJvlykn9prX12AuVsnOQlrbWPVNUHk/x3hhaKc5J8bRJHzVW1KsP7/oMM4eHJGca1bZHkm0l+o7X23+Mudw71emCS92dolfz71tq1o/svyNBi8oMxPP8vZehm3iVDS/huSd7bWjvo7v53Ekb12TvJMzN0a16YITQcN83vY1W9McnvJLkqw7q/JMmqJP82+lyOfQc7CjK75Kev/0kZgsZxrbW3j7ms+yS5deY1jA4injH6+UmGA5c/H/eB2/qaCRpV9dQk98vwmXhehvflkRl6rfabxHezhnH078rQO/ZnGbbBeyV51rjKq6r/meSFSf4uyaeSfGe+9jujg/bHZHiN22X4LHwhw4ma/6+1dvmU6vGqDMObfpLkcxm2wxcn+fok100Ns028OMkHMzTinJHkXzL0En0/Q2NGa619f1J1uCeEyvUwOiJ9Q4ausB9l2Kj8/qg7eFxlPDtDN9dXq+qXkuyb4Uv1nQwfqlNaaxeMq7x11GXJaAzb05Msb639/azH7pvhiPkDrbWfTKEuM11y98+woXlohi/Xv2f4HE+l5aiGkyOePKrDTCvdBRnWw9i+TKOW6Y0zjJf5boad1pZJrstwcHNJG04UmLqqen6SAzO05N03w5H8z7fWXjjr6L7n+WdOxtkpw5CCX0yyY4aegv/ZhvG9YzfrM7bPqNyLkpzfWvvq6PHtM3zm79da+4NJ1OFu6rYkQyvV0zKEuxdkaE09Lck/jqsVaVZo2iFDt+dpo3Vynwzr5KkZukK3TfK0Npw0N45yX5thqM/7M7yeC2bqk6F16vlJ/qC1dtM4yrunZn1GTs4Qrk+pqt0zbA8uSvLN1tp1YyzvtRm2c2e31q6u4WTA12UYgvODJB9qw3jfcZS1cZLfyBBWH5xhm/PNDN3ekxyrPrsOM+v3hRkO3s8eDTfZJkOr5Ux3/OrW2iFTqM8mGRqQlmYYT7xnhpOyliR5Y2vt21Oow/2T/G6Gg+utM4Tac5N8trU20e73e0KovBuzx62NNuq3ZRiI/dTR319srf1ozGWen2H85EmjjftDM+xUn57hRJDzWmuvGWeZd1OXmR3Mp5J8sLV2QlUdlOHL/fHW2smz19GE6zITcP80w3i3K5JcmSFkXZLkrNbalydU9saj7oZnZPSe1zBX2A4ZTqS6tLX27kmUfad6zLwfH0hyVWvtjyZd5t3U5XlJXpqhJfHiDAc7F/WEylmv76FJdm6tfXq0Ud8yw47kl5P8bWvtP8b0MmaXPbMz2yZDULtgVG4y7FjPa629f9zlzrFOB2boAv9UG8bYzgT5xyd5YoYxxa8ec9m/liG8/DBDy+xJMz0SNUzb9YjW2v8bY3lLMvSAvDTDwcTFSf4xw+fqsnGVMw6j9X9Eko+Ncx2spZwtkvxlhoPLG5N8LUOQuHCcLdOzvncPSnL1aDu7S4bGjCcl+V5r7fXjKGuO9VmSYZjJ5hkacC7M0KDy7Qz73yckuW5S2/s71WXvDCfpHNxaO3d03+4Zxs2+d8Jl/8y2tIax1ftm+J78zeyGnoVCqJyDqnp3hlaCKzJ8wP8lyWWttWvHGapGR6FHtNaeMvpivSDDPHlnZWjufnWb8li6Ufffma21HavqJRm6Ab6YoeXo90Zjb6ZZny9l6J65IUOL4a6jnw+01j45wXK3SPJ/k+zdhpNVXp7kltbaR+unA8l7y5gJzk/IcFT86xlaA09orX1u1nJbJfnvaX0WZtXroRm6oVZmGE/01TG3zs6U85cZWgNfN+ux7hbQOdbhtRkC0x+OWsmemqEL6sGttZdOqx6z6rNRhulDts/QM3JZhtB7bobvwHZJbmqtXTLmcrfIMI50pwzB4gkZDuB+c9KtMzWM4XtVkhdlCBbfSfKqNt6TX+6xUSva/8rwXrwjw0HtFRPq8n54hm71PTIcWC3L0IL41QwH0mMbz1/DCYHXZpg+7pzRAcz9Moznm0qwn3Ug9aAMrZK7ZmihW5Hkn1prfzKNesyqz/0y9Mo8OcMBzlRC3Kyg/+AMB1tbZOh6P3tmu19V95nvlvu1ESrvwqw3da8M41j+JsNZd4/IMLXAdUkOHWdLZVUdnuTa1trbR93gv5lhTOXho/Kvaa29eVzlzbFOOyX5/zLMBfnKDCdOfC/DYO6xzcu4jjrMbGi2zdCK+6rW2o2zHn9UkjXjCHZrKXsm6Lw+yYrW2h+M/v6VDBvgd7bW/nVMZc28zjMyTOvxuxlayx6doQvmgNbaR2YvO45y16Ne/5hksww7+fsl+XGG1oOPjDPUjFrrn99aW1Ojsx9rGOv1T621fx5XOXdR9soMXa3/0GaND53ZgE8zVM5a75tn2O5sn+HEoacl+Upr7ZWTKO9O9903w1CTQzOE2D+c5Ouvqs1ndx/XMLb4V6YdJu5ODdP3PD/DvJ2PznACyTeSfHjc4X5U3tattStG27nHZegh2SXDMIHjx1TGkgzbtF/MEOB+kqGb9V8ydH1Pc6L1TdqsrvYahmIcnqFl+Nga/wlidy5/5nt33ySbtda+X1UvyHAg8aUMJ4h+ZZLrZNZ+5/gk/2P0s0WGnoOvZNjmjvUE4XFx9vddm/nAPD3J4a21E5LbB0vvmWTZOAPlyLlJnjFqJXhThjMKPzL6cH0vP+2Om6ZvZehmfkeSN4+6Xt6YYcD0tD0nw1QOx1XV+5Nc3Fr7bmvtW5MqcFYX015JPjQKlA/N0GL74gw7l7GEytGG7GEZWun+b1W9tbX2C6MW4l/L8F5M/KzHu6jXVkm2aK09edR6sl2GHeoeGbqkxmLUGvq9DGOW0n46XnefJH87rnLuVObMAeQjM3zOn5PkAVV1boZ1flkbTe4/zVbK0XpfOgpZ5yU5r6o+n+EyqaeN6j6JE3T+OMN7cGobZh/4dg0nDW49Wk9jPaCZtRN/dZIXjj5fM2MWz89wVviCMTqg/ViSj40+ry/I0B05tlkJZq2T7ZIcUVUvHW3nvlXD+NZtM7RYjsXoM/ThJB8ebYNemWFmjZvbFKesGR1A/U4NJ/5dmOS7o33OtRlaZ5NhyM0kbTQq4/cyzPpxeYbP47EZTpDarLW29yQrMNrnV5IdWmuPT5IapjLcN8O+5ysZ5qpdcITKuzD6QleGM6yeW1XnZZhw9rIMXdKT8JkMX+ZzM5yc8Rezjtqfm59OIzE1o536787crqqfy7DTndrloWbtwM7JMNXS7kkOSPK90Rf+H6bQNfa+UZkPyTAM4RtV9b8znFwwTntkmFB8VX46/+nXkvxnGw3In2agnNUy9/iM5ghswxm4362qf84wzm9srTOttctH37V31zBH5MYZusB+0MZ0UshazExTc0iGYSavy9Dl+6oMrbGfzjDx9lSNvmuHjVpuz0pyQWvtezVMQr16tNjYQu5MiM3QIrJ7kl+pqm9m2IEdmOFAN8nt0xyNs9ydRmW8JsNZx8/OcBm6S5I8qbV29bjKuydmtRw9KsM2erMMw2H+ubV2VMZ/AYiZYHNAhq7o/57V3bldkvu3MZ6kUcPlD9dkmIf1siTvrGHOyrEcMK+H7TIMtdgpw3fxe6N67NRa+1Iy+av7zNq+tgzb4P/IcCW5b2S4GMVEZ92YdaC4c5Iv1nBlvQtG3d4fGv0sWLq/78boKPEvkjwwQ0vBRRk+WN9srX1vguU+MMN4vR+ONvIvTvL61tpUrjt+p7ps0lq7eTQM4MFtmL7k59poKpn5MFone2SY5mS7DNdDnejYklHrwNMzTP5+cVU9J8mbWmtPGXM5W2UYXrFRhiD79xlOxti4tfZb026lnFWvt2eY5PxTGeYN/HJrbSwtJaOQ9MjW2nmj21tmaBF4WIaN+XcyzFM40Xnhqur/ZJgO5sej29tl+O6d21o7fZpd36Pyt88QtO6f4TNxY4azUFe11nadYLlbJnl4hh37rhm6Wr84iS7oqnp4a+27NYxb/3aG6bP2aa39TlUdmSRtHqaSuitV9ZUMB7ZHZggXP0xyaoYGgLGPL6+q05O8rbX2xar6H204SfCvM7TgHTGmMp6WYVjRF0Y/38kwq8P7kjy+zRpqNA01jGN8YoYD2W0znKxzVmvt/0x7+zc62Hl6hs/laZPez9yp7PdkOMg6M0OX+/kZhsBNrOt/HITKdRiFiV0zBJjtMxzFn9imN2D350dl39gmeCLK3ZQ/c4R+UpKPttYmNvH0XZQ/0w20c4au5isytNydN6rXsjbmyZ/XUZ9HZtj5PSnJ0jaBeTFH5WyU4Qoir8twFuw72nAlnakGm1l1eUqGEwZ2zdB6/98ZWjbe3TrHstZwqcuWofXt+Rm6XS+pqkdkmLtwYnMTzvp87Zhh3NbXMlzb/ZJpr+e1GW1/dsww1OBRGVqvzm6tfXbSO9gapjK5td1x/PI4T0zcJEML3/syzIX5rgxn/X6mDTNNvDvJ51trp4yjvHtq1vCIPTNcuvBFGcaU71nDvLVPTfKLk2hoqKrfTbJ9a+33ZuqSoYFj397vxah18skZxoTOTHz+ixm61W/NcCb4VAP97M/0KFwumfL2fea9fn2GA/tzM/QSvjJDuD20tXbaFOvzuAzTPD07w7XPv5xhKqN5a9RZF93fd2MU6P47Q6vM2aMWlWdkimMZRmOaphbkZu1k79Nau2kU3DbPcAbey6dVjxmjumyZYUf/lSRvz3DEdmUNV1n44KTKHg1/qFE1Zupxamvt0RnjmNK1BcXR7XdX1ZczjGW9atb9UzUq88wMV1bZPEPL1VMybPDHcXLURhlOPjggw7jhnarqmgzr+OvJ5E5MmvWcD81wCbrHZhjucXH9/+2dZ7hV1bWG3wECNsQawGDB2FDE3oggUkSIvRsRe8GoIWrMjTW5KYZoLNg1YuyI1wZcRLEECyqWa8WCgopGBMUWjXKRcX98c3kWBL0qa6192Ge8z3Meztn7cObce68155ijfMPsNXTv1yT0mq6L2SiX8nVUqPBVAUMZBmXu/l8C5ZH1yz9f8GfQBnXMuQUd1G5C0jH7mtl01EXqwgLH+17k7rk+wHVI0uz19NgIpIJQmEGZM2x6oJzeQ0w5rY8g7/2jBRiUm6Diz1boM3jY3U8wFUP+Eun//m1hxviezE3zOwDomjzWlR2kc+M0R2lemV7tI6gpQ5E91v+N3P23WJrPU8iwxdR4ZI/GbFBCeCr/jZxnbkfUAqs3KuV/CG2sr3oj6GxTFrkF7WSkiTbRVBnYydWLucqq43zl9QooFHwWqka/EQng9vumv7EQYy8+f9gneQ02SmHoQuQc0kHlCrQ5PeQpNzT32m9Em9Yfaxj67ocMyVlIs/GJFIbLwnHf+5pI+UJ/BNqhQ+4KKOVkZdSStA1wkpdQVbuAuRjyxG6PxKzbprGfKXvsr5lPdg0cDmzvkjQq7RqYz0s0ANjV3feswCO6CTJk5gJ3oc27HaqmrrlRmWHKc56JohRdUV75SHTQLFSzMF2Lj3lKeTJpla6HCoSe8VQ4thB//zwkgzTEJN9zMTAb7XcTkKxWl4UZ43vMaRAqzvrCpIBxjruPqkV0JjcnA1q5WtIuj/Qxy7wXMqPy5yjd6Au07w33gtKNSsfd42sBX8jNvC4NOWRTUJhgp1rPreTX3Rxt7pcg6R5oOHxsAbSowZxGIU/W+cCg9Nhg4IiSxlsO9fU9Cdg89/iA7GfkpSvq/d4TuAdVN56PKs2XQh68N4CV8p9DRe95s/Rv//T+j0Th7qxN6dFFvAdIRPrX6ft2yPO8NZLQ2AYVRZX1GrPrugdwLqowb58eWxroVdX7/XVzS9/fk607RV13XzPmfmgDBTgT6Ju/Fop+fekaXzL9vBQ6XNyGqqlLe53fcZ7ZfbBm7tpYGRWtvZnuhaUKHK95+ndd4M8lvq4ngVVyP09AnsstgGVr8P72RXrIIHWDC2vwWWfrQc/0XgxL6/FWuXmWtgbnxl8vXVsboH3uCaTG8PcqP5vv+xXh7xzz5Vd9jkIPbd19E5NW1rk00jL+IkivP/NUDAeON7PrgTZmNhiFf3qgcElVc2qJbuw3UCVuqxSCPRQYWNKwbVDIoT3wGzObhm7sOzz1WfWCTqvp72TyJMuTOiWgvKbZSDZpZg1P6/ujrh5rIy/lA8BFqHjlywI8192QmD3uPj3l8a3masWYFQ6UQm7eWdeew4APzewVVG39aFljf4e5gfL3RqXHi5YPWi592w8VvN2Yxvl1er6snOXB6PD0rqnKfTSqNN4Q5RDfaWY71eia/4rc+LsAXc3sTHd/AuhkKuJ4y4sVPc+UCM4HupvZ2+5+/ldPFhApSrmUGyPpnjtRYU4L1DWm6ihc9no7I0cOrirnY9JcK4nO5Pb+LFJyJ8qvbo8UPo5Fnt0yo3SZqsKayKh+DjkazjOpDuzmjTz0DZFTOQ+5C6YDCkluhPqrgk5PH3oFvT5ryONmNhO42BV26IkMiFXQwnOMV1QJmFtMugAvuMKso1Bv4B7AZE/VwkXj7q8DfzYVCiyG8u2aA7uZ2ZtIwqhwqQ13n4VyRy9LuU1H0mBU2df+xxJwpUC0QsVpTyD9xoPdfYpJCHhibl7fN/S9oM1tVRQZWGCuaRm4+zhgXFq4+6FNZFeU/lKqfEieXLi7F3rfmyO1iaElDrspOjh0A8YkI7OFqyisO6q+vbqEcZshQfex6HPvBXyGjMslkLRNzQulclyK1oI/mJoAXOapbWWReENl76+AI4DfpVSk64Ar3X1SAWM8mfLD90QRqSWRsP1KZvapF9i7/FvM5cuUP9gZ2NHMlkYpEI+5+ztVGJSJbB3bD5jo7qcDmDraDEEOjCFlTiB3vR8BzDCz1snAxqVR+ucyxy+KyKlcAKY2YXPQRTYU9QHeFqnoV9rRpkpM7QH3Q6dyQ8b131Fy8AcVz2UZd//YzJ5DlY5T0+OroZNc4X3X09/PTqzLo3BMF5Po7NrAGSgseqw30m4GRZJe93rIO38JuhZeRAeuLkUs+Ol93hMJDS+JKhz7AR+X5CHLxs0+57boPp+ZHSrNrCtwuLsfXNb43zCvJVFo8nmkkdcKydbMRIbMzBLG3BmFuz9AHvKnUCeV81F1/3VF51KbCoH2QZv1o2msGSj8O6dW+cPzzTEz8jsgpYdX03v1F1S48RcvS/eA+QAAHK9JREFUMN/W1FRiJDowz849vjPKOV3P3Vcoarzc318FCWrvje6/w12i86ViEo6fiSJfG6SvLYDl0f47yd3PKnseaS7ZevBrpHLxcyT8PjvlN3Z098EVzKM10qHshg5ZVwKXe3kavYUTRmUit4B0BXq4CiMM9V/eB1Uo3uQLmSDdWDGzFX3etnT9kbzLjugUe427/6miuayK2sKtRYMQcz40/zPgr97QbaWMOeyFcjb75B7rAAx1993LGrcxka6Bie7+Xno/BqN+x2+7KkUL3fhrsbmZ2V+RMfUASneZjA4Pn6c1oKqe46u4ei3vDfR096OSN2kTYHOUx3dMwYbdlii89jnyFE1Ch8r+yMCfCvyizJCfSZf1cFQUNdor7N7ybTGzccjgeQvtAx2AQahFa2FOBlNP8bEo/N8OONdzbRiLNuwXMH4zdK29XEWY1czOQN63jVFoeWpKf+mCPNdvu/tVZb/u+ebUHrgcRYjuQc6E45Aj4Ykq5pCbS3bY3gZFx8pK9yqUMCoTZrYNOh0NRhvpObnTS5ZQXriXoLFgZg+inMWbkXTJB+nxNsiwdncvMww3/3zWQJ2LVkGL+Xh0guuKemBvVvL4P0S5jVlhSjN0bbR190Fljl1LTMLySyIj5lrPiWyb2VaoGvG55E0qZbEve3NLYd3uKJc10+pbD3gXGbPdkHf8pao2NFM17j9pCMOdnjdmzWx5d59V5HzM7FLgZNQ56B0k7vxxNh7wqasSt2gv5anoOhqPFAXaIaOtMyqIHIqaP9R0c8qt/yeiXM/n0KHjdZSD+La7v13CuOsg0f39UE7f34Gr3P2/qzSwysSUK78N8ojfhu69V1CqzdPu/r6lHt8VGNO7IeWBJ1wdvTI90hVRN51HvODq/m+Yy4/RQXKWu1+fHmuN8swLT7cogzAqE8movByFVoehPKLXUm7RCOSpG13LOZZFymnJhG97I4PieWRMjarVImZmPwHuRdV4ByGP5bNIdqJ0QWQz+ymSDZmFQhFLAIO9gLymxopJ4mcAqsCegUR/WySDphUy6CtZYMsiXVd/QKHlv6OQ43TkITGk2Te5QoOyJRI47ozyd9dH6gMTUXHUlKLnYSqO+VcyGi9BRjboEDUKpfoUnk9qZqsDpyG5qK2Q5uwMtJGuk35tg1qnl8z/2afUoD7IoBzp7h8WbOBnkbLOqNf8R+nxzVGO3bru3q2IsRobZrYF2n82RSoAH6N78KoKxm6OUi+WQaH4SagwbopJ7u0zL7mDTe6z3xPYDXlH33f3HdL98k6ZUbmiCaOSrzaZp1GoNesg0BudUh5F4biNvYQcvsZEMhraoM1tB7Torwjc4O6/r2gO2Q3WA1jO3W/LPdcCeQpLyS/JbxJpsZmLPBLbpu9LyeNsTKTX3QfljX2BjJpnkVbricA/3H1wVWHhMjF1RzoUSdj8Lwo9jnP3+2s8p17o3muN8iofcfdhBY9zFdpET3G1YTV0gDgMHayfdvf+RY6ZG3uxbKM2sy7IM/4+ymVbuejXujCY2a3Ig/Y0qlY/AKUFDHT3J0sY72ngAFflb/7xmueYFsmCDPLk3NgMFck94+43VnGwS/vevsBP0/jjkITPg8CbZXzO842faUPfijSYeyKZqt8nr34Ldz+jzDkUSRiVgJn9D5ILuj4ZNMuijbUXqv6+290fqOUcq8JSgUz6vi16H2Z5Ra2pcjfYWBR+vd7MjkF9YG9391sqmMPZwOooJPgiWlymFe2daMyY2Q6oGGEvdB+shLx5p6dT/CJrVKbwus/nifox8hb2RGkgu7oqLiuZT7rm90eRgY+Tcb8GKmSZVPQGm4yXPi65qh8ifdAPkadmGNrIXq43Y+a7YCom2o2GyvSpKEy/DdDdC+r1nfv8twVOc/feudBvKxSlubJsj1lV5F7vD5Axtx3KYZyAOliVHvLOzSV7n89DTqQH0aHqYOQ5/bO7n1PBPFqjiElP9D4McPfXTL3fL1iUoqRN3qg0dXM4z927p9P6LiiX71F0kj+hLM9YYyB3U62JxJ83QAvoY8BYr0GLOjNbCVVer59CAocjA2c94Dh3n17CmNlCtx3q2nMBSgNYA2gJfIT6vtatpzLnJT4YndDvzT3XBq0XjV4n7buwIOM4HSo+rMI7n7vuNkeV1tumQ+2xqHvH0KINeDPbB9jf3Xc2s5VRUVxHJNc1CLjXKyrKm29ejeLAlvtMBgB3JcPbgKXd/ZOyDG2TFvDynuRs0mM7I/3IHkWPVyty68zlKJ2gN7r+ZqMGIxfkI1RVzAele3XzeYtVb0fFUuNLHj+73k5EaSBLp3tzSyTvt2mZ4xdNs1pPoBEwAFV5gbxy+6Dqzz7IS3lEjeZVCbnT76ko1L0qOqkdAoxNC13VtAVeMrPdkYbeYORJXrMMgzKRbWY9UXuwq939TNRr/C7g+Xo2KGEeYe1fo97Xzc1siElDsnO9GZTQoA1nItPtnU51Av+Z/ugA1Psa9P5vBmxtZl1L8AgviTzwoHDuysAQd78VdbXp/nX/sUwag0GZkbzZB6CDLKijyidm1iEZRGXoxt4H7Gdmh5kUMEAh2etKGKtm5NaZTZH24wfImXMcqqwHdE9WNKVlkIzXT3JjN0f3xcSv+09FkDMomyPB9RWALU2FsyegCMIiRRiV0mTraFLSPwWFOIanC/8ddMHVJWa2opn1SBd0Z3c/DQn8DkG9td+nxI4m38BktLH/HoXAX0RtAUvtrpIWsWWBHcysU7rhp7n7De5+eVljNwayBdzU8/4ld5+GvFYbALcDB+aMrrrDRXbAuhIl71cxbrbBrgNMMbOLUKHCQFQNvjEUvsHeC+xhZi8jPb6rUSQAVHE8No3ZvMAxFxncfW4y5K9FhiWAm+SerjCzVmUYwO7+LPAbpHDxWzN7Ca3BdWVUwlcFSM8i3d813f1Zd78LddW5B6o7ZLiUTkYCZ5jZCDP7BSqUe8nd/1XWuMkzP9ckq3WdqzhtdyTjdw0qCj2vrPHLom43ie/AOLRwPIVaAQ7xho4CO9BQuFOPHI5Cu/8EJqfT8eLJ3T/ezAaiG79SXJVuP8t+TuHAvujkViZroZPySsiIfdnMJqHF5R8lj11Tcgv4YsBrJg25Diinclugf5W5TrXEKxb6T1yCru/ZqHjmI1PHof8oeiB3fzP97T4oxD4OvpLR6o5khgpvCbkoYGbXouK04S7R9w3MbBB6rwzl3ZdZiXsDUptogaSVXvSKuphVzFtIo3Ip4Bkz64s62P0geYQrXWfcfYSZPYUiY11pSIErk61NahubkiIHLsWFx01NPwoXuq+CJm9Uuvu7QN+UxzcnLeaLI52wll5BZ4Ea0he13ptqZqcBiwMfp5PaD5CUwexv/AslYGYtXBWp26Fq7+FmtmvZ4Vd3fyXlm22M8ny2QN1dbkInx6bAeOQl6wD8p7t/mnIsh6fnmwFNztgog2zjNLM13f0OU8Hge66WpIOAV116fYUXRaWD83/l5tIarQd3ucTuF9lCrO+LqShmIirE2cvMXkX3/4vAIcmTWCrpelgDFebUozGZqXhsC4xInrpX0aHqARpaEVa2zliDGsGnyGP4WhXjorB/J7TePmFmnwD/7e6vII/1uyjta5GiyRfqLAhT/9veqKvGqFrPpwxMBUoPA9u5+6O5xw9BFY9zgLPcfUIN5pYlco8Abnb3mysadzm0sMxJi90yqPLzhXSjNxnMrKWrRdmmwNlA76bouSqbtMFehwrBHkA9jyeb9OlauSqwSzfwUni9FWqT+GlT8EgviPR5tEJaod2R6sSaKD1glJekgmHzVoAPdfcN6+0zSGHe95FsVW933yP3XHtgbnLyVDkng6+M+cuQZ/i8qg5VZrYe0sy9GUkprY1UGNohdYZpZc+haMKobKKYJBR6ozDLUihv7mJ3n2Rmi1d5Ss55bFpmnlFTtfFTwNplGjM5A3ZHVKTVGyXMP4SEsV9196qKNmpOyhf7wtQK8xF3f8rM2rr7u03Re1U26eDSH8h62i+L8rrHZWHpoBpy69BWqLPLYqh4cR0kffOyu59dwrgtsjUmpZ286+6X5jxodUFaY3+DikEvQ0WQrdz9nylftbm731jBPLLPeX1UWX9oMi5fBzbzGnXOS0b3KuhA85FXWAFfJGFUNlHM7FGUJzfLJFsxAOmFvQ6MRuLXn1ZxUs6d0k8G7nH3ialIoJO7P1/Fid3MHke5tecincKNkMfiqHr1Vi+I3IL7DLCbu0+p9ZyaAin8vAa6D/sCF7n7ZfXmrWqs5K77dYFbUbOLL9Jz2fpUmJGXG28d4FR3PyA9vh7q5LbIdFD5NqT0svdRp64LUS7/XLTXjEHFcce4+91lX/O5z/OXqP3yb9PhbruUhlLZ4Tntc3PTtbAHSj/5ZxVjl0VUfzdBTC0pv0wGZTN3H+nue6P8jkuRG75jhZuZmSqLV0lzAN1oz5taeJWS+5uFPtKJ9XNUdd7W3XdH8hLPATVtF1c2aVHLwk9ZGKgn0mmckr1HQfGY2ZJm1s3MlnL3T9z9GXf/Jcrhq1lXnyZKthcehHL9vki59QCbmtkBRXoNc2vrbKC1ma2QjKlJwIrJuKwn/oTyJScBP3H39ZHjojtS+bjB3e+G8qu+cwbjj4C1TDJRH7v7HfM9XwWe1txOwJnJa7tIr7lNvlCnKeLuD5mKYKBBJw+X8OuV6asS0kL6Zfp+OHC8mV0PtDFpZPYBelCCbmBu8eoAXIG8k5n4bWuSYVX0uI2JXGrBTWY2GVUbP0SDPmsU5hSMqVPL4sgruSvwrEne5xGkl7p2lsMbXspqyN0H7YER6bEsBWgg8rIVQjIaVkVh7qlm9i7Q1d1HmdlxwM5I9mlSUWM2AjZFOYIfmFlLM7sP5RHfB1zh7m9CdQL4KdTcCkmm/dGk8vEY8IS7f1L2+Bk5A7Y/kvGDRXzNDaOyiZLlLpaZr/gtedzMZqJ8zlHJS3YR8lpORSGRsvM7n0WFSQ68Z2b/gaoTS+352sjYHzgNtQc7F3gZGsX1UY8MQTl6n6NUiwnoQLM7Ul24Duqv3/MiwhjgVDP7AHgTfT49kHZgUeyAhL7HmNkElLt9elp3pgDH0yBOv8hjUtR409WZqANwEiqIvBZp4X5uZme69EHLTnPqDkx09+lmdjSKjPVAXsutUJvGv5Q5h9xc2iL5ujdR/UAWFVukD5JhVAa15lgkuHyhSfi5A1pk9/AS9QJzBTpdgR7u/sfkQRiHCnZuoaHDSd3j7tPM7ArgHDPrhSRUZkROX7GkDXZt1JauB+re9TDS7WuDjIpZ6dejKKp6RiED/yQUHWkP3OTubxQ4xtNIuqsnsFMa539RLudl9RACnY98B6f9aejg9FAy3n9ZRcjZpO7R190fMLMTUHrTk+5+jpmtiKSkppY8h2zf2Q/oBhyFWkGeYFIeqDr8XjhRqBPUDDNb0efttdofeQR2RILs13hJPYhTXukc1AJyYlpYsuT5pVACd02qAKvGzJZLYamVUTrEUBTuO6HKUFBTIKV43Ozut6SfT0NR7tL7jAf/P9YgpbU5sDwyhv5RYIFOS5TPnqX8bI5ky3oBX6DDxZkFG7E1xdRU4z4U0m0NHAmM8Yb+38+7etyX7plPBUMtUO3AZ+gA9wLq5FNZK14zG4+klX4FPOzuV5nZmcB4dx9bxRzKIozKoGaY+pv+C2l0/VfmmTTJCR2ENtuhJY29DXA5knEZhlrVvZa8cyOQQTu6jLEbE2a2Ecohew+FYbZBBUsroM4iB7u6PAQLiakz1D+AA4EHXDJN9wCnu/uEKqtOA5GrBF4P5W93RKLTDwNPu/vHBY93NHAicBVwhydBdVOv8b7oQP0Lr0HTiTJJa/qCOjjdC2yTdy5UNJ9WwGYozakTCkMPd/e/VTD2WsDl7r6dqeFBt+Sdfhw4wt3/p+w5lEkYlUFNSNXeW6A8lt4oJPI8MnBGlRlyNbOfoBDUCjS04ewNvIZac+2NJEUqObXWEjP7EUqDWQ61KZ2BtPk+Ai5AOp1DajfD+iFV2vdH3vGVUPhtY2BDb0JaqI2JnFF5G8qh3heYhkK2nwBXe4HNF9I1MAC1P+0MvALcAYz0RVDo+vtiktDaC137P69YxqcmTS5y11oHtO/MAX7o7geb2ZbA2e7erazxqyKMyqCmpBNjG7TA7oCMzBWRxEQpIcF0OjwX9fH9MnmQ+qCF5T3gbnd/oIyxFwVyi9/VwAyXzE1QIGa2PKryPR5dcxOR96Ku1QYaI2a2AgrHbmnS7x2EPIa9gCPd/eWSxm2PvNa7ozVwKnCgV9xVphaknNHKOjhZI2pykV77McAJqEhvCoqYPenuZ1UxhzIJozKoOWa2TBZmShVxfYBZXkJLNFN7yvPcvXu6uXcBbkAeypkoj/CtosdtTOQW2K2RHudBwFhkyN+X+712SAA/8ipLxMw6IuNyjLvfGcVR1WJmA5HHeBhaG3qZ2dIo97VfSWO2cfVfz37eCNjb3U8uY7xAWI2aXJhZP1SY89fs4GhmuwCbo8rzK4AH6yFiEUZlUDmWOlOY2ZrIqNkAJU0/Box198I04RYw9jlIf/I/zWx74GAk53AOqUDF3U8va/zGQK4g6X4kn/Ez5B1eF2mkHeruw/O/W7vZBkG5pEhFG5SScAZwDarMnuPuxxY4TnbfHYb0SVdD7XGHuXupVcdNmdz7vj4q0OkBPO7um5hEx88Fji4zSmBmO6ECpXbAdFRHcHM9plhFR52gcnKVlKciY2ZV5P4/BBhrEj0vi6eAjinkdQoKOQ1PVYfvAMuUOHajIC2wqwBLpGKkFd19c2Rgj0OFOplHMwzKoK5x9w/d/Q13fwKlIQxEm/9lBY/jZtYZGRdnAG2B7YHJZvZSWpOCgsmtYbVscjEGXVeDUMj7p8AEMxtm6nteN4ROZVApSQ+sMxKZ7ezuB5m6K/wBWAvYE+W5lMU4FP54ChWmDMmFoXagoXCn3tkCddHZCPgwPfY86vLxJITweVC/5FJAVkcFI5sCtwHnIzmx6QWPt1qSCToIVX63B25x90FmdiFAmRGaAKhhk4u0ls5KX4+b2SUo5WJH4Bgzu8PrpN97GJVB1RwOtEQ6lJOThtni7j4eGJ/ym54ta/CUBN836ZXNcfePTD1+9wBauvvTZY3dyHgYabM1AzCzA4CuuZ+jm0tQz2SVxrehg+YM4HfAb9HBtjCj0iRqfbqpucBnqJPM0DQuqLNSky0MLBNrBE0ucoWPWwKHoq5Zt6BC0fvMrG29GJQQRmVQPX2R9uFUk/Dz4sDHZvYLdLO9U4VGm88rbL4EMBttKk2CzBOT9PHuAn6PpE3OyH6lRlMLglIxs6WTLuAawHR3Pyn33OnAHmb2QoGpH21Qx5xbUKXvTcD9wL5mNh3JGF1Y0FjBvGxtZnNQW8yJ2YMusfERyCtdqBbp/CSDsj3q7X0u2udOBk4xs971VhgaOZVBZaTK6y1RLhHu/mrSBRuBEuPXReGnSnH3D9z95jKr/xoDyYCcB1e/3bNRvk8nd5+RPV71/IKgIi40s7tQN5P5oyIvAT2LzCV29/fc/SjUkvEtZFCADM3fAX9y99eLGi/4N4Yh2aa1k+LFSunxq9B+VBpJzQSUbjTa3S9w99PcvRMwGkXI6orwVAZVMhAJjF9taoV4O3Cxuw8zsxvc/fPaTq9+SQK/V6TT+UOZFl4uzH0U8p5E6DuoW1Lo8wqUS7cJcKiZrYuE/l9DXVYuLXi8JVG9yFOp8vsUYDdk7IyJe60crKHJxb4oV74ZcD3wWtIj3RBpVZbJzWb2GUp7mDjfc3OQI6WuCEmhoDLSjdzf3WeZ2c6os8R2wOvo1PYXpIsYF2XBmDp57IaMx7ZoMb0dLXT/QlXwm7n7zJARCuqd1HRhJWB9oB+KlHRGecbdi8pxS2k926HWj8uide494Og07p3AThEZKB6rcZMLa+gatwVqf7sbMBLl1L6N1uKz3f2FsuZQC8KoDCrB1Gt7iLv/2HItuVI1+C7oJDnQ3Z+r5TybAqZuLnsBx6IChdmos0Ufi/7TQRMjefGXR95L3P3qAv/2CUiq62J0cOuFvFbvoHvwTXc/qKjxAtGYmlyY2RLA0kAXpMu8A/JQ3urue1YxhyoJozKoDDNr6e6zI7zaeEjdXI5EIfHR8dkETYG8N97MfgVc4O6fZY0ZChxnCVRlPBAZNOejg1zz1AAi7rcSaGxNLvIdlJIeaT/gbXe/v6o5VEUYlUEQBEGTIifz0h043903Lnm8dkhOrS0q2Bhb5nhNHTMbgLzCJwK3Igm1S9x9mpmdiho+lNZk4xu6xk0E7qxnTdIwKoMgCIImg5m18NRj2czOQIL/l5bgpTwV+AIYj0Sv2yHjojOSFhqKtHJjEy6YVHV9DQozv4HyVjNP4UPAMVVoEpvZ34BpqMp8DuozvgzK8zyv7PFrQVR/B0EQBHVNrv/zOqg97AHpqZtR1TcFG5SrAx2RJuFJqAp5Bqo4Xyf92th6K9JoLNSyyUUj6BpXU8KoDIIgCOqanDdwNtA65bXNcvdJZvbDlPM2qcDxXjezIzND1cy6IGmh94EfASuHQVk+NWpyUdOucbUmjMogCIKgbknVv6uiMPdUM3sX6Oruo8zsOGBn4F6gMKMS5vV8unveiJhc5DjBt8PdP0Ce6bJpFF3jakXkVAZBEAR1i5n1Q236xgATgLWB45HXagpwFvBilmdZwXxCB7ZOSVJGDwPbufujuccPQTqVc4Cz3H1CjaZYOmFUBkEQBHVL6rt8ICqWWAq1R1wO9eK+LPUBD0MvWGjM7DygN9ACXWtZ17hJZrZ4U+gaF0ZlEARBUJeYWUvgy0wL0sw2R1IzvVBl9lvAme7+Ru1mGdQL0TVOvTCDIAiCoB45DBVLnGZmXdz9cXf/E8p7uwj4EnW3CYKFInWN+zIZlM3cfaS77w10Qv3kdwU61rNBCeGpDIIgCOqU1PN+AGqJ2Bl4BbgDGOnu02o5t6D+iK5xYVQGQRAETYBcbuXuQBvUi/vApGkYBEEBhFEZBEEQ1DX53svp542Avd395BpOKwjqjjAqgyAIgroj10XnMJTPthqqxh3m7lNrO7sgqE+iUCcIgiCoO5JB2Rk4EjgDaAtsjwp3XkpddYIgKJAwKoMgCIK6wsxWS98eBFwFtAducfctUSXuPe7+fo2mFwR1SxiVQRAEQd1gZi2A081sK+Az4FpgD+DB9CufA3fXaHpBUNeEURkEQRDUE21Q15xbkPD0qsD9wL5m1hPYF3j26/97EATflyjUCYIgCOqO1If5l8Bc4C5gF6AdcKO7X1jLuQVBvRJGZRAEQVAXmJkBS6I6nc/MbCngFNTVZBgwpqmKUgdBFSxW6wkEQRAEQUEMRiHvd81sWdRv+WFgQ9RJ504z28nd59ZwjkFQt4RRGQRBENQLzYA1gLGoY04vVKwzGlgCmBkGZRCUR4S/gyAIgrrAzJYA9gEGAo8C5wMzgObuPqcp92QOgioIozIIgiCoK8ysHXA4Ejwf7e5jazylIGgShFEZBEEQLPKY2anAF8B4YBaq9N4A6AxMAYYCczw2vSAojcipDIIgCBZpzGx1oCPwA+Ak4GkU9t4EWCf92lh3f6EW8wuCpkJ4KoMgCIJFHjNbzN3npO+7IGmh94EfASu7+7Bazi8ImgJhVAZBEARBEAQLTbRpDIIgCOqWJIgeBEEFhKcyCIIgCIIgWGjCUxkEQRAEQRAsNGFUBkEQBEEQBAtNGJVBEARBEATBQhNGZRAEQRAEQbDQhFEZBEEQBEEQLDRhVAZBEARBEAQLTRiVQRAEQRAEwULzf6WV+hNgIEr/AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.figure(figsize=(10, 10), dpi=80)\n", "\n", "unique, counts = np.unique(labels, return_counts=True)\n", "plt.bar(CLASS_LABELS[unique], counts)\n", "plt.xticks(rotation=70)\n", "\n", "unique, counts = np.unique(y_train, return_counts=True)\n", "plt.bar(CLASS_LABELS[unique], counts)\n", "plt.xticks(rotation=70)\n", "\n", "unique, counts = np.unique(y_test, return_counts=True)\n", "plt.bar(CLASS_LABELS[unique], counts)\n", "plt.xticks(rotation=70)\n", "\n", "unique, counts = np.unique(y_val, return_counts=True)\n", "plt.bar(CLASS_LABELS[unique], counts)\n", "plt.xticks(rotation=70)\n", "\n", "plt.legend([\"All\", \"Train\", \"Test\", \"Validation\"])\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QaZxGMgKZznS" }, "outputs": [], "source": [ "def train(config=None):\n", " with wandb.init(config=config):\n", " config = wandb.config\n", " \n", " # Generate new model\n", " model = Transformer(\n", " num_layers=config.num_layers,\n", " embed_dim=config.embed_layer_size,\n", " mlp_dim=config.fc_layer_size,\n", " num_heads=config.num_heads,\n", " num_classes=18,\n", " dropout_rate=config.dropout,\n", " attention_dropout_rate=config.attention_dropout,\n", " )\n", "\n", " # adapt on training dataset - must be before model.compile !!!\n", " model.input_norm.adapt(X_train, batch_size=config.batch_size)\n", " print(model.input_norm.variables)\n", "\n", " # Select optimizer\n", " if config.optimizer == \"adam\":\n", " optim = Adam(\n", " global_clipnorm=config.global_clipnorm,\n", " amsgrad=config.amsgrad,\n", " )\n", " elif config.optimizer == \"adamw\":\n", " optim = AdamW(\n", " weight_decay=config.weight_decay,\n", " amsgrad=config.amsgrad,\n", " global_clipnorm=config.global_clipnorm,\n", " exclude_from_weight_decay=[\"position\"]\n", " )\n", " else:\n", " raise ValueError(\"The used optimizer is not in list of available\")\n", "\n", " model.compile(\n", " loss=smoothed_sparse_categorical_crossentropy(label_smoothing=config.label_smoothing),\n", " optimizer=optim,\n", " metrics=[\"accuracy\"],\n", " )\n", "\n", " # Train model\n", " model.fit(\n", " X_train,\n", " y_train,\n", " batch_size=config.batch_size,\n", " epochs=config.epochs,\n", " validation_data=(X_val, y_val),\n", " callbacks=[\n", " LearningRateScheduler(cosine_schedule(base_lr=config.learning_rate, total_steps=config.epochs, warmup_steps=config.warmup_steps)),\n", " PrintLR(),\n", " WandbCallback(monitor=\"val_accuracy\", mode='max', save_weights_only=True),\n", " EarlyStopping(monitor=\"val_accuracy\", mode='max', min_delta=0.001, patience=5),\n", " ],\n", " verbose=1\n", " )\n", "\n", " model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "c2f96abecad54565be62d18c1b5c1e68", "fa0aba1429524429af176534638122db", "0054c6582b4c45faa323add90dd34770", "634c65b48e0b40359f6158364fb54ad7", "ebdfa2b37e5540e39bd6624f22eeb19a", "7198820fcadc4aaf875ee617dd605fbc", "18adf181e391491a9426d17e69a8c574", "dc1f7e46eaf643999d482c3b17e3bee6" ] }, "id": "J743-OTSSsZy", "outputId": "1ea18bc2-7308-4e10-cabf-7c94643fab4a" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: lwikvs2y with config:\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tamsgrad: False\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tattention_dropout: 0.1\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 64\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tdropout: 0.1\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tembed_layer_size: 128\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 50\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tfc_layer_size: 256\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tglobal_clipnorm: 3\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tlabel_smoothing: 0.1\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tlearning_rate: 0.001\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tnum_heads: 6\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \tnum_layers: 3\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \toptimizer: adam\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \twarmup_steps: 10\n" ] }, { "data": { "text/html": [ "\n", " Syncing run earnest-sweep-1 to Weights & Biases (docs).
\n", "Sweep page: https://wandb.ai/markub/imu-transformer/sweeps/cfdl7wcr
\n", "\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[, , ]\n", "Epoch 1/50\n", " 6/939 [..............................] - ETA: 3:08 - loss: 2.8825 - accuracy: 0.1228WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0914s vs `on_train_batch_end` time: 0.0915s). Check your callbacks.\n", "939/939 [==============================] - 226s 234ms/step - loss: 1.8975 - accuracy: 0.4512 - val_loss: 1.4693 - val_accuracy: 0.6309 - lr: 1.0000e-04\n", "Epoch 2/50\n", "939/939 [==============================] - 224s 239ms/step - loss: 1.2502 - accuracy: 0.7161 - val_loss: 1.0970 - val_accuracy: 0.7766 - lr: 2.0000e-04\n", "Epoch 3/50\n", "939/939 [==============================] - 224s 238ms/step - loss: 1.0634 - accuracy: 0.7823 - val_loss: 1.0177 - val_accuracy: 0.8070 - lr: 3.0000e-04\n", "Epoch 4/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.9973 - accuracy: 0.8063 - val_loss: 0.9463 - val_accuracy: 0.8246 - lr: 4.0000e-04\n", "Epoch 5/50\n", "939/939 [==============================] - 224s 239ms/step - loss: 0.9527 - accuracy: 0.8252 - val_loss: 0.9526 - val_accuracy: 0.8252 - lr: 5.0000e-04\n", "Epoch 6/50\n", "939/939 [==============================] - 233s 248ms/step - loss: 0.9277 - accuracy: 0.8355 - val_loss: 0.9304 - val_accuracy: 0.8317 - lr: 6.0000e-04\n", "Epoch 7/50\n", "939/939 [==============================] - 226s 240ms/step - loss: 0.9065 - accuracy: 0.8444 - val_loss: 0.8776 - val_accuracy: 0.8602 - lr: 7.0000e-04\n", "Epoch 8/50\n", "939/939 [==============================] - 224s 239ms/step - loss: 0.8888 - accuracy: 0.8529 - val_loss: 0.8554 - val_accuracy: 0.8703 - lr: 8.0000e-04\n", "Epoch 9/50\n", "939/939 [==============================] - 224s 239ms/step - loss: 0.8734 - accuracy: 0.8596 - val_loss: 0.9027 - val_accuracy: 0.8493 - lr: 9.0000e-04\n", "Epoch 10/50\n", "939/939 [==============================] - 232s 247ms/step - loss: 0.8616 - accuracy: 0.8657 - val_loss: 0.8845 - val_accuracy: 0.8542 - lr: 0.0010\n", "Epoch 11/50\n", "939/939 [==============================] - 227s 242ms/step - loss: 0.8363 - accuracy: 0.8779 - val_loss: 0.8222 - val_accuracy: 0.8856 - lr: 9.9846e-04\n", "Epoch 12/50\n", "939/939 [==============================] - 232s 248ms/step - loss: 0.8288 - accuracy: 0.8815 - val_loss: 0.8512 - val_accuracy: 0.8751 - lr: 9.9384e-04\n", "Epoch 13/50\n", "939/939 [==============================] - 234s 249ms/step - loss: 0.8128 - accuracy: 0.8888 - val_loss: 0.8171 - val_accuracy: 0.8859 - lr: 9.8619e-04\n", "Epoch 14/50\n", "939/939 [==============================] - 226s 241ms/step - loss: 0.8014 - accuracy: 0.8944 - val_loss: 0.7949 - val_accuracy: 0.8972 - lr: 9.7553e-04\n", "Epoch 15/50\n", "939/939 [==============================] - 232s 247ms/step - loss: 0.7910 - accuracy: 0.8990 - val_loss: 0.8334 - val_accuracy: 0.8826 - lr: 9.6194e-04\n", "Epoch 16/50\n", "939/939 [==============================] - 226s 241ms/step - loss: 0.7824 - accuracy: 0.9037 - val_loss: 0.8004 - val_accuracy: 0.8956 - lr: 9.4550e-04\n", "Epoch 17/50\n", "939/939 [==============================] - 225s 239ms/step - loss: 0.7732 - accuracy: 0.9078 - val_loss: 0.7767 - val_accuracy: 0.9084 - lr: 9.2632e-04\n", "Epoch 18/50\n", "939/939 [==============================] - 232s 247ms/step - loss: 0.7614 - accuracy: 0.9136 - val_loss: 0.7578 - val_accuracy: 0.9181 - lr: 9.0451e-04\n", "Epoch 19/50\n", "939/939 [==============================] - 226s 241ms/step - loss: 0.7565 - accuracy: 0.9163 - val_loss: 0.7581 - val_accuracy: 0.9151 - lr: 8.8020e-04\n", "Epoch 20/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.7475 - accuracy: 0.9202 - val_loss: 0.7378 - val_accuracy: 0.9265 - lr: 8.5355e-04\n", "Epoch 21/50\n", "939/939 [==============================] - 225s 239ms/step - loss: 0.7413 - accuracy: 0.9231 - val_loss: 0.7479 - val_accuracy: 0.9215 - lr: 8.2472e-04\n", "Epoch 22/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.7364 - accuracy: 0.9255 - val_loss: 0.7346 - val_accuracy: 0.9281 - lr: 7.9389e-04\n", "Epoch 23/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.7311 - accuracy: 0.9279 - val_loss: 0.7497 - val_accuracy: 0.9220 - lr: 7.6125e-04\n", "Epoch 24/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.7251 - accuracy: 0.9307 - val_loss: 0.7317 - val_accuracy: 0.9298 - lr: 7.2700e-04\n", "Epoch 25/50\n", "939/939 [==============================] - 224s 238ms/step - loss: 0.7216 - accuracy: 0.9324 - val_loss: 0.7182 - val_accuracy: 0.9356 - lr: 6.9134e-04\n", "Epoch 26/50\n", "939/939 [==============================] - 226s 241ms/step - loss: 0.7163 - accuracy: 0.9348 - val_loss: 0.7221 - val_accuracy: 0.9340 - lr: 6.5451e-04\n", "Epoch 27/50\n", "939/939 [==============================] - 233s 249ms/step - loss: 0.7107 - accuracy: 0.9373 - val_loss: 0.7117 - val_accuracy: 0.9390 - lr: 6.1672e-04\n", "Epoch 28/50\n", "939/939 [==============================] - 227s 242ms/step - loss: 0.7077 - accuracy: 0.9391 - val_loss: 0.7110 - val_accuracy: 0.9397 - lr: 5.7822e-04\n", "Epoch 29/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.7030 - accuracy: 0.9409 - val_loss: 0.7051 - val_accuracy: 0.9416 - lr: 5.3923e-04\n", "Epoch 30/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.6987 - accuracy: 0.9429 - val_loss: 0.6998 - val_accuracy: 0.9432 - lr: 5.0000e-04\n", "Epoch 31/50\n", "939/939 [==============================] - 233s 248ms/step - loss: 0.6951 - accuracy: 0.9448 - val_loss: 0.6992 - val_accuracy: 0.9447 - lr: 4.6077e-04\n", "Epoch 32/50\n", "939/939 [==============================] - 227s 241ms/step - loss: 0.6931 - accuracy: 0.9459 - val_loss: 0.6999 - val_accuracy: 0.9443 - lr: 4.2178e-04\n", "Epoch 33/50\n", "939/939 [==============================] - 225s 239ms/step - loss: 0.6892 - accuracy: 0.9474 - val_loss: 0.6952 - val_accuracy: 0.9458 - lr: 3.8328e-04\n", "Epoch 34/50\n", "939/939 [==============================] - 224s 239ms/step - loss: 0.6837 - accuracy: 0.9505 - val_loss: 0.6854 - val_accuracy: 0.9508 - lr: 3.4549e-04\n", "Epoch 35/50\n", "939/939 [==============================] - 232s 247ms/step - loss: 0.6752 - accuracy: 0.9549 - val_loss: 0.6496 - val_accuracy: 0.9714 - lr: 3.0866e-04\n", "Epoch 36/50\n", "939/939 [==============================] - 234s 249ms/step - loss: 0.6197 - accuracy: 0.9840 - val_loss: 0.6156 - val_accuracy: 0.9858 - lr: 2.7300e-04\n", "Epoch 37/50\n", "939/939 [==============================] - 234s 249ms/step - loss: 0.6016 - accuracy: 0.9919 - val_loss: 0.6100 - val_accuracy: 0.9891 - lr: 2.3875e-04\n", "Epoch 38/50\n", "939/939 [==============================] - 226s 241ms/step - loss: 0.5956 - accuracy: 0.9943 - val_loss: 0.6126 - val_accuracy: 0.9877 - lr: 2.0611e-04\n", "Epoch 39/50\n", "939/939 [==============================] - 233s 248ms/step - loss: 0.5924 - accuracy: 0.9957 - val_loss: 0.6096 - val_accuracy: 0.9894 - lr: 1.7528e-04\n", "Epoch 40/50\n", "939/939 [==============================] - 227s 241ms/step - loss: 0.5902 - accuracy: 0.9965 - val_loss: 0.6141 - val_accuracy: 0.9880 - lr: 1.4645e-04\n", "Epoch 41/50\n", "939/939 [==============================] - 225s 240ms/step - loss: 0.5881 - accuracy: 0.9973 - val_loss: 0.6074 - val_accuracy: 0.9908 - lr: 1.1980e-04\n", "Epoch 42/50\n", "939/939 [==============================] - 226s 240ms/step - loss: 0.5868 - accuracy: 0.9979 - val_loss: 0.6050 - val_accuracy: 0.9915 - lr: 9.5491e-05\n", "Epoch 43/50\n", "939/939 [==============================] - 233s 248ms/step - loss: 0.5866 - accuracy: 0.9979 - val_loss: 0.6042 - val_accuracy: 0.9914 - lr: 7.3680e-05\n", "Epoch 44/50\n", "939/939 [==============================] - 227s 241ms/step - loss: 0.5851 - accuracy: 0.9986 - val_loss: 0.6060 - val_accuracy: 0.9910 - lr: 5.4497e-05\n", "Epoch 45/50\n", "939/939 [==============================] - 226s 240ms/step - loss: 0.5845 - accuracy: 0.9988 - val_loss: 0.6055 - val_accuracy: 0.9914 - lr: 3.8060e-05\n", "Epoch 46/50\n", "939/939 [==============================] - 225s 239ms/step - loss: 0.5837 - accuracy: 0.9991 - val_loss: 0.6056 - val_accuracy: 0.9918 - lr: 2.4472e-05\n", "Model: \"transformer\"\n", "_________________________________________________________________\n", " Layer (type) Output Shape Param # \n", "=================================================================\n", " normalization (Normalizatio multiple 13 \n", " n) \n", " \n", " positional_embedding (Posit multiple 39296 \n", " ionalEmbedding) \n", " \n", " encoder (Encoder) multiple 462080 \n", " \n", " encoder_1 (Encoder) multiple 462080 \n", " \n", " encoder_2 (Encoder) multiple 462080 \n", " \n", " layer_normalization_6 (Laye multiple 256 \n", " rNormalization) \n", " \n", " dense_7 (Dense) multiple 2322 \n", " \n", "=================================================================\n", "Total params: 1,428,127\n", "Trainable params: 1,428,114\n", "Non-trainable params: 13\n", "_________________________________________________________________\n" ] }, { "data": { "text/html": [ "
Waiting for W&B process to finish, PID 498... (success)." ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c2f96abecad54565be62d18c1b5c1e68", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(Label(value=' 5.52MB of 5.52MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "
\n", "

Run history:


accuracy▁▄▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇█████████
epoch▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇████
loss█▅▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁
lr▁▂▃▄▄▅▆▇██████▇▇▇▇▇▇▆▆▅▅▅▅▄▄▄▃▃▃▂▂▂▂▁▁▁▁
val_accuracy▁▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇██████████
val_loss█▅▄▄▄▄▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁

\n", "

Run summary:


accuracy0.99912
best_epoch45
best_val_accuracy0.99177
epoch45
loss0.58374
lr2e-05
val_accuracy0.99177
val_loss0.60557
\n", "
\n", "Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 1 other file(s)\n", "
Synced earnest-sweep-1: https://wandb.ai/markub/imu-transformer/runs/lwikvs2y
\n", "Find logs at: ./wandb/run-20220201_003202-lwikvs2y/logs
\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: Sweep Agent: Waiting for job.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Sweep Agent: Exiting.\n" ] } ], "source": [ "wandb.agent(sweep_id, train, count=32)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "Training_posledna_verzia_3-3.ipynb", "provenance": [] }, "interpreter": { "hash": "9185113d2128201d66faecd4f34fb34e89a635073a034991399523e584519355" }, "kernelspec": { "display_name": "Python 3.9.6 64-bit ('base': conda)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" }, "orig_nbformat": 4, "widgets": { "application/vnd.jupyter.widget-state+json": { "0054c6582b4c45faa323add90dd34770": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "LabelModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "LabelModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "LabelView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7198820fcadc4aaf875ee617dd605fbc", "placeholder": "​", "style": "IPY_MODEL_ebdfa2b37e5540e39bd6624f22eeb19a", "value": " 5.55MB of 5.55MB uploaded (0.00MB deduped)\r" } }, "18adf181e391491a9426d17e69a8c574": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "634c65b48e0b40359f6158364fb54ad7": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_dc1f7e46eaf643999d482c3b17e3bee6", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_18adf181e391491a9426d17e69a8c574", "value": 1 } }, "7198820fcadc4aaf875ee617dd605fbc": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c2f96abecad54565be62d18c1b5c1e68": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "VBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "VBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "VBoxView", "box_style": "", "children": [ "IPY_MODEL_0054c6582b4c45faa323add90dd34770", "IPY_MODEL_634c65b48e0b40359f6158364fb54ad7" ], "layout": "IPY_MODEL_fa0aba1429524429af176534638122db" } }, "dc1f7e46eaf643999d482c3b17e3bee6": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ebdfa2b37e5540e39bd6624f22eeb19a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "fa0aba1429524429af176534638122db": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } } } } }, "nbformat": 4, "nbformat_minor": 0 }