{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.16","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install transformers[sentencepiece] huggingface -q","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:47:53.058331Z","iopub.execute_input":"2023-05-12T11:47:53.058582Z","iopub.status.idle":"2023-05-12T11:48:11.528155Z","shell.execute_reply.started":"2023-05-12T11:47:53.058558Z","shell.execute_reply":"2023-05-12T11:48:11.527021Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ntensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 3.20.2 which is incompatible.\ntensorflow-metadata 1.13.1 requires protobuf<5,>=3.20.3, but you have protobuf 3.20.2 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.4; however, version 23.1.2 is available.\nYou should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n#import seaborn as sns\nimport os\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Input, Dense, GlobalMaxPool1D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.data import Dataset\n\nimport transformers\nfrom transformers import AutoTokenizer, TFAutoModel\n\nfrom huggingface_hub import notebook_login, push_to_hub_keras, from_pretrained_keras","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:49:51.308904Z","iopub.execute_input":"2023-05-12T11:49:51.309754Z","iopub.status.idle":"2023-05-12T11:50:50.942522Z","shell.execute_reply.started":"2023-05-12T11:49:51.309710Z","shell.execute_reply":"2023-05-12T11:50:50.941305Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stderr","text":"D0512 11:50:24.269871341      14 config.cc:119]                        gRPC EXPERIMENT tcp_frame_size_tuning               OFF (default:OFF)\nD0512 11:50:24.269909000      14 config.cc:119]                        gRPC EXPERIMENT tcp_rcv_lowat                       OFF (default:OFF)\nD0512 11:50:24.269913299      14 config.cc:119]                        gRPC EXPERIMENT peer_state_based_framing            OFF (default:OFF)\nD0512 11:50:24.269916201      14 config.cc:119]                        gRPC EXPERIMENT flow_control_fixes                  ON  (default:ON)\nD0512 11:50:24.269918879      14 config.cc:119]                        gRPC EXPERIMENT memory_pressure_controller          OFF (default:OFF)\nD0512 11:50:24.269921601      14 config.cc:119]                        gRPC EXPERIMENT unconstrained_max_quota_buffer_size OFF (default:OFF)\nD0512 11:50:24.269925176      14 config.cc:119]                        gRPC EXPERIMENT new_hpack_huffman_decoder           ON  (default:ON)\nD0512 11:50:24.269927811      14 config.cc:119]                        gRPC EXPERIMENT event_engine_client                 OFF (default:OFF)\nD0512 11:50:24.269930386      14 config.cc:119]                        gRPC EXPERIMENT monitoring_experiment               ON  (default:ON)\nD0512 11:50:24.269932936      14 config.cc:119]                        gRPC EXPERIMENT promise_based_client_call           OFF (default:OFF)\nD0512 11:50:24.269935469      14 config.cc:119]                        gRPC EXPERIMENT free_large_allocator                OFF (default:OFF)\nD0512 11:50:24.269938067      14 config.cc:119]                        gRPC EXPERIMENT promise_based_server_call           OFF (default:OFF)\nD0512 11:50:24.269940970      14 config.cc:119]                        gRPC EXPERIMENT transport_supplies_client_latency   OFF (default:OFF)\nD0512 11:50:24.269952451      14 config.cc:119]                        gRPC EXPERIMENT event_engine_listener               OFF (default:OFF)\nI0512 11:50:24.270176104      14 ev_epoll1_linux.cc:122]               grpc epoll fd: 66\nD0512 11:50:24.270188925      14 ev_posix.cc:144]                      Using polling engine: epoll1\nD0512 11:50:24.270209220      14 dns_resolver_ares.cc:822]             Using ares dns resolver\nD0512 11:50:24.270651874      14 lb_policy_registry.cc:46]             registering LB policy factory for \"priority_experimental\"\nD0512 11:50:24.270665013      14 lb_policy_registry.cc:46]             registering LB policy factory for \"outlier_detection_experimental\"\nD0512 11:50:24.270669110      14 lb_policy_registry.cc:46]             registering LB policy factory for \"weighted_target_experimental\"\nD0512 11:50:24.270672580      14 lb_policy_registry.cc:46]             registering LB policy factory for \"pick_first\"\nD0512 11:50:24.270675869      14 lb_policy_registry.cc:46]             registering LB policy factory for \"round_robin\"\nD0512 11:50:24.270679086      14 lb_policy_registry.cc:46]             registering LB policy factory for \"weighted_round_robin_experimental\"\nD0512 11:50:24.270685803      14 lb_policy_registry.cc:46]             registering LB policy factory for \"ring_hash_experimental\"\nD0512 11:50:24.270702631      14 lb_policy_registry.cc:46]             registering LB policy factory for \"grpclb\"\nD0512 11:50:24.270729400      14 lb_policy_registry.cc:46]             registering LB policy factory for \"rls_experimental\"\nD0512 11:50:24.270744660      14 lb_policy_registry.cc:46]             registering LB policy factory for \"xds_cluster_manager_experimental\"\nD0512 11:50:24.270749029      14 lb_policy_registry.cc:46]             registering LB policy factory for \"xds_cluster_impl_experimental\"\nD0512 11:50:24.270752476      14 lb_policy_registry.cc:46]             registering LB policy factory for \"cds_experimental\"\nD0512 11:50:24.270758870      14 lb_policy_registry.cc:46]             registering LB policy factory for \"xds_cluster_resolver_experimental\"\nD0512 11:50:24.270762669      14 lb_policy_registry.cc:46]             registering LB policy factory for \"xds_override_host_experimental\"\nD0512 11:50:24.270766303      14 lb_policy_registry.cc:46]             registering LB policy factory for \"xds_wrr_locality_experimental\"\nD0512 11:50:24.270771070      14 certificate_provider_registry.cc:35]  registering certificate provider factory for \"file_watcher\"\nI0512 11:50:24.273060617      14 socket_utils_common_posix.cc:408]     Disabling AF_INET6 sockets because ::1 is not available.\nI0512 11:50:24.288500755      14 socket_utils_common_posix.cc:337]     TCP_USER_TIMEOUT is available. TCP_USER_TIMEOUT will be used thereafter\nE0512 11:50:24.296032187      14 oauth2_credentials.cc:236]            oauth_fetch: UNKNOWN:C-ares status is not ARES_SUCCESS qtype=A name=metadata.google.internal. is_balancer=0: Domain name not found {created_time:\"2023-05-12T11:50:24.29601562+00:00\", grpc_status:2}\n","output_type":"stream"}]},{"cell_type":"code","source":"## Setting up TPUs\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\nprint('Running on TPU ', tpu.master())\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.TPUStrategy(tpu)\nprint(\"REPLICAS: \", tpu_strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:50:50.944280Z","iopub.execute_input":"2023-05-12T11:50:50.944798Z","iopub.status.idle":"2023-05-12T11:51:00.695080Z","shell.execute_reply.started":"2023-05-12T11:50:50.944771Z","shell.execute_reply":"2023-05-12T11:51:00.693913Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"Running on TPU  \nINFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\nINFO:tensorflow:Initializing the TPU system: local\nINFO:tensorflow:Finished initializing TPU system.\nINFO:tensorflow:Found TPU system:\nINFO:tensorflow:*** Num TPU Cores: 8\nINFO:tensorflow:*** Num TPU Workers: 1\nINFO:tensorflow:*** Num TPU Cores Per Worker: 8\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:0, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:1, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:2, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:3, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:4, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:5, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:6, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:7, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\nREPLICAS:  8\n","output_type":"stream"}]},{"cell_type":"code","source":"class Config:\n    EPOCHS = 4\n    MODEL = \"bert-base-multilingual-uncased\"\n    BUFFER_SIZE = 1000\n    BATCH_SIZE = 16*tpu_strategy.num_replicas_in_sync\n    MAX_LEN = 192\n    LEARNING_RATE = 2e-5\n    WEIGHT_DECAY = 1e-6\n    RANDOM_STATE = 42","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:00.696282Z","iopub.execute_input":"2023-05-12T11:51:00.696560Z","iopub.status.idle":"2023-05-12T11:51:00.702270Z","shell.execute_reply.started":"2023-05-12T11:51:00.696536Z","shell.execute_reply":"2023-05-12T11:51:00.701120Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"input_dir = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification\"\ntrain1 = pd.read_csv(os.path.join(input_dir, \"jigsaw-toxic-comment-train.csv\"))\ntrain2 = pd.read_csv(os.path.join(input_dir, \"jigsaw-unintended-bias-train.csv\"))\nval = pd.read_csv(os.path.join(input_dir,\"validation.csv\"))\ntest = pd.read_csv(os.path.join(input_dir,\"test.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:00.704483Z","iopub.execute_input":"2023-05-12T11:51:00.704815Z","iopub.status.idle":"2023-05-12T11:51:27.504621Z","shell.execute_reply.started":"2023-05-12T11:51:00.704792Z","shell.execute_reply":"2023-05-12T11:51:27.503561Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"train1 = train1.iloc[:,1:3]\ntrain2 = train2.iloc[:,1:3]\nval = val.loc[:,[\"comment_text\",\"toxic\"]]\ntest.rename(columns={\"content\":\"comment_text\"}, inplace=True)\nsub = test[['id']]\ntrain2.toxic = (train2.toxic>0.5).astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:27.505989Z","iopub.execute_input":"2023-05-12T11:51:27.506326Z","iopub.status.idle":"2023-05-12T11:51:27.643971Z","shell.execute_reply.started":"2023-05-12T11:51:27.506301Z","shell.execute_reply":"2023-05-12T11:51:27.643077Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train1,\n                   train2.query(\"toxic==1\"),\n                   train2.query(\"toxic==0\").sample(n=200000, random_state=Config.RANDOM_STATE)])\ntrain.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:27.645230Z","iopub.execute_input":"2023-05-12T11:51:27.645543Z","iopub.status.idle":"2023-05-12T11:51:27.973540Z","shell.execute_reply.started":"2023-05-12T11:51:27.645516Z","shell.execute_reply":"2023-05-12T11:51:27.972585Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:27.974748Z","iopub.execute_input":"2023-05-12T11:51:27.975079Z","iopub.status.idle":"2023-05-12T11:51:27.983728Z","shell.execute_reply.started":"2023-05-12T11:51:27.975052Z","shell.execute_reply":"2023-05-12T11:51:27.982961Z"},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"(535775, 2)"},"metadata":{}}]},{"cell_type":"code","source":"test.rename(columns={\"content\":\"comment_text\"}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:27.984792Z","iopub.execute_input":"2023-05-12T11:51:27.985120Z","iopub.status.idle":"2023-05-12T11:51:27.995577Z","shell.execute_reply.started":"2023-05-12T11:51:27.985070Z","shell.execute_reply":"2023-05-12T11:51:27.994720Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"import re\ntrain['comment_text'] = train['comment_text'].apply(lambda x: re.sub('\\n',' ',x).strip())\nval['comment_text'] = val['comment_text'].apply(lambda x: re.sub('\\n',' ',x).strip())\ntest['comment_text'] = test['comment_text'].apply(lambda x: re.sub('\\n',' ',x).strip())","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:27.996790Z","iopub.execute_input":"2023-05-12T11:51:27.997135Z","iopub.status.idle":"2023-05-12T11:51:29.490559Z","shell.execute_reply.started":"2023-05-12T11:51:27.997088Z","shell.execute_reply":"2023-05-12T11:51:29.489600Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"seq_len = [len(i.split()) for i in train.comment_text]\n\npd.Series(seq_len).hist(bins = 30)\nprint(np.mean(seq_len))\nprint(max(seq_len))","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:29.493564Z","iopub.execute_input":"2023-05-12T11:51:29.493937Z","iopub.status.idle":"2023-05-12T11:51:32.280356Z","shell.execute_reply.started":"2023-05-12T11:51:29.493911Z","shell.execute_reply":"2023-05-12T11:51:32.279438Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"56.28243572395129\n2321\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAkIAAAGdCAYAAAD+JxxnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+BklEQVR4nO3de3BU9f3/8VcSkw0BNuFiElICRFEhykVCDfutOlxCFsw4UqODymhEhIFv4jSkBZv+MNzawWK5WYJpqxA6SgU61VagIWuQUMsCEki5CaMWv7Ff2GAVWAiwWZLz+6OT82UJQhYWVjnPx8xO3fN5n7Of/byT8OrZc5IIwzAMAQAAWFBkuCcAAAAQLgQhAABgWQQhAABgWQQhAABgWQQhAABgWQQhAABgWQQhAABgWQQhAABgWbeEewLfZs3NzTpy5Ig6duyoiIiIcE8HAAC0gWEYOnXqlFJSUhQZeflzPgShyzhy5IhSU1PDPQ0AAHAVvvjiC3Xv3v2yNQShy+jYsaOk/yyk3W4P6bH9fr8qKyuVnZ2t6OjokB4bbUMPwo8ehB89CD96EHper1epqanmv+OXQxC6jJaPw+x2+3UJQnFxcbLb7Xzhhwk9CD96EH70IPzowfXTlstauFgaAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABY1i3hnoDV3TNro3xNEUHv9/nLOddhNgAAWAtnhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGVdUxB6+eWXFRERocLCQnPbuXPnlJ+fry5duqhDhw7Kzc1VfX19wH51dXXKyclRXFycEhMTNW3aNJ0/fz6gZvPmzRo0aJBsNpt69+6t8vLyVq9fWlqqXr16KTY2VpmZmdqxY0fAeFvmAgAArOuqg9BHH32k3/zmN+rfv3/A9qlTp+q9997T2rVrVV1drSNHjujRRx81x5uampSTk6PGxkZt3bpVK1euVHl5uUpKSsyaw4cPKycnR8OGDVNtba0KCwv1/PPPa+PGjWbN6tWrVVRUpJkzZ2rXrl0aMGCAnE6njh071ua5AAAAa7uqIHT69GmNGzdOv/vd79SpUydz+8mTJ/XGG29o4cKFGj58uDIyMrRixQpt3bpV27ZtkyRVVlbqwIEDevPNNzVw4ECNHj1ac+fOVWlpqRobGyVJZWVlSktL04IFC9S3b18VFBToscce06JFi8zXWrhwoSZOnKjx48crPT1dZWVliouL0/Lly9s8FwAAYG1XFYTy8/OVk5OjrKysgO01NTXy+/0B2/v06aMePXrI7XZLktxut/r166ekpCSzxul0yuv1av/+/WbNxcd2Op3mMRobG1VTUxNQExkZqaysLLOmLXMBAADWdkuwO7z99tvatWuXPvroo1ZjHo9HMTExSkhICNielJQkj8dj1lwYglrGW8YuV+P1enX27FkdP35cTU1Nl6w5ePBgm+dyMZ/PJ5/PZz73er2SJL/fL7/ff8l9rlbL8WyRxjXtj6vXsoasZfjQg/CjB+FHD0IvmLUMKgh98cUX+tGPfiSXy6XY2NigJ/ZtN2/ePM2ePbvV9srKSsXFxV2X15w7uPmq9tuwYUOIZ2JdLpcr3FOwPHoQfvQg/OhB6Jw5c6bNtUEFoZqaGh07dkyDBg0ytzU1NWnLli1aunSpNm7cqMbGRp04cSLgTEx9fb2Sk5MlScnJya3u7mq5k+vCmovv7qqvr5fdble7du0UFRWlqKioS9ZceIwrzeVixcXFKioqMp97vV6lpqYqOztbdru9LUvUZn6/Xy6XSy/tjJSvOSLo/ffNcoZ0PlbU0oORI0cqOjo63NOxJHoQfvQg/OhB6LV8otMWQQWhESNGaO/evQHbxo8frz59+ujFF19UamqqoqOjVVVVpdzcXEnSoUOHVFdXJ4fDIUlyOBz6xS9+oWPHjikxMVHSf1Kw3W5Xenq6WXPxGQ+Xy2UeIyYmRhkZGaqqqtKYMWMkSc3NzaqqqlJBQYEkKSMj44pzuZjNZpPNZmu1PTo6+rp9cfqaI+RrCj4I8c0SOtezv2gbehB+9CD86EHoBLOOQQWhjh076p577gnY1r59e3Xp0sXcPmHCBBUVFalz586y2+164YUX5HA4NGTIEElSdna20tPT9fTTT2v+/PnyeDyaMWOG8vPzzRAyefJkLV26VNOnT9dzzz2nTZs2ac2aNVq/fr35ukVFRcrLy9PgwYN13333afHixWpoaND48eMlSfHx8VecCwAAsLagL5a+kkWLFikyMlK5ubny+XxyOp1atmyZOR4VFaV169ZpypQpcjgcat++vfLy8jRnzhyzJi0tTevXr9fUqVO1ZMkSde/eXa+//rqczv/7OGjs2LH68ssvVVJSIo/Ho4EDB6qioiLgAuorzQUAAFjbNQehzZs3BzyPjY1VaWmpSktLv3Gfnj17XvFi36FDh2r37t2XrSkoKDA/CruUtswFAABYF39rDAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWFZQQei1115T//79ZbfbZbfb5XA49Ne//tUcHzp0qCIiIgIekydPDjhGXV2dcnJyFBcXp8TERE2bNk3nz58PqNm8ebMGDRokm82m3r17q7y8vNVcSktL1atXL8XGxiozM1M7duwIGD937pzy8/PVpUsXdejQQbm5uaqvrw/m7QIAgJtcUEGoe/fuevnll1VTU6OdO3dq+PDheuSRR7R//36zZuLEiTp69Kj5mD9/vjnW1NSknJwcNTY2auvWrVq5cqXKy8tVUlJi1hw+fFg5OTkaNmyYamtrVVhYqOeff14bN240a1avXq2ioiLNnDlTu3bt0oABA+R0OnXs2DGzZurUqXrvvfe0du1aVVdX68iRI3r00UevapEAAMDNKagg9PDDD+uhhx7SHXfcoTvvvFO/+MUv1KFDB23bts2siYuLU3Jysvmw2+3mWGVlpQ4cOKA333xTAwcO1OjRozV37lyVlpaqsbFRklRWVqa0tDQtWLBAffv2VUFBgR577DEtWrTIPM7ChQs1ceJEjR8/Xunp6SorK1NcXJyWL18uSTp58qTeeOMNLVy4UMOHD1dGRoZWrFihrVu3BswVAABY2y1Xu2NTU5PWrl2rhoYGORwOc/tbb72lN998U8nJyXr44Yf10ksvKS4uTpLkdrvVr18/JSUlmfVOp1NTpkzR/v37de+998rtdisrKyvgtZxOpwoLCyVJjY2NqqmpUXFxsTkeGRmprKwsud1uSVJNTY38fn/Acfr06aMePXrI7XZryJAhl3xPPp9PPp/PfO71eiVJfr9ffr//apbpG7UczxZpXNP+uHota8hahg89CD96EH70IPSCWcugg9DevXvlcDh07tw5dejQQe+8847S09MlSU899ZR69uyplJQU7dmzRy+++KIOHTqkP/3pT5Ikj8cTEIIkmc89Hs9la7xer86ePavjx4+rqanpkjUHDx40jxETE6OEhIRWNS2vcynz5s3T7NmzW22vrKw0w1yozR3cfFX7bdiwIcQzsS6XyxXuKVgePQg/ehB+9CB0zpw50+baoIPQXXfdpdraWp08eVJ//OMflZeXp+rqaqWnp2vSpElmXb9+/dStWzeNGDFCn332mW6//fZgX+qGKy4uVlFRkfnc6/UqNTVV2dnZAR/xhYLf75fL5dJLOyPla44Iev99s5whnY8VtfRg5MiRio6ODvd0LIkehB89CD96EHotn+i0RdBBKCYmRr1795YkZWRk6KOPPtKSJUv0m9/8plVtZmamJOnTTz/V7bffruTk5FZ3d7XcyZWcnGz+78V3d9XX18tut6tdu3aKiopSVFTUJWsuPEZjY6NOnDgRcFbowppLsdlsstlsrbZHR0dfty9OX3OEfE3BByG+WULnevYXbUMPwo8ehB89CJ1g1vGaf49Qc3NzwHU1F6qtrZUkdevWTZLkcDi0d+/egLu7XC6X7Ha7+fGaw+FQVVVVwHFcLpd5HVJMTIwyMjICapqbm1VVVWXWZGRkKDo6OqDm0KFDqqurC7ieCQAAWFtQZ4SKi4s1evRo9ejRQ6dOndKqVau0efNmbdy4UZ999plWrVqlhx56SF26dNGePXs0depUPfjgg+rfv78kKTs7W+np6Xr66ac1f/58eTwezZgxQ/n5+eaZmMmTJ2vp0qWaPn26nnvuOW3atElr1qzR+vXrzXkUFRUpLy9PgwcP1n333afFixeroaFB48ePlyTFx8drwoQJKioqUufOnWW32/XCCy/I4XB844XSAADAeoIKQseOHdMzzzyjo0ePKj4+Xv3799fGjRs1cuRIffHFF3r//ffNUJKamqrc3FzNmDHD3D8qKkrr1q3TlClT5HA41L59e+Xl5WnOnDlmTVpamtavX6+pU6dqyZIl6t69u15//XU5nf93TczYsWP15ZdfqqSkRB6PRwMHDlRFRUXABdSLFi1SZGSkcnNz5fP55HQ6tWzZsmtZKwAAcJMJKgi98cYb3ziWmpqq6urqKx6jZ8+eV7zjaejQodq9e/dlawoKClRQUPCN47GxsSotLVVpaekV5wQAAKyJvzUGAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsK6gg9Nprr6l///6y2+2y2+1yOBz661//ao6fO3dO+fn56tKlizp06KDc3FzV19cHHKOurk45OTmKi4tTYmKipk2bpvPnzwfUbN68WYMGDZLNZlPv3r1VXl7eai6lpaXq1auXYmNjlZmZqR07dgSMt2UuAADA2oIKQt27d9fLL7+smpoa7dy5U8OHD9cjjzyi/fv3S5KmTp2q9957T2vXrlV1dbWOHDmiRx991Ny/qalJOTk5amxs1NatW7Vy5UqVl5erpKTErDl8+LBycnI0bNgw1dbWqrCwUM8//7w2btxo1qxevVpFRUWaOXOmdu3apQEDBsjpdOrYsWNmzZXmAgAAEFQQevjhh/XQQw/pjjvu0J133qlf/OIX6tChg7Zt26aTJ0/qjTfe0MKFCzV8+HBlZGRoxYoV2rp1q7Zt2yZJqqys1IEDB/Tmm29q4MCBGj16tObOnavS0lI1NjZKksrKypSWlqYFCxaob9++Kigo0GOPPaZFixaZ81i4cKEmTpyo8ePHKz09XWVlZYqLi9Py5cslqU1zAQAAuOVqd2xqatLatWvV0NAgh8Ohmpoa+f1+ZWVlmTV9+vRRjx495Ha7NWTIELndbvXr109JSUlmjdPp1JQpU7R//37de++9crvdAcdoqSksLJQkNTY2qqamRsXFxeZ4ZGSksrKy5Ha7JalNc7kUn88nn89nPvd6vZIkv98vv99/lSt1aS3Hs0Ua17Q/rl7LGrKW4UMPwo8ehB89CL1g1jLoILR37145HA6dO3dOHTp00DvvvKP09HTV1tYqJiZGCQkJAfVJSUnyeDySJI/HExCCWsZbxi5X4/V6dfbsWR0/flxNTU2XrDl48KB5jCvN5VLmzZun2bNnt9peWVmpuLi4b9zvWswd3HxV+23YsCHEM7Eul8sV7ilYHj0IP3oQfvQgdM6cOdPm2qCD0F133aXa2lqdPHlSf/zjH5WXl6fq6upgD/OtVFxcrKKiIvO51+tVamqqsrOzZbfbQ/pafr9fLpdLL+2MlK85Iuj9981yhnQ+VtTSg5EjRyo6Ojrc07EkehB+9CD86EHotXyi0xZBB6GYmBj17t1bkpSRkaGPPvpIS5Ys0dixY9XY2KgTJ04EnImpr69XcnKyJCk5ObnV3V0td3JdWHPx3V319fWy2+1q166doqKiFBUVdcmaC49xpblcis1mk81ma7U9Ojr6un1x+poj5GsKPgjxzRI617O/aBt6EH70IPzoQegEs47X/HuEmpub5fP5lJGRoejoaFVVVZljhw4dUl1dnRwOhyTJ4XBo7969AXd3uVwu2e12paenmzUXHqOlpuUYMTExysjICKhpbm5WVVWVWdOWuQAAAAR1Rqi4uFijR49Wjx49dOrUKa1atUqbN2/Wxo0bFR8frwkTJqioqEidO3eW3W7XCy+8IIfDYV6cnJ2drfT0dD399NOaP3++PB6PZsyYofz8fPNMzOTJk7V06VJNnz5dzz33nDZt2qQ1a9Zo/fr15jyKioqUl5enwYMH67777tPixYvV0NCg8ePHS1Kb5gIAABBUEDp27JieeeYZHT16VPHx8erfv782btyokSNHSpIWLVqkyMhI5ebmyufzyel0atmyZeb+UVFRWrdunaZMmSKHw6H27dsrLy9Pc+bMMWvS0tK0fv16TZ06VUuWLFH37t31+uuvy+n8v2tixo4dqy+//FIlJSXyeDwaOHCgKioqAi6gvtJcAAAAIgzDuLr7ty3A6/UqPj5eJ0+evC4XS2/YsEHTd0Rd1TVCn7+cE9L5WFFLDx566CE+lw8TehB+9CD86EHoBfPvN39rDAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWBZBCAAAWFZQQWjevHn6/ve/r44dOyoxMVFjxozRoUOHAmqGDh2qiIiIgMfkyZMDaurq6pSTk6O4uDglJiZq2rRpOn/+fEDN5s2bNWjQINlsNvXu3Vvl5eWt5lNaWqpevXopNjZWmZmZ2rFjR8D4uXPnlJ+fry5duqhDhw7Kzc1VfX19MG8ZAADcxIIKQtXV1crPz9e2bdvkcrnk9/uVnZ2thoaGgLqJEyfq6NGj5mP+/PnmWFNTk3JyctTY2KitW7dq5cqVKi8vV0lJiVlz+PBh5eTkaNiwYaqtrVVhYaGef/55bdy40axZvXq1ioqKNHPmTO3atUsDBgyQ0+nUsWPHzJqpU6fqvffe09q1a1VdXa0jR47o0UcfDXqRAADAzemWYIorKioCnpeXlysxMVE1NTV68MEHze1xcXFKTk6+5DEqKyt14MABvf/++0pKStLAgQM1d+5cvfjii5o1a5ZiYmJUVlamtLQ0LViwQJLUt29fffjhh1q0aJGcTqckaeHChZo4caLGjx8vSSorK9P69eu1fPly/fSnP9XJkyf1xhtvaNWqVRo+fLgkacWKFerbt6+2bdumIUOGBPPWAQDATSioIHSxkydPSpI6d+4csP2tt97Sm2++qeTkZD388MN66aWXFBcXJ0lyu93q16+fkpKSzHqn06kpU6Zo//79uvfee+V2u5WVlRVwTKfTqcLCQklSY2OjampqVFxcbI5HRkYqKytLbrdbklRTUyO/3x9wnD59+qhHjx5yu92XDEI+n08+n8987vV6JUl+v19+vz/o9bmcluPZIo1r2h9Xr2UNWcvwoQfhRw/Cjx6EXjBredVBqLm5WYWFhfrBD36ge+65x9z+1FNPqWfPnkpJSdGePXv04osv6tChQ/rTn/4kSfJ4PAEhSJL53OPxXLbG6/Xq7NmzOn78uJqami5Zc/DgQfMYMTExSkhIaFXT8joXmzdvnmbPnt1qe2VlpRnkQm3u4Oar2m/Dhg0hnol1uVyucE/B8uhB+NGD8KMHoXPmzJk21151EMrPz9e+ffv04YcfBmyfNGmS+d/9+vVTt27dNGLECH322We6/fbbr/blboji4mIVFRWZz71er1JTU5WdnS273R7S1/L7/XK5XHppZ6R8zRFB779vljOk87Gilh6MHDlS0dHR4Z6OJdGD8KMH4UcPQq/lE522uKogVFBQoHXr1mnLli3q3r37ZWszMzMlSZ9++qluv/12JScnt7q7q+VOrpbripKTk1vd3VVfXy+73a527dopKipKUVFRl6y58BiNjY06ceJEwFmhC2suZrPZZLPZWm2Pjo6+bl+cvuYI+ZqCD0J8s4TO9ewv2oYehB89CD96EDrBrGNQd40ZhqGCggK988472rRpk9LS0q64T21trSSpW7dukiSHw6G9e/cG3N3lcrlkt9uVnp5u1lRVVQUcx+VyyeFwSJJiYmKUkZERUNPc3KyqqiqzJiMjQ9HR0QE1hw4dUl1dnVkDAACsLagzQvn5+Vq1apX+/Oc/q2PHjua1NvHx8WrXrp0+++wzrVq1Sg899JC6dOmiPXv2aOrUqXrwwQfVv39/SVJ2drbS09P19NNPa/78+fJ4PJoxY4by8/PNszGTJ0/W0qVLNX36dD333HPatGmT1qxZo/Xr15tzKSoqUl5engYPHqz77rtPixcvVkNDg3kXWXx8vCZMmKCioiJ17txZdrtdL7zwghwOB3eMAQAASUEGoddee03Sf35p4oVWrFihZ599VjExMXr//ffNUJKamqrc3FzNmDHDrI2KitK6des0ZcoUORwOtW/fXnl5eZozZ45Zk5aWpvXr12vq1KlasmSJunfvrtdff928dV6Sxo4dqy+//FIlJSXyeDwaOHCgKioqAi6gXrRokSIjI5Wbmyufzyen06lly5YFtUAAAODmFVQQMozL3+qdmpqq6urqKx6nZ8+eV7zraejQodq9e/dlawoKClRQUPCN47GxsSotLVVpaekV5wQAAKyHvzUGAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsK6ggNG/ePH3/+99Xx44dlZiYqDFjxujQoUMBNefOnVN+fr66dOmiDh06KDc3V/X19QE1dXV1ysnJUVxcnBITEzVt2jSdP38+oGbz5s0aNGiQbDabevfurfLy8lbzKS0tVa9evRQbG6vMzEzt2LEj6LkAAADrCioIVVdXKz8/X9u2bZPL5ZLf71d2drYaGhrMmqlTp+q9997T2rVrVV1drSNHjujRRx81x5uampSTk6PGxkZt3bpVK1euVHl5uUpKSsyaw4cPKycnR8OGDVNtba0KCwv1/PPPa+PGjWbN6tWrVVRUpJkzZ2rXrl0aMGCAnE6njh071ua5AAAAa7slmOKKioqA5+Xl5UpMTFRNTY0efPBBnTx5Um+88YZWrVql4cOHS5JWrFihvn37atu2bRoyZIgqKyt14MABvf/++0pKStLAgQM1d+5cvfjii5o1a5ZiYmJUVlamtLQ0LViwQJLUt29fffjhh1q0aJGcTqckaeHChZo4caLGjx8vSSorK9P69eu1fPly/fSnP23TXAAAgLUFFYQudvLkSUlS586dJUk1NTXy+/3Kysoya/r06aMePXrI7XZryJAhcrvd6tevn5KSkswap9OpKVOmaP/+/br33nvldrsDjtFSU1hYKElqbGxUTU2NiouLzfHIyEhlZWXJ7Xa3eS4X8/l88vl85nOv1ytJ8vv98vv9V7VG36TleLZI45r2x9VrWUPWMnzoQfjRg/CjB6EXzFpedRBqbm5WYWGhfvCDH+iee+6RJHk8HsXExCghISGgNikpSR6Px6y5MAS1jLeMXa7G6/Xq7NmzOn78uJqami5Zc/DgwTbP5WLz5s3T7NmzW22vrKxUXFzcNy3FNZk7uPmq9tuwYUOIZ2JdLpcr3FOwPHoQfvQg/OhB6Jw5c6bNtVcdhPLz87Vv3z59+OGHV3uIb53i4mIVFRWZz71er1JTU5WdnS273R7S1/L7/XK5XHppZ6R8zRFB779vljOk87Gilh6MHDlS0dHR4Z6OJdGD8KMH4UcPQq/lE522uKogVFBQoHXr1mnLli3q3r27uT05OVmNjY06ceJEwJmY+vp6JScnmzUX393VcifXhTUX391VX18vu92udu3aKSoqSlFRUZesufAYV5rLxWw2m2w2W6vt0dHR1+2L09ccIV9T8EGIb5bQuZ79RdvQg/CjB+FHD0InmHUM6q4xwzBUUFCgd955R5s2bVJaWlrAeEZGhqKjo1VVVWVuO3TokOrq6uRwOCRJDodDe/fuDbi7y+VyyW63Kz093ay58BgtNS3HiImJUUZGRkBNc3OzqqqqzJq2zAUAAFhbUGeE8vPztWrVKv35z39Wx44dzWtt4uPj1a5dO8XHx2vChAkqKipS586dZbfb9cILL8jhcJgXJ2dnZys9PV1PP/205s+fL4/HoxkzZig/P988GzN58mQtXbpU06dP13PPPadNmzZpzZo1Wr9+vTmXoqIi5eXlafDgwbrvvvu0ePFiNTQ0mHeRtWUuAADA2oIKQq+99pokaejQoQHbV6xYoWeffVaStGjRIkVGRio3N1c+n09Op1PLli0za6OiorRu3TpNmTJFDodD7du3V15enubMmWPWpKWlaf369Zo6daqWLFmi7t276/XXXzdvnZeksWPH6ssvv1RJSYk8Ho8GDhyoioqKgAuorzQXAABgbUEFIcO48q3esbGxKi0tVWlp6TfW9OzZ84p3PQ0dOlS7d+++bE1BQYEKCgquaS4AAMC6+FtjAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsoIOQlu2bNHDDz+slJQURURE6N133w0Yf/bZZxURERHwGDVqVEDN119/rXHjxslutyshIUETJkzQ6dOnA2r27NmjBx54QLGxsUpNTdX8+fNbzWXt2rXq06ePYmNj1a9fP23YsCFg3DAMlZSUqFu3bmrXrp2ysrL0ySefBPuWAQDATSroINTQ0KABAwaotLT0G2tGjRqlo0ePmo8//OEPAePjxo3T/v375XK5tG7dOm3ZskWTJk0yx71er7Kzs9WzZ0/V1NTolVde0axZs/Tb3/7WrNm6dauefPJJTZgwQbt379aYMWM0ZswY7du3z6yZP3++Xn31VZWVlWn79u1q3769nE6nzp07F+zbBgAAN6Fbgt1h9OjRGj169GVrbDabkpOTLzn28ccfq6KiQh999JEGDx4sSfr1r3+thx56SL/61a+UkpKit956S42NjVq+fLliYmJ09913q7a2VgsXLjQD05IlSzRq1ChNmzZNkjR37ly5XC4tXbpUZWVlMgxDixcv1owZM/TII49Ikn7/+98rKSlJ7777rp544olg3zoAALjJBB2E2mLz5s1KTExUp06dNHz4cP385z9Xly5dJElut1sJCQlmCJKkrKwsRUZGavv27frhD38ot9utBx98UDExMWaN0+nUL3/5Sx0/flydOnWS2+1WUVFRwOs6nU7zo7rDhw/L4/EoKyvLHI+Pj1dmZqbcbvclg5DP55PP5zOfe71eSZLf75ff77/2hblAy/FskcY17Y+r17KGrGX40IPwowfhRw9CL5i1DHkQGjVqlB599FGlpaXps88+089+9jONHj1abrdbUVFR8ng8SkxMDJzELbeoc+fO8ng8kiSPx6O0tLSAmqSkJHOsU6dO8ng85rYLay48xoX7XarmYvPmzdPs2bNbba+srFRcXFxblyAocwc3X9V+F18PhavncrnCPQXLowfhRw/Cjx6EzpkzZ9pcG/IgdOGZln79+ql///66/fbbtXnzZo0YMSLULxdSxcXFAWeZvF6vUlNTlZ2dLbvdHtLX8vv9crlcemlnpHzNEUHvv2+WM6TzsaKWHowcOVLR0dHhno4l0YPwowfhRw9Cr+UTnba4Lh+NXei2225T165d9emnn2rEiBFKTk7WsWPHAmrOnz+vr7/+2ryuKDk5WfX19QE1Lc+vVHPheMu2bt26BdQMHDjwknO12Wyy2WyttkdHR1+3L05fc4R8TcEHIb5ZQud69hdtQw/Cjx6EHz0InWDW8br/HqF//etf+uqrr8ww4nA4dOLECdXU1Jg1mzZtUnNzszIzM82aLVu2BHzG53K5dNddd6lTp05mTVVVVcBruVwuORwOSVJaWpqSk5MDarxer7Zv327WAAAAaws6CJ0+fVq1tbWqra2V9J+Lkmtra1VXV6fTp09r2rRp2rZtmz7//HNVVVXpkUceUe/eveV0/uejnL59+2rUqFGaOHGiduzYob///e8qKCjQE088oZSUFEnSU089pZiYGE2YMEH79+/X6tWrtWTJkoCPrX70ox+poqJCCxYs0MGDBzVr1izt3LlTBQUFkqSIiAgVFhbq5z//uf7yl79o7969euaZZ5SSkqIxY8Zc47IBAICbQdAfje3cuVPDhg0zn7eEk7y8PL322mvas2ePVq5cqRMnTiglJUXZ2dmaO3duwEdOb731lgoKCjRixAhFRkYqNzdXr776qjkeHx+vyspK5efnKyMjQ127dlVJSUnA7xr6r//6L61atUozZszQz372M91xxx169913dc8995g106dPV0NDgyZNmqQTJ07o/vvvV0VFhWJjY4N92wAA4CYUdBAaOnSoDOObb/neuHHjFY/RuXNnrVq16rI1/fv319/+9rfL1jz++ON6/PHHv3E8IiJCc+bM0Zw5c644JwAAYD38rTEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZBCEAAGBZQQehLVu26OGHH1ZKSooiIiL07rvvBowbhqGSkhJ169ZN7dq1U1ZWlj755JOAmq+//lrjxo2T3W5XQkKCJkyYoNOnTwfU7NmzRw888IBiY2OVmpqq+fPnt5rL2rVr1adPH8XGxqpfv37asGFD0HMBAADWFXQQamho0IABA1RaWnrJ8fnz5+vVV19VWVmZtm/frvbt28vpdOrcuXNmzbhx47R//365XC6tW7dOW7Zs0aRJk8xxr9er7Oxs9ezZUzU1NXrllVc0a9Ys/fa3vzVrtm7dqieffFITJkzQ7t27NWbMGI0ZM0b79u0Lai4AAMC6bgl2h9GjR2v06NGXHDMMQ4sXL9aMGTP0yCOPSJJ+//vfKykpSe+++66eeOIJffzxx6qoqNBHH32kwYMHS5J+/etf66GHHtKvfvUrpaSk6K233lJjY6OWL1+umJgY3X333aqtrdXChQvNwLRkyRKNGjVK06ZNkyTNnTtXLpdLS5cuVVlZWZvmAgAArC2k1wgdPnxYHo9HWVlZ5rb4+HhlZmbK7XZLktxutxISEswQJElZWVmKjIzU9u3bzZoHH3xQMTExZo3T6dShQ4d0/Phxs+bC12mpaXmdtswFAABYW9BnhC7H4/FIkpKSkgK2JyUlmWMej0eJiYmBk7jlFnXu3DmgJi0trdUxWsY6deokj8dzxde50lwu5vP55PP5zOder1eS5Pf75ff7L/fWg9ZyPFukcU374+q1rCFrGT70IPzoQfjRg9ALZi1DGoS+6+bNm6fZs2e32l5ZWam4uLjr8ppzBzdf1X4XXxiOq+dyucI9BcujB+FHD8KPHoTOmTNn2lwb0iCUnJwsSaqvr1e3bt3M7fX19Ro4cKBZc+zYsYD9zp8/r6+//trcPzk5WfX19QE1Lc+vVHPh+JXmcrHi4mIVFRWZz71er1JTU5WdnS273X7lBQiC3++Xy+XSSzsj5WuOCHr/fbOcIZ2PFbX0YOTIkYqOjg73dCyJHoQfPQg/ehB6LZ/otEVIg1BaWpqSk5NVVVVlhg2v16vt27drypQpkiSHw6ETJ06opqZGGRkZkqRNmzapublZmZmZZs3/+3//T36/3/yicLlcuuuuu9SpUyezpqqqSoWFhebru1wuORyONs/lYjabTTabrdX26Ojo6/bF6WuOkK8p+CDEN0voXM/+om3oQfjRg/CjB6ETzDoGfbH06dOnVVtbq9raWkn/uSi5trZWdXV1ioiIUGFhoX7+85/rL3/5i/bu3atnnnlGKSkpGjNmjCSpb9++GjVqlCZOnKgdO3bo73//uwoKCvTEE08oJSVFkvTUU08pJiZGEyZM0P79+7V69WotWbIk4GzNj370I1VUVGjBggU6ePCgZs2apZ07d6qgoECS2jQXAABgbUGfEdq5c6eGDRtmPm8JJ3l5eSovL9f06dPV0NCgSZMm6cSJE7r//vtVUVGh2NhYc5+33npLBQUFGjFihCIjI5Wbm6tXX33VHI+Pj1dlZaXy8/OVkZGhrl27qqSkJOB3Df3Xf/2XVq1apRkzZuhnP/uZ7rjjDr377ru65557zJq2zAUAAFhX0EFo6NChMoxvvtMpIiJCc+bM0Zw5c76xpnPnzlq1atVlX6d///7629/+dtmaxx9/XI8//vg1zQUAAFgXf2sMAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYFkEIAABYVtB/awzfDr1+uv6q9/385ZwQzgQAgO8uzggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLCnkQmjVrliIiIgIeffr0McfPnTun/Px8denSRR06dFBubq7q6+sDjlFXV6ecnBzFxcUpMTFR06ZN0/nz5wNqNm/erEGDBslms6l3794qLy9vNZfS0lL16tVLsbGxyszM1I4dO0L9dgEAwHfYdTkjdPfdd+vo0aPm48MPPzTHpk6dqvfee09r165VdXW1jhw5okcffdQcb2pqUk5OjhobG7V161atXLlS5eXlKikpMWsOHz6snJwcDRs2TLW1tSosLNTzzz+vjRs3mjWrV69WUVGRZs6cqV27dmnAgAFyOp06duzY9XjLAADgO+i6BKFbbrlFycnJ5qNr166SpJMnT+qNN97QwoULNXz4cGVkZGjFihXaunWrtm3bJkmqrKzUgQMH9Oabb2rgwIEaPXq05s6dq9LSUjU2NkqSysrKlJaWpgULFqhv374qKCjQY489pkWLFplzWLhwoSZOnKjx48crPT1dZWVliouL0/Lly6/HWwYAAN9Bt1yPg37yySdKSUlRbGysHA6H5s2bpx49eqimpkZ+v19ZWVlmbZ8+fdSjRw+53W4NGTJEbrdb/fr1U1JSklnjdDo1ZcoU7d+/X/fee6/cbnfAMVpqCgsLJUmNjY2qqalRcXGxOR4ZGamsrCy53e5vnLfP55PP5zOfe71eSZLf75ff77+mNblYy/FskUZIjxvMa1tdyzqwHuFDD8KPHoQfPQi9YNYy5EEoMzNT5eXluuuuu3T06FHNnj1bDzzwgPbt2yePx6OYmBglJCQE7JOUlCSPxyNJ8ng8ASGoZbxl7HI1Xq9XZ8+e1fHjx9XU1HTJmoMHD37j3OfNm6fZs2e32l5ZWam4uLi2LUCQ5g5uvi7HvZwNGzbc8Nf8NnO5XOGeguXRg/CjB+FHD0LnzJkzba4NeRAaPXq0+d/9+/dXZmamevbsqTVr1qhdu3ahfrmQKi4uVlFRkfnc6/UqNTVV2dnZstvtIX0tv98vl8ull3ZGytccEdJjX8m+Wc4b+nrfVi09GDlypKKjo8M9HUuiB+FHD8KPHoReyyc6bXFdPhq7UEJCgu688059+umnGjlypBobG3XixImAs0L19fVKTk6WJCUnJ7e6u6vlrrILay6+06y+vl52u13t2rVTVFSUoqKiLlnTcoxLsdlsstlsrbZHR0dfty9OX3OEfE03NgjxjRboevYXbUMPwo8ehB89CJ1g1vG6/x6h06dP67PPPlO3bt2UkZGh6OhoVVVVmeOHDh1SXV2dHA6HJMnhcGjv3r0Bd3e5XC7Z7Xalp6ebNRceo6Wm5RgxMTHKyMgIqGlublZVVZVZAwAAEPIg9JOf/ETV1dX6/PPPtXXrVv3whz9UVFSUnnzyScXHx2vChAkqKirSBx98oJqaGo0fP14Oh0NDhgyRJGVnZys9PV1PP/20/vGPf2jjxo2aMWOG8vPzzbM1kydP1j//+U9Nnz5dBw8e1LJly7RmzRpNnTrVnEdRUZF+97vfaeXKlfr44481ZcoUNTQ0aPz48aF+ywAA4Dsq5B+N/etf/9KTTz6pr776Srfeeqvuv/9+bdu2TbfeeqskadGiRYqMjFRubq58Pp+cTqeWLVtm7h8VFaV169ZpypQpcjgcat++vfLy8jRnzhyzJi0tTevXr9fUqVO1ZMkSde/eXa+//rqczv+79mXs2LH68ssvVVJSIo/Ho4EDB6qioqLVBdQAAMC6Qh6E3n777cuOx8bGqrS0VKWlpd9Y07Nnzyve2TR06FDt3r37sjUFBQUqKCi4bA0AALAu/tYYAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwrFvCPQHceL1+uv6q9/385ZwQzgQAgPDijBAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsghAAALAsSwSh0tJS9erVS7GxscrMzNSOHTvCPSUAAPAtcEu4J3C9rV69WkVFRSorK1NmZqYWL14sp9OpQ4cOKTExMdzT+87p9dP1V73v5y/nhHAmAABcu5v+jNDChQs1ceJEjR8/Xunp6SorK1NcXJyWL18e7qkBAIAwu6nPCDU2NqqmpkbFxcXmtsjISGVlZcntdreq9/l88vl85vOTJ09Kkr7++mv5/f6Qzs3v9+vMmTO6xR+ppuaIkB7726r3T9aE5XW3F4+45PaWHnz11VeKjo6+wbOCRA++DehB+NGD0Dt16pQkyTCMK9be1EHo3//+t5qampSUlBSwPSkpSQcPHmxVP2/ePM2ePbvV9rS0tOs2R1x/XReEewYAgHA4deqU4uPjL1tzUwehYBUXF6uoqMh83tzcrK+//lpdunRRRERoz9p4vV6lpqbqiy++kN1uD+mx0Tb0IPzoQfjRg/CjB6FnGIZOnTqllJSUK9be1EGoa9euioqKUn19fcD2+vp6JScnt6q32Wyy2WwB2xISEq7nFGW32/nCDzN6EH70IPzoQfjRg9C60pmgFjf1xdIxMTHKyMhQVVWVua25uVlVVVVyOBxhnBkAAPg2uKnPCElSUVGR8vLyNHjwYN13331avHixGhoaNH78+HBPDQAAhNlNH4TGjh2rL7/8UiUlJfJ4PBo4cKAqKipaXUB9o9lsNs2cObPVR3G4cehB+NGD8KMH4UcPwivCaMu9ZQAAADehm/oaIQAAgMshCAEAAMsiCAEAAMsiCAEAAMsiCIVBaWmpevXqpdjYWGVmZmrHjh3hntJNY9asWYqIiAh49OnTxxw/d+6c8vPz1aVLF3Xo0EG5ubmtfuFmXV2dcnJyFBcXp8TERE2bNk3nz5+/0W/lO2PLli16+OGHlZKSooiICL377rsB44ZhqKSkRN26dVO7du2UlZWlTz75JKDm66+/1rhx42S325WQkKAJEybo9OnTATV79uzRAw88oNjYWKWmpmr+/PnX+619Z1ypB88++2yr74tRo0YF1NCDazNv3jx9//vfV8eOHZWYmKgxY8bo0KFDATWh+vmzefNmDRo0SDabTb1791Z5efn1fns3NYLQDbZ69WoVFRVp5syZ2rVrlwYMGCCn06ljx46Fe2o3jbvvvltHjx41Hx9++KE5NnXqVL333ntau3atqqurdeTIET366KPmeFNTk3JyctTY2KitW7dq5cqVKi8vV0lJSTjeyndCQ0ODBgwYoNLS0kuOz58/X6+++qrKysq0fft2tW/fXk6nU+fOnTNrxo0bp/3798vlcmndunXasmWLJk2aZI57vV5lZ2erZ8+eqqmp0SuvvKJZs2bpt7/97XV/f98FV+qBJI0aNSrg++IPf/hDwDg9uDbV1dXKz8/Xtm3b5HK55Pf7lZ2drYaGBrMmFD9/Dh8+rJycHA0bNky1tbUqLCzU888/r40bN97Q93tTMXBD3XfffUZ+fr75vKmpyUhJSTHmzZsXxlndPGbOnGkMGDDgkmMnTpwwoqOjjbVr15rbPv74Y0OS4Xa7DcMwjA0bNhiRkZGGx+Mxa1577TXDbrcbPp/vus79ZiDJeOedd8znzc3NRnJysvHKK6+Y206cOGHYbDbjD3/4g2EYhnHgwAFDkvHRRx+ZNX/961+NiIgI43//938NwzCMZcuWGZ06dQrowYsvvmjcdddd1/kdffdc3APDMIy8vDzjkUce+cZ96EHoHTt2zJBkVFdXG4YRup8/06dPN+6+++6A1xo7dqzhdDqv91u6aXFG6AZqbGxUTU2NsrKyzG2RkZHKysqS2+0O48xuLp988olSUlJ02223ady4caqrq5Mk1dTUyO/3B6x/nz591KNHD3P93W63+vXrF/ALN51Op7xer/bv339j38hN4PDhw/J4PAFrHh8fr8zMzIA1T0hI0ODBg82arKwsRUZGavv27WbNgw8+qJiYGLPG6XTq0KFDOn78+A16N99tmzdvVmJiou666y5NmTJFX331lTlGD0Lv5MmTkqTOnTtLCt3PH7fbHXCMlhr+Dbl6BKEb6N///reamppa/VbrpKQkeTyeMM3q5pKZmany8nJVVFTotdde0+HDh/XAAw/o1KlT8ng8iomJafWHdC9cf4/Hc8n+tIwhOC1rdrmveY/Ho8TExIDxW265RZ07d6YvITJq1Cj9/ve/V1VVlX75y1+qurpao0ePVlNTkyR6EGrNzc0qLCzUD37wA91zzz2SFLKfP99U4/V6dfbs2evxdm56N/2f2IC1jB492vzv/v37KzMzUz179tSaNWvUrl27MM4MCJ8nnnjC/O9+/fqpf//+uv3227V582aNGDEijDO7OeXn52vfvn0B1yfi24szQjdQ165dFRUV1eougfr6eiUnJ4dpVje3hIQE3Xnnnfr000+VnJysxsZGnThxIqDmwvVPTk6+ZH9axhCcljW73Nd8cnJyq5sFzp8/r6+//pq+XCe33Xabunbtqk8//VQSPQilgoICrVu3Th988IG6d+9ubg/Vz59vqrHb7fyfvatEELqBYmJilJGRoaqqKnNbc3Ozqqqq5HA4wjizm9fp06f12WefqVu3bsrIyFB0dHTA+h86dEh1dXXm+jscDu3duzfgHwWXyyW73a709PQbPv/vurS0NCUnJwesudfr1fbt2wPW/MSJE6qpqTFrNm3apObmZmVmZpo1W7Zskd/vN2tcLpfuuusuderU6Qa9m5vHv/71L3311Vfq1q2bJHoQCoZhqKCgQO+88442bdqktLS0gPFQ/fxxOBwBx2ip4d+QaxDuq7Wt5u233zZsNptRXl5uHDhwwJg0aZKRkJAQcJcArt6Pf/xjY/Pmzcbhw4eNv//970ZWVpbRtWtX49ixY4ZhGMbkyZONHj16GJs2bTJ27txpOBwOw+FwmPufP3/euOeee4zs7GyjtrbWqKioMG699VajuLg4XG/pW+/UqVPG7t27jd27dxuSjIULFxq7d+82/ud//scwDMN4+eWXjYSEBOPPf/6zsWfPHuORRx4x0tLSjLNnz5rHGDVqlHHvvfca27dvNz788EPjjjvuMJ588klz/MSJE0ZSUpLx9NNPG/v27TPefvttIy4uzvjNb35zw9/vt9HlenDq1CnjJz/5ieF2u43Dhw8b77//vjFo0CDjjjvuMM6dO2cegx5cmylTphjx8fHG5s2bjaNHj5qPM2fOmDWh+Pnzz3/+04iLizOmTZtmfPzxx0ZpaakRFRVlVFRU3ND3ezMhCIXBr3/9a6NHjx5GTEyMcd999xnbtm0L95RuGmPHjjW6detmxMTEGN/73veMsWPHGp9++qk5fvbsWeO///u/jU6dOhlxcXHGD3/4Q+Po0aMBx/j888+N0aNHG+3atTO6du1q/PjHPzb8fv+NfivfGR988IEhqdUjLy/PMIz/3EL/0ksvGUlJSYbNZjNGjBhhHDp0KOAYX331lfHkk08aHTp0MOx2uzF+/Hjj1KlTATX/+Mc/jPvvv9+w2WzG9773PePll1++UW/xW+9yPThz5oyRnZ1t3HrrrUZ0dLTRs2dPY+LEia3+zxc9uDaXWn9JxooVK8yaUP38+eCDD4yBAwcaMTExxm233RbwGghehGEYxo0+CwUAAPBtwDVCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsghCAADAsv4/2I1qxbjTQ/MAAAAASUVORK5CYII="},"metadata":{}}]},{"cell_type":"markdown","source":"### Tokenization","metadata":{}},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(Config.MODEL)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-12T11:51:32.281478Z","iopub.execute_input":"2023-05-12T11:51:32.281773Z","iopub.status.idle":"2023-05-12T11:51:33.112847Z","shell.execute_reply.started":"2023-05-12T11:51:32.281748Z","shell.execute_reply":"2023-05-12T11:51:33.111873Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stderr","text":"Downloading (…)okenizer_config.json: 100%|██████████| 28.0/28.0 [00:00<00:00, 10.1kB/s]\nDownloading (…)lve/main/config.json: 100%|██████████| 625/625 [00:00<00:00, 382kB/s]\nDownloading (…)solve/main/vocab.txt: 100%|██████████| 872k/872k [00:00<00:00, 13.9MB/s]\nDownloading (…)/main/tokenizer.json: 100%|██████████| 1.72M/1.72M [00:00<00:00, 97.0MB/s]\n","output_type":"stream"}]},{"cell_type":"code","source":"def encoder(text_data, tokenizer=tokenizer, max_len=Config.MAX_LEN):\n    return tokenizer(text_data.comment_text.values.tolist(), \n                     max_length=max_len, \n                     truncation=True, \n                     padding=\"max_length\",\n                     add_special_tokens=True,\n                     return_tensors=\"tf\",\n                     return_token_type_ids = False)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:33.113957Z","iopub.execute_input":"2023-05-12T11:51:33.114337Z","iopub.status.idle":"2023-05-12T11:51:33.120170Z","shell.execute_reply.started":"2023-05-12T11:51:33.114311Z","shell.execute_reply":"2023-05-12T11:51:33.119390Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"encoded_train = encoder(text_data = train)\nencoded_val = encoder(text_data = val)\nencoded_test = encoder(text_data = test)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:51:33.121147Z","iopub.execute_input":"2023-05-12T11:51:33.121426Z","iopub.status.idle":"2023-05-12T11:52:16.634801Z","shell.execute_reply.started":"2023-05-12T11:51:33.121402Z","shell.execute_reply":"2023-05-12T11:52:16.633568Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"train_dataset = (tf.data.Dataset.from_tensor_slices((dict(encoded_train), train[\"toxic\"]))\n                .repeat()\n                .shuffle(Config.BUFFER_SIZE)\n                .batch(Config.BATCH_SIZE)\n                .prefetch(tf.data.AUTOTUNE))\n\nval_dataset = (tf.data.Dataset.from_tensor_slices((dict(encoded_val), val[\"toxic\"]))\n                .batch(Config.BATCH_SIZE)\n                .prefetch(tf.data.AUTOTUNE))\n\ntest_dataset = tf.data.Dataset.from_tensor_slices(dict(encoded_test)).batch(Config.BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:52:16.636039Z","iopub.execute_input":"2023-05-12T11:52:16.636353Z","iopub.status.idle":"2023-05-12T11:52:16.668330Z","shell.execute_reply.started":"2023-05-12T11:52:16.636312Z","shell.execute_reply":"2023-05-12T11:52:16.667443Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"def model_builder(transformer, max_len=Config.MAX_LEN):\n    input_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_ids\")\n    masks = Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n    \n    bert_layers = transformer.bert(input_ids, attention_mask=masks)[0]\n    \n    \"\"\"intermediate = Dense(1024, activation='relu')(bert_layers)\n    output = Dense(1, activation=\"sigmoid\", name=\"output_layer\")(intermediate)\"\"\"\n    \n    out = GlobalMaxPool1D()(bert_layers)\n    out = Dense(768, activation=\"relu\")(out)\n    out = Dropout(0.1)(out)\n    out = Dense(384, activation=\"relu\")(out)\n    output = Dense(1, activation=\"sigmoid\")(out)\n    model = Model(inputs=[input_ids, masks], outputs=output)\n    model.layers[2].trainable = True\n    \n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=Config.LEARNING_RATE, weight_decay=Config.WEIGHT_DECAY),\n                  loss=tf.keras.losses.BinaryCrossentropy(),\n                  metrics=tf.keras.metrics.AUC())\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:52:16.669447Z","iopub.execute_input":"2023-05-12T11:52:16.669741Z","iopub.status.idle":"2023-05-12T11:52:16.679253Z","shell.execute_reply.started":"2023-05-12T11:52:16.669715Z","shell.execute_reply":"2023-05-12T11:52:16.678393Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"with tpu_strategy.scope():\n    transformer = TFAutoModel.from_pretrained(Config.MODEL)\n    model = model_builder(transformer=transformer)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:52:16.680310Z","iopub.execute_input":"2023-05-12T11:52:16.680659Z","iopub.status.idle":"2023-05-12T11:52:51.834986Z","shell.execute_reply.started":"2023-05-12T11:52:16.680636Z","shell.execute_reply":"2023-05-12T11:52:51.833917Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stderr","text":"Downloading tf_model.h5: 100%|██████████| 999M/999M [00:10<00:00, 99.5MB/s] \nSome layers from the model checkpoint at bert-base-multilingual-uncased were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']\n- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nAll the layers of TFBertModel were initialized from the model checkpoint at bert-base-multilingual-uncased.\nIf your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n","output_type":"stream"}]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:52:51.836335Z","iopub.execute_input":"2023-05-12T11:52:51.836733Z","iopub.status.idle":"2023-05-12T11:52:51.881506Z","shell.execute_reply.started":"2023-05-12T11:52:51.836705Z","shell.execute_reply":"2023-05-12T11:52:51.880546Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"Model: \"model\"\n__________________________________________________________________________________________________\n Layer (type)                   Output Shape         Param #     Connected to                     \n==================================================================================================\n input_ids (InputLayer)         [(None, 192)]        0           []                               \n                                                                                                  \n attention_mask (InputLayer)    [(None, 192)]        0           []                               \n                                                                                                  \n bert (TFBertMainLayer)         TFBaseModelOutputWi  167356416   ['input_ids[0][0]',              \n                                thPoolingAndCrossAt               'attention_mask[0][0]']         \n                                tentions(last_hidde                                               \n                                n_state=(None, 192,                                               \n                                 768),                                                            \n                                 pooler_output=(Non                                               \n                                e, 768),                                                          \n                                 past_key_values=No                                               \n                                ne, hidden_states=N                                               \n                                one, attentions=Non                                               \n                                e, cross_attentions                                               \n                                =None)                                                            \n                                                                                                  \n global_max_pooling1d (GlobalMa  (None, 768)         0           ['bert[0][0]']                   \n xPooling1D)                                                                                      \n                                                                                                  \n dense (Dense)                  (None, 768)          590592      ['global_max_pooling1d[0][0]']   \n                                                                                                  \n dropout_37 (Dropout)           (None, 768)          0           ['dense[0][0]']                  \n                                                                                                  \n dense_1 (Dense)                (None, 384)          295296      ['dropout_37[0][0]']             \n                                                                                                  \n dense_2 (Dense)                (None, 1)            385         ['dense_1[0][0]']                \n                                                                                                  \n==================================================================================================\nTotal params: 168,242,689\nTrainable params: 168,242,689\nNon-trainable params: 0\n__________________________________________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"train_steps_per_epoch = train.shape[0]//Config.BATCH_SIZE\n\nhistory=model.fit(train_dataset,\n                  validation_data=val_dataset,\n                  steps_per_epoch=train_steps_per_epoch,\n                  epochs=Config.EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:52:51.882653Z","iopub.execute_input":"2023-05-12T11:52:51.883060Z","iopub.status.idle":"2023-05-12T12:13:12.713842Z","shell.execute_reply.started":"2023-05-12T11:52:51.883032Z","shell.execute_reply":"2023-05-12T12:13:12.712319Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"Epoch 1/2\nWARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n","output_type":"stream"},{"name":"stderr","text":"WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n","output_type":"stream"},{"name":"stdout","text":"4185/4185 [==============================] - ETA: 0s - loss: 0.0486 - auc: 0.9973","output_type":"stream"},{"name":"stderr","text":"2023-05-12 12:03:44.473468: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] model_pruner failed: INVALID_ARGUMENT: Graph does not contain terminal node Add/ReadVariableOp.\n2023-05-12 12:03:44.751296: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] model_pruner failed: INVALID_ARGUMENT: Graph does not contain terminal node Add/ReadVariableOp.\n","output_type":"stream"},{"name":"stdout","text":"4185/4185 [==============================] - 666s 136ms/step - loss: 0.0486 - auc: 0.9973 - val_loss: 0.6711 - val_auc: 0.7589\nEpoch 2/2\n4185/4185 [==============================] - 553s 132ms/step - loss: 0.0420 - auc: 0.9980 - val_loss: 0.6677 - val_auc: 0.7927\n","output_type":"stream"}]},{"cell_type":"code","source":"model.evaluate(val_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:14:40.714960Z","iopub.execute_input":"2023-05-12T12:14:40.715901Z","iopub.status.idle":"2023-05-12T12:14:44.186043Z","shell.execute_reply.started":"2023-05-12T12:14:40.715867Z","shell.execute_reply":"2023-05-12T12:14:44.184917Z"},"trusted":true},"execution_count":21,"outputs":[{"name":"stdout","text":"63/63 [==============================] - 3s 39ms/step - loss: 0.1322 - auc: 0.9915\n","output_type":"stream"},{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"[0.1321573555469513, 0.9915268421173096]"},"metadata":{}}]},{"cell_type":"code","source":"val_steps_per_epoch = val.shape[0]//Config.BATCH_SIZE\nval_history=model.fit(val_dataset.repeat(),\n                  steps_per_epoch=val_steps_per_epoch,\n                  epochs=2)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:13:29.464786Z","iopub.execute_input":"2023-05-12T12:13:29.465239Z","iopub.status.idle":"2023-05-12T12:14:32.640293Z","shell.execute_reply.started":"2023-05-12T12:13:29.465206Z","shell.execute_reply":"2023-05-12T12:14:32.639052Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stdout","text":"Epoch 1/2\n62/62 [==============================] - 8s 131ms/step - loss: 0.3363 - auc: 0.8208\nEpoch 2/2\n62/62 [==============================] - 54s 131ms/step - loss: 0.1986 - auc: 0.9492\n","output_type":"stream"}]},{"cell_type":"code","source":"preds = model.predict(test_dataset)\nsub['toxic'] = preds\nsub.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:14:57.376394Z","iopub.execute_input":"2023-05-12T12:14:57.377409Z","iopub.status.idle":"2023-05-12T12:15:33.710506Z","shell.execute_reply.started":"2023-05-12T12:14:57.377371Z","shell.execute_reply":"2023-05-12T12:15:33.709250Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stderr","text":"2023-05-12 12:15:02.970518: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] model_pruner failed: INVALID_ARGUMENT: Graph does not contain terminal node AssignAddVariableOp.\n2023-05-12 12:15:03.224869: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] model_pruner failed: INVALID_ARGUMENT: Graph does not contain terminal node AssignAddVariableOp.\n","output_type":"stream"},{"name":"stdout","text":"499/499 [==============================] - 36s 49ms/step\n","output_type":"stream"}]},{"cell_type":"code","source":"model.save(\"mbert-fine-tuned-2-hiddenstates\")","metadata":{"execution":{"iopub.status.busy":"2023-05-12T10:44:30.231587Z","iopub.execute_input":"2023-05-12T10:44:30.231921Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stderr","text":"WARNING:absl:Found untraced functions such as _update_step_xla, encoder_layer_call_fn, encoder_layer_call_and_return_conditional_losses, pooler_layer_call_fn, pooler_layer_call_and_return_conditional_losses while saving (showing 5 of 829). These functions will not be directly callable after loading.\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Pushing Model to Hugging Face","metadata":{}},{"cell_type":"code","source":"!huggingface-cli login --token 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","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from huggingface_hub import push_to_hub_keras\npush_to_hub_keras(model, 'Multilingual-Toxic-Comment-Roberta')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading model from Hub","metadata":{}},{"cell_type":"code","source":"from huggingface_hub import from_pretrained_keras\nm = from_pretrained_keras('shivansh-ka/Multilingual-Toxic-Comment-Roberta')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = m.predict(test_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['toxic'] = preds\nsub.to_csv(\"submission.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}