Upload Optimisation_Model (3).ipynb

model gpt 2 fine tuné avec un jeu de données sur des tweets parlant du covid

Optimisation_Model (3).ipynb

@@ -0,0 +1,619 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "-C4hOJzui0GC",
+        "outputId": "53b258f1-0b19-4c1e-95a2-1c45e053be4d"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Mounted at /content/drive\n"
+          ]
+        }
+      ],
+      "source": [
+        "from google.colab import drive\n",
+        "drive.mount('/content/drive')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "C9_nGmZplkwn",
+        "outputId": "27ea388c-1391-466e-e7f9-1155c9b1880b"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m465.3/465.3 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m950.8/950.8 kB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m21.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+            "tensorflow 2.15.0 requires keras<2.16,>=2.15.0, but you have keras 3.0.5 which is incompatible.\u001b[0m\u001b[31m\n",
+            "\u001b[0mCollecting clean-text\n",
+            "  Downloading clean_text-0.6.0-py3-none-any.whl (11 kB)\n",
+            "Collecting emoji<2.0.0,>=1.0.0 (from clean-text)\n",
+            "  Downloading emoji-1.7.0.tar.gz (175 kB)\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m175.4/175.4 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "Collecting ftfy<7.0,>=6.0 (from clean-text)\n",
+            "  Downloading ftfy-6.1.3-py3-none-any.whl (53 kB)\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.4/53.4 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hRequirement already satisfied: wcwidth<0.3.0,>=0.2.12 in /usr/local/lib/python3.10/dist-packages (from ftfy<7.0,>=6.0->clean-text) (0.2.13)\n",
+            "Building wheels for collected packages: emoji\n",
+            "  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for emoji: filename=emoji-1.7.0-py3-none-any.whl size=171033 sha256=0223548bdb25e799ce902bbd67d1aca1a2f166851e765cdbd400528ff496aa31\n",
+            "  Stored in directory: /root/.cache/pip/wheels/31/8a/8c/315c9e5d7773f74b33d5ed33f075b49c6eaeb7cedbb86e2cf8\n",
+            "Successfully built emoji\n",
+            "Installing collected packages: emoji, ftfy, clean-text\n",
+            "Successfully installed clean-text-0.6.0 emoji-1.7.0 ftfy-6.1.3\n",
+            "Collecting unicode\n",
+            "  Downloading unicode-2.9-py2.py3-none-any.whl (14 kB)\n",
+            "Installing collected packages: unicode\n",
+            "Successfully installed unicode-2.9\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install -q --upgrade keras-nlp\n",
+        "!pip install -q --upgrade keras  # Upgrade to Keras 3.\n",
+        "!pip install clean-text\n",
+        "!pip install unicode\n",
+        "# !pip install keras==2.15.0\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {
+        "id": "Ik6OXLR8lkzt"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"  # or \"tensorflow\" or \"torch\"\n",
+        "\n",
+        "import keras_nlp\n",
+        "import keras\n",
+        "\n",
+        "# améliore la rapidité d'entrainement\n",
+        "keras.mixed_precision.set_global_policy(\"mixed_float16\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "U3-rKDHxtFxS",
+        "outputId": "64ce0f1f-1563-41f8-ddda-dba344b6a16c"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "<class 'pandas.core.frame.DataFrame'>\n",
+            "RangeIndex: 44955 entries, 0 to 44954\n",
+            "Data columns (total 1 columns):\n",
+            " #   Column         Non-Null Count  Dtype \n",
+            "---  ------         --------------  ----- \n",
+            " 0   OriginalTweet  44955 non-null  object\n",
+            "dtypes: object(1)\n",
+            "memory usage: 351.3+ KB\n"
+          ]
+        }
+      ],
+      "source": [
+        "import pandas as pd\n",
+        "\n",
+        "df = pd.concat([\n",
+        "    pd.read_csv('/content/drive/MyDrive/Corona_NLP_test.csv')[['OriginalTweet']],\n",
+        "    pd.read_csv('/content/drive/MyDrive/Corona_NLP_train.csv', encoding='latin-1', on_bad_lines='skip')[['OriginalTweet']]\n",
+        "\n",
+        "])\n",
+        "df.reset_index(inplace=True, drop=True)\n",
+        "df.info()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vxunnbfhFKqj"
+      },
+      "source": [
+        "# **Pré-traitement**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "P3g_M1s_tGqj",
+        "outputId": "8f22765c-ce40-42a4-d860-80155e732acb"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "0\n",
+            "Tous les éléments sont uniques.\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(df['OriginalTweet'].isnull().sum())\n",
+        "df.dropna(subset=['OriginalTweet'], how='any', inplace=True)\n",
+        "df.reset_index(inplace=True, drop=True)\n",
+        "\n",
+        "unique_elements = not df['OriginalTweet'].duplicated().any()\n",
+        "\n",
+        "if unique_elements:\n",
+        "    print(\"Tous les éléments sont uniques.\")\n",
+        "else:\n",
+        "    print(\"Certains éléments ne sont pas uniques.\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "id": "RbFJC39ytGsZ",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "b820a9b7-2622-498f-ed29-56309e781749"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "WARNING:root:Since the GPL-licensed package `unidecode` is not installed, using Python's `unicodedata` package which yields worse results.\n"
+          ]
+        }
+      ],
+      "source": [
+        "import cleantext\n",
+        "import re\n",
+        "# corrige les caractères unicode mal formaté\n",
+        "# converti les caracteres non ASCII en leur équivalent exemple : é devient e\n",
+        "\n",
+        "df['OriginalTweet'] = df['OriginalTweet'].apply(lambda x: cleantext.clean(x,\n",
+        "                                                                         fix_unicode=True,\n",
+        "                                                                         to_ascii=True,\n",
+        "                                                                         lower=True,\n",
+        "                                                                         no_emails=True,\n",
+        "                                                                         no_urls=True,\n",
+        "                                                                         ))\n",
+        "df['OriginalTweet'] = df['OriginalTweet'].str.replace('<url>', '')\n",
+        "# traitement des character speciaux\n",
+        "df['OriginalTweet'] = df['OriginalTweet'].apply(lambda x: re.sub(r'\\W', ' ', x))\n",
+        "# on enleve les espaces en trop\n",
+        "df['OriginalTweet'] = df['OriginalTweet'].str.replace(r'\\s{2,}', ' ', regex=True)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bH-ODHW4tKvu"
+      },
+      "outputs": [],
+      "source": [
+        "df.info()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 40,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 385
+        },
+        "id": "Vb3n637UlqjX",
+        "outputId": "73c15e07-6bc7-49be-da1d-72a4ea49bb77"
+      },
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "\u001b[1mPreprocessor: \"gpt2_causal_lm_preprocessor_8\"\u001b[0m\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Preprocessor: \"gpt2_causal_lm_preprocessor_8\"</span>\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+              "┃\u001b[1m \u001b[0m\u001b[1mTokenizer (type)                                  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m                                            Vocab #\u001b[0m\u001b[1m \u001b[0m┃\n",
+              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+              "│ gpt2_tokenizer (\u001b[38;5;33mGPT2Tokenizer\u001b[0m)                     │                                              \u001b[38;5;34m50,257\u001b[0m │\n",
+              "└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+              "┃<span style=\"font-weight: bold\"> Tokenizer (type)                                   </span>┃<span style=\"font-weight: bold\">                                             Vocab # </span>┃\n",
+              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+              "│ gpt2_tokenizer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GPT2Tokenizer</span>)                     │                                              <span style=\"color: #00af00; text-decoration-color: #00af00\">50,257</span> │\n",
+              "└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "\u001b[1mModel: \"gpt2_causal_lm_8\"\u001b[0m\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"gpt2_causal_lm_8\"</span>\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape             \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to              \u001b[0m\u001b[1m \u001b[0m┃\n",
+              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+              "│ padding_mask (\u001b[38;5;33mInputLayer\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n",
+              "├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n",
+              "│ token_ids (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n",
+              "├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n",
+              "│ gpt2_backbone (\u001b[38;5;33mGPT2Backbone\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1280\u001b[0m)        │     \u001b[38;5;34m774,030,080\u001b[0m │ padding_mask[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],        │\n",
+              "│                               │                           │                 │ token_ids[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]            │\n",
+              "├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n",
+              "│ token_embedding               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m50257\u001b[0m)       │      \u001b[38;5;34m64,328,960\u001b[0m │ gpt2_backbone[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]        │\n",
+              "│ (\u001b[38;5;33mReversibleEmbedding\u001b[0m)         │                           │                 │                            │\n",
+              "└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+              "┃<span style=\"font-weight: bold\"> Layer (type)                  </span>┃<span style=\"font-weight: bold\"> Output Shape              </span>┃<span style=\"font-weight: bold\">         Param # </span>┃<span style=\"font-weight: bold\"> Connected to               </span>┃\n",
+              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+              "│ padding_mask (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n",
+              "├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n",
+              "│ token_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n",
+              "├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n",
+              "│ gpt2_backbone (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GPT2Backbone</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1280</span>)        │     <span style=\"color: #00af00; text-decoration-color: #00af00\">774,030,080</span> │ padding_mask[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],        │\n",
+              "│                               │                           │                 │ token_ids[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]            │\n",
+              "├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n",
+              "│ token_embedding               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">50257</span>)       │      <span style=\"color: #00af00; text-decoration-color: #00af00\">64,328,960</span> │ gpt2_backbone[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]        │\n",
+              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">ReversibleEmbedding</span>)         │                           │                 │                            │\n",
+              "└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m774,030,080\u001b[0m (2.88 GB)\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">774,030,080</span> (2.88 GB)\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m774,030,080\u001b[0m (2.88 GB)\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">774,030,080</span> (2.88 GB)\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
+            ],
+            "text/html": [
+              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
+              "</pre>\n"
+            ]
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "# initialisation pour l'utilisation du modèle GPT-2\n",
+        "import pandas as pd\n",
+        "# from tensorflow.keras.preprocessing.text import Tokenizer\n",
+        "from keras_nlp.models import GPT2CausalLMPreprocessor, GPT2CausalLM\n",
+        "\n",
+        "preprocessor = GPT2CausalLMPreprocessor.from_preset(\n",
+        "    \"gpt2_large_en\",\n",
+        "    sequence_length=128,\n",
+        ")\n",
+        "\n",
+        "gpt2_lm = GPT2CausalLM.from_preset(\n",
+        "    \"gpt2_large_en\",\n",
+        "    preprocessor=preprocessor\n",
+        ")\n",
+        "\n",
+        "gpt2_lm.summary()\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PtHNonkjlqlV"
+      },
+      "outputs": [],
+      "source": [
+        "print(df)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "d08uzWyPlqpJ",
+        "outputId": "18f3b943-5267-4a33-9391-e5f106fcd857"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "['trending new yorkers encounter empty supermarket shelves pictured wegmans in brooklyn sold out online grocers foodkick maxdelivery as coronavirus fearing shoppers stock up ', 'when i couldn t find hand sanitizer at fred meyer i turned to amazon but 114 97 for a 2 pack of purell check out how coronavirus concerns are driving up prices ', 'find out how you can protect yourself and loved ones from coronavirus ', ' panic buying hits newyork city as anxious shoppers stock up on food medical supplies after healthcare worker in her 30s becomes bigapple 1st confirmed coronavirus patient or a bloomberg staged event qanon qanon2018 qanon2020 election2020 cdc ', ' toiletpaper dunnypaper coronavirus coronavirusaustralia coronavirusupdate covid_19 9news corvid19 7newsmelb dunnypapergate costco one week everyone buying baby milk powder the next everyone buying up toilet paper ', 'do you remember the last time you paid 2 99 a gallon for regular gas in los angeles prices at the pump are going down a look at how the coronavirus is impacting prices 4pm abc7 ', 'voting in the age of coronavirus hand sanitizer supertuesday ', ' drtedros we can t stop covid19 without protecting healthworkers prices of surgical masks have increased six fold n95 respirators have more than trebled gowns cost twice as much drtedros coronavirus', 'hi twitter i am a pharmacist i sell hand sanitizer for a living or i do when any exists like masks it is sold the fuck out everywhere should you be worried no use soap should you visit twenty pharmacies looking for the last bottle no pharmacies are full of sick people ', 'anyone been in a supermarket over the last few days went to do my normal shop last night is the sight that greeted me barmy btw what s so special about tinned tomatoes covid_19 dublin ']\n"
+          ]
+        }
+      ],
+      "source": [
+        "text_df = list(df['OriginalTweet'].astype(str).values)\n",
+        "print(text_df[:10])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 41,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "dawoMENVFphM",
+        "outputId": "48232747-b2d8-4ea3-f148-fa7f7f1ad7b8"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Epoch 1/5\n",
+            "\u001b[1m1405/1405\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1236s\u001b[0m 568ms/step - accuracy: 0.3385 - loss: 1.2210\n",
+            "Epoch 2/5\n",
+            "\u001b[1m1405/1405\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m425s\u001b[0m 302ms/step - accuracy: 0.4055 - loss: 1.0146\n",
+            "Epoch 3/5\n",
+            "\u001b[1m1405/1405\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m430s\u001b[0m 305ms/step - accuracy: 0.4726 - loss: 0.8486\n",
+            "Epoch 4/5\n",
+            "\u001b[1m1405/1405\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m430s\u001b[0m 305ms/step - accuracy: 0.5486 - loss: 0.6875\n",
+            "Epoch 5/5\n",
+            "\u001b[1m1405/1405\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m431s\u001b[0m 306ms/step - accuracy: 0.6267 - loss: 0.5447\n"
+          ]
+        }
+      ],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "\n",
+        "nb_epochs = 5\n",
+        "with tf.device('/device:GPU:0'):\n",
+        "\n",
+        "    learning_rate = keras.optimizers.schedules.ExponentialDecay(\n",
+        "    initial_learning_rate=5e-5,\n",
+        "    decay_steps=len(text_df) * nb_epochs,\n",
+        "    decay_rate=0.96,\n",
+        "    staircase=True\n",
+        "    )\n",
+        "\n",
+        "    # l'otpmisation PolynomialDecay peut être utilisé à la place de Exponential\n",
+        "    # Decay (la précision n'est pas mauvaise)\n",
+        "\n",
+        "    # initial_learning_rate peut être mis à 1e-3\n",
+        "    # learning_rate = keras.optimizers.schedules.PolynomialDecay(\n",
+        "    #     initial_learning_rate=5e-5,\n",
+        "    #     decay_steps=len(text_df) * nb_epochs,\n",
+        "    #     end_learning_rate=0.0,\n",
+        "    # )\n",
+        "\n",
+        "    # on va crée un callbacks dans le cas ou l'entrainement devient régréssant\n",
+        "    early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=nb_epochs, restore_best_weights=True)\n",
+        "\n",
+        "    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
+        "    gpt2_lm.compile(\n",
+        "        optimizer=keras.optimizers.Adam(learning_rate=learning_rate),\n",
+        "        loss=loss,\n",
+        "        weighted_metrics=[\"accuracy\"],\n",
+        "    )\n",
+        "\n",
+        "    gpt2_lm.fit(\n",
+        "        x=text_df,\n",
+        "        epochs=nb_epochs,\n",
+        "        # batch_size = 32,\n",
+        "        callbacks=[early_stopping]\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Sauvegarde du modèle\n",
+        "\n",
+        "On sauvegarde le model pour éviter de relancer toutes les cellules."
+      ],
+      "metadata": {
+        "id": "fT70Qhsf3fCh"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from tensorflow.keras.models import load_model, save_model\n",
+        "\n",
+        "save_model(gpt2_lm, '/content/drive/MyDrive/my_model.keras')"
+      ],
+      "metadata": {
+        "id": "Hki3ab9jIowZ"
+      },
+      "execution_count": 44,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Réutilisation d'un modèle sauvegardé"
+      ],
+      "metadata": {
+        "id": "7wRf7dmvI9OV"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "gpt2_lm = load_model('/content/drive/MyDrive/my_model.keras')"
+      ],
+      "metadata": {
+        "id": "vBMqnFGGJUZg"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 43,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "33qOkv9FlqrE",
+        "outputId": "607200e4-94ab-4557-91e2-25f5d7494b2f"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\n",
+            "GPT-2 output:\n",
+            "The symptoms of the covid 19 virus can be so severe that a healthcare worker may be too scared to come into work to make money the government needs to step in and save these workers now \n",
+            "___________________________________________________________\n",
+            "\n",
+            "GPT-2 output:\n",
+            "The symptoms of the covid19 coronavirus vary from person to person i have observed that when i have left my house to go to the supermarket the streets have been full of people i have been told that it has been like this every day this week\n",
+            "___________________________________________________________\n",
+            "\n",
+            "GPT-2 output:\n",
+            "The symptoms of the covid2019 covid_19 pandemic vary from person to person and may include fever coughing chest congestion shortness of breath muscle aches and pains in various parts of your body learn more about what to look for in your next health exam \n",
+            "___________________________________________________________\n",
+            "\n",
+            "GPT-2 output:\n",
+            "The symptoms of the covid2019 coronavirus vary from person to person but the virus is generally passed in contact with someone who has it sneezing or talking back coughing into a tissue or using a cart or a shopping cart \n",
+            "___________________________________________________________\n",
+            "\n",
+            "GPT-2 output:\n",
+            "The symptoms of the covid19 coronavirus are typically mild and self contained adults and children should not be encouraged to engage in activities or go to the beach children under 18 are advised to stay at home \n",
+            "___________________________________________________________\n"
+          ]
+        }
+      ],
+      "source": [
+        "# model gpt-2 de base\n",
+        "for i in range(5):\n",
+        "# Ne pas laisser d'espace avant le \" ça un impacte énorme !\n",
+        "  seed_text = \"The symptoms of the covid\"\n",
+        "  # on peu rajouter le paramètre max_length si on veut\n",
+        "  generated_text = gpt2_lm.generate(seed_text, max_length=100)\n",
+        "  print(\"\\nGPT-2 output:\")\n",
+        "  print(generated_text)\n",
+        "  print('___________________________________________________________')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mMGwwyCa6tnt"
+      },
+      "outputs": [],
+      "source": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "K50lvyaJ6tc6"
+      },
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "A100",
+      "provenance": [],
+      "machine_shape": "hm"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}