refactoring de requirements.txt

api.py

@@ -1,51 +0,0 @@
-import uvicorn
-from fastapi import FastAPI, Form, Request
-from fastapi.staticfiles import StaticFiles
-from fastapi.templating import Jinja2Templates
-
-from inference import inferenceAPI
-
-
-# ------ MODELE --------------------------------------------------------------
-# appel de la fonction inference, adaptee pour une entree txt
-def summarize(text: str):
-    return " ".join(inferenceAPI(text))
-
-
-# ----------------------------------------------------------------------------------
-
-
-# -------- API ---------------------------------------------------------------------
-app = FastAPI()
-
-# static files pour envoi du css au navigateur
-templates = Jinja2Templates(directory="templates")
-app.mount("/", StaticFiles(directory="templates", html=True), name="templates")
-
-
-@app.get("/")
-async def index(request: Request):
-    return templates.TemplateResponse("index.html.jinja", {"request": request})
-
-
-# retourner le texte, les predictions et message d'erreur si formulaire envoye vide
-@app.post("/")
-async def prediction(request: Request, text: str = Form(None)):
-    if not text:
-        error = "Merci de saisir votre texte."
-        return templates.TemplateResponse(
-            "index.html.jinja", {"request": request, "text": error}
-        )
-    else:
-        summary = summarize(text)
-        return templates.TemplateResponse(
-            "index.html.jinja", {"request": request, "text": text, "summary": summary}
-        )
-
-
-# ------------------------------------------------------------------------------------
-
-
-# lancer le serveur et le recharger a chaque modification sauvegardee
-# if __name__ == "__main__":
-#     uvicorn.run("api:app", port=8000, reload=True)

requirements.txt

@@ -1,56 +1,14 @@
-absl-py==1.4.0
-aiohttp==3.8.4
-aiosignal==1.3.1
-alembic==1.9.4
 anyascii==0.3.1
 anyio==3.6.2
-async-timeout==4.0.2
-attrs==22.2.0
-banal==1.0.6
-blis==0.7.9
-catalogue==2.0.8
-certifi==2022.12.7
-charset-normalizer==3.0.1
-click==8.1.3
-confection==0.0.4
-contourpy==1.0.7
-contractions==0.1.73
-cycler==0.11.0
-cymem==2.0.7
-dataloader==2.0
-dataset==1.6.0
-datasets==2.10.1
-dill==0.3.6
-en-core-web-lg==3.5.0
-evaluate==0.4.0
-fastapi==0.91.0
-filelock==3.9.0
-flake8==6.0.0
-fonttools==4.38.0
-frozenlist==1.3.3
-fsspec==2023.3.0
-greenlet==2.0.2
-h11==0.14.0
-huggingface-hub==0.12.1
 certifi==2022.12.7
 charset-normalizer==3.1.0
-click==8.1.3
-fastapi==0.92.0
+contractions==0.1.73
+fastapi==0.94.0
 filelock==3.9.0
+huggingface-hub==0.13.2
 idna==3.4
-importlib-metadata==6.0.0
-importlib-resources==5.12.0
 Jinja2==3.1.2
-joblib==1.2.0
-kiwisolver==1.4.4
-langcodes==3.3.0
-Mako==1.2.4
 MarkupSafe==2.1.2
-matplotlib==3.7.0
-mccabe==0.7.0
-multidict==6.0.4
-multiprocess==0.70.14
-murmurhash==1.0.9
 numpy==1.24.2
 nvidia-cublas-cu11==11.10.3.66
 nvidia-cuda-nvrtc-cu11==11.7.99
@@ -58,56 +16,21 @@ nvidia-cuda-runtime-cu11==11.7.99
 nvidia-cudnn-cu11==8.5.0.96
 packaging==23.0
 pandas==1.5.3
-pathy==0.10.1
-Pillow==9.4.0
-preshed==3.0.8
-protobuf==3.20.0
 pyahocorasick==2.0.0
-pyarrow==11.0.0
-pycodestyle==2.10.0
-pydantic==1.10.4
-pyflakes==3.0.1
-pyparsing==3.0.9
+pydantic==1.10.6
 python-dateutil==2.8.2
-python-multipart==0.0.5
+python-multipart==0.0.6
 pytz==2022.7.1
 PyYAML==6.0
 regex==2022.10.31
 requests==2.28.2
-responses==0.18.0
-rouge-score==0.1.2
-scikit-learn==1.2.1
-scipy==1.10.0
-sentencepiece==0.1.97
 six==1.16.0
-smart-open==6.3.0
 sniffio==1.3.0
-spacy==3.5.0
-spacy-legacy==3.0.12
-spacy-loggers==1.0.4
-SQLAlchemy==1.4.46
-srsly==2.4.5
-starlette==0.24.0
-summarizer==0.0.7
+starlette==0.26.1
 textsearch==0.0.24
-thinc==8.1.7
-threadpoolctl==3.1.0
-tokenizers==0.13.2
-tomli==2.0.1
-torch==1.13.1
-tqdm==4.64.1
-transformers==4.26.1
-typer==0.7.0
-typing-extensions==4.4.0
-urllib3==1.26.14
-starlette==0.25.0
 tokenizers==0.13.2
 torch==1.13.1
 tqdm==4.65.0
+transformers==4.26.1
 typing_extensions==4.5.0
 urllib3==1.26.15
-uvicorn==0.20.0
-wasabi==1.1.1
-xxhash==3.2.0
-yarl==1.8.2
-zipp==3.14.0

src/api.py

@@ -1,31 +1,30 @@
-import uvicorn
 from fastapi import FastAPI, Form, Request
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-import re
 
-from src.inference import inferenceAPI
-from src.inference_t5 import inferenceAPI_t5
+from inference_lstm import inferenceAPI
+from inference_t5 import inferenceAPI
 
 
 # ------ INFERENCE MODEL --------------------------------------------------------------
 # appel de la fonction inference, adaptee pour une entree txt
 def summarize(text: str):
-    if choisir_modele.var == 'lstm' :
+    if choisir_modele.var == "lstm":
         return " ".join(inferenceAPI(text))
     elif choisir_modele.var == "fineTunedT5":
         text = inferenceAPI_t5(text)
 
+
 # ----------------------------------------------------------------------------------
 
 
 def choisir_modele(choixModele):
     print("ON A RECUP LE CHOIX MODELE")
-    if choixModele == "lstm" :
-        choisir_modele.var ='lstm'
+    if choixModele == "lstm":
+        choisir_modele.var = "lstm"
     elif choixModele == "fineTunedT5":
         choisir_modele.var = "fineTunedT5"
-    else :
+    else:
         "le modele n'est pas defini"
 
 
@@ -41,29 +40,29 @@ app.mount("/templates", StaticFiles(directory="templates"), name="templates")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
+
 @app.get("/model")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
+
 @app.get("/predict")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
 
 @app.post("/model")
-async def choix_model(request: Request, choixModel:str = Form(None)):
+async def choix_model(request: Request, choixModel: str = Form(None)):
     print(choixModel)
     if not choixModel:
         erreur_modele = "Merci de saisir un modèle."
         return templates.TemplateResponse(
-        "index.html.jinja", {"request": request, "text": erreur_modele}
+            "index.html.jinja", {"request": request, "text": erreur_modele}
         )
-    else :
+    else:
         choisir_modele(choixModel)
         print("C'est bon on utilise le modèle demandé")
-        return templates.TemplateResponse(
-        "index.html.jinja", {"request": request}
-        )
+        return templates.TemplateResponse("index.html.jinja", {"request": request})
 
 
 # retourner le texte, les predictions et message d'erreur si formulaire envoye vide

src/dataloader.py

@@ -52,10 +52,15 @@ class Data(torch.utils.data.Dataset):
 
     def __getitem__(self, idx):
         row = self.data.iloc[idx]
-        text = row["text"].translate(str.maketrans("", "", string.punctuation)).split()
+        text = row["text"].translate(
+            str.maketrans(
+                "", "", string.punctuation)).split()
         summary = (
-            row["summary"].translate(str.maketrans("", "", string.punctuation)).split()
-        )
+            row["summary"].translate(
+                str.maketrans(
+                    "",
+                    "",
+                    string.punctuation)).split())
         summary = ["<start>", *summary, "<end>"]
         sample = {"text": text, "summary": summary}
 
@@ -106,7 +111,8 @@ class Data(torch.utils.data.Dataset):
             tokenized_texts.append(text)
 
         if text_type == "summary":
-            return [["<start>", *summary, "<end>"] for summary in tokenized_texts]
+            return [["<start>", *summary, "<end>"]
+                    for summary in tokenized_texts]
         return tokenized_texts
 
     def get_words(self) -> list:
@@ -157,8 +163,10 @@ class Vectoriser:
 
     def __init__(self, vocab=None) -> None:
         self.vocab = vocab
-        self.word_count = Counter(word.lower().strip(",.\\-") for word in self.vocab)
-        self.idx_to_token = sorted([t for t, c in self.word_count.items() if c > 1])
+        self.word_count = Counter(word.lower().strip(",.\\-")
+                                  for word in self.vocab)
+        self.idx_to_token = sorted(
+            [t for t, c in self.word_count.items() if c > 1])
         self.token_to_idx = {t: i for i, t in enumerate(self.idx_to_token)}
 
     def load(self, path):
@@ -167,7 +175,8 @@ class Vectoriser:
             self.word_count = Counter(
                 word.lower().strip(",.\\-") for word in self.vocab
             )
-            self.idx_to_token = sorted([t for t, c in self.word_count.items() if c > 1])
+            self.idx_to_token = sorted(
+                [t for t, c in self.word_count.items() if c > 1])
             self.token_to_idx = {t: i for i, t in enumerate(self.idx_to_token)}
 
     def save(self, path):

src/fine_tune_T5.py

@@ -1,49 +1,55 @@
-import re
 import os
+import re
 import string
+
 import contractions
-import torch
 import datasets
-from datasets import Dataset
+import evaluate
 import pandas as pd
+import torch
+from datasets import Dataset
 from tqdm import tqdm
-import evaluate
-from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, AutoConfig
-from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer
-from transformers import DataCollatorForSeq2Seq
+from transformers import (AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer,
+                          DataCollatorForSeq2Seq, Seq2SeqTrainer,
+                          Seq2SeqTrainingArguments)
 
 
 def clean_text(texts):
-    '''This fonction makes clean text for the future use'''
+    """This fonction makes clean text for the future use"""
     texts = texts.lower()
     texts = contractions.fix(texts)
     texts = texts.translate(str.maketrans("", "", string.punctuation))
-    texts = re.sub(r'\n', ' ', texts)
+    texts = re.sub(r"\n", " ", texts)
     return texts
 
 
 def datasetmaker(path=str):
-    '''This fonction take the jsonl file, read it to a dataframe,
-     remove the colums not needed for the task and turn it into a file type Dataset
-    '''
+    """This fonction take the jsonl file, read it to a dataframe,
+    remove the colums not needed for the task and turn it into a file type Dataset
+    """
     data = pd.read_json(path, lines=True)
-    df = data.drop(['url',
-                    'archive',
-                    'title',
-                    'date',
-                    'compression',
-                    'coverage',
-                    'density',
-                    'compression_bin',
-                    'coverage_bin',
-                    'density_bin'],
-                   axis=1)
+    df = data.drop(
+        [
+            "url",
+            "archive",
+            "title",
+            "date",
+            "compression",
+            "coverage",
+            "density",
+            "compression_bin",
+            "coverage_bin",
+            "density_bin",
+        ],
+        axis=1,
+    )
     tqdm.pandas()
-    df['text'] = df.text.apply(lambda texts: clean_text(texts))
-    df['summary'] = df.summary.apply(lambda summary: clean_text(summary))
+    df["text"] = df.text.apply(lambda texts: clean_text(texts))
+    df["summary"] = df.summary.apply(lambda summary: clean_text(summary))
     dataset = Dataset.from_dict(df)
     return dataset
 
+
 # voir si le model par hasard esr déjà bien
 
 # test_text = dataset['text'][0]
@@ -60,20 +66,33 @@ def generate_batch_sized_chunks(list_elements, batch_size):
         yield list_elements[i: i + batch_size]
 
 
-def calculate_metric(dataset, metric, model, tokenizer,
-                     batch_size, device,
-                     column_text='text',
-                     column_summary='summary'):
+def calculate_metric(
+    dataset,
+    metric,
+    model,
+    tokenizer,
+    batch_size,
+    device,
+    column_text="text",
+    column_summary="summary",
+):
     article_batches = list(
-        str(generate_batch_sized_chunks(dataset[column_text], batch_size)))
+        str(generate_batch_sized_chunks(dataset[column_text], batch_size))
+    )
     target_batches = list(
-        str(generate_batch_sized_chunks(dataset[column_summary], batch_size)))
+        str(generate_batch_sized_chunks(dataset[column_summary], batch_size))
+    )
 
     for article_batch, target_batch in tqdm(
-            zip(article_batches, target_batches), total=len(article_batches)):
-
-        inputs = tokenizer(article_batch, max_length=1024, truncation=True,
-                           padding="max_length", return_tensors="pt")
+        zip(article_batches, target_batches), total=len(article_batches)
+    ):
+        inputs = tokenizer(
+            article_batch,
+            max_length=1024,
+            truncation=True,
+            padding="max_length",
+            return_tensors="pt",
+        )
         # parameter for length penalty ensures that the model does not
         # generate sequences that are too long.
         summaries = model.generate(
@@ -81,16 +100,18 @@ def calculate_metric(dataset, metric, model, tokenizer,
             attention_mask=inputs["attention_mask"].to(device),
             length_penalty=0.8,
             num_beams=8,
-            max_length=128)
+            max_length=128,
+        )
 
         # Décode les textes
         # renplacer les tokens, ajouter des textes décodés avec les rédéfences
         # vers la métrique.
         decoded_summaries = [
             tokenizer.decode(
-                s,
-                skip_special_tokens=True,
-                clean_up_tokenization_spaces=True) for s in summaries]
+                s, skip_special_tokens=True, clean_up_tokenization_spaces=True
+            )
+            for s in summaries
+        ]
 
         decoded_summaries = [d.replace("", " ") for d in decoded_summaries]
 
@@ -100,59 +121,60 @@ def calculate_metric(dataset, metric, model, tokenizer,
 
     # compute et return les ROUGE scores.
     results = metric.compute()
-    rouge_names = ['rouge1', 'rouge2', 'rougeL', 'rougeLsum']
+    rouge_names = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
     rouge_dict = dict((rn, results[rn]) for rn in rouge_names)
-    return pd.DataFrame(rouge_dict, index=['T5'])
+    return pd.DataFrame(rouge_dict, index=["T5"])
 
 
 def convert_ex_to_features(example_batch):
-    input_encodings = tokenizer(example_batch['text'],
-                                max_length=1024, truncation=True)
+    input_encodings = tokenizer(
+        example_batch["text"],
+        max_length=1024,
+        truncation=True)
 
     labels = tokenizer(
-        example_batch['summary'],
+        example_batch["summary"],
         max_length=128,
         truncation=True)
 
     return {
-        'input_ids': input_encodings['input_ids'],
-        'attention_mask': input_encodings['attention_mask'],
-        'labels': labels['input_ids']
+        "input_ids": input_encodings["input_ids"],
+        "attention_mask": input_encodings["attention_mask"],
+        "labels": labels["input_ids"],
     }
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
+    train_dataset = datasetmaker("data/train_extract.jsonl")
 
-    train_dataset = datasetmaker('data/train_extract.jsonl')
+    dev_dataset = datasetmaker("data/dev_extract.jsonl")
 
-    dev_dataset = datasetmaker('data/dev_extract.jsonl')
+    test_dataset = datasetmaker("data/test_extract.jsonl")
 
-    test_dataset = datasetmaker('data/test_extract.jsonl')
-
-    dataset = datasets.DatasetDict({'train': train_dataset,
-                                    'dev': dev_dataset, 'test': test_dataset})
+    dataset = datasets.DatasetDict(
+        {"train": train_dataset, "dev": dev_dataset, "test": test_dataset}
+    )
 
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-    tokenizer = AutoTokenizer.from_pretrained('google/mt5-small')
+    tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
     mt5_config = AutoConfig.from_pretrained(
-        'google/mt5-small',
+        "google/mt5-small",
         max_length=128,
         length_penalty=0.6,
         no_repeat_ngram_size=2,
         num_beams=15,
     )
-    model = (AutoModelForSeq2SeqLM
-             .from_pretrained('google/mt5-small', config=mt5_config)
-             .to(device))
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        "google/mt5-small", config=mt5_config
+    ).to(device)
 
     dataset_pt = dataset.map(
         convert_ex_to_features,
-        remove_columns=[
-            "summary",
-            "text"],
+        remove_columns=["summary", "text"],
         batched=True,
-        batch_size=128)
+        batch_size=128,
+    )
 
     data_collator = DataCollatorForSeq2Seq(
         tokenizer, model=model, return_tensors="pt")
@@ -182,8 +204,8 @@ if __name__ == '__main__':
         args=training_args,
         data_collator=data_collator,
         # compute_metrics = calculate_metric,
-        train_dataset=dataset_pt['train'],
-        eval_dataset=dataset_pt['dev'].select(range(10)),
+        train_dataset=dataset_pt["train"],
+        eval_dataset=dataset_pt["dev"].select(range(10)),
         tokenizer=tokenizer,
     )
 
@@ -197,8 +219,9 @@ if __name__ == '__main__':
         tokenizer,
         batch_size=2,
         device=device,
-        column_text='text',
-        column_summary='summary')
+        column_text="text",
+        column_summary="summary",
+    )
     print(score)
 
     # Fine Tuning terminés et à sauvgarder
@@ -211,9 +234,7 @@ if __name__ == '__main__':
         trainer.model.save_pretrained("t5_summary")
     tokenizer.save_pretrained("t5_summary")
     # load local model
-    model = (AutoModelForSeq2SeqLM
-             .from_pretrained("t5_summary")
-             .to(device))
+    model = AutoModelForSeq2SeqLM.from_pretrained("t5_summary").to(device)
 
     # mettre en usage : TEST
 

src/fine_tune_t5.py

@@ -1,204 +0,0 @@
-import torch
-import datasets
-from datasets import Dataset, DatasetDict
-import pandas as pd
-from tqdm import tqdm
-import re
-import os
-import nltk
-import string
-import contractions
-from transformers import pipeline
-import evaluate
-from transformers import AutoModelForSeq2SeqLM, AutoTokenizer,AutoConfig
-from transformers import Seq2SeqTrainingArguments ,Seq2SeqTrainer
-from transformers import DataCollatorForSeq2Seq
-
-# cuda out of memory
-os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:200"
-
-nltk.download('stopwords')
-nltk.download('punkt')
-
-
-def clean_data(texts):
-    texts = texts.lower()
-    texts = contractions.fix(texts)
-    texts = texts.translate(str.maketrans("", "", string.punctuation))
-    texts = re.sub(r'\n',' ',texts)
-    return texts
-
-def datasetmaker (path=str):
-    data = pd.read_json(path, lines=True)
-    df = data.drop(['url','archive','title','date','compression','coverage','density','compression_bin','coverage_bin','density_bin'],axis=1)
-    tqdm.pandas()
-    df['text'] = df.text.apply(lambda texts : clean_data(texts))
-    df['summary'] = df.summary.apply(lambda summary : clean_data(summary))
-    # df['text'] = df['text'].map(str)
-    # df['summary'] = df['summary'].map(str)
-    dataset = Dataset.from_dict(df)
-    return dataset
-
-#voir si le model par hasard esr déjà bien
-
-# test_text = dataset['text'][0]
-# pipe = pipeline('summarization',model = model_ckpt)
-# pipe_out = pipe(test_text)
-# print (pipe_out[0]['summary_text'].replace('.<n>','.\n'))
-# print(dataset['summary'][0])
-
-def generate_batch_sized_chunks(list_elements, batch_size):
-    """split the dataset into smaller batches that we can process simultaneously
-    Yield successive batch-sized chunks from list_of_elements."""
-    for i in range(0, len(list_elements), batch_size):
-        yield list_elements[i : i + batch_size]
-
-def calculate_metric(dataset, metric, model, tokenizer,
-                               batch_size, device,
-                               column_text='text',
-                               column_summary='summary'):
-    article_batches = list(str(generate_batch_sized_chunks(dataset[column_text], batch_size)))
-    target_batches = list(str(generate_batch_sized_chunks(dataset[column_summary], batch_size)))
-
-    for article_batch, target_batch in tqdm(
-        zip(article_batches, target_batches), total=len(article_batches)):
-
-        inputs = tokenizer(article_batch, max_length=1024,  truncation=True,
-                        padding="max_length", return_tensors="pt")
-
-        summaries = model.generate(input_ids=inputs["input_ids"].to(device),
-                         attention_mask=inputs["attention_mask"].to(device),
-                         length_penalty=0.8, num_beams=8, max_length=128)
-        ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''
-
-        # Décode les textes
-        # renplacer les tokens, ajouter des textes décodés avec les rédéfences vers la métrique.
-        decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,
-                                clean_up_tokenization_spaces=True)
-               for s in summaries]
-
-        decoded_summaries = [d.replace("", " ") for d in decoded_summaries]
-
-
-        metric.add_batch(predictions=decoded_summaries, references=target_batch)
-
-    #compute et return les ROUGE scores.
-    results = metric.compute()
-    rouge_names = ['rouge1','rouge2','rougeL','rougeLsum']
-    rouge_dict = dict((rn, results[rn] ) for rn in rouge_names )
-    return pd.DataFrame(rouge_dict, index = ['T5'])
-
-
-def convert_ex_to_features(example_batch):
-    input_encodings = tokenizer(example_batch['text'],max_length = 1024,truncation = True)
-
-    labels =tokenizer(example_batch['summary'], max_length = 128, truncation = True )
-
-    return {
-        'input_ids' : input_encodings['input_ids'],
-        'attention_mask': input_encodings['attention_mask'],
-        'labels': labels['input_ids']
-    }
-
-if __name__=='__main__':
-
-    train_dataset = datasetmaker('data/train_extract_100.jsonl')
-
-    dev_dataset = datasetmaker('data/dev_extract_100.jsonl')
-
-    test_dataset = datasetmaker('data/test_extract_100.jsonl')
-
-    dataset = datasets.DatasetDict({'train':train_dataset,'dev':dev_dataset ,'test':test_dataset})
-
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-    tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
-    mt5_config = AutoConfig.from_pretrained(
-    "google/mt5-small",
-    max_length=128,
-    length_penalty=0.6,
-    no_repeat_ngram_size=2,
-    num_beams=15,
-    )
-    model = (AutoModelForSeq2SeqLM
-            .from_pretrained("google/mt5-small", config=mt5_config)
-            .to(device))
-
-    dataset_pt= dataset.map(convert_ex_to_features,remove_columns=["summary", "text"],batched = True,batch_size=128)
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model,return_tensors="pt")
-
-
-    training_args = Seq2SeqTrainingArguments(
-        output_dir = "mt5_sum",
-        log_level = "error",
-        num_train_epochs = 10,
-        learning_rate = 5e-4,
-        #   lr_scheduler_type = "linear",
-        warmup_steps = 0,
-        optim = "adafactor",
-        weight_decay = 0.01,
-        per_device_train_batch_size = 2,
-        per_device_eval_batch_size = 1,
-        gradient_accumulation_steps = 16,
-        evaluation_strategy = "steps",
-        eval_steps = 100,
-        predict_with_generate=True,
-        generation_max_length = 128,
-        save_steps = 500,
-        logging_steps = 10,
-        # push_to_hub = True
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model = model,
-        args = training_args,
-        data_collator = data_collator,
-        # compute_metrics = calculate_metric,
-        train_dataset=dataset_pt['train'],
-        eval_dataset=dataset_pt['dev'].select(range(10)),
-        tokenizer = tokenizer,
-    )
-
-    trainer.train()
-    rouge_metric = evaluate.load("rouge")
-
-    score = calculate_metric(test_dataset, rouge_metric, trainer.model, tokenizer,
-                                batch_size=2, device=device,
-                                column_text='text',
-                                column_summary='summary')
-    print (score)
-
-
-    #Fine Tuning terminés et à sauvgarder
-
-
-
-    # save fine-tuned model in local
-    os.makedirs("./summarization_t5", exist_ok=True)
-    if hasattr(trainer.model, "module"):
-        trainer.model.module.save_pretrained("./summarization_t5")
-    else:
-        trainer.model.save_pretrained("./summarization_t5")
-    tokenizer.save_pretrained("./summarization_t5")
-    # load local model
-    model = (AutoModelForSeq2SeqLM
-            .from_pretrained("./summarization_t5")
-            .to(device))
-
-
-    # mettre en usage : TEST
-
-
-    # gen_kwargs = {"length_penalty": 0.8, "num_beams":8, "max_length": 128}
-    # sample_text = dataset["test"][0]["text"]
-    # reference = dataset["test"][0]["summary"]
-    # pipe = pipeline("summarization", model='./summarization_t5')
-
-    # print("Text:")
-    # print(sample_text)
-    # print("\nReference Summary:")
-    # print(reference)
-    # print("\nModel Summary:")
-    # print(pipe(sample_text, **gen_kwargs)[0]["summary_text"])

src/inference.py

@@ -1,56 +0,0 @@
-"""
- Allows to predict the summary for a given entry text
-"""
-import pickle
-
-import torch
-
-from src import dataloader
-from src.model import Decoder, Encoder, EncoderDecoderModel
-
-with open("model/vocab.pkl", "rb") as vocab:
-    words = pickle.load(vocab)
-vectoriser = dataloader.Vectoriser(words)
-
-
-def inferenceAPI(text: str) -> str:
-    """
-    Predict the summary for an input text
-    --------
-    Parameter
-        text: str
-            the text to sumarize
-    Return
-        str
-            The summary for the input text
-    """
-    text = text.split()
-    # On défini les paramètres d'entrée pour le modèle
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
-    encoder.to(device)
-    decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
-    decoder.to(device)
-
-    # On instancie le modèle
-    model = EncoderDecoderModel(encoder, decoder, vectoriser, device)
-
-    # model.load_state_dict(torch.load("model/model.pt", map_location=device))
-    # model.eval()
-    # model.to(device)
-
-    # On vectorise le texte
-    source = vectoriser.encode(text)
-    source = source.to(device)
-
-    # On fait passer le texte dans le modèle
-    with torch.no_grad():
-        output = model(source).to(device)
-        output.to(device)
-        output = output.argmax(dim=-1)
-    return vectoriser.decode(output)
-
-
-# if __name__ == "__main__":
-#     # inference()
-#     print(inferenceAPI("If you choose to use these attributes in logged messages, you need to exercise some care. In the above example, for instance, the Formatter has been set up with a format string which expects ‘clientip’ and ‘user’ in the attribute dictionary of the LogRecord. If these are missing, the message will not be logged because a string formatting exception will occur. So in this case, you always need to pass the extra dictionary with these keys."))

src/inference_t5.py

@@ -1,10 +1,11 @@
 """
  Allows to predict the summary for a given entry text
 """
-import torch
-import contractions
 import re
 import string
+
+import contractions
+import torch
 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
 
@@ -12,7 +13,7 @@ def clean_text(texts: str) -> str:
     texts = texts.lower()
     texts = contractions.fix(texts)
     texts = texts.translate(str.maketrans("", "", string.punctuation))
-    texts = re.sub(r'\n', ' ', texts)
+    texts = re.sub(r"\n", " ", texts)
     return texts
 
 
@@ -31,32 +32,32 @@ def inferenceAPI(text: str) -> str:
     # On défini les paramètres d'entrée pour le modèle
     text = clean_text(text)
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    tokenizer = (AutoTokenizer.from_pretrained("Linggg/t5_summary"))
+    tokenizer = AutoTokenizer.from_pretrained("Linggg/t5_summary")
     # load local model
-    model = (AutoModelForSeq2SeqLM
-             .from_pretrained("Linggg/t5_summary")
-             .to(device))
+    model = AutoModelForSeq2SeqLM.from_pretrained("Linggg/t5_summary").to(device)
 
     text_encoding = tokenizer(
         text,
         max_length=1024,
-        padding='max_length',
+        padding="max_length",
         truncation=True,
         return_attention_mask=True,
         add_special_tokens=True,
-        return_tensors='pt'
+        return_tensors="pt",
     )
     generated_ids = model.generate(
-        input_ids=text_encoding['input_ids'],
-        attention_mask=text_encoding['attention_mask'],
+        input_ids=text_encoding["input_ids"],
+        attention_mask=text_encoding["attention_mask"],
         max_length=128,
         num_beams=8,
         length_penalty=0.8,
-        early_stopping=True
+        early_stopping=True,
     )
 
     preds = [
-        tokenizer.decode(gen_id, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        tokenizer.decode(
+            gen_id, skip_special_tokens=True, clean_up_tokenization_spaces=True
+        )
         for gen_id in generated_ids
     ]
     return "".join(preds)

src/model.py

@@ -25,7 +25,8 @@ class Encoder(torch.nn.Module):
         # on s'en servira pour les mots inconnus
         self.embeddings = torch.nn.Embedding(vocab_size, embeddings_dim)
         self.embeddings.to(device)
-        self.hidden = torch.nn.LSTM(embeddings_dim, hidden_size, dropout=dropout)
+        self.hidden = torch.nn.LSTM(
+            embeddings_dim, hidden_size, dropout=dropout)
         # Comme on va calculer la log-vraisemblance,
         # c'est le log-softmax qui nous intéresse
         self.dropout = torch.nn.Dropout(dropout)
@@ -61,7 +62,8 @@ class Decoder(torch.nn.Module):
         # on s'en servira pour les mots inconnus
         self.vocab_size = vocab_size
         self.embeddings = torch.nn.Embedding(vocab_size, embeddings_dim)
-        self.hidden = torch.nn.LSTM(embeddings_dim, hidden_size, dropout=dropout)
+        self.hidden = torch.nn.LSTM(
+            embeddings_dim, hidden_size, dropout=dropout)
         self.output = torch.nn.Linear(hidden_size, vocab_size)
         # Comme on va calculer la log-vraisemblance,
         # c'est le log-softmax qui nous intéresse
@@ -100,32 +102,36 @@ class EncoderDecoderModel(torch.nn.Module):
         # The ratio must be inferior to 1 to allow text compression
         assert summary_len < 1, f"number lesser than 1 expected, got {summary_len}"
 
-        target_len = int(
-            summary_len * source.shape[0]
-        )  # Expected summary length (in words)
-        target_vocab_size = self.decoder.vocab_size  # Word Embedding length
+        # Expected summary length (in words)
+        target_len = int(summary_len * source.shape[0])
+        # Word Embedding length
+        target_vocab_size = self.decoder.vocab_size
 
-        # Output of the right format (expected summmary length x word embedding length)
-        # filled with zeros. On each iteration, we will replace one of the row of this
-        # matrix with the choosen word embedding
+        # Output of the right format (expected summmary length x word
+        # embedding length) filled with zeros. On each iteration, we
+        # will replace one of the row of this matrix with the choosen
+        # word embedding
         outputs = torch.zeros(target_len, target_vocab_size)
 
-        # put the tensors on the device (useless if CPU bus very useful in case of GPU)
+        # put the tensors on the device (useless if CPU bus very useful in
+        # case of GPU)
         outputs.to(self.device)
         source.to(self.device)
 
-        # last hidden state of the encoder is used as the initial hidden state of the decoder
-        hidden, cell = self.encoder(source)  # Encode the input text
-        input = self.vectoriser.encode(
-            "<start>"
-        )  # Encode the first word of the summary
+        # last hidden state of the encoder is used
+        # as the initial hidden state of the decoder
+
+        # Encode the input text
+        hidden, cell = self.encoder(source)
+        # Encode the first word of the summary
+        input = self.vectoriser.encode("<start>")
 
         # put the tensors on the device
         hidden.to(self.device)
         cell.to(self.device)
         input.to(self.device)
 
-        ### BEAM SEARCH ###
+        #   BEAM SEARCH   #
         # If you wonder, b stands for better
         values = None
         b_outputs = torch.zeros(target_len, target_vocab_size).to(self.device)
@@ -134,14 +140,16 @@ class EncoderDecoderModel(torch.nn.Module):
         for i in range(1, target_len):
             # On va déterminer autant de mot que la taille du texte souhaité
             # insert input token embedding, previous hidden and previous cell states
-            # receive output tensor (predictions) and new hidden and cell states.
+            # receive output tensor (predictions) and new hidden and cell
+            # states.
 
             # replace predictions in a tensor holding predictions for each token
             # logging.debug(f"output : {output}")
 
             ####### DÉBUT DU BEAM SEARCH ##########
             if values is None:
-                # On calcule une première fois les premières probabilité de mot après <start>
+                # On calcule une première fois les premières probabilité de mot
+                # après <start>
                 output, hidden, cell = self.decoder(input, hidden, cell)
                 output.to(self.device)
                 b_hidden = hidden
@@ -152,7 +160,8 @@ class EncoderDecoderModel(torch.nn.Module):
                 values, indices = output.topk(num_beams, sorted=True)
 
             else:
-                # On instancie le dictionnaire qui contiendra les scores pour chaque possibilité
+                # On instancie le dictionnaire qui contiendra les scores pour
+                # chaque possibilité
                 scores = {}
 
                 # Pour chacune des meilleures valeurs, on va calculer l'output
@@ -160,7 +169,8 @@ class EncoderDecoderModel(torch.nn.Module):
                     indice.to(self.device)
 
                     # On calcule l'output
-                    b_output, b_hidden, b_cell = self.decoder(indice, b_hidden, b_cell)
+                    b_output, b_hidden, b_cell = self.decoder(
+                        indice, b_hidden, b_cell)
 
                     # On empêche le modèle de se répéter d'un mot sur l'autre en mettant
                     # de force la probabilité du mot précédent à 0
@@ -179,7 +189,8 @@ class EncoderDecoderModel(torch.nn.Module):
                 # Et du coup on rempli la place de i-1 à la place de i
                 b_outputs[i - 1] = b_output.to(self.device)
 
-                # On instancies nos nouvelles valeurs pour la prochaine itération
+                # On instancies nos nouvelles valeurs pour la prochaine
+                # itération
                 values, indices = b_output.topk(num_beams, sorted=True)
 
             ##################################

src/train.py

@@ -150,16 +150,24 @@ if __name__ == "__main__":
     words = train_dataset.get_words()
     vectoriser = dataloader.Vectoriser(words)
 
-    train_dataset = dataloader.Data("data/train_extract.jsonl", transform=vectoriser)
-    dev_dataset = dataloader.Data("data/dev_extract.jsonl", transform=vectoriser)
+    train_dataset = dataloader.Data(
+        "data/train_extract.jsonl",
+        transform=vectoriser)
+    dev_dataset = dataloader.Data(
+        "data/dev_extract.jsonl",
+        transform=vectoriser)
 
     train_dataloader = torch.utils.data.DataLoader(
-        train_dataset, batch_size=2, shuffle=True, collate_fn=dataloader.pad_collate
-    )
+        train_dataset,
+        batch_size=2,
+        shuffle=True,
+        collate_fn=dataloader.pad_collate)
 
     dev_dataloader = torch.utils.data.DataLoader(
-        dev_dataset, batch_size=4, shuffle=True, collate_fn=dataloader.pad_collate
-    )
+        dev_dataset,
+        batch_size=4,
+        shuffle=True,
+        collate_fn=dataloader.pad_collate)
 
     for i_batch, batch in enumerate(train_dataloader):
         print(i_batch, batch[0], batch[1])
@@ -169,7 +177,8 @@ if __name__ == "__main__":
     print("Device check. You are using:", device)
 
     ### RÉSEAU ENTRAÎNÉ ###
-    # Pour s'assurer que les résultats seront les mêmes à chaque run du notebook
+    # Pour s'assurer que les résultats seront les mêmes à chaque run du
+    # notebook
     torch.use_deterministic_algorithms(True)
     torch.manual_seed(0)
     random.seed(0)
@@ -178,9 +187,8 @@ if __name__ == "__main__":
     encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
     decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
 
-    trained_classifier = EncoderDecoderModel(encoder, decoder, vectoriser, device).to(
-        device
-    )
+    trained_classifier = EncoderDecoderModel(
+        encoder, decoder, vectoriser, device).to(device)
 
     print(next(trained_classifier.parameters()).device)
     # print(train_dataset.is_cuda)
@@ -194,7 +202,6 @@ if __name__ == "__main__":
 
     torch.save(trained_classifier.state_dict(), "model/model.pt")
     vectoriser.save("model/vocab.pkl")
-    trained_classifier.push_to_hub("SummaryProject-LSTM")
 
     print(f"test summary : {vectoriser.decode(dev_dataset[6][1])}")
     print(