Repository: ceb10n/fastapi-translations-management

Issue:
Refactor cli to isolate the docs analisis

Discussion:

Patch:
diff --git a/fastapi_translations/cli.py b/fastapi_translations/cli.py
index 06e5331..3169b6a 100644
--- a/fastapi_translations/cli.py
+++ b/fastapi_translations/cli.py
@@ -1,58 +1,15 @@
-import fnmatch
-import os
-from enum import Enum, unique
-from pathlib import Path
 from typing import Annotated
 
 import typer
 from rich.console import Console
-from rich.progress import track
 
 from fastapi_translations import printer
-from fastapi_translations.git import FastAPIGitDocs
-from fastapi_translations.models import DocFile, Summary
+from fastapi_translations.translations import Languages, Summary
 
 console = Console()
 
 app = typer.Typer(rich_markup_mode="rich")
 
-restricted_list = [
-    "*reference/*",
-    "*release-notes.md",
-    "*fastapi-people.md",
-    "*external-links.md",
-    "*newsletter.md",
-    "*management-tasks.md",
-    "*management.md",
-    "*contributing.md",
-]
-
-@unique
-class Languages(Enum):
-    az = "az"
-    bn = "bn"
-    de = "de"
-    em = "em"
-    es = "es"
-    fa = "fa"
-    fr = "fr"
-    he = "he"
-    hu = "hu"
-    id = "id"
-    it = "it"
-    ja = "ja"
-    ko = "ko"
-    pl = "pl"
-    pt = "pt"
-    ru = "ru"
-    tr = "tr"
-    uk = "uk"
-    ur = "ur"
-    vi = "vi"
-    yo = "yo"
-    zh = "zh"
-    zh_hant = "zh-hant"
-
 
 @app.command("report")
 def report(
@@ -60,77 +17,30 @@ def report(
         Languages,
         typer.Option(
             ...,
-            "--language", "-l",
-            help="The language to check for translations report"
+            "--language",
+            "-l",
+            help="The language to check for translations report",
         ),
     ],
     save_csv: Annotated[
         bool,
         typer.Option(
-            "--csv", "-c",
-            help="Save all missing and outdated translations to a csv file"
-        )
-    ] = False
+            "--csv",
+            "-c",
+            help="Save all missing and outdated translations to a csv file",
+        ),
+    ] = False,
 ) -> None:
     """Generate a report for the translated docs"""
-    base_docs_path = Path("docs")
-    en_docs_path = Path("docs/en")
-
-    summary = Summary(lang=lang.value)
     console.clear()
-    git = FastAPIGitDocs()
-
-    for root, _, files in track(
-        os.walk(en_docs_path),
-        "🚶 Walking through 📂 docs looking for 🔠 translations",
-    ):
-        for file in files:
-            if file.endswith(".md"):
-                file_relative_path = os.path.relpath(
-                    os.path.join(root, file), en_docs_path
-                )
-                if file_in_reserved_list(file_relative_path):
-                    continue
-
-                translated_path = os.path.join(
-                    base_docs_path, lang.value, file_relative_path
-                )
-                translation_exists = os.path.exists(translated_path)
-
-                original_doc_date = git.get_commit_date_for(
-                    os.path.join(root, file)
-                )
-                translation_is_outdated = False
-                translated_date = git.get_commit_date_for(translated_path)
-
-                if translated_date and original_doc_date:
-                    translation_is_outdated = translated_date > original_doc_date
+    summary = Summary(lang=lang.value)
+    summary.generate()
 
-                doc = DocFile(
-                    translation_lang=lang.value,
-                    original_file=os.path.join(root, file),
-                    original_commit=original_doc_date,
-                    translation_file=translated_path,
-                    translation_exists=translation_exists,
-                    translation_commit=translated_date,
-                    translation_is_outdated=translation_is_outdated
-                )
-                summary.append_file(doc)
     printer.print_table(summary, console, 10)
 
     if save_csv:
         printer.print_to_csv(summary)
 
 
-def file_in_reserved_list(file: str) -> bool:
-    for reserved in restricted_list:
-        if fnmatch.fnmatch(file, reserved):
-            print(f"{file} matches {reserved}")
-            return True
-
-    print(f"{file} doesn't match {reserved}")
-    return False
-
-
 def main() -> None:
     app()
diff --git a/fastapi_translations/docs.py b/fastapi_translations/docs.py
new file mode 100644
index 0000000..3d83c8a
--- /dev/null
+++ b/fastapi_translations/docs.py
@@ -0,0 +1,10 @@
+restricted_list = [
+    "*reference/*",
+    "*release-notes.md",
+    "*fastapi-people.md",
+    "*external-links.md",
+    "*newsletter.md",
+    "*management-tasks.md",
+    "*management.md",
+    "*contributing.md",
+]
diff --git a/fastapi_translations/models.py b/fastapi_translations/models.py
deleted file mode 100644
index 119e77a..0000000
--- a/fastapi_translations/models.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from datetime import datetime
-
-from pydantic import BaseModel, computed_field
-
-
-class DocFile(BaseModel):
-    official_lang: str = "en"
-    translation_lang: str
-    original_file: str
-    original_commit: datetime | None
-    translation_file: str | None = None
-    translation_exists: bool
-    translation_commit: datetime | None
-    translation_is_outdated: bool
-
-
-class Summary(BaseModel):
-    lang: str
-    files_analyzed: int = 0
-    files_translated: int = 0
-    files_outdated: int = 0
-    files_missing_translation: int = 0
-    files: list[DocFile] = []
-
-    @computed_field  # type: ignore
-    @property
-    def percentage_translated(self) -> float:
-        try:
-            return (
-                100 * float(self.files_translated) / float(self.files_analyzed)
-            )
-        except Exception:
-            return 0.0
-
-    @computed_field  # type: ignore
-    @property
-    def percentage_missing_translation(self) -> float:
-        try:
-            return (
-                100
-                * float(self.files_missing_translation)
-                / float(self.files_analyzed)
-            )
-        except Exception:
-            return 0.0
-
-    @computed_field  # type: ignore
-    @property
-    def percentage_outdated_translation(self) -> float:
-        try:
-            return 100 * float(self.files_outdated) / float(self.files_analyzed)
-        except Exception:
-            return 0.0
-
-    def append_file(self, doc: DocFile) -> None:
-        self.files.append(doc)
-        self.files_analyzed += 1
-
-        if doc.translation_exists:
-            self.files_translated += 1
-
-        if not doc.translation_exists:
-            self.files_missing_translation += 1
-
-        if doc.translation_is_outdated:
-            self.files_outdated += 1
-
-    def first_outdated_files(self, lenght: int = 10) -> list[DocFile]:
-        return list(filter(lambda d: d.translation_is_outdated, self.files))[
-            :lenght
-        ]
-
-    def first_missing_translation_files(
-        self, lenght: int = 10
-    ) -> list[DocFile]:
-        return list(filter(lambda d: not d.translation_exists, self.files))[
-            :lenght
-        ]
diff --git a/fastapi_translations/printer.py b/fastapi_translations/printer.py
index 606cade..6a67cd4 100644
--- a/fastapi_translations/printer.py
+++ b/fastapi_translations/printer.py
@@ -3,7 +3,7 @@ import csv
 from rich.console import Console
 from rich.table import Table
 
-from fastapi_translations.models import DocFile, Summary
+from fastapi_translations.translations import Document, Summary
 
 
 def print_table(summary: Summary, console: Console, table_size: int = 10) -> None:
@@ -60,7 +60,7 @@ def print_table(summary: Summary, console: Console, table_size: int = 10) -> Non
 
 
 def print_to_csv(summary: Summary) -> None:
-    header = DocFile.model_fields.keys()
+    header = Document.model_fields.keys()
     with open(f"fastapi-translations-lang-{summary.lang}.csv", "w", newline="") as f:
         writer = csv.DictWriter(f, header)
         writer.writeheader()
diff --git a/fastapi_translations/translations/__init__.py b/fastapi_translations/translations/__init__.py
new file mode 100644
index 0000000..acc43ab
--- /dev/null
+++ b/fastapi_translations/translations/__init__.py
@@ -0,0 +1,6 @@
+__all__ = ["Document", "Languages", "Summary"]
+
+
+from .document import Document
+from .languages import Languages
+from .summary import Summary
diff --git a/fastapi_translations/translations/document.py b/fastapi_translations/translations/document.py
new file mode 100644
index 0000000..7237957
--- /dev/null
+++ b/fastapi_translations/translations/document.py
@@ -0,0 +1,21 @@
+from datetime import datetime
+
+from pydantic import BaseModel, computed_field
+
+
+class Document(BaseModel):
+    official_lang: str = "en"
+    translation_lang: str
+    original_file: str
+    original_commit: datetime | None
+    translation_file: str | None = None
+    translation_exists: bool
+    translation_commit: datetime | None
+
+    @computed_field  # type: ignore
+    @property
+    def translation_is_outdated(self) -> bool:
+        if not self.original_commit or not self.translation_commit:
+            return False
+
+        return self.original_commit > self.translation_commit
diff --git a/fastapi_translations/translations/languages.py b/fastapi_translations/translations/languages.py
new file mode 100644
index 0000000..13126d9
--- /dev/null
+++ b/fastapi_translations/translations/languages.py
@@ -0,0 +1,36 @@
+from enum import Enum, unique
+
+
+@unique
+class Languages(Enum):
+    """Languages that the FastAPI documentation has translation.
+
+    For more information about the languages, check `FastAPI docs`_.
+
+    .. _FastAPI docs: https://github.com/fastapi/fastapi/tree/master/docs
+    """
+
+    az = "az"
+    bn = "bn"
+    de = "de"
+    em = "em"
+    es = "es"
+    fa = "fa"
+    fr = "fr"
+    he = "he"
+    hu = "hu"
+    id = "id"
+    it = "it"
+    ja = "ja"
+    ko = "ko"
+    nl = "nl"
+    pl = "pl"
+    pt = "pt"
+    ru = "ru"
+    tr = "tr"
+    uk = "uk"
+    ur = "ur"
+    vi = "vi"
+    yo = "yo"
+    zh = "zh"
+    zh_hant = "zh-hant"
diff --git a/fastapi_translations/translations/summary.py b/fastapi_translations/translations/summary.py
new file mode 100644
index 0000000..eafc0cf
--- /dev/null
+++ b/fastapi_translations/translations/summary.py
@@ -0,0 +1,169 @@
+import fnmatch
+import os
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from pydantic import BaseModel, computed_field
+from rich.progress import Progress, TaskID
+
+from ..git import FastAPIGitDocs
+from .document import Document
+
+base_docs_path = Path("docs")
+"""The base path for FastAPI documentation."""
+
+en_docs_path = Path("docs/en")
+"""The base path for FastAPI english documentation;"""
+
+restricted_list = [
+    "*reference/*",
+    "*release-notes.md",
+    "*fastapi-people.md",
+    "*external-links.md",
+    "*newsletter.md",
+    "*management-tasks.md",
+    "*management.md",
+    "*contributing.md",
+]
+"""list[str]: Restricted files and/or folders.
+
+FastAPI docs states that not all documents should be translated.
+You can find the reference in `Contributing`_.
+
+.. _Contributing: https://fastapi.tiangolo.com/contributing/#dont-translate-these-pages
+"""
+
+
+class Summary(BaseModel):
+    lang: str
+    files_analyzed: int = 0
+    files_translated: int = 0
+    files_outdated: int = 0
+    files_missing_translation: int = 0
+    files: list[Document] = []
+
+    @computed_field  # type: ignore
+    @property
+    def percentage_translated(self) -> float:
+        try:
+            return (
+                100 * float(self.files_translated) / float(self.files_analyzed)
+            )
+        except Exception:
+            return 0.0
+
+    @computed_field  # type: ignore
+    @property
+    def percentage_missing_translation(self) -> float:
+        try:
+            return (
+                100
+                * float(self.files_missing_translation)
+                / float(self.files_analyzed)
+            )
+        except Exception:
+            return 0.0
+
+    @computed_field  # type: ignore
+    @property
+    def percentage_outdated_translation(self) -> float:
+        try:
+            return 100 * float(self.files_outdated) / float(self.files_analyzed)
+        except Exception:
+            return 0.0
+
+    def generate(self) -> None:
+        git = FastAPIGitDocs()
+        with Progress() as progress:
+            futures = []
+
+            for root, _, files in os.walk(en_docs_path):
+                if root.startswith("docs/en/docs/img") or root.startswith(
+                    "docs/en/docs/js"
+                ):
+                    continue
+
+                task = progress.add_task(
+                    f"🚶 Walking through 📂 {root} looking for 🔠 translations",
+                    start=False,
+                )
+                with ThreadPoolExecutor(max_workers=50) as pool:
+                    future = pool.submit(
+                        self.process_file, git, root, files, progress, task
+                    )
+                    futures.append(future)
+
+    def process_file(
+        self,
+        git: FastAPIGitDocs,
+        root_dir: str,
+        files: list[str],
+        progress: Progress,
+        task: TaskID,
+    ) -> None:
+        progress.update(task, total=len(files))
+        progress.start_task(task)
+
+        for file in files:
+            if file.endswith(".md"):
+                file_relative_path = os.path.relpath(
+                    os.path.join(root_dir, file), en_docs_path
+                )
+                if self.file_in_reserved_list(file_relative_path):
+                    progress.update(task, advance=1)
+                    continue
+
+                translated_path = os.path.join(
+                    base_docs_path, self.lang, file_relative_path
+                )
+                translation_exists = os.path.exists(translated_path)
+
+                original_doc_date = git.get_commit_date_for(
+                    os.path.join(root_dir, file)
+                )
+
+                translated_date = git.get_commit_date_for(translated_path)
+
+                document = Document(
+                    translation_lang=self.lang,
+                    original_file=os.path.join(root_dir, file),
+                    original_commit=original_doc_date,
+                    translation_file=translated_path,
+                    translation_exists=translation_exists,
+                    translation_commit=translated_date,
+                )
+                self.append_document(document)
+
+            progress.update(task, advance=1)
+
+    def file_in_reserved_list(self, file: str) -> bool:
+        for reserved in restricted_list:
+            if fnmatch.fnmatch(file, reserved):
+                return True
+
+        return False
+
+    def append_document(self, doc: Document) -> None:
+        self.files.append(doc)
+        self.files_analyzed += 1
+
+        if doc.translation_exists:
+            self.files_translated += 1
+
+        if not doc.translation_exists:
+            self.files_missing_translation += 1
+
+        if doc.translation_is_outdated:
+            self.files_outdated += 1
+
+    def first_outdated_files(self, lenght: int = 10) -> list[Document]:
+        return list(filter(lambda d: d.translation_is_outdated, self.files))[
+            :lenght
+        ]
+
+    def first_missing_translation_files(
+        self, lenght: int = 10
+    ) -> list[Document]:
+        return list(filter(lambda d: not d.translation_exists, self.files))[
+            :lenght
+        ]


Test patch:
diff --git a/tests/translations/__init__.py b/tests/translations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/translations/document_test.py b/tests/translations/document_test.py
new file mode 100644
index 0000000..d62ccd6
--- /dev/null
+++ b/tests/translations/document_test.py
@@ -0,0 +1,16 @@
+import datetime
+
+from fastapi_translations.translations import Document
+
+
+def test_translation_is_outdated_should_return_true_if_english_document_has_older_commit_than_the_translated_document():
+    document = Document(
+        translation_lang="es",
+        original_file="/test/file1.md",
+        translation_file="/test/file1.md",
+        translation_exists=True,
+        original_commit=datetime.datetime(2025, 1, 28, 0, 0, 0),
+        translation_commit=datetime.datetime(2025, 1, 29, 0, 0, 0),
+    )
+
+    assert not document.translation_is_outdated
diff --git a/tests/models_test.py b/tests/translations/summary_test.py
similarity index 68%
rename from tests/models_test.py
rename to tests/translations/summary_test.py
index c6ab40b..5d7320d 100644
--- a/tests/models_test.py
+++ b/tests/translations/summary_test.py
@@ -1,40 +1,37 @@
 import datetime
 
-from fastapi_translations.models import DocFile, Summary
+from fastapi_translations.translations import Document, Summary
 
 
-def test_must_compute_summary_when_adding_docfile():
-    doc1 = DocFile(
+def test_must_compute_summary_when_adding_document():
+    doc1 = Document(
         translation_lang="es",
         original_file="/test/file1.md",
-        original_commit=datetime.datetime(2024, 1, 1, 0, 0, 0),
         translation_file="/test/file1.md",
         translation_exists=True,
+        original_commit=datetime.datetime(2024, 1, 1, 0, 0, 0),
         translation_commit=datetime.datetime(2024, 1, 15, 0, 0, 0),
-        translation_is_outdated=True
     )
-    doc2 = DocFile(
+    doc2 = Document(
         translation_lang="es",
         original_file="/test/file2.md",
-        original_commit=datetime.datetime(2024, 1, 1, 0, 0, 0),
         translation_file="/test/file2.md",
         translation_exists=True,
+        original_commit=datetime.datetime(2024, 1, 2, 0, 0, 0),
         translation_commit=datetime.datetime(2024, 1, 1, 0, 0, 0),
-        translation_is_outdated=False
     )
-    doc3 = DocFile(
+    doc3 = Document(
         translation_lang="es",
         original_file="/test/file3.md",
         original_commit=datetime.datetime(2024, 1, 1, 0, 0, 0),
         translation_file=None,
         translation_exists=False,
         translation_commit=None,
-        translation_is_outdated=False
     )
     summary = Summary(lang="es")
-    summary.append_file(doc1)
-    summary.append_file(doc2)
-    summary.append_file(doc3)
+    summary.append_document(doc1)
+    summary.append_document(doc2)
+    summary.append_document(doc3)
 
     assert summary.files_analyzed == 3
     assert summary.files_missing_translation == 1