initial commit

Baseline initial commit

.cursorindexingignore

@@ -0,0 +1,3 @@
+
+# Don't index SpecStory auto-save files, but allow explicit context inclusion via @ references
+.specstory/**

.gitattributes

@@ -0,0 +1 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,227 @@
+# data
+# data/
+# !pdf/
+# !output_dir/
+
+md_to_pdf*
+html_to_md*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Certificates
+*.pem
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

.python-version

@@ -0,0 +1 @@
+3.12

.specstory/.gitignore

@@ -0,0 +1,2 @@
+# SpecStory explanation file
+/.what-is-this.md

README.md

@@ -0,0 +1,178 @@
+---
+#[project]
+#name: "parserpdf"
+name: "parser2md"
+#title: "parserPDF"
+title: "parser2md"
+emoji: 📝
+colorFrom: yellow
+colorTo: purple
+sdk: gradio
+sdk_version: 5.0.1
+app_file: main.py
+pinned: false
+license: mit
+short_description: 'PDF & HTML parser to markdown'
+version: "0.1.0"
+readme: "README.md"
+requires-python: ">=3.12"
+dependencies: []
+owner: "research-semmyk"
+---
+# parserPDF
+
+[![Gradio](https://img.shields.io/badge/Gradio-SDK-amber?logo=gradio)](https://www.gradio.app/)
+[![Python](https://img.shields.io/badge/Python->=3.12-blue?logo=python)](https://www.python.org/)
+[![MIT License](https://img.shields.io/badge/License-MIT-yellow?logo=mit)](LICENSE)
+
+A Gradio-based web application for converting PDF and HTML documents to Markdown format. Powered by the Marker library (a pipeline of deep learning models for document parsing) and optional LLM integration for enhanced processing. Supports batch processing of files and directories via an intuitive UI.
+
+## Features
+- **PDF to Markdown**: Extract text, tables, and images from PDFs using Marker.
+- **HTML to Markdown**: Convert HTML files to clean Markdown.
+- **Batch Processing**: Upload multiple files or entire directories.
+- **LLM Integration**: Optional use of Hugging Face or OpenAI models for advanced conversion (e.g., via Llama or GPT models).
+- **Customizable Settings**: Adjust model parameters, output formats (Markdown/HTML), page ranges, and more via the UI.
+- **Output Management**: Generated Markdown files saved to a configurable output directory, with logs and download links.
+
+## Project Structure
+```
+parserpdf/
+├── README.md                   # Project documentation
+├── requirements.txt            # Python dependencies
+├── main.py                     # Entry point – launches the Gradio UI
+├── pyproject.toml              # Project configuration
+├── .env                        # Environment variables (e.g., API tokens)
+├── .gitignore                  # Git ignore rules
+├── converters/                 # Conversion logic
+│   ├── __init__.py
+│   ├── extraction_converter.py # Document extraction utilities
+│   ├── pdf_to_md.py            # Marker-based PDF → Markdown
+│   ├── html_to_md.py           # HTML → Markdown
+│   └── md_to_pdf.py            # Markdown → PDF (pending full implementation)
+├── file_handler/               # File handling utilities
+│   ├── __init__.py
+│   └── file_utils.py           # Helpers for files, directories, and paths
+├── llm/                        # LLM client integrations
+│   ├── __init__.py
+│   ├── hf_client.py            # Hugging Face client wrapper
+│   ├── openai_client.py        # Marker OpenAI client
+│   ├── llm_login.py            # Authentication handlers
+│   └── provider_validator.py   # Provider validation
+├── ui/                         # Gradio UI components
+│   ├── __init__.py
+│   └── gradio_ui.py            # UI layout and event handlers
+├── utils/                      # Utility modules
+│   ├── __init__.py
+│   ├── config.py               # Configuration constants
+│   ├── config.ini              # config file for settings
+│   ├── logger.py               # Logging wrapper
+│   ├── lib_loader.py           # loads weasyprint lib dependencies to environ
+│   ├── get_config.py           # helper for getting configurations
+│   ├── get_arg_name.py         # helper for getting argument names
+│   └── utils.py                # General utilities and helpers
+├── data/                       # Sample data and outputs (gitignored)
+│   ├── output_dir/             # Output directory
+│   ├── pdf/                    # Sample PDFs
+├── logs/                       # Log files (gitignored)
+├── tests/                      # Unit tests
+├── tests_converter.py          # tests for converters
+└── scrapyard/                  # Development scraps
+
+
+[Projected]
+├── transformers/
+│   ├── __init__.py
+│   ├── marker.py           # Marker class
+│   └── marker_utils.py     # helpers for Marker class
+
+```
+
+## Installation
+1. Clone the repository:
+   ```
+   git clone <repo-url>
+   cd parserpdf
+   ```
+
+2. Create a virtual environment and install dependencies:
+   ```
+   python -m venv venv
+   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   pip install -r requirements.txt
+   ```
+
+3. Set up environment variables (optional for LLM features):
+   - Create a `.env` file with your API tokens, e.g.:
+     ```
+     HF_TOKEN=hf_xxx
+     OPENAI_API_KEY=sk-xxx
+     ```
+
+4. Install Marker (if not in requirements.txt):
+   ```
+   pip install marker-pdf
+   ```
+
+## Usage
+1. Run the application:
+   ```
+   python main.py
+   ```
+
+2. Open the provided local URL (e.g., http://127.0.0.1:7860) in your browser.
+
+3. In the UI:
+   - Upload PDF/HTML files or directories via the "PDF & HTML ➜ Markdown" tab.
+   - Configure LLM/Marker settings in the accordions (e.g., select provider, model, tokens).
+   - Click "Process All Uploaded Files" to convert.
+   - View logs, JSON output, and download generated Markdown files.
+
+### Example Workflow
+- Upload a PDF directory.
+- Set model to `meta-llama/Llama-4-Maverick-17B-128E-Instruct` (Hugging Face).
+- Enable LLM if needed, set page range (e.g., "1-10").
+- Process: Outputs Markdown files with extracted text/images to `output_dir`.
+
+## Configuration
+- Edit `utils/config.py` or `utils/config.ini` for defaults (e.g., model ID, output dir).
+- UI overrides: Adjust sliders for max tokens, temperature, workers, etc.
+
+## LLM Providers
+- **Hugging Face**: Supports inference providers like Fireworks AI, Together AI.
+- **OpenAI**: Compatible via router (default: https://router.huggingface.co/v1).
+- Login via UI or CLI: `huggingface-cli login`.
+
+## Output
+- Markdown files saved to `output_dir` (default: `./output_dir`).
+- Images extracted as JPEGs alongside Markdown.
+- Logs in `logs/` and UI textbox.
+
+## Limitations & TODO
+- Markdown → PDF is pending full implementation.
+- HTML tab is deprecated; use main tab for mixed uploads.
+- Large files/directories may require increased `max_workers`.
+- No JSON/chunks output yet (flagged for future).
+
+## Contributing
+Fork the repo, create a branch, and submit a PR. 
+
+Ensure tests pass: - verify the application's functionality.
+```
+pytest tests/
+```
+Test Structure
+- tests/test_converters.py: Tests PDF/HTML/Markdown converters, including init, conversion, batch processing, and error handling.
+- tests/test_file_handler.py: Tests file collection utilities (PDF/HTML/MD paths), data processing, and output directory creation.
+- tests/test_utils.py: Tests logging setup, config loading, utility functions like is_dict/is_list_of_dicts, and configuration access.
+- tests/test_llm.py: Tests LLM login, provider validation, Hugging Face/OpenAI client initialization, and API interactions.
+- tests/test_main_ui.py: Tests main application logic, UI building, batch conversion, file accumulation, and ProcessPoolExecutor integration.
+
+
+## License
+MIT License. See [LICENSE](LICENSE) for details.
+
+## Acknowledgments
+- Built with [Gradio](https://gradio.app/) for the UI.
+- PDF parsing via [Marker](https://github.com/VikParuchuri/marker).
+- LLM integrations using Hugging Face Transformers and OpenAI APIs.

converters/__init__.py

@@ -0,0 +1 @@
+

converters/extraction_converter.py

@@ -0,0 +1,264 @@
+import os
+from pathlib import Path
+import traceback
+#import time
+from typing import Dict, Any, Type, Optional, Union #, BaseModel
+from pydantic import BaseModel
+
+from marker.models import create_model_dict
+#from marker.converters.extraction import ExtractionConverter as MarkerExtractor  ## structured pydantic extraction
+from marker.converters.pdf import PdfConverter as MarkerConverter  ## full document convertion/extraction
+from marker.config.parser import ConfigParser  ## Process custom configuration
+from marker.services.openai import OpenAIService as MarkerOpenAIService
+#from sympy import Union
+
+#from llm.hf_client import HFChatClient
+from llm.openai_client import OpenAIChatClient
+from file_handler.file_utils import collect_pdf_paths, collect_html_paths, collect_markdown_paths, create_outputdir
+from utils.lib_loader import load_library
+
+from utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+# Full document converter
+class DocumentConverter:
+    """ 
+    Business logic wrapper using Marker OpenAI LLM Services to
+    convert documents (PDF, HTML files) into markdowns + assets. 
+    """
+
+    def __init__(self,
+        #provider: str,
+        model_id: str,
+        #base_url: str,
+        hf_provider: str,
+        #endpoint_url: str,
+        #backend_choice: str,
+        #system_message: str,
+        #max_tokens: int,
+        temperature: float,
+        top_p: float,
+        #stream: bool,
+        api_token: str,
+        openai_base_url: str = "https://router.huggingface.co/v1",
+        openai_image_format: Optional[str] = "webp",
+        #max_workers: Optional[str] = 4,
+        max_retries: Optional[int] = 2,
+        output_format: str = "markdown",
+        output_dir: Optional[Union[str, Path]] = "output_dir",
+        use_llm: Optional[bool] = None,  #bool = False,  #Optional[bool] = False,  #True,
+        page_range: Optional[str] = None,  #str = None  #Optional[str] = None,  
+        ):
+
+        #self.converter = None  #MarkerConverter
+        self.model_id = model_id  #"model_name"
+        self.openai_api_key = api_token  ## to replace dependency on self.client.openai_api_key
+        self.openai_base_url = openai_base_url  #,  #self.base_url,
+        self.temperature = temperature   #, self.client.temperature,
+        self.top_p = top_p               # self.client.top_p,
+        self.llm_service = MarkerOpenAIService
+        self.openai_image_format = openai_image_format  #"png"  #better compatibility
+        self.max_retries = max_retries  ## pass to __call__
+        self.output_dir = output_dir
+        self.use_llm = use_llm[0] if isinstance(use_llm, tuple) else use_llm,  #False,  #True,
+        #self.page_range = page_range[0] if isinstance(page_range, tuple) else page_range   ##SMY: iterating twice because self.page casting as hint type tuple!
+        self.page_range = page_range if page_range else None
+        # self.page_range = page_range[0] if isinstance(page_range, tuple) else page_range if isinstance(page_range, str) else None,  ##Example: "0,4-8,16"  ##Marker parses as List[int]  #]debug  #len(pdf_file)
+        '''
+        if isinstance(page_range, tuple | str):
+            self.page_range = page_range[0] if isinstance(page_range, tuple) else page_range
+        else:
+            self.page_range = None
+        '''
+
+        # 0) Instantiate the LLM Client (OPENAIChatClient): Get a provider‐agnostic chat function
+        ##SMY: #future. Plan to integrate into Marker: uses its own LLM services (clients). As at 1.9.2, there's no huggingface client service.
+        try:
+            self.client = OpenAIChatClient(
+            model_id=model_id,
+            hf_provider=hf_provider,
+            #base_url=base_url,
+            api_token=api_token,
+            temperature=temperature,
+            top_p=top_p,
+            )
+            logger.log(level=20, msg="✔️ OpenAIChatClient instantiated:", extra={"model_id": self.client.model_id, "chatclient": str(self.client)})
+
+        except Exception as exc:
+            tb = traceback.format_exc()   #exc.__traceback__
+            logger.exception(f"✗ Error initialising OpenAIChatClient: {exc}\n{tb}")
+            raise RuntimeError(f"✗ Error initialising OpenAIChatClient: {exc}\n{tb}")  #.with_traceback(tb)
+
+        # 1) # Define the custom configuration for the Hugging Face LLM.
+                # Use typing.Dict and typing.Any for flexible dictionary type hints 
+        try:
+            self.config_dict: Dict[str, Any] = self.get_config_dict(model_id=model_id, llm_service=str(self.llm_service), output_format=output_format)
+            #self.config_dict.pop("page_range") if self.config_dict.get("page_range")[0] is None else None  ##SMY: execute if page_range is none. `else None` ensures valid syntactic expression
+
+            ##SMY: if falsely empty tuple () or None, pop the "page_range" key-value pair, else do nothing if truthy tuple value (i.e. keep as-is)
+            self.config_dict.pop("page_range", None) if not self.config_dict.get("page_range") else None
+
+            logger.log(level=20, msg="✔️ config_dict custom configured:", extra={"service": "openai"})  #, "config": str(self.config_dict)})
+
+        except Exception as exc:
+            tb = traceback.format_exc()   #exc.__traceback__
+            logger.exception(f"✗ Error configuring custom config_dict: {exc}\n{tb}")
+            raise RuntimeError(f"✗ Error configuring custom config_dict: {exc}\n{tb}")  #.with_traceback(tb)
+
+        # 2) Use the Marker's ConfigParser to process configuration.
+            # The `ConfigParser` class is explicitly imported and used as the type hint.
+        try:
+            config_parser: ConfigParser = ConfigParser(self.config_dict)
+            logger.log(level=20, msg="✔️ parsed/processed custom config_dict:", extra={"config": str(config_parser)})  #.config_dict)})
+
+        except Exception as exc:
+            tb = traceback.format_exc()   #exc.__traceback__
+            logger.exception(f"✗ Error parsing/processing custom config_dict: {exc}\n{tb}")
+            raise RuntimeError(f"✗ Error parsing/processing custom config_dict: {exc}\n{tb}")  #.with_traceback(tb)
+        
+        # 3) Create the artifact dictionary and retrieve the LLM service.
+        try:
+            #self.artifact_dict: Dict[str, Any] = self.get_create_model_dict  ##SMY: Might have to eliminate function afterall
+            self.artifact_dict: Dict[str, Type[BaseModel]] = create_model_dict()  ##SMY: BaseModel for Any??
+            #logger.log(level=20, msg="✔️ Create artifact_dict and llm_service retrieved:", extra={"llm_service": self.llm_service})
+        
+        except Exception as exc:
+            tb = traceback.format_exc()   #exc.__traceback__
+            logger.exception(f"✗ Error creating artifact_dict or retrieving LLM service: {exc}\n{tb}")
+            raise RuntimeError(f"✗ Error creating artifact_dict or retrieving LLM service: {exc}\n{tb}")  #.with_traceback(tb)
+
+        # 4) Instantiate Marker's MarkerConverter (PdfConverter) with config managed by config_parser
+        try:
+            llm_service_str = str(self.llm_service).split("'")[1]  ## SMY: split and slicing  ##Gets the string value
+
+            # sets api_key required by Marker 
+            os.environ["OPENAI_API_KEY"] = self.openai_api_key or api_token  ## to handle Marker's assertion test on OpenAI
+            logger.log(level=20, msg="self.converter: instantiating MarkerConverter:", extra={"llm_service_str": llm_service_str, "api_token": api_token})  ##debug
+            
+            #self.converter: MarkerConverter = MarkerConverter(
+            self.converter = MarkerConverter(
+                #artifact_dict=self.artifact_dict,
+                artifact_dict=create_model_dict(),
+                config=config_parser.generate_config_dict(),
+                #llm_service=self.llm_service  ##SMY expecting str but self.llm_service, is service object marker.services of type BaseServices
+                llm_service=llm_service_str    ##resolve
+            )
+            
+            logger.log(level=20, msg="✔️ MarkerConverter instantiated successfully:", extra={"converter.config": str(self.converter.config.get("openai_base_url")), "use_llm":self.converter.use_llm})
+            #return self.converter  ##SMY: to query why did I comment out?. Bingo: "__init__() should return None, not 'PdfConverter'"
+        except Exception as exc:
+            tb = traceback.format_exc
+            logger.exception(f"✗ Error initialising MarkerExtractor: {exc}\n{tb}")
+            raise RuntimeError(f"✗ Error initialising MarkerExtractor: {exc}\n{tb}")
+        
+        # Define the custom configuration for HF LLM.
+    def get_config_dict(self, model_id: str, llm_service=MarkerOpenAIService, output_format: Optional[str] = "markdown" ) -> Dict[str, Any]:
+        """ Define the custom configuration for the Hugging Face LLM. """
+
+        try:
+            ## Enable higher quality processing with LLMs.  ## See MarkerOpenAIService,  
+            #llm_service = llm_service.removeprefix("<class '").removesuffix("'>")  # e.g <class 'marker.services.openai.OpenAIService'>
+            llm_service  = str(llm_service).split("'")[1]  ## SMY: split and slicing
+            self.use_llm = self.use_llm[0]
+            self.page_range = self.page_range[0] if isinstance(self.page_range, tuple) else self.page_range #if isinstance(self.page_range, str) else None,  ##SMY: passing as hint type tuple!
+            
+
+            config_dict = {
+                "output_format" : output_format,     #"markdown",
+                "openai_model"   : self.model_id,    #self.client.model_id,  #"model_name"
+                "openai_api_key" : self.client.openai_api_key,   #self.client.openai_api_key,  #self.api_token,
+                "openai_base_url": self.openai_base_url,  #self.client.base_url,  #self.base_url,
+                "temperature"    : self.temperature,      #self.client.temperature,
+                "top_p"          : self.top_p,            #self.client.top_p,
+                "openai_image_format": self.openai_image_format, #"webp",  #"png"  #better compatibility
+                "max_retries"    : self.max_retries,  #3,  ## pass to __call__
+                "output_dir"     : self.output_dir,
+                "use_llm"        : self.use_llm,      #False,  #True,
+                "page_range"     : self.page_range,   #]debug  #len(pdf_file)
+            }
+            return config_dict
+        except Exception as exc:
+            tb = traceback.format_exc()   #exc.__traceback__
+            logger.exception(f"✗ Error configuring custom config_dict: {exc}\n{tb}")
+            raise RuntimeError(f"✗ Error configuring custom config_dict: {exc}\n{tb}")  #").with_traceback(tb)
+            #raise
+
+    ##SMY: flagged for deprecation
+    ##SMY: marker prefer default artifact dictionary (marker.models.create_model_dict) instead of overridding
+    #def get_extraction_converter(self, chat_fn):
+    def get_create_model_dict(self):
+        """
+        Wraps the LLM chat_fn into marker’s artifact_dict
+        and returns an ExtractionConverter for PDFs & HTML.
+        """
+        return create_model_dict() 
+        #artifact_dict = create_model_dict(inhouse_chat_model=chat_fn)      
+        #return artifact_dict            
+
+## SMY: Kept for future implementation (and historic reasoning). Keeping the classes separate to avoid confusion with the original implementation
+'''
+class DocumentExtractor:
+    """ 
+    Business logic wrapper using HFChatClient and Marker to
+    convert documents (PDF, HTML files) into markdowns + assets
+    Wrapper around the Marker extraction converter for PDFs & HTML. 
+    """
+
+    def __init__(self,
+        provider: str,
+        model_id: str,
+        hf_provider: str,
+        endpoint_url: str,
+        backend_choice: str,
+        system_message: str,
+        max_tokens: int,
+        temperature: float,
+        top_p: float,
+        stream: bool,
+        api_token: str,
+        ):
+        # 1) Instantiate the LLM Client (HFChatClient): Get a provider‐agnostic chat function
+        try:
+            self.client = HFChatClient(
+            provider=provider,    
+            model_id=model_id,
+            hf_provider=hf_provider,
+            endpoint_url=endpoint_url,
+            backend_choice=backend_choice,       #choices=["model-id", "provider", "endpoint"]
+            system_message=system_message,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            stream=stream,
+            api_token=api_token,
+            )
+            logger.log(level=20, msg="✔️ HFChatClient instantiated:", extra={"model_id": model_id, "chatclient": str(self.client)})
+
+        except Exception as exc:
+            tb = traceback.format_exc()   #exc.__traceback__
+            logger.exception(f"✗ Error initialising HFChatClient: {exc}")
+            raise RuntimeError(f"✗ Error initialising HFChatClient: {exc}").with_traceback(tb)
+            #raise
+
+        # 2) Build Marker's artifact dict using the client's chat method
+        self.artifact_dict = self.get_extraction_converter(self.client)
+        
+        # 3) Instantiate Marker's ExtractionConverter (ExtractionConverter)
+        try:
+            self.extractor = MarkerExtractor(artifact_dict=self.artifact_dict)
+        except Exception as exc:
+            logger.exception(f"✗ Error initialising MarkerExtractor: {exc}")
+            raise RuntimeError(f"✗ Error initialising MarkerExtractor: {exc}")
+    
+    ##SMY: marker prefer default artifact dictionary (marker.models.create_model_dict) instead of overridding
+    def get_extraction_converter(self, chat_fn):
+        """
+        Wraps the LLM chat_fn into marker’s artifact_dict
+        and returns an ExtractionConverter for PDFs & HTML.
+        """
+        
+        artifact_dict = create_model_dict(inhouse_chat_model=chat_fn)
+        return artifact_dict
+'''
+

converters/pdf_to_md.py

@@ -0,0 +1,332 @@
+# converters/pdf_to_md.py
+import os
+from pathlib import Path
+from typing import List, Dict, Optional, Union
+import traceback  ## Extract, format and print information about Python stack traces.
+import time
+
+#from llm.hf_client import HFChatClient
+from converters.extraction_converter import DocumentConverter  #, DocumentExtractor #as docextractor #ExtractionConverter  #get_extraction_converter  ## SMY: should disuse
+from file_handler.file_utils import collect_pdf_paths, collect_html_paths, collect_markdown_paths, create_outputdir, write_markdown, dump_images
+
+
+from utils import config
+from utils.lib_loader import set_weasyprint_library
+from utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+# Define global variables
+docconverter: DocumentConverter = None
+converter = None  #DocumentConverter
+#converter:DocumentConverter.converter = None
+
+# Define docextractor in the pool as serialised object and passed to each worker process.
+# Note: DocumentConverter must be "picklable".
+def init_worker(#self,
+    provider: str,
+    model_id: str,
+    #base_url,
+    hf_provider: str,
+    endpoint_url: str,
+    backend_choice: str,
+    system_message: str,
+    max_tokens: int,
+    temperature: float,
+    top_p: float,
+    stream: bool,
+    api_token: str,
+    openai_base_url: str,  #: str = "https://router.huggingface.co/v1",
+    openai_image_format: str,  #: str | None = "webp",
+    max_workers: int,
+    max_retries: int,  #: int | None = 2,
+    output_format: str,  #: str = "markdown",
+    output_dir: str,  #: Union | None = "output_dir",
+    use_llm: bool,  #: bool | None = False,
+    page_range: str,  #: str | None = None
+    ):
+    
+    #'''
+    """ 
+    instantiate DocumentConverter/DocumentExtractor for use in each pool worker
+    Args:
+
+    """
+
+    ## moved to class
+    #    Initialise the global `converter` in each worker
+    # Define global variables
+    global docconverter
+    global converter
+    
+
+    ##SMY: kept for future implementation. Replaced with DocumentConverter.
+    '''
+    # 1) Instantiate the DocumentExtractor
+    logger.log(level=20, msg="initialising docextractor:", extra={"model_id": model_id, "hf_provider": hf_provider})
+    try:
+        docextractor = DocumentExtractor(   
+            provider=provider,     
+            model_id=model_id,
+            hf_provider=hf_provider,
+            endpoint_url=endpoint_url,
+            backend_choice=backend_choice,
+            system_message=system_message,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            stream=stream,
+            api_token=api_token,
+        )
+        logger.log(level=20, msg="✔️ docextractor initialised:", extra={"model_id": model_id, "hf_provider": hf_provider})
+    except Exception as exc:
+        #logger.error(f"Failed to initialise DocumentExtractor: {exc}")
+        tb = traceback.format_exc()
+        logger.exception(f"init_worker: Error initialising DocumentExtractor → {exc}\n{tb}", exc_info=True)
+        return f"✗ init_worker: error initialising DocumentExtractor → {exc}\n{tb}"
+    
+    self.docextractor = docextractor
+    '''
+
+    #'''
+    # 1) Instantiate the DocumentConverter
+    logger.log(level=20, msg="initialising docconverter:", extra={"model_id": model_id, "hf_provider": hf_provider})  ##debug
+    try:
+        docconverter = DocumentConverter(
+            model_id,  #: str,
+            hf_provider,  #: str,
+            temperature,  #: float,
+            top_p,  #: float,
+            api_token,  #: str,
+            openai_base_url,  #: str = "https://router.huggingface.co/v1",
+            openai_image_format,  #: str | None = "webp",
+            max_retries,  #: int | None = 2,
+            output_format,  #: str = "markdown",
+            output_dir,  #: Union | None = "output_dir",
+            use_llm,  #: bool | None = False,
+            page_range,  #: str | None = None
+        )
+        logger.log(level=20, msg="✔️ docextractor initialised:", extra={"docconverter model_id": docconverter.converter.config.get("openai_model"), "docconverter use_llm": docconverter.converter.use_llm, "docconverter output_dir": docconverter.output_dir})
+    except Exception as exc:
+        #logger.error(f"Failed to initialise DocumentConverter: {exc}")  #debug
+        tb = traceback.format_exc()
+        logger.exception(f"init_worker: Error initialising DocumentConverter → {exc}\n{tb}", exc_info=True)
+        return f"✗ init_worker: error initialising DocumentConverter → {exc}\n{tb}"
+    
+    #docconverter = docconverter
+    converter = docconverter.converter
+    #self.llm_service = docconverter.llm_service  ##duplicate?
+    #self.model_id = model_id   ##duplicate?
+    #'''
+
+class PdfToMarkdownConverter:
+    """
+    Wrapper around the Marker library that converts PDFs to Markdown.
+    """
+
+    #def __init__(self, options: Dict | None = None):
+    def __init__(self, options: Dict | None = None): #extractor: DocumentExtractor, options: Dict | None = None):
+        self.options = options or {}
+        self.output_dir_string = ''
+        #self.OUTPUT_DIR = config.OUTPUT_DIR     ##flag unused
+        #self.MAX_RETRIES = config.MAX_RETRIES   ##flag unused
+        #self.docconverter = None  #DocumentConverter
+        #self.converter = self.docconverter.converter #None 
+
+    # This global will be set (re-initialised) in each worker after init_worker runs
+
+    ## moved from extraction_converter ( to standalone extract_to_md)
+    #def extract(self, src_path: str, output_dir: str) -> Dict[str, int, Union[str, Path]]:
+    def extract(self, src_path: str, output_dir: str) -> Dict:
+    #def extract(src_path: str, output_dir: str) -> Dict[str, int]:  #, extractor: DocumentExtractor) -> Dict[str, int]:
+        """
+        Convert one file (PDF/HTML) to Markdown + images.
+        Writes a `.md` file and any extracted images under `output_dir`.
+        Returns a dict with metadata, e.g. {"filename": <file.name>, "images": <count>, "filepath": <filepath>}.
+        """
+        
+        try:
+            ## SMY: TODO: convert htmls to PDF. Marker will by default attempt weasyprint which typically raise 'libgobject-2' error on Win
+            # Set a new environment variable
+            set_weasyprint_library()  ##utils.lib_loader.set_weasyprint_library()            
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception(f"Error loading weasyprint backend dependency → {exc}\n{tb}", exc_info=True)  # Log the full traceback
+            raise RuntimeWarning(f"✗ error during loading weasyprint backend dependency → {exc}\n{tb}")
+
+        
+        # Run Marker conversion with LLM if use_llm is true
+        try:
+            #rendered = self.docconverter.converter(src_path, use_llm=True)
+            #rendered = self.docconverter.converter(src_path)
+            rendered = converter(src_path)
+            logger.log(level=20, msg=f"✓ File extraction successful for {Path(src_path).name}")
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception(f"Error during file extraction → {exc}\n{tb}", exc_info=True)  # Log the full traceback
+            
+            return f"✗ error during extraction → {exc}\n{tb}"
+
+        # Write Markdown file
+        '''
+        base = Path(str_path).stem   ## Get filename without extension
+        md_path = output_dir / f"{base}.md"  # Join output dir and new markdown file with the slash operator
+        
+        with open(md_path, "w", encoding="utf-8") as f:
+            f.write(rendered.markdown)
+        '''
+        try:
+            md_file = write_markdown(src_path=src_path, output_dir=output_dir, rendered=rendered)
+            #debug md_file = "debug_md_file dummy name" ##debug
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception(f"✗ error creating md_file → {exc}\n{tb}", exc_info=True)
+            #return f"✗ error creating md_file → {exc}\n{tb}"        
+
+        # Dump extracted images
+        #debug images_count = 100  ##debug
+        try:
+            images_count, image_path = dump_images(src_path, output_dir, rendered)
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception(f"✗ error counting and creating image_path → {exc}\n{tb}", exc_info=True)
+            #return f"✗ error counting andcreating image_path → {exc}\n{tb}"
+        
+        #return {"images": len(rendered.images), "file": md_file}  ##debug
+        return {"file": md_file.name, "images": images_count, "filepath": md_file, "image_path": image_path}  ####SMY should be Dict[str, int, str]. Dicts are not necessarily ordered.
+
+    #def convert_files(src_path: str, output_dir: str, max_retries: int = 2) -> str:
+    def convert_files(self, src_path: str, output_dir_string: str = None, max_retries: int = 2) -> Union[Dict, str]:  #str:    
+    #def convert_files(self, src_path: str) -> str:    
+        """
+        Worker task: use `extractor` to convert file with retry/backoff.
+        Returns a short log line.
+        """
+
+        try:
+            output_dir = create_outputdir(root=src_path, output_dir_string=self.output_dir_string)
+            logger.info(f"✓ output_dir created: {output_dir}")  #{create_outputdir(src_path)}"
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception("✗ error creating output_dir → {exc}\n{tb}", exc_info=True)
+            return f"✗ error creating output_dir → {exc}\n{tb}"
+
+        try:
+            #if Path(src_path).suffix.lower() not in {".pdf", ".html", ".htm"}:
+            #if not Path(src_path).name.endswith(tuple({".pdf", ".html"})):  #,".docx", ".doc", ".pptx", ".ppt", ".xlsx", ".xls"})):
+            if not Path(src_path).name.endswith((".pdf", ".html")):  #,".docx", ".doc", ".pptx", ".ppt", ".xlsx", ".xls"})):            
+                logger.log(level=20, msg=f"skipped {Path(src_path).name}", exc_info=True)
+                return f"skipped {Path(src_path).name}"
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception("✗ error during suffix extraction → {exc}\n{tb}", exc_info=True)
+            return f"✗ error during suffix extraction → {exc}"
+
+        #max_retries = self.MAX_RETRIES
+        for attempt in range(1, max_retries + 1):
+            try:
+                info = self.extract(str(src_path), str(output_dir.stem))  #extractor.converter(str(src_path), str(output_dir))  #
+                logger.log(level=20, msg=f"✓ : info about extracted {Path(src_path).name}: ", extra={"info": str(info)})
+                '''  ##SMY: moving formating to calling Gradio
+                img_count = info.get("images", 0)
+                md_filename = info.get("file", 0)
+                md_filepath = info.get("filepath", 0)
+                #return f"✓ {src_path.name} ({img_count} images)"
+                return f"✓ {md_filename}: ({img_count} images)", md_filepath
+                '''
+                return info  ##SMY: simply return the dict
+            except Exception as exc:
+                if attempt == max_retries:
+                    tb = traceback.format_exc()
+                    return f"✗ {info.get("file")} → {exc}\n{tb}"
+                    #return f"✗ {md_filename} → {exc}\n{tb}"
+                
+                #time.sleep(2 ** attempt)
+                # Exponential backoff before retry
+                logger.warning(f"Attempt {attempt} failed for {Path(src_path).name}: {exc}. Retrying in {2 ** attempt}s...")
+
+                time.sleep(2 ** attempt)
+    
+    ## SMY: unused
+    #=====================  discarded
+    '''
+    def convert(self, pdf_path: Path) -> str:
+        """
+        Convert a single PDF file to Markdown string.
+
+        Parameters
+        ----------
+        pdf_path : pathlib.Path
+            Path to the source PDF.
+
+        Returns
+        -------
+        str
+            The extracted Markdown content.
+        """
+        logger.info(f"Converting {pdf_path} → Markdown")
+        try:
+            md_text = self.marker.extract_markdown(str(pdf_path))
+            return md_text
+        except Exception as exc:
+            logger.exception("Marker failed to convert PDF.")
+            raise RuntimeError(f"Failed to convert {pdf_path}") from exc
+
+    
+    def batch_convert(self, pdf_paths: List[Path]) -> Dict[str, str]:
+        """
+        Convert multiple PDFs and return a mapping of filename → Markdown.
+
+        Parameters
+        ----------
+        pdf_paths : list[pathlib.Path]
+            List of PDF files to process.
+
+        Returns
+        -------
+        dict
+            Mapping from original file name (without extension) to Markdown string.
+        """
+        results = {}
+        for p in pdf_paths:
+            try:
+                md = self.convert(p)
+                key = p.stem  # filename without .pdf
+                results[key] = md
+            except Exception as exc:
+                logger.warning(f"Skipping {p}: {exc}")
+        return results
+
+    def convert_file(self, src_path: Path, extractor: DocumentConverter): #DocumentExtractor):  #-> str:
+        """
+        Converts one PDF or HTML file to Markdown + images
+        with retry/backoff on errors.
+        """
+        path    = src_path
+        out_dir = path.parent / self.OUTPUT_DIR
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+        for attempt in range(1, self.MAX_RETRIES + 1):
+            try:
+                rendered = extractor.converter(str(path), use_llm=True)
+
+                # Write Markdown
+                md_file = out_dir / f"{path.stem}.md"
+                md_file.write_text(rendered.markdown, encoding="utf-8")
+
+                # Dump images
+                for name, content in rendered.images.items():
+                    (out_dir / name).write_bytes(content)
+
+                print(f"[ok]   {path.name}")
+                return
+
+            except Exception as e:
+                if attempt == self.MAX_RETRIES:
+                    print(f"[fail] {path.name} after {attempt} attempts")
+                    traceback.print_exc()
+                else:
+                    backoff = 2 ** attempt
+                    print(f"[retry] {path.name} in {backoff}s ({e})")
+                    time.sleep(backoff)
+    '''

data/output_dir/.gitignore

@@ -0,0 +1,4 @@
+**
+# !*.md
+!COSE_ITAssetsRedefinition_Adesemowo2021_1-s2.0-S0167404820304041-main/
+!.gitignore

data/output_dir/COSE_ITAssetsRedefinition_Adesemowo2021_1-s2.0-S0167404820304041-main/.gitignore

@@ -0,0 +1,4 @@
+!*.md
+!*.jpeg
+# !COSE_ITAssetsRedefinition_Adesemowo2021_1-s2.0-S0167404820304041-main/
+!.gitignore

data/pdf/.gitignore

@@ -0,0 +1,3 @@
+**
+!*.pdf
+!.gitignore

data/pdf/COSE_ITAssetsRedefinition_Adesemowo2021_1-s2.0-S0167404820304041-main.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e971eac65cce7be288302e2b1faf8c622b62bb9c8fedb60a3f88ff385c3104c
+size 2137689

file_handler/__init__.py

@@ -0,0 +1 @@
+

file_handler/file_utils.py

@@ -0,0 +1,296 @@
+# file_handler/file_utils.py
+#import os
+from pathlib import Path
+from itertools import chain
+from typing import List, Union, Any, Mapping
+from PIL import Image
+
+import utils.config as config
+
+##SMY: Might be deprecated vis duplicated. See marker/marker/config/parser.py  ~ https://github.com/datalab-to/marker/blob/master/marker/config/parser.py#L169
+#def create_outputdir(root: Union[str, Path], out_dir:Union[str, Path] = None) -> Path:  #List[Path]:
+def create_outputdir(root: Union[str, Path], output_dir_string:str = None) -> Path:  #List[Path]:
+    """ Create output dir under the input folder """
+    
+    '''  ##preserved for future implementation if needed again
+    root = root if isinstance(root, Path) else Path(root)  
+    #root = Path(root)
+    if not root.exists():
+        raise FileNotFoundError(f"Root path {root} does not exist: cannot create output dir.")
+    out_dir = out_dir if out_dir else "output_md"  ## SMY: default to outputdir in config file = "output_md"
+    output_dir = root.parent / out_dir  #"md_output"  ##SMY: concatenating output str with src Path
+    '''
+
+    ## map to img_path. Opt to putting output within same output_md folder rather than individual source folders
+    output_dir_string = output_dir_string if output_dir_string else "output_dir"  ##redundant SMY: default to outputdir in config file = "output_md"
+    output_dir = Path("data") / output_dir_string  #"output_md"  ##SMY: concatenating output str with src Path
+    output_dir.mkdir(mode=0o2644, parents=True, exist_ok=True)
+    return output_dir
+
+def is_file_with_extension(path_obj: Path) -> bool:
+    """
+    Checks if a pathlib.Path object is a file and has a non-empty extension.
+    """
+    path_obj = path_obj if isinstance(path_obj, Path) else Path(path_obj) if isinstance(path_obj, str) else None
+    return path_obj.is_file() and bool(path_obj.suffix)
+
+def process_dicts_data(data:Union[dict, list[dict]]):
+    """ Returns formatted JSON string for a single dictionary or a list of dictionaries"""
+    import json
+    from pathlib import WindowsPath
+    #from typing import dict, list
+
+    # Serialise WindowsPath objects to strings using custom json.JSoNEncoder subclass
+    class PathEncoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, WindowsPath):
+                return str(obj)
+            # Let the base class default method raise the TypeError for other types
+            return json.JSONEncoder.default(self, obj)
+
+    # Convert the list of dicts to a formatted JSON string
+    formatted_string = json.dumps(data, indent=4, cls=PathEncoder)
+    
+    return formatted_string
+
+##NB: Python =>3.10, X | Y equiv to the type checker as Union[X, Y]
+def collect_pdf_html_paths(root: Union[str, Path]) -> List[Path]:
+    """
+    Recursively walk *root* and return a list of all PDF files.
+    """
+    root = Path(root)
+    patterns = ["*.pdf", "*.html"]  #, "*.htm*"]
+    if not root.exists():
+        raise FileNotFoundError(f"Root path {root} does not exist.")
+    #pdfs_htmls = [p for p in root.rglob("*.pdf", "*.html", "*.htm*") if p.is_file()]
+    #pdfs_htmls = [chain.from_iterable(root.rglob(pattern) for pattern in patterns)]
+    # Use itertools.chain to combine the generators from multiple rglob calls
+    pdfs_htmls = list(chain.from_iterable(root.rglob(pattern) for pattern in patterns))
+
+    return pdfs_htmls
+
+def collect_pdf_paths(root: Union[str, Path]) -> List[Path]:
+    """
+    Recursively walk *root* and return a list of all PDF files.
+    """
+    root = Path(root)
+    if not root.exists():
+        raise FileNotFoundError(f"Root path {root} does not exist.")
+    pdfs = [p for p in root.rglob("*.pdf") if p.is_file()]
+    return pdfs
+
+def collect_html_paths(root: Union[str, Path]) -> List[Path]:
+    """
+    Recursively walk *root* and return a list of all PDF files.
+    """
+    root = Path(root)
+    if not root.exists():
+        raise FileNotFoundError(f"Root path {root} does not exist.")
+    htmls = [p for p in root.rglob("*.html", ".htm") if p.is_file()]
+
+    ## SMY: TODO: convert htmls to PDF. Marker will by default attempt weasyprint which typically raise 'libgobject-2' error on Win
+    
+    return htmls
+
+def collect_markdown_paths(root: Union[str, Path]) -> List[Path]:
+    """
+    Recursively walk *root* and return a list of all Markdown files.
+    """
+    root = Path(root)
+    md_files = [p for p in root.rglob("*.md") if p.is_file()]
+    return md_files
+
+#m __future__ import annotations
+def write_markdown(
+    src_path: Union[str, Path],
+    output_dir: Union[str, Path],
+    rendered: Any,
+) -> Path:
+    
+    """
+    Write the Markdown representation of a source file to an output directory.
+
+    Parameters
+    ----------
+    src_path : str | Path
+        Path to the original source file. Only its base name is used for naming
+        the resulting Markdown file.
+    output_dir : str | Path
+        Directory where the Markdown file will be written. It was created if it does not
+        exist with create_outputdir().
+    rendered : object
+        Object that provides a ``markdown`` attribute containing the text to write.
+
+    Returns
+    -------
+    pathlib.Path
+        The full path of the written Markdown file.
+
+    Raises
+    ------
+    FileNotFoundError
+        If *src_path* does not point to an existing file.
+    OSError
+        If writing the file fails for any reason (e.g. permission denied).
+    AttributeError
+        If *rendered* does not expose a ``markdown`` attribute.
+
+    Notes
+    -----
+    The function is intentionally lightweight: it only handles path resolution,
+    directory creation, and file I/O. All rendering logic should be performed before
+    calling this helper.
+    """
+    src = Path(src_path)
+    if not src.is_file():
+        raise FileNotFoundError(f"Source file does not exist: {src}")
+
+    #out_dir = Path(output_dir)
+    #out_dir.mkdir(parents=True, exist_ok=True)
+
+    md_name = f"{src.stem}.md"
+    if isinstance(output_dir, Path):
+        md_path = output_dir / f"{src.stem}" / md_name
+    else:
+        #md_path = Path(src.parent) / f"{Path(output_dir).stem}" / f"{src.stem}" / md_name
+        
+        ## Opt to putting output within same output_md folder rather than individual source folders
+        #md_path = Path("data\\pdf") / "output_md" / f"{src.stem}" / md_name  ##debug
+        md_path = Path("data") / output_dir / f"{src.stem}" / md_name  ##debug
+    ##SMY: [resolved] Permission Errno13 - https://stackoverflow.com/a/57454275
+    md_path.parent.mkdir(mode=0o2644, parents=True, exist_ok=True)  ##SMY: create nested md_path if not exists
+    md_path.parent.chmod(0)
+
+    try:
+        markdown_text = getattr(rendered, "markdown")  ##SMY: get extracted markdown
+    except AttributeError as exc:  # pragma: no cover
+        raise AttributeError(
+            "Extractor Rendered object must have a 'markdown' attribute"
+        ) from exc
+
+    with md_path.open(mode="w", encoding="utf-8") as md_f:
+        md_f.write(markdown_text)    ##SMY: write markdown content to markdown file
+
+    return md_path               ##SMY: return the markdown file  #✓ 
+    #return {"files": md_path}   ##SMY: return dict of file with markdown filename.
+
+# Dummp Markdown extracted images
+def dump_images(
+    src_path: Union[str, Path],
+    output_dir: Union[str, Path],
+    rendered: Any,
+) -> int:
+    
+    """
+    Dump the images  of the Markdown representation of a source file to an output directory.
+
+    Parameters
+    ----------
+    src_path : str | Path
+        Path to the original source file. Only its base name is used for naming
+        the resulting Markdown file.
+    output_dir : str | Path
+        Directory where the Markdown file will be written. It was created if it does not
+        exist with create_outputdir().
+    rendered : object
+        Object that provides a ``markdown`` attribute containing the text to write.
+
+    Returns
+    -------
+    Number of images dumped from the  Markdown file.
+    """
+
+    try:
+        images: Image.Image = getattr(rendered, "images")
+    except TypeError as exc:  # pragma: no cover
+        raise AttributeError(
+            "Extracted images from rendered.images must be a mapping of str -> PIL.Image"
+        ) from exc
+    
+    # Initialise variables
+    images_count = 0
+    img_path_list = []
+    ##SMY: See marker.output.save_output()  : https://github.com/datalab-to/marker/blob/master/marker/output.py
+    #for img_name, img_bytes in images.items():
+
+    src = Path(src_path)  ##SMY: keep uniform with write_markdown. No need is exists anymore
+    for img_name, img in images.items():
+        # Resolve the full path and make sure any sub‑directories exist.
+        #img_path = Path(output_dir) / src_path / img_name    ##SMY: image files  ##concatenate Path + str
+        #img_path = create_outputdir(src_path) / img_name
+        
+        if isinstance(output_dir, Path):
+            img_path = output_dir.stem / img_name            
+        else:
+            # #img_path = Path(output_dir) / f"{src.stem}" / img_name   ##SMY: create markdown file ##SMY concatenating Path with str
+            # #img_path = Path(output_dir) / img_name   ##SMY: create markdown file ##SMY concatenating Path with str
+            #img_path = Path(src.parent) / f"{Path(output_dir).stem}" / f"{src.stem}" / img_name
+            
+            #img_path = Path("data\\pdf") / "output_md" / f"{src.stem}" / img_name  ##debug
+            img_path = Path("data") / output_dir / f"{src.stem}" / img_name  ##debug
+        #img_path.mkdir(mode=0o777, parents=True, exist_ok=True)  ##SMY: create nested img_path if not exists
+        #img_path.parent.mkdir(parents=True, exist_ok=True)
+
+        img.save(img_path)    ##SMY: save images (of type PIL.Image.Image) to markdown folder
+        images_count += 1
+        #img_path_list = img_path_list.append(img_path)
+        img_path_list.append(img_path)
+
+    return images_count, img_path_list        ##SMY: return number of images and path
+    #return images.items().count
+    #return len(images)
+
+# Dummp Markdown extracted images  ##SMY: Marked for deprecated
+'''
+def dump_images(
+    src_path: Union[str, Path],
+    output_dir: Union[str, Path],
+    rendered: Any,
+) -> int:
+    
+    """
+    Dump the images  of the Markdown representation of a source file to an output directory.
+
+    Parameters
+    ----------
+    src_path : str | Path
+        Path to the original source file. Only its base name is used for naming
+        the resulting Markdown file.
+    output_dir : str | Path
+        Directory where the Markdown file will be written. It was created if it does not
+        exist with create_outputdir().
+    rendered : object
+        Object that provides a ``markdown`` attribute containing the text to write.
+
+    Returns
+    -------
+    Number of images dumped from the  Markdown file.
+    """
+
+    try:
+        images: Mapping[str, bytes] = getattr(rendered, "images")
+    except TypeError as exc:  # pragma: no cover
+        raise AttributeError(
+            "Extracted images from rendered.images must be a mapping of str -> bytes"
+        ) from exc
+
+    images_count = 0
+    ##SMY: See marker.output.save_output()  : https://github.com/datalab-to/marker/blob/master/marker/output.py
+    #for img_name, img_bytes in images.items():
+    for img_name, img in images.items():
+        # Resolve the full path and make sure any sub‑directories exist.
+        img_path = Path(output_dir) / src_path / img_name    ##SMY: image files  ##concatenate Path + str
+        img_path.parent.mkdir(parents=True, exist_ok=True)
+
+        #'' '
+        #with img_path.open("wb") as fp:
+        #    fp.write(img_bytes)    ##SMY: write images to markdown folder
+        #images_count += 1
+        #'' '
+        img.save(img_path)    ##SMY: save images (of type PIL.Image.Image) to markdown folder
+        images_count += 1
+
+    return images_count        ##SMY: return number of images
+    #return images.items().count
+    #return len(images)
+'''

llm/hf_client.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+from typing import Iterable, Literal, Optional
+import os
+import time
+import traceback
+from huggingface_hub import InferenceClient, login, logout as hf_logout
+
+from llm.llm_login import login_huggingface, is_login_huggingface
+
+from utils.logger import get_logger
+
+## Get logger instance
+logger = get_logger(__name__)
+
+
+class HFChatClient:
+    """
+    Provider‐agnostic LLM client interface.
+    Encapsulate `huggingface_hub.InferenceClient` setup and chat calls.
+
+    Backends:
+    - model: plain HF model id (e.g., "HuggingFaceH4/zephyr-7b-beta")
+    - provider: provider-routed id (e.g., "openai/gpt-oss-120b:fireworks-ai")
+    - endpoint: full inference endpoint URL (e.g., "http://localhost:1234").
+    """
+
+    def __init__(self,
+        #api_token: str,
+        #model_id: str = "gpt2",
+        provider: str = "huggingface",  ## "huggingface2", "openai"
+        model_id: str = "openai/gpt-oss-120b", ##default_model
+        hf_provider: str = "huggingface",
+        endpoint_url: Optional[str] = None,
+        #backend: Literal["model", "provider", "endpoint"] = [],
+        backend_choice: Optional[str] = None,  #choices=["model-id", "provider", "endpoint"]
+        system_message: str = "",
+        max_tokens: int = 4096,
+        temperature: float = 0.0,
+        top_p: float = 0.1,
+        stream: bool = False,
+        api_token: Optional[str] = None
+        ) -> None:
+        
+        try:
+            self.model_id = model_id
+            self.provider = provider.lower()
+            self.hf_provider = hf_provider.lower()
+            self.endpoint_url = endpoint_url
+            #self.backend = backend
+            #self.backend_literal: Literal["model", "provider", "endpoint"] = (
+            '''
+            self.backend: Literal["model", "provider", "endpoint"] = (
+                "model" if backend_choice == "Hugging Face Model ID" else (
+                    "provider" if backend_choice == "HF Provider Route" else "endpoint")
+                ),
+            '''
+            self.backend: Literal["model", "provider", "endpoint"] = (
+                "model" if backend_choice == "model-id" else (
+                    "provider" if backend_choice == "provider" else "endpoint")
+                )  ## see Gradio backend_choice dropdown
+            self.system_message = system_message
+            self.max_tokens = max_tokens
+            self.temperature = temperature
+            self.top_p = top_p
+            self.stream = stream
+            self.token = api_token if api_token else None   #""  # invalid; preserved
+            #self.token = token or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")  ## not preferred
+
+            self.base_url = "https://router.huggingface.co/v1"  #%22"  #HF API proxy
+        except Exception as exc:
+            #logger.error(f"client_init_failed", extra={"error": str(exc)}")
+            tb = traceback.format_exc()
+            logger.exception(f'✗ client_init_failed", extra={"error": str(exc)}\n{tb}', exc_info=True)
+            raise RuntimeError(f"✗ Failed to initialise client: {exc}\n{tb}")
+
+        ##SMY: //TOBE: Deprecated : Moved to llm.llm_login
+        '''
+        # # Disable implicit token propagation for determinism
+        # Explicitly disable implicit token propagation; we rely on explicit auth or env var
+        os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
+        
+        # Privacy-first login: try interactive CLI first; fallback to provided/env token only if needed
+        try:
+            login()
+            time.sleep(15)  ##SMY pause for login. Helpful: pool async opex 
+            logger.info("hf_login", extra={"mode": "cli"})
+        except Exception as exc:
+            # Respect common env var names; prefer explicit token arg when provided
+            fallback_token = self.token or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
+            if fallback_token:
+                try:
+                    login(token=fallback_token)
+                    self.token = fallback_token
+                    logger.info("hf_login", extra={"mode": "token"})
+                except Exception as exc_token:
+                    logger.warning("hf_login_failed", extra={"error": str(exc_token)})
+            else:
+                logger.warning("hf_login_failed", extra={"error": str(exc)})
+                # Silent fallback; client will still work if token is passed directly
+                #pass
+        '''        
+        login_huggingface(self.token) if not is_login_huggingface() else logger.log(level=20, msg=f"logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+
+    @staticmethod
+    def _normalise_history(history: list, system_message: str, latest_user_message: str) -> list[dict]:
+        """ 
+        `prompt` prefixed by system_message if set
+        Normalise chat history to list of {"role": role, "content": content} dicts.
+        Supports both dict and tuple formats for history items.
+        """
+        messages: list[dict] = []
+        if system_message:
+            messages.append({"role": "system", "content": system_message})
+        for item in history or []:
+            if isinstance(item, dict) and "role" in item and "content" in item:
+                if item["role"] in ("user", "assistant"):
+                    messages.append({"role": item["role"], "content": item["content"]})
+            elif isinstance(item, (list, tuple)) and len(item) == 2:
+                usr, asst = item
+                if usr:
+                    messages.append({"role": "user", "content": usr})
+                if asst:
+                    messages.append({"role": "assistant", "content": asst})
+        messages.append({"role": "user", "content": latest_user_message})
+        return messages
+
+    @staticmethod
+    def _initialise_client(self,
+        backend: Literal["model", "provider", "endpoint"], 
+        model_id: Optional[str] = None, 
+        hf_provider: Optional[str] = None, 
+        endpoint_url: Optional[str] = None, 
+        token: Optional[str] = None) -> InferenceClient:
+
+        try:
+            match backend:
+                case "endpoint" | "model":
+                    logger.debug("_initialise_client: initialising with:", extra={"model":model_id})  ## debug
+                    hf_client = InferenceClient(model=model_id or endpoint_url, token=token)   #endpoint=target)   ##, token=api_token or self.token)
+                    logger.log(20, "client: ", extra={"model":model_id})  ## debug
+                case "provider":
+                    logger.info("_initialise_client: initialising with:", extra={"provider":hf_provider})  ## debug
+                    hf_client = InferenceClient(provider=hf_provider, model=model_id, token=token)  ##, token=api_token or self.token)
+                    #client = client(model = model_id, provider=provider, token=token)   ##target
+                    logger.log(20, "client: ", extra={"backend":backend})  ## debug
+                case _:
+                    raise ValueError("Invalid backend.")
+            return hf_client
+        except Exception as exc:
+            logger.log(40, "_initialise_client: client_init_failed", extra={"error": str(exc)})  ## debug
+            raise RuntimeError(f"_initialise_client: Failed to initialise client: {exc}")
+
+    ## wrap HF client for marker
+    def chat_fn(
+        self,
+        message: str,
+        history: list = [],
+        ) -> Iterable[str]:
+        """    
+        messages = self._normalise_history(history, system_message, message)
+        token = api_token or self.token
+        """
+        ## set prompt and token
+        messages = self._normalise_history(message, history, self.system_message)
+        #token = api_token or self.token
+        #token = self.token  ## redundant
+
+        logger.log(20,"chat: initialising client", extra={
+            "backend": self.backend, "model": self.model_id, "provider": self.hf_provider, "endpoint": self.endpoint_url,
+            "stream": self.stream, "max_tokens": self.max_tokens, "temperature": self.temperature, "top_p": self.top_p,
+        })
+
+        ## initialised client
+        try:
+            client = self._initialise_client(self, self.backend, self.model_id, self.hf_provider, self.endpoint_url, self.token)  #api_token)
+            logger.log(20, "chat: client initialised")  ## debug
+        except Exception as exc:
+            ##logger.error
+            logger.log(40,"chat client_init_failed", extra={"error": str(exc)})
+            raise RuntimeError(f"chat: Failed to initialise client: {exc}")
+        
+        logger.log(20, "chat_start", extra={
+                "backend": self.backend, "model": self.model_id, "provider": self.hf_provider, "endpoint": self.endpoint_url,
+            "stream": self.stream, "max_tokens": self.max_tokens, "temperature": self.temperature, "top_p": self.top_p,
+                })
+        
+        if self.stream:
+            acc = ""
+            for chunk in client.chat_completion(
+                messages=messages,
+                #model=client.model,  ## moved back to client initialise
+                max_tokens=self.max_tokens,
+                stream=True,
+                temperature=self.temperature,
+                top_p=self.top_p,
+            ):
+                delta = getattr(chunk.choices[0].delta, "content", None) or ""
+                if delta:
+                    acc += delta
+                    yield acc
+            return
+
+        result = client.chat_completion(
+            messages=messages,
+            #model=client.model,  ## moved back to client initialised
+            max_tokens=self.max_tokens,
+            stream=False,
+            temperature=self.temperature,
+            top_p=self.top_p,
+            )
+        yield result.choices[0].message.content
+
+        '''
+        ## future consideration
+        response = client.text_generation(
+            #model=model_name,
+            inputs=prompt,
+            parameters={
+                "max_new_tokens": max_new_tokens,
+                "temperature": temperature,
+            },
+        )
+        return response[0].generated_text
+        '''
+
+    def logout(self) -> bool:
+        """Logout from Hugging Face and clear in-process tokens.
+
+        Returns True on success, False otherwise.
+        """
+        try:
+            hf_logout()
+        except Exception as exc:
+            logger.error("hf_logout_failed", extra={"error": str(exc)})
+            return False
+        # Clear process environment tokens
+        for key in ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN"):
+            if key in os.environ:
+                os.environ.pop(key, None)
+        self.token = None
+        logger.info("hf_logout_success")
+        return True
+

llm/llm_login.py

@@ -0,0 +1,70 @@
+from huggingface_hub import login, logout
+import os
+import traceback
+from time import sleep
+from typing import Optional
+
+from utils.logger import get_logger
+
+## Get logger instance
+logger = get_logger(__name__)
+
+def login_huggingface(token: Optional[str] = None):
+    """
+    Login to Hugging Face account. Prioritize CLI login for privacy and determinism.
+
+    Attempts to log in to Hugging Face Hub.
+    First, it tries to log in interactively via the Hugging Face CLI.
+    If that fails, it falls back to using a token provided as an argument or
+    found in the environment variables HF_TOKEN or HUGGINGFACEHUB_API_TOKEN.
+
+    If both methods fail, it logs a warning and continues without logging in.
+    """
+
+    logger.info("Attempting Hugging Face login...")
+        
+    # Disable implicit token propagation for determinism
+    # Explicitly disable implicit token propagation; we rely on explicit auth or env var
+    os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
+
+    token = token
+    # Privacy-first login: try interactive CLI first; fallback to provided/env token only if needed
+    try:
+        login()
+        sleep(5)  ##SMY pause for login. Helpful: pool async opex 
+        logger.info("✔️ hf_login already", extra={"mode": "cli"})
+    except Exception as exc:
+        # Respect common env var names; prefer explicit token arg when provided
+        fallback_token = token or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
+        if fallback_token:
+            try:
+                login(token=fallback_token)
+                token = fallback_token
+                logger.info("✔️ hf_login through fallback", extra={"mode": "token"})  ##SMY: This only displays if token is provided
+            except Exception as exc_token:
+                logger.warning("❌ hf_login_failed", extra={"error": str(exc_token)})
+        else:
+            logger.warning("❌ hf_login_failed", extra={"error": str(exc)})
+            # Silent fallback; client will still work if token is passed directly
+            #pass
+
+def is_login_huggingface():
+    from huggingface_hub import HfApi
+    from huggingface_hub.utils import HfHubHTTPError
+
+    try:
+        HfApi().whoami()
+        logger.log(level=20, msg=("✔️ You are logged in."), extra={"is_logged_in": True})
+        return True
+    except HfHubHTTPError as exc:
+        # A 401 status code indicates an authentication error.
+        if exc.response.status_code == 401:
+            print("⚠️ You are not logged in. You can still access public models.")
+        else:
+            # Handle other HTTP errors if necessary
+            #print(f"An unexpected HTTP error occurred: {exc}")
+            tb = traceback.format_exc()
+            logger.exception(f"✗ An unexpected HTTP error occurred: → {exc}\n{tb}", exc_info=True)
+            #raise RuntimeError(f"✗ An unexpected HTTP error occurred: → {exc}\n{tb}") from exc
+            return False
+        

llm/openai_client.py

@@ -0,0 +1,91 @@
+
+from __future__ import annotations
+
+from typing import Optional   #Iterable, Literal
+#import os
+#import time
+import traceback
+#from huggingface_hub import InferenceClient, login, logout as hf_logout
+
+from llm.llm_login import login_huggingface, is_login_huggingface
+
+import dotenv
+#dotenv.load_dotenv(".env")
+
+
+from utils.logger import get_logger
+
+## Get logger instance
+logger = get_logger(__name__)
+
+
+class OpenAIChatClient:
+    """
+    Provider‐agnostic OpenAI-based LLM client interface.
+    Compatible with `huggingface_hub.InferenceClient` setup and chat calls.
+
+    - base_url="https://router.huggingface.co/v1",
+    - api_key=os.environ["HF_TOKEN"],
+    """
+
+    def __init__(self,
+                 model_id: Optional[str] = None,
+                 hf_provider: Optional[str] = None,
+                 base_url: Optional[str] = "https://router.huggingface.co/v1",  #None,
+                 api_token: Optional[str] = None,
+                 temperature: Optional[float] = 0.2,
+                 top_p: Optional[float] = 0.2,
+                ) -> None:
+        
+        try:
+            openai_api_key_env = dotenv.get_key(".env", "OPENAI_API_KEY")
+            self.model_id = f"{model_id}:{hf_provider}" if hf_provider is not None else model_id  ##concatenate so HF can pipe to Hf provider
+            self.hf_provider = hf_provider
+            self.base_url = base_url  #"https://router.huggingface.co/v1"  #%22"  #HF API proxy
+            #self.token = api_token if api_token else None   ##debug
+            self.token = openai_api_key_env if openai_api_key_env else api_token  #dotenv.get_key(".env", "OPENAI_API_KEY")
+            #self.token = token or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")  ## not preferred
+            login_huggingface(self.token) if not is_login_huggingface() else logger.log(level=20, msg=f"logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+            #self.fake_token = api_token or "a1b2c3" #or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
+            self.openai_api_key = self.token  #self.fake_token
+            self.temperature = temperature
+            self.top_p = top_p
+
+            logger.log(level=2, msg="initialised OpenAIChatClient:", extra={"base_url": self.base_url, "openai_api_key": self.openai_api_key})
+            
+        except Exception as exc:
+            #logger.error(f"OpenAI client_init_failed", extra={"error": str(exc)}")
+            tb = traceback.format_exc()
+            logger.exception(f'✗ OpenAI client_init_failed", extra={"error": str(exc)}\n{tb}', exc_info=True)
+            raise RuntimeError(f"✗ Failed to initialise OpenAI client: {exc}\n{tb}")
+      
+        #login_huggingface(self.token) if not is_login_huggingface() else logger.log(level=20, msg=f"logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+
+####IN PROGRESS
+# 
+"""
+## HuggingFace API-proxy Inference Provider - https://huggingface.co/docs/inference-providers/index?python-clients=openai
+## https://huggingface.co/openai/gpt-oss-20b?inference_api=true&inference_provider=fireworks-ai&language=python&client=openai
+
+import os
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="https://router.huggingface.co/v1",
+    api_key=os.environ["HF_TOKEN"],
+)
+
+stream = client.chat.completions.create(
+    model="openai/gpt-oss-20b:fireworks-ai",
+    messages=[
+        {
+            "role": "user",
+            "content": "What is the capital of France?"
+        }
+    ],
+    stream=True,
+)
+
+for chunk in stream:
+    print(chunk.choices[0].delta.content, end="")
+"""

llm/provider_validator.py

@@ -0,0 +1,116 @@
+"""Validate provider names against Hugging Face Inference Providers list.
+
+Source: https://huggingface.co/docs/inference-providers/index
+
+Functions:
+- get_supported_providers() -> set[str]
+- normalize_provider(text: str) -> str | None
+- is_valid_provider(text: str) -> bool
+- suggest_providers(text: str, limit: int = 3) -> list[str]
+
+Supports common aliases (e.g., "together-ai" -> "together", "fireworks" -> "fireworks-ai").
+"""
+
+from __future__ import annotations
+
+from difflib import get_close_matches
+from typing import Iterable
+
+
+# Canonical provider slugs from docs (table and provider URLs)
+_CANONICAL: set[str] = {
+    "cerebras",
+    "cohere",
+    "fal-ai",
+    "featherless-ai",
+    "fireworks-ai",
+    "groq",
+    "hf-inference",
+    "hyperbolic",
+    "nebius",
+    "novita",
+    "nscale",
+    "replicate",
+    "sambanova",
+    "together",
+}
+
+# Common aliases users may type; maps to canonical slug
+_ALIASES: dict[str, str] = {
+    "together-ai": "together",
+    "fireworks": "fireworks-ai",
+    "falai": "fal-ai",
+    "featherless": "featherless-ai",
+    "hf": "hf-inference",
+    "huggingface": "hf-inference",
+}
+
+
+def _to_key(text: str) -> str:
+    return (text or "").strip().lower()
+
+
+def get_supported_providers(extra: Iterable[str] | None = None) -> set[str]:
+    """Return set of canonical provider slugs.
+
+    Optionally extend with additional slugs via `extra`.
+    """
+    return _CANONICAL | set(map(_to_key, (extra or [])))
+
+
+def normalize_provider(text: str) -> str | None:
+    """Return canonical provider slug for `text`, if known; else None.
+
+    Accepts canonical slugs and common aliases.
+    """
+    key = _to_key(text)
+    if not key:
+        return None
+    if key in _CANONICAL:
+        return key
+    if key in _ALIASES:
+        return _ALIASES[key]
+    return None
+
+
+def is_valid_provider(text: str) -> bool:
+    """True if `text` is a known provider or alias."""
+    return normalize_provider(text) is not None
+
+
+def suggest_providers(text: str, limit: int = 3) -> list[str]:
+    """Suggest close canonical matches for `text`.
+
+    Uses difflib to match against canonical slugs; returns up to `limit` suggestions.
+    """
+    key = _to_key(text)
+    if not key:
+        return []
+    # Search both canonical and alias keys to be helpful, then map to canonical
+    candidates = list(_CANONICAL | set(_ALIASES))
+    suggestions = get_close_matches(key, candidates, n=limit, cutoff=0.6)
+    canon = []
+    for s in suggestions:
+        canon_slug = s if s in _CANONICAL else _ALIASES.get(s)
+        if canon_slug and canon_slug not in canon:
+            canon.append(canon_slug)
+    return canon[:limit]
+
+
+if __name__ == "__main__":
+    import sys
+
+    query = " ".join(sys.argv[1:])
+    if not query:
+        print("Usage: python provider_validator.py <provider-name>")
+        raise SystemExit(2)
+
+    norm = normalize_provider(query)
+    if norm:
+        print(f"valid: {norm}")
+    else:
+        print("invalid")
+        suggestions = suggest_providers(query)
+        if suggestions:
+            print("did_you_mean:", ", ".join(suggestions))
+

main.py

@@ -0,0 +1,22 @@
+# main.py
+import os
+from pathlib import Path
+
+from ui.gradio_ui import build_interface
+from utils.logger import get_logger, setup_logging
+
+setup_logging()  ## set logging
+#logger = get_logger("pypdfmd")
+logger = get_logger("parserpdf")
+
+if __name__ == "__main__":
+    # Ensure the working directory is clean
+    #os.chdir(os.path.dirname(__file__))
+    ## script working dir absolute path
+    script_dir = Path(__file__).resolve().parent
+    ## change the cwd to the script's dir
+    os.chdir(script_dir)    ##Path.cwd()
+
+    demo = build_interface()
+    #demo.launch(debug=True, show_error=True ,ssr_mode=True)  #(share=True)  # share=True for public link; remove in production
+    demo.launch(debug=True, show_error=True)

pyproject.toml

@@ -0,0 +1,9 @@
+[project]
+#name = "parserpdf"
+name = "parser2md"
+version = "0.1.0"
+description = "PDF & HTML parser to markdown"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = []
+owner "research-semmyk"

requirements.txt

@@ -0,0 +1,5 @@
+gradio>=4.0
+#marker==1.3.0          # pip install marker (GitHub: https://github.com/datalab-to/marker)
+#pandoc==2.3            # for Markdown → PDF conversion
+#weasyprint==59.0       # optional fallback if pandoc is not available
+python-magic==0.4.27   # file‑type detection

tests/test_converters.py

@@ -0,0 +1,98 @@
+# tests/test_converters.py
+# run with pytest tests/.
+
+import pytest
+import os
+from unittest.mock import patch, MagicMock
+from pathlib import Path
+
+from converters.pdf_to_md import PdfToMarkdownConverter
+from converters.html_to_md import HtmlToMarkdownConverter
+from converters.md_to_pdf import MarkdownToPdfConverter
+from converters.extraction_converter import DocumentConverter
+
+@pytest.fixture
+def sample_pdf_path():
+    # Create a temporary PDF file for testing
+    pdf_path = Path("tests/sample.pdf")
+    pdf_path.write_bytes(b"%PDF-1.4\nSample PDF content")
+    yield pdf_path
+    if pdf_path.exists():
+        pdf_path.unlink()
+
+@pytest.fixture
+def sample_html_path():
+    html_path = Path("tests/sample.html")
+    html_path.write_text("<html><body><h1>Test</h1><p>Hello World</p></body></html>")
+    yield html_path
+    if html_path.exists():
+        html_path.unlink()
+
+@pytest.fixture
+def sample_md_path():
+    md_path = Path("tests/sample.md")
+    md_path.write_text("# Test\nHello World")
+    yield md_path
+    if md_path.exists():
+        md_path.unlink()
+
+def test_pdf_to_markdown_converter_init():
+    converter = PdfToMarkdownConverter()
+    assert isinstance(converter, PdfToMarkdownConverter)
+    assert hasattr(converter, 'output_dir_string')
+
+@patch('converters.pdf_to_md.Marker')  # Assuming Marker is imported in pdf_to_md.py
+def test_pdf_to_markdown_convert_file(mock_marker, sample_pdf_path):
+    mock_marker.convert_single.return_value = {"markdown": "# Converted\nContent", "images": []}
+    
+    converter = PdfToMarkdownConverter()
+    result = converter.convert_file(sample_pdf_path)
+    
+    assert isinstance(result, dict)
+    assert "markdown" in result
+    assert "filepath" in result
+    mock_marker.convert_single.assert_called_once_with(str(sample_pdf_path), prefer_latex=False)
+
+def test_html_to_markdown_converter(sample_html_path):
+    converter = HtmlToMarkdownConverter()
+    result = converter.batch_convert([sample_html_path])
+    
+    assert isinstance(result, dict)
+    assert Path(sample_html_path.name) in result
+    assert result[Path(sample_html_path.name)].startswith("# Test")
+
+def test_markdown_to_pdf_converter(sample_md_path):
+    converter = MarkdownToPdfConverter()
+    output_dir = Path("tests/output_pdf")
+    output_dir.mkdir(exist_ok=True)
+    
+    pdf_files = converter.batch_convert([sample_md_path], output_dir)
+    
+    assert isinstance(pdf_files, list)
+    if pdf_files:
+        pdf_path = pdf_files[0]
+        assert pdf_path.exists()
+        assert pdf_path.suffix == ".pdf"
+        pdf_path.unlink()
+    
+    output_dir.rmdir()
+
+@patch('converters.extraction_converter.get_token')
+def test_document_converter_login(mock_get_token):
+    mock_get_token.return_value = "test_token"
+    converter = DocumentConverter()
+    assert converter.client.token == "test_token"
+
+def test_pdf_to_markdown_batch_convert(tmp_path):
+    # Test batch with multiple files
+    pdf1 = tmp_path / "test1.pdf"
+    pdf2 = tmp_path / "test2.pdf"
+    pdf1.write_bytes(b"%PDF-1.4")
+    pdf2.write_bytes(b"%PDF-1.4")
+    
+    converter = PdfToMarkdownConverter()
+    with patch.object(converter, 'convert_file', return_value={"markdown": "test", "filepath": str(pdf1)}):
+        results = converter.batch_convert([pdf1, pdf2])
+    
+    assert len(results) == 2
+    assert all("markdown" in res for res in results)

tests/test_file_handler.py

@@ -0,0 +1,115 @@
+# tests/test_file_handler.py
+# run with pytest tests/.
+
+import pytest
+from pathlib import Path
+import tempfile
+from unittest.mock import patch
+
+from file_handler.file_utils import (
+    collect_pdf_paths, collect_html_paths, collect_markdown_paths,
+    process_dicts_data, create_outputdir
+)
+
+@pytest.fixture
+def temp_dir_with_pdfs():
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        tmpdir = Path(tmpdirname)
+        # Create sample PDF files
+        (tmpdir / "doc1.pdf").touch()
+        (tmpdir / "subfolder/doc2.pdf").mkdir(parents=True)
+        (tmpdir / "subfolder/doc2.pdf").touch()
+        (tmpdir / "not_pdf.txt").touch()
+        yield tmpdir
+
+@pytest.fixture
+def temp_dir_with_html():
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        tmpdir = Path(tmpdirname)
+        (tmpdir / "page1.html").touch()
+        (tmpdir / "subfolder/page2.htm").mkdir(parents=True)
+        (tmpdir / "subfolder/page2.htm").touch()
+        (tmpdir / "not_html.md").touch()
+        yield tmpdir
+
+@pytest.fixture
+def temp_dir_with_md():
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        tmpdir = Path(tmpdirname)
+        (tmpdir / "note1.md").touch()
+        (tmpdir / "subfolder/note2.md").mkdir(parents=True)
+        (tmpdir / "subfolder/note2.md").touch()
+        (tmpdir / "not_md.pdf").touch()
+        yield tmpdir
+
+def test_collect_pdf_paths(temp_dir_with_pdfs):
+    paths = collect_pdf_paths(str(temp_dir_with_pdfs))
+    assert len(paths) == 2
+    assert all(p.suffix.lower() == '.pdf' for p in paths)
+    assert Path(str(temp_dir_with_pdfs) / "doc1.pdf") in paths
+    assert Path(str(temp_dir_with_pdfs) / "subfolder/doc2.pdf") in paths
+
+def test_collect_pdf_paths_no_pdfs(temp_dir_with_html):
+    paths = collect_pdf_paths(str(temp_dir_with_html))
+    assert len(paths) == 0
+
+def test_collect_html_paths(temp_dir_with_html):
+    paths = collect_html_paths(str(temp_dir_with_html))
+    assert len(paths) == 2
+    assert all(p.suffix.lower() in ['.html', '.htm'] for p in paths)
+    assert Path(str(temp_dir_with_html) / "page1.html") in paths
+    assert Path(str(temp_dir_with_html) / "subfolder/page2.htm") in paths
+
+def test_collect_html_paths_no_html(temp_dir_with_pdfs):
+    paths = collect_html_paths(str(temp_dir_with_pdfs))
+    assert len(paths) == 0
+
+def test_collect_markdown_paths(temp_dir_with_md):
+    paths = collect_markdown_paths(str(temp_dir_with_md))
+    assert len(paths) == 2
+    assert all(p.suffix.lower() == '.md' for p in paths)
+    assert Path(str(temp_dir_with_md) / "note1.md") in paths
+    assert Path(str(temp_dir_with_md) / "subfolder/note2.md") in paths
+
+def test_collect_markdown_paths_no_md(temp_dir_with_pdfs):
+    paths = collect_markdown_paths(str(temp_dir_with_pdfs))
+    assert len(paths) == 0
+
+def test_process_dicts_data():
+    sample_logs = [
+        {"filepath": Path("file1.md"), "markdown": "Content1", "image_path": ["img1.jpg"]},
+        {"filepath": Path("file2.md"), "markdown": "Content2", "image_path": []},
+        {"error": "Conversion failed for file3"}
+    ]
+    result = process_dicts_data(sample_logs)
+    assert "file1.md" in result
+    assert "Content1" in result
+    assert "img1.jpg" in result
+    assert "Conversion failed" in result
+
+def test_process_dicts_data_empty():
+    result = process_dicts_data([])
+    assert result == ""
+
+def test_process_dicts_data_invalid():
+    with pytest.raises(ValueError):
+        process_dicts_data([{"invalid": "data"}])
+
+def test_create_outputdir(tmp_path):
+    output_dir = tmp_path / "test_output"
+    create_outputdir(str(output_dir))
+    assert output_dir.exists()
+    assert output_dir.is_dir()
+
+def test_create_outputdir_existing(tmp_path):
+    output_dir = tmp_path / "test_output"
+    output_dir.mkdir()
+    create_outputdir(str(output_dir))
+    assert output_dir.exists()
+    assert output_dir.is_dir()
+
+@patch('pathlib.Path.mkdir')
+def test_create_outputdir_error(mock_mkdir):
+    mock_mkdir.side_effect = OSError("Permission denied")
+    with pytest.raises(OSError):
+        create_outputdir("protected_dir")

tests/test_llm.py

@@ -0,0 +1,115 @@
+# tests/test_llm.py
+# run with pytest tests/.
+# 
+# import pytest
+from unittest.mock import patch, MagicMock
+import huggingface_hub
+from huggingface_hub import get_token
+
+from llm.llm_login import login_huggingface
+from llm.provider_validator import is_valid_provider, suggest_providers
+from llm.hf_client import HFChatClient  # Assuming this exists
+from llm.openai_client import OpenAIClient  # Assuming this exists
+
+def test_login_huggingface_success():
+    with patch('huggingface_hub.login') as mock_login:
+        api_token = "hf_test_token"
+        login_huggingface(api_token)
+        mock_login.assert_called_once_with(token=api_token, add_to_git_credential=False)
+
+def test_login_huggingface_no_token():
+    with patch('huggingface_hub.login') as mock_login:
+        with pytest.raises(ValueError, match="API token required"):
+            login_huggingface(None)
+
+@patch('huggingface_hub.login')
+def test_login_huggingface_error(mock_login):
+    mock_login.side_effect = Exception("Login failed")
+    with pytest.raises(Exception, match="Login failed"):
+        login_huggingface("invalid_token")
+
+def test_is_valid_provider():
+    assert is_valid_provider("huggingface") is True
+    assert is_valid_provider("openai") is True
+    assert is_valid_provider("invalid_provider") is False
+    assert is_valid_provider("") is False
+    assert is_valid_provider(None) is False
+
+def test_suggest_providers():
+    suggestions = suggest_providers("hugngface")  # Typo example
+    assert isinstance(suggestions, list)
+    assert "huggingface" in suggestions
+    
+    no_suggestions = suggest_providers("completely_unknown")
+    assert isinstance(no_suggestions, list)
+    assert len(no_suggestions) == 0
+
+@patch('llm.hf_client.HFChatClient.__init__')
+def test_hf_client_init(mock_init):
+    mock_init.return_value = None
+    client = HFChatClient(model_id="test-model", api_token="test_token")
+    mock_init.assert_called_once_with(
+        model_id="test-model",
+        api_token="test_token",
+        # Add other expected params based on actual __init__
+    )
+
+@patch('llm.hf_client.login_huggingface')
+@patch('llm.hf_client.get_token')
+def test_hf_client_token(mock_get_token, mock_login):
+    mock_get_token.return_value = "cached_token"
+    mock_login.return_value = None
+    
+    client = HFChatClient(model_id="test-model")
+    assert client.api_token == "cached_token"
+
+@patch('openai.OpenAI')
+def test_openai_client_init(mock_openai):
+    mock_client = MagicMock()
+    mock_openai.return_value = mock_client
+    
+    client = OpenAIClient(api_key="sk_test_key", base_url="https://api.openai.com/v1")
+    mock_openai.assert_called_once_with(
+        api_key="sk_test_key",
+        base_url="https://api.openai.com/v1"
+    )
+    assert client.client == mock_client
+
+@patch('openai.OpenAI')
+def test_openai_client_chat(mock_openai):
+    mock_client = MagicMock()
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock(content="Hello!")]
+    mock_client.chat.completions.create.return_value = mock_response
+    mock_openai.return_value = mock_client
+    
+    client = OpenAIClient(api_key="sk_test_key")
+    response = client.chat("Hello", model="gpt-3.5-turbo")
+    
+    assert response == "Hello!"
+    mock_client.chat.completions.create.assert_called_once_with(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+
+def test_provider_validator_edge_cases():
+    # Test with non-string inputs
+    assert is_valid_provider(123) is False
+    assert suggest_providers(123) == []
+    
+    # Test case insensitivity
+    assert is_valid_provider("HUGGINGFACE") is True
+    assert is_valid_provider("OpEnAi") is True
+
+@patch('huggingface_hub.get_token')
+def test_get_token_from_env(mock_get_token):
+    mock_get_token.return_value = None
+    with patch.dict('os.environ', {'HUGGINGFACE_HUB_TOKEN': 'env_token'}):
+        token = get_token()
+        assert token == 'env_token'
+
+@patch('huggingface_hub.get_token')
+def test_get_token_from_cache(mock_get_token):
+    mock_get_token.return_value = 'cached_token'
+    token = get_token()
+    assert token == 'cached_token'

tests/test_main_ui.py

@@ -0,0 +1,148 @@
+# tests/test_main_ui.py
+# run with pytest tests/.
+
+import pytest
+from unittest.mock import patch, MagicMock
+from pathlib import Path
+
+from main import build_interface  # Wait, main imports from ui, but test main logic
+from ui.gradio_ui import convert_batch, build_interface, accumulate_files, clear_state, pdf_files_wrap
+from utils.logger import get_logger
+
+logger = get_logger("test_main_ui")
+
+@pytest.fixture
+def mock_gradio():
+    with patch('gradio.Blocks') as mock_blocks, \
+         patch('gradio.Markdown') as mock_md, \
+         patch('gradio.Accordion') as mock_accordion, \
+         patch('gradio.Dropdown') as mock_dropdown, \
+         patch('gradio.Textbox') as mock_textbox, \
+         patch('gradio.Slider') as mock_slider, \
+         patch('gradio.Checkbox') as mock_checkbox, \
+         patch('gradio.Button') as mock_button, \
+         patch('gradio.File') as mock_file, \
+         patch('gradio.UploadButton') as mock_upload, \
+         patch('gradio.State') as mock_state, \
+         patch('gradio.Tab') as mock_tab, \
+         patch('gradio.JSON') as mock_json, \
+         patch('gradio.Files') as mock_files, \
+         patch('gradio.Gallery') as mock_gallery:
+        yield {
+            'Blocks': mock_blocks, 'Markdown': mock_md, 'Accordion': mock_accordion,
+            'Dropdown': mock_dropdown, 'Textbox': mock_textbox, 'Slider': mock_slider,
+            'Checkbox': mock_checkbox, 'Button': mock_button, 'File': mock_file,
+            'UploadButton': mock_upload, 'State': mock_state, 'Tab': mock_tab,
+            'JSON': mock_json, 'Files': mock_files, 'Gallery': mock_gallery
+        }
+
+def test_build_interface(mock_gradio):
+    demo = build_interface()
+    assert demo is not None
+    # Verify UI components are created
+    mock_gradio['Blocks'].assert_called_once_with(title="parserPDF", css=MagicMock())
+    mock_gradio['Markdown'].assert_called()  # Title markdown
+    mock_gradio['Accordion'].assert_any_call("⚙️ LLM Model Settings", open=False)
+    mock_gradio['Tab'].assert_any_call(" 📄 PDF & HTML ➜ Markdown")
+
+def test_convert_batch_no_files():
+    result = convert_batch([], 0, "huggingface", "test-model", "fireworks-ai", "", "model-id", 
+                          "system", 1024, 0.0, 0.1, False, "token", 
+                          "https://router.huggingface.co/v1", "webp", 4, 2, "markdown", 
+                          "output_dir", False, None)
+    assert "No files uploaded" in result[0]
+
+@patch('ui.gradio_ui.login_huggingface')
+@patch('ui.gradio_ui.ProcessPoolExecutor')
+@patch('ui.gradio_ui.pdf2md_converter.convert_files')
+def test_convert_batch_success(mock_convert, mock_pool, mock_login):
+    mock_result = MagicMock()
+    mock_convert.return_value = {"filepath": Path("test.md"), "image_path": ["img.jpg"], "markdown": "content"}
+    mock_pool.return_value.__enter__.return_value.map.return_value = [mock_result]
+    mock_login.return_value = None
+    
+    pdf_files = [MagicMock(name="test.pdf")]
+    result = convert_batch(pdf_files, 1, "huggingface", "test-model", "fireworks-ai", "", "model-id", 
+                          "system", 1024, 0.0, 0.1, False, "token", 
+                          "https://router.huggingface.co/v1", "webp", 4, 2, "markdown", 
+                          "output_dir", False, None)
+    
+    assert len(result) == 3
+    assert "test.md" in result[0]
+    assert "img.jpg" in result[2][0]
+    mock_pool.assert_called_once()
+    mock_convert.assert_called_once_with("test.pdf")
+
+@patch('ui.gradio_ui.ProcessPoolExecutor')
+def test_convert_batch_pool_error(mock_pool):
+    mock_pool.side_effect = Exception("Pool error")
+    pdf_files = [MagicMock(name="test.pdf")]
+    result = convert_batch(pdf_files, 1, "huggingface", "test-model", "fireworks-ai", "", "model-id", 
+                          "system", 1024, 0.0, 0.1, False, "token", 
+                          "https://router.huggingface.co/v1", "webp", 4, 2, "markdown", 
+                          "output_dir", False, None)
+    assert "Error during ProcessPoolExecutor" in result[0]
+
+def test_accumulate_files():
+    # Test initial accumulation
+    new_files = [MagicMock(name="/tmp/file1.pdf"), MagicMock(name="/tmp/file2.html")]
+    state = []
+    updated_state, message = accumulate_files(new_files, state)
+    assert len(updated_state) == 2
+    assert "/tmp/file1.pdf" in updated_state
+    assert "Accumulated 2 file(s)" in message
+    
+    # Test adding to existing state
+    new_files2 = [MagicMock(name="/tmp/file3.pdf")]
+    updated_state2, message2 = accumulate_files(new_files2, updated_state)
+    assert len(updated_state2) == 3
+    assert "Accumulated 3 file(s)" in message2
+    
+    # Test no new files
+    _, message3 = accumulate_files([], updated_state2)
+    assert "No new files uploaded" in message3
+
+def test_clear_state():
+    result = clear_state()
+    assert len(result) == 4
+    assert result[0] == []  # cleared file list
+    assert result[1] == "Files list cleared."  # message
+    assert result[2] == []  # cleared file btn
+    assert result[3] == []  # cleared dir btn
+
+def test_pdf_files_wrap():
+    # Single file
+    single_file = "single.pdf"
+    wrapped = pdf_files_wrap(single_file)
+    assert isinstance(wrapped, list)
+    assert len(wrapped) == 1
+    assert wrapped[0] == single_file
+    
+    # List of files
+    files_list = ["file1.pdf", "file2.html"]
+    wrapped_list = pdf_files_wrap(files_list)
+    assert wrapped_list == files_list
+    
+    # None input
+    assert pdf_files_wrap(None) == [None]
+
+@patch('ui.gradio_ui.os.chdir')
+@patch('ui.gradio_ui.Path')
+def test_main_launch(mock_path, mock_chdir):
+    mock_script_dir = MagicMock()
+    mock_path.return_value.resolve.return_value.parent = mock_script_dir
+    mock_chdir.return_value = None
+    
+    # Test main execution path
+    with patch('builtins.__name__', '__main__'):
+        from main import main  # Assuming main has a main function, or test the if __name__ logic indirectly
+        # Since main.py is simple, test the key parts
+        demo = MagicMock()
+        with patch('ui.gradio_ui.build_interface', return_value=demo):
+            with patch('gradio.Interface.launch') as mock_launch:
+                # Execute main logic
+                import main
+                main.main()  # If it has main(), or just import runs it
+                
+                mock_chdir.assert_called_once_with(mock_script_dir)
+                mock_launch.assert_called_once_with(debug=True, show_error=True)

tests/test_utils.py

@@ -0,0 +1,94 @@
+# tests/test_utils.py
+# run with pytest tests/.
+# 
+# import pytest
+import logging
+from unittest.mock import patch, MagicMock
+from pathlib import Path
+
+from utils.logger import get_logger, setup_logging
+from utils.utils import is_dict, is_list_of_dicts
+from utils.config import TITLE, DESCRIPTION  # Assuming these are defined
+from utils.get_config import get_config_value  # If separate module
+
+def test_setup_logging(capsys):
+    setup_logging()
+    captured = capsys.readouterr()
+    assert "Logging configured" in captured.out or captured.err  # Assuming it prints config message
+
+def test_get_logger():
+    logger = get_logger("test_logger")
+    assert isinstance(logger, logging.Logger)
+    assert logger.name == "test_logger"
+
+@patch('logging.getLogger')
+def test_get_logger_custom(mock_get_logger):
+    mock_logger = MagicMock()
+    mock_get_logger.return_value = mock_logger
+    logger = get_logger("custom_test")
+    mock_get_logger.assert_called_once_with("custom_test")
+    assert logger == mock_logger
+
+def test_is_dict():
+    assert is_dict({"key": "value"}) is True
+    assert is_dict({"key": [1, 2]}) is True
+    assert is_dict([]) is False
+    assert is_dict("string") is False
+    assert is_dict(123) is False
+    assert is_dict(None) is False
+
+def test_is_list_of_dicts():
+    assert is_list_of_dicts([{"a": 1}, {"b": 2}]) is True
+    assert is_list_of_dicts([]) is False  # Empty list not considered list of dicts
+    assert is_list_of_dicts([{"a": 1}, "string"]) is False
+    assert is_list_of_dicts("not_list") is False
+    assert is_list_of_dicts([1, 2]) is False
+    assert is_list_of_dicts(None) is False
+
+def test_config_constants():
+    # Test if config values are as expected (update based on actual config.py)
+    assert TITLE == "parserPDF"  # Or whatever the actual value is
+    assert DESCRIPTION.startswith("PDF parser")  # Partial match for description
+
+@patch('utils.get_config.configparser.ConfigParser')
+def test_get_config_value(mock_configparser):
+    mock_config = MagicMock()
+    mock_config.get.return_value = "test_value"
+    mock_configparser.return_value = mock_config
+    
+    value = get_config_value("SECTION", "KEY")
+    mock_config.get.assert_called_once_with("SECTION", "KEY")
+    assert value == "test_value"
+
+@patch('utils.get_config.configparser.ConfigParser')
+def test_get_config_value_default(mock_configparser):
+    mock_config = MagicMock()
+    mock_config.get.side_effect = KeyError("No such key")
+    mock_configparser.return_value = mock_config
+    
+    value = get_config_value("SECTION", "NONEXISTENT", default="fallback")
+    assert value == "fallback"
+    mock_config.get.assert_called_once_with("SECTION", "NONEXISTENT")
+
+def test_logger_levels(caplog):
+    # Test logging at different levels
+    logger = get_logger("level_test")
+    
+    with caplog.at_level(logging.DEBUG):
+        logger.debug("Debug message")
+        assert "Debug message" in caplog.text
+    
+    with caplog.at_level(logging.INFO):
+        logger.info("Info message")
+        assert "Info message" in caplog.text
+    
+    with caplog.at_level(logging.ERROR):
+        logger.error("Error message")
+        assert "Error message" in caplog.text
+
+def test_setup_logging_file(tmp_path):
+    log_file = tmp_path / "test.log"
+    with patch.dict('os.environ', {'LOG_FILE': str(log_file)}):
+        setup_logging()
+        assert log_file.exists()
+        log_file.unlink()  # Cleanup

tests/tests_converter.py

@@ -0,0 +1,19 @@
+# tests/test_conversion.py
+# run with pytest tests/.
+
+from pathlib import Path
+from converters.pdf_to_md import PdfToMarkdownConverter
+from converters.md_to_pdf import MarkdownToPdfConverter
+
+def test_sample_pdf():
+    pdf = Path("tests/sample.pdf")
+    converter = PdfToMarkdownConverter()
+    md = converter.convert(pdf)
+    assert isinstance(md, str) and len(md) > 0
+
+def test_markdown_to_pdf(tmp_path):
+    md_file = tmp_path / "test.md"
+    md_file.write_text("# Hello\nThis is a test.")
+    conv = MarkdownToPdfConverter()
+    pdf_path = conv.convert(md_file)
+    assert pdf_path.exists() and pdf_path.suffix == ".pdf"

ui/gradio_ui.py

@@ -0,0 +1,850 @@
+# ui/gradio_ui.py
+import gradio as gr
+
+from pathlib import Path, WindowsPath
+import traceback  ## Extract, format and print information about Python stack traces.
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from typing import Optional, Union #, Dict, List, Any, Tuple
+
+from huggingface_hub import get_token
+import file_handler
+import file_handler.file_utils
+from utils.config import TITLE, DESCRIPTION, DESCRIPTION_PDF_HTML, DESCRIPTION_PDF, DESCRIPTION_HTML, DESCRIPTION_MD
+from utils.utils import is_dict, is_list_of_dicts
+from file_handler.file_utils import process_dicts_data, collect_pdf_paths, collect_html_paths, collect_markdown_paths, create_outputdir  ## should move to handling file
+#from llm.hf_client import HFChatClient  ## SMY: unused. See converters.extraction_converter
+from llm.provider_validator import is_valid_provider, suggest_providers
+from llm.llm_login import login_huggingface
+
+from converters.extraction_converter import DocumentConverter as docconverter  #DocumentExtractor #as docextractor
+from converters.pdf_to_md import PdfToMarkdownConverter, init_worker
+from converters.md_to_pdf import MarkdownToPdfConverter
+from converters.html_to_md import HtmlToMarkdownConverter
+
+from utils.get_config import get_config_value
+from utils.logger import get_logger
+
+logger = get_logger(__name__)   ##NB: setup_logging()  ## set logging
+
+# Instantiate converters class once – they are stateless
+pdf2md_converter = PdfToMarkdownConverter()
+#html2md_converter = HtmlToMarkdownConverter()
+md2pdf_converter = MarkdownToPdfConverter()
+
+
+# pool executor to convert files called by Gradio
+def convert_batch(
+    pdf_files, #: list[str],
+    pdf_files_count: int,
+    provider: str,
+    model_id: str,
+    #base_url: str
+    hf_provider: str,
+    endpoint: str,
+    backend_choice: str,
+    system_message: str,
+    max_tokens: int,
+    temperature: float,
+    top_p: float,
+    stream: bool,
+    api_token: str,
+    #max_workers: int,
+    #max_retries: int,
+    openai_base_url: str = "https://router.huggingface.co/v1",
+    openai_image_format: Optional[str] = "webp",
+    max_workers: Optional[int] = 4,
+    max_retries: Optional[int] = 2,
+    output_format: str = "markdown",
+    #output_dir: Optional[Union[str, Path]] = "output_dir",
+    output_dir_string: str = "output_dir_default",
+    use_llm: bool = False,   #Optional[bool] = False,  #True,
+    page_range: str = None,  #Optional[str] = None,  
+) -> str:
+    """
+    Handles the conversion process using multiprocessing.
+    Spins up a pool and converts all uploaded files in parallel.
+    Aggregates per-file logs into one string.
+    Receives Gradio component values, starting with the list of uploaded file paths
+    """
+
+    # explicitly wrap file object in a list
+    #pdf_files = pdf_files_wrap(pdf_files)   ##Flag:  deprecation
+
+    ## debug
+    #logger.log(level=30, msg="pdf_files_inputs", extra={"input_arg[0]:": pdf_files[0]})
+
+    #if not files:
+    if not pdf_files or pdf_files is None:  ## Check if files is None. This handles the case where no files are uploaded.
+        logger.log(level=30, msg="Initialising ProcessPool: No files uploaded.", extra={"pdf_files": pdf_files, "files_len": pdf_files_count})
+        return "Initialising ProcessPool: No files uploaded."
+    
+    # Get config values if not provided
+    model_id = get_config_value("MARKER_CAP", "MODEL_ID") if not model_id else model_id
+    openai_base_url = get_config_value( "MARKER_CAP", "OPENAI_BASE_URL") if not openai_base_url else openai_base_url
+    openai_image_format = get_config_value( "MARKER_CAP", "OPENAI_IMAGE_FORMAT") if not openai_image_format else openai_image_format
+    max_workers = get_config_value("MARKER_CAP", "MAX_WORKERS") if not max_workers else max_workers
+    max_retries = get_config_value("MARKER_CAP", "MAX_RETRIES") if not max_retries else max_retries
+    output_format = get_config_value("MARKER_CAP", "OUTPUT_FORMAT") if not output_format else output_format
+    output_dir_string = str(get_config_value("MARKER_CAP", "OUTPUT_DIR") if not output_dir_string else output_dir_string)
+    use_llm = get_config_value("MARKER_CAP", "USE_LLM") if not use_llm else use_llm
+    page_range = get_config_value("MARKER_CAP", "PAGE_RANGE") if not page_range else page_range
+         
+    # Create the initargs tuple from the Gradio inputs: # 'files' is an iterable, and handled separately.
+    init_args = (
+            provider,            
+            model_id,
+            #base_url,
+            hf_provider,
+            endpoint,
+            backend_choice,
+            system_message,
+            max_tokens,
+            temperature,
+            top_p,
+            stream,
+            api_token,
+            openai_base_url,
+            openai_image_format,
+            max_workers,
+            max_retries,
+            output_format,
+            output_dir_string,
+            use_llm,
+            page_range,
+        )
+    
+    #global docextractor   ##SMY: deprecated.
+    try:
+        login_huggingface(api_token)  ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+    except Exception as exc:  # Catch all exceptions
+        tb = traceback.format_exc()
+        logger.exception(f"✗ Error during login_huggingface → {exc}\n{tb}", exc_info=True) # Log the full traceback
+        return f"✗ An error occurred during login_huggingface → {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+
+    try:
+        # Create a pool with init_worker initialiser
+        with ProcessPoolExecutor(
+            max_workers=max_workers,
+            initializer=init_worker,
+            initargs=init_args
+        ) as pool:
+            #global docextractor
+            logger.log(level=30, msg="Initialising ProcessPool: pool:", extra={"pdf_files": pdf_files, "files_len": len(pdf_files), "model_id": model_id, "output_dir": output_dir_string})  #pdf_files_count
+            
+            # Map the files (pdf_files) to the conversion function (pdf2md_converter.convert_file)
+            # The 'docconverter' argument is implicitly handled by the initialiser
+            
+            #futures = [pool.map(pdf2md_converter.convert_files, f) for f in pdf_files]
+            #logs = [f.result() for f in as_completed(futures)]
+            #futures = [pool.submit(pdf2md_converter.convert_files, file) for file in pdf_files]
+            #logs = [f.result() for f in futures]
+            
+            try:
+                pdf2md_converter.output_dir_string = output_dir_string   ##SMY: attempt setting directly to resolve pool.map iterable
+                #result_convert = pool.map(pdf2md_converter.convert_files, pdf_files, max_retries)
+                results = pool.map(pdf2md_converter.convert_files, pdf_files)  ##SMY iterables  #output_dir_string)
+            except Exception as exc:
+                # Raise the exception to stop the Gradio app
+                #raise  # Re-raise the exception to halt execution
+                logger.exception("Error during pooling file conversion", exc_info=True)  # Log the full traceback
+                traceback.print_exc()  # Print the exception traceback
+                return f"An error occurred during pool.map: {str(exc)}", f"Error: {exc}", f"Error: {exc}"  ## return the exception message
+    
+            #'''
+            logger.log(level=20, msg="ProcessPoolExecutor pool result:", extra={"results": str(results)})
+            logs = []
+            logs_files_images = []
+            #logs.extend(results)   ## performant pythonic
+            #logs = list[results]  ## 
+            logs = [result for result in results]  ## pythonic list comprehension
+            ## logs : [file , images , filepath, image_path]
+            
+            #logs_files_images = logs_files.extend(logs_images)  #zip(logs_files, logs_images)   ##SMY: in progress
+            for log in logs:
+                #logs_files_images.append(log.get("filepath", "Error or No filepath"))  # if all(isinstance(log, dict) for item in logs))
+                #logs_files_images.extend(list(image for image in log.get("image_path", "Error or no image_path")))
+
+                logs_files_images.append(log.get("filepath") if is_dict(logs) or isinstance(log, Path) else "Error or no image_path")  # isinstance(log, (dict, str))
+                logs_files_images.extend(list(image for image in log.get("image_path", "Error or no image_path")))
+                
+            
+            #logs_files_images.append(logs_filepath) ## to del
+            #logs_files_images.extend(logs_images)   ## to del
+            #'''
+    except Exception as exc:
+        tb = traceback.format_exc()
+        logger.exception(f"✗ Error during ProcessPoolExecutor → {exc}\n{tb}" , exc_info=True)  # Log the full traceback
+        #traceback.print_exc()  # Print the exception traceback
+        return f"✗ An error occurred during ProcessPoolExecutor→ {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+
+    '''
+    logger.log(level=20, msg="ProcessPoolExecutor pool result:", extra={"results": str(results)})
+    logs = []
+    #logs.extend(results)   ## performant pythonic
+    #logs = list[results]  ## 
+    logs = [result for result in results]  ## pythonic list comprehension
+    '''
+
+    try:
+        logs_return = file_handler.file_utils.process_dicts_data(logs)   #"\n".join(log for log in logs)  ##SMY outputs to gr.JSON component with no need for json.dumps(data, indent=)
+        #logs_files_images_return = "\n".join(path for path in logs_files_images)  ##TypeError: sequence item 0: expected str instance, WindowsPath found
+        
+        ##convert the List of Path objects to List of string for gr.Files output
+        #logs_files_images_return = list(str(path) for path in logs_files_images)  
+        
+        ## # Convert any Path objects to strings, but leave strings as-is
+        logs_files_images_return = list(str(path) if isinstance(path, Path) else path for path in logs_files_images)
+        return logs_return, logs_return, logs_files_images_return
+        #return "\n".join(logs), "\n".join(logs_files_images)    #"\n".join(logs_files)
+    except Exception as exc:
+        tb = traceback.format_exc()
+        logger.exception(f"✗ Error during returning result logs → {exc}\n{tb}" , exc_info=True)  # Log the full traceback
+        #traceback.print_exc()  # Print the exception traceback
+        return f"✗ An error occurred during returning result logs→ {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+
+
+    #return "\n".join(log for log in logs), "\n".join(str(path) for path in logs_files_images)
+    #print(f'logs_files_images: {"\n".join(str(path) for path in logs_files_images)}')
+        
+# files wrapping into list  ##SMY: Flagged for deprecation
+def pdf_files_wrap(files: list[str]):
+    # explicitly wrap file object in a list
+    return [files] if not isinstance(files, list) else files
+    #return [files]
+
+##====================
+## SMY: moved to logic file: See pdf_to_md.py. Currently unused
+def convert_pdfs_to_md(file: gr.File | None, folder: str | None) -> dict:
+    """
+    Gradio callback for PDF → Markdown.
+    Accepts either a single file or a folder path (recursively).
+    Leverages Marker, a pipeline of deep learning models, for conversion
+    Returns a dictionary of filename → Markdown string.
+    """
+    if not file and not folder:
+        return {"error": "Please provide a PDF file or a folder."}
+
+    pdf_paths = []
+
+    # Single file
+    if file:
+        pdf_path = Path(file.name)
+        pdf_paths.append(pdf_path)
+
+    # Folder (recursively)
+    if folder:
+        try:
+            pdf_paths.extend(collect_pdf_paths(folder))
+        except Exception as exc:
+            logger.exception("Folder traversal failed.")
+            return {"error": str(exc)}
+
+    if not pdf_paths:
+        return {"error": "No PDF files found."}
+
+    results = pdf2md_converter.batch_convert(pdf_paths)
+    # Gradio expects a dict of {filename: content}
+    return results
+
+## SMY: to refactor and moved to logic file. Currently unused
+def convert_htmls_to_md(file: gr.File | None, folder: str | None) -> dict:
+    """
+    Gradio callback for HTML → Markdown.
+    Accepts either a single file or a folder path (recursively).
+    Returns a dictionary of filename → Markdown string.
+    """
+    if not file and not folder:
+        return {"error": "Please provide a HTML file or a folder."}
+
+    html_paths = []
+
+    # Single file
+    if file:
+        html_path = Path(file.name)
+        html_paths.append(html_path)
+
+    # Folder (recursively)
+    if folder:
+        try:
+            html_paths.extend(collect_html_paths(folder))
+        except Exception as exc:
+            logger.exception("Folder traversal failed.")
+            return {"error": str(exc)}
+
+    if not html_paths:
+        return {"error": "No HTML files found."}
+
+    results = html2md_converter.batch_convert(html_paths)
+    # Gradio expects a dict of {filename: content}
+    return results
+
+## SMY: to refactor and moved to logic file
+def convert_md_to_pdf(file: gr.File | None, folder: str | None) -> list[gr.File]:
+    """
+    Gradio callback for Markdown → PDF.
+    Returns a list of generated PDF files (as Gradio File objects).
+    """
+    if not file and not folder:
+        return []
+
+    md_paths = []
+
+    # Single file
+    if file:
+        md_path = Path(file.name)
+        md_paths.append(md_path)
+
+    # Folder
+    if folder:
+        try:
+            md_paths.extend(collect_markdown_paths(folder))
+        except Exception as exc:
+            logger.exception("Folder traversal failed.")
+            return []
+
+    if not md_paths:
+        return []
+
+    output_dir = Path("./generated_pdfs")
+    output_dir.mkdir(exist_ok=True)
+
+    pdf_files = md2pdf_converter.batch_convert(md_paths, output_dir)
+    # Convert to Gradio File objects
+    gr_files = [gr.File(path=str(p)) for p in pdf_files]
+    return gr_files
+##====================
+
+def build_interface() -> gr.Blocks:
+    """
+    Assemble the Gradio Blocks UI.
+    """
+        
+    # Use custom CSS to style the file component
+    custom_css = """
+    .file-or-directory-area {
+        border: 2px dashed #ccc;
+        padding: 20px;
+        text-align: center;
+        border-radius: 8px;
+        margin-bottom: 10px;
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+    }
+    .file-or-directory-area:hover {
+        border-color: #007bff;
+        background-color: #f8f9fa;
+    }
+    .gradio-upload-btn {
+        margin-top: 10px;
+    }
+    """
+
+    def is_file_with_extension(path_obj: Path) -> bool:
+        """
+        Checks if a pathlib.Path object is a file and has a non-empty extension.
+        """
+        path_obj = path_obj if isinstance(path_obj, Path) else Path(path_obj) if isinstance(path_obj, str) else None
+        return path_obj.is_file() and bool(path_obj.suffix)
+
+    def accumulate_files(uploaded_files, current_state):
+        """
+        Accumulates newly uploaded files with the existing state.
+        """
+        # Initialize state if it's the first run
+        if current_state is None:
+            current_state = []
+        
+        # If no files were uploaded in this interaction, return the current state unchanged
+        if not uploaded_files:
+            return current_state, f"No new files uploaded. Still tracking {len(current_state)} file(s)."
+        
+        # Get the temporary paths of the newly uploaded files
+        # call is_file_with_extension to check if pathlib.Path object is a file and has a non-empty extension
+        new_file_paths = [f.name for f in uploaded_files if is_file_with_extension(Path(f.name))]  #Path(f.name) and Path(f.name).is_file() and bool(Path(f.name).suffix)]  #Path(f.name).suffix.lower() !=""]
+
+        # Concatenate the new files with the existing ones in the state
+        updated_files = current_state + new_file_paths
+        updated_filenames = [Path(f).name for f in updated_files]
+        
+        # Return the updated state and a message to the user
+        #file_info = "\n".join(updated_files)
+        filename_info = "\n".join(updated_filenames)
+        #message = f"Accumulated {len(updated_files)} file(s) total.\n\nAll file paths:\n{file_info}"
+        message = f"Accumulated {len(updated_files)} file(s) total: \n{filename_info}"
+        
+        return updated_files, message
+
+    def clear_state():
+        """
+        Clears the accumulated state of uloaded file list, output textbox, files and directory upload.
+        """
+        return [], "Files list cleared.", [], []
+
+    # with gr.Blocks(title=TITLE) as demo
+    with gr.Blocks(title=TITLE, css=custom_css) as demo:
+        gr.Markdown(f"## {DESCRIPTION}")
+
+        # Clean UI: Model parameters hidden in collapsible accordion
+        with gr.Accordion("⚙️ LLM Model Settings", open=False):
+            gr.Markdown(f"#### **Backend Configuration**")
+            system_message = gr.Textbox(
+                label="System Message",
+                lines=2
+            )
+            with gr.Row():
+                provider_dd = gr.Dropdown(
+                    choices=["huggingface", "openai"],
+                    label="Provider",
+                    value="huggingface",
+                    #allow_custom_value=True
+                )
+                backend_choice = gr.Dropdown(
+                    choices=["model-id", "provider", "endpoint"],
+                    label="HF Backend Choice"
+                )  ## SMY: ensure HFClient maps correctly 
+                model_tb = gr.Textbox(
+                    label="Model ID",
+                    value="meta-llama/Llama-4-Maverick-17B-128E-Instruct",  #image-Text-to-Text  #"openai/gpt-oss-120b",  ##Text-to-Text
+                )
+                endpoint_tb = gr.Textbox(
+                    label="Endpoint",
+                    placeholder="Optional custom endpoint"
+                )
+            with gr.Row():
+                max_token_sl = gr.Slider(
+                    label="Max Tokens",
+                    minimum=1,
+                    maximum=131172,  #65536,  #32768,  #16384,  #8192,
+                    value=1024,  #512,
+                    step=1
+                )
+                temperature_sl = gr.Slider(
+                    label="Temperature",
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.0,
+                    step=0.1  #0.01
+                )
+                top_p_sl = gr.Slider(
+                    label="Top-p",
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.1,
+                    step=0.1  #0.01
+                )
+                stream_cb = gr.Checkbox(
+                    label="LLM Streaming",
+                    value=False
+                )
+            with gr.Row():
+                api_token_tb = gr.Textbox(
+                    label="API Token [OPTIONAL]",
+                    type="password",
+                    placeholder="hf_xxx or openai key"
+                )
+                hf_provider_dd = gr.Dropdown(
+                    choices=["fireworks-ai", "together-ai", "openrouter-ai", "hf-inference"],
+                    value="fireworks-ai",
+                    label="Provider",
+                    allow_custom_value=True,  # let users type new providers as they appear
+                )
+
+        # Validate provider on change; warn but allow continue
+        def on_provider_change(provider_value: str):
+            if not provider_value:
+                return
+            if not is_valid_provider(provider_value):
+                sug = suggest_providers(provider_value)
+                extra = f" Suggestions: {', '.join(sug)}." if sug else ""
+                gr.Warning(
+                    f"Provider not on HF provider list. See https://huggingface.co/docs/inference-providers/index.{extra}"
+                )
+        hf_provider_dd.change(on_provider_change, inputs=hf_provider_dd, outputs=None)
+
+        # Clean UI: Model parameters hidden in collapsible accordion
+        with gr.Accordion("⚙️ Marker Settings", open=False):
+            gr.Markdown(f"#### **Marker Configuration**")
+            with gr.Row():
+                openai_base_url_tb = gr.Textbox(
+                    label="OpenAI Base URL: Default HuggingFace",
+                    value="https://router.huggingface.co/v1",
+                    lines=1,
+                    max_lines=1,
+                )
+                openai_image_format_dd = gr.Dropdown(
+                    choices=["webp", "png", "jpeg"],
+                    label="OpenAI Image Format",
+                    value="webp",
+                )
+                output_format_dd = gr.Dropdown(
+                    choices=["markdown", "html"],  #, "json", "chunks"],  ##SMY: To be enabled later
+                    #choices=["markdown", "html", "json", "chunks"],
+                    label="Output Format",
+                    value="markdown",
+                )
+                output_dir_tb = gr.Textbox(
+                    label="Output Directory",
+                    value="output_dir",  #"output_md",
+                    lines=1,
+                    max_lines=1,
+                )
+            with gr.Row():
+                max_workers_sl = gr.Slider(
+                    label="Max Worker",
+                    minimum=1,
+                    maximum=7,
+                    value=4,
+                    step=1  
+                )
+                max_retries_sl = gr.Slider(
+                    label="Max Retry",
+                    minimum=1,
+                    maximum=3,
+                    value=2,
+                    step=1  #0.01
+                )
+                use_llm_cb = gr.Checkbox(
+                    label="Use LLM for Marker conversion",
+                    value=False
+                )
+                page_range_tb = gr.Textbox(
+                    label="Page Range (Optional)",
+                    placeholder="Example: 0,1-5,8,12-15",
+                    lines=1,
+                    max_lines=1,
+                )
+
+        # Initialise gr.State
+        state_max_workers = gr.State(4)  #max_workers_sl,
+        state_max_retries = gr.State(2) #max_retries_sl,
+
+        def update_state_stored_value(new_component_input):
+            """ Updates stored state: use for max_workers and max_retries """
+            return new_component_input
+        
+        # Update gr.State values on slider components change. NB: initial value of `gr.State` must be able to be deepcopied
+        max_workers_sl.change(update_state_stored_value, inputs=max_workers_sl, outputs=state_max_workers)
+        max_retries_sl.change(update_state_stored_value, inputs=max_retries_sl, outputs=state_max_retries)
+
+
+        with gr.Accordion("🤗 HuggingFace Logout", open=False):
+            # Logout controls
+            def do_logout():
+                #ok = docextractor.client.logout()
+                ok = docconverter.client.logout()
+                # Reset token textbox on successful logout
+                msg = "✅ Logged out of Hugging Face and cleared tokens." if ok else "⚠️ Logout failed."
+                return gr.update(value=""), gr.update(visible=True, value=msg)
+            
+            logout_status = gr.Markdown(visible=False)
+            logout_btn = gr.Button("Logout from Hugging Face", variant="stop")
+
+            logout_btn.click(fn=do_logout, inputs=None, outputs=[api_token_tb, logout_status]) 
+
+        
+        # The gr.State component to hold the accumulated list of files
+        uploaded_file_list = gr.State([])   ##NB: initial value of `gr.State` must be able to be deepcopied
+
+        # --- PDF & HTML → Markdown tab ---
+        with gr.Tab(" 📄 PDF & HTML ➜ Markdown"):
+            gr.Markdown(f"#### {DESCRIPTION_PDF_HTML}")
+
+            ### flag4deprecation  #earlier implementation
+            '''
+            pdf_files = gr.File(
+                label="Upload PDF, HTML or PDF and HTMLfiles",
+                file_count="directory", ## handle directory and files upload #"multiple",
+                type="filepath",
+                file_types=["pdf", ".pdf"],
+                #size="small",
+            )
+            pdf_files_count = gr.TextArea(label="Files Count", interactive=False, lines=1)
+            with gr.Row():
+                btn_pdf_count = gr.Button("Count Files")
+                #btn_pdf_upload = gr.UploadButton("Upload files")
+                btn_pdf_convert = gr.Button("Convert PDF(s)")
+            '''
+
+            with gr.Column(elem_classes=["file-or-directory-area"]):
+                with gr.Row():
+                    file_btn = gr.UploadButton(
+                    #file_btn = gr.File(
+                        label="Upload Multiple Files",
+                        file_count="multiple",
+                        file_types=["file"],
+                        #height=25,  #"sm",
+                        size="sm",
+                        elem_classes=["gradio-upload-btn"]
+                    )
+                    dir_btn = gr.UploadButton(
+                    #dir_btn = gr.File(
+                        label="Upload a Directory",
+                        file_count="directory",
+                        #file_types=["file"],  #Warning: The `file_types` parameter is ignored when `file_count` is 'directory'
+                        #height=25,  #"0.5",
+                        size="sm",
+                        elem_classes=["gradio-upload-btn"]
+                    )
+            with gr.Accordion("Display uploaded", open=True):
+                # Displays the accumulated file paths
+                output_textbox = gr.Textbox(label="Accumulated Files", lines=3) #, max_lines=4)  #10
+            
+            with gr.Row():
+                process_button = gr.Button("Process All Uploaded Files", variant="primary")
+                clear_button = gr.Button("Clear All Uploads", variant="secondary")
+
+            # Event handler for the multiple file upload button
+            file_btn.upload(
+                fn=accumulate_files,
+                inputs=[file_btn, uploaded_file_list],
+                outputs=[uploaded_file_list, output_textbox]
+            )
+
+            # Event handler for the directory upload button
+            dir_btn.upload(
+                fn=accumulate_files,
+                inputs=[dir_btn, uploaded_file_list],
+                outputs=[uploaded_file_list, output_textbox]
+            )
+
+            # Event handler for the "Clear" button
+            clear_button.click(
+                fn=clear_state,
+                inputs=None,
+                outputs=[uploaded_file_list, output_textbox, file_btn, dir_btn],
+            )
+
+        # --- PDF → Markdown tab ---
+        with gr.Tab(" 📄 PDF ➜ Markdown (Flag for DEPRECATION)", interactive=False, visible=True):  #False
+            gr.Markdown(f"#### {DESCRIPTION_PDF}")
+
+            files_upload_pdf = gr.File(
+                label="Upload PDF files",
+                file_count="directory", ## handle directory and files upload #"multiple",
+                type="filepath",
+                file_types=["pdf", ".pdf"],
+                #size="small",
+            )
+            files_count = gr.TextArea(label="Files Count", interactive=False, lines=1)  #pdf_files_count
+            with gr.Row():
+                btn_pdf_count = gr.Button("Count Files")
+                #btn_pdf_upload = gr.UploadButton("Upload files")
+                btn_pdf_convert = gr.Button("Convert PDF(s)")
+        
+        # --- 📃 HTML → Markdown tab ---
+        with gr.Tab("🕸️ HTML ➜ Markdown: (Flag for DEPRECATION)", interactive=False, visible=False):
+            gr.Markdown(f"#### {DESCRIPTION_HTML}")
+
+            files_upload_html = gr.File(
+                label="Upload HTML files",
+                file_count="multiple",
+                type="filepath",
+                file_types=["html", ".html", "htm", ".htm"]
+            )
+            #btn_html_convert = gr.Button("Convert HTML(s)")
+            html_files_count = gr.TextArea(label="Files Count", interactive=False, lines=1)
+            with gr.Row():
+                btn_html_count = gr.Button("Count Files")
+                #btn_pdf_upload = gr.UploadButton("Upload files")
+                btn_html_convert = gr.Button("Convert PDF(s)")
+
+
+        # --- Markdown → PDF tab ---
+        with gr.Tab("PENDING: Markdown ➜ PDF", interactive=False):
+            gr.Markdown(f"#### {DESCRIPTION_MD}")
+
+            md_files = gr.File(
+                label="Upload Markdown files",
+                file_count="multiple",
+                type="filepath",
+                file_types=["md", ".md"]
+            )
+            btn_md_convert = gr.Button("Convert Markdown to PDF)")
+            output_pdf = gr.Gallery(label="Generated PDFs", elem_id="pdf_gallery")
+
+            '''
+            md_input = gr.File(label="Upload a single Markdown file", file_count="single")
+            md_folder_input = gr.Textbox(
+                label="Or provide a folder path (recursively)",
+                placeholder="/path/to/folder",
+            )
+            convert_md_btn = gr.Button("Convert Markdown to PDF")
+            output_pdf = gr.Gallery(label="Generated PDFs", elem_id="pdf_gallery")
+
+            convert_md_btn.click(
+                fn=convert_md_to_pdf,
+                inputs=[md_input, md_folder_input],
+                outputs=output_pdf,
+            )
+            '''
+
+        # A Files component to display individual processed files as download links
+        with gr.Accordion("⏬ View and Download processed files", open=False):
+            with gr.Row():
+                files_individual_JSON = gr.JSON(label="Serialised JSON list", max_height=250)
+                files_individual_downloads = gr.Files(label="Individual Processed Files")
+
+        ## Displays processed file paths
+        with gr.Accordion("View processing log", open=False):
+            log_output = gr.Textbox(
+                label="Conversion Logs",
+                lines=5,
+                #max_lines=25,
+                interactive=False
+            )
+        
+        # file inputs
+        ## [wierd] NB: inputs_arg is a list of Gradio component objects, not the values of those components.
+        ## inputs_arg variable captures the state of these components at the time the list is created. 
+        ## When btn_convert.click() is called later, it uses the list as it was initially defined
+        ##
+        ## SMY: Gradio component values are not directly mutable.
+        ## Instead, you should pass the component values to a function,
+        ## and then use the return value of the function to update the component.
+        ## Discarding for now. #//TODO: investigate further.
+        ## SMY: Solved: using gr.State 
+        inputs_arg = [
+            #pdf_files,
+            ##pdf_files_wrap(pdf_files),  # wrap pdf_files in a list (if not already)
+            uploaded_file_list,
+            files_count,  #pdf_files_count,
+            provider_dd,
+            model_tb,
+            hf_provider_dd,
+            endpoint_tb,
+            backend_choice,
+            system_message,
+            max_token_sl,
+            temperature_sl,
+            top_p_sl,
+            stream_cb,
+            api_token_tb,
+            #gr.State(4),   # max_workers
+            #gr.State(3),    # max_retries
+            openai_base_url_tb,
+            openai_image_format_dd,
+            state_max_workers, #gr.State(4),  #max_workers_sl,
+            state_max_retries, #gr.State(2), #max_retries_sl,
+            output_format_dd,
+            output_dir_tb,
+            use_llm_cb,
+            page_range_tb,
+        ]
+
+        ## debug
+        #logger.log(level=30, msg="About to execute btn_pdf_convert.click", extra={"files_len": pdf_files_count, "pdf_files": pdf_files})
+        
+        try:
+            #logger.log(level=30, msg="input_arg[0]: {input_arg[0]}")
+            process_button.click(
+            #pdf_files.upload( 
+                fn=convert_batch,
+                inputs=inputs_arg,
+                outputs=[log_output, files_individual_JSON, files_individual_downloads],
+            )
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception(f"✗ Error during process_button.click → {exc}\n{tb}", exc_info=True)
+            return f"✗ An error occurred during process_button.click → {exc}\n{tb}"            
+
+        ##gr.File .upload() event, fire only after a file has been uploaded
+        # Event handler for the pdf file upload button
+        files_upload_pdf.upload(
+            fn=accumulate_files,
+            inputs=[files_upload_pdf, uploaded_file_list],
+            outputs=[uploaded_file_list, log_output]
+        )
+        #inputs_arg[0] = files_upload
+        btn_pdf_convert.click(
+        #pdf_files.upload( 
+            fn=convert_batch,
+            outputs=[log_output, files_individual_downloads],
+            inputs=inputs_arg, 
+        )
+        '''
+            inputs = [
+                pdf_files,
+                #pdf_files_wrap(pdf_files),  # wrap pdf_files in a list (if not already)            
+                pdf_files_count,
+                provider_dd,
+                model_tb,
+                hf_provider_dd,
+                endpoint_tb,
+                backend_choice,
+                system_message,
+                max_token_sl,
+                temperature_sl,
+                top_p_sl,
+                stream_cb,
+                api_token_tb,
+                #gr.State(4),   # max_workers
+                #gr.State(3),    # max_retries
+                openai_base_url_tb,
+                openai_image_format_dd,
+                state_max_workers, #gr.State(max_workers_sl),  #max_workers_sl,
+                state_max_retries, #gr.State(max_retries_sl), #max_retries_sl,
+                output_format_dd,
+                output_dir_tb,
+                use_llm_cb,
+                page_range_tb,
+            ],
+        '''   
+        #    )
+
+        # reuse the same business logic for HTML tab
+        # Event handler for the pdf file upload button
+        files_upload_html.upload(
+            fn=accumulate_files,
+            inputs=[files_upload_html, uploaded_file_list],
+            outputs=[uploaded_file_list, log_output]
+        )
+        #inputs_arg[0] = html_files
+        btn_html_convert.click(
+            fn=convert_batch,
+            inputs=inputs_arg,
+            outputs=[log_output, files_individual_downloads]
+        )
+
+        def get_file_count(file_list):
+            """
+            Counts the number of files in the list.
+
+            Args:
+                file_list (list): A list of temporary file objects.
+            Returns:
+                str: A message with the number of uploaded files.
+            """
+            if file_list:
+                return f"{len(file_list)}", f"Upload: {len(file_list)} files: \n {file_list}"  #{[pdf_files.value]}"
+            else:
+                return "No files uploaded.", "No files uploaded."        # Count files button
+        
+        btn_pdf_count.click(
+            fn=get_file_count,
+            inputs=[files_upload_pdf],
+            outputs=[files_count, log_output]
+        )
+        btn_html_count.click(
+            fn=get_file_count,
+            inputs=[files_upload_html],
+            outputs=[html_files_count, log_output]
+        )        
+        
+        # Validate files upload on change; warn but allow continue
+        def on_pdf_files_change(pdf_files_value: list[str]):
+            # explicitly wrap file object in a list
+            pdf_files_value = pdf_files_wrap(pdf_files_value)
+            #if not isinstance(pdf_files_value, list):
+            #    pdf_files_value = [pdf_files_value]
+
+            pdf_files_path = [file.name for file in pdf_files_value]
+            pdf_files_len = len(pdf_files_value)  #len(pdf_files_path)
+            if pdf_files_value:
+                #return            
+                return pdf_files_path, pdf_files_len
+        #pdf_files.change(on_pdf_files_change, inputs=pdf_files, outputs=[log_output, pdf_files_count])  #, postprocess=False)  ##debug
+
+
+    return demo
+

utils/config.ini

@@ -0,0 +1,158 @@
+[marker]
+provider=openai
+#model_id=openai/gpt-oss-120b
+## Marker will return "LLM did not return a valid response" if model is not 'Image-Text-to-Text'
+## because of OpenAI inference failed: Errorcode: 400 ... "Unsupported ChatMessageContent type: image_url"
+## Note that Marker works pretty well using it's own transformer-based model without LLM
+model_id=meta-llama/Llama-4-Maverick-17B-128E-Instruct
+hf_provider=fireworks-ai
+endpoint_url=""
+backend_choice=provider
+system_message=""
+max_tokens=8192
+temperature=0.2
+top_p=0.2
+stream=True
+api_token=a1b2c3
+openai_model=openai/gpt-oss-120b
+openai_api_key=a1b2c3
+openai_base_url=https://router.huggingface.co/v1
+openai_image_format=webp
+#max_retries=3
+
+#[Configuration]
+use_llm=True
+output_format=markdown
+input_dir=inputs
+output_dir=output_md
+max_workers=4
+max_retries=2
+extract_images=True
+output_image_format=png
+output_encoding=utf-8
+debug_data_folder=debug_data
+
+[unsure]
+image_output_dir="images"
+image_output_format="png"
+base_dir=Path(__file__).resolve().parent.parent
+###
+# Create a Path object from the current file's location, resolve it to an absolute path,
+# and then get its parent's parent using chained .parent calls or the parents[] attribute.
+#grandparent_dir = Path(__file__).resolve().parent.parent  #os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+###
+
+[libraries]
+libobject_path = C:\\Dat\\dev\\gtk3-runtime\\bin
+
+
+# from config.ini  ##SMY: future plan to merge
+[MARKER_CAP]
+#[marker]
+PROVIDER       = openai
+#MODEL_ID       = openai/gpt-oss-120b
+## Marker will return "LLM did not return a valid response" if model is not 'Image-Text-to-Text'
+## because of OpenAI inference failed: Errorcode: 400 ... "Unsupported ChatMessageContent type: image_url"
+## Note that Marker works pretty well using it's own transformer-based model without LLM
+MODEL_ID=meta-llama/Llama-4-Maverick-17B-128E-Instruct
+HF_PROVIDER    = fireworks-ai
+ENDPOINT_URL   = ""
+BACKEND_CHOiCE = provider
+SYSTEM_MESSAGE = ""
+MAX_TOKENS     = 8192
+TEMMPERATURE   = 0.2
+TOP_P          = 0.2
+STREAM         = True
+API_TOKEN      = a1b2c3
+OPENAI_MODEL        = openai/gpt-oss-120b
+OPENAI_API_KEY      = a1b2c3
+OPENAI_BASE_URL     = https://router.huggingface.co/v1
+OPENAI_IMAGE_FORMAT = webp
+
+#[CONFIGURATION]
+MAX_WORKERS         = 4
+MAX_RETRIES         = 2
+OUTPUT_FORMAT       = markdown
+INPUT_DIR           = inputs
+OUTPUT_DIR          = output_dir
+USE_LLM             = False
+EXTRACT_IMAGES      = True
+OUTPUT_IMAGE_FORMAT = png
+OUTPUT_ENCODING     = utf-8
+DEBUG_DATA_FOLDER   = debug_data
+
+[UNSURE_CAP]
+IMAGE_OUTPUT_DIR    = images
+IMAGE_OUTPUT_FORMAT = png
+BASE_DIR            = Path(__file__).resolve().parent.parent
+###
+# Create a Path object from the current file's location, resolve it to an absolute path
+# Get its parent's parent using chained .parent calls or the parents[] attribute.
+#grandparent_dir = Path(__file__).resolve().parent.parent  #os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+###
+
+[LIBRARIES_CAP]
+LIBOBJECT_PATH = C:\\Dat\\dev\\gtk3-runtime\\bin
+WEASYPRINT_DLL_DIRECTORIES = C:\\Dat\\dev\\gtk3-runtime\\bin
+
+[GLOBAL_CAP]
+# Globals within each worker process
+HF_MODEL       ="openai/gpt-oss-120b"
+HF_TOKEN       = ""
+HF_CLIENT      = None
+ARTIFACT_DICT  = None
+PDF_CONVERTER  = None
+HTML_CONVERTER = None
+
+[marker_dict]
+## "meta-llama/Llama-4-Maverick-17B-128E-Instruct:fireworks-ai"
+provider:"openai"  #provider,     
+model_id:"openai/gpt-oss-120b"  #model_id,  #"meta-llama/Llama-4-Maverick-17B-128E-Instruct:fireworks-ai"
+hf_provider:"fireworks-ai"   #hf_provider,
+endpoint_url:""  #endpoint_url,
+backend_choice:"provider"  #backend_choice,
+system_message:""   #system_message,
+max_tokens:8192  #max_tokens,
+temperature:0.2  #temperature,
+top_p:0.2   #top_p,
+stream:"stream"
+api_token:"a1b2c3"  #get_token,
+output_format:"markdown"  #output_format,  #"markdown",
+openai_model:"openai/gpt-oss-120b"  #self.client.model_id,  #"model_name"
+openai_api_key:"a1b2c3"  #self.client.openai_api_key,  #self.api_token,
+openai_base_url:"https://router.huggingface.co/v1"  #self.client.base_url,  #self.base_url,
+#temperature=self.client.temperature,
+#top_p=self.client.top_p,
+openai_image_format:"webp"  #"png"  #better compatibility
+max_retries:3  ## pass to __call__
+
+
+[marker_nostrip]
+provider="openai"
+model_id="openai/gpt-oss-120b"
+hf_provider="fireworks-ai"
+endpoint_url=""
+backend_choice="provider"
+system_message=""
+max_tokens=8192
+temperature=0.2
+top_p=0.2
+stream=True
+api_token="a1b2c3"
+openai_model="openai/gpt-oss-120b"
+openai_api_key="a1b2c3"
+openai_base_url="https://router.huggingface.co/v1"
+openai_image_format="webp"
+#max_retries=3
+
+#[Configuration]
+use_llm=True
+output_format="markdown"
+input_dir="inputs"
+output_dir="output_md"
+max_workers=4
+max_retries=2
+extract_images=True
+output_image_format="png"
+output_encoding=utf-8
+debug_data_folder="debug_data"

utils/config.py

@@ -0,0 +1,83 @@
+# utils/config.py
+
+import os
+
+"""
+Centralised configuration constants.
+##SMY: TODO: Create Class Settings(BaseSettings)  leveraging from pydantic_settings import BaseSettings
+"""
+
+# UI text
+TITLE = "PyPDFMd – PDF & HTML ↔ Markdown Converter"
+DESCRIPTION = (
+    "Drag‑and‑drop a single PDF/HTML, a folder to convert to Markdown."
+    "Or upload Markdown/LaTeX files and generate a polished PDF."
+)
+DESCRIPTION_PDF_HTML = (
+    "Upload a single or multiple PDF or HTML, a folder or an entire directory tree "
+    "to convert to Markdown."
+)
+DESCRIPTION_PDF = (
+    "Drag‑and‑drop a single PDF, a folder of PDFs or an entire directory tree "
+    "to convert to Markdown."
+)
+DESCRIPTION_HTML = (
+    "Drag‑and‑drop a single HTML, a folder of HTMLs or an entire directory tree "
+    "to convert to Markdown."
+)
+DESCRIPTION_MD = (
+    "Upload Markdown/LaTeX files and generate a polished PDF."
+)
+
+# Conversion defaults
+DEFAULT_MARKER_OPTIONS = {
+    "include_images": True,
+    "image_format": "png",
+}
+
+# Configuration
+MAX_WORKERS    = int(os.getenv("MAX_WORKERS", "4"))
+MAX_RETRIES    = int(os.getenv("MAX_RETRIES", "2"))  #3
+INPUT_DIR      = os.getenv("INPUT_DIR",  "inputs")  # unused
+OUTPUT_DIR     = os.getenv("OUTPUT_DIR", "md_output")
+USE_LLM             = bool(os.getenv("USE-LLM", False))  #True
+EXTRACT_IMAGES      = bool(os.getenv("EXTRACT_IMAGES", True))  #True
+OUTPUT_IMAGE_FORMAT = os.getenv("OUTPUT_IMAGE_FORMAT", "png")  #png
+OUTPUT_ENCODING     = os.getenv("OUTPUT_ENCODING", "utf-8")  #utf-8
+DEBUG_DATA_FOLDER   = os.getenv("DEBUG_DATA_FOLDER", "debug_data")  #debug_data
+
+# Global
+HF_MODEL       = os.getenv("HF_MODEL", "gpt2")  # swap for a chat-capable model
+HF_TOKEN       = os.getenv("HF_TOKEN")          # your Hugging Face token
+
+
+
+## //TODO: 
+# from config.ini  ##SMY: future plan to merge
+api_token="a1b2c3"
+OUTPUT_FORMAT       = "markdown"   #output_format
+OPENAI_MODEL        = "openai/gpt-oss-120b"   #openai_model
+OPENAI_API_KEY      = "a1b2c3"  #openai_api_key
+OPENAI_BASE_URL     = "https://router.huggingface.co/v1"   ##openai_base_url
+OPENAI_IMAGE_FORMAT = "webp"    #openai_image_format
+OUTPUT_IMAGE_FORMAT = "png"
+#max_retries=3 
+
+#[marker]
+PROVIDER       = "openai"  #provider
+MODEL_ID       = "openai/gpt-oss-120b"  #model_id
+HF_PROVIDER    = "fireworks-ai"  #hf_provider
+ENDPOINT_URL   = ""  #endpoint_url
+BACKEND_CHOiCE = "provider"  #backend_choice
+SYSTEM_MESSAGE = ""  #system_message
+MAX_TOKENS     = 8192  #max_tokens
+TEMMPERATURE   = 0.2  #temperature
+TOP_P          = 0.2  #top_p
+STREAM         = True  #stream
+
+# Globals within each worker process
+hf_client      = None
+artifact_dict  = None
+pdf_converter  = None
+html_converter = None
+

utils/get_arg_name.py

@@ -0,0 +1,19 @@
+import inspect
+
+def get_arg_name_as_string(arg):
+    """
+    Returns the name of the argument passed to the function as a string.
+    This works by inspecting the calling frame's local variables.
+
+    example usage:
+    def my_function(x):
+        arg_name = get_arg_name_as_string(arg_x)
+        print(f"The argument name is: {arg_name}")  # Outputs: "The argument name is: arg_x"
+    """
+    frame = inspect.currentframe().f_back # Get the frame of the caller
+    arg_name = None
+    for name, value in frame.f_locals.items():
+        if value is arg:
+            arg_name = name
+            break
+    return arg_name

utils/get_config.py

@@ -0,0 +1,18 @@
+from configparser import ConfigParser as config
+from typing import Union
+from pathlib import Path
+#from utils.get_arg_name import get_arg_name_as_string
+import traceback
+
+def get_config_value(section:str, parameter:str, fallback:str=None, configfile: Union[str, Path]="utils\\config.ini"):
+    """ Load config file, locate section, read parameter and return value """
+    
+    try:
+        cfg = config()
+        cfg.read(configfile)
+        param_value = cfg[section].get(option=parameter, fallback=fallback)  #"C:\\Dat\\dev\\gtk3-runtime\\bin")
+        return param_value
+    except Exception as exc:
+        tb = traceback.format_exc()
+        raise RuntimeWarning(f"Error loading config: {exc}\n{tb}")
+        #pass

utils/lib_loader.py

@@ -0,0 +1,130 @@
+import os
+from pathlib import Path
+import sys
+import ctypes
+from typing import Union
+from configparser import ConfigParser as config
+from venv import logger
+from utils.get_arg_name import get_arg_name_as_string
+from utils.get_config import get_config_value
+import traceback
+
+from utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+def set_weasyprint_library(libpath: Union[str, Path] = None, config_file: Union[str, Path] = "utils\\config.ini"):
+    """ Loads Weasyprint backend dependency libraries to environment """
+    # Check if the current platform is Windows
+    if sys.platform == 'win32':
+        
+        #libgobject_path =  #"/path/to/your/custom/glib/install/lib/libgobject-2.0.so.0"
+        if not libpath:
+            '''cfg = config()
+            cfg.read(config_file)  #"utils\\config.ini")
+            lib_path = cfg["LIBRARIES_CAP"].get(f"WEASYPRINT_DLL_DIRECTORIES", "C:\\Dat\\dev\\gtk3-runtime\\bin")
+            '''
+            lib_path = get_config_value("LIBRARIES_CAP", "WEASYPRINT_DLL_DIRECTORIES") if not libpath else "C:\\Dat\\dev\\gtk3-runtime\\bin"
+
+            # Check if the file exists before attempting to load it
+            #if not os.path.exists(libobject):
+            if not Path(lib_path).exists():
+                raise FileNotFoundError(f"The specified Weasyprint DLL Directory does not exist: {lib_path}. Follow Weasyprint installation guide or provide a valid GTK3-runtime path.")
+            #logger.exception(f"gobject library path: {libgobject_path}")  ##debug
+            
+        try:
+            # Set a new environment variable
+            lib_path = lib_path  ##SMY: on dev machine, using extracted 'portable' GTK3 rather than installing 'MSYS2'
+            os.environ["WEASYPRINT_DLL_DIRECTORIES"] = lib_path
+            #logger.info(f"sets Weasyprint DLL library path: {lib_path}")   #debug
+            
+        except Exception as exc:
+            tb = traceback.format_exc()
+            logger.exception(f"Error setting environ: weasyprint backend dependency → {exc}\n{tb}", exc_info=True)  # Log the full traceback
+            
+            raise RuntimeWarning(f"✗ error during setting environ: weasyprint backend dependency → {exc}\n{tb}")
+        
+
+def load_library(libobject_name: Union[str, Path]):
+    """ 
+    Loads Weasyprint backend dependency libraries 
+    usage:  list(map(load_library, library_list))  ##SMY: map the load_library function to each item in library_list
+    The library list was starting to grow excessively, opt to setting environ   
+    """
+    # Check if the current platform is Windows
+    if sys.platform == 'win32':
+        
+        #libgobject_path =  #"/path/to/your/custom/glib/install/lib/libgobject-2.0.so.0"
+        cfg = config()
+        cfg.read("utils\\config.ini")
+        lib_path = cfg["libraries"].get(f"libobject_path", "C:\\Dat\\dev\\gtk3-runtime\\bin")
+        lib_object_dll = get_arg_name_as_string(libobject_name)  ## future use
+
+        # Construct the path to libgobject-2.0.dll
+        #libgobject_path = os.path.join(os.environ.get('GLIB_PREFIX', 'C:\\glib'), 'bin', 'libgobject-2.0-0.dll')
+        libobject = f"{lib_path}\\{libobject_name}.dll"   ##libgobject-2.0-0.dll"
+        #print(f"Loading gobject library: {libgobject}")   #debug
+        
+        # Check if the file exists before attempting to load it
+        #if not os.path.exists(libobject):
+        if not Path(libobject).exists():
+            raise FileNotFoundError(f"The specified library file does not exist: {libobject}")
+        #print(f"gobject library path: {libgobject_path}")  ##debug
+
+        # Load the library using ctypes
+        try:
+            ctypes_libgobject = ctypes.CDLL(libobject)
+            #msg = f"libgobject-2.0-0.dll loaded successfully via ctypes. {str(ctypes_libgobject)}"
+            #print(msg)  ##debug
+        except OSError as exc:
+            tb = traceback.format_exc()
+            raise RuntimeWarning(f"Failed to load library: {exc}\n{tb}")  ##raise RuntimeError
+
+## Test
+#load_library("libpango-1.0-0")
+#load_library("libgobject-2.0-0")
+
+
+##SMY: Original implementation: TODO: for refactoring
+def load_libgobject():
+    # Check if the current platform is Windows
+    if sys.platform == 'win32':
+        
+        #libgobject_path =  #"/path/to/your/custom/glib/install/lib/libgobject-2.0.so.0"
+        cfg = config()
+        cfg.read("utils\\config.ini")
+        libgobject_path = cfg["libraries"].get("libgobject_path", "C:\\Dat\\dev\\gtk3-runtime\\bin")
+
+        # Construct the path to libgobject-2.0.dll
+        #libgobject_path = os.path.join(os.environ.get('GLIB_PREFIX', 'C:\\glib'), 'bin', 'libgobject-2.0-0.dll')
+        libgobject = f"{libgobject_path}\\libgobject-2.0-0.dll"
+        #print(f"Loading gobject library: {libgobject}")   #debug
+        
+        # Check if the file exists before attempting to load it
+        if not os.path.exists(libgobject):
+            raise FileNotFoundError(f"The specified library file does not exist: {libgobject}")
+        #print(f"gobject library path: {libgobject_path}")  ##debug
+
+        # Load the library using ctypes
+        try:
+            ctypes_libgobject = ctypes.CDLL(libgobject)
+            #msg = f"libgobject-2.0-0.dll loaded successfully via ctypes. {str(ctypes_libgobject)}"
+            #print(msg)  ##debug
+
+            return ctypes_libgobject
+        except OSError as exc:
+            tb = traceback.format_exc()
+            raise RuntimeWarning(f"Failed to load library: {exc}\n{tb}")  ##raise RuntimeError
+
+
+    # Load the library using ctypes (Linux/macOS)
+    # Construct the path to libgobject-2.0.so.0 in the custom GLib installation
+    #libgobject_path = os.path.join(os.environ.get('GLIB_PREFIX', '/opt/glib'), 'lib', 'libgobject-2.0.so.0') 
+    #print("This script is intended to run on Unix-like systems, not Windows.")
+    else:
+        # Load the library using ctypes (Linux/macOS)
+        # Construct the path to libgobject-2.0.so.0 in the custom GLib installation
+        libgobject_path = os.path.join(os.environ.get('GLIB_PREFIX', '/opt/glib'), 'lib', 'libgobject-2.0.so.0') 
+        #print("This script is intended to run on Unix-like systems, not Windows.")
+
+        return libgobject_path

utils/logger.py

@@ -0,0 +1,81 @@
+# utils/logger.py
+
+import json
+import logging
+import sys
+from datetime import datetime, timezone
+
+'''
+def get_logger(name: str) -> logging.Logger:
+    """
+    Returns a logger configured with a console handler.
+    """
+    logger = logging.getLogger(name)
+    if not logger.handlers:
+        logger.setLevel(logging.INFO)
+        ch = logging.StreamHandler()
+        formatter = logging.Formatter(
+            "[%(asctime)s] %(levelname)s - %(name)s: %(message)s",
+            datefmt="%H:%M:%S",
+        )
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+    return logger
+'''
+
+class JsonFormatter(logging.Formatter):
+    """Minimal JSON formatter for structured logs."""
+
+    def format(self, record: logging.LogRecord) -> str:  #
+        payload = {
+            #"ts": datetime.now(timezone.utc).isoformat(),  ## default to 'YYYY-MM-DD HH:MM:SS.mmmmmm',
+            "ts": datetime.now(timezone.utc).strftime("%Y-%m-%d : %H:%M:%S"),  ## SMY: interested in datefmt="%H:%M:%S",
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        # Include extra attributes (fields not in default LogRecord)
+        for key, value in record.__dict__.items():
+            if key in ("args", "msg", "levelno", "levelname", "name", "pathname", "filename",
+                       "module", "exc_info", "exc_text", "stack_info", "lineno", "funcName",
+                       "created", "msecs", "relativeCreated", "thread", "threadName",
+                       "processName", "process"):
+                continue
+            payload[key] = value
+        return json.dumps(payload, ensure_ascii=False)
+
+#def setup_logging(level: int = logging.INFO) -> None:  ## Causing non-stop logging on HF spaces
+def setup_logging(level: int = None) -> None:
+    """Configure root logger with JSON output to both stdout and file.
+    
+    Args:
+        level: Logging level. If None, uses WARNING for production (HF Spaces) 
+               and INFO for local development.
+    """
+    if level is None:
+        # Auto-detect environment: WARNING for production, INFO for local dev
+        import os
+        is_production = os.getenv("SPACE_ID") or os.getenv("HF_SPACE_ID") or os.getenv("HUGGINGFACE_SPACE_ID")
+        level = logging.WARNING if is_production else logging.INFO
+    
+    # Console handler
+    console_handler = logging.StreamHandler(stream=sys.stdout)
+    console_handler.setFormatter(JsonFormatter())  #, datefmt="%H:%M:%S",)  ##explicit time format
+    
+    # File handler
+    #file_handler = logging.FileHandler("logs/app_logging_scrap.log", mode="a", encoding="utf-8")
+    file_handler = logging.FileHandler("logs/app_logging.log", mode="a", encoding="utf-8")
+    file_handler.setFormatter(JsonFormatter())
+    
+    root = logging.getLogger()
+    root.handlers.clear()
+    root.addHandler(console_handler)
+    root.addHandler(file_handler)
+    root.setLevel(level)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a module logger configured with console handler using defined JSON format."""
+    return logging.getLogger(name)
+
+

utils/utils.py

@@ -0,0 +1,15 @@
+def is_dict(variable):
+    """Checks if a variable is a dict."""
+    if isinstance(variable, dict):
+        return True
+        
+    return False
+
+def is_list_of_dicts(variable):
+    """Checks if a variable is a list containing only dicts."""
+    
+    if isinstance(variable, list):
+        # Return True only if the list is empty or all elements are dicts.
+        return all(isinstance(item, dict) for item in variable)
+        
+    return False