Repository: plantnet/gbif-dl

Issue:
Add docs
Add a simple doc pages, using [pdoc3](https://pdoc3.github.io/pdoc/). This should also be integrated in #5, and can probably only enabled after the project is public.

A readme how to create the docs should be included in the `pdocs` folder.

Discussion:

Patch:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000..0eb247b
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,26 @@
+name: Docs
+
+on:
+  push:
+    branches:
+      - master
+jobs:
+  test:
+    name: Docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+            pip install -e .['docs']
+            pip install ghp-import
+      - name: deploy docs
+        run: |
+          echo '<meta http-equiv=refresh content=0;url=polars/index.html>' > target/doc/index.html && \
+          pdoc --html gbif_dl --output-dir target/doc --template-dir pdoc --force && \
+          ghp-import -n target/doc && \
+          git push -qf https://${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git gh-pages
\ No newline at end of file
diff --git a/gbif_dl/__init__.py b/gbif_dl/__init__.py
index a87cf87..4e48a26 100644
--- a/gbif_dl/__init__.py
+++ b/gbif_dl/__init__.py
@@ -1,3 +1,15 @@
+"""
+__gbif-dl__ provides easy access to media data from the GBIF database to be used for training machine learning models.
+It wraps the GBIF API and supports directly querying the api to obtain and download a list of urls.
+Existing queries can also be obtained using the download api of GBIF simply by providing a GBIF DOI key.
+The package provides an efficient downloader that uses pythons modern asyncio modules to speed up downloading of
+many small files as typically occur in downloading image files.
+
+This is the python package API documentation.
+
+Please checkout [github repository](https://github.com/plantnet/gbif-dl) for more information.
+"""
+
 from . import io
 from .generators import dwca
 from .generators import api
diff --git a/gbif_dl/generators/__init__.py b/gbif_dl/generators/__init__.py
index e69de29..f0afb73 100644
--- a/gbif_dl/generators/__init__.py
+++ b/gbif_dl/generators/__init__.py
@@ -0,0 +1,3 @@
+"""
+Generate urls from GBIF based on queries or darwin-core archives.
+"""
diff --git a/gbif_dl/generators/api.py b/gbif_dl/generators/api.py
index 82b19e1..5e0aa4d 100644
--- a/gbif_dl/generators/api.py
+++ b/gbif_dl/generators/api.py
@@ -1,3 +1,8 @@
+"""
+This module creates the interface to the GBIF API. By providing queries to the api, users can use 
+this module to obtain lists of urls of media data to be downloaded using the [io](gbif_dl.io) module.
+"""
+
 import pygbif
 import itertools as it
 import random
@@ -29,11 +34,8 @@ def gbif_query_generator(
         label (str, optional): Output label name. Defaults to `None`.
         subset (str, optional): Subset name. Defaults to `None`.
 
-    Returns:
-        str: [description]
-
     Yields:
-        Iterator[str]: [description]
+        MediaData
     """
     offset = 0
 
@@ -84,7 +86,7 @@ def gbif_count(mediatype: str = "StillImage", *args, **kwargs) -> str:
     return pygbif.occurrences.search(limit=0, mediatype=mediatype, *args, **kwargs)["count"]
 
 
-def dproduct(dicts):
+def _dproduct(dicts):
     """Returns the products of dicts"""
     return (dict(zip(dicts, x)) for x in it.product(*dicts.values()))
 
@@ -104,38 +106,32 @@ def generate_urls(
     """Provides url generator from given query
 
     Args:
-        queries (Dict):
-            dictionary of queries supported by the GBIF api
+        queries (Dict): dictionary of queries supported by the GBIF api
         label (str, optional): Output label name.
             Defaults to `None` which yields all metadata.
-        nb_samples (int):
-            Limit the total number of samples retrieved from the API.
+        nb_samples (int): Limit the total number of samples retrieved from the API.
             When set to -1 and `split_streams_by` is not `None`,
             a minimum number of samples will be calculated
             from using the number of available samples per stream.
             Defaults to `None` which retrieves all samples from all streams until
             all streams are exchausted.
-        nb_samples_per_stream (int):
-            Limit the maximum number of items to be retrieved per stream.
+        nb_samples_per_stream (int): Limit the maximum number of items to be retrieved per stream.
             Defaults to `None` which retrieves all samples from stream until
             stream generator is exhausted.
-        split_streams_by (Optional[Union[str, List]], optional):
-            Stream identifiers to be balanced. Defaults to None.
-        subset_streams (Optional[Union[str, Dict]], optional):
-            Map certain streams into separate subsets, by setting the `subset`
-            metadata. Supports a remainder value of `"*"` which acts as a
-            wildcard. E.g. `subset_streams={"train": { "speciesKey": [5352251, 3190653]},
+        split_streams_by (Optional[Union[str, List]], optional): Stream identifiers to be balanced.
+            Defaults to None.
+        subset_streams (Optional[Union[str, Dict]], optional): Map certain streams into
+            separate subsets, by setting the `subset` metadata. Supports a remainder
+            value of `"*"` which acts as a wildcard. Defaults to None.
+            E.g. `subset_streams={"train": { "speciesKey": [5352251, 3190653]},
             "test": { "speciesKey": "*" }}` will move species of 5352251 and 3190653
             into `train` whereas all other species will go into test.
-            Defaults to None.
-        weighted_streams (int):
-            Calculates sampling weights for all streams and applies them during
+        weighted_streams (int): Calculates sampling weights for all streams and applies them during
             sampling. To be combined with nb_samples not `None`.
             Defaults to `False`.
         cache_requests (bool, optional): Enable GBIF API cache.
             Can significantly improve API requests. Defaults to False.
-        mediatype (str):
-            supported GBIF media type. Can be `StillImage`, `MovingImage`, `Sound`.
+        mediatype (str): supported GBIF media type. Can be `StillImage`, `MovingImage`, `Sound`.
             Defaults to `StillImage`.
 
     Returns:
@@ -164,7 +160,7 @@ def generate_urls(
 
         # for each b in balance_queries, create a separate stream
         # later we control the sampling processs of these streams to balance them
-        for b in dproduct(balance_queries):
+        for b in _dproduct(balance_queries):
             subset = None
             # for each stream we wrap into pescador Streamers for additional features
             for key, value in b.items():
@@ -202,7 +198,7 @@ def generate_urls(
 
         if verbose:
             nb_queries = [
-                gbif_count(mediatype=mediatype, **q, **b) for b in dproduct(balance_queries)
+                gbif_count(mediatype=mediatype, **q, **b) for b in _dproduct(balance_queries)
             ]
             print(sum(nb_queries))
 
@@ -211,14 +207,14 @@ def generate_urls(
         if nb_samples == -1:
             # calculate the miniumum number of samples available per stream
             nb_samples = min(
-                [gbif_count(mediatype=mediatype, **q, **b) for b in dproduct(balance_queries)]
+                [gbif_count(mediatype=mediatype, **q, **b) for b in _dproduct(balance_queries)]
             ) * len(streams)
 
         if weighted_streams:
             weights = np.array(
                 [
                     float(gbif_count(mediatype=mediatype, **q, **b))
-                    for b in dproduct(balance_queries)
+                    for b in _dproduct(balance_queries)
                 ]
             )
             weights /= np.max(weights)
diff --git a/gbif_dl/generators/dwca.py b/gbif_dl/generators/dwca.py
index a3ab667..c74d114 100644
--- a/gbif_dl/generators/dwca.py
+++ b/gbif_dl/generators/dwca.py
@@ -1,12 +1,16 @@
+"""
+This module creates the interface to the GBIF API via a [download/doi functionality](https://www.gbif.org/data-processing).
+Give a link to a GBIF darwincore archive, users can use this module to obtain lists of urls of media data 
+to be downloaded using the [io](gbif_dl.io) module.
+"""
+
 import pygbif
 from pathlib import Path
 import random
 import requests
 import hashlib
-import mimetypes
 import re
 import tempfile
-import shutil
 from typing import Optional
 import os
 
@@ -87,7 +91,7 @@ def doi_to_gbif_key(doi: str) -> str:
                 return gbif_key
 
 
-def _is_doi(identifier: str) -> bool:
+def is_doi(identifier: str) -> bool:
     """Validates if identifier is a valid DOI
 
     Args:
@@ -133,7 +137,7 @@ def generate_urls(
     Returns:
         Iterable: item generator that yields files from generator
     """
-    if _is_doi:
+    if is_doi:
         key = doi_to_gbif_key(identifier)
     else:
         key = identifier
diff --git a/gbif_dl/io.py b/gbif_dl/io.py
index c25431c..7a33298 100644
--- a/gbif_dl/io.py
+++ b/gbif_dl/io.py
@@ -1,7 +1,10 @@
+"""
+Async based fast downloader.
+"""
 import asyncio
 import inspect
 from pathlib import Path
-from typing import AsyncGenerator, Callable, Dict, Generator, Union, Optional
+from typing import AsyncGenerator, Callable, Generator, Union, Optional
 import sys
 import json
 import hashlib
@@ -40,7 +43,7 @@ async def download_single(
     is_valid_file: Optional[Callable[[bytes], bool]] = None,
     overwrite: bool = False,
     proxy: Optional[str] = None,
-    random_subsets: Optional[Dict] = None,
+    random_subsets: Optional[dict] = None,
 ):
     """Async function to download single url to disk
 
@@ -51,16 +54,13 @@ async def download_single(
         is_valid_file (optional): A function that takes bytes
             and checks if the bytes originate from a valid file
             (used to check of corrupt files). Defaults to None.
-        overwrite (bool):
-            overwrite files with existing `baseline` signature, Defaults to False.
-        proxy (str):
-            proxy server url. Authentication credentials can be passed in URL.
+        overwrite (bool): overwrite files with existing `baseline` signature, Defaults to False.
+        proxy (str): proxy server url. Authentication credentials can be passed in URL.
             e.g `proxy="http://user:pass@some.proxy.com"`.
             Proxy can also be used globally using environmental variables.
             See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html.
-        random_subsets (Dict):
-            add random splits/subsets given as a dict of class names and it's propability.
-            e.g. `{'train': 0.9, test': 0.1}` will result in 90% of the items
+        random_subsets (dict[str, float]): add random splits/subsets given as a dict of class names
+            and it's propability. e.g. `{'train': 0.9, test': 0.1}` will result in 90% of the items
             go into a `train` subfolder and 10% go into a `test` subfolder.
             The propabilities have to sum up to `1.0` to avoid an error.
     """
@@ -132,14 +132,14 @@ async def download_single(
             await fp.write(json.dumps(label))
 
 
-async def download_queue(
+async def _download_queue(
     queue: asyncio.Queue,
     session: RetryClient,
     root: str,
     is_valid_file: Optional[Callable[[bytes], bool]] = None,
     overwrite: bool = False,
     proxy: Optional[str] = None,
-    random_subsets: Optional[Dict] = None,
+    random_subsets: Optional[dict] = None,
 ):
     """Consumes items from download queue
 
@@ -150,15 +150,12 @@ async def download_queue(
         is_valid_file (optional): A function that takes bytes
             and checks if the bytes originate from a valid file
             (used to check of corrupt files). Defaults to None.
-        overwrite (bool):
-            overwrite files with existing `baseline` signature, Defaults to False.
-        proxy (str):
-            proxy server url. Authentication credentials can be passed in URL.
+        overwrite (bool): overwrite files with existing `baseline` signature, Defaults to False.
+        proxy (str): proxy server url. Authentication credentials can be passed in URL.
             e.g `proxy="http://user:pass@some.proxy.com"`.
             Proxy can also be used globally using environmental variables.
             See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html.
-        random_subsets (Dict):
-            add random subset given as a dict of class names and it's propability.
+        random_subsets (dict[str, float]): add random subset given as a dict of class names and it's propability.
             e.g. `{'train': 0.9, test': 0.1}` will result in 90% of the items
             go into a `train` subfolder and 10% go into a `test` subfolder.
             The propabilities have to sum up to `1.0` to avoid an error.
@@ -172,7 +169,7 @@ async def download_queue(
         queue.task_done()
 
 
-async def download_from_asyncgen(
+async def _download_from_asyncgen(
     items: AsyncGenerator,
     root: str = "data",
     tcp_connections: int = 64,
@@ -183,38 +180,28 @@ async def download_from_asyncgen(
     overwrite: bool = False,
     is_valid_file: Optional[Callable[[bytes], bool]] = None,
     proxy: Optional[str] = None,
-    random_subsets: Optional[Dict] = None,
+    random_subsets: Optional[dict] = None,
 ):
     """Asynchronous downloader that takes an interable and downloads it
 
     Args:
-        items (Union[Generator, AsyncGenerator]):
-            (async/sync) generator that yiels a standardized dict of urls
-        root (str, optional):
-            Root path of downloads. Defaults to "data".
-        tcp_connections (int, optional):
-            Maximum number of concurrent TCP connections. Defaults to 128.
-        nb_workers (int, optional):
-            Maximum number of workers. Defaults to 128.
-        batch_size (int, optional):
-            Maximum queue batch size. Defaults to 8.
-        retries (int, optional):
-            Maximum number of retries. Defaults to 3.
-        verbose (bool, if isinstance(e, Iterable):ptional):
-            Activate verbose. Defaults to False.
-        overwrite (bool):
-            overwrite files with existing `baseline` signature, Defaults to False.
+        items (Union[Generator, AsyncGenerator]): (async/sync) generator that yiels a standardized dict of urls
+        root (str, optional): Root path of downloads. Defaults to "data".
+        tcp_connections (int, optional): Maximum number of concurrent TCP connections. Defaults to 128.
+        nb_workers (int, optional): Maximum number of workers. Defaults to 128.
+        batch_size (int, optional): Maximum queue batch size. Defaults to 8.
+        retries (int, optional): Maximum number of retries. Defaults to 3.
+        verbose (bool, Optional): Activate verbose. Defaults to False.
+        overwrite (bool): overwrite files with existing `baseline` signature, Defaults to False.
         is_valid_file (optional): A function that takes bytes
             and checks if the bytes originate from a valid file
             (used to check of corrupt files). Defaults to None.
             overwrite existing files, Defaults to False.
-        proxy (str):
-            proxy server url. Authentication credentials can be passed in URL.
+        proxy (str): proxy server url. Authentication credentials can be passed in URL.
             e.g `proxy="http://user:pass@some.proxy.com"`.
             Proxy can also be used globally using environmental variables.
             See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html.
-        random_subsets (Dict):
-            add random subset given as a dict of class names and it's propability.
+        random_subsets (dict[str, float]): add random subset given as a dict of class names and it's propability.
             e.g. `{'train': 0.9, 'test': 0.1}` will result in 90% of the items
             go into a `train` subfolder and 10% go into a `test` subfolder.
             The propabilities have to sum up to `1.0` to avoid an error.
@@ -236,7 +223,7 @@ async def download_from_asyncgen(
         loop = asyncio.get_event_loop()
         workers = [
             loop.create_task(
-                download_queue(
+                _download_queue(
                     queue,
                     session,
                     root=root,
@@ -273,38 +260,28 @@ def download(
     overwrite: bool = False,
     is_valid_file: Optional[Callable[[bytes], bool]] = None,
     proxy: Optional[str] = None,
-    random_subsets: Optional[Dict] = None,
+    random_subsets: Optional[dict] = None,
 ):
     """Core download function that takes an interable (sync or async)
 
     Args:
-        items (Union[Generator, AsyncGenerator, Iterable]):
-            (async/sync) generator or list that yiels a standardized dict of urls
-        root (str, optional):
-            Root path of downloads. Defaults to "data".
-        tcp_connections (int, optional):
-            Maximum number of concurrent TCP connections. Defaults to 128.
-        nb_workers (int, optional):
-            Maximum number of workers. Defaults to 128.
-        batch_size (int, optional):
-            Maximum queue batch size. Defaults to 8.
-        retries (int, optional):
-            Maximum number of retries. Defaults to 3.
-        verbose (bool, optional):
-            Activate verbose. Defaults to False.
-        overwrite (bool):
-            overwrite files with existing `baseline` signature, Defaults to False.
+        items (Union[Generator, AsyncGenerator, Iterable]): (async/sync) generator
+            or list that yiels a standardized dict of urls
+        root (str, optional): Root path of downloads. Defaults to "data".
+        tcp_connections (int, optional): Maximum number of concurrent TCP connections. Defaults to 128.
+        nb_workers (int, optional): Maximum number of workers. Defaults to 128.
+        batch_size (int, optional): Maximum queue batch size. Defaults to 8.
+        retries (int, optional): Maximum number of retries. Defaults to 3.
+        verbose (bool, optional): Activate verbose. Defaults to False.
+        overwrite (bool): overwrite files with existing `baseline` signature, Defaults to False.
         is_valid_file (optional): A function that takes bytes
             and checks if the bytes originate from a valid file
             (used to check of corrupt files). Defaults to None.
             overwrite existing files, Defaults to False.
-        proxy (str):
-            proxy server url. Authentication credentials can be passed in URL.
-            e.g `proxy="http://user:pass@some.proxy.com"`.
-            Proxy can also be used globally using environmental variables.
+        proxy (str): Proxy server url. Authentication credentials can be passed in URL. e.g
+            `proxy="http://user:pass@some.proxy.com"`. Proxy can also be used globally using environmental variables.
             See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html.
-        random_subsets (Dict):
-            add random subset given as a dict of class names and it's propability.
+        random_subsets (dict[str, float]): add random subset given as a dict of class names and it's propability.
             e.g. `{'train': 0.9, test': 0.1}` will result in 90% of the items
             go into a `train` subfolder and 10% into a `test` subfolder.
             The propabilities have to sum up to `1.0` to avoid an error.
@@ -327,7 +304,7 @@ def download(
             raise RuntimeError("Make sure that weight probabilities add up to one")
 
     return run_async(
-        download_from_asyncgen,
+        _download_from_asyncgen,
         items,
         root=root,
         tcp_connections=tcp_connections,
diff --git a/gbif_dl/utils.py b/gbif_dl/utils.py
index dd06993..9ea7ac2 100644
--- a/gbif_dl/utils.py
+++ b/gbif_dl/utils.py
@@ -1,3 +1,6 @@
+"""
+Utility functions
+"""
 import asyncio
 import functools
 import threading
diff --git a/pdoc/config.mako b/pdoc/config.mako
new file mode 100644
index 0000000..9e9f669
--- /dev/null
+++ b/pdoc/config.mako
@@ -0,0 +1,53 @@
+<%!
+    # Template configuration. Copy over in your template directory
+    # (used with `--template-dir`) and adapt as necessary.
+    # Note, defaults are loaded from this distribution file, so your
+    # config.mako only needs to contain values you want overridden.
+    # You can also run pdoc with `--config KEY=VALUE` to override
+    # individual values.
+    html_lang = 'en'
+    show_inherited_members = False
+    extract_module_toc_into_sidebar = True
+    list_class_variables_in_index = True
+    sort_identifiers = True
+    show_type_annotations = True
+    # Show collapsed source code block next to each item.
+    # Disabling this can improve rendering speed of large modules.
+    show_source_code = True
+    # If set, format links to objects in online source code repository
+    # according to this template. Supported keywords for interpolation
+    # are: commit, path, start_line, end_line.
+    git_link_template = 'https://github.com/plantnet/gbif-dl/blob/{commit}/{path}#L{start_line}-L{end_line}'
+    #git_link_template = 'https://gitlab.com/USER/PROJECT/blob/{commit}/{path}#L{start_line}-L{end_line}'
+    #git_link_template = 'https://bitbucket.org/USER/PROJECT/src/{commit}/{path}#lines-{start_line}:{end_line}'
+    #git_link_template = 'https://CGIT_HOSTNAME/PROJECT/tree/{path}?id={commit}#n{start-line}'
+    # A prefix to use for every HTML hyperlink in the generated documentation.
+    # No prefix results in all links being relative.
+    link_prefix = ''
+    # Enable syntax highlighting for code/source blocks by including Highlight.js
+    syntax_highlighting = True
+    # Set the style keyword such as 'atom-one-light' or 'github-gist'
+    #     Options: https://github.com/highlightjs/highlight.js/tree/master/src/styles
+    #     Demo: https://highlightjs.org/static/demo/
+    hljs_style = 'github'
+    # If set, insert Google Analytics tracking code. Value is GA
+    # tracking id (UA-XXXXXX-Y).
+    google_analytics = ''
+    # If set, insert Google Custom Search search bar widget above the sidebar index.
+    # The whitespace-separated tokens represent arbitrary extra queries (at least one
+    # must match) passed to regular Google search. Example:
+    #google_search_query = 'inurl:github.com/USER/PROJECT  site:PROJECT.github.io  site:PROJECT.website'
+    google_search_query = ''
+    # Enable offline search using Lunr.js. For explanation of 'fuzziness' parameter, which is
+    # added to every query word, see: https://lunrjs.com/guides/searching.html#fuzzy-matches
+    # If 'index_docstrings' is False, a shorter index is built, indexing only
+    # the full object reference names.
+    #lunr_search = {'fuzziness': 1, 'index_docstrings': True}
+    lunr_search = None
+    # If set, render LaTeX math syntax within \(...\) (inline equations),
+    # or within \[...\] or $$...$$ or `.. math::` (block equations)
+    # as nicely-formatted math formulas using MathJax.
+    # Note: in Python docstrings, either all backslashes need to be escaped (\\)
+    # or you need to use raw r-strings.
+    latex_math = True
+%>
\ No newline at end of file
diff --git a/setup.py b/setup.py
index fa323df..aefb487 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ setup(
         "tqdm",
         "typing-extensions; python_version < '3.8'",
     ],
-    extras_require={"tests": ["pytest"]},
+    extras_require={"tests": ["pytest"], "docs": ["pdoc3"]},
     # entry_points={"console_scripts": ["gbif_dl=gbif_dl.cli:download"]},
     packages=find_packages(),
     include_package_data=True,


Test patch:
diff --git a/tests/test_dwca.py b/tests/test_dwca.py
index 23d79b7..8852cc0 100644
--- a/tests/test_dwca.py
+++ b/tests/test_dwca.py
@@ -13,8 +13,8 @@ def nb_samples(request):
 
 
 def test_is_doi(doi):
-    assert gbif_dl.dwca._is_doi(doi) is True
-    assert gbif_dl.dwca._is_doi("xyz") is False
+    assert gbif_dl.dwca.is_doi(doi) is True
+    assert gbif_dl.dwca.is_doi("xyz") is False
 
 
 def test_doi_to_gbif_key(doi):