Repository: FicHub/fichub-cli (language: python)

Issue:
Get metadata in JSON
Ala `fanficfare --meta-only --json-meta "$url" `. It’d be good if the keys (JSON structure) are compatible with fanficfare, too.


Discussion / hints:

Fix patch:
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 27a443b..1893426 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.5
+current_version = 0.3.6
 commit = True
 tag = False
 parse = ^
diff --git a/README.md b/README.md
index 02044ec..ebd984c 100644
--- a/README.md
+++ b/README.md
@@ -35,14 +35,21 @@ Usage: fichub_cli [OPTIONS]
   https://github.com/FicHub/fichub-cli/issues
 
 Options:
-  -u, --url TEXT         The url of the fanfiction enclosed within quotes
-  -i, --infile TEXT      Give a filename to read URLs from
-  -l, --list_url TEXT    Enter a comma separated list of urls to download, enclosed within quotes
-  -o, --out_dir TEXT     Absolute/Relative path to the Output directory for files (default: Current Directory)
-  -f, --format TEXT      Download Format: epub (default), mobi, pdf or html
-  --force                Force overwrite of an existing file
-  -s, --supported_sites  List of supported sites
-  -d, --debug            Debug mode
+  -u, --url TEXT         The url of the fanfiction enclosed within quotes.
+  -i, --infile TEXT      Give a filename to read URLs from.
+  -l, --list-url TEXT    Enter a comma separated list of urls to download,
+                         enclosed within quotes.
+  -o, --out-dir TEXT     Absolute path to the Output directory for files
+                         (default: Current Directory).
+  -f, --format TEXT      Download Format: epub (default), mobi, pdf or html.
+  --force                Force overwrite of an existing file.
+  --get-urls TEXT        Get all story urls found from a page. Currently
+                         supports archiveofourown.org only.
+  -s, --supported-sites  List of supported sites.
+  -d, --debug            Show the log in the console for debugging.
+  --meta-json TEXT       Fetch only the metadata for the fanfiction in json
+                         format.
+  --log                  Save the logfile for debugging.
   -v, --version          Display version & quit.
   --help                 Show this message and exit.
 ```
@@ -67,6 +74,18 @@ fichub_cli -i urls.txt
 fichub_cli -l "https://www.fanfiction.net/s/11191235/1/Harry-Potter-and-the-Prince-of-Slytherin,https://www.fanfiction.net/s/13720575/1/A-Cadmean-Victory-Remastered"
 ```
 
+- To get all story urls found from a page. Currently supports archiveofourown.org only.
+
+```
+fichub_cli --get-urls https://archiveofourown.org/users/flamethrower/
+```
+
+- To fetch only the metadata for the fanfiction in json format.
+
+```
+fichub_cli --meta-json "https://www.fanfiction.net/s/12933896/1/Things-you-cannot-leave-behind"
+```
+
 ### Default Configuration
 
 - The fanfiction will be downloaded in epub format. To change it, use `-f` followed by the format.
diff --git a/fichub_cli/cli.py b/fichub_cli/cli.py
index 65ed349..4187f10 100644
--- a/fichub_cli/cli.py
+++ b/fichub_cli/cli.py
@@ -2,7 +2,7 @@
 import sys
 from loguru import logger
 from datetime import datetime
-from colorama import init, Fore
+from colorama import init, Fore, Style
 
 from .utils.processing import get_format_type
 from .utils.fetch_data import FetchData
@@ -13,21 +13,23 @@
 
 # @logger.catch  # for internal debugging
 @click.command(no_args_is_help=True)
-@click.option('-u', '--url', help='The url of the fanfiction enclosed within quotes ')
-@click.option('-i', '--infile', help='Give a filename to read URLs from')
-@click.option('-l', '--list-url', 'list_url',  help='Enter a comma separated list of urls to download, enclosed within quotes')
-@click.option('-o', '--out-dir', 'out_dir', default="", help='Absolute path to the Output directory for files (default: Current Directory)')
-@click.option('-f', '--format', '_format', default="epub", help='Download Format: epub (default), mobi, pdf or html')
-@click.option('--force', default=False, help=' Force overwrite of an existing file', is_flag=True)
+@click.option('-u', '--url', help='The url of the fanfiction enclosed within quotes.')
+@click.option('-i', '--infile', help='Give a filename to read URLs from.')
+@click.option('-l', '--list-url', 'list_url',  help='Enter a comma separated list of urls to download, enclosed within quotes.')
+@click.option('-o', '--out-dir', 'out_dir', default="", help='Absolute path to the Output directory for files (default: Current Directory).')
+@click.option('-f', '--format', '_format', default="epub", help='Download Format: epub (default), mobi, pdf or html.')
+@click.option('--force', default=False, help=' Force overwrite of an existing file.', is_flag=True)
 @click.option('--get-urls', 'get_urls', default=None, help='Get all story urls found from a page. Currently supports archiveofourown.org only.')
-@click.option('-s', '--supported-sites', 'supported_sites', default=False, help='List of supported sites', is_flag=True)
-@click.option('-d', '--debug', default=False, help='Show the log in the console for debugging', is_flag=True)
-@click.option('--log', default=False, help='Save the logfile for debugging', is_flag=True)
-@click.option('-a', '--automated', default=False, help='For internal testing only', is_flag=True, hidden=True)
+@click.option('-s', '--supported-sites', 'supported_sites', default=False, help='List of supported sites.', is_flag=True)
+@click.option('-d', '--debug', default=False, help='Show the log in the console for debugging.', is_flag=True)
+@click.option('--meta-json', 'meta_json', default=None, help='Fetch only the metadata for the fanfiction in json format.')
+@click.option('--log', default=False, help='Save the logfile for debugging.', is_flag=True)
+@click.option('-a', '--automated', default=False, help='For internal testing only.', is_flag=True, hidden=True)
 @click.option('-v', '--version', default=False, help='Display version & quit.', is_flag=True)
 def run_cli(infile: str, url: str, list_url: str, _format: str, get_urls: str,
             out_dir: str, debug: bool, version: bool, log: bool,
-            supported_sites: bool, force: bool, automated: bool):
+            supported_sites: bool, force: bool, automated: bool,
+            meta_json: str):
     """
     A CLI for the fichub.net API
 
@@ -62,33 +64,40 @@ def run_cli(infile: str, url: str, list_url: str, _format: str, get_urls: str,
         fic = FetchData(debug)
         fic.get_urls_from_page(get_urls)
 
+    elif meta_json:
+        fic = FetchData(debug)
+        fic.get_metadata(meta_json)
+
     if version:
-        click.echo("Version: 0.3.5")
+        click.echo("Version: 0.3.6")
         sys.exit(0)
 
     if supported_sites:
         click.echo(Fore.GREEN + """
-    Supported Sites
-
-        - SpaceBattles, SufficientVelocity, QuestionableQuesting (XenForo)
-        - FanFiction.net, FictionPress
-        - Archive Of Our Own
-        - Royal Road
-        - Harry Potter Fanfic Archive
-        - Sink Into Your Eyes
-        - AdultFanfiction.org
-        - Worm, Ward
-
-    Partial support (or not tested recently):
-
-        - XenForo based sites (Bulbagarden Forums, The Fanfiction Forum, Fanfic Paradise)
-        - Fiction Alley
-        - Fiction Hunt
-        - The Sugar Quill (largely untested)
-        - FanficAuthors (minimal)
-        - Harry Potter Fanfiction (archive from pre-revival)
-
-    To report issues upstream for these sites, visit https://fichub.net/#contact
+Supported Sites:""" + Style.RESET_ALL + """
+
+    - SpaceBattles, SufficientVelocity, QuestionableQuesting (XenForo)
+    - FanFiction.net, FictionPress
+    - Archive Of Our Own
+    - Royal Road
+    - Harry Potter Fanfic Archive
+    - Sink Into Your Eyes
+    - AdultFanfiction.org
+    - Worm, Ward
+""" + Fore.GREEN + """
+Partial support (or not tested recently):""" + Style.RESET_ALL + """
+
+    - XenForo based sites
+        - Bulbagarden Forums
+        - The Fanfiction Forum
+        - Fanfic Paradise
+    - Fiction Alley
+    - Fiction Hunt
+    - The Sugar Quill(largely untested)
+    - FanficAuthors(minimal)
+    - Harry Potter Fanfiction(archive from pre-revival)
+""" + Fore.BLUE + """
+To report issues upstream for these sites, visit https://fichub.net/#contact
 """)
         sys.exit(0)
 
diff --git a/fichub_cli/utils/fetch_data.py b/fichub_cli/utils/fetch_data.py
index 9cc13e5..c55057c 100644
--- a/fichub_cli/utils/fetch_data.py
+++ b/fichub_cli/utils/fetch_data.py
@@ -1,7 +1,7 @@
 import re
 import requests
 from tqdm import tqdm
-from colorama import Fore
+from colorama import Fore, Style
 from loguru import logger
 from bs4 import BeautifulSoup
 from rich.console import Console
@@ -25,6 +25,8 @@ def __init__(self, format_type="epub", out_dir="", force=False,
         self.exit_status = 0
 
     def get_fic_with_infile(self, infile: str):
+        if self.debug:
+            logger.debug("Calling get_fic_with_infile()")
 
         try:
             with open(infile, "r") as f:
@@ -37,19 +39,16 @@ def get_fic_with_infile(self, infile: str):
             exit(1)
 
         init_log(self.debug, self.force)
-        if self.debug:
-            logger.debug("Calling get_fic_with_infile()")
-
         with tqdm(total=len(urls), ascii=False,
                   unit="file", bar_format=bar_format) as pbar:
 
             for url in urls:
 
+                download_processing_log(self.debug, url)
                 supported_url, self.exit_status = check_url(
-                    pbar, url, self.debug, self.exit_status)
+                    url, self.debug, self.exit_status)
                 if supported_url:
                     try:
-                        download_processing_log(self.debug, url)
                         fic = FicHub(self.debug, self.automated,
                                      self.exit_status)
                         fic.get_fic_metadata(url, self.format_type)
@@ -77,27 +76,27 @@ def get_fic_with_infile(self, infile: str):
                         pass  # skip the unsupported url
 
                 else:  # skip the unsupported url
+                    pbar.update(1)
                     continue
 
     def get_fic_with_list(self, list_url: str):
 
-        urls = list_url.split(",")
-
-        init_log(self.debug, self.force)
         if self.debug:
             logger.debug("Calling get_fic_with_list()")
 
+        urls = list_url.split(",")
+
+        init_log(self.debug, self.force)
         with tqdm(total=len(urls), ascii=False,
                   unit="file", bar_format=bar_format) as pbar:
 
             for url in urls:
-
+                download_processing_log(self.debug, url)
                 supported_url,  self.exit_status = check_url(
-                    pbar, url, self.debug, self.exit_status)
+                    url, self.debug, self.exit_status)
 
                 if supported_url:
                     try:
-                        download_processing_log(self.debug, url)
                         fic = FicHub(self.debug, self.automated,
                                      self.exit_status)
                         fic.get_fic_metadata(url, self.format_type)
@@ -124,24 +123,24 @@ def get_fic_with_list(self, list_url: str):
                         pass  # skip the unsupported url
 
                 else:  # skip the unsupported url
+                    pbar.update(1)
                     continue
 
     def get_fic_with_url(self, url: str):
 
-        init_log(self.debug, self.force)
         if self.debug:
             logger.debug("Calling get_fic_with_url()")
 
+        init_log(self.debug, self.force)
         with tqdm(total=1, ascii=False,
                   unit="file", bar_format=bar_format) as pbar:
 
+            download_processing_log(self.debug, url)
             supported_url, self.exit_status = check_url(
-                pbar, url, self.debug, self.exit_status)
+                url, self.debug, self.exit_status)
 
             if supported_url:
                 try:
-                    download_processing_log(self.debug, url)
-
                     fic = FicHub(self.debug, self.automated, self.exit_status)
                     fic.get_fic_metadata(url, self.format_type)
 
@@ -166,8 +165,8 @@ def get_fic_with_url(self, url: str):
                     self.exit_status = 1
                     pass  # skip the unsupported url
 
-            else:  # skip the unsupported url
-                pass
+            else:
+                pbar.update(1)
 
     def get_urls_from_page(self, get_urls: str):
 
@@ -208,17 +207,30 @@ def get_urls_from_page(self, get_urls: str):
                 if ao3_works_list:
                     found_flag = True
                     tqdm.write(Fore.GREEN +
-                               f"\nFound {len(ao3_works_list)} works urls.")
+                               f"\nFound {len(ao3_works_list)} works urls." +
+                               Style.RESET_ALL)
                     ao3_works_list = '\n'.join(ao3_works_list)
                     tqdm.write(ao3_works_list)
 
                 if ao3_series_list:
                     found_flag = True
                     tqdm.write(Fore.GREEN +
-                               f"\nFound {len(ao3_series_list)} series urls.")
+                               f"\nFound {len(ao3_series_list)} series urls." +
+                               Style.RESET_ALL)
                     ao3_series_list = '\n'.join(ao3_series_list)
                     tqdm.write(ao3_series_list)
 
             if found_flag is False:
                 tqdm.write(Fore.RED + "\nFound 0 urls.")
                 self.exit_status = 1
+
+    def get_metadata(self, url: str):
+
+        with console.status("[bold green]Processing..."):
+            supported_url, self.exit_status = check_url(
+                url, self.debug, self.exit_status)
+
+            if supported_url:
+                fic = FicHub(self.debug, self.automated, self.exit_status)
+                fic.get_fic_extraMetadata(url)
+                console.print(fic.fic_extraMetadata)
diff --git a/fichub_cli/utils/fichub.py b/fichub_cli/utils/fichub.py
index d8b74e1..447f71f 100644
--- a/fichub_cli/utils/fichub.py
+++ b/fichub_cli/utils/fichub.py
@@ -3,9 +3,10 @@
 from colorama import Fore, Style
 from tqdm import tqdm
 from loguru import logger
+import json
 
 headers = {
-    'User-Agent': 'fichub_cli/0.3.5',
+    'User-Agent': 'fichub_cli/0.3.6',
 }
 
 
@@ -70,11 +71,12 @@ def get_fic_metadata(self, url: str, format_type: int):
             self.exit_status = 1
             if self.debug:
                 logger.error(
-                    f"\nSkipping unsupported URL: {url}\nTo see the supported site list, fichub_cli -s")
+                    f"Skipping unsupported URL: {url}")
             else:
                 tqdm.write(
-                    Fore.RED + f"\nSkipping unsupported URL: {url}" +
-                    Style.RESET_ALL + "\nTo see the supported site list, fichub_cli -s")
+                    Fore.RED + f"Skipping unsupported URL: {url}" +
+                    Style.RESET_ALL + Fore.CYAN +
+                    "\nTo see the supported site list, use -s flag")
 
     def get_fic_data(self, download_url: str):
 
@@ -88,3 +90,23 @@ def get_fic_data(self, download_url: str):
         if self.debug:
             logger.debug(
                 f"GET: {self.response_data.status_code}: {self.response_data.url}")
+
+    def get_fic_extraMetadata(self, url: str):
+
+        params = {'q': url}
+        if self.automated:  # for internal testing
+            params['automated'] = 'true'
+            if self.debug:
+                logger.debug(
+                    "--automated flag was passed. Internal Testing mode is on.")
+
+        response = requests.get(
+            "https://fichub.net/api/v0/epub", params=params,
+            allow_redirects=True, headers=headers
+        )
+
+        if self.debug:
+            logger.debug(f"GET: {response.status_code}: {response.url}")
+
+        response = response.json()
+        self.fic_extraMetadata = json.dumps(response['meta'], indent=4)
diff --git a/fichub_cli/utils/processing.py b/fichub_cli/utils/processing.py
index 9928205..88d6942 100644
--- a/fichub_cli/utils/processing.py
+++ b/fichub_cli/utils/processing.py
@@ -30,7 +30,7 @@ def get_format_type(_format: str = "epub") -> int:
     return format_type
 
 
-def check_url(pbar, url: str, debug: bool = False,
+def check_url(url: str, debug: bool = False,
               exit_status: int = 0) -> Tuple[bool, int]:
 
     if re.search(r"\barchiveofourown.org/series\b", url):
@@ -43,16 +43,16 @@ def check_url(pbar, url: str, debug: bool = False,
         unsupported_flag = False
 
     if unsupported_flag:
-        pbar.update(1)
         exit_status = 1
 
         if debug:
             logger.error(
-                f"Skipping unsupported URL: {url}\nTo see the supported site list, fichub_cli -s")
+                f"Skipping unsupported URL: {url}")
         else:
             tqdm.write(
                 Fore.RED + f"Skipping unsupported URL: {url}" +
-                Style.RESET_ALL + "\nTo see the supported site list, fichub_cli -s")
+                Style.RESET_ALL + Fore.CYAN +
+                "\nTo see the supported site list, use -s flag")
 
         return False, exit_status
 
@@ -84,7 +84,8 @@ def save_data(out_dir: str, file_name:  str, download_url: str,
         else:
             tqdm.write(
                 Fore.RED +
-                f"{out_dir+file_name} is already the latest version. Skipping download. Use --force flag to overwrite.")
+                f"{out_dir+file_name} is already the latest version. Skipping download." +
+                Style.RESET_ALL + Fore.CYAN + " Use --force flag to overwrite.")
 
     else:
         if force and debug:
diff --git a/setup.py b/setup.py
index eadd2d9..2d2a2bb 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@
     description="A CLI for the fichub.net API",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    version='0.3.5',
+    version='0.3.6',
     license='MIT',
     url="https://github.com/FicHub/fichub-cli",
     packages=find_packages(include=['fichub_cli', 'fichub_cli.*']),


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 6bf3983..f757d3f 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -47,4 +47,4 @@ def test_cli_version():
 
     assert not result.exception
     assert result.exit_code == 0
-    assert result.output.strip() == 'Version: 0.3.5'
+    assert result.output.strip() == 'Version: 0.3.6'