Repository: alan-turing-institute/repro-catalogue

Issue:
Versioning
Implement semantic versioning to give every commit to master a unique release number. I usually like to have this maintained in `setup.py` (numpy is a good example). Any commits on a development branch have additional versioning information.

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..dd5cb91
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,10 @@
+language: python
+python:
+  - "3.6"
+  - "3.7"
+install:
+  - pip install -r requirements.txt
+  - pip install -r requirements-dev.txt
+  - pip install .
+script:
+  - pytest
diff --git a/catalogue/catalogue.py b/catalogue/catalogue.py
index 9817397..abca5b2 100644
--- a/catalogue/catalogue.py
+++ b/catalogue/catalogue.py
@@ -169,6 +169,7 @@ def hash_output(output_data):
     else:
         raise AssertionError("Provided input {} is not a file or directory".format(output_data))
 
+
 def hash_code(repo_path):
     """
     Get commit digest for current HEAD commit
@@ -231,9 +232,9 @@ def construct_dict(timestamp, args):
     return results
 
 
-def store_hash(hash_dict, timestamp, store):
+def store_hash(hash_dict, timestamp, store, ext="json"):
     """
-    Save hash information to <timestamp.json> file.
+    Save hash information to <timestamp.ext> file.
 
     Parameters
     ----------
@@ -243,17 +244,17 @@ def store_hash(hash_dict, timestamp, store):
         timestamp (will be used as name of file)
     store: str
         directory where to store the file
+    ext: str
+        the extension of the file to store the hash info in, default is "json"
 
     Returns
     -------
     None
     """
-    assert isinstance(timestamp, str)
-    assert len(timestamp) == 15, "bad format for timestamp"
 
-    if not os.path.exists(store):
-        os.makedirs(store)
-    with open(os.path.join(store, "{}.json".format(timestamp)),"w") as f:
+    os.makedirs(store, exist_ok=True)
+    
+    with open(os.path.join(store, "{}.{}".format(timestamp, ext)),"w") as f:
         json.dump(hash_dict, f)
 
 
diff --git a/catalogue/compare.py b/catalogue/compare.py
index e9752fd..a9bc746 100644
--- a/catalogue/compare.py
+++ b/catalogue/compare.py
@@ -1,13 +1,14 @@
 import os
 from . import catalogue as ct
-from .utils import create_timestamp, CATALOGUE_DIR
+from .utils import create_timestamp
 
 def compare(args):
     """
     Compares two hash files
 
-    Compares two hash files, given as input arguments to the command line tool. Prints results
-    on the command line.
+    Compares two hash files, given as input arguments to the command line tool.
+    If only one file is given that input is compared to the current state.
+    Prints results on the command line.
     """
 
     assert len(args.hashes) == 1 or len(args.hashes) == 2, "compare can only accept 1 or 2 hash files"
@@ -15,18 +16,19 @@ def compare(args):
     if args.csv is None:
         hash_dict_1 = ct.load_hash(args.hashes[0])
     else:
-        hash_dict_1 = ct.load_csv(os.path.join(CATALOGUE_DIR, args.csv), args.hashes[0])
+        hash_dict_1 = ct.load_csv(os.path.join(args.catalogue_results, args.csv), args.hashes[0])
 
     if len(args.hashes) == 2:
         if args.csv is None:
             hash_dict_2 = ct.load_hash(args.hashes[1])
         else:
-            hash_dict_2 = ct.load_csv(os.path.join(CATALOGUE_DIR, args.csv), args.hashes[1])
+            hash_dict_2 = ct.load_csv(os.path.join(args.catalogue_results, args.csv), args.hashes[1])
     else:
         hash_dict_2 = ct.construct_dict(create_timestamp(), args)
 
     print_comparison(compare_hashes(hash_dict_1, hash_dict_2))
 
+
 def compare_hashes(hash_dict_1, hash_dict_2):
     """
     Compare two hash dictionaries
@@ -97,6 +99,7 @@ def compare_hashes(hash_dict_1, hash_dict_2):
 
     return { "matches" : matches, "differs" : differs, "failures" : failures }
 
+
 def print_comparison(compare_dict):
     """
     Print a nicely formatted summary of hash comparisons
@@ -112,13 +115,15 @@ def print_comparison(compare_dict):
         raise KeyError("comparison dictionary input to print_comparison is missing a value")
 
     print("\n".join([
-            "results differ in {} places:".format(len(differs)),
+            "NOTE we expect the timestamp hashes to differ.",
+            "\nhashes differ in {} places:".format(len(differs)),
             "===========================",
             *differs,
-            "results match in {} places:".format(len(matches)),
+            "\nhashes match in {} places:".format(len(matches)),
             "==========================",
             *matches,
-            "results could not be compared in {} places:".format(len(failures)),
+            "\nhashes could not be compared in {} places:".format(len(failures)),
             "==========================================",
-            *failures
+            *failures,
+            ""
             ]))
diff --git a/catalogue/engage.py b/catalogue/engage.py
index 050efc0..53d4e97 100644
--- a/catalogue/engage.py
+++ b/catalogue/engage.py
@@ -6,9 +6,8 @@ from git import InvalidGitRepositoryError
 
 from . import catalogue as ct
 from .compare import compare_hashes, print_comparison
-from .utils import create_timestamp, check_paths_exists, CATALOGUE_DIR
+from .utils import create_timestamp, check_paths_exists
 
-CATALOGUE_LOCK_PATH = os.path.join(CATALOGUE_DIR, ".lock")
 
 def git_query(repo_path, commit_changes=False):
     """
@@ -69,31 +68,56 @@ def git_query(repo_path, commit_changes=False):
 
 
 def engage(args):
+    """
+    The `catalogue engage` command.
+
+    The engage command is used prior to running an analysis. The `args` contain
+    paths to `input_data` and `code` (which must be a git repo).
+
+    The engage command:
+        - does a `git_query()` check of the `code` repo
+        - gets hashes for the input_data and code (from `construct_dict()`)
+        - saves the hashes to a `.lock` file
+
+    Once engaged (a `.lock` file exists), the command cannot be run again until
+    `disengage` has been run.
+    """
     assert check_paths_exists(args), 'Not all provided filepaths exist.'
 
     if git_query(args.code, True):
         try:
-            assert not os.path.exists(CATALOGUE_LOCK_PATH)
+            assert not os.path.exists(os.path.join(args.catalogue_results, ".lock"))
         except AssertionError:
             print("Already engaged (.lock file exists). To disengage run 'catalogue disengage...'")
             print("See 'catalogue disengage --help' for details")
         else:
-            if not os.path.exists(CATALOGUE_DIR):
-                os.makedirs(CATALOGUE_DIR)
-
             hash_dict = ct.construct_dict(create_timestamp(), args)
-            with open(CATALOGUE_LOCK_PATH, "w") as f:
-                json.dump(hash_dict, f)
+            ct.store_hash(hash_dict, "", args.catalogue_results, ext="lock")
             print("'catalogue engage' succeeded. Proceed with analysis")
 
 
 def disengage(args):
+    """
+    The `catalogue disengage` command.
+
+    The disengage command is run just after finishing an analysis. It cannot be run
+    unless `engage` was run first.
+
+    The `args` contain paths to `input_data`, `code` (must be a git repo) and `output_data`.
+
+    The disengage command:
+        - reads hashes stored in the `.lock` file created during `engage`
+        - gets hashes for the `input_data`, `code` and `output_data` (from `construct_dict()`)
+        - compares the two sets of hashes (if the same, saves the hashes)
+        - prints the results of the comparison
+    """
     assert check_paths_exists(args), 'Not all provided filepaths exist.'
 
     timestamp = create_timestamp()
     try:
-        lock_dict = ct.load_hash(CATALOGUE_LOCK_PATH)
-        os.remove(CATALOGUE_LOCK_PATH)
+        LOCK_FILE_PATH = os.path.join(args.catalogue_results, ".lock")
+        lock_dict = ct.load_hash(LOCK_FILE_PATH)
+        os.remove(LOCK_FILE_PATH)
     except FileNotFoundError:
         print("Not currently engaged (could not find .lock file). To engage run 'catalogue engage...'")
         print("See 'catalogue engage --help' for details")
@@ -106,7 +130,7 @@ def disengage(args):
             # add engage timestamp to hash_dict
             hash_dict["timestamp"].update({"engage": lock_dict["timestamp"]["engage"]})
             if args.csv is None:
-                ct.store_hash(hash_dict, timestamp, CATALOGUE_DIR)
+                ct.store_hash(hash_dict, timestamp, args.catalogue_results)
             else:
-                ct.save_csv(hash_dict, timestamp, os.path.join(CATALOGUE_DIR, args.csv))
+                ct.save_csv(hash_dict, timestamp, os.path.join(args.catalogue_results, args.csv))
         print_comparison(compare)
diff --git a/catalogue/parser.py b/catalogue/parser.py
index bfa2224..913ff73 100644
--- a/catalogue/parser.py
+++ b/catalogue/parser.py
@@ -11,7 +11,7 @@ def main():
 
     This is the main function that is called when running the tool. The function parses
     the arguments supplied and calls the appropriate function (`engage`, `disengage`,
-    `checkhashes`, or `compare`). The details of each of these functions is described in
+    or `compare`). The details of each of these functions is described in
     the appropriate docstrings.
 
     engage
@@ -70,6 +70,15 @@ def main():
                              " that is a git repository. Default is the current working directory."),
         default='.')
 
+    common_parser.add_argument(
+        '--catalogue_results',
+        type=str,
+        metavar='catalogue_results',
+        help=textwrap.dedent("This argument should be the path (full or relative) to the directory where any" +
+                            " files created by catalogue should be stored. Default is catalogue_results."),
+        default='catalogue_results'
+    )
+
     output_parser = argparse.ArgumentParser(add_help=False)
     output_parser.add_argument(
         '--output_data',
diff --git a/catalogue/utils.py b/catalogue/utils.py
index 342328f..792befd 100644
--- a/catalogue/utils.py
+++ b/catalogue/utils.py
@@ -3,7 +3,6 @@ import os
 
 from datetime import datetime
 
-CATALOGUE_DIR = "catalogue_results"
 
 def create_timestamp():
     return datetime.now().strftime("%Y%m%d-%H%M%S")
@@ -22,6 +21,7 @@ def check_paths_exists(args):
     ---------
     Boolean indicating if all filepaths exist.
     """
-    paths = [value for key, value in vars(args).items() if key not in ["command", "func", "csv"]]
+    paths = [value for key, value in vars(args).items()
+            if key not in ["command", "func", "csv", "catalogue_results"]]
     path_checks = [os.path.exists(path) for path in paths]
     return all(path_checks)
diff --git a/setup.py b/setup.py
index 4959d08..41e756a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,31 @@
 from setuptools import setup, find_packages
 
+# version information
+MAJOR = 1
+MINOR = 0
+MICRO = 0
+PRERELEASE = 0
+ISRELEASED = True
+version = "{}.{}.{}".format(MAJOR, MINOR, MICRO)
+
+if not ISRELEASED:
+    version += ".dev{}".format(PRERELEASE)
+
+# write version information to file
+
+def write_version_file(version):
+    "writes version file that is read when importing version number"
+    version_file = """'''
+Version file automatically created by setup.py file
+'''
+version = '{}'
+    """.format(version)
+
+    with open("catalogue/version.py", "w") as fh:
+        fh.write(version_file)
+
+write_version_file(version)
+
 # Source dependencies from requirements.txt file.
 try:
     with open("requirements.txt", "r") as f:
@@ -9,17 +35,17 @@ except FileNotFoundError:
     install_packages = []
 
 setup(
-    name="catalogue",
-    version="",
+    name="repro-catalogue",
+    version=version,
     install_requires=install_packages,
     include_package_data=True,
-    python_requires=">=3.7",
-    author="",
+    python_requires=">=3.6",
+    author='The Alan Turing Institute Research Engineering Group',
     author_email="",
-    url="",
+    url="https://github.com/alan-turing-institute/repro-catalogue",
     # this should be a whitespace separated string of keywords, not a list
     keywords="cli-tool management version-control hashing",
-    description="",
+    description="Tool for reproducible analyses",
     long_description=open("./README.md", "r").read(),
     long_description_content_type="text/markdown",
     license="MIT",


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 42268ee..2e6d5ab 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -33,7 +33,7 @@ def fixture4():
 @pytest.fixture
 def empty_hash():
     """
-    Hash produced if no input is provided. 
+    Hash produced if no input is provided.
     """
     return hashlib.sha512().hexdigest()
 
@@ -64,6 +64,7 @@ def test_args(git_repo):
     args = argparse.Namespace(
         command = "engage",
         input_data = os.path.join(git_repo, "data"),
+        catalogue_results = "catalogue_results",
         code = git_repo,
         csv = None
     )
diff --git a/tests/test_catalogue.py b/tests/test_catalogue.py
index 9b72b79..199baf1 100644
--- a/tests/test_catalogue.py
+++ b/tests/test_catalogue.py
@@ -215,11 +215,15 @@ def test_store_hash(tmpdir):
     ct.store_hash(hash_dict, timestamp, tmpdir.strpath)
     assert file.read() == '{"hello": "world"}'
 
-    # invalid timestamp
-    with pytest.raises(AssertionError):
-        ct.store_hash(hash_dict, 123456789, tmpdir.strpath)
-    with pytest.raises(AssertionError):
-        ct.store_hash(hash_dict, "20200430", tmpdir.strpath)
+    # save to file in a new subdirectory
+    new_file = tmpdir.join("results", '{}.json'.format(timestamp))
+    ct.store_hash(hash_dict, timestamp, os.path.join(tmpdir.strpath, "results"))
+    assert new_file.read() == '{"hello": "world"}'
+
+    # save to .lock file
+    file = tmpdir.join('.lock')
+    ct.store_hash(hash_dict, "", tmpdir.strpath, ext="lock")
+    assert file.read() == '{"hello": "world"}'
 
     # not all inputs provided
     with pytest.raises(TypeError):
diff --git a/tests/test_compare.py b/tests/test_compare.py
index f1db0c9..1b00b7b 100644
--- a/tests/test_compare.py
+++ b/tests/test_compare.py
@@ -16,6 +16,7 @@ def test_compare_json(fixture1, fixture2, fixtures_dir, capsys, git_repo):
         command = "compare",
         input_data = "data",
         output_data = "results",
+        catalogue_results = "catalogue_results",
         code = git_repo,
         csv = None
     )
@@ -68,6 +69,7 @@ def test_compare_csv(fixture4, fixtures_dir, git_repo, workspace, capsys):
         command = "compare",
         input_data = "data",
         output_data = "results",
+        catalogue_results = "catalogue_results",
         code = git_repo,
         csv = fixture4
     )