Repository: emsig/libdlf (language: python)

Issue:
ToDo for Python package
- [ ] Replace pure txt files by binary npz files (check if load is actually faster)
- [ ] Replace licence of package to BSD-3 (license of filter remains CC-BY-4.0)


Discussion / hints:

Fix patch:
diff --git a/packages/create_python.py b/packages/create_python.py
index 6aba90e..6604529 100644
--- a/packages/create_python.py
+++ b/packages/create_python.py
@@ -1,7 +1,9 @@
+import os
 import json
 import shutil
 import pathlib
 import subprocess
+import numpy as np
 from os.path import abspath
 
 # Get git version
@@ -14,15 +16,19 @@
     version = version[0][1:]
 
 # Create package directory
-pathlib.Path(abspath("python/libdlf")).mkdir(parents=True, exist_ok=True)
+path_libdlf = abspath("python/libdlf")
+path_lib = os.path.join(path_libdlf, 'lib')
+pathlib.Path(path_libdlf).mkdir(parents=True, exist_ok=True)
 
 # Copy library to python package
-shutil.copytree(abspath('../lib'), abspath('python/libdlf/lib'),
-                dirs_exist_ok=True)
+shutil.copytree(abspath('../lib'), path_lib, dirs_exist_ok=True)
 
-# Copy README and LICENSE
+# Copy README
 shutil.copyfile('../README.md', 'python/README.md')
-shutil.copyfile('../LICENSE', 'python/LICENSE')
+
+# Copy LICENSEs
+shutil.copyfile('../LICENSE', 'python/libdlf/lib/LICENSE')
+shutil.copyfile('LICENSE', 'python/LICENSE')
 
 # Create setup.py
 setup = f"""# -*- coding: utf-8 -*-
@@ -41,7 +47,7 @@
     author="The emsig community",
     author_email="info@emsig.xyz",
     url="https://github.com/emsig/libdlf",
-    license="CC-BY-4.0",
+    license="BSD-3-Clause",
     packages=["libdlf"],
     include_package_data=True,
     install_requires=["numpy"],
@@ -57,19 +63,20 @@
 
 # Create MANIFEST.in
 with open(abspath("python/MANIFEST.in"), "w") as fm:
-    fm.write("include libdlf/lib/filters.json\n")
-    fm.write("include libdlf/lib/*/*.txt\n")
+    fm.write("include libdlf/lib/LICENSE\n")
+    fm.write("include libdlf/lib/*/*.npz\n")
     fm.write("include LICENSE\n")
     fm.write("exclude MANIFEST.in\n")
     fm.write("exclude setup.cfg\n")
 
+
 # Read json
-with open(abspath('python/libdlf/lib/filters.json')) as fj:
+with open(os.path.join(path_lib, 'filters.json')) as fj:
     lib = json.load(fj)
 
 
 # Create init file
-with open(abspath('python/libdlf/__init__.py'), 'w') as fi:
+with open(os.path.join(path_libdlf, '__init__.py'), 'w') as fi:
 
     # Loop over transforms and add them
     for transform, flist in lib.items():
@@ -82,7 +89,7 @@
 for transform, flist in lib.items():
 
     # Create transform file and loop over filters
-    with open(abspath(f"python/libdlf/{transform}.py"), "w") as ft:
+    with open(os.path.join(path_libdlf, f"{transform}.py"), "w") as ft:
 
         # Imports
         ft.write("import os\n")
@@ -94,10 +101,7 @@
         ft.write("',\n]\n\n")
 
         # Path of the library
-        ft.write("_LIBPATH = os.path.abspath(os.path.dirname(__file__))\n")
-
-        # Cache for the filters
-        ft.write("_CACHE = {}\n\n\n")
+        ft.write("_LIBPATH = os.path.abspath(os.path.dirname(__file__))\n\n\n")
 
         # Number of filters for this transform
         nr_filt = len(flist)
@@ -108,6 +112,12 @@
             # File path and name of the filter
             fname = f"{filt['file']}"
 
+            # Convert txt to npz
+            txtfile = np.loadtxt(os.path.join(path_libdlf, fname), unpack=True)
+            np.savez_compressed(
+                os.path.join(path_libdlf, fname[:-4]), dlf=txtfile
+            )
+
             # Start the function
             ft.write(f"def {filt['name']}():")
 
@@ -150,16 +160,26 @@
 
             # Write function; we use np.loadtxt to read the files
             ft.write(
-                f"    if '{filt['name']}' not in _CACHE.keys():\n"
-                f"        fname = '{fname}'\n"
-                f"        _CACHE['{filt['name']}'] = np.loadtxt(\n"
-                "            os.path.join(_LIBPATH, fname), unpack=True)\n"
-                f"    return _CACHE['{filt['name']}']\n")
+                f"    if getattr({filt['name']}, 'cache', None) is None:\n"
+                f"        fname = '{fname[:-3]}npz'\n"
+                f"        {filt['name']}.cache = np.load(\n"
+                "            os.path.join(_LIBPATH, fname))['dlf']\n"
+                f"    return {filt['name']}.cache\n")
+
+            # Add values to function
+            ft.write(
+                f"\n\n{filt['name']}.values = {filt['values'].split(',')}\n")
 
             # Empty lines after function (except for last filter)
             if f_i < nr_filt-1:
                 ft.write("\n\n")
 
+            # Remove txt file.
+            os.remove(os.path.join(path_libdlf, fname))
+
+# Remove json file.
+os.remove(os.path.join(path_lib, 'filters.json'))
+
 
 # Write tests
 test_filters = r"""import libdlf


Test patch:
diff --git a/tests/test_json.py b/tests/test_json.py
index c8d1d51..3495ee5 100644
--- a/tests/test_json.py
+++ b/tests/test_json.py
@@ -1,6 +1,5 @@
 import os
 import json
-import pytest
 
 
 class TestJson:
@@ -19,7 +18,6 @@ def test_transforms(self):
         # Assert #dir == #categories
         assert len(self.filters) == len(transforms)
 
-
         keys = ['name', 'author', 'year', 'appendix', 'points', 'values',
                 'file']