Repository: HEPCloud/decisionengine

Issue:
Add postgresql.sql to distributed decisionengine rpm 
There is a file src/decisionengine/framework/dataspace/datasources/postgresql.sql
which is needed to initialize the Postgres database.  This is not currently packaged with the RPM.
Please add it to the RPM again.

Steve Timm


Comments:

Patch:
diff --git a/package/rpm/install_section b/package/rpm/install_section
index 82c2966f..aa7d0ad1 100644
--- a/package/rpm/install_section
+++ b/package/rpm/install_section
@@ -28,3 +28,7 @@ echo "%dir %attr(0750,decisionengine,decisionengine) %{_sysconfdir}/decisionengi
 mkdir -p %{buildroot}/%{_sysconfdir}/sysconfig
 touch %{buildroot}/%{_sysconfdir}/sysconfig/decisionengine
 echo "%attr(0640,root,decisionengine) %config(noreplace) %{_sysconfdir}/sysconfig/decisionengine" >> INSTALLED_FILES
+
+mkdir -p %{buildroot}/%{_defaultdocdir}/%{name}/datasources/
+install -m 0644 src/decisionengine/framework/dataspace/datasources/postgresql.sql %{buildroot}/%{_defaultdocdir}/%{name}/datasources/postgresql.sql
+echo "%doc %{_defaultdocdir}/%{name}" >> INSTALLED_FILES
diff --git a/src/decisionengine/framework/dataspace/datablock.py b/src/decisionengine/framework/dataspace/datablock.py
index ccfcca83..7d071fd4 100644
--- a/src/decisionengine/framework/dataspace/datablock.py
+++ b/src/decisionengine/framework/dataspace/datablock.py
@@ -445,7 +445,6 @@ class DataBlock:
         Check if the dataproduct for a given key or any key is expired
         """
         self.logger.info('datablock is checking for expired dataproducts')
-        pass
 
     def mark_expired(self, expiration_time):
         """
@@ -453,4 +452,3 @@ class DataBlock:
         and mark it as expired if expiration_time <= current time
         """
         self.logger.info('datablock is marking expired dataproducts')
-        pass
diff --git a/src/decisionengine/framework/dataspace/datasources/postgresql.py b/src/decisionengine/framework/dataspace/datasources/postgresql.py
index 097a72ff..c06e4eb5 100644
--- a/src/decisionengine/framework/dataspace/datasources/postgresql.py
+++ b/src/decisionengine/framework/dataspace/datasources/postgresql.py
@@ -183,7 +183,6 @@ class Postgresql(ds.DataSource):
                 SELECT_TASKMANAGERS += " AND "
             else:
                 SELECT_TASKMANAGERS += " WHERE "
-                have_where = True
             SELECT_TASKMANAGERS += " tm.datestamp <=  '" + end_time + "'"
         try:
             return self._select_dictresult(SELECT_TASKMANAGERS +
@@ -470,10 +469,6 @@ class Postgresql(ds.DataSource):
             except psycopg2.Error:  # pragma: no cover
                 pass
             raise
-        except psycopg2.Error:
-            if db:
-                db.rollback()
-            raise
         finally:
             list([x.close if x else None for x in (cursor, db)])
 
@@ -497,10 +492,6 @@ class Postgresql(ds.DataSource):
             except psycopg2.Error:  # pragma: no cover
                 pass
             raise
-        except psycopg2.Error:
-            if db:
-                db.rollback()
-            raise
         finally:
             list([x.close if x else None for x in (cursor, db)])
 
diff --git a/src/decisionengine/framework/logicengine/Rule.py b/src/decisionengine/framework/logicengine/Rule.py
index 39923625..87282717 100644
--- a/src/decisionengine/framework/logicengine/Rule.py
+++ b/src/decisionengine/framework/logicengine/Rule.py
@@ -23,8 +23,8 @@ class Rule:
         return self.expr.evaluate(evaluated_facts)
 
     def __str__(self):  # pragma: no cover
-        return f"name: {self.name}\n"
-        f"expression: '{self.expr}'\n"
-        f"actions: {self.actions}\n"
-        f"false_actions: {self.false_actions}\n"
-        f"facts: {self.new_facts}"
+        return (f"name: {self.name}\n"
+                f"expression: '{self.expr}'\n"
+                f"actions: {self.actions}\n"
+                f"false_actions: {self.false_actions}\n"
+                f"facts: {self.new_facts}")
diff --git a/src/decisionengine/framework/modules/SourceProxy.py b/src/decisionengine/framework/modules/SourceProxy.py
index 5ed09edb..09030431 100644
--- a/src/decisionengine/framework/modules/SourceProxy.py
+++ b/src/decisionengine/framework/modules/SourceProxy.py
@@ -11,6 +11,7 @@ import decisionengine.framework.dataspace.datablock as datablock
 import decisionengine.framework.dataspace.dataspace as dataspace
 from decisionengine.framework.modules import Source
 from decisionengine.framework.modules.Source import Parameter
+from decisionengine.framework.modules.translate_product_name import translate_all
 
 RETRIES = 10
 RETRY_TO = 60
@@ -27,7 +28,7 @@ class SourceProxy(Source.Source):
             raise RuntimeError(
                 'SourceProxy misconfigured. Must have {} defined'.format(must_have))
         self.source_channel = config['channel_name']
-        self.data_keys = config['Dataproducts']
+        self.data_keys = translate_all(config['Dataproducts'])
         self.retries = config.get('retries', RETRIES)
         self.retry_to = config.get('retry_timeout', RETRY_TO)
         self.logger = logging.getLogger()
@@ -35,7 +36,7 @@ class SourceProxy(Source.Source):
         # Hack - it is possible for a subclass to declare @produces,
         #        in which case, we do not want to override that.
         if not self._produces:
-            self._produces = {k: Any for k in self.data_keys}
+            self._produces = {new_name: Any for new_name in self.data_keys.values()}
 
     def post_create(self, global_config):
         self.dataspace = dataspace.DataSpace(global_config)
@@ -85,18 +86,12 @@ class SourceProxy(Source.Source):
         filled_keys = []
         for _ in range(self.retries):
             if len(filled_keys) != len(self.data_keys):
-                for k in self.data_keys:
-                    if isinstance(k, tuple) or isinstance(k, list):
-                        k_in = k[0]
-                        k_out = k[1]
-                    else:
-                        k_in = k
-                        k_out = k
+                for k_in, k_out in self.data_keys.items():
                     if k_in not in filled_keys:
                         try:
                             rc[k_out] = pd.DataFrame(
                                 self._get_data(data_block, k_in))
-                            filled_keys.append(k)
+                            filled_keys.append(k_in)
                         except KeyError as ke:
                             self.logger.debug("KEYERROR %s", ke)
             if len(filled_keys) == len(self.data_keys):
diff --git a/src/decisionengine/framework/modules/translate_product_name.py b/src/decisionengine/framework/modules/translate_product_name.py
new file mode 100644
index 00000000..480bba16
--- /dev/null
+++ b/src/decisionengine/framework/modules/translate_product_name.py
@@ -0,0 +1,39 @@
+# The functionality provided here supports the translation of one
+# string to another using the syntax 'old -> new', subject to the
+# following constraints:
+#
+#   - The 'old' and 'new' strings may contain the following
+#     characters: a-z, A-Z, 0-9, and _ (underscore).
+#
+#   - If '->' is provided, it must be surrounded by at least one space
+#     on either side.
+#
+# The behavior is the following:
+#
+#   - translate("old") == ("old", None)
+#   - translate("old -> new") == ("old", "new")
+#   - translate_all(["old", "old1 -> new1"]) == {"old": "old", "old1": "new1"}
+
+import re
+
+
+def translate(spec):
+    """Break apart the string 'old -> new' into a tuple ('old', 'new')"""
+    match = re.fullmatch(R'(\w+)(\s+\->\s+(\w+))?', spec)
+    if match is None:
+        raise RuntimeError(f"The specification '{spec}' does not match the supported pattern "
+                           '"old_name[ -> new_name]",\n'
+                           "where the product names can consist of the characters a-z, a-Z, 0-9, "
+                           "and an underscore '_'.\nIf an optional new name is specified, the '->' "
+                           "token must be surrounded by at least\none space on either side.")
+    return match.group(1, 3)
+
+
+def translate_all(specs):
+    result = {}
+    for entry in specs:
+        old, new = translate(entry)
+        if new is None:
+            new = old
+        result[old] = new
+    return result


Test patch:
diff --git a/src/decisionengine/framework/modules/tests/test_translate_product_name.py b/src/decisionengine/framework/modules/tests/test_translate_product_name.py
new file mode 100644
index 00000000..cae42936
--- /dev/null
+++ b/src/decisionengine/framework/modules/tests/test_translate_product_name.py
@@ -0,0 +1,24 @@
+from decisionengine.framework.modules.translate_product_name import translate, translate_all
+
+import pytest
+
+def test_translate_none():
+    with pytest.raises(RuntimeError, match="does not match the supported pattern"):
+        translate("")
+    assert translate("old") == ("old", None)
+
+def test_translate_simple():
+    assert translate("old -> new") == ('old', 'new')
+
+def test_translate_with_underscores():
+    assert translate("old_v0 -> new_v1") == ('old_v0', 'new_v1')
+
+def test_translate_illegal_characters():
+    with pytest.raises(RuntimeError, match="does not match the supported pattern"):
+        translate("old-v0 -> new-v1")
+    with pytest.raises(RuntimeError, match="does not match the supported pattern"):
+        translate("old-<3=->=new-<3")
+
+def test_translate_all():
+    specs = ["old", "old1 -> new1"]
+    assert translate_all(specs) == {"old": "old", "old1": "new1"}