Repository: containers-tools/cct

Issue:
CCT_MODULE_PATH variable is wrongly set in script modules
this variable and the CCT_MODULE_PATH_NAME should point to directory with image.yaml 

Discussion:

Patch:
diff --git a/cct/cfg.py b/cct/cfg.py
new file mode 100644
index 0000000..87ca19b
--- /dev/null
+++ b/cct/cfg.py
@@ -0,0 +1,2 @@
+dogen = False
+artifacts = []
diff --git a/cct/change_processor.py b/cct/change_processor.py
index d8b7025..a47ee4b 100644
--- a/cct/change_processor.py
+++ b/cct/change_processor.py
@@ -8,6 +8,7 @@ of the MIT license. See the LICENSE file for details.
 
 import logging
 
+from cct import cfg
 from cct.module import ModuleManager, ModuleRunner, Module
 
 logger = logging.getLogger('cct')
@@ -21,10 +22,10 @@ class ChangeProcessor(object):
         self.modules_dir = modules_dir
         self.artifacts_dir = artifacts_dir
 
-    def process(self, fetch_only=False):
+    def process(self):
         logger.debug("processing change %s" % self.config)
         for change in self.config:
-            return self._process_change(change, fetch_only)
+            return self._process_change(change)
 
     def _merge_environment(self, change_env, module_env):
         if change_env is None:
@@ -44,7 +45,9 @@ class ChangeProcessor(object):
             env_dict[variable] = env[variable]
         return env_dict
 
-    def _process_change(self, change_cfg, fetch_only):
+    def _process_change(self, change_cfg):
+        if 'name' not in change_cfg:
+            change_cfg['name'] = ''
         logger.info("processing change %s" % change_cfg['name'])
         change_env = self._create_env_dict(change_cfg.get('environment'))
         if 'description' not in change_cfg:
@@ -58,18 +61,16 @@ class ChangeProcessor(object):
                 override = module['override'] if 'override' in module else False
                 mr.install_module(url, ver, override)
 
-        if fetch_only:
-            artifacts = {}
-            for _, module in mr.modules.items():
-                for _, artifact in module.artifacts.items():
-                    artifacts[artifact.filename] = artifact.hash
-            return artifacts
+        if cfg.dogen:
+            return
 
         steps = []
         for modules in change_cfg['changes']:
             for module_name, operations in modules.items():
                 if module_name not in mr.modules:
-                        raise Exception("Module %s cannot be found" % module_name)
+                    msg = "Module %s cannot be found" % module_name
+                    logger.error(msg)
+                    raise Exception(msg)
                 module = Module(module_name, None, None)
                 module.instance = mr.modules[module_name]
                 module._update_env(change_env)
@@ -82,8 +83,6 @@ class ChangeProcessor(object):
             runner.run()
         except:
             raise
-        finally:
-            runner.print_result_report()
 
 
 class Change(object):
@@ -113,9 +112,3 @@ class ChangeRunner(object):
                 logger.error("module %s failed processing steps" % module.name)
                 self.results.append(module)
                 raise
-
-    def print_result_report(self):
-        for module in self.results:
-            print("Processed module: %s" % module.name)
-            for operation in module.operations:
-                print("  %-30s: %s" % (operation.command, operation.state))
diff --git a/cct/cli/main.py b/cct/cli/main.py
index 223b8b2..35003a5 100755
--- a/cct/cli/main.py
+++ b/cct/cli/main.py
@@ -13,7 +13,8 @@ import os
 import sys
 import urllib2
 
-from cct import setup_logging, version
+
+from cct import setup_logging, version, cfg
 from cct.change_processor import ChangeProcessor
 from cct.module import ModuleManager
 from urlparse import urlparse
@@ -32,7 +33,6 @@ class CCT_CLI(object):
         self.parser = MyParser(description='Container configuration tool')
 
     def setup_arguments(self):
-        self.parser.add_argument('--fetch-only', action="store_true", help="cct will fetch modules and artifacts only")
         self.parser.add_argument('--modules-dir', nargs='?', default="%s/%s" % (os.getcwd(), 'modules'), help='directory from where modules are executed')
         self.parser.add_argument('--artifacts-dir', nargs='?', default="%s/%s" % (os.getcwd(), 'artifacts'), help='directory where artifacts are stored')
         self.parser.add_argument('-v', '--verbose', action="store_true", help='verbose output')
@@ -59,10 +59,7 @@ class CCT_CLI(object):
         stream = open(file, 'r')
         return yaml.load(stream)
 
-    def fetch_artifacts(self, changes, modules_dir, artifacts_dir):
-        return self.process_changes(changes, modules_dir, artifacts_dir, True)
-
-    def process_changes(self, changes, modules_dir, artifacts_dir, fetch_only):
+    def process_changes(self, changes, modules_dir, artifacts_dir):
         for change in changes:
             if change is '':
                 continue
@@ -72,7 +69,7 @@ class CCT_CLI(object):
             else:
                 change = self.process_file(change)
             cp = ChangeProcessor(change, modules_dir, artifacts_dir)
-            return cp.process(fetch_only)
+            return cp.process()
 
     def run(self):
         self.setup_arguments()
@@ -110,7 +107,7 @@ class CCT_CLI(object):
             else:
                 changes += args.changes
             try:
-                self.process_changes(changes, args.modules_dir, args.artifacts_dir, args.fetch_only)
+                self.process_changes(changes, args.modules_dir, args.artifacts_dir)
             except KeyboardInterrupt:
                 pass
             except Exception:
diff --git a/cct/module.py b/cct/module.py
index 2ab2434..5db6002 100644
--- a/cct/module.py
+++ b/cct/module.py
@@ -19,6 +19,7 @@ import subprocess
 import traceback
 import yaml
 
+from cct import cfg
 from cct.errors import CCTError
 from cct.lib.git import clone_repo, get_tag_or_branch
 
@@ -90,7 +91,8 @@ class ModuleManager(object):
                 self.check_module_sh(candidate)
         elif 'script' in language:
             for candidate in filter(lambda f: os.path.isdir(os.path.join(directory, f)), os.listdir(directory)):
-                self.check_module_script(os.path.join(directory, candidate))
+                if candidate != '.git':
+                    self.check_module_script(os.path.join(directory, candidate))
         else:
             pattern = os.path.join(os.path.abspath(directory), '*.py')
             for candidate in glob.glob(pattern):
@@ -124,9 +126,8 @@ class ModuleManager(object):
         self.modules[name].override = self.override
 
     def check_module_script(self, candidate):
-        module_name = "cct.module." + candidate.split('/')[-1]
-        logger.debug("Importing module %s to %s" % (os.path.abspath(candidate), module_name))
-        name = "%s.%s" % (os.path.dirname(candidate).split('/')[-1], module_name.split('.')[-1])
+        name = "%s.%s" % (os.path.dirname(candidate).split('/')[-1], candidate.split('/')[-1])
+        logger.debug("Importing script from %s to %s" % (os.path.abspath(candidate), name))
         self.modules[name] = ScriptModule(name, os.path.dirname(candidate), self.artifacts_dir)
         self.modules[name].version = self.version
         self.modules[name].override = self.override
@@ -174,7 +175,7 @@ class ModuleManager(object):
 
         for method in dir(module):
             if callable(getattr(module, method)):
-                if method[0] in string.ascii_lowercase and method not in ['run', 'setup', 'teardown']:
+                if method[0] in string.ascii_lowercase and method not in ['setup', 'teardown']:
                     print("  %s: %s" % (method, getattr(module, method).__doc__))
 
         if getattr(module, "teardown").__doc__:
@@ -211,11 +212,11 @@ class ModuleRunner(object):
     def run(self):
         self.module.instance.setup()
         for operation in self.module.operations:
-            if operation.command in ['setup', 'run', 'teardown']:
+            if operation.command in ['setup', 'teardown']:
                 continue
             if operation.command == 'user':
                 logger.info("setting uid to %s" % operation.args[0])
-                self.module.uid=operation.args[0]
+                self.module.uid = operation.args[0]
                 continue
             self.module.instance._process_environment(operation)
             try:
@@ -237,6 +238,7 @@ class ModuleRunner(object):
 
 class Module(object):
     artifacts = {}
+    modules_dirs = {}
 
     def __init__(self, name, directory, artifacts_dir, version=None):
         self.name = name
@@ -251,6 +253,7 @@ class Module(object):
         self.uid = os.getuid()
         if not directory:
             return
+        self.modules_dirs[os.path.splitext(name)[0]] = directory
         with open(os.path.join(directory, "module.yaml")) as stream:
             config = yaml.load(stream)
             if 'artifacts' in config:
@@ -297,9 +300,13 @@ class Module(object):
 
     def _get_artifacts(self, artifacts, destination):
         for artifact in artifacts:
-            cct_artifact = CctArtifact(**artifact)
-            cct_artifact.fetch(destination)
-            self.artifacts[cct_artifact.name] = cct_artifact
+            if cfg.dogen:
+                if artifact not in cfg.artifacts:
+                    cfg.artifacts.append(artifact)
+            else:
+                cct_artifact = CctArtifact(**artifact)
+                cct_artifact.fetch(destination)
+                self.artifacts[cct_artifact.name] = cct_artifact
 
     def setup(self):
         pass
@@ -345,13 +352,15 @@ class CctArtifact(object):
     """
     Object representing artifact file for changes
     name - name of the file
-    md5sum - md5sum
+    md5 - hash of artifact
+    sha256 - hash of artifact
+    sha1 - has of artifact
     """
-    def __init__(self, name, chksum, artifact="", hint=""):
+    def __init__(self, name, md5=None, sha1=None, sha256=None, artifact="", hint=""):
         self.name = name
-        self.chksum = chksum
-        self.alg = chksum.split(':')[0]
-        self.hash = chksum.split(':')[1]
+        self.sums = {'sha1': sha1,
+                     'sha256': sha256,
+                     'md5': md5}
         self.artifact = self.replace_variables(artifact) if '$' in artifact else artifact
         self.filename = os.path.basename(artifact)
         self.path = None
@@ -364,16 +373,6 @@ class CctArtifact(object):
         self.path = os.path.join(directory, self.filename)
 
         url = self.artifact
-        if 'CCT_ARTIFACT_CACHE' in os.environ:
-            cache = os.environ['CCT_ARTIFACT_CACHE']
-            logger.info('Using CCT_ARTIFACT_CACHE=%s to fetch artifact' % cache)
-            for var in [v for v in dir(self) if not callable(getattr(self, v))]:
-                if var.startswith('_'):
-                    continue
-                token = '#%s#' % var
-                cache = cache.replace(token, getattr(self, var))
-            url = cache
-
         if self.check_sum():
             logger.info("Using cached artifact for %s" % self.filename)
             return
@@ -394,18 +393,19 @@ class CctArtifact(object):
             if self.hint:
                 raise CCTError('hash is not correct for artifact: "%s". %s' % (self.path, self.hint))
             else:
-                raise CCTError("artifact from %s doesn't match required chksum %s" % (url, self.chksum))
+                raise CCTError("artifact from %s doesn't match required chksum" % url)
 
     def check_sum(self):
         if not os.path.exists(self.path):
             return False
-        hash = getattr(hashlib, self.chksum[:self.chksum.index(':')])()
-        with open(self.path, "rb") as f:
-            for block in iter(lambda: f.read(65536), b""):
-                hash.update(block)
-        if self.chksum[self.chksum.index(':') + 1:] == hash.hexdigest():
-            return True
-        return False
+        for alg, sum in self.sums.items():
+            hash = getattr(hashlib, alg)()
+            with open(self.path, "rb") as f:
+                for block in iter(lambda: f.read(65536), b""):
+                    hash.update(block)
+            if sum is not None and sum != hash.hexdigest():
+                return False
+        return True
 
     def replace_variables(self, string):
         var_regex = re.compile('\$\{.*\}')
@@ -433,7 +433,6 @@ class Operation(object):
 
 
 class ScriptModule(Module):
-    modules_dirs = {}
 
     def __init__(self, name, directory, artifacts_dir):
         Module.__init__(self, name, directory, artifacts_dir)
@@ -442,7 +441,6 @@ class ScriptModule(Module):
         self.names = {}
         for script in filter(lambda f: os.path.isfile(f), glob.glob(pattern)):
             self.names[os.path.basename(script).replace('-', '_').replace('.', '_')] = script
-            self.modules_dirs[os.path.splitext(name)[0]] = os.path.dirname(script)
 
     def __getattr__(self, name):
         def wrapper(*args, **kwargs):
@@ -456,7 +454,7 @@ class ScriptModule(Module):
         cmd = 'bash -x %s %s' % (script, " ".join(args))
         try:
             env = dict(os.environ)
-            mod_dir = os.path.dirname(script)
+            mod_dir = os.path.dirname(os.path.dirname(script))
             env['CCT_MODULE_PATH'] = mod_dir
             logger.info('Created CCT_MODULE_PATH environment variable for module %s' % mod_dir)
 
diff --git a/os.yaml b/os.yaml
deleted file mode 100644
index ad7d120..0000000
--- a/os.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-- name: openshift.eap.setup
-  description: execute steps for eap
-  environment:
-    - foo: gfoo
-    - bar: gbar
-  changes:
-    - openshift:
-        - setup: /tmp/install/
-        - setup_jolokia: version=1.3.2.redhat-1
-        - add_custom_launch_script:
-        - setup_liveness_probe:
-        - setup_readiness_probe:
-        - setup_logging:
-        - add_custom_configfile:
-        - add_amq_rar:
-        - inject_maven_settings:
-        - link_java_db_drivers:
-        - remove_console:
-        - setup_jdk: version=1.8.0-openjdk
-        - setup_s2i:
-        - add_openshift_ping: version=1.0.0.Beta7-redhat-1
-        - setup_deployment_dir:
diff --git a/samples/dummy.yaml b/samples/dummy.yaml
index 996f9c5..6d4d034 100644
--- a/samples/dummy.yaml
+++ b/samples/dummy.yaml
@@ -1,8 +1,9 @@
 - name: samples.jboss_cli
   description: execute samples jboss_cli steps
+  modules:
+    - url: https://github.com/containers-tools/base
   changes:
     - base.Dummy:
       - dump: data-source remove --name=ExampleDS
     - base.Shell:
-      - url: https://github.com/containers-tools/base
       - shell: ls -l
diff --git a/samples/eap_xml.yaml b/samples/eap_xml.yaml
deleted file mode 100644
index 22af030..0000000
--- a/samples/eap_xml.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-- name: samples.jboss_cli
-  description: execute samples jboss_cli steps
-  changes:
-    - base.File:
-        - copy: standalone.xml standalone-openshift.xml
-        - copy: source=standalone.xml destination=standalone2.xml
-    - base.XML:
-      - setup: standalone-openshift.xml
-      - insert: >
-          xpath=/ns:server/ns:extensions
-          snippet='<extension module="org.jboss.ctt.dummy" />'
diff --git a/samples/git_standalone_xml.yaml b/samples/git_standalone_xml.yaml
deleted file mode 100644
index 8925fda..0000000
--- a/samples/git_standalone_xml.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-- name: samples.jboss.config
-  description: clone repository with standalone.xml and use it for wildky
-  changes:
-    - git:
-      - checkout: https://github.com/l-d-j/cct-config-files path=cct-config-files
-    - file:
-      - move: /opt/jboss/wildfly/standalone/configuration/standalone.xml /opt/jboss/wildfly/standalone/configuration/standalone.xml.bkp
-      - move: cct-config-files/wildfly/8/standalone.xml /opt/jboss/wildfly/standalone/configuration/standalone.xml
\ No newline at end of file
diff --git a/samples/jboss_cli.yaml b/samples/jboss_cli.yaml
deleted file mode 100644
index 1dfbbf7..0000000
--- a/samples/jboss_cli.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-- name: samples.jboss_cli
-  description: execute samples jboss_cli steps
-  changes:
-    - jboss.Cli:
-      # set jboss home - uses $JBOSS_HOME by default
-      - setup:
-      - run_cli: data-source remove --name=ExampleDS
diff --git a/samples/wildfly_log_debug.yaml b/samples/wildfly_log_debug.yaml
deleted file mode 100644
index 3296a82..0000000
--- a/samples/wildfly_log_debug.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-- name: samples.xmledit
-  description: change console logging to debug
-  changes:
-    - base.XML:
-      - setup: /opt/jboss/wildfly/standalone/configuration/standalone.xml namespaces=ns:urn:jboss:domain:3.0|lns:urn:jboss:domain:logging:3.0
-      - replace_attribute: >
-          xpath=/ns:server/ns:profile/lns:subsystem/lns:console-handler/lns:level
-          attribute=name
-          value=DEBUG


Test patch:
diff --git a/tests/test_change_processor.py b/tests/test_change_processor.py
index d9e9a85..54a4fc4 100755
--- a/tests/test_change_processor.py
+++ b/tests/test_change_processor.py
@@ -50,7 +50,7 @@ class TestModule(unittest.TestCase):
         }]
         destination = tempfile.mkdtemp()
         changeProcessor = ChangeProcessor(config, destination, '/tmp/')
-        changeProcessor.process(fetch_only=False)
+        changeProcessor.process()
         shutil.rmtree(destination)
 
 if __name__ == '__main__':
diff --git a/tests/test_unit_module.py b/tests/test_unit_module.py
index b40659b..de1fb30 100644
--- a/tests/test_unit_module.py
+++ b/tests/test_unit_module.py
@@ -45,7 +45,7 @@ class TestModules(unittest.TestCase):
             "artifacts": [
                 {
                     "artifact": url,
-                    "chksum": chksum,
+                    "md5": chksum,
                     "name": "cct",
                 }
             ]
@@ -56,18 +56,18 @@ class TestModules(unittest.TestCase):
 
     def test_artifacts_fetching(self):
         url = "https://github.com/containers-tools/cct/archive/v0.2.0.zip"
-        chksum = "md5:607468ba87490a2b8daa4b6a73168620"
+        chksum = "607468ba87490a2b8daa4b6a73168620"
         self.get_artifact(url, chksum)
 
     def test_artifacts_fetching_wrong_url(self):
         url = "https://github.com/containers-tools/cct/archive/0.0.1.zip33"
-        chksum = "md5:must_be_wrong_too"
+        chksum = "must_be_wrong_too"
         with self.assertRaises(CCTError):
             self.get_artifact(url, chksum)
 
     def test_artifacts_fetching_wrong_chksum(self):
         url = "https://github.com/containers-tools/cct/archive/0.2.0.zip"
-        chksum = "md5:foo"
+        chksum = "foo"
         with self.assertRaises(CCTError):
             self.get_artifact(url, chksum)