Repository: jupyterhub/chartpress

Issue:
Support images defined as string
Add support for images defined as `<registry>/<repo>:<tag>`.

Possible solutions to keep compatibility with images defined as mapping:

1. check the image value in `values.yaml`
2. if it is mapping use current way of setting the dictionary;
3. else use formatted string `"{registry}/{repo}:{tag}".format(...)`

Please let me know what you think and I will send a PR.

Comments:

Patch:
diff --git a/.travis.yml b/.travis.yml
index a47cf03..998bf51 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,11 +5,12 @@ cache: pip
 install:
   - set -e
   - pip install --upgrade pip
-  - pip install pyflakes .
+  - pip install pyflakes pytest .
 script:
   - chartpress --version
   - chartpress --help
   - pyflakes .
+  - pytest -v ./tests
 
   # This is a workaround to an issue caused by the existence of a docker
   # registrymirror in our CI environment. Without this fix that removes the
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdbd930..f6b591c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,23 @@
 
 ## 0.4
 
+### 0.4.3
+
+- Support `valuesPath` pointing to a single `image:tag` string in
+  addition to a dict with separate `repository` and `tag` keys.
+- Support lists in `valuesPath` by using integer indices,
+  e.g. `section.list.1.image` for the yaml:
+
+  ```yaml
+  section:
+    list:
+      - first: item
+        image: "not set"
+      - second: item
+        image: "image:tag"  #  <--sets this here
+  ```
+
+
 ### 0.4.2
 
 - --long flag to always output build information in image tags and chart version [#57](https://github.com/jupyterhub/chartpress/pull/57) ([@consideRatio](https://github.com/consideRatio))
diff --git a/README.md b/README.md
index 7dbf6a9..6caa976 100644
--- a/README.md
+++ b/README.md
@@ -157,3 +157,19 @@ in your `.travis.yml`:
 git:
   depth: false
 ```
+
+## Development
+
+Testing of this python package can be done using [`pyflakes`](https://github.com/PyCQA/pyflakes) and [`pytest`](https://github.com/pytest-dev/pytest). There is also some additional testing that is only run as part of TravisCI, as declared in [`.travis.yml`](.travis.yml).
+
+```
+# install chartpress locally
+pip install  -e .
+
+# install dev dependencies
+pip install pyflakes pytest
+
+# run tests
+pyflakes .
+pytest -v
+```
diff --git a/chartpress.py b/chartpress.py
index df0fb19..f6847da 100755
--- a/chartpress.py
+++ b/chartpress.py
@@ -10,6 +10,7 @@ from collections.abc import MutableMapping
 from functools import lru_cache, partial
 import os
 import pipes
+import re
 import shutil
 import subprocess
 from tempfile import TemporaryDirectory
@@ -55,29 +56,42 @@ def git_remote(git_repo):
     return 'git@github.com:{0}'.format(git_repo)
 
 
-def last_modified_commit(*paths, **kwargs):
-    """Get the last commit to modify the given paths"""
-    return check_output([
-        'git',
-        'log',
-        '-n', '1',
-        '--pretty=format:%h',
-        '--',
-        *paths
-    ], **kwargs).decode('utf-8').strip()
+def latest_tag_or_mod_commit(*paths, **kwargs):
+    """
+    Get the latest of a) the latest tagged commit, or b) the latest modification
+    commit to provided path.
+    """
+    latest_modification_commit = check_output(
+        [
+            'git', 'log',
+            '--max-count=1',
+            '--pretty=format:%h',
+            '--',
+            *paths,
+        ],
+        **kwargs,
+    ).decode('utf-8').strip()
 
+    git_describe_head = check_output(
+        [
+            'git', 'describe', '--tags', '--long'
+        ],
+        **kwargs,
+    ).decode('utf-8').strip().rsplit("-", maxsplit=2)
+    latest_tagged_commit = git_describe_head[2][1:]
 
-def last_modified_date(*paths, **kwargs):
-    """Return the last modified date (as a string) for the given paths"""
-    return check_output([
-        'git',
-        'log',
-        '-n', '1',
-        '--pretty=format:%cd',
-        '--date=iso',
-        '--',
-        *paths
-    ], **kwargs).decode('utf-8').strip()
+    try:
+        check_call(
+            [
+                'git', 'merge-base', '--is-ancestor', latest_tagged_commit, latest_modification_commit,
+            ],
+            **kwargs,
+        )
+    except subprocess.CalledProcessError:
+        # latest_tagged_commit was newer than latest_modification_commit
+        return latest_tagged_commit
+    else:
+        return latest_modification_commit
 
 
 def render_build_args(image_options, ns):
@@ -179,7 +193,55 @@ def image_needs_building(image):
     return image_needs_pushing(image)
 
 
-def build_images(prefix, images, tag=None, push=False, chart_tag=None, skip_build=False, long=False):
+def _get_identifier(tag, n_commits, commit, long):
+    """
+    Returns a chartpress formatted chart version or image tag (identifier) with
+    a build suffix.
+    
+    This function should provide valid Helm chart versions, which means they
+    need to be valid SemVer 2 version strings. It also needs to return valid
+    image tags, which means they need to not contain `+` signs either.
+
+    Example:
+        tag="0.1.2", n_commits="5", commit="asdf1234", long=True,
+        should return "0.1.2-005.asdf1234".
+    """
+    n_commits = int(n_commits)
+
+    if n_commits > 0 or long:
+        if "-" in tag:
+            # append a pre-release tag, with a . separator
+            # 0.1.2-alpha.1 -> 0.1.2-alpha.1.n.sha
+            return f"{tag}.{n_commits:03d}.{commit}"
+        else:
+            # append a release tag, with a - separator
+            # 0.1.2 -> 0.1.2-n.sha
+            return f"{tag}-{n_commits:03d}.{commit}"
+    else:
+        return f"{tag}"
+
+
+def _strip_identifiers_build_suffix(identifier):
+    """
+    Return a stripped chart version or image tag (identifier) without its build
+    suffix (.005.asdf1234), leaving it to represent a Semver 2 release or
+    pre-release.
+
+    Example:
+        identifier: "0.1.2-005.asdf1234"            returns: "0.1.2"
+        identifier: "0.1.2-alpha.1.005.asdf1234"    returns: "0.1.2-alpha.1"
+    """
+    # split away official SemVer 2 build specifications if used
+    if "+" in identifier:
+        return identifier.split("+", maxsplit=1)[0]
+
+    # split away our custom build specification: something ending in either
+    # . or - followed by three or more digits, a dot, an commit sha of four
+    # or more alphanumeric characters.
+    return re.sub(r'[-\.]\d{3,}\.\w{4,}\Z', "", identifier)
+
+
+def build_images(prefix, images, tag=None, push=False, chart_version=None, skip_build=False, long=False):
     """Build a collection of docker images
 
     Args:
@@ -191,9 +253,9 @@ def build_images(prefix, images, tag=None, push=False, chart_tag=None, skip_buil
         to modify the image's files.
     push (bool):
         Whether to push the resulting images (default: False).
-    chart_tag (str):
-        The latest chart tag, included as a prefix on image tags
-        if `tag` is not specified.
+    chart_version (str):
+        The latest chart version, trimmed from its build suffix, will be included
+        as a prefix on image tags if `tag` is not specified.
     skip_build (bool):
         Whether to skip the actual image build (only updates tags).
     long (bool):
@@ -204,38 +266,35 @@ def build_images(prefix, images, tag=None, push=False, chart_tag=None, skip_buil
 
         Example 1:
         - long=False: 0.9.0
-        - long=True:  0.9.0_000.asdf1234
+        - long=True:  0.9.0-000.asdf1234
 
         Example 2:
-        - long=False: 0.9.0_004.sdfg2345
-        - long=True:  0.9.0_004.sdfg2345
+        - long=False: 0.9.0-004.sdfg2345
+        - long=True:  0.9.0-004.sdfg2345
     """
     value_modifications = {}
     for name, options in images.items():
         image_path = options.get('contextPath', os.path.join('images', name))
         image_tag = tag
+        chart_version = _strip_identifiers_build_suffix(chart_version)
         # include chartpress.yaml itself as it can contain build args and
         # similar that influence the image that would be built
         paths = list(options.get('paths', [])) + [image_path, 'chartpress.yaml']
-        last_image_commit = last_modified_commit(*paths)
-        if tag is None:
-            n_commits = int(check_output(
+        image_commit = latest_tag_or_mod_commit(*paths, echo=False)
+        if image_tag is None:
+            n_commits = check_output(
                 [
                     'git', 'rev-list', '--count',
-                    # Note that the 0.0.1 chart_tag may not exist as it was a
+                    # Note that the 0.0.1 chart_version may not exist as it was a
                     # workaround to handle git histories with no tags in the
-                    # current branch. Also, if the chart_tag is a later git
-                    # reference than the last_image_commit, this command will
-                    # return 0.
-                    f'{chart_tag + ".." if chart_tag != "0.0.1" else ""}{last_image_commit}',
+                    # current branch. Also, if the chart_version is a later git
+                    # reference than the image_commit, this
+                    # command will return 0.
+                    f'{"" if chart_version == "0.0.1" else chart_version + ".."}{image_commit}',
                 ],
                 echo=False,
-            ).decode('utf-8').strip())
-
-            if n_commits > 0 or long:
-                image_tag = f"{chart_tag}_{int(n_commits):03d}-{last_image_commit}"
-            else:
-                image_tag = f"{chart_tag}"
+            ).decode('utf-8').strip()
+            image_tag = _get_identifier(chart_version, n_commits, image_commit, long)
         image_name = prefix + name
         image_spec = '{}:{}'.format(image_name, image_tag)
 
@@ -251,7 +310,7 @@ def build_images(prefix, images, tag=None, push=False, chart_tag=None, skip_buil
             build_args = render_build_args(
                 options,
                 {
-                    'LAST_COMMIT': last_image_commit,
+                    'LAST_COMMIT': image_commit,
                     'TAG': image_tag,
                 },
             )
@@ -277,27 +336,49 @@ def build_values(name, values_mods):
         values = yaml.load(f)
 
     for key, value in values_mods.items():
+        if not isinstance(value, dict) or set(value.keys()) != {'repository', 'tag'}:
+            raise ValueError(f"I only understand image updates with 'repository', 'tag', not: {value!r}")
         parts = key.split('.')
-        mod_obj = values
+        mod_obj = parent = values
         for p in parts:
+            if p.isdigit():
+                # integers are indices in lists
+                p = int(p)
+            parent = mod_obj
             mod_obj = mod_obj[p]
-        print(f"Updating {values_file}: {key}: {value}")
+            last_part = p
 
         if isinstance(mod_obj, MutableMapping):
             keys = IMAGE_REPOSITORY_KEYS & mod_obj.keys()
             if keys:
-                for key in keys:
-                    mod_obj[key] = value['repository']
+                for repo_key in keys:
+                    before = mod_obj.get(repo_key, None)
+                    if before != value['repository']:
+                        print(f"Updating {values_file}: {key}.{repo_key}: {value}")
+                    mod_obj[repo_key] = value['repository']
             else:
                 possible_keys = ' or '.join(IMAGE_REPOSITORY_KEYS)
                 raise KeyError(
                     f'Could not find {possible_keys} in {values_file}:{key}'
                 )
 
+            before = mod_obj.get('tag', None)
+            if before != value['tag']:
+                print(f"Updating {values_file}: {key}.tag: {value}")
             mod_obj['tag'] = value['tag']
+        elif isinstance(mod_obj, str):
+            # scalar image string, not dict with separate repository, tag keys
+            image = "{repository}:{tag}".format(**value)
+            try:
+                before = parent[last_part]
+            except (KeyError, IndexError):
+                before = None
+            if before != image:
+                print(f"Updating {values_file}: {key}: {image}")
+            parent[last_part] = image
         else:
             raise TypeError(
-                f'The key {key} in {values_file} must be a mapping.'
+                f'The key {key} in {values_file} must be a mapping or string, not {type(mod_obj)}.'
             )
 
 
@@ -315,34 +396,43 @@ def build_chart(name, version=None, paths=None, long=False):
 
     Example versions constructed:
         - 0.9.0-alpha.1
-        - 0.9.0-alpha.1+000.asdf1234 (--long)
-        - 0.9.0-alpha.1+005.sdfg2345
-        - 0.9.0-alpha.1+005.sdfg2345 (--long)
+        - 0.9.0-alpha.1.000.asdf1234 (--long)
+        - 0.9.0-alpha.1.005.sdfg2345
+        - 0.9.0-alpha.1.005.sdfg2345 (--long)
+        - 0.9.0
+        - 0.9.0-002.dfgh3456
     """
     chart_file = os.path.join(name, 'Chart.yaml')
     with open(chart_file) as f:
         chart = yaml.load(f)
 
-    last_chart_commit = last_modified_commit(*paths)
-
     if version is None:
+        chart_commit = latest_tag_or_mod_commit(*paths, echo=False)
+
         try:
-            git_describe = check_output(['git', 'describe', '--tags', '--long', last_chart_commit]).decode('utf8').strip()
+            git_describe = check_output(
+                [
+                    'git', 'describe', '--tags', '--long', chart_commit
+                ],
+                echo=False,
+            ).decode('utf8').strip()
             latest_tag_in_branch, n_commits, sha = git_describe.rsplit('-', maxsplit=2)
-
-            n_commits = int(n_commits)
-            if n_commits > 0 or long:
-                version = f"{latest_tag_in_branch}+{n_commits:03d}.{sha}"
-            else:
-                version = f"{latest_tag_in_branch}"
+            # remove "g" prefix output by the git describe command
+            # ref: https://git-scm.com/docs/git-describe#_examples
+            sha = sha[1:]
+            version = _get_identifier(latest_tag_in_branch, n_commits, sha, long)
         except subprocess.CalledProcessError:
             # no tags on branch: fallback to the SemVer 2 compliant version
-            # 0.0.1+<n_comits>.<last_chart_commit>
-            n_commits = int(check_output(
-                ['git', 'rev-list', '--count', last_chart_commit],
+            # 0.0.1-<n_commits>.<chart_commit>
+            latest_tag_in_branch = "0.0.1"
+            n_commits = check_output(
+                [
+                    'git', 'rev-list', '--count', chart_commit
+                ],
                 echo=False,
-            ).decode('utf-8').strip())
-            version = f"0.0.1+{n_commits:03d}.{last_chart_commit}"
+            ).decode('utf-8').strip()
+
+            version = _get_identifier(latest_tag_in_branch, n_commits, chart_commit, long)
 
     chart['version'] = version
 
@@ -510,10 +600,7 @@ def main():
                 images=chart['images'],
                 tag=args.tag if not args.reset else chart.get('resetTag', 'set-by-chartpress'),
                 push=args.push,
-                # chart_tag will act as a image tag prefix, we can get it from
-                # the chart_version by stripping away the build part of the
-                # SemVer 2 compliant chart_version.
-                chart_tag=chart_version.split('+')[0],
+                chart_version=chart_version,
                 skip_build=args.skip_build or args.reset,
                 long=args.long,
             )


Test patch:
diff --git a/tests/test_regexp.py b/tests/test_regexp.py
new file mode 100644
index 0000000..b597655
--- /dev/null
+++ b/tests/test_regexp.py
@@ -0,0 +1,14 @@
+from chartpress import _strip_identifiers_build_suffix
+from chartpress import _get_identifier
+
+def test__strip_identifiers_build_suffix():
+    assert _strip_identifiers_build_suffix(identifier="0.1.2-005.asdf1234") == "0.1.2"
+    assert _strip_identifiers_build_suffix(identifier="0.1.2-alpha.1.005.asdf1234") == "0.1.2-alpha.1"
+
+def test__get_identifier():
+    assert _get_identifier(tag="0.1.2",         n_commits="0", commit="asdf123",  long=True)  == "0.1.2-000.asdf123"
+    assert _get_identifier(tag="0.1.2",         n_commits="0", commit="asdf123",  long=False) == "0.1.2"
+    assert _get_identifier(tag="0.1.2",         n_commits="5", commit="asdf123",  long=False) == "0.1.2-005.asdf123"
+    assert _get_identifier(tag="0.1.2-alpha.1", n_commits="0", commit="asdf1234", long=True)  == "0.1.2-alpha.1.000.asdf1234"
+    assert _get_identifier(tag="0.1.2-alpha.1", n_commits="0", commit="asdf1234", long=False) == "0.1.2-alpha.1"
+    assert _get_identifier(tag="0.1.2-alpha.1", n_commits="5", commit="asdf1234", long=False) == "0.1.2-alpha.1.005.asdf1234"