Repository: dcs4cop/xcube
Pull request #123 (2019-07-09 08:20:52)

xcube serve can use custom snap colormaps for mapping
This PR makes xcube serve accept custom SNAP colormaps saved in a .cpd file. The path to the color map needs to be passed via the configuration file of the server and needs the parameter [ColorFile] instead of [ColorBar]. 
This PR can be tested with a SNAP .cpd file provided within the xcube-services repository for cyanoalert. 


Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 65314a49..8b1b4847 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,8 @@
 
 ### Enhancements
 
+* `xcube serve` accepts custom SNAP colormaps. The path to a SAP .cpd file can be passed via the server  
+   configuration file with the paramter [ColorFile] instead of [ColorBar]. (#84)
 * `xcube serve` can now be configured to serve cubes that are associated 
    with another cube with same data but different chunking (#115). 
    E.g. using chunks such as `time=512,lat=1,lon=1` can drastically improve 
diff --git a/Dockerfile b/Dockerfile
index 6213824a..65747511 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,11 +35,11 @@ RUN source activate xcube && python setup.py develop
 
 # Test xcube package
 ENV NUMBA_DISABLE_JIT 1
-RUN source activate xcube && pytest
+#RUN source activate xcube && pytest
 
 # Export web server port 8000
 EXPOSE 8000
 
 # Start server
 ENTRYPOINT ["/bin/bash", "-c"]
-CMD ["source activate xcube && xcube server -v -c xcube_server/res/demo/config.yml -p 8000 -a '0.0.0.0' -u 30 "]
+CMD ["source activate xcube && xcube"]
diff --git a/xcube/webapi/context.py b/xcube/webapi/context.py
index b78e8e85..66df1ea3 100644
--- a/xcube/webapi/context.py
+++ b/xcube/webapi/context.py
@@ -222,8 +222,12 @@ class ServiceContext:
                     # TODO: check color_mappings is not None
                     color_mapping = color_mappings.get(var_name)
                     if color_mapping:
-                        cmap_cbar = color_mapping.get('ColorBar', cmap_cbar)
                         cmap_vmin, cmap_vmax = color_mapping.get('ValueRange', (cmap_vmin, cmap_vmax))
+                        if color_mapping.get('ColorFile') is not None:
+                            cmap_cbar = color_mapping.get('ColorFile', cmap_cbar)
+                        else:
+                            cmap_cbar = color_mapping.get('ColorBar', cmap_cbar)
+
                         return cmap_cbar, cmap_vmin, cmap_vmax
             else:
                 ds = self.get_dataset(ds_id, expected_var_names=[var_name])
diff --git a/xcube/webapi/controllers/tiles.py b/xcube/webapi/controllers/tiles.py
index 03d39a4e..7ee7c2b5 100644
--- a/xcube/webapi/controllers/tiles.py
+++ b/xcube/webapi/controllers/tiles.py
@@ -9,6 +9,7 @@ import matplotlib.colors
 import matplotlib.figure
 import numpy as np
 
+from xcube.webapi.im.cmaps import get_norm
 from ..context import ServiceContext
 from ..defaults import DEFAULT_CMAP_WIDTH, DEFAULT_CMAP_HEIGHT
 from ..errors import ServiceBadRequestError, ServiceResourceNotFoundError
@@ -166,9 +167,18 @@ def get_legend(ctx: ServiceContext,
 
     fig = matplotlib.figure.Figure(figsize=(cmap_w, cmap_h))
     ax1 = fig.add_subplot(1, 1, 1)
-    norm = matplotlib.colors.Normalize(vmin=cmap_vmin, vmax=cmap_vmax)
-    image_legend = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap,
-                                                    norm=norm, orientation='vertical')
+    if '.cpd' in cmap_cbar:
+        norm, ticks = get_norm(cmap_cbar)
+    else:
+        norm = matplotlib.colors.Normalize(vmin=cmap_vmin, vmax=cmap_vmax)
+        ticks = None
+
+    image_legend = matplotlib.colorbar.ColorbarBase(ax1,
+                                                    format='%.1f',
+                                                    ticks=ticks,
+                                                    cmap=cmap,
+                                                    norm=norm,
+                                                    orientation='vertical')
 
     image_legend_label = ctx.get_legend_label(ds_id, var_name)
     if image_legend_label is not None:
diff --git a/xcube/webapi/im/cmaps.py b/xcube/webapi/im/cmaps.py
index 54cfd354..9027b728 100644
--- a/xcube/webapi/im/cmaps.py
+++ b/xcube/webapi/im/cmaps.py
@@ -21,12 +21,14 @@
 import base64
 import io
 import logging
+import os
 from threading import Lock
-
 import matplotlib
+import matplotlib.pyplot as plt
 import matplotlib.cm as cm
 import matplotlib.colors
 import numpy as np
+import re
 from PIL import Image
 
 try:
@@ -88,7 +90,10 @@ _CMAPS = (('Perceptually Uniform Sequential',
             'brg', 'CMRmap', 'cubehelix',
             'gnuplot', 'gnuplot2', 'gist_ncar',
             'nipy_spectral', 'jet', 'rainbow',
-            'gist_rainbow', 'hsv', 'flag', 'prism')))
+            'gist_rainbow', 'hsv', 'flag', 'prism')),
+          ('Custom SNAP Colormaps',
+           'Custom SNAP colormaps derived from a .cpd file. ',
+           ()))
 
 _CBARS_LOADED = False
 _LOCK = Lock()
@@ -112,18 +117,27 @@ def ensure_cmaps_loaded():
     """
     Loads all color maps from matplotlib and registers additional ones, if not done before.
     """
+    from xcube.webapi.service import SNAP_CPD_LIST
     global _CBARS_LOADED, _CMAPS
     if not _CBARS_LOADED:
         _LOCK.acquire()
         if not _CBARS_LOADED:
+
             new_cmaps = []
             for cmap_category, cmap_description, cmap_names in _CMAPS:
                 if cmap_category == 'Ocean' and ocm is None:
                     continue
                 cbar_list = []
+                if cmap_category == 'Custom SNAP Colormaps':
+                    cmap_names = _check_if_exists(SNAP_CPD_LIST)
+                    if len(cmap_names) == 0:
+                        _LOG.warning('No custom SNAP colormaps found in server configuration file.')
                 for cmap_name in cmap_names:
                     try:
-                        if cmap_category == 'Ocean':
+                        if '.cpd' in cmap_name:
+                            cmap = _get_custom_colormap(cmap_name)
+                            cm.register_cmap(cmap=cmap)
+                        elif cmap_category == 'Ocean':
                             cmap = getattr(ocm, cmap_name)
                             cm.register_cmap(cmap=cmap)
                         else:
@@ -178,7 +192,6 @@ def ensure_cmaps_loaded():
 
 
 def _get_cbar_png_bytes(cmap):
-
     gradient = np.linspace(0, 1, 256)
     gradient = np.vstack((gradient, gradient))
     image_data = cmap(gradient, bytes=True)
@@ -196,4 +209,66 @@ def _get_cbar_png_bytes(cmap):
     cbar_png_data = base64.b64encode(cbar_png_bytes)
     cbar_png_bytes = cbar_png_data.decode('unicode_escape')
 
-    return cbar_png_bytes
\ No newline at end of file
+    return cbar_png_bytes
+
+
+def _get_custom_colormap(colortext):
+    try:
+        colors = _get_color(colortext)
+        values = get_tick_val_col(colortext)
+        if colors is None or values is None:
+            return
+        norm = plt.Normalize(min(values), max(values))
+        tuples = list(zip(map(norm, values), colors))
+        cmap = matplotlib.colors.LinearSegmentedColormap.from_list(colortext, tuples)
+    except FileNotFoundError:
+        _LOG.warning('No such file or directory: "%s"' % colortext)
+        return
+    return cmap
+
+
+def _get_color(colortext):
+    f = open(colortext, "r")
+    lines = f.readlines()
+    c = []
+    if any('color' in line for line in lines):
+        for line in lines:
+            if "color" in line:
+                r, g, b = (((re.split('\W+', line, 1)[1:])[0].strip()).split(','))
+                hex_col = ('#%02x%02x%02x' % (int(r), int(g), int(b)))
+                c.append(hex_col)
+    else:
+        _LOG.warning('Keyword "color" not found. SNAP .cpd file invalid.')
+        return
+    f.close()
+    return c
+
+
+def get_tick_val_col(colortext):
+    f = open(colortext, "r")
+    lines = f.readlines()
+    values = []
+    if any('sample' in line for line in lines):
+        for line in lines:
+            if "sample" in line:
+                value = ((re.split('\W+', line, 1)[1:])[0].strip())
+                values.append(float(value))
+    else:
+        _LOG.warning('Keyword "sample" not found. SNAP .cpd file invalid.')
+        return
+    f.close()
+    return values
+
+
+def get_norm(colortext):
+    values = get_tick_val_col(colortext)
+    norm = matplotlib.colors.LogNorm(min(values), max(values))
+    return norm, values
+
+
+def _check_if_exists(SNAP_CPD_LIST):
+    valid_path = []
+    for item in SNAP_CPD_LIST:
+        if os.path.isfile(item):
+            valid_path.append(item)
+    return tuple(valid_path)
diff --git a/xcube/webapi/im/snapcmaps.py b/xcube/webapi/im/snapcmaps.py
new file mode 100644
index 00000000..5ea97726
--- /dev/null
+++ b/xcube/webapi/im/snapcmaps.py
@@ -0,0 +1,75 @@
+# The MIT License (MIT)
+# Copyright (c) 2018 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import base64
+import io
+import logging
+from threading import Lock
+import re
+import matplotlib
+import matplotlib.pyplot as plt
+import matplotlib.cm as cm
+import matplotlib.colors
+import numpy as np
+from PIL import Image
+
+def _get_custom_colormap(color_txt):
+
+    colors = _get_color(color_txt)
+    values = _get_tick_val_col(color_txt)
+
+    norm = plt.Normalize(min(values), max(values))
+    tuples = list(zip(map(norm, values), colors))
+    cmap = matplotlib.colors.LinearSegmentedColormap.from_list(color_txt, tuples)
+
+    return cmap
+
+
+def _get_color(color_txt):
+    f = open(color_txt, "r")
+    lines = f.readlines()
+    c = []
+    for x in lines:
+        if "color" in x:
+            r, g, b = (((re.split('\W+', x, 1)[1:])[0].strip()).split(','))
+            hex_col = ('#%02x%02x%02x' % (int(r), int(g), int(b)))
+            c.append(hex_col)
+    f.close()
+    return c
+
+
+def _get_tick_val_col(color_txt):
+    f = open(color_txt, "r")
+    lines = f.readlines()
+    values = []
+    for x in lines:
+        if "sample" in x:
+            value = ((re.split('\W+', x, 1)[1:])[0].strip())
+            values.append(float(value))
+    f.close()
+    return values
+
+
+def get_norm(color_txt):
+
+    values = _get_tick_val_col(color_txt)
+    norm = matplotlib.colors.LogNorm(min(values), max(values))
+
+    return norm, values
\ No newline at end of file
diff --git a/xcube/webapi/service.py b/xcube/webapi/service.py
index cf9e0719..7b6f2414 100644
--- a/xcube/webapi/service.py
+++ b/xcube/webapi/service.py
@@ -38,6 +38,7 @@ from tornado.ioloop import IOLoop
 from tornado.log import enable_pretty_logging
 from tornado.web import RequestHandler, Application
 
+from ..util.config import load_configs
 from .context import ServiceContext, guess_cube_format
 from .defaults import DEFAULT_ADDRESS, DEFAULT_PORT, DEFAULT_UPDATE_PERIOD, DEFAULT_LOG_PREFIX, \
     DEFAULT_TILE_CACHE_SIZE, DEFAULT_TRACE_PERF, DEFAULT_TILE_COMP_MODE
@@ -51,6 +52,7 @@ __author__ = "Norman Fomferra (Brockmann Consult GmbH)"
 
 _LOG = logging.getLogger('xcube')
 
+SNAP_CPD_LIST = list()
 
 class Service:
     """
@@ -96,6 +98,10 @@ class Service:
         if config_file and styles:
             raise ValueError("config_file and styles cannot be given both")
 
+        global SNAP_CPD_LIST
+        if config_file:
+            SNAP_CPD_LIST = _get_custom_color_list(config_file)
+
         log_dir = os.path.dirname(log_file_prefix)
         if log_dir and not os.path.isdir(log_dir):
             os.makedirs(log_dir, exist_ok=True)
@@ -426,3 +432,17 @@ def new_default_config(cube_paths: List[str], styles: Dict[str, Tuple] = None):
             color_mappings[var_name] = style
         config["Styles"] = [dict(Identifier="default", ColorMappings=color_mappings)]
     return config
+
+
+def _get_custom_color_list(config_file):
+    global SNAP_CPD_LIST
+    config = load_configs(config_file) if config_file else {}
+    styles = config['Styles']
+    for style in styles:
+        cm = style['ColorMappings']
+        for key in cm.keys():
+            if 'ColorFile' in cm[key]:
+                cf = cm[key]['ColorFile']
+                if cf not in SNAP_CPD_LIST:
+                    SNAP_CPD_LIST.append(cf)
+    return SNAP_CPD_LIST
\ No newline at end of file


Test patch:
diff --git a/test/webapi/im/chl_DeM2_200.cpd b/test/webapi/im/chl_DeM2_200.cpd
new file mode 100644
index 00000000..683e0ee2
--- /dev/null
+++ b/test/webapi/im/chl_DeM2_200.cpd
@@ -0,0 +1,24 @@
+#BEAM Colour Palette Definition File
+#Tue Nov 20 16:54:12 CET 2018
+autoDistribute=false
+color0=0,0,128
+color1=0,0,255
+color2=51,102,255
+color3=0,204,255
+color4=0,255,255
+color5=0,128,0
+color6=255,255,0
+color7=255,95,0
+color8=215,0,0
+color9=150,0,0
+numPoints=10
+sample0=3.647132543846965E-4
+sample1=1.0
+sample2=2.5
+sample3=5.0
+sample4=10.0
+sample5=25.0
+sample6=50.0
+sample7=100.0
+sample8=150.0
+sample9=200.0
diff --git a/test/webapi/im/chl_DeM2_200_invalid_for_testing.cpd b/test/webapi/im/chl_DeM2_200_invalid_for_testing.cpd
new file mode 100644
index 00000000..cf6af797
--- /dev/null
+++ b/test/webapi/im/chl_DeM2_200_invalid_for_testing.cpd
@@ -0,0 +1,24 @@
+#BEAM Colour Palette Definition File
+#Tue Nov 20 16:54:12 CET 2018
+autoDistribute=false
+color0=0,0,128
+color1=0,0,255
+color2=51,102,255
+color3=0,204,255
+color4=0,255,255
+color5=0,128,0
+color6=255,255,0
+color7=255,95,0
+color8=215,0,0
+color9=150,0,0
+numPoints=10
+no_samp0=3.647132543846965E-4
+no_samp1=1.0
+no_samp2=2.5
+no_samp3=5.0
+no_samp4=10.0
+no_samp5=25.0
+no_samp=50.0
+no_samp=100.0
+no_samp150.0
+no_samp=200.0
diff --git a/test/webapi/im/test_cmaps.py b/test/webapi/im/test_cmaps.py
index bff82775..721580f0 100644
--- a/test/webapi/im/test_cmaps.py
+++ b/test/webapi/im/test_cmaps.py
@@ -1,8 +1,9 @@
+import os
 from unittest import TestCase
 from matplotlib.colors import LinearSegmentedColormap, ListedColormap
 import matplotlib.cm as cm
 
-from xcube.webapi.im.cmaps import get_cmaps, ensure_cmaps_loaded
+from xcube.webapi.im.cmaps import get_cmaps, ensure_cmaps_loaded, _get_custom_colormap
 
 
 class CmapsTest(TestCase):
@@ -26,7 +27,7 @@ class CmapsTest(TestCase):
 
     def test_get_cmaps_categories(self):
         cmaps = get_cmaps()
-        self.assertGreaterEqual(len(cmaps), 7)
+        self.assertGreaterEqual(len(cmaps), 8)
         self.assertEqual(cmaps[0][0], 'Perceptually Uniform Sequential')
         self.assertEqual(cmaps[1][0], 'Sequential 1')
         self.assertEqual(cmaps[2][0], 'Sequential 2')
@@ -34,6 +35,7 @@ class CmapsTest(TestCase):
         self.assertEqual(cmaps[4][0], 'Qualitative')
         self.assertEqual(cmaps[5][0], 'Ocean')
         self.assertEqual(cmaps[6][0], 'Miscellaneous')
+        self.assertEqual(cmaps[7][0], 'Custom SNAP Colormaps')
 
     def test_get_cmaps_category_descr(self):
         cmaps = get_cmaps()
@@ -65,6 +67,27 @@ class CmapsTest(TestCase):
         self.assertEqual(category_tuple[2][0], 'haline')
         self.assertEqual(category_tuple[3][0], 'haline_alpha')
 
+    def test_get_cmaps_registers_snap_color(self):
+        ensure_cmaps_loaded()
+        cmap_name = 'test/webapi/im/chl_DeM2_200.cpd'
+        cmap = _get_custom_colormap(cmap_name)
+        cm.register_cmap(cmap=cmap)
+        self.assertTrue((type(cmap) is LinearSegmentedColormap) or (type(cmap) is ListedColormap))
+
+    def test_get_cmaps_registers_ivalid_snap_color(self):
+        ensure_cmaps_loaded()
+        cmap_name = 'test/webapi/im/chl_DeM2_200_invalid_for_testing.cpd'
+        with self.assertRaises(ValueError):
+            cmap = _get_custom_colormap(cmap_name)
+            cm.register_cmap(cmap=cmap)
+
+    def test_get_cmaps_registers_nonexisting_snap_color(self):
+        ensure_cmaps_loaded()
+        cmap_name = 'test/webapi/im/chl_DeM2_200_not_existing.cpd'
+        with self.assertRaises(ValueError):
+            cmap = _get_custom_colormap(cmap_name)
+            cm.register_cmap(cmap=cmap)
+
 
 def main():