Repository: ome/ome-model

Issue:
OriginalMetadata from 2003-FC lost in transform process
Issue is a follow up to image.sc thread https://forum.image.sc/t/bio-formats-returns-all-metadata-except-ome-meta-data/52295/2

Using the sample file provided in the thread and correcting validation errors shows that the original metadata (stored in 2003-FC  spec) gets lost when going through the transform update process. The particular transform the issue occurs with is 
`2008-09-to-2009-09.xsl`

The data that gets lost is stored using the `OriginalMetadata` tag such as below (adding the CA namespace didn't resolve the issue):
```
<CustomAttributes>
<OriginalMetadata ID="OriginalMetadata:3" Name="Acquisition Parameters/Channel 2" Value="Fluorescence"/>
</CustomAttributes>
```

In the transform this should be handled by the following, transforming the metadata to an XMLAnnotation:
```
  <xsl:template match="CA:*">
    <xsl:element name="{name()}" namespace="{$newCANS}">
      <xsl:apply-templates select="@*|node()"/>
    </xsl:element>
  </xsl:template>

  <!-- Transform the CustomAttributes into XMLAnnotation -->
  <xsl:template match="CA:CustomAttributes">
    <xsl:if test="count(@*|node()) &gt; 0">
      <xsl:element name="StructuredAnnotations" namespace="{$newSANS}">
        <xsl:element name="XMLAnnotation" namespace="{$newSANS}">
          <xsl:attribute name="ID">Annotation:1</xsl:attribute>
          <xsl:element name="Value" namespace="{$newSANS}">
            <xsl:apply-templates select="@*|node()"/>
          </xsl:element>
        </xsl:element>
      </xsl:element>
    </xsl:if>
  </xsl:template>
```


Comments:

Patch:
diff --git a/.appveyor.yml b/.appveyor.yml
deleted file mode 100644
index 2f7748a0..00000000
--- a/.appveyor.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-environment:
-  AV_PROJECTS: 'c:\projects'
-  AV_OME_M2: 'c:\projects\m2'
-  AV_OME_PYTHON: 'c:\projects\python'
-  AV_OME_SOURCE: 'c:\projects\source'
-
-# Note that only Oracle JDK is provided.
-  matrix:
-    - java: 10
-    - java: 1.8
-    - java: 1.7
-
-cache:
-  - '%AV_OME_M2% -> appveyor.yml'
-  - '%AV_OME_PYTHON% -> appveyor.yml'
-
-os: 'Visual Studio 2015'
-clone_folder: '%AV_OME_SOURCE%'
-clone_depth: 5
-platform: x64
-
-init:
-  - git config --global core.autocrlf input
-  - refreshenv
-  - 'if [%java%] == [1.7] set "JAVA_HOME=C:\Program Files\Java\jdk1.7.0"'
-  - 'if [%java%] == [1.8] set "JAVA_HOME=C:\Program Files\Java\jdk1.8.0"'
-  - 'if [%java%] == [10] set "JAVA_HOME=C:\Program Files\Java\jdk10"'
-  - PATH=%JAVA_HOME%\bin;%PATH%
-  - 'set "MAVEN_OPTS=-Dmaven.repo.local=%AV_OME_M2%"'
-  - 'if [%java%] == [1.7] set "MAVEN_OPTS=%MAVEN_OPTS% -Dhttps.protocols=TLSv1.2"'
-
-build_script:
-  # Cached venv is available from this point.
-  - 'if NOT EXIST "%AV_OME_PYTHON%\" set AV_OME_CREATE_VENV=true'
-  - 'if [%AV_OME_CREATE_VENV%] == [true] C:\Python27-x64\python -m pip install virtualenv'
-  - 'if [%AV_OME_CREATE_VENV%] == [true] C:\Python27-x64\python -m virtualenv %AV_OME_PYTHON%'
-  - PATH=%AV_OME_PYTHON%;%AV_OME_PYTHON%\scripts;%PATH%
-  - 'if [%AV_OME_CREATE_VENV%] == [true] python -m pip install sphinx'
-  - mvn install
diff --git a/ome-xml/src/main/java/ome/xml/meta/MetadataConverter.java b/ome-xml/src/main/java/ome/xml/meta/MetadataConverter.java
index e76c1160..c787950a 100644
--- a/ome-xml/src/main/java/ome/xml/meta/MetadataConverter.java
+++ b/ome-xml/src/main/java/ome/xml/meta/MetadataConverter.java
@@ -2301,7 +2301,7 @@ public final class MetadataConverter {
 
         int wellAnnotationRefCount = 0;
         try {
-          src.getWellAnnotationRefCount(i, q);
+          wellAnnotationRefCount = src.getWellAnnotationRefCount(i, q);
         }
         catch (NullPointerException e) { }
         for (int a=0; a<wellAnnotationRefCount; a++) {
diff --git a/ome_model/experimental.py b/ome_model/experimental.py
index e5f4ecfd..aa9f8630 100644
--- a/ome_model/experimental.py
+++ b/ome_model/experimental.py
@@ -102,7 +102,7 @@ class Image(object):
     def __init__(self,
                  name,
                  sizeX, sizeY, sizeZ, sizeC, sizeT,
-                 tiffs=[],
+                 tiffs=None,
                  order="XYZTC",
                  type="uint16",
                  ):
@@ -123,8 +123,9 @@ class Image(object):
             'Planes': [],
         }
         Image.ID += 1
-        for tiff in tiffs:
-            self.add_tiff(tiff)
+        if tiffs:
+            for tiff in tiffs:
+                self.add_tiff(tiff)
 
     def add_channel(self, name=None, color=None, samplesPerPixel=1):
         self.data["Channels"].append(
@@ -223,12 +224,16 @@ def parse_tiff(tiff):
     return (m.group("channel"), m.group("time"), m.group("slice"))
 
 
-def create_companion(plates=[], images=[], out=None):
+def create_companion(plates=None, images=None, out=None):
     """
     Create a companion OME-XML for a given experiment.
     Assumes 2D TIFFs
     """
     root = ET.Element("OME", attrib=OME_ATTRIBUTES)
+    if not plates:
+        plates = []
+    if not images:
+        images = []
 
     for plate in plates:
         p = ET.SubElement(root, "Plate", attrib=plate.data['Plate'])
diff --git a/specification/src/main/resources/transforms/2008-09-to-2009-09.xsl b/specification/src/main/resources/transforms/2008-09-to-2009-09.xsl
index 53c1d1ab..8e6a5c52 100644
--- a/specification/src/main/resources/transforms/2008-09-to-2009-09.xsl
+++ b/specification/src/main/resources/transforms/2008-09-to-2009-09.xsl
@@ -154,6 +154,25 @@
         </xsl:element>
       </xsl:for-each>
 
+      <!-- Transform the CustomAttributes into XMLAnnotation -->
+      <xsl:if test="(count(//*[local-name() = 'StructuredAnnotations'])) = 0">
+        <xsl:if test="(count(//*[local-name() = 'CustomAttributes'])) &gt; 0">
+          <xsl:element name="StructuredAnnotations" namespace="{$newSANS}">
+            <xsl:comment>Append Custom Attributes as XMLAnnotation</xsl:comment>
+            <xsl:for-each select="//*[local-name() = 'CustomAttributes']">
+              <xsl:if test="count(@*|node()) &gt; 0">
+                <xsl:variable name="annotationIndex" select="position()" />
+                <xsl:element name="XMLAnnotation" namespace="{$newSANS}">
+                  <xsl:attribute name="ID"><xsl:value-of select="concat('Annotation:CustomAttributes', $annotationIndex)"/></xsl:attribute>
+                  <xsl:element name="Value" namespace="{$newSANS}">
+                    <xsl:apply-templates select="@*|node()"/>
+                  </xsl:element>
+                </xsl:element>
+              </xsl:if>
+            </xsl:for-each>
+          </xsl:element>
+        </xsl:if>
+      </xsl:if>
     </OME>
   </xsl:template>
 
@@ -181,6 +200,27 @@
     </xsl:element>
   </xsl:template>
 
+  <xsl:template match="SA:StructuredAnnotations">
+    <xsl:element name="{name()}" namespace="{$newSANS}">
+      <xsl:apply-templates select="@*|node()"/>
+      <!-- Transform the CustomAttributes into XMLAnnotation -->
+      <xsl:if test="(count(//*[local-name() = 'CustomAttributes'])) &gt; 0">
+        <xsl:comment>Append Custom Attributes as XMLAnnotation</xsl:comment>
+        <xsl:for-each select="//*[local-name() = 'CustomAttributes']">
+          <xsl:if test="count(@*|node()) &gt; 0">
+          <xsl:variable name="annotationIndex" select="position()" />
+            <xsl:element name="XMLAnnotation" namespace="{$newSANS}">
+              <xsl:attribute name="ID"><xsl:value-of select="concat('Annotation:CustomAttributes', $annotationIndex)"/></xsl:attribute>
+              <xsl:element name="Value" namespace="{$newSANS}">
+                <xsl:apply-templates select="@*|node()"/>
+              </xsl:element>
+            </xsl:element>
+          </xsl:if>
+        </xsl:for-each>
+      </xsl:if>
+    </xsl:element>
+  </xsl:template>
+
   <xsl:template match="SA:List">
     <xsl:apply-templates select="*"/>
   </xsl:template>
@@ -849,18 +889,12 @@
     </xsl:element>
   </xsl:template>
 
-  <!-- Transform the CustomAttributes into XMLAnnotation -->
+  <!-- Remove CustomAttributes -->
   <xsl:template match="CA:CustomAttributes">
-    <xsl:if test="count(@*|node()) &gt; 0">
-      <xsl:element name="StructuredAnnotations" namespace="{$newSANS}">
-        <xsl:element name="XMLAnnotation" namespace="{$newSANS}">
-          <xsl:attribute name="ID">Annotation:1</xsl:attribute>
-          <xsl:element name="Value" namespace="{$newSANS}">
-            <xsl:apply-templates select="@*|node()"/>
-          </xsl:element>
-        </xsl:element>
-      </xsl:element>
-    </xsl:if>
+    <xsl:variable name="caCount" select="count(preceding::CA:CustomAttributes | ancestor::CA:CustomAttributes) + 1"/> 
+    <xsl:element name="SA:AnnotationRef" namespace="{$newSANS}">
+      <xsl:attribute name="ID"><xsl:value-of select="concat('Annotation:CustomAttributes', $caCount)"/></xsl:attribute>
+    </xsl:element>
   </xsl:template>
 
   <!--
@@ -886,8 +920,14 @@
       <xsl:apply-templates
           select="@* [not(name() = 'DefaultPixels' or name() = 'AcquiredPixels')]"/>
       <xsl:for-each
-          select="* [not(local-name(.) = 'Thumbnail' or local-name(.) = 'DisplayOptions' or local-name(.) = 'Region' or local-name(.) = 'CustomAttributes' or local-name(.) = 'LogicalChannel')]">
+          select="* [not(local-name(.) = 'Thumbnail' or local-name(.) = 'DisplayOptions' or local-name(.) = 'Region' or local-name(.) = 'LogicalChannel')]">
         <xsl:choose>
+          <xsl:when test="local-name(.) ='CustomAttributes'">
+            <xsl:variable name="caCount" select="count(preceding::CA:CustomAttributes | ancestor::CA:CustomAttributes) + 1"/> 
+            <xsl:element name="SA:AnnotationRef" namespace="{$newSANS}">
+              <xsl:attribute name="ID"><xsl:value-of select="concat('Annotation:CustomAttributes', $caCount)"/></xsl:attribute>
+            </xsl:element>
+          </xsl:when>
           <xsl:when test="local-name(.) ='Description'">
             <xsl:apply-templates select="current()"/>
           </xsl:when>


Test patch:
diff --git a/test/unit/test_plate.py b/test/unit/test_plate.py
new file mode 100644
index 00000000..c7d09633
--- /dev/null
+++ b/test/unit/test_plate.py
@@ -0,0 +1,128 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# #L%
+
+
+from ome_model.experimental import Plate, Image, create_companion
+import xml.etree.ElementTree as ElementTree
+
+NS = {'OME': 'http://www.openmicroscopy.org/Schemas/OME/2016-06'}
+ElementTree.register_namespace('OME', NS['OME'])
+
+
+class TestPlate(object):
+
+    def test_minimal_plate(self, tmpdir):
+        f = str(tmpdir.join('plate.companion.ome'))
+
+        p = Plate("test", 1, 1)
+        well = p.add_well(0, 0)
+        i = Image("test", 256, 512, 3, 4, 5)
+        well.add_wellsample(0, i)
+        create_companion(plates=[p], out=f)
+
+        root = ElementTree.parse(f).getroot()
+        plates = root.findall('OME:Plate', namespaces=NS)
+        assert len(plates) == 1
+        assert plates[0].attrib['Name'] == 'test'
+        wells = plates[0].findall('OME:Well', namespaces=NS)
+        assert len(wells) == 1
+        assert wells[0].attrib['Row'] == '0'
+        assert wells[0].attrib['Column'] == '0'
+        wellsamples = wells[0].findall('OME:WellSample', namespaces=NS)
+        assert len(wellsamples) == 1
+        imagerefs = wellsamples[0].findall('OME:ImageRef', namespaces=NS)
+        assert len(imagerefs) == 1
+        imageid = imagerefs[0].attrib['ID']
+        images = root.findall('OME:Image', namespaces=NS)
+        assert len(images) == 1
+        assert images[0].attrib['Name'] == 'test'
+        assert images[0].attrib['ID'] == imageid
+        pixels = images[0].findall('OME:Pixels', namespaces=NS)
+        assert len(pixels) == 1
+        assert pixels[0].attrib['SizeX'] == '256'
+        assert pixels[0].attrib['SizeY'] == '512'
+        assert pixels[0].attrib['SizeZ'] == '3'
+        assert pixels[0].attrib['SizeC'] == '4'
+        assert pixels[0].attrib['SizeT'] == '5'
+        assert pixels[0].attrib['DimensionOrder'] == 'XYZTC'
+        assert pixels[0].attrib['Type'] == 'uint16'
+        channels = pixels[0].findall('OME:Channel', namespaces=NS)
+        assert len(channels) == 0
+
+    def test_multiple_rows_columns_wellsamples(self, tmpdir):
+        f = str(tmpdir.join('plate.companion.ome'))
+
+        p = Plate("test", 4, 5)
+        for row in range(4):
+            for column in range(5):
+                well = p.add_well(row, column)
+                for field in range(6):
+                    i = Image("test", 256, 512, 3, 4, 5)
+                    well.add_wellsample(field, i)
+        create_companion(plates=[p], out=f)
+
+        root = ElementTree.parse(f).getroot()
+        plates = root.findall('OME:Plate', namespaces=NS)
+        assert len(plates) == 1
+        assert plates[0].attrib['Name'] == 'test'
+        wells = plates[0].findall('OME:Well', namespaces=NS)
+        assert len(wells) == 20
+        imageids = []
+        for i in range(20):
+            wellsamples = wells[i].findall('OME:WellSample', namespaces=NS)
+            assert len(wellsamples) == 6
+            for ws in wellsamples:
+                imagerefs = ws.findall('OME:ImageRef', namespaces=NS)
+                assert len(imagerefs) == 1
+                imageids.append(imagerefs[0].attrib['ID'])
+        assert len(imageids) == 120
+        images = root.findall('OME:Image', namespaces=NS)
+        assert len(images) == 120
+        assert [x.attrib['ID'] for x in images] == imageids
+
+    def test_multiple_plates_one_per_file(self, tmpdir):
+
+        files = [str(tmpdir.join('%s.companion.ome' % i)) for i in range(4)]
+        for i in range(4):
+            p = Plate("test %s" % i, 1, 1)
+            well = p.add_well(0, 0)
+            img = Image("test %s" % i, 256, 512, 3, 4, 5)
+            well.add_wellsample(0, img)
+            create_companion(plates=[p], out=files[i])
+
+        for i in range(4):
+            root = ElementTree.parse(files[i]).getroot()
+            plates = root.findall('OME:Plate', namespaces=NS)
+            assert len(plates) == 1
+            assert plates[0].attrib['Name'] == 'test %s' % i
+            wells = plates[0].findall('OME:Well', namespaces=NS)
+            assert len(wells) == 1
+            wellsamples = wells[0].findall('OME:WellSample', namespaces=NS)
+            assert len(wellsamples) == 1
+            imagerefs = wellsamples[0].findall('OME:ImageRef', namespaces=NS)
+            assert len(imagerefs) == 1
+            images = root.findall('OME:Image', namespaces=NS)
+            assert len(images) == 1
+            assert images[0].attrib['Name'] == 'test %s' % i