Repository: XLSForm/pyxform
Pull request #169 (2017-12-19 06:43:36)

Feature/external instance #107
PR for #107. More or less the same as pulldata and select_one_from_file etc. in terms of adding a supplementary instance to the form model, but this just does the instance part without any effects elsewhere on the form (no controls, bindings, etc.).

Includes some tidy up of the other functions that generate instances to make it possible to validate across all of them e.g. to check that duplicates aren't being added. Also I think this may qualify as a bug fix but for example if a user has a form using the same select from file e.g. `select_one_from_file cities.csv` with item1 and item2, then 2 identical copies of the instance would be added - now only one is added.

Includes some other minor cleanups:

- Survey's `__repr__` calling `unicode(self)` caused a recursion crash, now uses `__unicode__` instead
- Exception that was acting like a NotImplementedError is now one
- PyxformTestCase didn't show why the compilation failed but now it does



Patch:
diff --git a/pyxform/builder.py b/pyxform/builder.py
index 105cd58..7296061 100644
--- a/pyxform/builder.py
+++ b/pyxform/builder.py
@@ -3,6 +3,7 @@ import os
 
 from pyxform import file_utils, utils
 from pyxform.errors import PyXFormError
+from pyxform.external_instance import ExternalInstance
 from pyxform.question import (InputQuestion, MultipleChoiceQuestion,
                               OsmUploadQuestion, Question, RangeQuestion,
                               TriggerQuestion, UploadQuestion)
@@ -92,6 +93,8 @@ class SurveyElementBuilder(object):
             d = self._sections[section_name]
             full_survey = self.create_survey_element_from_dict(d)
             return full_survey.children
+        elif d[u"type"] == u"xml-external":
+            return ExternalInstance(**d)
         else:
             return self._create_question_from_dict(
                 d, copy_json_dict(QUESTION_TYPE_DICT), self._add_none_option)
diff --git a/pyxform/external_instance.py b/pyxform/external_instance.py
new file mode 100644
index 0000000..c2c54ca
--- /dev/null
+++ b/pyxform/external_instance.py
@@ -0,0 +1,13 @@
+from pyxform.survey_element import SurveyElement
+from pyxform.utils import node
+
+
+class ExternalInstance(SurveyElement):
+
+    def xml_control(self):
+        """
+        No-op since there is no associated form control to place under <body/>.
+
+        Exists here because there's a soft abstractmethod in SurveyElement.
+        """
+        pass
diff --git a/pyxform/instance_info.py b/pyxform/instance_info.py
new file mode 100644
index 0000000..b2d339c
--- /dev/null
+++ b/pyxform/instance_info.py
@@ -0,0 +1,11 @@
+
+
+class InstanceInfo(object):
+    """Standardise Instance details relevant during XML generation."""
+
+    def __init__(self, type, context, name, src, instance):
+        self.type = type
+        self.context = context
+        self.name = name
+        self.src = src
+        self.instance = instance
diff --git a/pyxform/question_type_dictionary.py b/pyxform/question_type_dictionary.py
index 0244acc..7de77bb 100644
--- a/pyxform/question_type_dictionary.py
+++ b/pyxform/question_type_dictionary.py
@@ -856,5 +856,8 @@ QUESTION_TYPE_DICT = \
             "bind": {
                 "type": "binary"
             }
+        },
+        "xml-external": {
+            # Only effect is to add an external instance.
         }
     }
diff --git a/pyxform/section.py b/pyxform/section.py
index 4d8410f..2be0b7a 100644
--- a/pyxform/section.py
+++ b/pyxform/section.py
@@ -1,3 +1,4 @@
+from pyxform.external_instance import ExternalInstance
 from pyxform.question import SurveyElement
 from pyxform.utils import node
 from pyxform.errors import PyXFormError
@@ -38,6 +39,8 @@ class Section(SurveyElement):
             if child.get(u"flat"):
                 for grandchild in child.xml_instance_array():
                     result.appendChild(grandchild)
+            elif isinstance(child, ExternalInstance):
+                continue
             else:
                 result.appendChild(child.xml_instance())
         return result
diff --git a/pyxform/survey.py b/pyxform/survey.py
index fc21dae..7e2370a 100644
--- a/pyxform/survey.py
+++ b/pyxform/survey.py
@@ -7,26 +7,25 @@ from collections import defaultdict
 from datetime import datetime
 
 from pyxform import constants
+from pyxform.external_instance import ExternalInstance
 from pyxform.errors import PyXFormError
+from pyxform.errors import ValidationError
 from pyxform.instance import SurveyInstance
+from pyxform.instance_info import InstanceInfo
 from pyxform.odk_validate import check_xform
 from pyxform.question import Question
 from pyxform.section import Section
 from pyxform.survey_element import SurveyElement
-from pyxform.utils import PatchedText, basestring, node, unicode
+from pyxform.utils import PatchedText, basestring, node, unicode, NSMAP
 
-nsmap = {
-    u"xmlns": u"http://www.w3.org/2002/xforms",
-    u"xmlns:h": u"http://www.w3.org/1999/xhtml",
-    u"xmlns:ev": u"http://www.w3.org/2001/xml-events",
-    u"xmlns:xsd": u"http://www.w3.org/2001/XMLSchema",
-    u"xmlns:jr": u"http://openrosa.org/javarosa",
-    u"xmlns:orx": u"http://openrosa.org/xforms"
-    }
 
-for prefix, uri in nsmap.items():
-    prefix = prefix.replace("xmlns", "").replace(":", "")
-    ETree.register_namespace(prefix, uri)
+def register_nsmap():
+    for prefix, uri in NSMAP.items():
+        prefix_no_xmlns = prefix.replace("xmlns", "").replace(":", "")
+        ETree.register_namespace(prefix_no_xmlns, uri)
+
+
+register_nsmap()
 
 
 class Survey(Section):
@@ -83,12 +82,14 @@ class Survey(Section):
                 if len(ns.split('=')) == 2 and ns.split('=')[0] != ''
             ]
             xmlns = u'xmlns:'
+            nsmap = NSMAP.copy()
             nsmap.update(dict([
                 (xmlns + k, v.replace('"', '').replace("'", ""))
                 for k, v in nslist if xmlns + k not in nsmap
             ]))
-
-        return nsmap
+            return nsmap
+        else:
+            return NSMAP
 
     def xml(self):
         """
@@ -112,64 +113,218 @@ class Survey(Section):
                     **nsmap
                     )
 
-    def _generate_static_instances(self):
+    @staticmethod
+    def _generate_static_instances(list_name, choice_list):
         """
         Generates <instance> elements for static data
         (e.g. choices for select type questions)
+
+        Note that per commit message 0578242 and in xls2json.py R539, an
+        instance is only output for select items defined in the choices sheet
+        when the item has a choice_filter, and it is that way for backwards
+        compatibility.
         """
-        for list_name, choice_list in self.choices.items():
-            instance_element_list = []
-            for idx, choice in zip(range(len(choice_list)), choice_list):
-                choice_element_list = []
-                # Add a unique id to the choice element incase there is itext
-                # it refrences
-                itext_id = '-'.join(['static_instance', list_name, str(idx)])
-                choice_element_list.append(node("itextId", itext_id))
+        instance_element_list = []
+        for idx, choice in enumerate(choice_list):
+            choice_element_list = []
+            # Add a unique id to the choice element in case there is itext
+            # it references
+            itext_id = '-'.join(['static_instance', list_name, str(idx)])
+            choice_element_list.append(node("itextId", itext_id))
+
+            for choicePropertyName, choicePropertyValue in choice.items():
+                if isinstance(choicePropertyValue, basestring) \
+                        and choicePropertyName != 'label':
+                    choice_element_list.append(
+                        node(choicePropertyName,
+                             unicode(choicePropertyValue))
+                    )
+            instance_element_list.append(node("item", *choice_element_list))
+        return InstanceInfo(
+            type=u"choice",
+            context=u"survey",
+            name=list_name,
+            src=None,
+            instance=node(
+                "instance",
+                node("root", *instance_element_list),
+                id=list_name
+            )
+        )
+
+    @staticmethod
+    def _get_dummy_instance():
+        """Instance content required by ODK Validate for select inputs."""
+        return node("root", node("item", node("name", "_"), node("label", "_")))
+
+    @staticmethod
+    def _generate_external_instances(element):
+        if isinstance(element, ExternalInstance):
+            name = element[u"name"]
+            src = "jr://file/{}.xml".format(name)
+            return InstanceInfo(
+                type=u"external",
+                context="[type: {t}, name: {n}]".format(
+                    t=element[u"parent"][u"type"],
+                    n=element[u"parent"][u"name"]
+                ),
+                name=name,
+                src=src,
+                instance=node(
+                    "instance",
+                    Survey._get_dummy_instance(),
+                    id=name,
+                    src=src
+                )
+            )
 
-                for choicePropertyName, choicePropertyValue in choice.items():
-                    if isinstance(choicePropertyValue, basestring) \
-                            and choicePropertyName != 'label':
-                        choice_element_list.append(
-                            node(choicePropertyName,
-                                 unicode(choicePropertyValue))
+    @staticmethod
+    def _validate_external_instances(instances):
+        """
+        Must have unique names.
+
+        - Duplications could come from across groups; this checks the form.
+        - Errors are pooled together into a (hopefully) helpful message.
+        """
+        seen = {}
+        for i in instances:
+            element = i.name
+            if seen.get(element) is None:
+                seen[element] = [i]
+            else:
+                seen[element].append(i)
+        errors = []
+        for element, copies in seen.items():
+            if 1 < len(copies):
+                contexts = ", ".join(x.context for x in copies)
+                errors.append(
+                    "Instance names must be unique within a form. "
+                    "The name '{i}' was found {c} time(s), "
+                    "under these contexts: {contexts}".format(
+                        i=element, c=len(copies), contexts=contexts))
+        if 0 < len(errors):
+            raise ValidationError("\n".join(errors))
+
+    @staticmethod
+    def _generate_pulldata_instances(element):
+        if 'calculate' in element['bind']:
+            calculate = element['bind']['calculate']
+            if calculate.startswith('pulldata('):
+                pieces = calculate.split('"') \
+                    if '"' in calculate else calculate.split("'")
+                if len(pieces) > 1 and pieces[1]:
+                    file_id = pieces[1]
+                    uri = "jr://file-csv/{}.csv".format(file_id)
+                    return InstanceInfo(
+                        type=u"pulldata",
+                        context="[type: {t}, name: {n}]".format(
+                            t=element[u"parent"][u"type"],
+                            n=element[u"parent"][u"name"]
+                        ),
+                        name=file_id,
+                        src=uri,
+                        instance=node(
+                            "instance",
+                            Survey._get_dummy_instance(),
+                            id=file_id,
+                            src=uri
                         )
-                instance_element_list.append(node("item",
-                                                  *choice_element_list))
-            yield node("instance", node("root", *instance_element_list),
-                       id=list_name)
+                    )
 
-    def _generate_pulldata_instances(self):
-        pulldata = []
-        for i in self.iter_descendants():
-            if 'calculate' in i['bind']:
-                calculate = i['bind']['calculate']
-                if calculate.startswith('pulldata('):
-                    pieces = calculate.split('"') \
-                        if '"' in calculate else calculate.split("'")
-                    if len(pieces) > 1 and pieces[1] not in pulldata:
-                        csv_id = pieces[1]
-                        pulldata.append(csv_id)
-
-                        yield node("instance", id=csv_id,
-                                   src="jr://file-csv/{}.csv".format(csv_id)
-                                   )
-
-    def _generate_from_file_instances(self):
-        for i in self.iter_descendants():
-            itemset = i.get('itemset')
-            if itemset and \
-                    (itemset.endswith('.csv') or itemset.endswith('.xml')):
-                file_id, ext = os.path.splitext(itemset)
-                uri = 'jr://%s/%s' % (
-                    'file' if ext == '.xml' else "file-%s" % ext[1:],
-                    itemset)
-                yield node(
+    @staticmethod
+    def _generate_from_file_instances(element):
+        itemset = element.get('itemset')
+        if itemset and (itemset.endswith('.csv') or itemset.endswith('.xml')):
+            file_id, ext = os.path.splitext(itemset)
+            uri = 'jr://%s/%s' % (
+                'file' if ext == '.xml' else "file-%s" % ext[1:], itemset)
+            return InstanceInfo(
+                type=u"file",
+                context="[type: {t}, name: {n}]".format(
+                    t=element[u"parent"][u"type"],
+                    n=element[u"parent"][u"name"]
+                ),
+                name=file_id,
+                src=uri,
+                instance=node(
                     "instance",
-                    node("root",
-                         node("item", node("name", "_"), node("label", "_"))),
+                    Survey._get_dummy_instance(),
                     id=file_id,
                     src=uri
                 )
+            )
+
+    def _generate_instances(self):
+        """
+        Get instances from all the different ways that they may be generated.
+
+        An opportunity to validate instances before output to the XML model.
+
+        Instance names used for the id attribute are generated as follows:
+
+        - xml-external: item name value (for type==xml-external)
+        - pulldata: first arg to calculation->pulldata()
+        - select from file: file name arg to type->itemset
+        - choices: list_name (for type==select_*)
+
+        Validation and business rules for output of instances:
+
+        - xml-external item name must be unique across the XForm and the form is
+          considered invalid if there is a duplicate name. This differs from
+          other item types which allow duplicates if not in the same group.
+        - for all instance sources, if the same instance name is encountered,
+          the following rules are used to allow re-using instances but prevent
+          overwriting conflicting instances:
+          - same id, same src URI: skip adding the second (duplicate) instance
+          - same id, different src URI: raise an error
+          - otherwise: output the instance
+
+        There are two other things currently supported by pyxform that involve
+        external files and are not explicitly handled here, but may be relevant
+        to future efforts to harmonise / simplify external data workflows:
+
+        - `search` appearance/function: works a lot like pulldata but the csv
+          isn't made explicit in the form.
+        - `select_one_external`: implicitly relies on a `itemsets.csv` file and
+          uses XPath-like expressions for querying.
+        """
+        instances = []
+        for i in self.iter_descendants():
+            i_ext = self._generate_external_instances(element=i)
+            i_pull = self._generate_pulldata_instances(element=i)
+            i_file = self._generate_from_file_instances(element=i)
+            instances += [x for x in [i_ext, i_pull, i_file] if x is not None]
+
+        # Append last so the choice instance is excluded on a name clash.
+        for k, v in self.choices.items():
+            instances += [
+                self._generate_static_instances(list_name=k, choice_list=v)]
+
+        # Check that external instances have unique names.
+        if 0 < len(instances):
+            ext_only = [x for x in instances if x.type == "external"]
+            self._validate_external_instances(instances=ext_only)
+
+        seen = {}
+        for i in instances:
+            if i.name in seen.keys() and seen[i.name].src != i.src:
+                # Instance id exists with different src URI -> error.
+                msg = "The same instance id will be generated for different " \
+                      "external instance source URIs. Please check the form. " \
+                      "Instance name: '{i}', Existing type: '{e}', " \
+                      "Existing URI: '{iu}', Duplicate type: '{d}', " \
+                      "Duplicate URI: '{du}', Duplicate context: '{c}'.".format(
+                          i=i.name, iu=seen[i.name].src, e=seen[i.name].type,
+                          d=i.type, du=i.src, c=i.context
+                      )
+                raise PyXFormError(msg)
+            elif i.name in seen.keys() and seen[i.name].src == i.src:
+                # Instance id exists with same src URI -> ok, don't duplicate.
+                continue
+            else:
+                # Instance doesn't exist yet -> add it.
+                yield i.instance
+            seen[i.name] = i
 
     def xml_model(self):
         """
@@ -183,9 +338,7 @@ class Survey(Section):
         if self._translations:
             model_children.append(self.itext())
         model_children += [node("instance", self.xml_instance())]
-        model_children += list(self._generate_static_instances())
-        model_children += list(self._generate_pulldata_instances())
-        model_children += list(self._generate_from_file_instances())
+        model_children += list(self._generate_instances())
         model_children += self.xml_bindings()
 
         if self.submission_url or self.public_key:
@@ -432,7 +585,7 @@ class Survey(Section):
         # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/
         xml_with_linebreaks = self.xml().toprettyxml(indent='  ')
         text_re = re.compile('(>)\n\s*(\s[^<>\s].*?)\n\s*(\s</)', re.DOTALL)
-        output_re = re.compile('\n.*(<output.*>)\n(  )*')
+        output_re = re.compile('\n.*(<output.*>)\n(\s\s)*')
         pretty_xml = text_re.sub(lambda m: ''.join(m.group(1, 2, 3)), xml_with_linebreaks)
         inline_output = output_re.sub('\g<1>', pretty_xml)
         inline_output = re.compile('<label>\s*\n*\s*\n*\s*</label>')\
@@ -440,7 +593,7 @@ class Survey(Section):
         return '<?xml version="1.0"?>\n' + inline_output
 
     def __repr__(self):
-        return unicode(self)
+        return self.__unicode__()
 
     def __unicode__(self):
         return "<pyxform.survey.Survey instance at %s>" % hex(id(self))
diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py
index b1b5d09..c1813ac 100644
--- a/pyxform/survey_element.py
+++ b/pyxform/survey_element.py
@@ -358,7 +358,7 @@ class SurveyElement(dict):
         The control depends on what type of question we're asking, it
         doesn't make sense to implement here in the base class.
         """
-        raise Exception("Control not implemented")
+        raise NotImplementedError("Control not implemented")
 
 
 def hashable(v):
diff --git a/pyxform/utils.py b/pyxform/utils.py
index b993690..a21d113 100644
--- a/pyxform/utils.py
+++ b/pyxform/utils.py
@@ -25,7 +25,15 @@ TAG_CHAR = r"[a-zA-Z:_0-9\-.]"
 XFORM_TAG_REGEXP = "%(start)s%(char)s*" % {
     "start": TAG_START_CHAR,
     "char": TAG_CHAR
-    }
+}
+NSMAP = {
+    u"xmlns": u"http://www.w3.org/2002/xforms",
+    u"xmlns:h": u"http://www.w3.org/1999/xhtml",
+    u"xmlns:ev": u"http://www.w3.org/2001/xml-events",
+    u"xmlns:xsd": u"http://www.w3.org/2001/XMLSchema",
+    u"xmlns:jr": u"http://openrosa.org/javarosa",
+    u"xmlns:orx": u"http://openrosa.org/xforms"
+}
 
 
 class DetachableElement(Element):
diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py
index 9853fac..8741427 100644
--- a/pyxform/xform2json.py
+++ b/pyxform/xform2json.py
@@ -6,8 +6,7 @@ import codecs
 import xml.etree.ElementTree as ETree
 from operator import itemgetter
 from pyxform import builder
-from pyxform.utils import basestring
-from pyxform.survey import nsmap
+from pyxform.utils import basestring, NSMAP
 
 
 # {{{ http://code.activestate.com/recipes/573463/ (r7)
@@ -181,7 +180,7 @@ class XFormToDict:
 
     def get_dict(self):
         json_str = json.dumps(self._dict)
-        for uri in nsmap.values():
+        for uri in NSMAP.values():
             json_str = json_str.replace('{%s}' % uri, '')
         return json.loads(json_str)
 


Test patch:
diff --git a/pyxform/tests/test_expected_output/pull_data.xml b/pyxform/tests/test_expected_output/pull_data.xml
index 0e01d4b..ccd3d38 100644
--- a/pyxform/tests/test_expected_output/pull_data.xml
+++ b/pyxform/tests/test_expected_output/pull_data.xml
@@ -12,7 +12,14 @@
           </meta>
         </pull_data>
       </instance>
-      <instance id="fruits" src="jr://file-csv/fruits.csv"/>
+      <instance id="fruits" src="jr://file-csv/fruits.csv">
+        <root>
+          <item>
+            <name>_</name>
+            <label>_</label>
+          </item>
+        </root>
+      </instance>
       <bind calculate="pulldata('fruits', 'name','name', 'mango')" nodeset="/pull_data/fruit" type="string"/>
       <bind nodeset="/pull_data/note_fruite" readonly="true()" type="string"/>
       <bind calculate="concat('uuid:', uuid())" nodeset="/pull_data/meta/instanceID" readonly="true()" type="string"/>
diff --git a/pyxform/tests_v1/pyxform_test_case.py b/pyxform/tests_v1/pyxform_test_case.py
index aba54e2..dd346d9 100644
--- a/pyxform/tests_v1/pyxform_test_case.py
+++ b/pyxform/tests_v1/pyxform_test_case.py
@@ -12,8 +12,8 @@ from unittest import TestCase
 from pyxform.builder import create_survey_element_from_dict
 from pyxform.errors import PyXFormError
 from pyxform.odk_validate import ODKValidateError, check_xform
-from pyxform.survey import nsmap
 from pyxform.tests_v1.test_utils.md_table import md_table_to_ss_structure
+from pyxform.utils import NSMAP, unicode
 from pyxform.xls2json import workbook_to_json
 
 
@@ -28,17 +28,17 @@ class PyxformMarkdown(object):
         if kwargs is None:
             kwargs = {}
         if autoname:
-            kwargs = self._autonameInputs(kwargs)
+            kwargs = self._autoname_inputs(kwargs)
         _md = []
         for line in md_raw.split('\n'):
-            if re.match(r'^\s+\#', line):
+            if re.match(r'^\s+#', line):
                 # ignore lines which start with pound sign
                 continue
-            elif re.match(r'^(.*)(\#[^\|]+)$', line):
+            elif re.match(r'^(.*)(#[^|]+)$', line):
                 # keep everything before the # outside of the last occurrence
                 # of |
                 _md.append(
-                    re.match(r'^(.*)(\#[^\|]+)$', line).groups()[0].strip())
+                    re.match(r'^(.*)(#[^|]+)$', line).groups()[0].strip())
             else:
                 _md.append(line.strip())
         md = '\n'.join(_md)
@@ -65,7 +65,8 @@ class PyxformMarkdown(object):
 
         return self._ss_structure_to_pyxform_survey(sheets, kwargs)
 
-    def _ss_structure_to_pyxform_survey(self, ss_structure, kwargs):
+    @staticmethod
+    def _ss_structure_to_pyxform_survey(ss_structure, kwargs):
         # using existing methods from the builder
         imported_survey_json = workbook_to_json(ss_structure)
         # ideally, when all these tests are working, this would be
@@ -77,7 +78,24 @@ class PyxformMarkdown(object):
 
         return survey
 
-    def _autonameInputs(self, kwargs):
+    @staticmethod
+    def _run_odk_validate(xml):
+        # On Windows, NamedTemporaryFile must be opened exclusively.
+        # So it must be explicitly created, opened, closed, and removed
+        tmp = tempfile.NamedTemporaryFile(suffix='.xml', delete=False)
+        tmp.close()
+        try:
+            with codecs.open(tmp.name, mode="w", encoding="utf-8") as fp:
+                fp.write(xml)
+                fp.close()
+            check_xform(tmp.name)
+        finally:
+            # Clean up the temporary file
+            os.remove(tmp.name)
+            assert not os.path.isfile(tmp.name)
+
+    @staticmethod
+    def _autoname_inputs(kwargs):
         """
         title and name are necessary for surveys, but not always convenient to
         include in test cases, so this will pull a default value
@@ -95,6 +113,7 @@ class PyxformMarkdown(object):
 
 
 class PyxformTestCase(PyxformMarkdown, TestCase):
+
     def assertPyxformXform(self, **kwargs):
         """
         PyxformTestCase.assertPyxformXform() named arguments:
@@ -139,10 +158,10 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
 
         try:
             if 'md' in kwargs.keys():
-                kwargs = self._autonameInputs(kwargs)
+                kwargs = self._autoname_inputs(kwargs)
                 survey = self.md_to_pyxform_survey(kwargs.get('md'), kwargs)
             elif 'ss_structure' in kwargs.keys():
-                kwargs = self._autonameInputs(kwargs)
+                kwargs = self._autoname_inputs(kwargs)
                 survey = self._ss_structure_to_pyxform_survey(
                     kwargs.get('ss_structure'), kwargs)
             else:
@@ -152,7 +171,9 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
             root = ETree.fromstring(xml.encode('utf-8'))
 
             # Ensure all namespaces are present, even if unused
-            root.attrib.update(**nsmap)
+            final_nsmap = NSMAP.copy()
+            final_nsmap.update(survey.get_nsmap())
+            root.attrib.update(final_nsmap)
 
             xml_nodes['xml'] = root
 
@@ -170,30 +191,13 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
             if debug:
                 print(xml)
             if run_odk_validate:
-                # On Windows, NamedTemporaryFile must be opened exclusively.
-                # So it must be explicitly created, opened, closed, and removed
-                tmp = tempfile.NamedTemporaryFile(suffix='.xml', delete=False)
-                tmp.close()
-                try:
-                    with codecs.open(
-                            tmp.name, mode="w", encoding="utf-8") as fp:
-                        if '_xml_append' in kwargs:
-                            xml += kwargs['_xml_append']
-                        fp.write(xml)
-                        fp.close()
-                    check_xform(tmp.name)
-                finally:
-                    # Clean up the temporary file
-                    os.remove(tmp.name)
-                    assert not os.path.isfile(tmp.name)
-
+                self._run_odk_validate(xml=xml)
                 if len(odk_validate_error__contains) > 0:
-                    raise PyxformTestError(
-                        "ODKValidateError was not raised"
-                        )
+                    raise PyxformTestError("ODKValidateError was not raised")
+
         except PyXFormError as e:
             survey = False
-            errors = [str(e)]
+            errors = [unicode(e)]
             if debug:
                 print("<xml unavailable />")
                 print("ERROR: '%s'" % errors[0])
@@ -201,9 +205,9 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
             if len(odk_validate_error__contains) is 0:
                 raise PyxformTestError("ODK Validate error was thrown but " +
                                        "'odk_validate_error__contains'" +
-                                       " was empty:" + e.message)
+                                       " was empty:" + unicode(e))
             for v_err in odk_validate_error__contains:
-                self.assertContains(e.message, v_err,
+                self.assertContains(unicode(e), v_err,
                                     msg_prefix='odk_validate_error__contains')
         else:
             survey = True
@@ -214,8 +218,8 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
 
                 def check_content(content):
                     text_arr = kwargs[contains_str]
-                    for text in text_arr:
-                        self.assertContains(content, text, msg_prefix=keyword)
+                    for i in text_arr:
+                        self.assertContains(content, i, msg_prefix=keyword)
 
                 return contains_str, check_content
 
@@ -240,7 +244,9 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
                 "Expected valid survey but compilation failed. "
                 "Try correcting the error with 'debug=True', "
                 "setting 'errored=True', "
-                "and or optionally 'error__contains=[...]'")
+                "and or optionally 'error__contains=[...]'"
+                "\nError(s): " + "\n".join(errors)
+            )
         elif survey and expecting_invalid_survey:
             raise PyxformTestError("Expected survey to be invalid.")
 
@@ -250,7 +256,8 @@ class PyxformTestCase(PyxformMarkdown, TestCase):
                 self.assertContains(joined_error, text,
                                     msg_prefix="error__contains")
 
-    def _assert_contains(self, content, text, msg_prefix):
+    @staticmethod
+    def _assert_contains(content, text, msg_prefix):
         if msg_prefix:
             msg_prefix += ": "
 
diff --git a/pyxform/tests_v1/test_xmldata.py b/pyxform/tests_v1/test_xmldata.py
new file mode 100644
index 0000000..d1d4895
--- /dev/null
+++ b/pyxform/tests_v1/test_xmldata.py
@@ -0,0 +1,362 @@
+from pyxform.errors import PyXFormError
+from pyxform.tests_v1.pyxform_test_case import PyxformTestCase
+from pyxform.tests_v1.pyxform_test_case import PyxformTestError
+try:
+    from unittest import skip
+except ImportError:
+    from unittest2 import skip
+
+
+class ExternalInstanceTests(PyxformTestCase):
+
+    def test_can__output_single_external_xml_item(self):
+        """Simplest possible example to include an external instance."""
+        self.assertPyxformXform(
+            md="""
+            | survey |              |        |       |
+            |        | type         | name   | label |
+            |        | xml-external | mydata |       |
+            """,
+            model__contains=[
+                '<instance id="mydata" src="jr://file/mydata.xml">'
+            ],
+            run_odk_validate=True
+        )
+
+    def test_cannot__use_same_external_xml_id_in_same_section(self):
+        """Duplicate external instances in the same section raises an error."""
+        with self.assertRaises(PyxformTestError) as ctx:
+            self.assertPyxformXform(
+                md="""
+                | survey |              |        |       |
+                |        | type         | name   | label |
+                |        | xml-external | mydata |       |
+                |        | xml-external | mydata |       |
+                """,
+                model__contains=[])
+        # This is caught first by existing validation rule.
+        self.assertIn("There are two survey elements named 'mydata'",
+                      repr(ctx.exception))
+
+    def test_can__use_unique_external_xml_in_same_section(self):
+        """Two unique external instances in the same section is OK."""
+        self.assertPyxformXform(
+            md="""
+            | survey |              |         |       |
+            |        | type         | name    | label |
+            |        | xml-external | mydata  |       |
+            |        | xml-external | mydata2 |       |
+            """,
+            model__contains=[
+                '<instance id="mydata" src="jr://file/mydata.xml">',
+                '<instance id="mydata2" src="jr://file/mydata2.xml">'
+            ],
+            run_odk_validate=True
+        )
+
+    def test_cannot__use_same_external_xml_id_across_groups(self):
+        """Duplicate external instances anywhere raises an error."""
+        with self.assertRaises(PyxformTestError) as ctx:
+            self.assertPyxformXform(
+                md="""
+                | survey |              |        |       |
+                |        | type         | name   | label |
+                |        | xml-external | mydata |       |
+                |        | begin group  | g1     |       |
+                |        | xml-external | mydata |       |
+                |        | end group    | g1     |       |
+                |        | begin group  | g2     |       |
+                |        | xml-external | mydata |       |
+                |        | end group    | g2     |       |
+                """,
+                model__contains=[])
+        self.assertIn("Instance names must be unique", repr(ctx.exception))
+        self.assertIn(
+            "The name \'mydata\' was found 3 time(s)", repr(ctx.exception))
+
+    def test_can__use_unique_external_xml_across_groups(self):
+        """Unique external instances anywhere is OK."""
+        self.assertPyxformXform(
+            md="""
+            | survey |              |         |                |
+            |        | type         | name    | label          |
+            |        | xml-external | mydata  |                |
+            |        | begin group  | g1      |                |
+            |        | xml-external | mydata1 |                |
+            |        | note         | note1   | It's note-able |
+            |        | end group    | g1      |                |
+            |        | begin group  | g2      |                |
+            |        | note         | note2   | It's note-able |
+            |        | xml-external | mydata2 |                |
+            |        | end group    | g2      |                |
+            |        | begin group  | g3      |                |
+            |        | note         | note3   | It's note-able |
+            |        | xml-external | mydata3 |                |
+            |        | end group    | g3      |                |
+            """,
+            model__contains=[
+                '<instance id="mydata" src="jr://file/mydata.xml">',
+                '<instance id="mydata1" src="jr://file/mydata1.xml">',
+                '<instance id="mydata2" src="jr://file/mydata2.xml">',
+                '<instance id="mydata3" src="jr://file/mydata3.xml">'
+            ],
+            run_odk_validate=True
+        )
+
+    def test_cannot__use_same_external_xml_id_with_mixed_types(self):
+        """Duplicate instances with other sources present raises an error."""
+        with self.assertRaises(PyxformTestError) as ctx:
+            self.assertPyxformXform(
+                md="""
+                | survey |                                      |      |       |
+                |        | type                                 | name | label | calculation  |
+                |        | begin group                          | g1   |       |              |
+                |        | xml-external                         | city |       |              |
+                |        | end group                            | g1   |       |              |
+                |        | xml-external                         | city |       |              |
+                |        | begin group                          | g2   |       |              |
+                |        | select_one_from_file cities.csv      | city | City  |              |
+                |        | end group                            | g2   |       |              |
+                |        | begin group                          | g3   |       |              |
+                |        | select_multiple_from_file cities.csv | city | City  |              |
+                |        | end group                            | g3   |       |              |
+                |        | begin group                          | g4   |       |              |
+                |        | calculate                            | city | City  | pulldata('fruits', 'name', 'name', 'mango') |
+                |        | end group                            | g4   |       |              |
+                """,
+                model__contains=[])
+        self.assertIn("The name 'city' was found 2 time(s)",
+                      repr(ctx.exception))
+
+    def test_can__use_all_types_together_with_unique_ids(self):
+        """Unique instances with other sources present are OK."""
+        self.assertPyxformXform(
+            md="""
+            | survey  |                                      |       |       |              |
+            |         | type                                 | name  | label | calculation  | choice_filter |
+            |         | begin group                          | g1    |       | | |
+            |         | xml-external                         | city1 |       | | |
+            |         | note                                 | note1 | Note  | | |
+            |         | end group                            | g1    |       | | |
+            |         | begin group                          | g2    |       | | |
+            |         | select_one_from_file cities.csv      | city2 | City2 | | |
+            |         | end group                            | g2    |       | | |
+            |         | begin group                          | g3    |       | | |
+            |         | select_multiple_from_file cities.csv | city3 | City3 | | |
+            |         | end group                            | g3    |       | | |
+            |         | begin group                          | g4    |       | | |
+            |         | calculate                            | city4 | City4 | pulldata('fruits', 'name', 'name', 'mango') | |
+            |         | note                                 | note4 | Note  | | |
+            |         | end group                            | g4    |       | | |
+            |         | select_one states                    | test  | Test  | | true() |
+            | choices |                                      |       |       | | |
+            |         | list_name                            | name  | label | | |
+            |         | states                               | 1     | Pass  | | |
+            |         | states                               | 2     | Fail  | | |
+            """,
+            model__contains=[
+                '<instance id="city1" src="jr://file/city1.xml">',
+"""
+      <instance id="cities" src="jr://file-csv/cities.csv">
+        <root>
+          <item>
+            <name>_</name>
+            <label>_</label>
+          </item>
+        </root>
+      </instance>
+""",
+                '<instance id="fruits" src="jr://file-csv/fruits.csv">',
+"""
+      <instance id="states">
+        <root>
+          <item>
+            <itextId>static_instance-states-0</itextId>
+            <name>1</name>
+          </item>
+          <item>
+            <itextId>static_instance-states-1</itextId>
+            <name>2</name>
+          </item>
+        </root>
+      </instance>
+"""
+            ],
+            run_odk_validate=True
+        )
+
+    def test_cannot__use_different_src_same_id__select_then_internal(self):
+        """Duplicate instance from internal choices raises an error: #88."""
+        md = """
+            | survey  |                                      |       |       |               |
+            |         | type                                 | name  | label | choice_filter |
+            |         | select_one_from_file states.csv      | state | State |               |
+            |         | select_one states                    | test  | Test  | state=/select_from_file_test/state |
+            | choices |                                      |       |       |               |
+            |         | list_name                            | name  | label |               |
+            |         | states                               | 1     | Pass  |               |
+            |         | states                               | 2     | Fail  |               |
+            """
+        with self.assertRaises(PyXFormError) as ctx:
+            survey = self.md_to_pyxform_survey(md_raw=md)
+            survey._to_pretty_xml()
+        self.assertIn(
+            "Instance name: 'states', "
+            "Existing type: 'file', Existing URI: 'jr://file-csv/states.csv', "
+            "Duplicate type: 'choice', Duplicate URI: 'None', "
+            "Duplicate context: 'survey'.",
+            repr(ctx.exception)
+        )
+
+    def test_cannot__use_different_src_same_id__external_then_pulldata(self):
+        """Duplicate instance from pulldata after xml-external raises an error."""
+        md = """
+            | survey |                                      |        |       |              |
+            |        | type                                 | name   | label | calculation  |
+            |        | begin group                          | g1     |       |              |
+            |        | xml-external                         | fruits |       |              |
+            |        | calculate                            | f_csv  | City  | pulldata('fruits', 'name', 'name', 'mango') |
+            |        | note                                 | note   | Fruity! ${f_csv} |   |
+            |        | end group                            | g1     |       |              |
+            """
+        with self.assertRaises(PyXFormError) as ctx:
+            survey = self.md_to_pyxform_survey(md_raw=md)
+            survey._to_pretty_xml()
+        self.assertIn(
+            "Instance name: 'fruits', "
+            "Existing type: 'external', Existing URI: 'jr://file/fruits.xml', "
+            "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', "
+            "Duplicate context: '[type: group, name: g1]'.",
+            repr(ctx.exception)
+        )
+
+    def test_cannot__use_different_src_same_id__pulldata_then_external(self):
+        """Duplicate instance from xml-external after pulldata raises an error."""
+        md = """
+            | survey |                                      |        |       |              |
+            |        | type                                 | name   | label | calculation  |
+            |        | begin group                          | g1     |       |              |
+            |        | calculate                            | f_csv  | City  | pulldata('fruits', 'name', 'name', 'mango') |
+            |        | xml-external                         | fruits |       |              |
+            |        | note                                 | note   | Fruity! ${f_csv} |   |
+            |        | end group                            | g1     |       |              |
+            """
+        with self.assertRaises(PyXFormError) as ctx:
+            survey = self.md_to_pyxform_survey(md_raw=md)
+            survey._to_pretty_xml()
+        self.assertIn(
+            "Instance name: 'fruits', "
+            "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', "
+            "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', "
+            "Duplicate context: '[type: group, name: g1]'.",
+            repr(ctx.exception)
+        )
+
+    def test_can__reuse_csv__selects_then_pulldata(self):
+        """Re-using the same csv external data source id and URI is OK."""
+        md = """
+            | survey |                                              |        |                                    |             |
+            |        | type                                         | name   | label                              | calculation |
+            |        | select_multiple_from_file pain_locations.csv | plocs  | Locations of pain this week.       | |
+            |        | select_one_from_file pain_locations.csv      | pweek  | Location of worst pain this week.  | |
+            |        | select_one_from_file pain_locations.csv      | pmonth | Location of worst pain this month. | |
+            |        | select_one_from_file pain_locations.csv      | pyear  | Location of worst pain this year.  | |
+            |        | calculate                                    | f_csv  | pd | pulldata('pain_locations', 'name', 'name', 'arm') |
+            |        | note                                         | note   | Arm ${f_csv} | |
+            """
+        expected = \
+"""
+      <instance id="pain_locations" src="jr://file-csv/pain_locations.csv">
+        <root>
+          <item>
+            <name>_</name>
+            <label>_</label>
+          </item>
+        </root>
+      </instance>
+"""
+        self.assertPyxformXform(
+            md=md, model__contains=[expected], run_odk_validate=True)
+        survey = self.md_to_pyxform_survey(md_raw=md)
+        xml = survey._to_pretty_xml()
+        self.assertEqual(1, xml.count(expected))
+
+    def test_can__reuse_csv__pulldata_then_selects(self):
+        """Re-using the same csv external data source id and URI is OK."""
+        md = """
+            | survey |                                              |        |       |             |
+            |        | type                                         | name   | label | calculation |
+            |        | calculate                                    | f_csv  | pd | pulldata('pain_locations', 'name', 'name', 'arm') |
+            |        | note                                         | note   | Arm ${f_csv} | |
+            |        | select_multiple_from_file pain_locations.csv | plocs  | Locations of pain this week.       | |
+            |        | select_one_from_file pain_locations.csv      | pweek  | Location of worst pain this week.  | |
+            |        | select_one_from_file pain_locations.csv      | pmonth | Location of worst pain this month. | |
+            |        | select_one_from_file pain_locations.csv      | pyear  | Location of worst pain this year.  | |
+            """
+        expected = \
+"""
+      <instance id="pain_locations" src="jr://file-csv/pain_locations.csv">
+        <root>
+          <item>
+            <name>_</name>
+            <label>_</label>
+          </item>
+        </root>
+      </instance>
+"""
+        self.assertPyxformXform(
+            md=md, model__contains=[expected], run_odk_validate=True)
+
+    def test_can__reuse_xml__selects_then_external(self):
+        """Re-using the same xml external data source id and URI is OK."""
+        md = """
+            | survey |                                              |        |        |
+            |        | type                                         | name   | label  | 
+            |        | select_multiple_from_file pain_locations.xml | plocs  | Locations of pain this week.       |
+            |        | select_one_from_file pain_locations.xml      | pweek  | Location of worst pain this week.  |
+            |        | select_one_from_file pain_locations.xml      | pmonth | Location of worst pain this month. |
+            |        | select_one_from_file pain_locations.xml      | pyear  | Location of worst pain this year.  |
+            |        | xml-external                                 | pain_locations  | |
+            """
+        expected = \
+"""
+      <instance id="pain_locations" src="jr://file/pain_locations.xml">
+        <root>
+          <item>
+            <name>_</name>
+            <label>_</label>
+          </item>
+        </root>
+      </instance>
+"""
+        survey = self.md_to_pyxform_survey(md_raw=md)
+        xml = survey._to_pretty_xml()
+        self.assertEqual(1, xml.count(expected))
+
+    def test_can__reuse_xml__external_then_selects(self):
+        """Re-using the same xml external data source id and URI is OK."""
+        md = """
+            | survey |                                              |        |        |
+            |        | type                                         | name   | label  | 
+            |        | xml-external                                 | pain_locations  | |
+            |        | select_multiple_from_file pain_locations.xml | plocs  | Locations of pain this week.       |
+            |        | select_one_from_file pain_locations.xml      | pweek  | Location of worst pain this week.  |
+            |        | select_one_from_file pain_locations.xml      | pmonth | Location of worst pain this month. |
+            |        | select_one_from_file pain_locations.xml      | pyear  | Location of worst pain this year.  |
+            """
+        expected = \
+"""
+      <instance id="pain_locations" src="jr://file/pain_locations.xml">
+        <root>
+          <item>
+            <name>_</name>
+            <label>_</label>
+          </item>
+        </root>
+      </instance>
+"""
+        self.assertPyxformXform(
+            md=md, model__contains=[expected], run_odk_validate=True)
+        survey = self.md_to_pyxform_survey(md_raw=md)
+        xml = survey._to_pretty_xml()
+        self.assertEqual(1, xml.count(expected))