Repository: scikit-hep/pyhf

Issue:
Combined workspaces have duplicated lumi/POI param configurations
# Description

When combining two workspaces with the same measurement name that has the same POI, the lumi and the POI param config are repeated twice. They should usually be compatible so we should just drop the extra copies.

# Expected Behavior

A single instance of the lumi and POI parameter configuration.

# Actual Behavior

Duplicated/extra instances of the lumi and POI parameter configuratoin.

# Steps to Reproduce

`pyhf combine a.json b.json`

# Checklist

- [x] Run `git fetch` to get the most up to date version of `master`
- [x] Searched through existing Issues to confirm this is not a duplicate issue
- [x] Filled out the Description, Expected Behavior, Actual Behavior, and Steps to Reproduce sections above or have edited/removed them in a way that fully describes the issue



Discussion:

Resolving patch:
diff --git a/src/pyhf/cli/spec.py b/src/pyhf/cli/spec.py
--- a/src/pyhf/cli/spec.py
+++ b/src/pyhf/cli/spec.py
@@ -238,12 +238,19 @@ def rename(workspace, output_file, channel, sample, modifier, measurement):
 @cli.command()
 @click.argument('workspace-one', default='-')
 @click.argument('workspace-two', default='-')
+@click.option(
+    '-j',
+    '--join',
+    default='none',
+    type=click.Choice(Workspace.valid_joins),
+    help='The join operation to apply when combining the two workspaces.',
+)
 @click.option(
     '--output-file',
     help='The location of the output json file. If not specified, prints to screen.',
     default=None,
 )
-def combine(workspace_one, workspace_two, output_file):
+def combine(workspace_one, workspace_two, join, output_file):
     """
     Combine two workspaces into a single workspace.
 
@@ -257,7 +264,7 @@ def combine(workspace_one, workspace_two, output_file):
 
     ws_one = Workspace(spec_one)
     ws_two = Workspace(spec_two)
-    combined_ws = Workspace.combine(ws_one, ws_two)
+    combined_ws = Workspace.combine(ws_one, ws_two, join=join)
 
     if output_file is None:
         click.echo(json.dumps(combined_ws, indent=4, sort_keys=True))
diff --git a/src/pyhf/workspace.py b/src/pyhf/workspace.py
--- a/src/pyhf/workspace.py
+++ b/src/pyhf/workspace.py
@@ -7,6 +7,8 @@
 """
 import logging
 import jsonpatch
+import copy
+import collections
 from . import exceptions
 from . import utils
 from .pdf import Model
@@ -16,11 +18,251 @@
 log = logging.getLogger(__name__)
 
 
+def _join_items(join, left_items, right_items, key='name'):
+    """
+    Join two lists of dictionaries along the given key.
+
+    This is meant to be as generic as possible for any pairs of lists of dictionaries for many join operations.
+
+    Args:
+        join (`str`): The join operation to apply. See ~pyhf.workspace.Workspace for valid join operations.
+        left_items (`list`): A list of dictionaries to join on the left
+        right_items (`list`): A list of dictionaries to join on the right
+
+    Returns:
+        :obj:`list`: A joined list of dictionaries.
+
+    """
+    if join == 'right outer':
+        primary_items, secondary_items = right_items, left_items
+    else:
+        primary_items, secondary_items = left_items, right_items
+    joined_items = copy.deepcopy(primary_items)
+    for secondary_item in secondary_items:
+        # outer join: merge primary and secondary, matching where possible
+        if join == 'outer' and secondary_item in primary_items:
+            continue
+        # left/right outer join: only add secondary if existing item (by key value) is not in primary
+        # NB: this will be slow for large numbers of items
+        elif join in ['left outer', 'right outer'] and secondary_item[key] in [
+            item[key] for item in joined_items
+        ]:
+            continue
+        joined_items.append(copy.deepcopy(secondary_item))
+    return joined_items
+
+
+def _join_versions(join, left_version, right_version):
+    """
+    Join two workspace versions.
+
+    Raises:
+      ~pyhf.exceptions.InvalidWorkspaceOperation: Versions are incompatible.
+
+    Args:
+        join (`str`): The join operation to apply. See ~pyhf.workspace.Workspace for valid join operations.
+        left_version (`str`): The left workspace version.
+        right_version (`str`): The right workspace version.
+
+    Returns:
+        :obj:`str`: The workspace version.
+
+    """
+    if left_version != right_version:
+        raise exceptions.InvalidWorkspaceOperation(
+            f"Workspaces of different versions cannot be combined: {left_version} != {right_version}"
+        )
+    return left_version
+
+
+def _join_channels(join, left_channels, right_channels):
+    """
+    Join two workspace channel specifications.
+
+    Raises:
+      ~pyhf.exceptions.InvalidWorkspaceOperation: Channel specifications are incompatible.
+
+    Args:
+        join (`str`): The join operation to apply. See ~pyhf.workspace.Workspace for valid join operations.
+        left_channels (`list`): The left channel specification.
+        right_channels (`list`): The right channel specification.
+
+    Returns:
+        :obj:`list`: A joined list of channels. Each channel follows the :obj:`defs.json#channel` `schema <https://scikit-hep.org/pyhf/likelihood.html#channel>`__
+
+    """
+
+    joined_channels = _join_items(join, left_channels, right_channels)
+    if join == 'none':
+        common_channels = set(c['name'] for c in left_channels).intersection(
+            c['name'] for c in right_channels
+        )
+        if common_channels:
+            raise exceptions.InvalidWorkspaceOperation(
+                f"Workspaces cannot have any channels in common with the same name: {common_channels}. You can also try a different join operation: {Workspace.valid_joins}."
+            )
+
+    elif join == 'outer':
+        counted_channels = collections.Counter(
+            channel['name'] for channel in joined_channels
+        )
+        incompatible_channels = [
+            channel for channel, count in counted_channels.items() if count > 1
+        ]
+        if incompatible_channels:
+            raise exceptions.InvalidWorkspaceOperation(
+                f"Workspaces cannot have channels in common with incompatible structure: {incompatible_channels}. You can also try a different join operation: {Workspace.valid_joins}."
+            )
+    return joined_channels
+
+
+def _join_observations(join, left_observations, right_observations):
+    """
+    Join two workspace observation specifications.
+
+    Raises:
+      ~pyhf.exceptions.InvalidWorkspaceOperation: Observation specifications are incompatible.
+
+    Args:
+        join (`str`): The join operation to apply. See ~pyhf.workspace.Workspace for valid join operations.
+        left_observations (`list`): The left observation specification.
+        right_observations (`list`): The right observation specification.
+
+    Returns:
+        :obj:`list`: A joined list of observations. Each observation follows the :obj:`defs.json#observation` `schema <https://scikit-hep.org/pyhf/likelihood.html#observations>`__
+
+    """
+    joined_observations = _join_items(join, left_observations, right_observations)
+    if join == 'none':
+        common_observations = set(
+            obs['name'] for obs in left_observations
+        ).intersection(obs['name'] for obs in right_observations)
+        if common_observations:
+            raise exceptions.InvalidWorkspaceOperation(
+                f"Workspaces cannot have any observations in common with the same name: {common_observations}. You can also try a different join operation: {Workspace.valid_joins}."
+            )
+
+    elif join == 'outer':
+        counted_observations = collections.Counter(
+            observation['name'] for observation in joined_observations
+        )
+        incompatible_observations = [
+            observation
+            for observation, count in counted_observations.items()
+            if count > 1
+        ]
+        if incompatible_observations:
+            raise exceptions.InvalidWorkspaceOperation(
+                f"Workspaces cannot have observations in common with incompatible structure: {incompatible_observations}. You can also try a different join operation: {Workspace.valid_joins}."
+            )
+    return joined_observations
+
+
+def _join_parameter_configs(measurement_name, left_parameters, right_parameters):
+    """
+    Join two measurement parameter config specifications.
+
+    Only uses by :method:`_join_measurements` when join='outer'.
+
+    Raises:
+      ~pyhf.exceptions.InvalidWorkspaceOperation: Parameter configuration specifications are incompatible.
+
+    Args:
+        measurement_name (`str`): The name of the measurement being joined (a detail for raising exceptions correctly)
+        left_parameters (`list`): The left parameter configuration specification.
+        right_parameters (`list`): The right parameter configuration specification.
+
+    Returns:
+        :obj:`list`: A joined list of parameter configurations. Each parameter configuration follows the :obj:`defs.json#config` schema
+
+    """
+    joined_parameter_configs = _join_items('outer', left_parameters, right_parameters)
+    counted_parameter_configs = collections.Counter(
+        parameter['name'] for parameter in joined_parameter_configs
+    )
+    incompatible_parameter_configs = [
+        parameter for parameter, count in counted_parameter_configs.items() if count > 1
+    ]
+    if incompatible_parameter_configs:
+        raise exceptions.InvalidWorkspaceOperation(
+            f"Workspaces cannot have a measurement ({measurement_name}) with incompatible parameter configs: {incompatible_parameter_configs}. You can also try a different join operation: {Workspace.valid_joins}."
+        )
+    return joined_parameter_configs
+
+
+def _join_measurements(join, left_measurements, right_measurements):
+    """
+    Join two workspace measurement specifications.
+
+    Raises:
+      ~pyhf.exceptions.InvalidWorkspaceOperation: Measurement specifications are incompatible.
+
+    Args:
+        join (`str`): The join operation to apply. See ~pyhf.workspace.Workspace for valid join operations.
+        left_measurements (`list`): The left measurement specification.
+        right_measurements (`list`): The right measurement specification.
+
+    Returns:
+        :obj:`list`: A joined list of measurements. Each measurement follows the :obj:`defs.json#measurement` `schema <https://scikit-hep.org/pyhf/likelihood.html#measurements>`__
+
+    """
+    joined_measurements = _join_items(join, left_measurements, right_measurements)
+    if join == 'none':
+        common_measurements = set(
+            meas['name'] for meas in left_measurements
+        ).intersection(meas['name'] for meas in right_measurements)
+        if common_measurements:
+            raise exceptions.InvalidWorkspaceOperation(
+                f"Workspaces cannot have any measurements in common with the same name: {common_measurements}. You can also try a different join operation: {Workspace.valid_joins}."
+            )
+
+    elif join == 'outer':
+        # need to store a mapping of measurement name to all measurement objects with that name
+        _measurement_mapping = {}
+        for measurement in joined_measurements:
+            _measurement_mapping.setdefault(measurement['name'], []).append(measurement)
+        # first check for incompatible POI
+        # then merge parameter configs
+        incompatible_poi = [
+            measurement_name
+            for measurement_name, measurements in _measurement_mapping.items()
+            if len(set(measurement['config']['poi'] for measurement in measurements))
+            > 1
+        ]
+        if incompatible_poi:
+            raise exceptions.InvalidWorkspaceOperation(
+                f"Workspaces cannot have the same measurements with incompatible POI: {incompatible_poi}."
+            )
+
+        joined_measurements = []
+        for measurement_name, measurements in _measurement_mapping.items():
+            if len(measurements) != 1:
+                new_measurement = {
+                    'name': measurement_name,
+                    'config': {
+                        'poi': measurements[0]['config']['poi'],
+                        'parameters': _join_parameter_configs(
+                            measurement_name,
+                            *[
+                                measurement['config']['parameters']
+                                for measurement in measurements
+                            ],
+                        ),
+                    },
+                }
+            else:
+                new_measurement = measurements[0]
+            joined_measurements.append(new_measurement)
+    return joined_measurements
+
+
 class Workspace(_ChannelSummaryMixin, dict):
     """
     A JSON-serializable object that is built from an object that follows the :obj:`workspace.json` `schema <https://scikit-hep.org/pyhf/likelihood.html#workspace>`__.
     """
 
+    valid_joins = ['none', 'outer', 'left outer', 'right outer']
+
     def __init__(self, spec, **config_kwargs):
         """Workspaces hold the model, data and measurements."""
         super(Workspace, self).__init__(spec, channels=spec['channels'])
@@ -68,9 +310,9 @@ def get_measurement(self, **config_kwargs):
           ~pyhf.exceptions.InvalidMeasurement: If the measurement was not found
 
         Args:
-            poi_name (str): The name of the parameter of interest to create a new measurement from
-            measurement_name (str): The name of the measurement to use
-            measurement_index (int): The index of the measurement to use
+            poi_name (`str`): The name of the parameter of interest to create a new measurement from
+            measurement_name (`str`): The name of the measurement to use
+            measurement_index (`int`): The index of the measurement to use
 
         Returns:
             :obj:`dict`: A measurement object adhering to the schema defs.json#/definitions/measurement
@@ -275,7 +517,7 @@ def _prune_and_rename(
             ],
             'observations': [
                 dict(
-                    observation,
+                    copy.deepcopy(observation),
                     name=rename_channels.get(observation['name'], observation['name']),
                 )
                 for observation in self['observations']
@@ -337,7 +579,7 @@ def rename(self, modifiers={}, samples={}, channels={}, measurements={}):
         )
 
     @classmethod
-    def combine(cls, left, right):
+    def combine(cls, left, right, join='none'):
         """
         Return a new workspace specification that is the combination of the two workspaces.
 
@@ -362,74 +604,34 @@ def combine(cls, left, right):
         Args:
             left (~pyhf.workspace.Workspace): A workspace
             right (~pyhf.workspace.Workspace): Another workspace
+            join (:obj:`str`): How to join the two workspaces. Pick from "none", "outer", "left outer", or "right outer".
 
         Returns:
             ~pyhf.workspace.Workspace: A new combined workspace object
 
         """
-        common_channels = set(left.channels).intersection(right.channels)
-        if common_channels:
-            raise exceptions.InvalidWorkspaceOperation(
-                "Workspaces cannot have any channels in common: {}".format(
-                    common_channels
-                )
-            )
-
-        common_measurements = set(left.measurement_names).intersection(
-            right.measurement_names
-        )
-        incompatible_poi = [
-            left.get_measurement(measurement_name=m)['config']['poi']
-            != right.get_measurement(measurement_name=m)['config']['poi']
-            for m in common_measurements
-        ]
-        if any(incompatible_poi):
-            raise exceptions.InvalidWorkspaceOperation(
-                "Workspaces cannot have any measurements with incompatible POI: {}".format(
-                    [
-                        m
-                        for m, i in zip(common_measurements, incompatible_poi)
-                        if incompatible_poi
-                    ]
-                )
+        if join not in Workspace.valid_joins:
+            raise ValueError(
+                f"Workspaces must be joined using one of the valid join operations ({Workspace.valid_joins}); not {join}"
             )
-        if left.version != right.version:
-            raise exceptions.InvalidWorkspaceOperation(
-                "Workspaces of different versions cannot be combined: {} != {}".format(
-                    left.version, right.version
-                )
+        if join in ['left outer', 'right outer']:
+            log.warning(
+                "You are using an unsafe join operation. This will silence exceptions that might be raised during a normal 'outer' operation."
             )
 
-        left_measurements = [
-            left.get_measurement(measurement_name=m)
-            for m in set(left.measurement_names) - set(common_measurements)
-        ]
-        right_measurements = [
-            right.get_measurement(measurement_name=m)
-            for m in set(right.measurement_names) - set(common_measurements)
-        ]
-        merged_measurements = [
-            dict(
-                name=m,
-                config=dict(
-                    poi=left.get_measurement(measurement_name=m)['config']['poi'],
-                    parameters=(
-                        left.get_measurement(measurement_name=m)['config']['parameters']
-                        + right.get_measurement(measurement_name=m)['config'][
-                            'parameters'
-                        ]
-                    ),
-                ),
-            )
-            for m in common_measurements
-        ]
+        new_version = _join_versions(join, left['version'], right['version'])
+        new_channels = _join_channels(join, left['channels'], right['channels'])
+        new_observations = _join_observations(
+            join, left['observations'], right['observations']
+        )
+        new_measurements = _join_measurements(
+            join, left['measurements'], right['measurements']
+        )
 
         newspec = {
-            'channels': left['channels'] + right['channels'],
-            'measurements': (
-                left_measurements + right_measurements + merged_measurements
-            ),
-            'observations': left['observations'] + right['observations'],
-            'version': left['version'],
+            'channels': new_channels,
+            'measurements': new_measurements,
+            'observations': new_observations,
+            'version': new_version,
         }
         return Workspace(newspec)


Test patch:
diff --git a/tests/test_workspace.py b/tests/test_workspace.py
--- a/tests/test_workspace.py
+++ b/tests/test_workspace.py
@@ -67,8 +67,9 @@ def test_get_measurement_fake(workspace_factory):
 
 def test_get_measurement_nonexist(workspace_factory):
     w = workspace_factory()
-    with pytest.raises(pyhf.exceptions.InvalidMeasurement):
+    with pytest.raises(pyhf.exceptions.InvalidMeasurement) as excinfo:
         w.get_measurement(measurement_name='nonexistent_measurement')
+    assert 'nonexistent_measurement' in str(excinfo.value)
 
 
 def test_get_workspace_measurement_priority(workspace_factory):
@@ -275,28 +276,334 @@ def test_rename_measurement(workspace_factory):
     assert renamed in new_ws.measurement_names
 
 
-def test_combine_workspace_same_channels(workspace_factory):
+@pytest.fixture(scope='session')
+def join_items():
+    left = [{'name': 'left', 'key': 'value'}, {'name': 'common', 'key': 'left'}]
+    right = [{'name': 'right', 'key': 'value'}, {'name': 'common', 'key': 'right'}]
+    return (left, right)
+
+
+def test_join_items_none(join_items):
+    left_items, right_items = join_items
+    joined = pyhf.workspace._join_items('none', left_items, right_items, key='name')
+    assert all(left in joined for left in left_items)
+    assert all(right in joined for right in right_items)
+
+
+def test_join_items_outer(join_items):
+    left_items, right_items = join_items
+    joined = pyhf.workspace._join_items('outer', left_items, right_items, key='name')
+    assert all(left in joined for left in left_items)
+    assert all(right in joined for right in right_items)
+
+
+def test_join_items_left_outer(join_items):
+    left_items, right_items = join_items
+    joined = pyhf.workspace._join_items(
+        'left outer', left_items, right_items, key='name'
+    )
+    assert all(left in joined for left in left_items)
+    assert not all(right in joined for right in right_items)
+
+
+def test_join_items_right_outer(join_items):
+    left_items, right_items = join_items
+    joined = pyhf.workspace._join_items(
+        'right outer', left_items, right_items, key='name'
+    )
+    assert not all(left in joined for left in left_items)
+    assert all(right in joined for right in right_items)
+
+
+@pytest.mark.parametrize("join", ['none', 'outer'])
+def test_combine_workspace_same_channels_incompatible_structure(
+    workspace_factory, join
+):
     ws = workspace_factory()
-    new_ws = ws.rename(channels={'channel2': 'channel3'})
+    new_ws = ws.rename(
+        channels={'channel2': 'channel3'},
+        samples={'signal': 'signal_other'},
+        measurements={'GaussExample': 'GaussExample2'},
+    ).prune(measurements=['GammaExample', 'ConstExample', 'LogNormExample'])
     with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
-        pyhf.Workspace.combine(ws, new_ws)
+        pyhf.Workspace.combine(ws, new_ws, join=join)
     assert 'channel1' in str(excinfo.value)
     assert 'channel2' not in str(excinfo.value)
 
 
-def test_combine_workspace_incompatible_poi(workspace_factory):
+@pytest.mark.parametrize("join", ['outer', 'left outer', 'right outer'])
+def test_combine_workspace_same_channels_outer_join(workspace_factory, join):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel2': 'channel3'})
+    combined = pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert 'channel1' in combined.channels
+
+
+@pytest.mark.parametrize("join", ['left outer', 'right outer'])
+def test_combine_workspace_same_channels_outer_join_unsafe(
+    workspace_factory, join, caplog
+):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel2': 'channel3'})
+    pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert 'using an unsafe join operation' in caplog.text
+
+
+@pytest.mark.parametrize("join", ['none', 'outer'])
+def test_combine_workspace_incompatible_poi(workspace_factory, join):
     ws = workspace_factory()
     new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
         measurements=['GammaExample', 'ConstExample', 'LogNormExample']
     )
-    new_ws = ws.rename(
+    new_ws = new_ws.rename(
         modifiers={new_ws.get_measurement()['config']['poi']: 'renamedPOI'}
     )
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation):
-        pyhf.Workspace.combine(ws, new_ws)
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert 'GaussExample' in str(excinfo.value)
+
+
+@pytest.mark.parametrize("join", ['none', 'outer', 'left outer', 'right outer'])
+def test_combine_workspace_diff_version(workspace_factory, join):
+    ws = workspace_factory()
+    ws.version = '1.0.0'
+    new_ws = ws.rename(
+        channels={'channel1': 'channel3', 'channel2': 'channel4'},
+        samples={
+            'background1': 'background3',
+            'background2': 'background4',
+            'signal': 'signal2',
+        },
+        modifiers={
+            'syst1': 'syst4',
+            'bkg1Shape': 'bkg3Shape',
+            'bkg2Shape': 'bkg4Shape',
+        },
+        measurements={
+            'ConstExample': 'OtherConstExample',
+            'LogNormExample': 'OtherLogNormExample',
+            'GaussExample': 'OtherGaussExample',
+            'GammaExample': 'OtherGammaExample',
+        },
+    )
+    new_ws['version'] = '1.2.0'
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert '1.0.0' in str(excinfo.value)
+    assert '1.2.0' in str(excinfo.value)
+
+
+@pytest.mark.parametrize("join", ['none'])
+def test_combine_workspace_duplicate_parameter_configs(workspace_factory, join):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert 'GaussExample' in str(excinfo.value)
+
+
+@pytest.mark.parametrize("join", ['outer', 'left outer', 'right outer'])
+def test_combine_workspace_duplicate_parameter_configs_outer_join(
+    workspace_factory, join
+):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    combined = pyhf.Workspace.combine(ws, new_ws, join=join)
+
+    poi = ws.get_measurement(measurement_name='GaussExample')['config']['poi']
+    ws_parameter_configs = [
+        parameter['name']
+        for parameter in ws.get_measurement(measurement_name='GaussExample')['config'][
+            'parameters'
+        ]
+    ]
+    new_ws_parameter_configs = [
+        parameter['name']
+        for parameter in new_ws.get_measurement(measurement_name='GaussExample')[
+            'config'
+        ]['parameters']
+    ]
+    combined_parameter_configs = [
+        parameter['name']
+        for parameter in combined.get_measurement(measurement_name='GaussExample')[
+            'config'
+        ]['parameters']
+    ]
+
+    assert poi in ws_parameter_configs
+    assert poi in new_ws_parameter_configs
+    assert poi in combined_parameter_configs
+    assert 'lumi' in ws_parameter_configs
+    assert 'lumi' in new_ws_parameter_configs
+    assert 'lumi' in combined_parameter_configs
+    assert len(combined_parameter_configs) == len(set(combined_parameter_configs))
+
+
+def test_combine_workspace_parameter_configs_ordering(workspace_factory):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    assert (
+        ws.get_measurement(measurement_name='GaussExample')['config']['parameters']
+        == new_ws.get_measurement(measurement_name='GaussExample')['config'][
+            'parameters'
+        ]
+    )
+
+
+def test_combine_workspace_observation_ordering(workspace_factory):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    assert ws['observations'][0]['data'] == new_ws['observations'][0]['data']
+
+
+def test_combine_workspace_deepcopied(workspace_factory):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
+        'bounds'
+    ] = [[0.0, 1.0]]
+    new_ws['observations'][0]['data'][0] = -10.0
+    assert (
+        ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
+            'bounds'
+        ]
+        != new_ws.get_measurement(measurement_name='GaussExample')['config'][
+            'parameters'
+        ][0]['bounds']
+    )
+    assert ws['observations'][0]['data'] != new_ws['observations'][0]['data']
+
+
+@pytest.mark.parametrize("join", ['fake join operation'])
+def test_combine_workspace_invalid_join_operation(workspace_factory, join):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    with pytest.raises(ValueError) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert join in str(excinfo.value)
+
+
+@pytest.mark.parametrize("join", ['none'])
+def test_combine_workspace_incompatible_parameter_configs(workspace_factory, join):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
+        'bounds'
+    ] = [[0.0, 1.0]]
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert 'GaussExample' in str(excinfo.value)
+
+
+@pytest.mark.parametrize("join", ['outer'])
+def test_combine_workspace_incompatible_parameter_configs_outer_join(
+    workspace_factory, join
+):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
+        'bounds'
+    ] = [[0.0, 1.0]]
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert 'GaussExample' in str(excinfo.value)
+    assert ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][
+        0
+    ]['name'] in str(excinfo.value)
+    assert new_ws.get_measurement(measurement_name='GaussExample')['config'][
+        'parameters'
+    ][0]['name'] in str(excinfo.value)
+
+
+def test_combine_workspace_incompatible_parameter_configs_left_outer_join(
+    workspace_factory,
+):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
+        'bounds'
+    ] = [[0.0, 1.0]]
+    combined = pyhf.Workspace.combine(ws, new_ws, join='left outer')
+    assert (
+        combined.get_measurement(measurement_name='GaussExample')['config'][
+            'parameters'
+        ][0]
+        == ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][
+            0
+        ]
+    )
+
+
+def test_combine_workspace_incompatible_parameter_configs_right_outer_join(
+    workspace_factory,
+):
+    ws = workspace_factory()
+    new_ws = ws.rename(channels={'channel1': 'channel3', 'channel2': 'channel4'}).prune(
+        measurements=['GammaExample', 'ConstExample', 'LogNormExample']
+    )
+    new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
+        'bounds'
+    ] = [[0.0, 1.0]]
+    combined = pyhf.Workspace.combine(ws, new_ws, join='right outer')
+    assert (
+        combined.get_measurement(measurement_name='GaussExample')['config'][
+            'parameters'
+        ][0]
+        == new_ws.get_measurement(measurement_name='GaussExample')['config'][
+            'parameters'
+        ][0]
+    )
+
+
+@pytest.mark.parametrize("join", ['none', 'outer'])
+def test_combine_workspace_incompatible_observations(workspace_factory, join):
+    ws = workspace_factory()
+    new_ws = ws.rename(
+        channels={'channel1': 'channel3', 'channel2': 'channel4'},
+        samples={
+            'background1': 'background3',
+            'background2': 'background4',
+            'signal': 'signal2',
+        },
+        modifiers={
+            'syst1': 'syst4',
+            'bkg1Shape': 'bkg3Shape',
+            'bkg2Shape': 'bkg4Shape',
+        },
+        measurements={
+            'GaussExample': 'OtherGaussExample',
+            'GammaExample': 'OtherGammaExample',
+            'ConstExample': 'OtherConstExample',
+            'LogNormExample': 'OtherLogNormExample',
+        },
+    )
+    new_ws['observations'][0]['name'] = ws['observations'][0]['name']
+    new_ws['observations'][0]['data'][0] = -10.0
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+        pyhf.Workspace.combine(ws, new_ws, join=join)
+    assert ws['observations'][0]['name'] in str(excinfo.value)
+    assert 'observations' in str(excinfo.value)
 
 
-def test_combine_workspace_diff_version(workspace_factory):
+def test_combine_workspace_incompatible_observations_left_outer(workspace_factory):
     ws = workspace_factory()
     new_ws = ws.rename(
         channels={'channel1': 'channel3', 'channel2': 'channel4'},
@@ -311,18 +618,53 @@ def test_combine_workspace_diff_version(workspace_factory):
             'bkg2Shape': 'bkg4Shape',
         },
         measurements={
+            'GaussExample': 'OtherGaussExample',
+            'GammaExample': 'OtherGammaExample',
             'ConstExample': 'OtherConstExample',
             'LogNormExample': 'OtherLogNormExample',
+        },
+    )
+    new_ws['observations'][0]['name'] = ws['observations'][0]['name']
+    new_ws['observations'][0]['data'][0] = -10.0
+    combined = pyhf.Workspace.combine(ws, new_ws, join='left outer')
+    assert (
+        combined.observations[ws['observations'][0]['name']]
+        == ws['observations'][0]['data']
+    )
+
+
+def test_combine_workspace_incompatible_observations_right_outer(workspace_factory):
+    ws = workspace_factory()
+    new_ws = ws.rename(
+        channels={'channel1': 'channel3', 'channel2': 'channel4'},
+        samples={
+            'background1': 'background3',
+            'background2': 'background4',
+            'signal': 'signal2',
+        },
+        modifiers={
+            'syst1': 'syst4',
+            'bkg1Shape': 'bkg3Shape',
+            'bkg2Shape': 'bkg4Shape',
+        },
+        measurements={
             'GaussExample': 'OtherGaussExample',
             'GammaExample': 'OtherGammaExample',
+            'ConstExample': 'OtherConstExample',
+            'LogNormExample': 'OtherLogNormExample',
         },
     )
-    new_ws.version = '0.0.0'
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation):
-        pyhf.Workspace.combine(ws, new_ws)
+    new_ws['observations'][0]['name'] = ws['observations'][0]['name']
+    new_ws['observations'][0]['data'][0] = -10.0
+    combined = pyhf.Workspace.combine(ws, new_ws, join='right outer')
+    assert (
+        combined.observations[ws['observations'][0]['name']]
+        == new_ws['observations'][0]['data']
+    )
 
 
-def test_combine_workspace(workspace_factory):
+@pytest.mark.parametrize("join", pyhf.Workspace.valid_joins)
+def test_combine_workspace(workspace_factory, join):
     ws = workspace_factory()
     new_ws = ws.rename(
         channels={'channel1': 'channel3', 'channel2': 'channel4'},
@@ -337,17 +679,13 @@ def test_combine_workspace(workspace_factory):
             'bkg2Shape': 'bkg4Shape',
         },
         measurements={
+            'GaussExample': 'OtherGaussExample',
+            'GammaExample': 'OtherGammaExample',
             'ConstExample': 'OtherConstExample',
             'LogNormExample': 'OtherLogNormExample',
         },
     )
-    combined = pyhf.Workspace.combine(ws, new_ws)
+    combined = pyhf.Workspace.combine(ws, new_ws, join=join)
     assert set(combined.channels) == set(ws.channels + new_ws.channels)
     assert set(combined.samples) == set(ws.samples + new_ws.samples)
     assert set(combined.parameters) == set(ws.parameters + new_ws.parameters)
-    combined_measurement = combined.get_measurement(measurement_name='GaussExample')
-    assert len(combined_measurement['config']['parameters']) == len(
-        ws.get_measurement(measurement_name='GaussExample')['config']['parameters']
-    ) + len(
-        new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters']
-    )