Repository: scikit-hep/pyhf

Issue:
Schema validation crashes when running in an environment without internet access
### Summary

In master and the 0.7.0 release candidate, pyhf operations involving model validation will crash in offline environments with a RefResolutionError. This is a common situation e.g. with worker nodes on HTC clusters.
The bug was introduced after 0.6.3, I think in #1753 where the [pre-loading was dropped](https://github.com/scikit-hep/pyhf/pull/1753/files#diff-01a944844c3739d996c27da33c727473ec48ebcac65f16b4001384bc3ae4e725L48).

### OS / Environment

```console
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="http://cern.ch/linux/"
BUG_REPORT_URL="http://cern.ch/linux/"

CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"
```


### Steps to Reproduce

I don't know a good way to prepare the environment to demonstrate this. 
But the below test exposes the attempt by the RefResolver to resolve the schema id through the https URL, and fails against the release candidate/master, but passes in 0.6.3
<!--- Paste your minimal failing Python example code between the quotes below -->
```python (paste below)
from functools import partial

import pytest
import jsonschema
import pyhf

def make_asserting_handler(origin):
    def asserting_handler(*args, **kwargs):
        raise AssertionError(
            f'called URL request handler from {origin} with args={args!r}, kwargs={kwargs!r} '
            'when no call should have been needed'
        )

    return asserting_handler


@pytest.fixture
def no_http_jsonschema_ref_resolving(monkeypatch):
    asserting_handler = make_asserting_handler('handlers')
    handlers = {
        'https': asserting_handler,
        'http': asserting_handler,
    }
    WrappedResolver = partial(jsonschema.RefResolver, handlers=handlers)
    monkeypatch.setattr('jsonschema.RefResolver', WrappedResolver, raising=True)

def test_preloaded_cache(
    no_http_jsonschema_ref_resolving,
):
    spec = {
        'channels': [
            {
                'name': 'singlechannel',
                'samples': [
                    {
                        'name': 'signal',
                        'data': [10],
                        'modifiers': [
                            {'name': 'mu', 'type': 'normfactor', 'data': None}
                        ],
                    },
                    {
                        'name': 'background',
                        'data': [20],
                        'modifiers': [
                            {
                                'name': 'uncorr_bkguncrt',
                                'type': 'shapesys',
                                'data': [30],
                            }
                        ],
                    },
                ],
            }
        ]
    }
    try:
        pyhf.schema.validate(spec, 'model.json')
    except AttributeError:
        pyhf.utils.validate(spec, 'model.json')
                                               
```


### File Upload (optional)

_No response_

### Expected Results

I expect schema validation to succeed without crashing even when there is no network access that allows resolving the https schema-ids.

### Actual Results

```console
jsonschema.exceptions.RefResolutionError: HTTPSConnectionPool(host='scikit-hep.org', port=443): Max retries exceeded with url: /pyhf/schemas/1.0.0/defs.json (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x2b2bb8457c40>: Failed to establish a new connection: [Errno 101] Network is unreachable'))
```


### pyhf Version

```console
pyhf, version 0.7.0rc2
```


### Code of Conduct

- [X] I agree to follow the Code of Conduct


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -40,6 +40,7 @@
             'scrapbook~=0.5.0',
             'jupyter',
             'graphviz',
+            'pytest-socket>=0.2.0',  # c.f. PR #1917
         ]
     )
 )
diff --git a/src/pyhf/schema/__init__.py b/src/pyhf/schema/__init__.py
--- a/src/pyhf/schema/__init__.py
+++ b/src/pyhf/schema/__init__.py
@@ -72,6 +72,8 @@ def __call__(self, new_path: pathlib.Path):
             self (pyhf.schema.Schema): Returns itself (for contextlib management)
         """
         self.orig_path, variables.schemas = variables.schemas, new_path
+        self.orig_cache = dict(variables.SCHEMA_CACHE)
+        variables.SCHEMA_CACHE.clear()
         return self
 
     def __enter__(self):
@@ -85,6 +87,7 @@ def __exit__(self, *args, **kwargs):
             None
         """
         variables.schemas = self.orig_path
+        variables.SCHEMA_CACHE = self.orig_cache
 
     @property
     def path(self):
diff --git a/src/pyhf/schema/loader.py b/src/pyhf/schema/loader.py
--- a/src/pyhf/schema/loader.py
+++ b/src/pyhf/schema/loader.py
@@ -39,3 +39,9 @@ def load_schema(schema_id: str):
             schema = json.load(json_schema)
             variables.SCHEMA_CACHE[schema['$id']] = schema
         return variables.SCHEMA_CACHE[schema['$id']]
+
+
+# pre-populate the cache to avoid network access
+# on first validation in standard usage
+# (not in pyhf.schema.variables to avoid circular imports)
+load_schema(f'{variables.SCHEMA_VERSION}/defs.json')


Test patch:
diff --git a/tests/test_schema.py b/tests/test_schema.py
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -1,6 +1,11 @@
-import pyhf
-import pytest
+import importlib
 import json
+import sys
+
+import pytest
+from pytest_socket import socket_disabled  # noqa: F401
+
+import pyhf
 
 
 @pytest.mark.parametrize('version', ['1.0.0'])
@@ -27,11 +32,20 @@ def test_schema_callable():
     assert callable(pyhf.schema)
 
 
-def test_schema_changeable(datadir, monkeypatch):
+@pytest.fixture
+def self_restoring_schema_globals():
+    old_path = pyhf.schema.path
+    old_cache = dict(pyhf.schema.variables.SCHEMA_CACHE)
+    yield old_path, old_cache
+    pyhf.schema(old_path)
+    pyhf.schema.variables.SCHEMA_CACHE = old_cache
+
+
+def test_schema_changeable(datadir, monkeypatch, self_restoring_schema_globals):
     monkeypatch.setattr(
         pyhf.schema.variables, 'schemas', pyhf.schema.variables.schemas, raising=True
     )
-    old_path = pyhf.schema.path
+    old_path, old_cache = self_restoring_schema_globals
     new_path = datadir / 'customschema'
 
     with pytest.raises(pyhf.exceptions.SchemaNotFound):
@@ -40,36 +54,47 @@ def test_schema_changeable(datadir, monkeypatch):
     pyhf.schema(new_path)
     assert old_path != pyhf.schema.path
     assert new_path == pyhf.schema.path
+    assert pyhf.schema.variables.SCHEMA_CACHE is not old_cache
+    assert len(pyhf.schema.variables.SCHEMA_CACHE) == 0
     assert pyhf.Workspace(json.load(open(new_path / 'custom.json')))
-    pyhf.schema(old_path)
+    assert len(pyhf.schema.variables.SCHEMA_CACHE) == 1
 
 
-def test_schema_changeable_context(datadir, monkeypatch):
+def test_schema_changeable_context(datadir, monkeypatch, self_restoring_schema_globals):
     monkeypatch.setattr(
         pyhf.schema.variables, 'schemas', pyhf.schema.variables.schemas, raising=True
     )
-    old_path = pyhf.schema.path
+    old_path, old_cache = self_restoring_schema_globals
     new_path = datadir / 'customschema'
 
     assert old_path == pyhf.schema.path
     with pyhf.schema(new_path):
         assert old_path != pyhf.schema.path
         assert new_path == pyhf.schema.path
+        assert pyhf.schema.variables.SCHEMA_CACHE is not old_cache
+        assert len(pyhf.schema.variables.SCHEMA_CACHE) == 0
         assert pyhf.Workspace(json.load(open(new_path / 'custom.json')))
+        assert len(pyhf.schema.variables.SCHEMA_CACHE) == 1
     assert old_path == pyhf.schema.path
+    assert old_cache == pyhf.schema.variables.SCHEMA_CACHE
 
 
-def test_schema_changeable_context_error(datadir, monkeypatch):
+def test_schema_changeable_context_error(
+    datadir, monkeypatch, self_restoring_schema_globals
+):
     monkeypatch.setattr(
         pyhf.schema.variables, 'schemas', pyhf.schema.variables.schemas, raising=True
     )
-    old_path = pyhf.schema.path
+    old_path, old_cache = self_restoring_schema_globals
     new_path = datadir / 'customschema'
 
     with pytest.raises(ZeroDivisionError):
         with pyhf.schema(new_path):
+            # this populates the current cache
+            pyhf.Workspace(json.load(open(new_path / 'custom.json')))
             raise ZeroDivisionError()
     assert old_path == pyhf.schema.path
+    assert old_cache == pyhf.schema.variables.SCHEMA_CACHE
 
 
 def test_no_channels():
@@ -567,3 +592,48 @@ def test_patchset_fail(datadir, patchset_file):
     patchset = json.load(open(datadir.joinpath(patchset_file)))
     with pytest.raises(pyhf.exceptions.InvalidSpecification):
         pyhf.schema.validate(patchset, 'patchset.json')
+
+
+def test_defs_always_cached(
+    socket_disabled,  # noqa: F811
+    isolate_modules,
+):
+    """
+    Schema definitions should always be loaded from the local files and cached at first import.
+
+    Otherwise pyhf will crash in contexts where the jsonschema.RefResolver cannot lookup the definition by the schema-id
+    (e.g. a cluster node without network access).
+    """
+    modules_to_clear = [name for name in sys.modules if name.split('.')[0] == 'pyhf']
+    for module_name in modules_to_clear:
+        del sys.modules[module_name]
+    pyhf = importlib.import_module('pyhf')
+
+    spec = {
+        'channels': [
+            {
+                'name': 'singlechannel',
+                'samples': [
+                    {
+                        'name': 'signal',
+                        'data': [10],
+                        'modifiers': [
+                            {'name': 'mu', 'type': 'normfactor', 'data': None}
+                        ],
+                    },
+                    {
+                        'name': 'background',
+                        'data': [20],
+                        'modifiers': [
+                            {
+                                'name': 'uncorr_bkguncrt',
+                                'type': 'shapesys',
+                                'data': [30],
+                            }
+                        ],
+                    },
+                ],
+            }
+        ]
+    }
+    pyhf.schema.validate(spec, 'model.json')  # may try to access network and fail