Repository: spacetelescope/acstools (language: python)

Issue:
TST: Replace ssbjenkins
How are we going to test?


Discussion / hints:
Apparently it is back, but with "a few minutes" delay. We'll see.

Still, we should still see if we can just use Actions.
It does kick off after a delay but no one can see the test results unless they are in internal network. Kind of defeats the purpose.

Fix patch:
diff --git a/.github/workflows/ci_workflows.yml b/.github/workflows/ci_workflows.yml
index 1f64b185..2b1b0f8e 100644
--- a/.github/workflows/ci_workflows.yml
+++ b/.github/workflows/ci_workflows.yml
@@ -8,6 +8,9 @@ on:
     # * is a special character in YAML so you have to quote this string
     - cron: '0 6 * * 2'
 
+env:
+  jref: "https://ssb.stsci.edu/cdbs/jref"
+
 jobs:
   initial_checks:
     name: Mandatory checks before CI
@@ -55,6 +58,47 @@ jobs:
         python -m pip install --upgrade bandit
         bandit -r . -c .bandit.yaml
 
+  tests:
+    runs-on: ubuntu-latest
+    needs: initial_checks
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+    - name: Install and build
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install -e .[test,all]
+    - name: Test
+      run: pytest --remote-data -v
+
+  dev_deps_tests:
+    runs-on: ubuntu-latest
+    needs: initial_checks
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Install and build
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps
+        python -m pip install -e .[test,all]
+    - name: Test
+      run: pytest --remote-data -v
+
   link_check:
     runs-on: ubuntu-latest
     needs: initial_checks
diff --git a/Jenkinsfile b/Jenkinsfile
deleted file mode 100644
index 996d81ac..00000000
--- a/Jenkinsfile
+++ /dev/null
@@ -1,26 +0,0 @@
-// Obtain files from source control system.
-if (utils.scm_checkout()) return
-
-// Define each build configuration, copying and overriding values as necessary.
-bc1 = new BuildConfig()
-bc1.nodetype = "linux"
-bc1.name = "release"
-// Would be nice if Jenkins can access /grp/hst/cdbs/xxxx directly.
-bc1.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory']
-bc1.conda_channels = ['http://ssb.stsci.edu/astroconda']
-bc1.conda_packages = ['python=3.7']
-bc1.build_cmds = ["pip install -e .[test,all]"]
-bc1.test_cmds = ["pytest --basetemp=tests_output --junitxml results.xml --bigdata -v"]
-bc1.failedUnstableThresh = 1
-bc1.failedFailureThresh = 6
-
-// Run with astropy dev and Python 3.8
-bc2 = utils.copy(bc1)
-bc2.name = "dev"
-bc2.conda_packages[0] = "python=3.8"
-bc2.build_cmds = ["pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps",
-                  "pip install -e .[test,all]"]
-
-// Iterate over configurations that define the (distibuted) build matrix.
-// Spawn a host of the given nodetype for each combination and run in parallel.
-utils.run([bc1, bc2])
diff --git a/JenkinsfileRT b/JenkinsfileRT
deleted file mode 100644
index 4b4f91cd..00000000
--- a/JenkinsfileRT
+++ /dev/null
@@ -1,33 +0,0 @@
-// Obtain files from source control system.
-// [skip ci] and [ci skip] have no effect here.
-if (utils.scm_checkout(['skip_disable':true])) return
-
-// Allow modification of the job configuration, affects all relevant build configs.
-// Pass this object in the argument list to the`run()` function below to apply these settings to the job's execution.
-jobconfig = new JobConfig()
-jobconfig.post_test_summary = true
-
-// Run nightly tests, which include the slow ones.
-bc = new BuildConfig()
-bc.nodetype = "RHEL-6"
-bc.name = "release"
-bc.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory',
-               'jref=/grp/hst/cdbs/jref/']
-bc.conda_channels = ['http://ssb.stsci.edu/astroconda']
-bc.conda_packages = ['python=3.6']
-bc.build_cmds = ["pip install -e .[test,all]"]
-bc.test_cmds = ["pytest --basetemp=tests_output --junitxml results.xml --bigdata --slow -v"]
-bc.failedUnstableThresh = 1
-bc.failedFailureThresh = 6
-
-// Astropy dev and Python 3.8
-bc1 = utils.copy(bc)
-bc1.name = "dev"
-bc1.conda_packages[0] = "python=3.8"
-bc1.build_cmds = ["pip install git+https://github.com/astropy/astropy.git#egg=astropy --upgrade --no-deps",
-                  "pip install -e .[test,all]"]
-
-// Iterate over configurations that define the (distributed) build matrix.
-// Spawn a host (or workdir) for each combination and run in parallel.
-// Also apply the job configuration defined in `jobconfig` above.
-utils.run([bc, bc1, jobconfig])
diff --git a/setup.cfg b/setup.cfg
index 8d879be9..94b2d807 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -48,12 +48,16 @@ all =
 test =
     pytest
     pytest-astropy-header
+    pytest-remotedata
     ci-watson
 docs =
     sphinx-automodapi
 
 [options.package_data]
 acstools = data/*
+acstools.tests =
+    data/input/*.fits
+    data/truth/*.fits
 
 [options.entry_points]
 console_scripts =
@@ -67,7 +71,6 @@ testpaths = "acstools" "doc"
 norecursedirs = build doc/build
 astropy_header = true
 xfail_strict = true
-junit_family=xunit2
 
 [flake8]
 # Ignoring these for now:


Test patch:
diff --git a/acstools/tests/.gitattributes b/acstools/tests/.gitattributes
new file mode 100644
index 00000000..20739a9d
--- /dev/null
+++ b/acstools/tests/.gitattributes
@@ -0,0 +1,1 @@
+data/** filter=lfs diff=lfs merge=lfs -text
diff --git a/acstools/tests/__init__.py b/acstools/tests/__init__.py
index fafc36df..7c5c17ab 100644
--- a/acstools/tests/__init__.py
+++ b/acstools/tests/__init__.py
@@ -1,5 +1,1 @@
 """Unit tests for ACSTOOLS."""
-
-# Currently, tests are on a different system due to large data files being
-# used. In the future, if there are tests that can be added to the package,
-# they should go in this directory.
diff --git a/acstools/tests/data/input/ja0x03ojq_flt.fits b/acstools/tests/data/input/ja0x03ojq_flt.fits
new file mode 100644
index 00000000..71ae38ca
--- /dev/null
+++ b/acstools/tests/data/input/ja0x03ojq_flt.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b1b6877bd0aa7ddb08956a833c6d355f4f1f1f49b2d909bf91b18d6d25e5b62
+size 41981760
diff --git a/acstools/tests/data/input/jb5g05ubq_flt.fits b/acstools/tests/data/input/jb5g05ubq_flt.fits
new file mode 100644
index 00000000..3372cd46
--- /dev/null
+++ b/acstools/tests/data/input/jb5g05ubq_flt.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50599bf8a56dd4856409c73136ec8ccace2a2412d593402f39900c414dc902fb
+size 167837760
diff --git a/acstools/tests/data/input/jc2z03cvq_blv_tmp.fits b/acstools/tests/data/input/jc2z03cvq_blv_tmp.fits
new file mode 100644
index 00000000..b0e2b416
--- /dev/null
+++ b/acstools/tests/data/input/jc2z03cvq_blv_tmp.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:596ac0365ec869d6f27f4c6a10a33578f139908c43a92d446311770fdffabedb
+size 167837760
diff --git a/acstools/tests/data/input/jc5001soq_flt.fits b/acstools/tests/data/input/jc5001soq_flt.fits
new file mode 100644
index 00000000..1fd07538
--- /dev/null
+++ b/acstools/tests/data/input/jc5001soq_flt.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a68b7bce05aa223b2f40bd8833938cbed22833294ea8cccd91c569119d129bc
+size 41944320
diff --git a/acstools/tests/data/input/jc8m10syq_flc.fits b/acstools/tests/data/input/jc8m10syq_flc.fits
new file mode 100644
index 00000000..f484c344
--- /dev/null
+++ b/acstools/tests/data/input/jc8m10syq_flc.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31169623b2e8af6f9cae420dd24788096234f946aca43e95d66a3bc3a35f5634
+size 167964480
diff --git a/acstools/tests/data/input/jc8o04ghq_flt.fits b/acstools/tests/data/input/jc8o04ghq_flt.fits
new file mode 100644
index 00000000..5aeeffb1
--- /dev/null
+++ b/acstools/tests/data/input/jc8o04ghq_flt.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f84c3b83b852ae5172d06dbff2548ccc094ee1a7948392f05fd72cb5c8e0fe
+size 167837760
diff --git a/acstools/tests/data/input/jc8o04ghq_mask1.fits b/acstools/tests/data/input/jc8o04ghq_mask1.fits
new file mode 100644
index 00000000..e2004968
--- /dev/null
+++ b/acstools/tests/data/input/jc8o04ghq_mask1.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77e4f669234e3de5afb528e77d7602b93efa966127e84eb912768e51a5daa9a4
+size 33595200
diff --git a/acstools/tests/data/input/jc8o04ghq_mask2.fits b/acstools/tests/data/input/jc8o04ghq_mask2.fits
new file mode 100644
index 00000000..2c812912
--- /dev/null
+++ b/acstools/tests/data/input/jc8o04ghq_mask2.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db2e2ab5c0a5336da9d2080679697bf302ce15184bbe5648d2058972fffb5e40
+size 33595200
diff --git a/acstools/tests/data/truth/ja0x03ojq_flt_ref.fits b/acstools/tests/data/truth/ja0x03ojq_flt_ref.fits
new file mode 100644
index 00000000..baeba9bd
--- /dev/null
+++ b/acstools/tests/data/truth/ja0x03ojq_flt_ref.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f147ff214763943ee1d06c4ba5930cd6d56c92560970b9bdddb9ac335a0cf7f
+size 41981760
diff --git a/acstools/tests/data/truth/jb5g05ubq_flt_ref.fits b/acstools/tests/data/truth/jb5g05ubq_flt_ref.fits
new file mode 100644
index 00000000..55634513
--- /dev/null
+++ b/acstools/tests/data/truth/jb5g05ubq_flt_ref.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f6de0ac8ed5a4d9ac2ae450722ec320e17a79108deff1e5c976072d76c09a3d
+size 167837760
diff --git a/acstools/tests/data/truth/jc2z03cvq_blv_tmp_ref.fits b/acstools/tests/data/truth/jc2z03cvq_blv_tmp_ref.fits
new file mode 100644
index 00000000..c28c5586
--- /dev/null
+++ b/acstools/tests/data/truth/jc2z03cvq_blv_tmp_ref.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6c092076af589b6e639f6c19c178a863f6055dac9b27a9f17f52159ae90af59
+size 167837760
diff --git a/acstools/tests/data/truth/jc5001soq_flt_ref.fits b/acstools/tests/data/truth/jc5001soq_flt_ref.fits
new file mode 100644
index 00000000..1a14aa8d
--- /dev/null
+++ b/acstools/tests/data/truth/jc5001soq_flt_ref.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d3e30fbb64330100ba4c49a9ed20d8f4c56b6fbff05442e6c66edea305a553d
+size 41944320
diff --git a/acstools/tests/data/truth/jc8m10syq_flc_ref.fits b/acstools/tests/data/truth/jc8m10syq_flc_ref.fits
new file mode 100644
index 00000000..36f6fd07
--- /dev/null
+++ b/acstools/tests/data/truth/jc8m10syq_flc_ref.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d991090716ea28cfa818f1338b96e9d69f931253c0380adf8427e24b6b6c4c8
+size 167964480
diff --git a/acstools/tests/data/truth/jc8o04ghq_flt_ref.fits b/acstools/tests/data/truth/jc8o04ghq_flt_ref.fits
new file mode 100644
index 00000000..73a07218
--- /dev/null
+++ b/acstools/tests/data/truth/jc8o04ghq_flt_ref.fits
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1575f7fedb6efe30300eb16e7bdcdd51ebe24404a89d12c04f9ec1bedb4d26d0
+size 167837760
diff --git a/acstools/tests/helpers.py b/acstools/tests/helpers.py
index 4b205eef..92f060f1 100644
--- a/acstools/tests/helpers.py
+++ b/acstools/tests/helpers.py
@@ -1,13 +1,14 @@
 """ACSTOOLS regression test helpers."""
 
 import os
+import shutil
 
 import pytest
-from ci_watson.artifactory_helpers import get_bigdata
-from ci_watson.hst_helpers import raw_from_asn, ref_from_image, download_crds
+from ci_watson.hst_helpers import ref_from_image, download_crds
 
 from astropy.io import fits
 from astropy.io.fits import FITSDiff
+from astropy.utils.data import get_pkg_data_filename
 
 __all__ = ['calref_from_image', 'BaseACSTOOLS']
 
@@ -66,12 +67,10 @@ def calref_from_image(input_image):
 
 # Base class for actual tests.
 # NOTE: Named in a way so pytest will not pick them up here.
-# NOTE: bigdata marker requires TEST_BIGDATA environment variable to
-#       point to a valid big data directory, whether locally or on Artifactory.
-# NOTE: envopt would point tests to "dev" or "stable".
+# NOTE: remote_data because reference files need internet connection
 # NOTE: _jail fixture ensures each test runs in a clean tmpdir.
-@pytest.mark.bigdata
-@pytest.mark.usefixtures('_jail', 'envopt')
+@pytest.mark.remote_data
+@pytest.mark.usefixtures('_jail')
 class BaseACSTOOLS:
     # Timeout in seconds for file downloads.
     timeout = 30
@@ -83,89 +82,52 @@ class BaseACSTOOLS:
     # To be defined by test class in actual test modules.
     detector = ''
 
-    @pytest.fixture(autouse=True)
-    def setup_class(self, envopt):
-        """
-        Class-level setup that is done at the beginning of the test.
-
-        Parameters
-        ----------
-        envopt : {'dev', 'stable'}
-            This is a ``pytest`` fixture that defines the test
-            environment in which input and truth files reside.
-
-        """
-        self.env = envopt
-
-    def get_input_file(self, filename):
-        """
-        Copy input file (ASN, RAW, etc) into the working directory.
-        If ASN is given, RAW files in the ASN table is also copied.
+    def get_input_file(self, filename, skip_ref=False):
+        """Copy input file into the working directory.
         The associated CRDS reference files are also copied or
-        downloaded, if necessary.
-
-        Data directory layout for CALCOS::
+        downloaded, if desired.
 
-            detector/
-                input/
-                truth/
+        Input file is from ``git lfs``, while the reference files from CDBS.
 
         Parameters
         ----------
         filename : str
-            Filename of the ASN/RAW/etc to copy over, along with its
-            associated files.
+            Filename to copy over, along with its reference files, if desired.
+
+        skip_ref : bool
+            Skip downloading reference files for the given input.
 
         """
-        # Copy over main input file.
-        dest = get_bigdata('scsb-acstools', self.env, self.detector, 'input',
-                           filename)
-
-        # For historical reason, need to remove ".orig" suffix if it exists.
-        if filename.endswith('.orig'):
-            newfilename = filename.rstrip('.orig')
-            os.rename(filename, newfilename)
-            filename = newfilename
-
-        if filename.endswith('_asn.fits'):
-            all_raws = raw_from_asn(filename)
-            for raw in all_raws:  # Download RAWs in ASN.
-                get_bigdata('scsb-acstools', self.env, self.detector, 'input',
-                            raw)
-        else:
-            all_raws = [filename]
-
-        first_pass = ('JENKINS_URL' in os.environ and
-                      'ssbjenkins' in os.environ['JENKINS_URL'])
-
-        for raw in all_raws:
-            ref_files = calref_from_image(raw)
-
-            for ref_file in ref_files:
-                # Special reference files that live with inputs.
-                if ('$' not in ref_file and
-                        os.path.basename(ref_file) == ref_file):
-                    get_bigdata('scsb-acstools', self.env, self.detector,
-                                'input', ref_file)
-                    continue
-
-                # Jenkins cannot see Central Storage on push event,
-                # and somehow setting, say, jref to "." does not work anymore.
-                # So, we need this hack.
-                if '$' in ref_file and first_pass:
-                    first_pass = False
-                    if not os.path.isdir('/grp/hst/cdbs'):
-                        ref_path = os.path.dirname(dest) + os.sep
-                        var = ref_file.split('$')[0]
-                        os.environ[var] = ref_path  # hacky hack hack
-
-                # Download reference files, if needed only.
-                download_crds(ref_file, timeout=self.timeout)
+        # Copy over main input file: The way calibration code was written,
+        # it usually assumes input is in the working directory.
+        src = get_pkg_data_filename(
+            os.path.join('data', 'input', filename), package='acstools.tests',
+            show_progress=False, remote_timeout=self.timeout)
+        dst = os.path.join(os.curdir, filename)
+        shutil.copyfile(src, dst)
+
+        if skip_ref:
+            return
+
+        ref_files = calref_from_image(src)
+        for ref_file in ref_files:
+            # Special reference files that live with inputs.
+            if ('$' not in ref_file and
+                    os.path.basename(ref_file) == ref_file):
+                refsrc = get_pkg_data_filename(
+                    os.path.join('data', 'input', ref_file),
+                    package='acstools.tests',
+                    show_progress=False, remote_timeout=self.timeout)
+                refdst = os.path.join(os.curdir, ref_file)
+                shutil.copyfile(refsrc, refdst)
+                continue
+
+            # Download reference files, if needed only.
+            download_crds(ref_file, timeout=self.timeout)
 
     def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
                         ignore_keywords_overwrite=None):
-        """
-        Compare ACSTOOLS output with "truth" using ``fitsdiff``.
+        """Compare ACSTOOLS output with "truth" using ``fitsdiff``.
 
         Parameters
         ----------
@@ -203,9 +165,11 @@ def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
             ignore_keywords = ignore_keywords_overwrite
 
         for actual, desired in outputs:
-            desired = get_bigdata('scsb-acstools', self.env, self.detector,
-                                  'truth', desired)
-            fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol,
+            desiredpath = get_pkg_data_filename(
+                os.path.join('data', 'truth', desired),
+                package='acstools.tests',
+                show_progress=False, remote_timeout=self.timeout)
+            fdiff = FITSDiff(actual, desiredpath, rtol=rtol, atol=atol,
                              ignore_keywords=ignore_keywords)
             creature_report += fdiff.report()
 
diff --git a/acstools/tests/test_polarization.py b/acstools/tests/test_polarization.py
index 3cb64a54..002e3894 100644
--- a/acstools/tests/test_polarization.py
+++ b/acstools/tests/test_polarization.py
@@ -1,8 +1,9 @@
+import numpy as np
 import pytest
 from astropy import units
 from astropy.tests.helper import assert_quantity_allclose
-import numpy as np
-from .. import polarization_tools
+
+from acstools import polarization_tools
 
 
 def test_tables():
@@ -48,7 +49,7 @@ def test_pol_class():
     but let's test that we get expected exceptions."""
 
     with pytest.raises(IndexError):
-        _ = polarization_tools.Polarization(1, 1, 1, 'F607W', 'WFC', 1)
+        polarization_tools.Polarization(1, 1, 1, 'F607W', 'WFC', 1)
 
     with pytest.raises(ValueError):
-        _ = polarization_tools.Polarization(1, 1, 1, 'F606W', 'WFB', 1)
+        polarization_tools.Polarization(1, 1, 1, 'F606W', 'WFB', 1)
diff --git a/acstools/tests/test_wfc_destripe.py b/acstools/tests/test_wfc_destripe.py
index 5fac3d64..67e08e38 100644
--- a/acstools/tests/test_wfc_destripe.py
+++ b/acstools/tests/test_wfc_destripe.py
@@ -1,10 +1,9 @@
 """Test standalone ACS destriping for post-SM4 WFC exposures."""
 
 import pytest
-from ci_watson.artifactory_helpers import get_bigdata
 
-from .helpers import BaseACSTOOLS
-from .. import acs_destripe
+from acstools import acs_destripe
+from acstools.tests.helpers import BaseACSTOOLS
 
 
 class TestDestripe(BaseACSTOOLS):
@@ -22,8 +21,7 @@ def _destripe_one(self, inputfile, outsuffix, outputfile, truthfile,
         else:
             # Get extra input masks
             for mfile in masks:
-                get_bigdata('scsb-acstools', self.env, self.detector, 'input',
-                            mfile)
+                self.get_input_file(mfile, skip_ref=True)
             acs_destripe.clean(inputfile, outsuffix,
                                mask1=masks[0], mask2=masks[1])
 
diff --git a/acstools/tests/test_wfc_satdet.py b/acstools/tests/test_wfc_satdet.py
index c2d3c6e5..326b2e09 100644
--- a/acstools/tests/test_wfc_satdet.py
+++ b/acstools/tests/test_wfc_satdet.py
@@ -1,13 +1,10 @@
-"""
-Test satellite trail detection.
+"""Test satellite trail detection.
 
 .. note:: Cannot test ``detsat()`` because PHT results change from run to run!
 
 """
-from ci_watson.artifactory_helpers import get_bigdata
-
-from .helpers import BaseACSTOOLS
-from .. import satdet
+from acstools import satdet
+from acstools.tests.helpers import BaseACSTOOLS
 
 
 class TestSatDet(BaseACSTOOLS):
@@ -21,8 +18,7 @@ def test_trail_mask(self):
         truthfile = rootname + '_flc_ref.fits'
 
         # Prepare input file.
-        get_bigdata('scsb-acstools', self.env, self.detector, 'input',
-                    inputfile)
+        self.get_input_file(inputfile, skip_ref=True)
 
         # Satellite trail masking.
         sciext = 4