Repository: AstarVienna/ScopeSim

Issue:
Don't show non-package directories in bug_report
When running `sim.bug_report()` for a cloned IRDB, `utils._get_all_irdb_pkgs()` will also show things like "irdb", "docs", ".github", "_REPORTS" etc.

Hints from issue comments:

Solution patch:
diff --git a/pyproject.toml b/pyproject.toml
index 82bbc518..9b32c0fe 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ScopeSim"
-version = "0.9.3a1"
+version = "0.9.3a2"
 description = "Generalised telescope observation simulator"
 license = "GPL-3.0-or-later"
 authors = ["Kieran Leschinski <kieran.leschinski@unive.ac.at>"]
diff --git a/scopesim/commands/user_commands.py b/scopesim/commands/user_commands.py
index 4361bad2..d6733e24 100644
--- a/scopesim/commands/user_commands.py
+++ b/scopesim/commands/user_commands.py
@@ -522,7 +522,7 @@ def add_packages_to_rc_search(local_path, package_list):
         plocal_path = patch_fake_symlinks(Path(local_path))
     except FileNotFoundError:
         # retry with mocks
-        plocal_path = patch_fake_symlinks(Path("./scopesim/tests/mocks"))
+        plocal_path = rc.__pkg_dir__ / "tests/mocks"
 
     for pkg in package_list:
         pkg_dir = plocal_path / pkg
diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py
index a5ebbba4..fefdd6b9 100644
--- a/scopesim/optics/fov.py
+++ b/scopesim/optics/fov.py
@@ -16,7 +16,13 @@ from synphot.units import PHOTLAM
 from . import fov_utils as fu
 from . import image_plane_utils as imp_utils
 
-from ..utils import from_currsys, quantify, has_needed_keywords, get_logger
+from ..utils import (
+    from_currsys,
+    quantify,
+    has_needed_keywords,
+    get_logger,
+    unit_includes_per_physical_type,
+)
 from ..source.source import Source
 
 
@@ -476,13 +482,16 @@ class FieldOfView:
             # Cube should be in PHOTLAM arcsec-2 for SpectralTrace mapping
             # Assumption is that ImageHDUs have units of PHOTLAM arcsec-2
             # ImageHDUs have photons/second/pixel.
-            # ..todo: Add a catch to get ImageHDU with BUNITs
             canvas_image_hdu = fits.ImageHDU(
                 data=np.zeros((self.header["NAXIS2"], self.header["NAXIS1"])),
                 header=self.header)
             # FIX: Do not scale source data - make a copy first.
-            # FIX: Use "Pixel scale conversion" as above.
-            field_data = field.data * self._pixarea(field.header).value / self.pixel_area
+            bunit = u.Unit(field.header.get("BUNIT", ""))
+            field_data = deepcopy(field.data)
+            if unit_includes_per_physical_type(bunit, "solid angle"):
+                # Field is in (PHOTLAM) / arcsec**2, need to scale by pixarea
+                field_data *= self._pixarea(field.header).value
+            field_data /= self.pixel_area
             field_hdu = fits.ImageHDU(data=field_data, header=field.header)
 
             canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu(
@@ -582,6 +591,10 @@ class FieldOfView:
 
         # 1. Make waveset and canvas cube (area, bin_width are applied at end)
         # TODO: Why is this not self.waveset? What's different?
+        # -> For non-cube input but cube output (e.g. 2D image in spec mode),
+        #    waverange needs to be resampled (I guess) to spectral_bin_width,
+        #    but self.waverange can only access the fields.
+        # -> Perhaps change that??
         wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds))
         fov_waveset = np.arange(
             self.meta["wave_min"].value, self.meta["wave_max"].value,
diff --git a/scopesim/source/source_fields.py b/scopesim/source/source_fields.py
index 7db133cb..3f1cd4ef 100644
--- a/scopesim/source/source_fields.py
+++ b/scopesim/source/source_fields.py
@@ -164,6 +164,9 @@ class TableSourceField(SpectrumSourceField):
     def __post_init__(self):
         """Validate input."""
         assert self.spectra, "Spectra must be non-empty for table source."
+        if not (uniquerefs := set(self.field["ref"])).issubset(self.spectra):
+            raise KeyError(f"Refs {uniquerefs.difference(self.spectra)} "
+                           "not found in spectra.")
         if "weight" not in self.field.colnames:
             self.field.add_column(
                 Column(name="weight", data=np.ones(len(self.field)))
diff --git a/scopesim/utils.py b/scopesim/utils.py
index bdd633b2..4eabcc37 100644
--- a/scopesim/utils.py
+++ b/scopesim/utils.py
@@ -150,8 +150,13 @@ def _get_required_packages():
 
 
 def _get_all_irdb_pkgs(root: Path):
+    def _name_valid(pkg_name):
+        if pkg_name in {"docs", "irdb"}:
+            return False
+        return not pkg_name.startswith(("_", "."))
+
     return [pkg_path for pkg_path in root.iterdir() if pkg_path.is_dir()
-            and not pkg_path.name.startswith("__")] if root.is_dir() else []
+            and _name_valid(pkg_path.name)] if root.is_dir() else []
 
 
 def _get_irdb_pkg_version(pkg_path: Path) -> str:
@@ -491,6 +496,14 @@ def unit_from_table(colname: str, table: Table,
     return u.Unit(default_unit)
 
 
+def unit_includes_per_physical_type(unit, physical_type):
+    """Check if one of the `unit`'s bases is of 1/`physical_type`."""
+    # TODO: Check again if there isn't any builtin functionality in astropy
+    #       for the same operation!
+    return any(1 / (base**power).physical_type == physical_type
+               for base, power in zip(unit.bases, unit.powers))
+
+
 def has_needed_keywords(header, suffix=""):
     """Check to see if the WCS keywords are in the header."""
     keys = {"CDELT1", "CRVAL1", "CRPIX1"}


Test patch:
diff --git a/scopesim/tests/conftest.py b/scopesim/tests/conftest.py
index 5f234e67..0be2a072 100644
--- a/scopesim/tests/conftest.py
+++ b/scopesim/tests/conftest.py
@@ -104,3 +104,17 @@ def protect_currsys():
     """Prevent modification of global currsys."""
     with patch("scopesim.rc.__currsys__"):
         yield
+
+
+@pytest.fixture(scope="function")
+def protect_config():
+    """Prevent modification of global config."""
+    with patch.dict("scopesim.rc.__config__"):
+        yield
+
+
+@pytest.fixture(scope="function")
+def protect_search_path():
+    """Prevent modification of global search_path."""
+    with patch("scopesim.rc.__search_path__"):
+        yield
diff --git a/scopesim/tests/test_utils_functions.py b/scopesim/tests/test_utils_functions.py
index 91dceeb9..7fd38c24 100644
--- a/scopesim/tests/test_utils_functions.py
+++ b/scopesim/tests/test_utils_functions.py
@@ -6,6 +6,7 @@ from unittest.mock import patch
 
 import numpy as np
 from astropy import wcs
+from astropy import units as u
 from astropy.io import ascii as ioascii, fits
 from astropy.table import Table
 
@@ -265,7 +266,12 @@ class TestSeq:
         assert arr[1:] - arr[:-1] == approx(step)
 
 
-@pytest.mark.usefixtures("protect_currsys")
+@pytest.mark.usefixtures("protect_config")
 def test_setting_instpkgspath():
     utils.link_irdb("bogus")
     assert rc.__config__["!SIM.file.local_packages_path"] == "bogus"
+
+
+def test_unit_includes_per_physical_type():
+    unit = u.Unit("photlam") / u.arcsec**2
+    assert utils.unit_includes_per_physical_type(unit, "solid angle")
diff --git a/scopesim/tests/tests_commands/test_UserCommands.py b/scopesim/tests/tests_commands/test_UserCommands.py
index f0af8ed1..a18b0808 100644
--- a/scopesim/tests/tests_commands/test_UserCommands.py
+++ b/scopesim/tests/tests_commands/test_UserCommands.py
@@ -113,7 +113,7 @@ class TestTrackIpAddress:
         _ = UserCommands(use_instrument="test_package")
 
 
-@pytest.mark.usefixtures("protect_currsys")
+@pytest.mark.usefixtures("no_file_error", "protect_search_path", "protect_config")
 class TestIffyPkgPaths:
     def test_finds_basic_instrument(self):
         UserCommands(use_instrument="basic_instrument")
diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py
index 4721103b..5ce12043 100644
--- a/scopesim/tests/tests_optics/test_FieldOfView.py
+++ b/scopesim/tests/tests_optics/test_FieldOfView.py
@@ -163,9 +163,8 @@ class TestExtractFrom:
         assert not np.isnan(fov.fields[0].data).any()
 
 
-# @pytest.mark.xfail(reason="apply make_cube's fov.waveset available to the outside ")
 class TestMakeCube:
-    @pytest.mark.xfail(reason="apply make_cube's fov.waveset available to the outside ")
+    @pytest.mark.xfail(reason="cube flux is broken")
     def test_makes_cube_from_table(self):
         src_table = so._table_source()            # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm
         fov = _fov_190_210_um()
@@ -187,7 +186,7 @@ class TestMakeCube:
             plt.imshow(cube.data[0, :, :], origin="lower")
             plt.show()
 
-    # @pytest.mark.xfail(reason="apply make_cube's fov.waveset available to the outside ")
+    @pytest.mark.xfail(reason="cube flux is broken")
     def test_makes_cube_from_imagehdu(self):
         src_image = so._image_source()            # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm
         fov = _fov_190_210_um()
@@ -206,7 +205,7 @@ class TestMakeCube:
             plt.imshow(cube.data[0, :, :], origin="lower")
             plt.show()
 
-    @pytest.mark.xfail(reason="apply make_cube's fov.waveset available to the outside ")
+    @pytest.mark.xfail(reason="cube flux is broken")
     def test_makes_cube_from_other_cube_imagehdu(self):
         import scopesim as sim
         sim.rc.__currsys__["!SIM.spectral.spectral_bin_width"] = 0.01
@@ -227,7 +226,7 @@ class TestMakeCube:
             plt.imshow(cube.data[0, :, :], origin="lower")
             plt.show()
 
-    @pytest.mark.xfail(reason="apply make_cube's fov.waveset available to the outside ")
+    @pytest.mark.xfail(reason="cube flux is broken")
     def test_makes_cube_from_two_similar_cube_imagehdus(self):
         src_cube = so._cube_source() + so._cube_source(dx=1)            # 2 cubes 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm
         fov = _fov_197_202_um()
@@ -247,7 +246,7 @@ class TestMakeCube:
             plt.imshow(cube.data[0, :, :], origin="lower")
             plt.show()
 
-    @pytest.mark.xfail(reason="apply make_cube's fov.waveset available to the outside ")
+    @pytest.mark.xfail(reason="cube flux is broken")
     def test_makes_cube_from_all_types_of_source_object(self):
         src_all = so._table_source() + \
                   so._image_source(dx=-4, dy=-4) + \
diff --git a/scopesim/tests/tests_source/test_source_Source.py b/scopesim/tests/tests_source/test_source_Source.py
index 0d8dda20..e01f52fd 100644
--- a/scopesim/tests/tests_source/test_source_Source.py
+++ b/scopesim/tests/tests_source/test_source_Source.py
@@ -16,7 +16,7 @@ from synphot.units import PHOTLAM
 
 from scopesim.source import source_utils
 from scopesim.source.source import Source
-from scopesim.source.source_fields import CubeSourceField
+from scopesim.source.source_fields import CubeSourceField, TableSourceField
 
 from scopesim.optics.image_plane import ImagePlane
 from scopesim.utils import convert_table_comments_to_dict
@@ -186,7 +186,8 @@ class TestSourceInit:
     def test_initialises_with_filename_and_spectrum(self, ii, dtype,
                                                     input_files, input_spectra):
         fname = input_files[ii]
-        src = Source(filename=fname, spectra=input_spectra[0])
+        spec = input_spectra if issubclass(dtype, Table) else input_spectra[0]
+        src = Source(filename=fname, spectra=spec)
         assert isinstance(src, Source)
         assert isinstance(src.spectra[0], SourceSpectrum)
         assert isinstance(src.fields[0].field, dtype)
@@ -448,6 +449,14 @@ def test_cube_source_field():
     _, ax = plt.subplots()
     csf.plot(ax, "red")
 
+
+def test_throws_for_invalid_ref():
+    with pytest.raises(KeyError):
+        # Minimal Table
+        tbl = Table(data=[[0, 1]], names=["ref"])
+        TableSourceField(tbl, {0: None})
+
+
 #
 # class TestScaleImageHDU:
 #     def test_scaling_properly_for_si_photlam_in_header(self):