Repository: ResonantGeoData/ResonantGeoData

Issue:
Unassigned permissions not working
Non-admin accounts are seeing an incorrect amount of spatial entries in the search results. Here are two results: 1) from my `@kitwar`e account which is an admin and one from my `@gmail` account that has no permissions. Using the changes from #401: 

https://github.com/ResonantGeoData/ResonantGeoData/blob/014ce2693a0a3e899d6af0a9d7822a5f1327268c/rgd/geodata/permissions.py#L108

You can see 475 results with the admin account (which is the correct amount) and 4949 results with the nonadmin account which hits that new code (this number is wildly incorrect): 

| admin | nonadmin |
| --- | --- |
| ![Screen Shot 2021-05-25 at 8 14 00 AM](https://user-images.githubusercontent.com/22067021/119513980-e9e3ce00-bd31-11eb-8a27-44c49b20d1d4.png) | ![Screen Shot 2021-05-25 at 8 13 51 AM](https://user-images.githubusercontent.com/22067021/119514301-21eb1100-bd32-11eb-941c-96ecd6b18a31.png) |




Discussion:

Resolving patch:
diff --git a/rgd/geodata/permissions.py b/rgd/geodata/permissions.py
--- a/rgd/geodata/permissions.py
+++ b/rgd/geodata/permissions.py
@@ -102,10 +102,12 @@ def filter_perm(user, queryset, role):
     user_path = (path + '__' if path != '' else path) + 'user'
     role_path = (path + '__' if path != '' else path) + 'role'
     queryset = annotate_queryset(queryset)
-    filtered = queryset.filter(**{user_path: user.pk}).exclude(**{role_path + '__lt': role})
+    filtered = (
+        queryset.filter(**{user_path: user.pk}).exclude(**{role_path + '__lt': role}).distinct()
+    )
     # Check setting for unassigned permissions
     if settings.RGD_GLOBAL_READ_ACCESS:
-        unassigned = queryset.filter(**{user_path + '__isnull': True})
+        unassigned = queryset.filter(**{user_path + '__isnull': True}).distinct()
         return unassigned | filtered
     return filtered
 


Test patch:
diff --git a/rgd/geodata/tests/conftest.py b/rgd/geodata/tests/conftest.py
--- a/rgd/geodata/tests/conftest.py
+++ b/rgd/geodata/tests/conftest.py
@@ -6,6 +6,7 @@
 from rest_framework.test import APIClient
 
 from . import factories
+from .data_fixtures import *  # noqa
 
 
 @pytest.fixture
diff --git a/rgd/geodata/tests/data_fixtures.py b/rgd/geodata/tests/data_fixtures.py
new file mode 100644
--- /dev/null
+++ b/rgd/geodata/tests/data_fixtures.py
@@ -0,0 +1,124 @@
+import pytest
+
+from rgd.geodata import models
+from rgd.geodata.datastore import datastore
+
+from . import factories
+
+
+@pytest.fixture
+def checksum_file():
+    return factories.ChecksumFileFactory()
+
+
+@pytest.fixture
+def checksum_file_url():
+    return factories.ChecksumFileFactory(
+        file=None,
+        url=datastore.get_url('stars.png'),
+        type=models.FileSourceType.URL,
+    )
+
+
+@pytest.fixture
+def geotiff_image_entry():
+    imagefile = factories.ImageFileFactory(
+        file__file__filename='paris_france_10.tiff',
+        file__file__from_path=datastore.fetch('paris_france_10.tiff'),
+    )
+    return imagefile.imageentry
+
+
+@pytest.fixture
+def astro_image():
+    name = 'astro.png'
+    imagefile = factories.ImageFileFactory(
+        file__file__filename=name,
+        file__file__from_path=datastore.fetch(name),
+    )
+    return imagefile.imageentry
+
+
+@pytest.fixture
+def sample_raster_a():
+    imagefile = factories.ImageFileFactory(
+        file__file__filename='20091021202517-01000100-VIS_0001.ntf',
+        file__file__from_path=datastore.fetch('20091021202517-01000100-VIS_0001.ntf'),
+    )
+    image_set = factories.ImageSetFactory(
+        images=[imagefile.imageentry.id],
+    )
+    raster = factories.RasterEntryFactory(
+        name='20091021202517-01000100-VIS_0001.ntf',
+        image_set=image_set,
+    )
+    return raster.rastermetaentry
+
+
+@pytest.fixture
+def sample_raster_b():
+    imagefile = factories.ImageFileFactory(
+        file__file__filename='cclc_schu_100.tif',
+        file__file__from_path=datastore.fetch('cclc_schu_100.tif'),
+    )
+    image_set = factories.ImageSetFactory(
+        images=[imagefile.imageentry.id],
+    )
+    raster = factories.RasterEntryFactory(
+        name='cclc_schu_100.tif',
+        image_set=image_set,
+    )
+    return raster.rastermetaentry
+
+
+@pytest.fixture
+def sample_raster_multi():
+    # These test files are dramatically downsampled for rapid testing
+    landsat_files = [
+        'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band1.tif',
+        'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band2.tif',
+        'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band3.tif',
+    ]
+    b1 = factories.ImageFileFactory(
+        file__file__filename=landsat_files[0],
+        file__file__from_path=datastore.fetch(landsat_files[0]),
+    )
+    b2 = factories.ImageFileFactory(
+        file__file__filename=landsat_files[1],
+        file__file__from_path=datastore.fetch(landsat_files[1]),
+    )
+    b3 = factories.ImageFileFactory(
+        file__file__filename=landsat_files[2],
+        file__file__from_path=datastore.fetch(landsat_files[2]),
+    )
+    image_set = factories.ImageSetFactory(
+        images=[
+            b1.imageentry.id,
+            b2.imageentry.id,
+            b3.imageentry.id,
+        ],
+    )
+    # Create a RasterEntry from the three band image entries
+    raster = factories.RasterEntryFactory(
+        name='Multi File Test',
+        image_set=image_set,
+    )
+    return raster.rastermetaentry
+
+
+@pytest.fixture
+def fmv_klv_file():
+    return factories.FMVFileFactory(
+        klv_file__filename='subset_metadata.klv',
+        klv_file__from_path=datastore.fetch('subset_metadata.klv'),
+    )
+
+
+@pytest.fixture
+def fmv_video_file():
+    return factories.FMVFileFactory(
+        file__file__filename='test_fmv.ts',
+        file__file__from_path=datastore.fetch('test_fmv.ts'),
+        klv_file=None,
+        web_video_file=None,
+    )
diff --git a/rgd/geodata/tests/test_api.py b/rgd/geodata/tests/test_api.py
--- a/rgd/geodata/tests/test_api.py
+++ b/rgd/geodata/tests/test_api.py
@@ -2,61 +2,6 @@
 from rest_framework import status
 
 from rgd.geodata import models
-from rgd.geodata.datastore import datastore
-
-from . import factories
-
-
-@pytest.fixture()
-def checksum_file():
-    return factories.ChecksumFileFactory()
-
-
-@pytest.fixture()
-def checksum_file_url():
-    return factories.ChecksumFileFactory(
-        file=None,
-        url=datastore.get_url('stars.png'),
-        type=models.FileSourceType.URL,
-    )
-
-
-@pytest.fixture()
-def landsat_image():
-    name = 'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band1.tif'
-    imagefile = factories.ImageFileFactory(
-        file__file__filename=name,
-        file__file__from_path=datastore.fetch(name),
-    )
-    return imagefile.imageentry
-
-
-@pytest.fixture()
-def astro_image():
-    name = 'astro.png'
-    imagefile = factories.ImageFileFactory(
-        file__file__filename=name,
-        file__file__from_path=datastore.fetch(name),
-    )
-    return imagefile.imageentry
-
-
-@pytest.fixture()
-def landsat_raster():
-    name = 'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band1.tif'
-    imagefile = factories.ImageFileFactory(
-        file__file__filename=name,
-        file__file__from_path=datastore.fetch(name),
-    )
-    image_set = factories.ImageSetFactory(
-        images=[imagefile.imageentry.id],
-    )
-    raster = factories.RasterEntryFactory(
-        name=name,
-        image_set=image_set,
-    )
-    raster.refresh_from_db()
-    return raster
 
 
 @pytest.mark.django_db(transaction=True)
@@ -102,9 +47,9 @@ def test_get_checksum_file(admin_api_client, checksum_file):
 
 
 @pytest.mark.django_db(transaction=True)
-def test_get_spatial_entry(api_client, landsat_raster):
+def test_get_spatial_entry(api_client, sample_raster_a):
     """Test individual GET for SpatialEntry model."""
-    pk = landsat_raster.rastermetaentry.spatial_id
+    pk = sample_raster_a.rastermetaentry.spatial_id
     response = api_client.get(f'/api/geodata/common/spatial_entry/{pk}')
     assert response.status_code == 200
     assert response.data
@@ -138,16 +83,16 @@ def test_create_get_subsampled_image(admin_api_client, astro_image):
 
 
 @pytest.mark.django_db(transaction=True)
-def test_create_and_download_cog(admin_api_client, landsat_image):
+def test_create_and_download_cog(admin_api_client, geotiff_image_entry):
     """Test POST for ConvertedImageFile model."""
     response = admin_api_client.post(
         '/api/geoprocess/imagery/cog',
-        {'source_image': landsat_image.id},
+        {'source_image': geotiff_image_entry.id},
     )
     assert response.status_code == 201
     assert response.data
     # Check that a COG was generated
-    cog = models.imagery.ConvertedImageFile.objects.get(source_image=landsat_image.id)
+    cog = models.imagery.ConvertedImageFile.objects.get(source_image=geotiff_image_entry.id)
     # NOTE: This doesn't actually verify the file is in COG format. Assumed.
     assert cog.converted_file
     # Also test download endpoint here:
diff --git a/rgd/geodata/tests/test_filters.py b/rgd/geodata/tests/test_filters.py
--- a/rgd/geodata/tests/test_filters.py
+++ b/rgd/geodata/tests/test_filters.py
@@ -1,47 +1,12 @@
 import pytest
 
-from rgd.geodata.datastore import datastore
+from rgd.geodata import models
 from rgd.geodata.filters import SpatialEntryFilter
-from rgd.geodata.models import RasterMetaEntry, SpatialEntry
-
-from . import factories
-
-
-@pytest.fixture
-def sample_raster_a():
-    imagefile = factories.ImageFileFactory(
-        file__file__filename='20091021202517-01000100-VIS_0001.ntf',
-        file__file__from_path=datastore.fetch('20091021202517-01000100-VIS_0001.ntf'),
-    )
-    image_set = factories.ImageSetFactory(
-        images=[imagefile.imageentry.id],
-    )
-    raster = factories.RasterEntryFactory(
-        name='20091021202517-01000100-VIS_0001.ntf',
-        image_set=image_set,
-    )
-    return RasterMetaEntry.objects.get(parent_raster=raster)
-
-
-@pytest.fixture
-def sample_raster_b():
-    imagefile = factories.ImageFileFactory(
-        file__file__filename='cclc_schu_100.tif',
-        file__file__from_path=datastore.fetch('cclc_schu_100.tif'),
-    )
-    image_set = factories.ImageSetFactory(
-        images=[imagefile.imageentry.id],
-    )
-    raster = factories.RasterEntryFactory(
-        name='cclc_schu_100.tif',
-        image_set=image_set,
-    )
-    return RasterMetaEntry.objects.get(parent_raster=raster)
 
 
 @pytest.mark.django_db(transaction=True)
 def test_q_distance(sample_raster_a, sample_raster_b):
-    assert SpatialEntry.objects.count() == 2
+    assert models.SpatialEntry.objects.count() == 2
     # Make sure all are returned sorted by distance
     filterset = SpatialEntryFilter(
         data={
@@ -49,7 +14,7 @@ def test_q_distance(sample_raster_a, sample_raster_b):
         }
     )
     assert filterset.is_valid()
-    qs = filterset.filter_queryset(SpatialEntry.objects.all())
+    qs = filterset.filter_queryset(models.SpatialEntry.objects.all())
     assert qs.count() == 2
     assert qs.first().spatial_id == sample_raster_a.spatial_id
     filterset = SpatialEntryFilter(
@@ -58,14 +23,14 @@ def test_q_distance(sample_raster_a, sample_raster_b):
         }
     )
     assert filterset.is_valid()
-    qs = filterset.filter_queryset(SpatialEntry.objects.all())
+    qs = filterset.filter_queryset(models.SpatialEntry.objects.all())
     assert qs.count() == 2
     assert qs.first().spatial_id == sample_raster_b.spatial_id
 
 
 @pytest.mark.django_db(transaction=True)
 def test_raster_intersects(sample_raster_a, sample_raster_b):
-    assert SpatialEntry.objects.count() == 2
+    assert models.SpatialEntry.objects.count() == 2
     filterset = SpatialEntryFilter(
         data={
             'q': f'SRID=4326;{sample_raster_a.outline.wkt}',
@@ -73,13 +38,13 @@ def test_raster_intersects(sample_raster_a, sample_raster_b):
         }
     )
     assert filterset.is_valid()
-    qs = filterset.filter_queryset(SpatialEntry.objects.all())
+    qs = filterset.filter_queryset(models.SpatialEntry.objects.all())
     assert qs.count() == 1
 
 
 @pytest.mark.django_db(transaction=True)
 def test_geojson_intersects(sample_raster_a, sample_raster_b):
-    assert SpatialEntry.objects.count() == 2
+    assert models.SpatialEntry.objects.count() == 2
     filterset = SpatialEntryFilter(
         data={
             'q': f'{sample_raster_a.outline.geojson}',
@@ -87,6 +52,6 @@ def test_geojson_intersects(sample_raster_a, sample_raster_b):
         }
     )
     assert filterset.is_valid()
-    qs = filterset.filter_queryset(SpatialEntry.objects.all())
+    qs = filterset.filter_queryset(models.SpatialEntry.objects.all())
     assert qs.count() == 1
     assert qs.first().spatial_id == sample_raster_a.spatial_id
diff --git a/rgd/geodata/tests/test_fmv.py b/rgd/geodata/tests/test_fmv.py
--- a/rgd/geodata/tests/test_fmv.py
+++ b/rgd/geodata/tests/test_fmv.py
@@ -2,36 +2,23 @@
 
 import pytest
 
-from rgd.geodata.datastore import datastore
 from rgd.geodata.models.fmv.base import FMVEntry
 from rgd.geodata.models.mixins import Status
 
-from . import factories
-
 NO_KWIVER = os.environ.get('NO_KWIVER', False)
 
 
 @pytest.mark.django_db(transaction=True)
-def test_populate_fmv_entry_from_klv_file():
+def test_populate_fmv_entry_from_klv_file(fmv_klv_file):
     # Since we provide the KLV file and the factory provides a dummpy MP4 file,
     #   the ETL routine will skip over those parts and just generate the FMVEntry
-    fmv_file = factories.FMVFileFactory(
-        klv_file__filename='subset_metadata.klv',
-        klv_file__from_path=datastore.fetch('subset_metadata.klv'),
-    )
-    fmv_entry = FMVEntry.objects.filter(fmv_file=fmv_file).first()
+    fmv_entry = FMVEntry.objects.filter(fmv_file=fmv_klv_file).first()
     assert fmv_entry.ground_frames is not None
 
 
 @pytest.mark.skipif(NO_KWIVER, reason='User set NO_KWIVER')
 @pytest.mark.django_db(transaction=True)
-def test_full_fmv_etl():
-    fmv_file = factories.FMVFileFactory(
-        file__file__filename='test_fmv.ts',
-        file__file__from_path=datastore.fetch('test_fmv.ts'),
-        klv_file=None,
-        web_video_file=None,
-    )
-    assert fmv_file.status == Status.SUCCEEDED, fmv_file.failure_reason
-    fmv_entry = FMVEntry.objects.get(fmv_file=fmv_file)
+def test_full_fmv_etl(fmv_video_file):
+    assert fmv_video_file.status == Status.SUCCEEDED, fmv_video_file.failure_reason
+    fmv_entry = FMVEntry.objects.get(fmv_file=fmv_video_file)
     assert fmv_entry.ground_frames is not None
diff --git a/rgd/geodata/tests/test_imagery.py b/rgd/geodata/tests/test_imagery.py
--- a/rgd/geodata/tests/test_imagery.py
+++ b/rgd/geodata/tests/test_imagery.py
@@ -24,12 +24,6 @@
     {'name': 'RomanColosseum_WV2mulitband_10.tif', 'centroid': {'x': 12.50, 'y': 41.89}},
 ]
 
-# These test files are dramatically downsampled for rapid testing
-LandsatFiles = [
-    'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band1.tif',
-    'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band2.tif',
-    'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band3.tif',
-]
 
 TOLERANCE = 2e-2
 
@@ -96,41 +90,16 @@ def test_repopulate_image_entry():
 
 
 @pytest.mark.django_db(transaction=True)
-def test_multi_file_raster():
+def test_multi_file_raster(sample_raster_multi):
     """Test the use case where a raster is generated from multiple files."""
-    b1 = factories.ImageFileFactory(
-        file__file__filename=LandsatFiles[0],
-        file__file__from_path=datastore.fetch(LandsatFiles[0]),
-    )
-    b2 = factories.ImageFileFactory(
-        file__file__filename=LandsatFiles[1],
-        file__file__from_path=datastore.fetch(LandsatFiles[1]),
-    )
-    b3 = factories.ImageFileFactory(
-        file__file__filename=LandsatFiles[2],
-        file__file__from_path=datastore.fetch(LandsatFiles[2]),
-    )
-    image_set = factories.ImageSetFactory(
-        images=[
-            b1.imageentry.id,
-            b2.imageentry.id,
-            b3.imageentry.id,
-        ],
-    )
-    # Create a RasterEntry from the three band image entries
-    raster = factories.RasterEntryFactory(
-        name='Multi File Test',
-        image_set=image_set,
-    )
-    meta = raster.rastermetaentry
-    assert raster.image_set.count == 3
-    assert meta.crs is not None
+    assert sample_raster_multi.parent_raster.image_set.count == 3
+    assert sample_raster_multi.crs is not None
 
 
 @pytest.mark.parametrize(
     'name',
     [
-        LandsatFiles[0],
+        'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band1.tif',
         'landcover_sample_2000.tif',
     ],
 )
diff --git a/rgd/geodata/tests/test_permissions.py b/rgd/geodata/tests/test_permissions.py
new file mode 100644
--- /dev/null
+++ b/rgd/geodata/tests/test_permissions.py
@@ -0,0 +1,22 @@
+from django.conf import settings
+import pytest
+
+from rgd.geodata import models
+from rgd.geodata.permissions import filter_read_perm
+
+
+@pytest.mark.django_db(transaction=True)
+def test_unassigned_permissions_complex(user_factory, user, sample_raster_a, sample_raster_multi):
+    prior = settings.RGD_GLOBAL_READ_ACCESS
+    settings.RGD_GLOBAL_READ_ACCESS = False
+    admin = user_factory(is_superuser=True)
+    admin_q = filter_read_perm(admin, models.RasterMetaEntry.objects.all())
+    basic_q = filter_read_perm(user, models.RasterMetaEntry.objects.all())
+    assert len(admin_q) == 2
+    assert len(basic_q) == 0
+    settings.RGD_GLOBAL_READ_ACCESS = True
+    admin_q = filter_read_perm(admin, models.RasterMetaEntry.objects.all())
+    basic_q = filter_read_perm(user, models.RasterMetaEntry.objects.all())
+    assert len(admin_q) == len(basic_q)
+    assert set(admin_q) == set(basic_q)
+    settings.RGD_GLOBAL_READ_ACCESS = prior
diff --git a/rgd/geodata/tests/test_tiles.py b/rgd/geodata/tests/test_tiles.py
--- a/rgd/geodata/tests/test_tiles.py
+++ b/rgd/geodata/tests/test_tiles.py
@@ -1,22 +1,9 @@
 import pytest
 
-from rgd.geodata.datastore import datastore
-
-from . import factories
-
-
-@pytest.fixture
-def image_entry():
-    imagefile = factories.ImageFileFactory(
-        file__file__filename='paris_france_10.tiff',
-        file__file__from_path=datastore.fetch('paris_france_10.tiff'),
-    )
-    return imagefile.imageentry
-
 
 @pytest.mark.django_db(transaction=True)
-def test_metadata(api_client, image_entry):
-    response = api_client.get(f'/api/geoprocess/imagery/{image_entry.pk}/tiles')
+def test_metadata(api_client, geotiff_image_entry):
+    response = api_client.get(f'/api/geoprocess/imagery/{geotiff_image_entry.pk}/tiles')
     metadata = response.data
     assert metadata['levels'] == 15
     assert metadata['sizeX'] == metadata['sizeY']
@@ -25,14 +12,14 @@ def test_metadata(api_client, image_entry):
 
 
 @pytest.mark.django_db(transaction=True)
-def test_tile(api_client, image_entry):
-    response = api_client.get(f'/api/geoprocess/imagery/{image_entry.pk}/tiles/1/0/0.png')
+def test_tile(api_client, geotiff_image_entry):
+    response = api_client.get(f'/api/geoprocess/imagery/{geotiff_image_entry.pk}/tiles/1/0/0.png')
     assert response.status_code == 200
     assert response['Content-Type'] == 'image/png'
 
 
 @pytest.mark.django_db(transaction=True)
-def test_thumbnail(api_client, image_entry):
-    response = api_client.get(f'/api/geoprocess/imagery/{image_entry.pk}/thumbnail')
+def test_thumbnail(api_client, geotiff_image_entry):
+    response = api_client.get(f'/api/geoprocess/imagery/{geotiff_image_entry.pk}/thumbnail')
     assert response.status_code == 200
     assert response['Content-Type'] == 'image/png'