Repository: hydroshare/hydroshare (language: python)

Issue:
[#5473] Aggregation meta file download fix
<!--

Please read, and add your text at the bottom of this message.

Thanks for contributing code to HydroShare. In order to maintain code quality and expedite this process, please assist the development team by making sure the following is present in this pull request.

For more information, see https://docs.google.com/document/d/1dzxqlZW5fKNEyQSeKiSFq-SmS-VOPCva95XXkBjPExs

-->

### Pull Request Checklist: 
- [x] Positive Test Case Written by Dev

<!-- Enter steps that a QA engineer, stakeholder, or user documentation writer would follow to test the positive or "successful" case of the functionality your code provides or fixes -->

- [x] Automated Testing

<!-- Our Jenkins Instance is set up to automatically test every commit from a pull request. Code coverage must not decrease so new functionality or code paths added during a bug fix must have appropriate tests written. Every test must pass, including PEP8 code formatting tests. -->

- [x] Sufficient User and Developer Documentation

<!-- Please email your positive test case lbrazil@cuahsi.org, who will make the decision regarding user documentation. -->

- [x] Passing Jenkins Build

<!-- Our Jenkins Instance is set up to automatically test every commit from a pull request. Every test must pass, including PEP8 code formatting tests. -->

- [x] Peer Code review and approval

<!-- This is the process by which a peer developer on the HydroShare team will read the changeset, provide feedback, and ultimately give a formal approval to the code before it passes PR status. -->

### Positive Test Case
1. Refer to issue #5473 to test. 



Discussion / hints:

Fix patch:
diff --git a/Dockerfile b/Dockerfile
index 2e844e7c67..6959b8f5a7 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,6 +4,8 @@ FROM hydroshare/hs_docker_base:305f2fa
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     locale-gen
 
+RUN pip install google-cloud-pubsub
+
 # https://www.digicert.com/kb/digicert-root-certificates.htm
 # Get the .pem file from digicert and add it to the bundle used by certifi
 # Could also use the REQUESTS_CA_BUNDLE environment variable to point to the .pem file
diff --git a/Dockerfile-multistage-node b/Dockerfile-multistage-node
index 3e6f8cefa9..182bf6f45a 100644
--- a/Dockerfile-multistage-node
+++ b/Dockerfile-multistage-node
@@ -28,6 +28,8 @@ RUN rm -rf static templates && \
 
 FROM hydroshare/hs_docker_base:305f2fa
 
+COPY --from=node-build /hydroshare /hydroshare
+
 # Set the locale. TODO - remove once we have a better alternative worked out
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     locale-gen
diff --git a/django_irods/storage.py b/django_irods/storage.py
index 833d1467f9..cc835cb4d5 100644
--- a/django_irods/storage.py
+++ b/django_irods/storage.py
@@ -21,6 +21,9 @@
     IRodsEnv,
 )
 
+import logging
+logger = logging.getLogger(__name__)
+
 
 @deconstructible
 class IrodsStorage(Storage):
@@ -334,11 +337,21 @@ def _list_files(self, path):
         fname_list = []
         fsize_list = []
 
+        single_resc_for_query = getattr(settings, 'IRODS_SINGLE_RESC_FOR_QUERY', "")
+        if single_resc_for_query:
+            resc_query = " AND DATA_RESC_NAME = '{}'".format(single_resc_for_query)
+        else:
+            resc_query = ""
+            logger.error(
+                "settings.IRODS_SINGLE_RESC_FOR_QUERY not set. "
+                "The resulting size might not be accurate if the file is replicated."
+            )
+
         # the query below returns name and size (separated in comma) of all data
         # objects/files under the path collection/directory
         qrystr = (
-            "select DATA_NAME, DATA_SIZE where DATA_REPL_STATUS != '0' "
-            "AND {}".format(IrodsStorage.get_absolute_path_query(path))
+            "select DATA_NAME, DATA_SIZE where DATA_REPL_STATUS = '1' "
+            "AND {}{}".format(IrodsStorage.get_absolute_path_query(path), resc_query)
         )
         stdout = self.session.run("iquest", None, "--no-page", "%s,%s", qrystr)[
             0
@@ -424,10 +437,19 @@ def size(self, name):
             )
         coll_name = file_info[0]
         file_name = file_info[1]
+        single_resc_for_query = getattr(settings, 'IRODS_SINGLE_RESC_FOR_QUERY', "")
+        if single_resc_for_query:
+            resc_query = " AND DATA_RESC_NAME = '{}'".format(single_resc_for_query)
+        else:
+            resc_query = ""
+            logger.error(
+                "settings.IRODS_SINGLE_RESC_FOR_QUERY not set. "
+                "The resulting size might not be accurate if the file is replicated."
+            )
         qrystr = (
-            "select DATA_SIZE where DATA_REPL_STATUS != '0' AND "
+            "select DATA_SIZE where DATA_REPL_STATUS = '1'{} AND "
             "{} AND DATA_NAME = '{}'".format(
-                IrodsStorage.get_absolute_path_query(coll_name), file_name
+                resc_query, IrodsStorage.get_absolute_path_query(coll_name), file_name
             )
         )
         stdout = self.session.run("iquest", None, "%s", qrystr)[0]
@@ -437,7 +459,7 @@ def size(self, name):
                 "{} cannot be found in iRODS to retrieve " "file size".format(name)
             )
         # remove potential '\n' from stdout
-        size_string = stdout.strip("0\n").replace("\n", "")
+        size_string = stdout.replace("\n", "")
         try:
             ret = int(float(size_string))
             return ret
diff --git a/django_irods/views.py b/django_irods/views.py
index 04435079b4..045d4237e1 100755
--- a/django_irods/views.py
+++ b/django_irods/views.py
@@ -1,30 +1,30 @@
 import datetime
-from dateutil import tz
+import logging
 import mimetypes
 import os
 import urllib
 from uuid import uuid4
 
+from dateutil import tz
 from django.conf import settings
-from django.http import HttpResponse, FileResponse, HttpResponseRedirect, JsonResponse
-from rest_framework.decorators import api_view
+from django.core.exceptions import ObjectDoesNotExist
+from django.http import (FileResponse, HttpResponse, HttpResponseRedirect,
+                         JsonResponse)
+from drf_yasg.utils import swagger_auto_schema
 from rest_framework import status
+from rest_framework.decorators import api_view
 
 from django_irods import icommands
 from hs_core.hydroshare.resource import check_resource_type
-from hs_core.task_utils import (
-    get_resource_bag_task,
-    get_or_create_task_notification,
-    get_task_user_id,
-    get_task_notification
-)
-
-from hs_core.signals import pre_download_file, pre_download_resource, pre_check_bag_flag
+from hs_core.signals import (pre_check_bag_flag, pre_download_file,
+                             pre_download_resource)
+from hs_core.task_utils import (get_or_create_task_notification,
+                                get_resource_bag_task, get_task_notification,
+                                get_task_user_id)
 from hs_core.tasks import create_bag_by_irods, create_temp_zip, delete_zip
-from hs_core.views.utils import authorize, ACTION_TO_AUTHORIZE
-from drf_yasg.utils import swagger_auto_schema
+from hs_core.views.utils import ACTION_TO_AUTHORIZE, authorize
+from hs_file_types.enums import AggregationMetaFilePath
 
-import logging
 logger = logging.getLogger(__name__)
 
 
@@ -285,6 +285,15 @@ def download(request, path, use_async=True, use_reverse_proxy=True,
             if bag_modified is None or bag_modified or not istorage.exists(irods_output_path):
                 res.setAVU("bag_modified", True)  # ensure bag_modified is set when irods_output_path does not exist
                 create_bag_by_irods(res_id, create_zip=False)
+        elif any([path.endswith(suffix) for suffix in AggregationMetaFilePath]):
+            # download aggregation meta xml/json schema file
+            try:
+                aggregation = res.get_aggregation_by_meta_file(path)
+            except ObjectDoesNotExist:
+                aggregation = None
+
+            if aggregation is not None and aggregation.metadata.is_dirty:
+                aggregation.create_aggregation_xml_documents()
 
     # If we get this far,
     # * path and irods_path point to true input
diff --git a/hs_composite_resource/models.py b/hs_composite_resource/models.py
index dc20c88db8..3d72a2142f 100644
--- a/hs_composite_resource/models.py
+++ b/hs_composite_resource/models.py
@@ -1,6 +1,6 @@
 import logging
 import os
-
+from typing import Union
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from mezzanine.pages.page_processors import processor_for
@@ -18,11 +18,23 @@
     RefTimeseriesLogicalFile,
     TimeSeriesLogicalFile
 )
-from hs_file_types.models.base import METADATA_FILE_ENDSWITH, RESMAP_FILE_ENDSWITH, SCHEMA_JSON_FILE_ENDSWITH
+from hs_file_types.enums import AggregationMetaFilePath
 from hs_file_types.utils import update_target_spatial_coverage, update_target_temporal_coverage
 
 logger = logging.getLogger(__name__)
 
+AggregationType = Union[
+    GenericLogicalFile,
+    FileSetLogicalFile,
+    GeoFeatureLogicalFile,
+    GeoRasterLogicalFile,
+    ModelProgramLogicalFile,
+    ModelInstanceLogicalFile,
+    NetCDFLogicalFile,
+    RefTimeseriesLogicalFile,
+    TimeSeriesLogicalFile
+]
+
 
 class CompositeResource(BaseResource):
     objects = ResourceManager("CompositeResource")
@@ -333,6 +345,38 @@ def get_aggregation_by_name(self, name, aggregations=None):
             raise ObjectDoesNotExist(
                 "No matching aggregation was found for name:{}".format(name))
 
+    def get_aggregation_by_meta_file(self, meta_file_path: str) -> AggregationType:
+        """Get an aggregation that matches the specified meta xml/json file path
+        :param  meta_file_path: directory path of the meta xml/json file
+        :return an aggregation object if found
+        :raises ObjectDoesNotExist if no matching aggregation is found
+        """
+        if __debug__:
+            assert any(meta_file_path.endswith(ext) for ext in AggregationMetaFilePath)
+
+        meta_file_path = self.get_relative_path(meta_file_path)
+        folder, base = os.path.split(meta_file_path)
+        if base.endswith(AggregationMetaFilePath.METADATA_FILE_ENDSWITH):
+            base_file_name_to_match = base[:-len(AggregationMetaFilePath.METADATA_FILE_ENDSWITH)]
+        elif base.endswith(AggregationMetaFilePath.RESMAP_FILE_ENDSWITH):
+            base_file_name_to_match = base[:-len(AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)]
+        else:
+            base_file_name_to_match = base[:-len(AggregationMetaFilePath.SCHEMA_JSON_FILE_ENDSWITH)]
+
+        for res_file in ResourceFile.list_folder(self, folder=folder, sub_folders=False):
+            if res_file.has_logical_file:
+                file_name_to_match = f"{base_file_name_to_match}{res_file.extension}"
+                if res_file.file_name == file_name_to_match:
+                    return res_file.logical_file
+        # check for folder aggregation
+        if folder and folder == base_file_name_to_match:
+            folder_aggregation = self.get_folder_aggregation_object(folder)
+            if folder_aggregation is not None:
+                return folder_aggregation
+
+        raise ObjectDoesNotExist("No matching aggregation was found for "
+                                 "meta file path:{}".format(meta_file_path))
+
     def get_fileset_aggregation_in_path(self, path, aggregations=None):
         """Get the first fileset aggregation in the path moving up (towards the root)in the path
         :param  path: directory path in which to search for a fileset aggregation
@@ -421,9 +465,9 @@ def set_parent_aggregation_dirty(path_to_search):
         # remove file extension from aggregation name (note: aggregation name is a file path
         # for all aggregation types except fileset/model aggregation
         file_name, _ = os.path.splitext(orig_path)
-        schema_json_file_name = file_name + SCHEMA_JSON_FILE_ENDSWITH
-        meta_xml_file_name = file_name + METADATA_FILE_ENDSWITH
-        map_xml_file_name = file_name + RESMAP_FILE_ENDSWITH
+        schema_json_file_name = file_name + AggregationMetaFilePath.SCHEMA_JSON_FILE_ENDSWITH
+        meta_xml_file_name = file_name + AggregationMetaFilePath.METADATA_FILE_ENDSWITH
+        map_xml_file_name = file_name + AggregationMetaFilePath.RESMAP_FILE_ENDSWITH
         if not is_new_path_a_folder:
             # case of file rename/move for single file aggregation
             schema_json_file_full_path = os.path.join(self.file_path, schema_json_file_name)
diff --git a/hs_core/authentication.py b/hs_core/authentication.py
index 1de0ae4ccf..eb01334415 100644
--- a/hs_core/authentication.py
+++ b/hs_core/authentication.py
@@ -130,7 +130,7 @@ def authenticate(self, request):
         _, value, *_ = request.headers.get('Authorization').split()
 
         decoded_username, decoded_password = (
-            base64.b64decode(value).decode("utf-8").split(":")
+            base64.b64decode(value).decode("utf-8").split(":", 1)
         )
         # authenticate against keycloak
         try:
diff --git a/hs_core/hydro_realtime_signal_processor.py b/hs_core/hydro_realtime_signal_processor.py
index 760d2ad276..470f78c5fd 100644
--- a/hs_core/hydro_realtime_signal_processor.py
+++ b/hs_core/hydro_realtime_signal_processor.py
@@ -3,6 +3,7 @@
 from django.conf import settings
 from django.db import models
 from hs_core.models import Date, BaseResource
+from hs_core.pubsub_discovery_processor import pub_update
 from hs_core.signals import post_spam_whitelist_change
 from hs_access_control.models import ResourceAccess
 from haystack.exceptions import NotHandled
@@ -62,6 +63,7 @@ def index_resource(signal_processor, instance: BaseResource):
                 try:
                     index = signal_processor.connections[using].get_unified_index().get_index(newsender)
                     index.update_object(newbase, using=using)
+                    pub_update(newbase.short_id, False)
                 except NotHandled:
                     logger.exception("Failure: changes to %s with short_id %s not added to Solr Index.",
                                      str(type(instance)), newbase.short_id)
@@ -71,6 +73,7 @@ def index_resource(signal_processor, instance: BaseResource):
                 try:
                     index = signal_processor.connections[using].get_unified_index().get_index(newsender)
                     index.remove_object(newbase, using=using)
+                    pub_update(newbase.short_id, True)
                 except NotHandled:
                     logger.exception("Failure: delete of %s with short_id %s failed.",
                                      str(type(instance)), newbase.short_id)
diff --git a/hs_core/management/commands/reset_file_size_system_metadata.py b/hs_core/management/commands/reset_file_size_system_metadata.py
new file mode 100644
index 0000000000..59575a82f6
--- /dev/null
+++ b/hs_core/management/commands/reset_file_size_system_metadata.py
@@ -0,0 +1,186 @@
+
+import time
+from django.utils import timezone
+from datetime import timedelta
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+from django.core.exceptions import ObjectDoesNotExist
+from django.conf import settings
+from django.db.models import Q, F
+
+from hs_core.hydroshare import current_site_url
+from hs_core.models import ResourceFile, BaseResource
+from theme.models import UserQuota
+
+current_site = current_site_url()
+_BATCH_SIZE = settings.BULK_UPDATE_CREATE_BATCH_SIZE
+
+
+def chunked_queryset(queryset, chunk_size=_BATCH_SIZE):
+    """Slice a queryset into chunks.
+    Code adapted from https://djangosnippets.org/snippets/10599/
+    """
+    if not queryset.exists():
+        return
+    queryset = queryset.order_by("pk")
+    pks = queryset.values_list("pk", flat=True)
+    start_pk = pks[0]
+    while True:
+        try:
+            end_pk = pks.filter(pk__gte=start_pk)[chunk_size]
+        except IndexError:
+            break
+        yield queryset.filter(pk__gte=start_pk, pk__lt=end_pk)
+        start_pk = end_pk
+    yield queryset.filter(pk__gte=start_pk)
+
+
+def update_file_sizes(resources, refreshed_weeks=None, modified_weeks=None):
+    total_resources = len(resources)
+    print(f"Updating file sizes for {total_resources} resources in Django")
+    res_count = 1
+    for res in resources:
+        print(f"\n{res_count}/{total_resources}: Updating file sizes for resource {res.short_id}")
+        start_time = time.time()
+        res_files = filter_files(res.files, refreshed_weeks=refreshed_weeks, modified_weeks=modified_weeks)
+        num_files = res_files.count()
+        print(f"Total files to update from resource {res.short_id}: {num_files}")
+        if num_files == 0:
+            print(f"Resource {res.short_id} has no files")
+            continue
+        print(f'{current_site}/resource/{res.short_id}: currently {res.size} bytes')
+        file_counter = 0
+        print("Updating files:", end=' ')
+        for res_file in res_files.iterator():
+            res_file.calculate_size(resource=res, save=True)
+            file_counter += 1
+            print(file_counter, end=', ')
+            if res_file._size <= 0:
+                print(f"File {res_file.short_path} was not found in iRODS.")
+        time_spent = time.time() - start_time
+        print(f"\nTime spent for resource {res.short_id}: {time_spent} seconds")
+        print(f"Updated {file_counter} files for resource {res.short_id}")
+        res_count += 1
+
+
+def filter_files(file_queryset, refreshed_weeks=None, modified_weeks=None):
+    if refreshed_weeks:
+        print(f"Only including files whose metadata has not been refreshed in the last {refreshed_weeks} weeks")
+        cuttoff_time = timezone.now() - timedelta(weeks=refreshed_weeks)
+        file_queryset = file_queryset.filter(Q(filesize_cache_updated__lte=cuttoff_time)
+                                             | Q(filesize_cache_updated__isnull=True))
+    if modified_weeks:
+        print(f"Only including resource files that have been modified in the last {modified_weeks} weeks")
+        cuttoff_time = timezone.now() - timedelta(weeks=modified_weeks)
+        file_queryset = file_queryset.filter(_modified_time__gte=cuttoff_time)
+    file_queryset = file_queryset.order_by(F('filesize_cache_updated').asc(nulls_first=True))
+    return file_queryset
+
+
+class Command(BaseCommand):
+    help = "Reset the filesize cached in django. Optionally update the file size cache by querying iRODS."
+
+    def add_arguments(self, parser):
+        parser.add_argument('--update', action='store_true', help='update file size cache in Django by querying iRODS')
+        parser.add_argument('--uid', type=int, help='filter to just a single user by uid')
+        parser.add_argument('--resource_id', type=str , help='filter to just a single resource by id')
+        parser.add_argument('--min_quota_django_model', type=int, help='filter to django UserQuota above this (in GB)')
+        parser.add_argument('--refreshed_weeks', type=int, dest='refreshed_weeks',
+                            help='include files not refreshed in the last X weeks')
+        parser.add_argument('--modified_weeks', type=int,
+                            help='include files that have been modified in the last X weeks')
+
+    def handle(self, *args, **options):
+        update = options['update'] if options['update'] else False
+        uid = options['uid'] if options['uid'] else None
+        resource_id = options['resource_id'] if options['resource_id'] else None
+        min_quota_django_model = int(options['min_quota_django_model']) if options['min_quota_django_model'] else 0
+        refreshed_weeks = options['refreshed_weeks'] if options['refreshed_weeks'] else None
+        modified_weeks = options['modified_weeks'] if options['modified_weeks'] else None
+
+        resources_to_modify = []
+
+        if resource_id:
+            if min_quota_django_model or uid:
+                raise CommandError("Cannot filter by resource_id in addition to min_quota_django_model or uid")
+            try:
+                res = BaseResource.objects.get(short_id=resource_id)
+                resources_to_modify = [res]
+            except ObjectDoesNotExist as e:
+                print(f"Resource with id {resource_id} not found: {e}")
+                raise CommandError(f"Resource with id {resource_id} not found: {e}")
+        elif uid:
+            if min_quota_django_model:
+                raise CommandError("Cannot filter by uid in addition to min_quota_django_model")
+            try:
+                user = User.objects.get(id=uid, is_active=True, is_superuser=False)
+                resources_to_modify = user.uaccess.owned_resources
+            except User.DoesNotExist:
+                print(f'Active user with id {uid} not found')
+        elif min_quota_django_model > 0:
+            uqs = UserQuota.objects.filter(user__is_active=True) \
+                .filter(user__is_superuser=False)
+            # TODO: once the quota system is moved from irods to django, used_value can no longer be filtered on
+            # It will be a property, so this will need to be updated
+            if min_quota_django_model > 0:
+                uqs = uqs.filter(used_value__gt=min_quota_django_model).order_by('-used_value')
+            num_uqs = uqs.count()
+            counter = 1
+            print(f'Found {num_uqs} users with quotas. Filtering out users with < {min_quota_django_model}GB')
+            start_time = time.time()
+            for uq in uqs:
+                user = uq.user
+                print(f'{counter}/{num_uqs}: \
+                      Getting resources for user: {user.username}, {current_site}/user/{user.id}/')
+                owned_resources = user.uaccess.owned_resources
+                for res in owned_resources:
+                    if res.get_quota_holder() == user:
+                        resources_to_modify.append(res)
+                counter += 1
+            time_spent = time.time() - start_time
+            print(f"Time spent collecting resources for {num_uqs} users: {time_spent} seconds")
+        else:
+            res_files = ResourceFile.objects.all()
+            if not refreshed_weeks and not modified_weeks:
+                # updating all files will time out, so we don't permit it.
+                raise CommandError("No filter specified. You must specify at least one filter.")
+            res_files = filter_files(res_files, refreshed_weeks=refreshed_weeks, modified_weeks=modified_weeks)
+            num_files = res_files.count()
+            print(f"Total files: {num_files}")
+            if update and num_files > 0:
+                file_counter = 1
+                chunk_number = 1
+                for chunk in chunked_queryset(res_files):
+                    start_time = time.time()
+                    print(f"Chunk {chunk_number}/{(num_files // _BATCH_SIZE) + 1}")
+                    for res_file in chunk.iterator():
+                        print(file_counter, end=', ')
+                        res_file.calculate_size(resource=res_file.resource, save=True)
+                        file_counter += 1
+                    time_spent = time.time() - start_time
+                    print(f"Time spent for chunk {chunk_number}: {time_spent} seconds")
+                    chunk_number += 1
+            else:
+                # reset the cache for the files
+                res_files.update(_size=-1)
+            print("Done")
+            return
+
+        if update:
+            update_file_sizes(resources_to_modify, refreshed_weeks=refreshed_weeks, modified_weeks=modified_weeks)
+        else:
+            for res in resources_to_modify:
+                print(f'Resetting filesizes in {current_site}/resource/{res.short_id}')
+
+                # using res.files instead of res.files.exclude(size=0) in case 0 values are cached incorrectly
+                res_files = filter_files(res.files, refreshed_weeks=refreshed_weeks, modified_weeks=modified_weeks)
+                res_files.update(_size=-1)
+
+                # set the updated date to now so that nightly celery task can update the size
+                res.updated = timezone.now().isoformat()
+                res_modified_date = res.metadata.dates.all().filter(type='modified').first()
+                if res_modified_date:
+                    res.metadata.update_element('date', res_modified_date.id)
+                else:
+                    res.metadata.create_element('date', type='modified', start_date=res.updated)
+        print("Done")
diff --git a/hs_core/migrations/0080_resourcefile_filesize_cache_updated.py b/hs_core/migrations/0080_resourcefile_filesize_cache_updated.py
new file mode 100644
index 0000000000..debdb1e518
--- /dev/null
+++ b/hs_core/migrations/0080_resourcefile_filesize_cache_updated.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-04-22 20:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hs_core', '0079_baseresource_bag_last_downloaded'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourcefile',
+            name='filesize_cache_updated',
+            field=models.DateTimeField(null=True),
+        ),
+    ]
diff --git a/hs_core/models.py b/hs_core/models.py
index 4ef8274af0..e090f5a2be 100755
--- a/hs_core/models.py
+++ b/hs_core/models.py
@@ -2599,8 +2599,9 @@ def is_schema_json_file(self, file_path):
         Note: this will return true for any file that ends with the schema.json ending
         We are taking the risk that user might create a file with the same filename ending
         """
-        from hs_file_types.models.base import SCHEMA_JSON_FILE_ENDSWITH
-        if file_path.endswith(SCHEMA_JSON_FILE_ENDSWITH):
+        from hs_file_types.enums import AggregationMetaFilePath
+
+        if file_path.endswith(AggregationMetaFilePath.SCHEMA_JSON_FILE_ENDSWITH):
             return True
         return False
 
@@ -2618,10 +2619,10 @@ def is_metadata_xml_file(self, file_path):
         Note: this will return true for any file that ends with the metadata endings
         We are taking the risk that user might create a file with the same filename ending
         """
-        from hs_file_types.models.base import (METADATA_FILE_ENDSWITH,
-                                               RESMAP_FILE_ENDSWITH)
-        if not (file_path.endswith(METADATA_FILE_ENDSWITH)
-                or file_path.endswith(RESMAP_FILE_ENDSWITH)):
+        from hs_file_types.enums import AggregationMetaFilePath
+
+        if not (file_path.endswith(AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+                or file_path.endswith(AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)):
             return False
         return True
 
@@ -3103,6 +3104,9 @@ class Meta:
     _modified_time = models.DateTimeField(null=True, blank=True)
     _checksum = models.CharField(max_length=255, null=True, blank=True)
 
+    # for tracking when size was last compared with irods
+    filesize_cache_updated = models.DateTimeField(null=True)
+
     def __str__(self):
         """Return resource filename or federated resource filename for string representation."""
         if self.resource.resource_federation_path:
@@ -3118,7 +3122,7 @@ def banned_symbols(cls):
     @classmethod
     def system_meta_fields(cls):
         """returns a list of system metadata fields"""
-        return ['_size', '_modified_time', '_checksum']
+        return ['_size', '_modified_time', '_checksum', 'filesize_cache_updated']
 
     @classmethod
     def create(cls, resource, file, folder='', source=None):
@@ -3372,6 +3376,7 @@ def calculate_size(self, resource=None, save=True):
                     self.resource_file.name == ''
             try:
                 self._size = self.fed_resource_file.size
+                self.filesize_cache_updated = now()
             except (SessionException, ValidationError):
                 logger = logging.getLogger(__name__)
                 logger.warning("file {} not found in iRODS".format(self.storage_path))
@@ -3382,12 +3387,13 @@ def calculate_size(self, resource=None, save=True):
                     self.fed_resource_file.name == ''
             try:
                 self._size = self.resource_file.size
+                self.filesize_cache_updated = now()
             except (SessionException, ValidationError):
                 logger = logging.getLogger(__name__)
                 logger.warning("file {} not found in iRODS".format(self.storage_path))
                 self._size = 0
         if save:
-            self.save(update_fields=["_size"])
+            self.save(update_fields=["_size", "filesize_cache_updated"])
 
     def set_system_metadata(self, resource=None, save=True):
         """Set system metadata (size, modified time, and checksum) for a file.
diff --git a/hs_core/pubsub_discovery_processor.py b/hs_core/pubsub_discovery_processor.py
new file mode 100644
index 0000000000..79fe165e46
--- /dev/null
+++ b/hs_core/pubsub_discovery_processor.py
@@ -0,0 +1,21 @@
+import json
+
+from django.conf import settings
+from google.auth import jwt
+from google.cloud import pubsub_v1
+
+publish_discoverable = getattr(settings, "PUBLISH_DISCOVERABLE", False)
+
+if publish_discoverable:
+    audience = "https://pubsub.googleapis.com/google.pubsub.v1.Publisher"
+    credentials = jwt.Credentials.from_service_account_file('service-account-pubsub.json', audience=audience)
+    credentials_pub = credentials.with_claims(audience=audience)
+    publisher = pubsub_v1.PublisherClient(credentials=credentials_pub)
+
+    topic_name = 'projects/{project_id}/topics/{topic}'.format(
+        project_id="apps-320517", topic='discovery_ids')
+
+
+def pub_update(resource_id: str, removed: bool = False):
+    if publish_discoverable:
+        publisher.publish(topic_name, json.dumps({"resource_id": resource_id, "removed": removed}).encode("utf-8"))
diff --git a/hs_file_types/enums.py b/hs_file_types/enums.py
new file mode 100644
index 0000000000..aa0e65eac2
--- /dev/null
+++ b/hs_file_types/enums.py
@@ -0,0 +1,7 @@
+import enum
+
+
+class AggregationMetaFilePath(str, enum.Enum):
+    RESMAP_FILE_ENDSWITH = "_resmap.xml"
+    METADATA_FILE_ENDSWITH = "_meta.xml"
+    SCHEMA_JSON_FILE_ENDSWITH = "_schema.json"
diff --git a/hs_file_types/models/base.py b/hs_file_types/models/base.py
index 11bb3f4803..b73d658167 100755
--- a/hs_file_types/models/base.py
+++ b/hs_file_types/models/base.py
@@ -25,10 +25,7 @@
     set_dirty_bag_flag, add_file_to_resource, resource_modified, get_file_from_irods
 from hs_core.models import ResourceFile, AbstractMetaDataElement, Coverage
 from hs_core.signals import post_remove_file_aggregation
-
-RESMAP_FILE_ENDSWITH = "_resmap.xml"
-METADATA_FILE_ENDSWITH = "_meta.xml"
-SCHEMA_JSON_FILE_ENDSWITH = "_schema.json"
+from ..enums import AggregationMetaFilePath
 
 
 class NestedLogicalFileMixin(object):
@@ -1524,9 +1521,9 @@ def get_xml_file_name(self, resmap=True):
         xml_file_name, _ = os.path.splitext(xml_file_name)
 
         if resmap:
-            xml_file_name += RESMAP_FILE_ENDSWITH
+            xml_file_name += AggregationMetaFilePath.RESMAP_FILE_ENDSWITH
         else:
-            xml_file_name += METADATA_FILE_ENDSWITH
+            xml_file_name += AggregationMetaFilePath.METADATA_FILE_ENDSWITH
         return xml_file_name
 
     def xml_file_short_path(self, resmap=True):
diff --git a/hs_file_types/models/base_model_program_instance.py b/hs_file_types/models/base_model_program_instance.py
index 8d54fb6f23..455d631612 100644
--- a/hs_file_types/models/base_model_program_instance.py
+++ b/hs_file_types/models/base_model_program_instance.py
@@ -13,7 +13,8 @@
 from hs_core.models import ResourceFile
 from hs_core.signals import post_remove_file_aggregation
 from hs_file_types.models import AbstractLogicalFile
-from hs_file_types.models.base import FileTypeContext, SCHEMA_JSON_FILE_ENDSWITH
+from hs_file_types.models.base import FileTypeContext
+from hs_file_types.enums import AggregationMetaFilePath
 from hydroshare import settings
 
 
@@ -35,7 +36,7 @@ def aggregation_name(self):
         """Returns aggregation name as per the aggregation naming rule defined in issue#2568"""
 
         if self.folder:
-            # this model program/instance aggregation has ben created from a folder
+            # this model program/instance aggregation has been created from a folder
             # aggregation folder path is the aggregation name
             return self.folder
         else:
@@ -60,7 +61,7 @@ def schema_short_file_path(self):
 
         json_file_name, _ = os.path.splitext(json_file_name)
 
-        json_file_name += SCHEMA_JSON_FILE_ENDSWITH
+        json_file_name += AggregationMetaFilePath.SCHEMA_JSON_FILE_ENDSWITH
 
         if self.folder:
             file_folder = self.folder
diff --git a/hydroshare/local_settings.cutover b/hydroshare/local_settings.cutover
index f79002f116..9b58a24b33 100644
--- a/hydroshare/local_settings.cutover
+++ b/hydroshare/local_settings.cutover
@@ -100,6 +100,7 @@ IRODS_ICOMMANDS_PATH = '/usr/bin'
 IRODS_HOST = 'hydrotest41.renci.org'
 IRODS_PORT = '1247'
 IRODS_DEFAULT_RESOURCE = 'hydrotest41Resc'
+IRODS_SINGLE_RESC_FOR_QUERY='hydrotest41Resc'
 IRODS_HOME_COLLECTION = '/hydrotest41Zone/home/hsproxy'
 IRODS_CWD = '/hydrotest41Zone/home/hsproxy'
 IRODS_ZONE = 'hydrotest41Zone'
diff --git a/hydroshare/local_settings.template b/hydroshare/local_settings.template
index b4e851b161..9c029cbb1a 100644
--- a/hydroshare/local_settings.template
+++ b/hydroshare/local_settings.template
@@ -100,6 +100,7 @@ IRODS_ICOMMANDS_PATH = '/usr/bin'
 IRODS_HOST = 'data.local.org'
 IRODS_PORT = '1247'
 IRODS_DEFAULT_RESOURCE = 'hydroshareReplResc'
+IRODS_SINGLE_RESC_FOR_QUERY = 'hydroshareReplResc'
 IRODS_HOME_COLLECTION = '/hydroshareZone/home/wwwHydroProxy'
 IRODS_CWD = '/hydroshareZone/home/wwwHydroProxy'
 IRODS_ZONE = 'hydroshareZone'
diff --git a/hydroshare/settings.py b/hydroshare/settings.py
index 12398b8812..3754dc14a7 100755
--- a/hydroshare/settings.py
+++ b/hydroshare/settings.py
@@ -146,7 +146,7 @@ def _(s): return s # noqa
 NIGHTLY_RESOURCE_REPAIR_DURATION = 60 * 60
 
 # Integer seconds that worker should allocate every night to generating filesystem metadata
-NIGHTLY_GENERATE_FILESYSTEM_METADATA_DURATION = 60 * 60
+NIGHTLY_GENERATE_FILESYSTEM_METADATA_DURATION = 60 * 60 * 4  # 4 hours
 
 # Should resource owners be notified of automated resource repair?
 NOTIFY_OWNERS_AFTER_RESOURCE_REPAIR = False
@@ -212,6 +212,11 @@ def _(s): return s # noqa
 OIDC_KEYCLOAK_URL = "https://auth.cuahsi.org/"
 OIDC_KEYCLOAK_REALM = "CUAHSI"
 
+# Enables publishing discoverable resources to Google PubSub (hs_core/pubsub_discovery_processor.py)
+# Requires a service account json file with PubSub permissions to be placed at the root of the
+# project with the name service-account-pubsub.json
+PUBLISH_DISCOVERABLE = False
+
 # List of finder classes that know how to find static files in
 # various locations.
 STATICFILES_FINDERS = (
diff --git a/irods/env-files/use-local-irods.env b/irods/env-files/use-local-irods.env
index 2be08d2580..892c138b2d 100644
--- a/irods/env-files/use-local-irods.env
+++ b/irods/env-files/use-local-irods.env
@@ -9,6 +9,7 @@ IRODS_ICOMMANDS_PATH='/usr/bin'
 IRODS_HOST='data.local.org'
 IRODS_PORT=1247
 IRODS_DEFAULT_RESOURCE='hydroshareReplResc'
+IRODS_SINGLE_RESC_FOR_QUERY='hydroshareReplResc'
 IRODS_HOME_COLLECTION='/hydroshareZone/home/wwwHydroProxy'
 IRODS_CWD='/hydroshareZone/home/wwwHydroProxy'
 IRODS_ZONE='hydroshareZone'
diff --git a/local-dev.yml b/local-dev.yml
index 97600236ec..efcbb2871a 100755
--- a/local-dev.yml
+++ b/local-dev.yml
@@ -21,6 +21,7 @@ services:
       IRODS_HOST: 'data.local.org'
       IRODS_PORT: '1247'
       IRODS_DEFAULT_RESOURCE: 'hydroshareReplResc'
+      IRODS_SINGLE_RESC_FOR_QUERY: 'hydroshareReplResc'
       IRODS_HOME_COLLECTION: '/hydroshareZone/home/wwwHydroProxy'
       IRODS_ZONE: 'hydroshareZone'
       IRODS_USERNAME: 'wwwHydroProxy'
@@ -63,6 +64,7 @@ services:
       IRODS_HOST: 'users.local.org'
       IRODS_PORT: '1247'
       IRODS_DEFAULT_RESOURCE: 'hydroshareReplResc'
+      IRODS_SINGLE_RESC_FOR_QUERY: 'hydroshareReplResc'
       IRODS_HOME_COLLECTION: '/hydroshareZone/home/wwwHydroProxy'
       IRODS_ZONE: 'hydroshareZone'
       IRODS_USERNAME: 'wwwHydroProxy'
diff --git a/theme/management/commands/report_resource_size_vs_quota_inconsistency.py b/theme/management/commands/report_resource_size_vs_quota_inconsistency.py
new file mode 100644
index 0000000000..829a67c32f
--- /dev/null
+++ b/theme/management/commands/report_resource_size_vs_quota_inconsistency.py
@@ -0,0 +1,222 @@
+
+import csv
+import math
+import time
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+from django.core.exceptions import ValidationError
+from django.conf import settings
+from django_irods.icommands import SessionException
+from django_irods.storage import IrodsStorage
+from django.utils.timezone import now
+
+from hs_core.hydroshare import convert_file_size_to_unit
+from hs_core.hydroshare import current_site_url
+from hs_core.models import ResourceFile
+from theme.models import UserQuota
+
+current_site = current_site_url()
+_BATCH_SIZE = settings.BULK_UPDATE_CREATE_BATCH_SIZE
+
+
+def get_dz_quota_usage_from_irods(username):
+    """
+    Query iRODS AVU to get quota usage for a user reported in iRODS quota microservices
+    :param username: the user name to get quota usage for.
+    :return: the quota usage from iRODS data zone
+    :raises: ValidationError if quota usage cannot be retrieved from iRODS
+    """
+    attname = username + '-usage'
+    istorage = IrodsStorage()
+    # get quota size for user in iRODS data zone by retrieving AVU set on irods bagit path
+    # collection
+    try:
+        uqDataZoneSize = istorage.getAVU(settings.IRODS_BAGIT_PATH, attname)
+        if uqDataZoneSize is None:
+            # user may not have resources in data zone, so corresponding quota size AVU may not
+            # exist for this user
+            uqDataZoneSize = -1
+        else:
+            uqDataZoneSize = float(uqDataZoneSize)
+    except SessionException:
+        # user may not have resources in data zone, so corresponding quota size AVU may not exist
+        # for this user
+        uqDataZoneSize = -1
+
+    if uqDataZoneSize < 0:
+        err_msg = 'no quota size AVU in data zone and user zone for user {}'.format(username)
+        print(err_msg)
+        raise ValidationError(err_msg)
+    else:
+        used_val = uqDataZoneSize
+    return used_val
+
+
+class Command(BaseCommand):
+    help = "Output potential quota inconsistencies between iRODS and Django resource aggregate filesize for all users"
+
+    def add_arguments(self, parser):
+        parser.add_argument('output_file_name_with_path', type=str, help='output file name with path')
+        parser.add_argument('--update', action='store_true', help='fix inconsistencies by recalculating in django')
+        parser.add_argument('--reset', action='store_true', help='reset resource file size in django when inconsistent')
+        parser.add_argument('--no_compare', action='store_true', help='reset/update filesizes without comparing quotas')
+        parser.add_argument('--uid', type=str, help='filter to just a single user by uid')
+        parser.add_argument('--min_quota_use_avu', type=int,
+                            help='filter to just users with AVU quota use above this size in GB')
+        parser.add_argument('--min_quota_django_model', type=int,
+                            help='filter to django UserQuota use above this size in GB')
+        parser.add_argument('--desc', action='store_true', help='order by descending quota use in iRODS Datazone')
+        parser.add_argument('--rel_tol', help='relative tolerance for comparing quota sizes, default is 0.01')
+
+    def handle(self, *args, **options):
+        uid = options['uid'] if options['uid'] else None
+        update = options['update'] if options['update'] else False
+        desc = options['desc'] if options['desc'] else False
+        reset = options['reset'] if options['reset'] else False
+        no_compare = options['no_compare'] if options['no_compare'] else False
+        min_quota_use_avu = int(options['min_quota_use_avu']) if options['min_quota_use_avu'] else 0
+        min_quota_django_model = int(options['min_quota_django_model']) if options['min_quota_django_model'] else 0
+        rel_tol = float(options['rel_tol']) if options['rel_tol'] else 0.01
+
+        if no_compare:
+            if not update and not reset:
+                print('Error: --no_compare option must be used with --update or --reset option')
+                return
+            if options['rel_tol']:
+                print('Warning: --rel_tol option is ignored when --no_compare is used')
+                print('Update or reset will be done even if comparison is within the tolerance.')
+            print('Using --no_compare option. Will reset/update file sizes even when no incosistency is detected.')
+
+        if min_quota_use_avu and min_quota_django_model:
+            print('Using --min_quota_use_avu and --min_quota_django_model')
+            if min_quota_use_avu != min_quota_django_model:
+                print('Warning: min_quota_use_avu and min_quota_django_model are different')
+            print('First quotas will be filtered by django model. Then users will be filtered by iRODS AVU.')
+
+        if update and reset:
+            print('Cannot use both --update and --reset options together')
+            return
+        uqs = UserQuota.objects.filter(user__is_active=True) \
+            .filter(user__is_superuser=False)
+        if uid:
+            try:
+                user = User.objects.get(id=uid, is_active=True, is_superuser=False)
+            except User.DoesNotExist:
+                print(f'Active user with id {uid} not found')
+            uqs = uqs.filter(user=user)
+        if min_quota_django_model > 0:
+            # TODO: once the quota system is moved from irods to django, used_value can no longer be filtered on
+            # It will be a property, so this will need to be updated
+            uqs = uqs.filter(used_value__gt=min_quota_django_model)
+        if desc:
+            uqs = uqs.order_by('-used_value')
+        num_uqs = uqs.count()
+        print(f"Number of user quotas to check: {num_uqs}")
+        counter = 1
+        csvfile = open(options['output_file_name_with_path'], 'w')
+        w = csv.writer(csvfile)
+        fields = [
+            'User',
+            'Starting summed resource sizes in Django',
+            'Updated summed resource sizes in Django',
+            'Quota reported in iRODS Datazone'
+        ]
+        w.writerow(fields)
+        for uq in uqs:
+            start_time = time.time()
+            user = uq.user
+            print("\n" + "*" * 80)
+            print(f'{counter}/{num_uqs}: Checking quota for user {user.username}, {current_site}/user/{user.id}/')
+            used_value_irods_dz = 0.0
+            try:
+                used_value_irods_dz = get_dz_quota_usage_from_irods(user.username)
+            except ValidationError:
+                # purposely ignore the error and continue to check for inconsistencies
+                # assumes that the user may not have any resources in the data zone
+                pass
+            used_value_irods_dz = convert_file_size_to_unit(used_value_irods_dz, "gb")
+            if used_value_irods_dz < min_quota_use_avu:
+                print(f"Quota usage in iRODS Datazone AVU is below {min_quota_use_avu} GB. Skipping.")
+                continue
+            # sum the resources sizes for all resources that the user is the quota holder for
+            owned_resources = user.uaccess.owned_resources
+            # filter resources above size limit
+            held_resources = []
+            total_size = 0
+            for res in owned_resources:
+                if res.get_quota_holder() == user:
+                    held_resources.append(res)
+                    res_size = res.size
+                    print(f"Resource {res.short_id} is held by {user.username}, size: {res_size} bytes")
+                    total_size += res_size
+            converted_total_size_django = convert_file_size_to_unit(int(total_size), 'gb')
+            print(f"Quota usage in iRODS Datazone from AVU: {used_value_irods_dz} GB")
+            print(f"Aggregate size of resources in Django: {converted_total_size_django} GB")
+            is_close = math.isclose(used_value_irods_dz, converted_total_size_django, rel_tol=rel_tol)
+            if not is_close or no_compare:
+                django_updated = ''
+                print('{} reported in django vs {} reported in iRODS for user {}'.format(
+                    converted_total_size_django, used_value_irods_dz, user.username), flush=True)
+
+                if update:
+                    print("Attempting to fix any inconsistency by updating file sizes in django")
+                    res_files = []
+                    updated_size = 0
+                    for res in held_resources:
+                        res_files = res.files.exclude(_size=0)
+                        num_files = res_files.count()
+                        print(f"Total files in resource {res.short_id}: {num_files}")
+                        if num_files == 0:
+                            print(f"Resource {res.short_id} has no files.")
+                            continue
+                        print(f'{current_site}/resource/{res.short_id}: currently {res.size} bytes')
+                        file_counter = 0
+                        # exclude files with size 0 as they don't exist in iRODS
+                        print("Updating files:")
+                        for res_file in res_files.iterator():
+                            # this is an expensive operation (3 irods calls per file) - about 1 min for 100 files
+                            # size, checksum and modified time are obtained from irods and assigned to
+                            # relevant fields of the resource file object
+                            res_file.calculate_size(resource=res, save=False)
+                            file_counter += 1
+                            print(f"{file_counter}/{num_files}")
+                            if res_file._size <= 0:
+                                print(f"File {res_file.short_path} was not found in iRODS.")
+
+                        ResourceFile.objects.bulk_update(res_files, '_size', batch_size=_BATCH_SIZE)
+                        print(f"Updated {file_counter} files for resource {res.short_id}")
+                        res.refresh_from_db()
+                        print(f'{current_site}/resource/{res.short_id}: now {res.size} bytes')
+                        # keep track of the updated size so that we can compare again
+                        updated_size += res.size
+                    converted_updated_size_django = convert_file_size_to_unit(int(updated_size), 'gb')
+                    django_updated = converted_updated_size_django
+                    if not math.isclose(used_value_irods_dz, converted_updated_size_django, rel_tol=rel_tol):
+                        print("Even after updating, an inconsistency remains!")
+                        print(f"Quota usage in iRODS Datazone AVU: {used_value_irods_dz} GB")
+                        print(f"Updated Total size of resources in Django: {converted_updated_size_django} GB")
+
+                elif reset:
+                    print("Resetting file size cache in django")
+                    for res in held_resources:
+                        print(f'Resetting all filesizes in {current_site}/resource/{res.short_id}')
+                        res.files.update(_size=-1)
+                        # set the updated date to now so that nightly celery task can update the size
+                        res.updated = now().isoformat()
+                        res_modified_date = res.metadata.dates.all().filter(type='modified').first()
+                        if res_modified_date:
+                            res.metadata.update_element('date', res_modified_date.id)
+                        else:
+                            res.metadata.create_element('date', type='modified', start_date=res.updated)
+                    django_updated = 'reset'
+                else:
+                    print('No action taken. Use --update or --reset to fix inconsistencies')
+                w.writerow([uq.user.username, converted_total_size_django, django_updated, used_value_irods_dz])
+            else:
+                print(f"Quota deemed consistent for user {user.username}, using {rel_tol} relative tolerance")
+            counter += 1
+            # Print the amount of time spent for this user
+            time_spent = time.time() - start_time
+            print(f"Time spent for user {user.username}: {time_spent} seconds")
+        csvfile.close()
+        print("Done")


Test patch:
diff --git a/hs_composite_resource/tests/test_composite_resource.py b/hs_composite_resource/tests/test_composite_resource.py
index 4256db41c6..394588641e 100644
--- a/hs_composite_resource/tests/test_composite_resource.py
+++ b/hs_composite_resource/tests/test_composite_resource.py
@@ -34,8 +34,7 @@
                                   ModelProgramLogicalFile, NetCDFLogicalFile,
                                   RefTimeseriesLogicalFile,
                                   TimeSeriesLogicalFile)
-from hs_file_types.models.base import (METADATA_FILE_ENDSWITH,
-                                       RESMAP_FILE_ENDSWITH)
+from hs_file_types.enums import AggregationMetaFilePath
 from hs_file_types.tests.utils import CompositeResourceTestMixin
 
 
@@ -3601,8 +3600,8 @@ def test_unzip(self):
         self.assertEqual(self.composite_resource.files.count(), 3)
         for res_file in self.composite_resource.files.all():
             # there should not be any resource files ending with _meta.xml or _resmap.xml
-            self.assertFalse(res_file.file_name.endswith(METADATA_FILE_ENDSWITH))
-            self.assertFalse(res_file.file_name.endswith(RESMAP_FILE_ENDSWITH))
+            self.assertFalse(res_file.file_name.endswith(AggregationMetaFilePath.METADATA_FILE_ENDSWITH))
+            self.assertFalse(res_file.file_name.endswith(AggregationMetaFilePath.RESMAP_FILE_ENDSWITH))
             # check file level system metadata
             self.assertGreater(res_file._size, 0)
             self.assertGreater(len(res_file._checksum), 0)
diff --git a/hs_file_types/tests/test_generic_metadata.py b/hs_file_types/tests/test_generic_metadata.py
index ceb217d1a7..8ebb530be9 100644
--- a/hs_file_types/tests/test_generic_metadata.py
+++ b/hs_file_types/tests/test_generic_metadata.py
@@ -10,7 +10,7 @@
 from hs_core.testing import MockIRODSTestCaseMixin
 from hs_core.views.utils import move_or_rename_file_or_folder, create_folder
 from hs_file_types.models import GenericLogicalFile, GenericFileMetaData
-from hs_file_types.models.base import METADATA_FILE_ENDSWITH, RESMAP_FILE_ENDSWITH
+from ..enums import AggregationMetaFilePath
 from .utils import CompositeResourceTestMixin
 
 
@@ -66,7 +66,7 @@ def test_create_aggregation_2(self):
         self.create_composite_resource()
         new_folder = 'generic_folder'
         ResourceFile.create_folder(self.composite_resource, new_folder)
-        # add the the txt file to the resource at the above folder
+        # add the txt file to the resource at the above folder
         self.add_file_to_resource(file_to_add=self.generic_file, upload_folder=new_folder)
         # there should be one resource file
         self.assertEqual(self.composite_resource.files.all().count(), 1)
@@ -680,8 +680,8 @@ def test_aggregation_xml_file_paths(self):
         logical_file = res_file.logical_file
         # file name without extension
         res_file_name, _ = os.path.splitext(res_file.file_name)
-        expected_meta_path = '{0}{1}'.format(res_file_name, METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}{1}'.format(res_file_name, RESMAP_FILE_ENDSWITH)
+        expected_meta_path = '{0}{1}'.format(res_file_name, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}{1}'.format(res_file_name, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
 
@@ -696,8 +696,10 @@ def test_aggregation_xml_file_paths(self):
         res_file = self.composite_resource.files.first()
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
-        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name, METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name, RESMAP_FILE_ENDSWITH)
+        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                 AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
 
@@ -709,8 +711,10 @@ def test_aggregation_xml_file_paths(self):
         res_file = self.composite_resource.files.first()
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
-        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name, METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name, RESMAP_FILE_ENDSWITH)
+        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                 AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
 
@@ -724,8 +728,9 @@ def test_aggregation_xml_file_paths(self):
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
         expected_meta_path = '{0}/{1}{2}'.format(folder_rename, res_file_name,
-                                                 METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}/{1}{2}'.format(folder_rename, res_file_name, RESMAP_FILE_ENDSWITH)
+                                                 AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}/{1}{2}'.format(folder_rename, res_file_name,
+                                                AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
diff --git a/hs_file_types/tests/test_geofeature_metadata.py b/hs_file_types/tests/test_geofeature_metadata.py
index e63b56ee68..c110c56619 100644
--- a/hs_file_types/tests/test_geofeature_metadata.py
+++ b/hs_file_types/tests/test_geofeature_metadata.py
@@ -9,12 +9,20 @@
 from hs_core.models import Coverage, ResourceFile
 from hs_core.testing import MockIRODSTestCaseMixin
 from hs_core.views.utils import remove_folder, move_or_rename_file_or_folder
-from hs_file_types.models import GeoFeatureLogicalFile, GenericLogicalFile, GenericFileMetaData, \
-    GeoFeatureFileMetaData
-from hs_file_types.models.base import RESMAP_FILE_ENDSWITH, METADATA_FILE_ENDSWITH
+from hs_file_types.models import (
+    GeoFeatureLogicalFile,
+    GenericLogicalFile,
+    GenericFileMetaData,
+    GeoFeatureFileMetaData,
+)
+from hs_file_types.enums import AggregationMetaFilePath
 from hs_file_types.models.geofeature import FieldInformation, GeometryInformation, OriginalCoverageGeofeature
-from .utils import assert_geofeature_file_type_metadata, CompositeResourceTestMixin, \
-    get_path_with_no_file_extension, get_number_of_decimal_places
+from .utils import (
+    assert_geofeature_file_type_metadata,
+    CompositeResourceTestMixin,
+    get_path_with_no_file_extension,
+    get_number_of_decimal_places,
+)
 
 
 class GeoFeatureFileTypeTest(MockIRODSTestCaseMixin, TransactionTestCase,
@@ -646,10 +654,10 @@ def test_aggregation_folder_rename(self):
 
         # test aggregation xml file paths
         shp_file_path = get_path_with_no_file_extension(logical_file.aggregation_name)
-        expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # test renaming folder
@@ -668,10 +676,12 @@ def test_aggregation_folder_rename(self):
         # test aggregation xml file paths
         new_shp_file_path = get_path_with_no_file_extension(logical_file.aggregation_name)
         self.assertNotEqual(new_shp_file_path, shp_file_path)
-        expected_meta_file_path = '{0}{1}'.format(new_shp_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(new_shp_file_path,
+                                                  AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(new_shp_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(new_shp_file_path,
+                                                 AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
@@ -702,10 +712,10 @@ def test_aggregation_parent_folder_rename(self):
 
         # test aggregation xml file paths
         shp_file_path = get_path_with_no_file_extension(logical_file.aggregation_name)
-        expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # create a folder to be the parent folder of the aggregation folder
@@ -740,10 +750,11 @@ def test_aggregation_parent_folder_rename(self):
         # test aggregation xml file paths after folder rename
         new_shp_file_path = get_path_with_no_file_extension(logical_file.aggregation_name)
         self.assertNotEqual(new_shp_file_path, shp_file_path)
-        expected_meta_file_path = '{0}{1}'.format(new_shp_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(new_shp_file_path,
+                                                  AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(new_shp_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(new_shp_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
@@ -772,10 +783,10 @@ def test_aggregation_folder_move(self):
         logical_file = res_file.logical_file
         shp_file_path = get_path_with_no_file_extension(logical_file.aggregation_name)
         # test aggregation xml file paths
-        expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # create a folder to move the aggregation folder there
@@ -798,10 +809,10 @@ def test_aggregation_folder_move(self):
         shp_file_path = get_path_with_no_file_extension(logical_file.aggregation_name)
 
         # test aggregation xml file paths
-        expected_meta_file_path = '{0}{1}'.format(shp_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(shp_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(shp_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
diff --git a/hs_file_types/tests/test_netcdf_metadata.py b/hs_file_types/tests/test_netcdf_metadata.py
index 2bf0be2b0e..94e2f93d37 100644
--- a/hs_file_types/tests/test_netcdf_metadata.py
+++ b/hs_file_types/tests/test_netcdf_metadata.py
@@ -10,9 +10,12 @@
 from hs_core.testing import MockIRODSTestCaseMixin
 from hs_core.views.utils import remove_folder, move_or_rename_file_or_folder
 from hs_file_types.models import NetCDFLogicalFile, NetCDFFileMetaData, OriginalCoverage, Variable
-from hs_file_types.models.base import METADATA_FILE_ENDSWITH, RESMAP_FILE_ENDSWITH
-from .utils import assert_netcdf_file_type_metadata, CompositeResourceTestMixin, \
-    get_path_with_no_file_extension
+from hs_file_types.enums import AggregationMetaFilePath
+from .utils import (
+    assert_netcdf_file_type_metadata,
+    CompositeResourceTestMixin,
+    get_path_with_no_file_extension,
+)
 
 
 class NetCDFFileTypeTest(MockIRODSTestCaseMixin, TransactionTestCase,
@@ -741,10 +744,10 @@ def test_aggregation_folder_rename(self):
 
         # test aggregation xml file paths
         nc_file_path = get_path_with_no_file_extension(nc_res_file.short_path)
-        expected_meta_file_path = '{0}{1}'.format(nc_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(nc_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # test renaming folder
@@ -764,10 +767,10 @@ def test_aggregation_folder_rename(self):
 
         # test aggregation xml file paths
         nc_file_path = get_path_with_no_file_extension(nc_res_file.short_path)
-        expected_meta_file_path = '{0}{1}'.format(nc_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(nc_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
@@ -800,10 +803,10 @@ def test_aggregation_parent_folder_rename(self):
 
         # test aggregation xml file paths
         nc_file_path = get_path_with_no_file_extension(nc_res_file.short_path)
-        expected_meta_file_path = '{0}{1}'.format(nc_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(nc_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # create a folder to be the parent folder of the aggregation folder
@@ -839,9 +842,9 @@ def test_aggregation_parent_folder_rename(self):
 
         # test aggregation xml file paths after folder rename
         nc_file_path = get_path_with_no_file_extension(nc_res_file.short_path)
-        expected_meta_file_path = '{0}{1}'.format(nc_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
-        expected_map_file_path = '{0}{1}'.format(nc_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
@@ -888,10 +891,10 @@ def test_aggregation_folder_move_1(self):
 
         # test aggregation xml file paths
         nc_file_path = get_path_with_no_file_extension(nc_res_file.short_path)
-        expected_meta_file_path = '{0}{1}'.format(nc_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(nc_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(nc_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
diff --git a/hs_file_types/tests/test_raster_metadata.py b/hs_file_types/tests/test_raster_metadata.py
index 30942298ef..5e96558058 100755
--- a/hs_file_types/tests/test_raster_metadata.py
+++ b/hs_file_types/tests/test_raster_metadata.py
@@ -11,7 +11,7 @@
 from hs_core.testing import MockIRODSTestCaseMixin
 from hs_core.views.utils import move_or_rename_file_or_folder
 from hs_file_types.models import GenericLogicalFile, GeoRasterFileMetaData, GeoRasterLogicalFile
-from hs_file_types.models.base import METADATA_FILE_ENDSWITH, RESMAP_FILE_ENDSWITH
+from hs_file_types.enums import AggregationMetaFilePath
 from hs_file_types.models.raster import BandInformation, CellInformation, OriginalCoverageRaster
 from .utils import CompositeResourceTestMixin, assert_raster_file_type_metadata, get_path_with_no_file_extension
 
@@ -979,10 +979,10 @@ def test_aggregation_folder_rename(self):
         aggregation_name = logical_file.aggregation_name
         # test aggregation xml file paths
         vrt_file_path = get_path_with_no_file_extension(aggregation_name)
-        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(vrt_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # test renaming folder
@@ -1002,10 +1002,10 @@ def test_aggregation_folder_rename(self):
 
         # test aggregation xml file paths
         vrt_file_path = get_path_with_no_file_extension(aggregation_name)
-        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(vrt_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
@@ -1037,10 +1037,10 @@ def test_aggregation_parent_folder_rename(self):
         # test aggregation xml file paths
         # test aggregation xml file paths
         vrt_file_path = get_path_with_no_file_extension(aggregation_name)
-        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
-        expected_map_file_path = '{0}{1}'.format(vrt_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # create a folder to be the parent folder of the aggregation folder
@@ -1075,10 +1075,10 @@ def test_aggregation_parent_folder_rename(self):
         aggregation_name = logical_file.aggregation_name
         # test aggregation xml file paths after folder rename
         vrt_file_path = get_path_with_no_file_extension(aggregation_name)
-        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, METADATA_FILE_ENDSWITH)
+        expected_meta_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
 
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
-        expected_map_file_path = '{0}{1}'.format(vrt_file_path, RESMAP_FILE_ENDSWITH)
+        expected_map_file_path = '{0}{1}'.format(vrt_file_path, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
 
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
diff --git a/hs_file_types/tests/test_reftimeseries_metadata.py b/hs_file_types/tests/test_reftimeseries_metadata.py
index aa4f76b664..052f7f6252 100644
--- a/hs_file_types/tests/test_reftimeseries_metadata.py
+++ b/hs_file_types/tests/test_reftimeseries_metadata.py
@@ -11,7 +11,7 @@
 from .utils import assert_ref_time_series_file_type_metadata, CompositeResourceTestMixin
 
 from hs_file_types.models import RefTimeseriesLogicalFile, RefTimeseriesFileMetaData
-from hs_file_types.models.base import METADATA_FILE_ENDSWITH, RESMAP_FILE_ENDSWITH
+from hs_file_types.enums import AggregationMetaFilePath
 
 
 class RefTimeseriesFileTypeTest(MockIRODSTestCaseMixin, TransactionTestCase,
@@ -234,8 +234,8 @@ def test_aggregation_xml_file_paths(self):
 
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
-        expected_meta_path = '{0}{1}'.format(res_file_name, METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}{1}'.format(res_file_name, RESMAP_FILE_ENDSWITH)
+        expected_meta_path = '{0}{1}'.format(res_file_name, AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}{1}'.format(res_file_name, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
 
@@ -250,8 +250,10 @@ def test_aggregation_xml_file_paths(self):
         res_file = self.composite_resource.files.first()
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
-        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name, METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name, RESMAP_FILE_ENDSWITH)
+        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                 AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
 
@@ -263,8 +265,10 @@ def test_aggregation_xml_file_paths(self):
         res_file = self.composite_resource.files.first()
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
-        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name, METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name, RESMAP_FILE_ENDSWITH)
+        expected_meta_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                 AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}/{1}{2}'.format(new_folder, res_file_name,
+                                                AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
 
@@ -278,8 +282,9 @@ def test_aggregation_xml_file_paths(self):
         logical_file = res_file.logical_file
         res_file_name, _ = os.path.splitext(res_file.file_name)
         expected_meta_path = '{0}/{1}{2}'.format(folder_rename, res_file_name,
-                                                 METADATA_FILE_ENDSWITH)
-        expected_map_path = '{0}/{1}{2}'.format(folder_rename, res_file_name, RESMAP_FILE_ENDSWITH)
+                                                 AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
+        expected_map_path = '{0}/{1}{2}'.format(folder_rename, res_file_name,
+                                                AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_path)
         self.assertEqual(logical_file.map_short_file_path, expected_map_path)
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
diff --git a/hs_file_types/tests/test_timeseries_metadata.py b/hs_file_types/tests/test_timeseries_metadata.py
index 1ce63c88be..913627ce2b 100644
--- a/hs_file_types/tests/test_timeseries_metadata.py
+++ b/hs_file_types/tests/test_timeseries_metadata.py
@@ -10,7 +10,7 @@
 from hs_core.testing import MockIRODSTestCaseMixin
 from hs_core.views.utils import remove_folder, move_or_rename_file_or_folder
 
-from hs_file_types.models.base import METADATA_FILE_ENDSWITH, RESMAP_FILE_ENDSWITH
+from hs_file_types.enums import AggregationMetaFilePath
 from hs_file_types.models.timeseries import (
     TimeSeriesLogicalFile,
     TimeSeriesFileMetaData,
@@ -775,11 +775,11 @@ def test_aggregation_folder_rename(self):
 
         # test aggregation xml file paths
         expected_meta_file_path = '{}/{}{}'.format(new_folder, base_sqlite_file_name,
-                                                   METADATA_FILE_ENDSWITH)
+                                                   AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
         expected_map_file_path = '{}/{}{}'.format(new_folder, base_sqlite_file_name,
-                                                  RESMAP_FILE_ENDSWITH)
+                                                  AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # test renaming folder
@@ -799,11 +799,11 @@ def test_aggregation_folder_rename(self):
 
         # test aggregation xml file paths
         expected_meta_file_path = '{}_1/{}{}'.format(new_folder, base_sqlite_file_name,
-                                                     METADATA_FILE_ENDSWITH)
+                                                     AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
         expected_map_file_path = '{}_1/{}{}'.format(new_folder, base_sqlite_file_name,
-                                                    RESMAP_FILE_ENDSWITH)
+                                                    AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
@@ -838,11 +838,12 @@ def test_aggregation_parent_folder_rename(self):
 
         # test aggregation xml file paths
         expected_meta_file_path = '{}/{}{}'.format(aggregation_folder_name,
-                                                   base_sqlite_file_name, METADATA_FILE_ENDSWITH)
+                                                   base_sqlite_file_name,
+                                                   AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
         expected_map_file_path = '{}/{}{}'.format(aggregation_folder_name,
-                                                  base_sqlite_file_name, RESMAP_FILE_ENDSWITH)
+                                                  base_sqlite_file_name, AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
         # create a folder to be the parent folder of the aggregation folder
@@ -881,13 +882,13 @@ def test_aggregation_parent_folder_rename(self):
         expected_meta_file_path = '{0}/{1}/{2}{3}'.format(parent_folder_rename,
                                                           aggregation_folder_name,
                                                           base_sqlite_file_name,
-                                                          METADATA_FILE_ENDSWITH)
+                                                          AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
 
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
         expected_map_file_path = '{0}/{1}/{2}{3}'.format(parent_folder_rename,
                                                          aggregation_folder_name,
                                                          base_sqlite_file_name,
-                                                         RESMAP_FILE_ENDSWITH)
+                                                         AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
 
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
@@ -940,13 +941,13 @@ def test_aggregation_folder_move(self):
         expected_meta_file_path = '{0}/{1}/{2}{3}'.format(parent_folder,
                                                           aggregation_folder_name,
                                                           base_sqlite_file_name,
-                                                          METADATA_FILE_ENDSWITH)
+                                                          AggregationMetaFilePath.METADATA_FILE_ENDSWITH)
         self.assertEqual(logical_file.metadata_short_file_path, expected_meta_file_path)
 
         expected_map_file_path = '{0}/{1}/{2}{3}'.format(parent_folder,
                                                          aggregation_folder_name,
                                                          base_sqlite_file_name,
-                                                         RESMAP_FILE_ENDSWITH)
+                                                         AggregationMetaFilePath.RESMAP_FILE_ENDSWITH)
 
         self.assertEqual(logical_file.map_short_file_path, expected_map_file_path)
 
diff --git a/hs_file_types/tests/test_view_functions.py b/hs_file_types/tests/test_view_functions.py
index 14af4fcb58..43f6aa192c 100644
--- a/hs_file_types/tests/test_view_functions.py
+++ b/hs_file_types/tests/test_view_functions.py
@@ -1,23 +1,47 @@
 import json
+import os.path
 
-from django.test import TestCase, RequestFactory
 from django.contrib.auth.models import Group
+from django.contrib.sessions.middleware import SessionMiddleware
+from django.test import RequestFactory, TestCase
 from django.urls import reverse
-
 from rest_framework import status
 
+from django_irods.views import download
 from hs_core import hydroshare
 from hs_core.models import ResourceFile
 from hs_core.testing import MockIRODSTestCaseMixin
-
-from hs_file_types.views import set_file_type, add_metadata_element, update_metadata_element, \
-    update_key_value_metadata, delete_key_value_metadata, add_keyword_metadata, \
-    delete_keyword_metadata, update_netcdf_file, update_dataset_name, update_refts_abstract, \
-    update_sqlite_file, update_timeseries_abstract, get_timeseries_metadata, remove_aggregation, \
-    delete_coverage_element, update_aggregation_coverage, move_aggregation, delete_aggregation
-from hs_file_types.models import GeoRasterLogicalFile, NetCDFLogicalFile, NetCDFFileMetaData, \
-    RefTimeseriesLogicalFile, TimeSeriesLogicalFile, GenericLogicalFile, FileSetLogicalFile
+from hs_file_types.models import (
+    FileSetLogicalFile,
+    GenericLogicalFile,
+    GeoRasterLogicalFile,
+    ModelProgramLogicalFile,
+    NetCDFFileMetaData,
+    NetCDFLogicalFile,
+    RefTimeseriesLogicalFile,
+    TimeSeriesLogicalFile,
+)
 from hs_file_types.tests.utils import CompositeResourceTestMixin
+from hs_file_types.views import (
+    add_keyword_metadata,
+    add_metadata_element,
+    delete_aggregation,
+    delete_coverage_element,
+    delete_key_value_metadata,
+    delete_keyword_metadata,
+    get_timeseries_metadata,
+    move_aggregation,
+    remove_aggregation,
+    set_file_type,
+    update_aggregation_coverage,
+    update_dataset_name,
+    update_key_value_metadata,
+    update_metadata_element,
+    update_netcdf_file,
+    update_refts_abstract,
+    update_sqlite_file,
+    update_timeseries_abstract,
+)
 
 
 class TestFileTypeViewFunctions(MockIRODSTestCaseMixin, TestCase, CompositeResourceTestMixin):
@@ -98,7 +122,7 @@ def test_create_raster_aggregation_from_folder(self):
         # create a folder to place the raster file
         new_folder = 'raster_folder'
         ResourceFile.create_folder(self.composite_resource, new_folder)
-        # add the the tif file to the resource at the above folder
+        # add the tif file to the resource at the above folder
         self.add_file_to_resource(file_to_add=self.raster_file, upload_folder=new_folder)
 
         self.assertEqual(self.composite_resource.files.all().count(), 1)
@@ -283,7 +307,7 @@ def test_create_fileset_aggregation(self):
         # create a folder and put a file in that folder
         new_folder = 'fileset_folder'
         ResourceFile.create_folder(self.composite_resource, new_folder)
-        # add the the text file to the resource at the above folder
+        # add the text file to the resource at the above folder
         self.add_file_to_resource(file_to_add=self.text_file, upload_folder=new_folder)
 
         self.assertEqual(self.composite_resource.files.all().count(), 1)
@@ -1720,6 +1744,133 @@ def test_update_file_set_coverage_from_contents(self):
         self.assertFalse(self.composite_resource.dangling_aggregations_exist())
         self.composite_resource.delete()
 
+    def test_aggregation_meta_xml_file_download(self):
+        # here we are testing that we can download meta/resmap xml file for an aggregation
+
+        # create generic/single file aggregation
+        self.create_composite_resource(file_to_upload=self.text_file)
+        res_file = self.composite_resource.files.first()
+        GenericLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
+        self.assertEqual(GenericLogicalFile.objects.count(), 1)
+        res_file = self.composite_resource.files.first()
+        self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
+
+        # download aggregation meta xml file
+        # compute meta xml file storage path
+        meta_file_storage_path = res_file.storage_path[:-4] + '_meta.xml'
+        url_params = {'path': meta_file_storage_path}
+        url = reverse('django_irods_download', kwargs=url_params)
+        url = f"{url}?zipped=False&aggregation=False"
+        request = self.factory.get(url)
+        request.user = self.user
+
+        self.add_session_to_request(request)
+        response = download(request, path=meta_file_storage_path)
+        # check that the download request was successful
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # download aggregation resmap xml file
+        # compute resmap  storage path
+        meta_file_storage_path = res_file.storage_path[:-4] + '_resmap.xml'
+        url_params = {'path': meta_file_storage_path}
+        url = reverse('django_irods_download', kwargs=url_params)
+        url = f"{url}?zipped=False&aggregation=False"
+        request = self.factory.get(url)
+        request.user = self.user
+
+        self.add_session_to_request(request)
+        response = download(request, path=meta_file_storage_path)
+        # check that the download request was successful
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+    def test_model_program_schema_json_file_download(self):
+        # here we are testing that we can download schema json file for a model program aggregation
+
+        # create model program aggregation
+        self.create_composite_resource(file_to_upload=self.text_file)
+        res_file = self.composite_resource.files.first()
+        ModelProgramLogicalFile.set_file_type(self.composite_resource, self.user, res_file.id)
+        self.assertEqual(ModelProgramLogicalFile.objects.count(), 1)
+        mp_aggr = ModelProgramLogicalFile.objects.first()
+        res_file = self.composite_resource.files.first()
+        self.assertEqual(res_file.logical_file_type_name, "ModelProgramLogicalFile")
+
+        # set metadata schema for model program aggregation
+        schema_file_path = 'pytest/assets/mi_schema.json'
+        with open(schema_file_path, 'r') as file_obj:
+            json_schema = file_obj.read()
+        assert len(json_schema) > 0
+        assert not mp_aggr.metadata_schema_json
+        mp_aggr.metadata_schema_json = json.loads(json_schema)
+        mp_aggr.save()
+
+        # download aggregation schema.json file
+        # compute schema file storage path
+        meta_file_storage_path = res_file.storage_path[:-4] + '_schema.json'
+        url_params = {'path': meta_file_storage_path}
+        url = reverse('django_irods_download', kwargs=url_params)
+        url = f"{url}?zipped=False&aggregation=False"
+        request = self.factory.get(url)
+        request.user = self.user
+
+        self.add_session_to_request(request)
+        response = download(request, path=meta_file_storage_path)
+        # check that the download request was successful
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+    def test_fileset_meta_xml_file_download(self):
+        # here we are testing that we can download meta xml file for a folder based aggregation
+
+        self.create_composite_resource()
+        # create a folder and put a file in that folder
+        new_folder = 'fileset_folder'
+        ResourceFile.create_folder(self.composite_resource, new_folder)
+        # add the text file to the resource at the above folder
+        self.add_file_to_resource(file_to_add=self.text_file, upload_folder=new_folder)
+
+        self.assertEqual(self.composite_resource.files.all().count(), 1)
+        # create fileset aggregation
+        FileSetLogicalFile.set_file_type(resource=self.composite_resource, user=self.user, folder_path=new_folder)
+        self.assertEqual(FileSetLogicalFile.objects.count(), 1)
+
+        # download aggregation meta xml file
+        # compute meta xml file storage path
+        meta_file_storage_path = os.path.join(self.composite_resource.file_path, new_folder)
+        meta_file_storage_path = os.path.join(meta_file_storage_path, f"{new_folder}_meta.xml")
+        url_params = {'path': meta_file_storage_path}
+        url = reverse('django_irods_download', kwargs=url_params)
+        url = f"{url}?zipped=False&aggregation=False"
+        request = self.factory.get(url)
+        request.user = self.user
+
+        self.add_session_to_request(request)
+        response = download(request, path=meta_file_storage_path)
+        # check that the download request was successful
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # download aggregation resmap xml file
+        # compute resmap  storage path
+        meta_file_storage_path = os.path.join(self.composite_resource.file_path, new_folder)
+        meta_file_storage_path = os.path.join(meta_file_storage_path, f"{new_folder}_resmap.xml")
+        url_params = {'path': meta_file_storage_path}
+        url = reverse('django_irods_download', kwargs=url_params)
+        url = f"{url}?zipped=False&aggregation=False"
+        request = self.factory.get(url)
+        request.user = self.user
+
+        self.add_session_to_request(request)
+        response = download(request, path=meta_file_storage_path)
+        # check that the download request was successful
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+    @staticmethod
+    def add_session_to_request(request):
+        """Use SessionMiddleware to add a session to the request."""
+        """Annotate a request object with a session"""
+        middleware = SessionMiddleware()
+        middleware.process_request(request)
+        request.session.save()
+
     def _add_delete_keywords_file_type(self, file_path, file_type):
         self.create_composite_resource(file_path)
         res_file = self.composite_resource.files.first()