Repository: hydroshare/hydroshare

Issue:
Mitigate missing geoservice registrations
**Description of the bug**
Last week we discovered that `update_web_services` tasks in Hydroshare were failing.
Turns out this was because the geoserver container had filled. We still don't know why the container ballooned.

For now, we want to add Jenkins monitoring to ensure that Geoserver is up, and to monitor the space on the container. This will be done as work for issue https://github.com/hydroshare/hydroshare/issues/5187

We also want a process to repair resources that are missing their data services registrations due to downtime of Geoserver (in the case that downtime overlapped with Hydroshare attempting registration/updates). That repair process will be completed in this issue https://github.com/hydroshare/hydroshare/issues/5183

**Additional information**
HS 2.8.1



Discussion:

Resolving patch:
diff --git a/hs_core/management/commands/register_webservices.py b/hs_core/management/commands/register_webservices.py
new file mode 100644
--- /dev/null
+++ b/hs_core/management/commands/register_webservices.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+"""
+Register web services for resource(s)
+"""
+
+from django.core.management.base import BaseCommand
+from hs_core.models import BaseResource
+from hs_core.hydroshare.utils import get_resource_by_shortkey
+from hs_core.tasks import check_geoserver_registrations
+from django.utils import timezone
+from datetime import timedelta
+from django.http.response import Http404
+
+
+class Command(BaseCommand):
+    help = "Register web services for resource(s)"
+
+    def add_arguments(self, parser):
+        parser.add_argument('resource_ids', nargs='*', type=str)
+        parser.add_argument('--days', type=int, dest='days', help='include resources updated in the last X days')
+        parser.add_argument('--async',
+                            action='store_true',  # True for presence, False for absence
+                            dest='run_async',
+                            help='web services requests should run async through celery')
+
+    def handle(self, *args, **options):
+        resources = options['resource_ids']
+        days = options['days']
+        run_async = options['run_async']
+
+        if resources and days:
+            print("Please only include either [resources] or [days] argument, but not both")
+            return
+
+        if resources:  # an array of resource short_id to check.
+            print("CHECKING RESOURCES PROVIDED")
+            found_resources = []
+            for rid in resources:
+                try:
+                    resource = get_resource_by_shortkey(rid)
+                    found_resources.append(resource)
+                except (BaseResource.DoesNotExist, Http404):
+                    msg = f"resource {rid} not found"
+                    print(msg)
+                    continue
+            resources = found_resources
+
+        elif days:
+            print(f"CHECKING ALL PUBLIC RESOURCES UPDATED IN LAST {days} DAYS")
+            cuttoff_time = timezone.now() - timedelta(days)
+            resources = BaseResource.objects.filter(updated__gte=cuttoff_time, raccess__public=True)
+
+        else:
+            print("CHECKING ALL PUBLIC RESOURCES")
+            resources = BaseResource.objects.filter(raccess__public=True)
+        if resources:
+            if run_async:
+                check_geoserver_registrations(resources, run_async=True)
+                print("WEB SERVICE CHECKS WILL RUN ASYNC. CHECK CELERY FOR STATUS.")
+            else:
+                check_geoserver_registrations(resources, run_async=False)
+                print("DONE UPDATING WEB SERVICES")
+        else:
+            print("No resources found matching provided args.")
diff --git a/hs_core/tasks.py b/hs_core/tasks.py
--- a/hs_core/tasks.py
+++ b/hs_core/tasks.py
@@ -38,6 +38,7 @@
 from theme.models import UserQuota, QuotaMessage, User
 from django_irods.icommands import SessionException
 from celery.result import states
+from celery.exceptions import TaskError
 
 from hs_core.models import BaseResource, ResourceFile, TaskNotification
 from hs_core.enums import RelationTypes
@@ -115,24 +116,25 @@ def setup_periodic_tasks(sender, **kwargs):
         # Hourly
         sender.add_periodic_task(crontab(minute=45), manage_task_hourly.s())
 
-        # Daily
-        sender.add_periodic_task(crontab(minute=30, hour=0), daily_innactive_group_requests_cleanup.s())
-        sender.add_periodic_task(crontab(minute=0, hour=1), nightly_periodic_task_check.s())
-        sender.add_periodic_task(crontab(minute=30, hour=1), nightly_repair_resource_files.s())
-        sender.add_periodic_task(crontab(minute=00, hour=12), daily_odm2_sync.s())
-        sender.add_periodic_task(crontab(minute=30, hour=22), nightly_metadata_review_reminder.s())
-        sender.add_periodic_task(crontab(minute=30, hour=23), nightly_zips_cleanup.s())
-        sender.add_periodic_task(crontab(minute=0, hour=2), nightly_cache_file_system_metadata.s())
+        # Daily (times in UTC)
+        sender.add_periodic_task(crontab(minute=0, hour=3), nightly_metadata_review_reminder.s())
+        sender.add_periodic_task(crontab(minute=30, hour=3), nightly_zips_cleanup.s())
+        sender.add_periodic_task(crontab(minute=0, hour=4), daily_odm2_sync.s())
+        sender.add_periodic_task(crontab(minute=30, hour=4), daily_innactive_group_requests_cleanup.s())
+        sender.add_periodic_task(crontab(minute=0, hour=5), check_geoserver_registrations.s())
+        sender.add_periodic_task(crontab(minute=30, hour=5), nightly_repair_resource_files.s())
+        sender.add_periodic_task(crontab(minute=0, hour=6), nightly_cache_file_system_metadata.s())
+        sender.add_periodic_task(crontab(minute=30, hour=6), nightly_periodic_task_check.s())
 
         # Weekly
-        sender.add_periodic_task(crontab(minute=30, hour=1, day_of_week=1), task_notification_cleanup.s())
+        sender.add_periodic_task(crontab(minute=0, hour=7, day_of_week=1), task_notification_cleanup.s())
 
         # Monthly
-        sender.add_periodic_task(crontab(minute=0, hour=1, day_of_month=1), update_from_geoconnex_task.s())
-        sender.add_periodic_task(crontab(minute=15, hour=0, day_of_week=1, day_of_month='1-7'),
+        sender.add_periodic_task(crontab(minute=30, hour=7, day_of_month=1), update_from_geoconnex_task.s())
+        sender.add_periodic_task(crontab(minute=0, hour=8, day_of_week=1, day_of_month='1-7'),
                                  send_over_quota_emails.s())
         sender.add_periodic_task(
-            crontab(minute=15, hour=1, day_of_month=1), monthly_group_membership_requests_cleanup.s())
+            crontab(minute=30, hour=8, day_of_month=1), monthly_group_membership_requests_cleanup.s())
 
 
 # Currently there are two different cleanups scheduled.
@@ -490,6 +492,58 @@ def send_over_quota_emails():
             logger.debug('user ' + u.username + ' does not have UserQuota foreign key relation')
 
 
+@celery_app.task(ignore_result=True, base=HydroshareTask)
+def check_geoserver_registrations(resources, run_async=True):
+    # Check to ensure resources have updated web services registrations
+
+    DISTRIBUTE_WEB_REGISTRATIONS_OVER = 2  # hours
+
+    if not settings.HSWS_ACTIVATED:
+        return
+
+    if not resources:
+        cuttoff_time = timezone.now() - timedelta(days=1)
+        resources = BaseResource.objects.filter(updated__gte=cuttoff_time, raccess__public=True)
+
+    if run_async:
+        # Evenly distribute requests
+        delay_between_res = (DISTRIBUTE_WEB_REGISTRATIONS_OVER * 60 * 60) / resources.count()
+
+        for current_res_num, res in enumerate(resources, start=0):
+            delay = delay_between_res * current_res_num
+            update_web_services.apply_async((
+                settings.HSWS_URL,
+                settings.HSWS_API_TOKEN,
+                settings.HSWS_TIMEOUT,
+                settings.HSWS_PUBLISH_URLS,
+                res.short_id
+            ), countdown=delay)
+    else:
+        failed_resources = {}
+        for res in resources:
+            response = update_web_services(
+                settings.HSWS_URL,
+                settings.HSWS_API_TOKEN,
+                settings.HSWS_TIMEOUT,
+                settings.HSWS_PUBLISH_URLS,
+                res.short_id
+            )
+            if not response['success']:
+                res_url = current_site_url() + res.get_absolute_url()
+                failed_resources[res_url] = response
+        if failed_resources:
+            err_msg = 'Attempt to update web services failed for the following resources:\n'
+            for res_url, resp in failed_resources:
+                err_msg += f'{res_url} => {json.dumps(resp)}\n'
+            if not settings.DISABLE_TASK_EMAILS:
+                subject = "Web services failed to update"
+                recipients = [settings.DEFAULT_SUPPORT_EMAIL]
+                send_mail(subject, err_msg, settings.DEFAULT_FROM_EMAIL, recipients)
+            else:
+                logger.error(err_msg)
+            raise TaskError(err_msg)
+
+
 @shared_task
 def add_zip_file_contents_to_resource(pk, zip_file_path):
     """Add zip file to existing resource and remove tmp zip file."""
@@ -927,7 +981,10 @@ def update_web_services(services_url, api_token, timeout, publish_urls, res_id):
                     )]
                     lf.metadata.extra_metadata["Web Services URL"] = url["message"]
                     lf.metadata.save()
-        return response.json()
+        if status.is_success(response.status_code):
+            return response.json()
+        else:
+            response.raise_for_status()
     except Exception as e:
         logger.error(f"Error updating web services: {str(e)}")
         raise


Test patch: