Repository: hydroshare/hydroshare

Issue:
Add ability to repair_resource ignoring recently repaired resources (batch)
**Describe the feature you'd like and what it will do**

Jenkins agent times out on repair_resource command
https://new-hs-ci.hydroshare.org/view/02-Deployment/job/Production-Deployment/job/prod-hsctl-command/268/console
```
FATAL: command execution failed
java.io.IOException
	at hudson.remoting.Channel.close(Channel.java:1499)
	at hudson.remoting.Channel.close(Channel.java:1455)
	at hudson.slaves.SlaveComputer.closeChannel(SlaveComputer.java:884)
	at hudson.slaves.SlaveComputer.kill(SlaveComputer.java:851)
	at hudson.model.AbstractCIBase.killComputer(AbstractCIBase.java:87)
	at jenkins.model.Jenkins.lambda$_cleanUpDisconnectComputers$11(Jenkins.java:3559)
	at hudson.model.Queue._withLock(Queue.java:1382)
	at hudson.model.Queue.withLock(Queue.java:1258)
	at jenkins.model.Jenkins._cleanUpDisconnectComputers(Jenkins.java:3555)
	at jenkins.model.Jenkins.cleanUp(Jenkins.java:3438)
	at hudson.WebAppMain.contextDestroyed(WebAppMain.java:441)
	at org.eclipse.jetty.server.handler.ContextHandler.callContextDestroyed(ContextHandler.java:1075)
	at org.eclipse.jetty.servlet.ServletContextHandler.callContextDestroyed(ServletContextHandler.java:584)
	at org.eclipse.jetty.server.handler.ContextHandler.contextDestroyed(ContextHandler.java:1038)
	at org.eclipse.jetty.servlet.ServletHandler.doStop(ServletHandler.java:319)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:94)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:201)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:108)
	at org.eclipse.jetty.security.SecurityHandler.doStop(SecurityHandler.java:437)
	at org.eclipse.jetty.security.ConstraintSecurityHandler.doStop(ConstraintSecurityHandler.java:423)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:94)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:201)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:108)
	at org.eclipse.jetty.server.session.SessionHandler.doStop(SessionHandler.java:520)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:94)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:201)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:108)
	at org.eclipse.jetty.server.handler.ContextHandler.stopContext(ContextHandler.java:1061)
	at org.eclipse.jetty.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:386)
	at org.eclipse.jetty.webapp.WebAppContext.stopWebapp(WebAppContext.java:1454)
	at org.eclipse.jetty.webapp.WebAppContext.stopContext(WebAppContext.java:1420)
	at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:1115)
	at org.eclipse.jetty.servlet.ServletContextHandler.doStop(ServletContextHandler.java:297)
	at org.eclipse.jetty.webapp.WebAppContext.doStop(WebAppContext.java:547)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:94)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:201)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:108)
	at org.eclipse.jetty.server.Server.doStop(Server.java:470)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:94)
	at winstone.Launcher.shutdown(Launcher.java:318)
	at winstone.ShutdownHook.run(ShutdownHook.java:25)
Caused: hudson.remoting.ChannelClosedException: Channel "hudson.remoting.Channel@30214e98:prod-nginx": Remote call on prod-nginx failed. The channel is closing down or has closed down
```

**Why is this feature important?**
We want to bulk repair resources, but this process times out because it is very long-running.
We could make the process asynchronous and run the repairs in parallel, but we run the risk of race conditions and these are potentially sensitive file operations on published resources.
Instead of making the process async, I suggest that we add the ability to run repairs in batches. Or to ignore resources that have recently been repaired

**Additional context**
Related to https://github.com/hydroshare/hydroshare/issues/5300
HS v2.12.3



Discussion:

Resolving patch:
diff --git a/hs_core/management/commands/repair_resource.py b/hs_core/management/commands/repair_resource.py
--- a/hs_core/management/commands/repair_resource.py
+++ b/hs_core/management/commands/repair_resource.py
@@ -17,7 +17,7 @@
 from hs_core.views.utils import get_default_admin_user
 from hs_core import hydroshare
 from django.utils import timezone
-from django.db.models import F
+from django.db.models import F, Q
 from datetime import timedelta
 
 import logging
@@ -28,7 +28,10 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('resource_ids', nargs='*', type=str)
-        parser.add_argument('--days', type=int, dest='days', help='include resources updated in the last X days')
+        parser.add_argument('--updated_since', type=int, dest='updated_since',
+                            help='include only resources updated in the last X days')
+        parser.add_argument('--ignore_repaired_since', type=int, dest='ignore_repaired_since',
+                            help='ignore resources repaired since X days ago')
         parser.add_argument(
             '--admin',
             action='store_true',  # True for presence, False for absence
@@ -52,11 +55,12 @@ def handle(self, *args, **options):
         logger = logging.getLogger(__name__)
         resources_ids = options['resource_ids']
         resources = BaseResource.objects.all()
-        days = options['days']
+        updated_since = options['updated_since']
         admin = options['admin']
         dry_run = options['dry_run']
         published = options['published']
         site_url = hydroshare.utils.current_site_url()
+        ignore_repaired_since = options['ignore_repaired_since']
 
         if resources_ids:  # an array of resource short_id to check.
             print("CHECKING RESOURCES PROVIDED")
@@ -67,13 +71,20 @@ def handle(self, *args, **options):
             print("FILTERING TO INCLUDE PUBLISHED RESOURCES ONLY")
             resources = resources.filter(raccess__published=True)
 
-        if days:
-            print(f"FILTERING TO INCLUDE RESOURCES UPDATED IN LAST {days} DAYS")
+        if updated_since:
+            print(f"FILTERING TO INCLUDE RESOURCES UPDATED IN LAST {updated_since} DAYS")
             if resources_ids:
-                print("Your supplied resource_ids will be filtered by the --days that you provided. ")
-            cuttoff_time = timezone.now() - timedelta(days)
+                print("Your supplied resource_ids will be filtered by the --updated_since days that you provided. ")
+            cuttoff_time = timezone.now() - timedelta(days=updated_since)
             resources = resources.filter(updated__gte=cuttoff_time)
 
+        if ignore_repaired_since:
+            print(f"FILTERING TO INCLUDE RESOURCES NOT REPAIRED IN THE LAST {ignore_repaired_since} DAYS")
+            if resources_ids:
+                print("Your supplied resource_ids will be filtered by the --ignore_repaired_since days provided. ")
+            cuttoff_time = timezone.now() - timedelta(days=ignore_repaired_since)
+            resources = resources.filter(Q(repaired__lt=cuttoff_time) | Q(repaired__isnull=True))
+
         if dry_run:
             print("CONDUCTING A DRY RUN: FIXES WILL NOT BE SAVED")
 
@@ -87,7 +98,7 @@ def handle(self, *args, **options):
         else:
             user = None
 
-        resources = resources.order_by(F('updated').asc(nulls_first=True))
+        resources = resources.order_by(F('repaired').asc(nulls_first=True))
 
         total_res_to_check = resources.count()
         current_resource = 0


Test patch: