Repository: ietf-tools/datatracker

Issue:
Submissions can be posted long after the staging files have been purged
### Describe the issue

In #7454, a submission was approved 60 days after it was submitted. Files older than 14 days are regularly purged from the staging directory where they are stored pending approval. The result was that the approved draft was posted but, because the staged files had long before been deleted, the actual contents are missing.

Stale submissions should be canceled on a schedule that is coordinated with the staging file purge. A submission should be canceled before its files are removed.

We already have a periodic Celery task that cancels submissions that have not yet been validated - a similar task should cancel stale submissions. It'd be adequate to run the new task on the daily schedule.

### Code of Conduct

- [X] I agree to follow the [IETF's Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md)


Discussion:

Resolving patch:
diff --git a/ietf/settings.py b/ietf/settings.py
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -817,6 +817,9 @@ def skip_unreadable_post(record):
 # Max time to allow for validation before a submission is subject to cancellation
 IDSUBMIT_MAX_VALIDATION_TIME = datetime.timedelta(minutes=20)
 
+# Age at which a submission expires if not posted
+IDSUBMIT_EXPIRATION_AGE = datetime.timedelta(days=14)
+
 IDSUBMIT_MANUAL_STAGING_DIR = '/tmp/'
 
 IDSUBMIT_FILE_TYPES = (
diff --git a/ietf/submit/tasks.py b/ietf/submit/tasks.py
--- a/ietf/submit/tasks.py
+++ b/ietf/submit/tasks.py
@@ -37,19 +37,34 @@ def process_and_accept_uploaded_submission_task(submission_id):
 @shared_task
 def cancel_stale_submissions():
     now = timezone.now()
-    stale_submissions = Submission.objects.filter(
+    # first check for submissions gone stale awaiting validation
+    stale_unvalidated_submissions = Submission.objects.filter(
         state_id='validating',
     ).annotate(
         submitted_at=Min('submissionevent__time'),
     ).filter(
         submitted_at__lt=now - settings.IDSUBMIT_MAX_VALIDATION_TIME,
     )
-    for subm in stale_submissions:
+    for subm in stale_unvalidated_submissions:
         age = now - subm.submitted_at
         log.log(f'Canceling stale submission (id={subm.id}, age={age})')
         cancel_submission(subm)
         create_submission_event(None, subm, 'Submission canceled: validation checks took too long')
 
+    # now check for expired submissions
+    expired_submissions = Submission.objects.exclude(
+        state_id__in=["posted", "cancel"],
+    ).annotate(
+        submitted_at=Min("submissionevent__time"),
+    ).filter(
+        submitted_at__lt=now - settings.IDSUBMIT_EXPIRATION_AGE,
+    )
+    for subm in expired_submissions:
+        age = now - subm.submitted_at
+        log.log(f'Canceling expired submission (id={subm.id}, age={age})')
+        cancel_submission(subm)
+        create_submission_event(None, subm, 'Submission canceled: expired without being posted')
+
 
 @shared_task(bind=True)
 def poke(self):


Test patch:
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -42,7 +42,7 @@
 from ietf.group.utils import setup_default_community_list_for_group
 from ietf.meeting.models import Meeting
 from ietf.meeting.factories import MeetingFactory
-from ietf.name.models import FormalLanguageName
+from ietf.name.models import DraftSubmissionStateName, FormalLanguageName
 from ietf.person.models import Person
 from ietf.person.factories import UserFactory, PersonFactory, EmailFactory
 from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory
@@ -3136,28 +3136,59 @@ def test_status_of_validating_submission(self):
         self.assertContains(r, s.name)
         self.assertContains(r, 'This submission is being processed and validated.', status_code=200)
 
-    @override_settings(IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30))
+    @override_settings(
+        IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30),
+        IDSUBMIT_EXPIRATION_AGE=datetime.timedelta(minutes=90),
+    )
     def test_cancel_stale_submissions(self):
+        # these will be lists of (Submission, "state_id") pairs
+        submissions_to_skip = []
+        submissions_to_cancel = []
+
+        # submissions in the validating state
         fresh_submission = SubmissionFactory(state_id='validating')
         fresh_submission.submissionevent_set.create(
             desc='fake created event',
             time=timezone.now() - datetime.timedelta(minutes=15),
         )
+        submissions_to_skip.append((fresh_submission, "validating"))
+
         stale_submission = SubmissionFactory(state_id='validating')
         stale_submission.submissionevent_set.create(
             desc='fake created event',
             time=timezone.now() - datetime.timedelta(minutes=30, seconds=1),
         )
+        submissions_to_cancel.append((stale_submission, "validating"))
+        
+        # submissions in other states
+        for state in DraftSubmissionStateName.objects.filter(used=True).exclude(slug="validating"):
+            to_skip = SubmissionFactory(state_id=state.pk)
+            to_skip.submissionevent_set.create(
+                desc="fake created event",
+                time=timezone.now() - datetime.timedelta(minutes=45),  # would be canceled if it were "validating"
+            )
+            submissions_to_skip.append((to_skip, state.pk))
+            to_expire = SubmissionFactory(state_id=state.pk)
+            to_expire.submissionevent_set.create(
+                desc="fake created event",
+                time=timezone.now() - datetime.timedelta(minutes=90, seconds=1),
+            )
+            if state.pk in ["posted", "cancel"]:
+                submissions_to_skip.append((to_expire, state.pk))  # these ones should not be expired regardless of age
+            else:
+                submissions_to_cancel.append(((to_expire, state.pk)))
 
         cancel_stale_submissions()
 
-        fresh_submission = Submission.objects.get(pk=fresh_submission.pk)
-        self.assertEqual(fresh_submission.state_id, 'validating')
-        self.assertEqual(fresh_submission.submissionevent_set.count(), 1)
+        for _subm, original_state_id in submissions_to_skip:
+            subm = Submission.objects.get(pk=_subm.pk)
+            self.assertEqual(subm.state_id, original_state_id)
+            self.assertEqual(subm.submissionevent_set.count(), 1)
 
-        stale_submission = Submission.objects.get(pk=stale_submission.pk)
-        self.assertEqual(stale_submission.state_id, 'cancel')
-        self.assertEqual(stale_submission.submissionevent_set.count(), 2)
+        for _subm, _ in submissions_to_cancel:
+            subm = Submission.objects.get(pk=_subm.pk)
+            self.assertEqual(subm.state_id, "cancel")
+            self.assertEqual(subm.submissionevent_set.count(), 2)
 
 
 class ApiSubmitTests(BaseSubmitTestCase):