Repository: oppia/oppia

Issue:
Exploration models remain cached after running the MigrateExplorationJob
<!--
  - Thanks for taking the time to report a bug in the Oppia project.
  - Before filing a new issue, please do a quick search to check that it hasn't
  - already been filed on the [issue tracker](https://github.com/oppia/oppia/issues)._
  -->

**Describe the bug**
It was observed that the server would return the outdated, cached ExplorationModels despite running MigrateExplorationJob which internally deletes migrated exploration IDs from the cache.

**To Reproduce**
Steps to reproduce the behavior:
 1. Run MigrateExplorationJob on the server with some unmigrated explorations.
 2. Check the history tab of a migrated exploration verify if there is a commit by OppiaMigrationBot that indicates a state schema version change.
 3. See error.

**Observed behavior**
History tab of a migrated exploration does NOT show the edit by OppiaMigrationBot.

**Expected behavior**
History tab of a migrated exploration should show the edit by OppiaMigrationBot.


Exploration models remain cached after running the MigrateExplorationJob
<!--
  - Thanks for taking the time to report a bug in the Oppia project.
  - Before filing a new issue, please do a quick search to check that it hasn't
  - already been filed on the [issue tracker](https://github.com/oppia/oppia/issues)._
  -->

**Describe the bug**
It was observed that the server would return the outdated, cached ExplorationModels despite running MigrateExplorationJob which internally deletes migrated exploration IDs from the cache.

**To Reproduce**
Steps to reproduce the behavior:
 1. Run MigrateExplorationJob on the server with some unmigrated explorations.
 2. Check the history tab of a migrated exploration verify if there is a commit by OppiaMigrationBot that indicates a state schema version change.
 3. See error.

**Observed behavior**
History tab of a migrated exploration does NOT show the edit by OppiaMigrationBot.

**Expected behavior**
History tab of a migrated exploration should show the edit by OppiaMigrationBot.




Discussion:
Flush the cache after any job is run
Flush the cache after any job is run

Resolving patch:
diff --git a/core/jobs/io/cache_io.py b/core/jobs/io/cache_io.py
new file mode 100644
--- /dev/null
+++ b/core/jobs/io/cache_io.py
@@ -0,0 +1,51 @@
+# coding: utf-8
+#
+# Copyright 2022 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides PTransforms for manipulating with cache."""
+
+from __future__ import annotations
+
+from core.domain import caching_services
+
+import apache_beam as beam
+from typing import Any
+
+
+# TODO(#15613): Here we use MyPy ignore because of the incomplete typing of
+# apache_beam library and absences of stubs in Typeshed, forces MyPy to
+# assume that PTransform class is of type Any. Thus to avoid MyPy's error (Class
+# cannot subclass 'PTransform' (has type 'Any')), we added an ignore here.
+class FlushCache(beam.PTransform): # type: ignore[misc]
+    """Flushes the memory caches."""
+
+    # Here we use type Any because we do not care about the type of items passed
+    # here.
+    def expand(
+        self, items: beam.PCollection[Any]
+    ) -> beam.pvalue.PDone:
+        """Flushes the memory caches.
+
+        Args:
+            items: PCollection. Items, can also contain just one item.
+
+        Returns:
+            PCollection. An empty PCollection.
+        """
+        return (
+            items
+            | beam.CombineGlobally(lambda _: [])
+            | beam.Map(lambda _: caching_services.flush_memory_caches())
+        )
diff --git a/core/jobs/jobs_manager.py b/core/jobs/jobs_manager.py
--- a/core/jobs/jobs_manager.py
+++ b/core/jobs/jobs_manager.py
@@ -26,6 +26,7 @@
 from core.domain import beam_job_services
 from core.jobs import base_jobs
 from core.jobs import job_options
+from core.jobs.io import cache_io
 from core.jobs.io import job_io
 from core.platform import models
 from core.storage.beam_job import gae_models as beam_job_models
@@ -130,7 +131,7 @@ def run_job(
 
     # NOTE: Exceptions raised within this context are logged and suppressed.
     with _job_bookkeeping_context(job_name) as run_model:
-        _ = job.run() | job_io.PutResults(run_model.id)
+        _ = job.run() | job_io.PutResults(run_model.id) | cache_io.FlushCache()
 
         run_result = pipeline.run()
 


Test patch:
diff --git a/core/jobs/io/cache_io_test.py b/core/jobs/io/cache_io_test.py
new file mode 100644
--- /dev/null
+++ b/core/jobs/io/cache_io_test.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+#
+# Copyright 2022 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for jobs.io.cache_io."""
+
+from __future__ import annotations
+
+from core.domain import caching_services
+from core.jobs import job_test_utils
+from core.jobs.io import cache_io
+
+import apache_beam as beam
+
+
+class FlushCacheTests(job_test_utils.PipelinedTestBase):
+
+    def test_cache_is_flushed(self) -> None:
+        items = [1] * 1000000
+
+        called_functions = {'flush_caches': False}
+
+        class MockMemoryCachingServices:
+
+            @staticmethod
+            def flush_caches() -> None:
+                """Flush cache."""
+                called_functions['flush_caches'] = True
+
+        with self.swap(
+            caching_services, 'memory_cache_services', MockMemoryCachingServices
+        ):
+            self.assert_pcoll_equal(
+                self.pipeline
+                | beam.Create(items)
+                | cache_io.FlushCache(),
+                [None]
+            )
+
+        self.assertTrue(called_functions['flush_caches'])
diff --git a/core/jobs/io/job_io_test.py b/core/jobs/io/job_io_test.py
--- a/core/jobs/io/job_io_test.py
+++ b/core/jobs/io/job_io_test.py
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Unit tests for jobs.io.job_run_results_io."""
+"""Unit tests for jobs.io.job_io."""
 
 from __future__ import annotations
 
diff --git a/core/jobs/io/ndb_io_test.py b/core/jobs/io/ndb_io_test.py
--- a/core/jobs/io/ndb_io_test.py
+++ b/core/jobs/io/ndb_io_test.py
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Provides an Apache Beam API for operating on NDB models."""
+"""Unit tests for jobs.io.ndb_io."""
 
 from __future__ import annotations
 
diff --git a/scripts/backend_test_shards.json b/scripts/backend_test_shards.json
--- a/scripts/backend_test_shards.json
+++ b/scripts/backend_test_shards.json
@@ -54,7 +54,8 @@
         "core.jobs.batch_jobs.skill_migration_jobs_test",
         "core.jobs.batch_jobs.story_migration_jobs_test",
         "core.jobs.batch_jobs.email_deletion_jobs_test",
-        "core.jobs.batch_jobs.blog_post_search_indexing_jobs_test"
+        "core.jobs.batch_jobs.blog_post_search_indexing_jobs_test",
+        "core.jobs.io.cache_io_test"
     ],
     "2": [
         "core.domain.rights_manager_test",