Repository: oppia/oppia

Issue:
Truncate Beam job run results if they exceed the length limit, so that the job run is still successful.
**Is your feature request related to a problem? Please describe.**

Job run results in Beam jobs have a limit of 1.5k chars. This results in some jobs erroring when the limit is too long.

**Describe the solution you'd like**

The Beam job infrastructure should truncate the job run result if it exceeds 1.5k chars. It seems (based on talking to @Lawful2002) that the output is in the form of a string. This issue proposes modifying our Beam job infrastructure so that, if the resulting string ends up being more than 1.48k chars, we chop off the last 20 chars and replace it with the string "...[TRUNCATED]...".


Discussion:
/cc @Lawful2002 This is based on our discussion in the LaCE Android team meeting and on the issue raised in [this PR](https://github.com/oppia/oppia/pull/15270#issuecomment-1109768252). Please add any other info as needed. Thanks!
@seanlip A single error/job result message is just a string (I've updated the issue description), the issue (the one discussed in the meeting) arose when I included a list `exp_link` (which is a list of URLs converted to a string) in the job result string which caused the overall message to exceed 1500 bytes.
<img width="324" alt="image" src="https://user-images.githubusercontent.com/73544247/165505775-27aebc73-6098-4a4e-8ddc-3203e7e4fd81.png">

Great, thanks for updating the issue @Lawful2002!

@vojtechjelinek please let us know if you need more clarification on the above. Thanks!

Resolving patch:
diff --git a/core/jobs/types/job_run_result.py b/core/jobs/types/job_run_result.py
--- a/core/jobs/types/job_run_result.py
+++ b/core/jobs/types/job_run_result.py
@@ -24,7 +24,10 @@
 
 from typing import Any, List, Tuple # isort: skip
 
-MAX_OUTPUT_BYTES = 1500
+# This is just to make sure that the output of the job have some reasonable
+# length. The maximum that model can hold is around 1 MB and this is much lower.
+MAX_OUTPUT_CHARACTERS = 5000
+TRUNCATED_MARK = '[TRUNCATED]'
 
 
 class JobRunResult:
@@ -52,9 +55,14 @@ def __init__(self, stdout: str = '', stderr: str = ''):
 
         self.stdout, self.stderr = stdout, stderr
 
-        if self.len_in_bytes() > MAX_OUTPUT_BYTES:
-            raise ValueError(
-                'JobRunResult must not exceed %d bytes' % MAX_OUTPUT_BYTES)
+        if len(self.stdout) > MAX_OUTPUT_CHARACTERS:
+            self.stdout = '%s%s' % (
+                self.stdout[:MAX_OUTPUT_CHARACTERS], TRUNCATED_MARK
+            )
+        if len(self.stderr) > MAX_OUTPUT_CHARACTERS:
+            self.stderr = '%s%s' % (
+                self.stderr[:MAX_OUTPUT_CHARACTERS], TRUNCATED_MARK
+            )
 
     @classmethod
     def as_stdout(cls, value: Any, use_repr: bool = False) -> JobRunResult:
@@ -107,7 +115,10 @@ def accumulate(cls, results: List[JobRunResult]) -> List[JobRunResult]:
         for i, result in enumerate(results):
             # Use i as a tie-breaker so that results, which don't implement the
             # comparison operators, don't get compared with one another.
-            heapq.heappush(results_heap, (result.len_in_bytes(), i, result))
+            heapq.heappush(
+                results_heap,
+                (len(result.stdout) + len(result.stderr), i, result)
+            )
 
         batches = []
         latest_batch_size, _, smallest = heapq.heappop(results_heap)
@@ -121,7 +132,8 @@ def accumulate(cls, results: List[JobRunResult]) -> List[JobRunResult]:
             # them is empty).
             padding = 2 if next_smallest.stdout and next_smallest.stderr else 1
 
-            if latest_batch_size + padding + result_size < MAX_OUTPUT_BYTES:
+            overall_size = latest_batch_size + padding + result_size
+            if overall_size <= MAX_OUTPUT_CHARACTERS:
                 latest_batch_size += padding + result_size
                 batches[-1].append(next_smallest)
             else:
@@ -135,15 +147,6 @@ def accumulate(cls, results: List[JobRunResult]) -> List[JobRunResult]:
             batched_results.append(JobRunResult(stdout=stdout, stderr=stderr))
         return batched_results
 
-    def len_in_bytes(self) -> int:
-        """Returns the number of bytes encoded by the JobRunResult instance.
-
-        Returns:
-            int. The number of bytes encoded by the JobRunResult instance.
-        """
-        output_bytes = (s.encode('utf-8') for s in (self.stdout, self.stderr))
-        return sum(len(output) for output in output_bytes)
-
     def __repr__(self) -> str:
         return '%s(stdout=%s, stderr=%s)' % (
             self.__class__.__name__,


Test patch:
diff --git a/core/jobs/io/job_io_test.py b/core/jobs/io/job_io_test.py
--- a/core/jobs/io/job_io_test.py
+++ b/core/jobs/io/job_io_test.py
@@ -53,7 +53,7 @@ def test_sharded_output(self) -> None:
             job_run_result.JobRunResult(stdout='ghi', stderr='789'),
         ]
 
-        with self.swap(job_run_result, 'MAX_OUTPUT_BYTES', 11):
+        with self.swap(job_run_result, 'MAX_OUTPUT_CHARACTERS', 8):
             self.assert_pcoll_empty(
                 self.pipeline
                 | beam.Create(messages)
diff --git a/core/jobs/types/job_run_result_test.py b/core/jobs/types/job_run_result_test.py
--- a/core/jobs/types/job_run_result_test.py
+++ b/core/jobs/types/job_run_result_test.py
@@ -55,9 +55,15 @@ def test_empty_result_raises_value_error(self) -> None:
         with self.assertRaisesRegex(ValueError, 'must not be empty'):  # type: ignore[no-untyped-call]
             job_run_result.JobRunResult()
 
-    def test_enormous_result_raises_value_error(self) -> None:
-        with self.assertRaisesRegex(ValueError, r'must not exceed \d+ bytes'):  # type: ignore[no-untyped-call]
-            job_run_result.JobRunResult(stdout='a' * 1501)
+    def test_enormous_stdout_result_is_truncated(self) -> None:
+        run_result = job_run_result.JobRunResult(stdout='a' * 5010)
+        self.assertEqual(run_result.stdout, '%s[TRUNCATED]' % ('a' * 5000))
+        self.assertEqual(run_result.stderr, '')
+
+    def test_enormous_stderr_result_is_truncated(self) -> None:
+        run_result = job_run_result.JobRunResult(stderr='a' * 5010)
+        self.assertEqual(run_result.stderr, '%s[TRUNCATED]' % ('a' * 5000))
+        self.assertEqual(run_result.stdout, '')
 
     def test_accumulate(self) -> None:
         single_job_run_result = job_run_result.JobRunResult.accumulate([
@@ -71,35 +77,46 @@ def test_accumulate(self) -> None:
         self.assertItemsEqual(  # type: ignore[no-untyped-call]
             single_job_run_result.stderr.split('\n'), ['123', '456'])
 
+    def test_accumulate_one_less_than_limit_is_not_truncated(self) -> None:
+        accumulated_results = job_run_result.JobRunResult.accumulate([
+            job_run_result.JobRunResult(stdout='', stderr='a' * 1999),
+            job_run_result.JobRunResult(stdout='', stderr='b' * 3000),
+        ])
+
+        self.assertEqual(len(accumulated_results), 1)
+
+        self.assertItemsEqual(  # type: ignore[no-untyped-call]
+            accumulated_results[0].stderr.split('\n'), ['a' * 1999, 'b' * 3000])
+
+    def test_accumulate_one_more_than_limit_case_is_split(self) -> None:
+        accumulated_results = job_run_result.JobRunResult.accumulate([
+            job_run_result.JobRunResult(stdout='', stderr='a' * 2000),
+            job_run_result.JobRunResult(stdout='', stderr='b' * 3000),
+        ])
+
+        self.assertEqual(len(accumulated_results), 2)
+
     def test_accumulate_with_enormous_outputs(self) -> None:
         accumulated_results = job_run_result.JobRunResult.accumulate([
             job_run_result.JobRunResult(
-                stdout='a' * 750, stderr='b' * 750),
+                stdout='a' * 5002, stderr='b' * 5002),
             job_run_result.JobRunResult(
-                stdout='a' * 500, stderr='b' * 500),
+                stdout='a' * 2000, stderr='b' * 2000),
             job_run_result.JobRunResult(
-                stdout='a' * 250, stderr='b' * 250),
+                stdout='a' * 1000, stderr='b' * 1000),
             job_run_result.JobRunResult(
-                stdout='a' * 100, stderr='b' * 100),
+                stdout='a' * 1000, stderr='b' * 1000),
             job_run_result.JobRunResult(
-                stdout='a' * 50, stderr='b' * 50),
+                stdout='a' * 2000, stderr='b' * 2000),
         ])
 
         # 100000 and 200000 are small enough ot fit as one, but the others will
         # each need their own result.
-        self.assertEqual(len(accumulated_results), 3)
+        self.assertEqual(len(accumulated_results), 4)
 
     def test_accumulate_with_empty_list(self) -> None:
         self.assertEqual(job_run_result.JobRunResult.accumulate([]), [])
 
-    def test_len_in_bytes(self) -> None:
-        result = job_run_result.JobRunResult(stdout='123', stderr='123')
-        self.assertEqual(result.len_in_bytes(), 6)
-
-    def test_len_in_bytes_of_unicode(self) -> None:
-        result = job_run_result.JobRunResult(stdout='😀', stderr='😀')
-        self.assertEqual(result.len_in_bytes(), 8)
-
     def test_equality(self) -> None:
         a_result = job_run_result.JobRunResult(stdout='abc', stderr='123')
         b_result = job_run_result.JobRunResult(stdout='def', stderr='456')