Repository: codalab/codalab-worksheets

Issue:
Requesting too little memory for a run can cause a worker to go down
Requesting as little as `10m` for a run can cause a worker to go down with the following error:
```
2020-06-27 03:09:59,180 Cannot start Docker container: Unable to start Docker container: 500 Server Error: 
Internal Server Error ("OCI runtime create failed: container_linux.go:349: starting container process caused 
"process_linux.go:449: container init caused \"process_linux.go:415: setting cgroup config for procHooks process caused 
\\\"failed to write \\\\\\\"10485760\\\\\\\" to \\\\\\\"/sys/fs/cgroup/memory/docker
/07dff7f7e04f43b430b20266840119e33a0070883eaa35d16774525aa352ee5c/memory.limit_in_bytes\\\\\\\": 
write /sys/fs/cgroup/memory/docker/07dff7f7e04f43b430b20266840119e33a0070883eaa35d16774525aa352ee5c/memory.limit_in_bytes: device or resource busy\\\"\"": unknown")
```
See this related issue: https://github.com/moby/moby/issues/38921


Discussion:

Patch:
diff --git a/codalab/lib/worksheet_util.py b/codalab/lib/worksheet_util.py
index 5b3868a7..6103459c 100644
--- a/codalab/lib/worksheet_util.py
+++ b/codalab/lib/worksheet_util.py
@@ -767,6 +767,7 @@ def interpret_items(schemas, raw_items, db_model=None):
                             'status': FetchStatusSchema.get_unknown_status(),
                             'header': header,
                             'rows': rows,
+                            'sort_keys': [bundle_info["sort_key"]],
                         }
                     )
                     .data
diff --git a/codalab/worker/docker_utils.py b/codalab/worker/docker_utils.py
index a5d2e43c..4ebf0b10 100644
--- a/codalab/worker/docker_utils.py
+++ b/codalab/worker/docker_utils.py
@@ -23,12 +23,16 @@ DEFAULT_CONTAINER_RUNNING_TIME = 0
 # Docker Registry HTTP API v2 URI prefix
 URI_PREFIX = 'https://hub.docker.com/v2/repositories/'
 
-# This specific error happens when a user specifies an image with an incompatible version of Cuda
+# This specific error happens when a user specifies an image with an incompatible version of CUDA
 NVIDIA_MOUNT_ERROR_REGEX = (
     '[\s\S]*OCI runtime create failed[\s\S]*stderr:[\s\S]*nvidia-container-cli: '
     'mount error: file creation failed:[\s\S]*nvidia-smi'
 )
-
+# This error happens when the memory requested by user is not enough to start a docker container
+MEMORY_LIMIT_ERROR_REGEX = (
+    '[\s\S]*OCI runtime create failed[\s\S]*failed to write[\s\S]*'
+    'memory.limit_in_bytes: device or resource busy[\s\S]*'
+)
 
 logger = logging.getLogger(__name__)
 client = docker.from_env(timeout=DEFAULT_TIMEOUT)
@@ -44,6 +48,8 @@ def wrap_exception(message):
                 error_message = format_error_message(e)
                 if re.match(NVIDIA_MOUNT_ERROR_REGEX, str(exception)):
                     raise DockerUserErrorException(error_message)
+                elif re.match(MEMORY_LIMIT_ERROR_REGEX, str(exception)):
+                    raise DockerUserErrorException(error_message)
                 else:
                     raise DockerException(error_message)
 


Test patch:
diff --git a/tests/worker/docker_utils_test.py b/tests/worker/docker_utils_test.py
index c1688097..a25c06cb 100644
--- a/tests/worker/docker_utils_test.py
+++ b/tests/worker/docker_utils_test.py
@@ -41,3 +41,25 @@ class DockerUtilsTest(unittest.TestCase):
         except Exception as e:
             self.assertEqual(str(e), 'Should throw DockerUserErrorException: ' + error)
             self.assertIsInstance(e, DockerUserErrorException)
+
+    def test_wrap_exception_with_memory_limit_error(self):
+        error = (
+            'Unable to start Docker container: 500 Server Error: Internal Server Error '
+            '("OCI runtime create failed: container_linux.go:349: starting container process '
+            'caused "process_linux.go:449: container init caused \"process_linux.go:415: '
+            'setting cgroup config for procHooks process caused \\\"failed to write\\\\\\\"8388608'
+            '\\\\\\\" to \\\\\\\"/sys/fs/cgroup/memory/docker/a5475e95e98bbb534870dfdf290e91251f54'
+            'e5c13be07a7b6819619a2dba48ef/memory.limit_in_bytes\\\\\\\":write /sys/fs/cgroup/memory'
+            '/docker/a5475e95e98bbb534870dfdf290e91251f54e5c13be07a7b6819619a2dba48ef/'
+            'memory.limit_in_bytes: device or resource busy\\\"\"": unknown'
+        )
+
+        @wrap_exception('Should throw DockerUserErrorException')
+        def throw_memory_error():
+            raise APIError(error)
+
+        try:
+            throw_memory_error()
+        except Exception as e:
+            self.assertEqual(str(e), 'Should throw DockerUserErrorException: ' + error)
+            self.assertIsInstance(e, DockerUserErrorException)