Repository: orchest/orchest
Issue: Web server is blocked when taking snapshots

**To Reproduce**  
Steps to reproduce the behaviour:

- have a project of large size, `head -c 10GB /dev/zero > data`
- create a job
- the webserver will be blocked while the snapshot is being copied

This happens for job and project deletion as well.

**Expected behaviour**  
The web server should not be blocked, i.e. other requests should happen concurrently.



Changed files:
services/orchest-webserver/app/app/utils.py, services/orchest-webserver/app/app/core/projects.py

diff --git a/services/orchest-webserver/app/app/core/projects.py b/services/orchest-webserver/app/app/core/projects.py
index 25cfd9601..2c6b1bdf8 100644
--- a/services/orchest-webserver/app/app/core/projects.py
+++ b/services/orchest-webserver/app/app/core/projects.py
@@ -1,5 +1,4 @@
 import os
-import shutil
 import subprocess
 import uuid
 from typing import Optional
@@ -18,6 +17,7 @@ from app.utils import (
     populate_default_environments,
     project_uuid_to_path,
     remove_project_jobs_directories,
+    rmtree,
 )
 from app.views.orchest_api import api_proxy_environment_builds
 
@@ -153,18 +153,11 @@ class DeleteProject(TwoPhaseFunction):
         """Remove a project from the fs and the orchest-api"""
 
         # Delete the project directory.
-        try:
-            project_path = project_uuid_to_path(project_uuid)
-            full_project_path = os.path.join(
-                current_app.config["PROJECTS_DIR"], project_path
-            )
-            shutil.rmtree(full_project_path)
-        except FileNotFoundError:
-            # If the `full_project_path` is not found, it means that the
-            # user has already performed the deletion operation. So we
-            # need to catch and then ignore this error, otherwise the DB
-            # deletion operation will not continue.
-            pass
+        project_path = project_uuid_to_path(project_uuid)
+        full_project_path = os.path.join(
+            current_app.config["PROJECTS_DIR"], project_path
+        )
+        rmtree(full_project_path)
 
         # Remove jobs directories related to project.
         remove_project_jobs_directories(project_uuid)
diff --git a/services/orchest-webserver/app/app/utils.py b/services/orchest-webserver/app/app/utils.py
index 6b42b19fa..2a84144a6 100644
--- a/services/orchest-webserver/app/app/utils.py
+++ b/services/orchest-webserver/app/app/utils.py
@@ -2,7 +2,7 @@ import hashlib
 import json
 import os
 import re
-import shutil
+import subprocess
 import uuid
 from urllib.parse import unquote
 
@@ -248,7 +248,7 @@ def delete_environment(app, project_uuid, environment_uuid):
     app.config["SCHEDULER"].add_job(requests.delete, args=[url])
 
     environment_dir = get_environment_directory(environment_uuid, project_uuid)
-    shutil.rmtree(environment_dir)
+    rmtree(environment_dir)
 
 
 def populate_default_environments(project_uuid):
@@ -348,7 +348,7 @@ def clear_folder(folder):
                 if os.path.isfile(file_path) or os.path.islink(file_path):
                     os.unlink(file_path)
                 elif os.path.isdir(file_path):
-                    shutil.rmtree(file_path)
+                    rmtree(file_path)
             except Exception as e:
                 current_app.logger.error(
                     "Failed to delete %s. Reason: %s" % (file_path, e)
@@ -358,13 +358,8 @@ def clear_folder(folder):
 
 
 def remove_dir_if_empty(path):
-    try:
-        if os.path.isdir(path) and not any(True for _ in os.scandir(path)):
-            shutil.rmtree(path, ignore_errors=True)
-    except FileNotFoundError as e:
-        current_app.logger.error(
-            "Failed to remove directory %s. Error: %s [%s]" % (path, e, type(e))
-        )
+    if os.path.isdir(path) and not any(True for _ in os.scandir(path)):
+        rmtree(path, ignore_errors=True)
 
 
 def pipeline_uuid_to_path(pipeline_uuid, project_uuid, job_uuid=None):
@@ -517,20 +512,6 @@ def write_config(app, key, value):
 
 
 def create_job_directory(job_uuid, pipeline_uuid, project_uuid):
-    def ignore_patterns(path, fnames):
-        """
-            Example:
-                path, fnames = \\
-                'docker/catching-error/testing', \\
-                ['hello.txt', 'some-dir']
-            """
-        # Ignore the ".orchest/pipelines" directory containing the
-        # logs and data directories.
-        if path.endswith(".orchest"):
-            return ["pipelines"]
-
-        # Ignore nothing.
-        return []
 
     snapshot_path = os.path.join(
         get_job_directory(pipeline_uuid, project_uuid, job_uuid),
@@ -543,7 +524,43 @@ def create_job_directory(job_uuid, pipeline_uuid, project_uuid):
         current_app.config["USER_DIR"], "projects", project_uuid_to_path(project_uuid)
     )
 
-    shutil.copytree(project_dir, snapshot_path, ignore=ignore_patterns)
+    copytree(project_dir, snapshot_path)
+    # Ignore the ".orchest/pipelines" directory containing the logs and
+    # data directories. Ignore errors because the directory might not be
+    # there.
+    rmtree(os.path.join(snapshot_path, ".orchest/pipelines"), ignore_errors=True)
+
+
+def rmtree(path, ignore_errors=False):
+    """A wrapped rm -rf.
+
+    If eventlet is being used and it's either patching all modules or
+    patchng subprocess, this function is not going to block the thread.
+
+    Raises:
+        OSError if it failed to copy.
+
+    """
+    exit_code = subprocess.call(f"rm -rf {path}", stderr=subprocess.STDOUT, shell=True)
+    if exit_code != 0 and not ignore_errors:
+        raise OSError(f"Failed to rm {path}: {exit_code}.")
+
+
+def copytree(source: str, target: str):
+    """A wrapped cp -r.
+
+    If eventlet is being used and it's either patching all modules or
+    patchng subprocess, this function is not going to block the thread.
+
+    Raises:
+        OSError if it failed to copy.
+
+    """
+    exit_code = subprocess.call(
+        f"cp -r {source} {target}", stderr=subprocess.STDOUT, shell=True
+    )
+    if exit_code != 0:
+        raise OSError(f"Failed to copy {source} to {target}, :{exit_code}.")
 
 
 def remove_job_directory(job_uuid, pipeline_uuid, project_uuid):
@@ -555,7 +572,7 @@ def remove_job_directory(job_uuid, pipeline_uuid, project_uuid):
     job_path = os.path.join(job_pipeline_path, job_uuid)
 
     if os.path.isdir(job_path):
-        shutil.rmtree(job_path, ignore_errors=True)
+        rmtree(job_path, ignore_errors=True)
 
     # Clean up parent directory if this job removal created empty
     # directories.
@@ -570,7 +587,7 @@ def remove_project_jobs_directories(project_uuid):
     )
 
     if os.path.isdir(project_jobs_path):
-        shutil.rmtree(project_jobs_path, ignore_errors=True)
+        rmtree(project_jobs_path, ignore_errors=True)
 
 
 def get_ipynb_template(language: str):