Repository: spack/spack

Issue:
File permissions and calls to "llnl.util.filesystem.remove_linked_tree"
### Steps to reproduce

Call from a script or a unit-test:

https://github.com/spack/spack/blob/f8201f4acc53bbb7efcde77a369546181b802d19/lib/spack/llnl/util/filesystem.py#L1183-L1209

with a path that fails to perform `shutil.rmtree`, e.g. the path of a file. That on linux leaves the file with different permissions because of how `onerror` handles the error.

### Error message

This was leading to issues like #29687 

### Information on your system

System is not relevant

@scheibelp proposed a possible fix for Windows in https://github.com/spack/spack/pull/29690#issuecomment-1076751911

### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have searched the issues of this repo and believe this is not a duplicate
- [X] I have run the failing commands in debug mode and reported the output


Discussion:
Apologies, I was out yesterday and didn't get a chance to respond to @scheibelp. Rather than pollute that PR, I'll post my response here. I think Peter has a fine solution, I'll push shortly.

Resolving patch:
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -1091,7 +1091,32 @@ def visit_directory_tree(root, visitor, rel_path='', depth=0):
     for f in dir_entries:
         if sys.version_info >= (3, 5, 0):
             rel_child = os.path.join(rel_path, f.name)
-            islink, isdir = f.is_symlink(), f.is_dir()
+            islink = f.is_symlink()
+            # On Windows, symlinks to directories are distinct from
+            # symlinks to files, and it is possible to create a
+            # broken symlink to a directory (e.g. using os.symlink
+            # without `target_is_directory=True`), invoking `isdir`
+            # on a symlink on Windows that is broken in this manner
+            # will result in an error. In this case we can work around
+            # the issue by reading the target and resolving the
+            # directory ourselves
+            try:
+                isdir = f.is_dir()
+            except OSError as e:
+                if is_windows and hasattr(e, 'winerror')\
+                   and e.winerror == 5 and islink:
+                    # if path is a symlink, determine destination and
+                    # evaluate file vs directory
+                    link_target = resolve_link_target_relative_to_the_link(f)
+                    # link_target might be relative but
+                    # resolve_link_target_relative_to_the_link
+                    # will ensure that if so, that it is relative
+                    # to the CWD and therefore
+                    # makes sense
+                    isdir = os.path.isdir(link_target)
+                else:
+                    raise e
+
         else:
             rel_child = os.path.join(rel_path, f)
             lexists, islink, isdir = lexists_islink_isdir(os.path.join(dir, f))
@@ -1099,7 +1124,7 @@ def visit_directory_tree(root, visitor, rel_path='', depth=0):
                 continue
 
         if not isdir:
-            # Handle files
+            # handle files
             visitor.visit_file(root, rel_child, depth)
         elif not islink and visitor.before_visit_dir(root, rel_child, depth):
             # Handle ordinary directories
@@ -1174,6 +1199,35 @@ def remove_if_dead_link(path):
         os.unlink(path)
 
 
+def readonly_file_handler(ignore_errors=False):
+    # TODO: generate stages etc. with write permissions wherever
+    # so this callback is no-longer required
+    """
+    Generate callback for shutil.rmtree to handle permissions errors on
+    Windows. Some files may unexpectedly lack write permissions even
+    though they were generated by Spack on behalf of the user (e.g. the
+    stage), so this callback will detect such cases and modify the
+    permissions if that is the issue. For other errors, the fallback
+    is either to raise (if ignore_errors is False) or ignore (if
+    ignore_errors is True). This is only intended for Windows systems
+    and will raise a separate error if it is ever invoked (by accident)
+    on a non-Windows system.
+    """
+    def error_remove_readonly(func, path, exc):
+        if not is_windows:
+            raise RuntimeError("This method should only be invoked on Windows")
+        excvalue = exc[1]
+        if is_windows and func in (os.rmdir, os.remove, os.unlink) and\
+           excvalue.errno == errno.EACCES:
+            # change the file to be readable,writable,executable: 0777
+            os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+            # retry
+            func(path)
+        elif not ignore_errors:
+            raise
+    return error_remove_readonly
+
+
 @system_path_filter
 def remove_linked_tree(path):
     """Removes a directory and its contents.
@@ -1181,23 +1235,18 @@ def remove_linked_tree(path):
     If the directory is a symlink, follows the link and removes the real
     directory before removing the link.
 
+    This method will force-delete files on Windows
+
     Parameters:
         path (str): Directory to be removed
     """
-    # On windows, cleaning a Git stage can be an issue
-    # as git leaves readonly files that Python handles
-    # poorly on Windows. Remove readonly status and try again
-    def onerror(func, path, exe_info):
-        os.chmod(path, stat.S_IWUSR)
-        try:
-            func(path)
-        except Exception as e:
-            tty.warn(e)
-            pass
-
     kwargs = {'ignore_errors': True}
+
+    # Windows readonly files cannot be removed by Python
+    # directly.
     if is_windows:
-        kwargs = {'onerror': onerror}
+        kwargs['ignore_errors'] = False
+        kwargs['onerror'] = readonly_file_handler(ignore_errors=True)
 
     if os.path.exists(path):
         if os.path.islink(path):
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -9,6 +9,7 @@
 import posixpath
 import re
 import shutil
+import sys
 import tempfile
 from contextlib import contextmanager
 
@@ -24,6 +25,7 @@
 import spack.util.spack_json as sjson
 from spack.error import SpackError
 
+is_windows = sys.platform == 'win32'
 # Note: Posixpath is used here as opposed to
 # os.path.join due to spack.spec.Spec.format
 # requiring forward slash path seperators at this stage
@@ -349,6 +351,14 @@ def remove_install_directory(self, spec, deprecated=False):
         path = self.path_for_spec(spec)
         assert(path.startswith(self.root))
 
+        # Windows readonly files cannot be removed by Python
+        # directly, change permissions before attempting to remove
+        if is_windows:
+            kwargs = {'ignore_errors': False,
+                      'onerror': fs.readonly_file_handler(ignore_errors=False)}
+        else:
+            kwargs = {}  # the default value for ignore_errors is false
+
         if deprecated:
             if os.path.exists(path):
                 try:
@@ -357,10 +367,9 @@ def remove_install_directory(self, spec, deprecated=False):
                     os.remove(metapath)
                 except OSError as e:
                     raise six.raise_from(RemoveFailedError(spec, path, e), e)
-
         elif os.path.exists(path):
             try:
-                shutil.rmtree(path)
+                shutil.rmtree(path, **kwargs)
             except OSError as e:
                 raise six.raise_from(RemoveFailedError(spec, path, e), e)
 


Test patch:
diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py
--- a/lib/spack/spack/test/llnl/util/link_tree.py
+++ b/lib/spack/spack/test/llnl/util/link_tree.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
-import sys
 
 import pytest
 
@@ -14,9 +13,6 @@
 
 from spack.stage import Stage
 
-pytestmark = pytest.mark.skipif(sys.platform == "win32",
-                                reason="does not run on windows")
-
 
 @pytest.fixture()
 def stage():