Repository: conda/conda

Issue:
conda clean -p breaks hard linking of new installs
@asmeurer 
Following on from conda/conda#68:

On Windows 8 (and probably equally on all other windows versions), `conda clean -p` removes all extracted packaged from the `pkgs` directory even if they are hard linked in some environments.

While all existing environments will continue to function, creation of a new environment with the same packages causes conda to extract the packages again and  to hard link to the newly extracted files. This massively increases disk usage and results in the opposite of what a user running `conda clean` was attempting to achieve.

The problem lies in `main_clean.py` and the fact that on Windows  `lstat(file).st_nlink` always returns 0, even if `file` is hard linked. (This seems to have been fixed from python 3.2 onwards: https://bugs.python.org/issue10027)

As a stop gap measure `conda clean -p` should be prevented from being run on windows until a better solution can be found.

```
C:\>conda list
# packages in environment at C:\Anaconda:
#
conda                     3.10.1                   py27_0
conda-env                 2.1.4                    py27_0
menuinst                  1.0.4                    py27_0
psutil                    2.2.1                    py27_0
pycosat                   0.6.1                    py27_0
python                    2.7.9                         1
pyyaml                    3.11                     py27_0
requests                  2.6.0                    py27_0
```



Discussion:
Yep, pretty annoying issue.

> The problem lies in main_clean.py and the fact that on Windows lstat(file).st_nlink always returns 0, even if file is hard linked. (This seems to have been fixed from python 3.2 onwards: https://bugs.python.org/issue10027)

If it's really the case then WinAPI **GetFileInformationByHandle** should be used. The nNumberOfLinks field of the BY_HANDLE_FILE_INFORMATION structure is exactly what we want. **ntfsutils** has very nice implementation of the thing here http://pydoc.net/Python/ntfsutils/0.1.3/ntfsutils.fs/ , I guess it can be borrowed. You know, `if getfileinfo(path).nNumberOfLinks > 1: ...`


Resolving patch:
diff --git a/conda/cli/main_clean.py b/conda/cli/main_clean.py
--- a/conda/cli/main_clean.py
+++ b/conda/cli/main_clean.py
@@ -65,6 +65,95 @@ def configure_parser(sub_parsers):
     p.set_defaults(func=execute)
 
 
+# work-around for python bug on Windows prior to python 3.2
+# https://bugs.python.org/issue10027
+# Adapted from the ntfsutils package, Copyright (c) 2012, the Mozilla Foundation
+class CrossPlatformStLink(object):
+    _st_nlink = None
+    
+    def __call__(self, path):
+        return self.st_nlink(path)
+
+    @classmethod
+    def st_nlink(cls, path):
+        if cls._st_nlink is None:
+            cls._initialize()
+        return cls._st_nlink(path)
+
+    @classmethod
+    def _standard_st_nlink(cls, path):
+        return lstat(path).st_nlink
+
+    @classmethod
+    def _windows_st_nlink(cls, path):
+        st_nlink = cls._standard_st_nlink(path)
+        if st_nlink != 0:
+            return st_nlink
+        else:
+            # cannot trust python on Windows when st_nlink == 0
+            # get value using windows libraries to be sure of its true value
+            # Adapted from the ntfsutils package, Copyright (c) 2012, the Mozilla Foundation
+            GENERIC_READ = 0x80000000
+            FILE_SHARE_READ = 0x00000001
+            OPEN_EXISTING = 3
+            hfile = cls.CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, 
+                                   OPEN_EXISTING, 0, None)
+            if hfile is None:
+                from ctypes import WinError
+                raise WinError()
+            info = cls.BY_HANDLE_FILE_INFORMATION()
+            rv = cls.GetFileInformationByHandle(hfile, info)
+            cls.CloseHandle(hfile)
+            if rv == 0:
+                from ctypes import WinError
+                raise WinError()
+            return info.nNumberOfLinks
+
+    @classmethod
+    def _initialize(cls):
+        if os.name != 'nt':
+            cls._st_nlink = cls._standard_st_nlink
+        else:
+            # http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858
+            import ctypes
+            from ctypes import POINTER, WinError
+            from ctypes.wintypes import DWORD, HANDLE, BOOL
+
+            cls.CreateFile = ctypes.windll.kernel32.CreateFileW
+            cls.CreateFile.argtypes = [ctypes.c_wchar_p, DWORD, DWORD, ctypes.c_void_p,
+                                       DWORD, DWORD, HANDLE]
+            cls.CreateFile.restype = HANDLE
+
+            # http://msdn.microsoft.com/en-us/library/windows/desktop/ms724211
+            cls.CloseHandle = ctypes.windll.kernel32.CloseHandle
+            cls.CloseHandle.argtypes = [HANDLE]
+            cls.CloseHandle.restype = BOOL
+
+            class FILETIME(ctypes.Structure):
+                _fields_ = [("dwLowDateTime", DWORD),
+                            ("dwHighDateTime", DWORD)]
+
+            class BY_HANDLE_FILE_INFORMATION(ctypes.Structure):
+                _fields_ = [("dwFileAttributes", DWORD),
+                            ("ftCreationTime", FILETIME),
+                            ("ftLastAccessTime", FILETIME),
+                            ("ftLastWriteTime", FILETIME),
+                            ("dwVolumeSerialNumber", DWORD),
+                            ("nFileSizeHigh", DWORD),
+                            ("nFileSizeLow", DWORD),
+                            ("nNumberOfLinks", DWORD),
+                            ("nFileIndexHigh", DWORD),
+                            ("nFileIndexLow", DWORD)]
+            cls.BY_HANDLE_FILE_INFORMATION = BY_HANDLE_FILE_INFORMATION
+
+            # http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952
+            cls.GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle
+            cls.GetFileInformationByHandle.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
+            cls.GetFileInformationByHandle.restype = BOOL
+            
+            cls._st_nlink = cls._windows_st_nlink
+
+
 def find_lock():
     from os.path import join
 
@@ -164,7 +253,8 @@ def find_pkgs():
     # TODO: This doesn't handle packages that have hard links to files within
     # themselves, like bin/python3.3 and bin/python3.3m in the Python package
     warnings = []
-
+    
+    cross_platform_st_nlink = CrossPlatformStLink()
     pkgs_dirs = defaultdict(list)
     for pkgs_dir in config.pkgs_dirs:
         if not os.path.exists(pkgs_dir):
@@ -180,11 +270,11 @@ def find_pkgs():
                     break
                 for fn in files:
                     try:
-                        stat = lstat(join(root, fn))
+                        st_nlink = cross_platform_st_nlink(join(root, fn))
                     except OSError as e:
                         warnings.append((fn, e))
                         continue
-                    if stat.st_nlink > 1:
+                    if st_nlink > 1:
                         # print('%s is installed: %s' % (pkg, join(root, fn)))
                         breakit = True
                         break


Test patch: