Repository: conda/conda-build

Issue:
Text replace on Windows broken
After building `numexpr` using `conda-build` (using this recipe https://github.com/ContinuumIO/anaconda-recipes/tree/master/numexpr), a user reported:

```
[testing] C:\Users\palm\Documents\antony\src\ipython>python -c "import numexpr"
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Users\palm\Miniconda3\envs\testing\lib\site-packages\numexpr\__init__
.py", line 24, in <module>
    from .__config__ import show as show_config, get_info
  File "C:\Users\palm\Miniconda3\envs\testing\lib\site-packages\numexpr\__config
__.py", line 5
    mkl_info={'library_dirs': ['C:\Users\palm\Miniconda3\envs\testing\\Library\\
lib'], 'define_macros': [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)], 'librarie
s': ['mkl_sequential_dll', 'mkl_core_dll', 'mkl_intel_lp64_dll', 'mkl_intel_thre
ad_dll'], 'include_dirs': ['C:\Users\palm\Miniconda3\envs\testing\\Library\\incl
ude']}
                              ^
SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in positio
n 2-3: truncated \UXXXXXXXX escape
```

Something strange is happening with the text replacement here.  The `numexpr` conda package we are talking about is: http://repo.continuum.io/pkgs/free/win-64/numexpr-2.6.0-np110py35_0.tar.bz2
It contains `info/has_prefix`, with:

```
"C:\\Minonda\\envs\\_build" text "Lib/site-packages/numexpr/__config__.py"
```

Something regarding the `\`'s is wrong here.  I'm not sure if the bug is in `conda-build` or `conda`.
Originally, `numexpr` has always been built using the internal Anaconda system, which by default would never create `info/has_prefix` on Windows.

CC @msarahan 



Discussion:
Ah, and the reason this wasn't caught using testing is because my prefix is `C:\Minonda\envs\...`, but
when you install the package into `C:\Users` (which contains `\U`) the wrong escaping strikes!

In the package you linked to, the contents of `Lib/site-packages/numexpr/__config__.py` seem to be escaped properly.  I think this bug might lie in conda's manipulation of that path when installing the package.

However, I want to know more about why you say you'd never create has_prefix on Windows - how would you fix the paths in these files?  What would make has_prefix unnecessary?

CC @kalefranz - should we move this one to conda?

On Windows, `info/has_prefix` files have never been necessary as far as Anaconda is concerned.
It would be nice to have a `no_has_prefix` option in `meta.yaml`, then I could fix the problem by adding:

```
build:
  no_has_prefix: True              [win]
```

I will inspect other new Windows packages which have been created using conda recipes, to see if they contain `info/has_prefix`, and if that might cause problems.

OK, but how does that `__config__.py` get handled with your build system?  It seems to have important paths in it that need to be reconfigured on end users' systems.  Is that true, or am I missing something?

`__config__.py` didn't get handled in the Anaconda build system.  I'm not sure what the `__config__.py` file is used for, but it never caused any problems not having it updated upon install.


Resolving patch:
diff --git a/conda_build/build.py b/conda_build/build.py
--- a/conda_build/build.py
+++ b/conda_build/build.py
@@ -269,6 +269,17 @@ def create_info_files(m, files, include_recipe=True):
     files_with_prefix = sorted(have_prefix_files(files))
     binary_has_prefix_files = m.binary_has_prefix_files()
     text_has_prefix_files = m.has_prefix_files()
+
+    ignore_files = m.ignore_prefix_files()
+    if ignore_files:
+        # do we have a list of files, or just ignore everything?
+        if hasattr(ignore_files, "__iter__"):
+            files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files]
+            binary_has_prefix_files = [f for f in binary_has_prefix_files if f[2] not in ignore_files]  # noqa
+            text_has_prefix_files = [f for f in text_has_prefix_files if f[2] not in ignore_files]
+        else:
+            files_with_prefix = []
+
     if files_with_prefix and not m.get_value('build/noarch_python'):
         auto_detect = m.get_value('build/detect_binary_files_with_prefix')
         if sys.platform == 'win32':
diff --git a/conda_build/metadata.py b/conda_build/metadata.py
--- a/conda_build/metadata.py
+++ b/conda_build/metadata.py
@@ -278,8 +278,8 @@ def _git_clean(source_meta):
     'build': ['number', 'string', 'entry_points', 'osx_is_app',
               'features', 'track_features', 'preserve_egg_dir',
               'no_link', 'binary_relocation', 'script', 'noarch_python',
-              'has_prefix_files', 'binary_has_prefix_files', 'script_env',
-              'detect_binary_files_with_prefix', 'rpaths',
+              'has_prefix_files', 'binary_has_prefix_files', 'ignore_prefix_files',
+              'detect_binary_files_with_prefix', 'rpaths', 'script_env',
               'always_include_files', 'skip', 'msvc_compiler',
               'pin_depends'  # pin_depends is experimental still
               ],
@@ -589,6 +589,16 @@ def has_prefix_files(self):
                                    "as the path delimiter on Windows")
         return ret
 
+    def ignore_prefix_files(self):
+        ret = self.get_value('build/ignore_prefix_files', False)
+        if type(ret) not in (list, bool):
+            raise RuntimeError('build/ignore_prefix_files should be boolean or a list of paths')
+        if sys.platform == 'win32':
+            if any('\\' in i for i in ret):
+                raise RuntimeError("build/ignore_prefix_files paths must use / "
+                                   "as the path delimiter on Windows")
+        return ret
+
     def always_include_files(self):
         return self.get_value('build/always_include_files', [])
 


Test patch:
diff --git a/tests/test-recipes/metadata/ignore_prefix_files/meta.yaml b/tests/test-recipes/metadata/ignore_prefix_files/meta.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/ignore_prefix_files/meta.yaml
@@ -0,0 +1,14 @@
+package:
+  name: conda-build-test-ignore-prefix-files
+  version: 1.0
+
+build:
+  ignore_prefix_files: True
+  script:
+    - echo %PREFIX%\\test.txt > %SCRIPTS%\\test.bat  # [win]
+    - echo %PREFIX%\\test2.txt > %SCRIPTS%\\test2.bat  # [win]
+    - echo ${PREFIX}/bin/test.txt > ${PREFIX}/test.sh  # [unix]
+    - echo ${PREFIX}/bin/test2.txt > ${PREFIX}/test2.sh  # [unix]
+
+about:
+  summary: test that ignore_prefix_files with boolean setting ignores all files
diff --git a/tests/test-recipes/metadata/ignore_prefix_files/run_test.py b/tests/test-recipes/metadata/ignore_prefix_files/run_test.py
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/ignore_prefix_files/run_test.py
@@ -0,0 +1,8 @@
+import os
+import sys
+
+# assumes that sys.prefix is <root>/envs/_test
+pkgs = os.path.join(sys.prefix, "..", "..", "pkgs")
+info_dir = os.path.join(pkgs, "conda-build-test-ignore-prefix-files-1.0-0", "info")
+assert os.path.isdir(info_dir)
+assert not os.path.isfile(os.path.join(info_dir, "has_prefix"))
diff --git a/tests/test-recipes/metadata/ignore_some_prefix_files/meta.yaml b/tests/test-recipes/metadata/ignore_some_prefix_files/meta.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/ignore_some_prefix_files/meta.yaml
@@ -0,0 +1,19 @@
+package:
+  name: conda-build-test-ignore-some-prefix-files
+  version: 1.0
+
+build:
+  #has_prefix_files:
+  #  - Scripts\\test.bat  # [win]
+  #  - test.sh  # [unix]
+  ignore_prefix_files:
+    - Scripts\\test2.bat  # [win]
+    - test2.sh  # [unix]
+  script:
+    - echo %PREFIX%\\test.txt > %SCRIPTS%\\test.bat  # [win]
+    - echo %PREFIX%\\test2.txt > %SCRIPTS%\\test2.bat  # [win]
+    - echo ${PREFIX}/bin/test.txt > ${PREFIX}/test.sh  # [unix]
+    - echo ${PREFIX}/bin/test2.txt > ${PREFIX}/test2.sh  # [unix]
+
+about:
+  summary: test that ignore_prefix_files with list setting ignores specified files
diff --git a/tests/test-recipes/metadata/ignore_some_prefix_files/run_test.py b/tests/test-recipes/metadata/ignore_some_prefix_files/run_test.py
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/ignore_some_prefix_files/run_test.py
@@ -0,0 +1,11 @@
+import os
+import sys
+
+# assumes that sys.prefix is <root>/envs/_test
+pkgs = os.path.normpath(os.path.join(sys.prefix, "..", "..", "pkgs"))
+info_dir = os.path.join(pkgs, "conda-build-test-ignore-some-prefix-files-1.0-0", "info")
+has_prefix_file = os.path.join(info_dir, "has_prefix")
+print(info_dir)
+assert os.path.isfile(has_prefix_file)
+with open(has_prefix_file) as f:
+    assert "test2" not in f.read()