Repository: mesonbuild/meson

Issue:
CMake subproject sandbox violation
**Describe the bug**
Meson seems to be hitting a sandbox violation when using a CMake subproject.

Offending CMake code: https://github.com/mongodb/mongo-c-driver/blob/master/src/libbson/CMakeLists.txt#L122=.

Log: https://github.com/hse-project/hse/runs/7198083520?check_suite_focus=true#step%3A12%3A360=

**To Reproduce**
Use mongo-c-driver as a subproject. You can clone hse-project/hse and -Dwrap_mode=forcefallback.

**Expected behavior**
Not hit a sandbox violation using the cmake module.

**system parameters**
* Is this a [cross build](https://mesonbuild.com/Cross-compilation.html) or just a plain native build (for the same computer)? no
* what operating system (e.g. MacOS Catalina, Windows 10, CentOS 8.0, Ubuntu 18.04, etc.) fc36
* what Python version are you using 3.10.5
* what `meson --version` 0.63.0
* what `ninja --version` if it's a Ninja build 1.10.2



Discussion:
I'm guessing this is somehow related to the way the cmake module indirectly does all its work inside `builddir/subprojects/{subproject_name}/__CMake_build/`, in this case probably grabbing a configured file and adding it as a files() object?
I can probably grab the generated meson.build file tomorrow. Will update since I assume that will assist in pinpointing the issue.
[meson.build.txt](https://github.com/mesonbuild/meson/files/9056847/meson.build.txt)

Relevant:

```
bson_shared_src = files(
  'src/libbson/src/bson/bcon.c',
  'src/libbson/src/bson/bson.c',
  'src/libbson/src/bson/bson-atomic.c',
  'src/libbson/src/bson/bson-clock.c',
  'src/libbson/src/bson/bson-context.c',
  'src/libbson/src/bson/bson-decimal128.c',
  'src/libbson/src/bson/bson-error.c',
  'src/libbson/src/bson/bson-iso8601.c',
  'src/libbson/src/bson/bson-iter.c',
  'src/libbson/src/bson/bson-json.c',
  'src/libbson/src/bson/bson-keys.c',
  'src/libbson/src/bson/bson-md5.c',
  'src/libbson/src/bson/bson-memory.c',
  'src/libbson/src/bson/bson-oid.c',
  'src/libbson/src/bson/bson-reader.c',
  'src/libbson/src/bson/bson-string.c',
  'src/libbson/src/bson/bson-timegm.c',
  'src/libbson/src/bson/bson-utf8.c',
  'src/libbson/src/bson/bson-value.c',
  'src/libbson/src/bson/bson-version-functions.c',
  'src/libbson/src/bson/bson-writer.c',
  'src/libbson/src/jsonsl/jsonsl.c',
  'src/common/common-b64.c',
  'src/common/common-md5.c',
  'src/common/common-thread.c',
  '/home/tpartin/Projects/hse/build-0.63/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src/bson/bson-config.h',
  '/home/tpartin/Projects/hse/build-0.63/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src/bson/bson-version.h',
  'src/libbson/src/bson/bcon.h',
  'src/libbson/src/bson/bson-atomic.h',
  'src/libbson/src/bson/bson-clock.h',
  'src/libbson/src/bson/bson-compat.h',
  'src/libbson/src/bson/bson-context.h',
  'src/libbson/src/bson/bson-decimal128.h',
  'src/libbson/src/bson/bson-endian.h',
  'src/libbson/src/bson/bson-error.h',
  'src/libbson/src/bson/bson.h',
  'src/libbson/src/bson/bson-iter.h',
  'src/libbson/src/bson/bson-json.h',
  'src/libbson/src/bson/bson-keys.h',
  'src/libbson/src/bson/bson-macros.h',
  'src/libbson/src/bson/bson-md5.h',
  'src/libbson/src/bson/bson-memory.h',
  'src/libbson/src/bson/bson-oid.h',
  'src/libbson/src/bson/bson-prelude.h',
  'src/libbson/src/bson/bson-reader.h',
  'src/libbson/src/bson/bson-string.h',
  'src/libbson/src/bson/bson-types.h',
  'src/libbson/src/bson/bson-utf8.h',
  'src/libbson/src/bson/bson-value.h',
  'src/libbson/src/bson/bson-version-functions.h',
  'src/libbson/src/bson/bson-writer.h',
  'src/libbson/src/bson/forwarding/bson.h'
)
```

Pretty obvious what is causing the problem! Seems like the cmake module is not removing the `PROJECT_BUILD_DIR` prefix.
I bisected the offending commit to:

```
$ git bisect good
e6e8159980e9a4c816223fcdac8c729d104c0c02 is the first bad commit
commit e6e8159980e9a4c816223fcdac8c729d104c0c02
Author: Eli Schwartz <eschwartz@archlinux.org>
Date:   Wed Mar 9 19:45:42 2022 -0500

    use shared implementation to convert files() strings to File objects
    
    This handles various edge cases:
    - checks for sandbox violations just like all other functions
    - warn for direntry issues
    - check for generated files referred to via strings instead of the
      returned object
    
    (All valid use cases for wanting to sneak around the checks, are made to
    work via commit bba588d8b03a9125bf5c4faaad31b70d39242b68.)

 mesonbuild/interpreter/interpreter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
```

But I don't think this is the root cause. The generated meson.build file still has full paths in the `files()` function as shown above.
I have an even better log, which tells you exactly what is going on inside the "sandbox violation" error:

```
project_root=PosixPath('/tmp/hse/subprojects/mongo-c-driver-1.20.1'), subproject_dir=PosixPath('/tmp/hse/subprojects/mongo-c-driver-1.20.1/subprojects'), list(norm.parents)=[PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src/bson'), PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src'), PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson'), PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src'), PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build'), PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1'), PosixPath('/tmp/hse/builddir/subprojects'), PosixPath('/tmp/hse/builddir'), PosixPath('/tmp/hse'), PosixPath('/tmp'), PosixPath('/')]
in validate_within_subproject: self.subproject='mongo-c-driver', subdir='subprojects/mongo-c-driver-1.20.1', fname='/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src/bson/bson-config.h'
norm=PosixPath('/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src/bson/bson-config.h')
```

This aligns with https://github.com/mesonbuild/meson/blob/6096337904da94486195b0d96b3913024b17b9d6/mesonbuild/interpreter/interpreter.py#L2862-L2880

```
project_root == '/tmp/hse/subprojects/mongo-c-driver-1.20.1'
subproject_dir == '/tmp/hse/subprojects/mongo-c-driver-1.20.1/subprojects'

# the file being checked
norm == '/tmp/hse/builddir/subprojects/mongo-c-driver-1.20.1/__CMake_build/src/libbson/src/bson/bson-config.h'
```

We then trigger from this code:
```python
if project_root not in norm.parents:
    raise InterpreterException(f'Sandbox violation: Tried to grab {inputtype} {norm.name} outside current (sub)project.')
```
> subprojects/mongo-c-driver-1.20.1/meson.build:2364:0: ERROR: Sandbox violation: Tried to grab file bson-config.h outside current (sub)project.

Which happens because the project_root is the current subproject source root, and the file is outside of that directory. It is, in fact, in the build directory instead. 

We don't usually allow specifying sources as e.g. `srcfile = meson.current_build_dir() / 'foo.h'`, because in theory you're supposed to do this as `srcfile = configure_file(..., output: 'foo.h')`. The cmake subproject violates this assumption because it wraps around cmake's configure_file and all meson sees is the generated output.

Like I said:
> in this case probably grabbing a configured file and adding it as a files() object?
What do you suggest as the solution?
I'm not entirely sure, because I cherish my right as described in https://mesonbuild.com/Mixing-build-systems.html to not worry about what cmake subprojects get up to. :D

I'm hoping @mensinda has an idea.
I mean, what is the bug in the cmake subproject? I am not seeing anything. I am happy to carry a patch downstream assuming there is not bug in the cmake module.
Same problem with https://github.com/AppImage/AppImageUpdate as subproject. With meson 0.62.0 works fine.
@mensinda ping?
OK, strangely working workaround: Just move the build directory outside the project directory and it works because [this code path is triggered in this case](https://github.com/mesonbuild/meson/blob/master/mesonbuild/interpreter/interpreter.py#L2890).

Also, I don't see any way how to get rid of the absolute paths in cases like this, since those files are generated with CMakes `configure_file`. Additionally, we can't just remove those files from the sources (which would be possible in these specific cases) because you could also generate `*.c` / `*.cpp` files this way.

So, seeing that the currently the location of the build dir affects the logic of whether files can or cannot be added, I would say this is a bug in the core Meson interpreter (and - to my knowledge at least - can't be reasonably resolved/fixed by the CMake module). The question then is how do we deal with this situation:

Do we make it consistent by always allowing adding files from the build directory (obviously this is my favorite :) ). Or do we disallow it, which would be a breaking change to previous versions and would force the CMake module to hack around this issue by doing very sketchy stuff like copying the files from the build dir to a "temporary" directory in the source dir...
Ignoring whether or not this is an interpreter bug, could the CMake module detect a path like `/home/tpartin/Projects/hse/build-0.63/subprojects/mongo-c-driver-1.20.1/__CMake_build/...` and remove the absolute path prefix?
> Ignoring whether or not this is an interpreter bug, could the CMake module detect a path like /home/tpartin/Projects/hse/build-0.63/subprojects/mongo-c-driver-1.20.1/__CMake_build/... and remove the absolute path prefix?

Using relative paths wouldn't change a thing, because then you would have a relative path pointing into the build directory and the same error would occur. The main problem is that these files *only* exist in the build directory and the CMake module must point to them somehow.

If this were to be fixed in the interpreter, then the fix would be fairly trivial:

```diff
diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py
index e5b61ddf3..6ccc95ef3 100644
--- a/mesonbuild/interpreter/interpreter.py
+++ b/mesonbuild/interpreter/interpreter.py
@@ -2887,6 +2887,8 @@ Try setting b_lundef to false instead.'''.format(self.coredata.options[OptionKey
             inputtype = 'directory'
         else:
             inputtype = 'file'
+        if builddir in norm.parents:
+            return
         if srcdir not in norm.parents:
             # Grabbing files outside the source tree is ok.
             # This is for vendor stuff like:
```

I just want to make sure everyone here is OK with this fix before I go ahead and make a PR with that.

---

The only workaround I can do purely in the CMake module (which I really want to avoid) is to detect usages of files in the build directory and then copy them to the subprojects source directory. This would be a "solution" that would impact the **sources** of a project. I am pretty sure we have a policy (or at least I hope so) that forbids modifying the actual source directory during the build process.
I'd like to reiterate, this ticket is covered by Meson's policy on mixing multiple build systems. Particularly, that means:
> Nor will we make changes to support these use cases that would worsen the user experience of users of plain Meson.

IMO weakening the sandbox "because cmake" constitutes worsening the experience of users of plain Meson, because we cannot deliver on our sanity check guarantees.

Speaking of build directories that lie outside of the source directory, I thought we did check this. We even have an explicit warning for it when we relax that use case:

```python
                if not strict and s.startswith(self.environment.get_build_dir()):
                    results.append(s)
                    mlog.warning(f'Source item {s!r} cannot be converted to File object, because it is a generated file. '
                                 'This will become a hard error in the future.', location=self.current_node)
                else:
                    self.validate_within_subproject(self.subdir, s)
                    results.append(mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, s))
```

Maybe `File.from_source_file()` isn't good enough if you have an absolute path as a string? `strict=True` is the default, by the way, and this warning is only triggered when passing a string to custom_target as terrifyingly documented in https://mesonbuild.com/Vala.html#gobject-introspection-and-language-bindings, because meson vala support is buggy and doesn't return a proper target for .gir files.

We absolutely should be enforcing this more consistently regardless of where the build directory is, and the fact that we don't is a bug.


> Or do we disallow it, which would be a breaking change to previous versions


It's not a breaking change, since we officially don't allow it and even error / warn about it in the common case.

> and would force the CMake module to hack around this issue by doing very sketchy stuff like copying the files from the build dir to a "temporary" directory in the source dir...

> The only workaround I can do purely in the CMake module (which I really want to avoid) is to detect usages of files in the build directory and then copy them to the subprojects source directory. This would be a "solution" that would impact the **sources** of a project. I am pretty sure we have a policy (or at least I hope so) that forbids modifying the actual source directory during the build process.

I agree that's also bad. Is that really necessary though?

Can the cmake module somehow exempt itself from the sandbox / builddir checks, without affecting pure Meson projects?
> Can the cmake module somehow exempt itself from the sandbox / builddir checks, without affecting pure Meson projects?

I guess I could introduce a new flag variable in the interpreter to disable those checks while the CMake subproject is being processed, but this is also not really clean... In general the CMake module "just" converts the CMake stuff to a Meson AST and then completely hands control over to the regular Meson logic.

Alternatively, could we allow this behavior in general if the subproject violation is also not violated in the build dir (so accessing everything in `build/subprojects/mongo-c-driver-1.20.1` is fine, while everything else in the build directory is off limits)?
> I'd like to reiterate, this ticket is covered by Meson's policy on mixing multiple build systems.

You can reiterate something as much as you want. It doesn't make the situation any better from an end user perspective where something worked for a very long time, and now it doesn't. It's not like writing a Meson build for mongo-c-driver is an easy task either, but I am getting the feeling that that is what I'll have to do if no consensus can be reached on the best course of action here.
A consensus will be reached and this will get fixed.

The comments I make about ***how*** it should get fixed, don't affect ***whether*** it will get fixed.
@mensinda 

> I guess I could introduce a new flag variable in the interpreter to disable those checks while the CMake subproject is being processed, but this is also not really clean... In general the CMake module "just" converts the CMake stuff to a Meson AST and then completely hands control over to the regular Meson logic.

I mean, let's face it, the generated AST isn't exactly clean either. :D Absolute paths, any case where the AST is relying on only being run after cmake in order to obtain output files produced by cmake, these are both cases where I'm not 100% on board with saying "this is just regular Meson logic".

So I think it's not too bad in cases where there's no other choice AND the file in question is bridging cmake for configuration and meson for building, that we add exceptions.

> Alternatively, could we allow this behavior in general if the subproject violation is also not violated in the build dir (so accessing everything in `build/subprojects/mongo-c-driver-1.20.1` is fine, while everything else in the build directory is off limits)?

Again, I don't think so. We already emit a warning for specifically custom_target:
> Source item {s!r} cannot be converted to File object, because it is a generated file. This will become a hard error in the future.

We have 3 possible states when passing strings to the builddir:
- `files()` -- used to be overlooked, is now an error
- `custom_target()` -- raises a warning, but is permitted purely for the benefit of vala
- everything else -- always an error

Current Meson design is I think clear that we don't generally want people to do this, and the only use cases for it seem to be:
- cmake, because of the internal limitation of producing files that cmake creates
- vala, because vala support is messy and `shared_library()` produces output files that aren't in its return value

I don't want to generally allow it because the former is an implementation detail of the cmake module, and the latter is a bug to be fixed (`input: meson.current_build_dir() / 'Foo-1.0.gir',` for crying out loud).
@mensinda ping?

Resolving patch:
diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py
--- a/mesonbuild/interpreter/interpreter.py
+++ b/mesonbuild/interpreter/interpreter.py
@@ -92,6 +92,7 @@
 from . import primitives as P_OBJ
 
 from pathlib import Path
+from enum import Enum
 import os
 import shutil
 import uuid
@@ -240,6 +241,16 @@ def dump_value(self, arr, list_sep, indent):
     KwargInfo('verbose', bool, default=False, since='0.62.0'),
 ]
 
+class InterpreterRuleRelaxation(Enum):
+    ''' Defines specific relaxations of the Meson rules.
+
+    This is intended to be used for automatically converted
+    projects (CMake subprojects, build system mixing) that
+    generate a Meson AST via introspection, etc.
+    '''
+
+    ALLOW_BUILD_DIR_FILE_REFFERENCES = 1
+
 permitted_dependency_kwargs = {
     'allow_fallback',
     'cmake_args',
@@ -279,6 +290,7 @@ def __init__(
                 mock: bool = False,
                 ast: T.Optional[mparser.CodeBlockNode] = None,
                 is_translated: bool = False,
+                relaxations: T.Optional[T.Set[InterpreterRuleRelaxation]] = None,
                 user_defined_options: T.Optional['argparse.Namespace'] = None,
             ) -> None:
         super().__init__(_build.environment.get_source_dir(), subdir, subproject)
@@ -294,6 +306,7 @@ def __init__(
         self.subproject_directory_name = subdir.split(os.path.sep)[-1]
         self.subproject_dir = subproject_dir
         self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt')
+        self.relaxations = relaxations or set()
         if not mock and ast is None:
             self.load_root_meson_file()
             self.sanity_check_ast()
@@ -937,11 +950,13 @@ def _do_subproject_meson(self, subp_name: str, subdir: str,
                              kwargs: kwargs.DoSubproject,
                              ast: T.Optional[mparser.CodeBlockNode] = None,
                              build_def_files: T.Optional[T.List[str]] = None,
-                             is_translated: bool = False) -> SubprojectHolder:
+                             is_translated: bool = False,
+                             relaxations: T.Optional[T.Set[InterpreterRuleRelaxation]] = None) -> SubprojectHolder:
         with mlog.nested(subp_name):
             new_build = self.build.copy()
             subi = Interpreter(new_build, self.backend, subp_name, subdir, self.subproject_dir,
                                default_options, ast=ast, is_translated=is_translated,
+                               relaxations=relaxations,
                                user_defined_options=self.user_defined_options)
             # Those lists are shared by all interpreters. That means that
             # even if the subproject fails, any modification that the subproject
@@ -1016,7 +1031,15 @@ def _do_subproject_cmake(self, subp_name: str, subdir: str, subdir_abs: str,
                 mlog.cmd_ci_include(meson_filename)
                 mlog.log()
 
-            result = self._do_subproject_meson(subp_name, subdir, default_options, kwargs, ast, [str(f) for f in cm_int.bs_files], is_translated=True)
+            result = self._do_subproject_meson(
+                    subp_name, subdir, default_options,
+                    kwargs, ast,
+                    [str(f) for f in cm_int.bs_files],
+                    is_translated=True,
+                    relaxations={
+                        InterpreterRuleRelaxation.ALLOW_BUILD_DIR_FILE_REFFERENCES,
+                    }
+            )
             result.cm_interpreter = cm_int
 
         mlog.log()
@@ -2887,6 +2910,8 @@ def validate_installable_file(fpath: Path) -> bool:
             inputtype = 'directory'
         else:
             inputtype = 'file'
+        if InterpreterRuleRelaxation.ALLOW_BUILD_DIR_FILE_REFFERENCES in self.relaxations and builddir in norm.parents:
+            return
         if srcdir not in norm.parents:
             # Grabbing files outside the source tree is ok.
             # This is for vendor stuff like:


Test patch:
diff --git a/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt b/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt
--- a/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt	
+++ b/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt	
@@ -17,7 +17,7 @@ generate_export_header(cmModLib)
 
 set_target_properties(cmModLib PROPERTIES VERSION 1.0.1)
 
-add_executable(testEXE main.cpp)
+add_executable(testEXE main.cpp "${CMAKE_CURRENT_BINARY_DIR}/config.h")
 
 target_link_libraries(cmModLib       ZLIB::ZLIB)
 target_link_libraries(cmModLibStatic ;ZLIB::ZLIB;)