Repository: mesonbuild/meson

Issue:
Allow passing GeneratedList to generator.process
Sometimes you want to chain two generators one after the other; for instance while building assembly with MSVC, you need to use `ml.exe` but it doesn't do pre-processing, so you need to call `cl.exe /EP /P /Fi@OUTPUT@ @INPUT@` to do the pre-processing, and then pass it to `ml.exe /Fo @OUTPUT@ /c @INPUT@` to get compiled object code which can be added as sources to an `executable` or `library`.

This is not trivial to do (I tried) but shouldn't be too difficult. Also implementing it on the Visual Studio backend has the same problem that is causing the `113 generatorcustom` test to fail there.

This is needed for building libffi with Meson without resorting to a wrapper script.


Discussion:
File objects too - if generator is defined in submodule - paths are relative to that submodule.

`bin2c.process('../../hello.zip')` - works

`bin2c.process(files('hello.zip'))` - syntax error

Any updates on this? Wrapper scripts are not trivial to write when e.g. subproject executables are involved in a compilation chain.
I am finding the exact same use case as described by this issue.
Same here. I needed to preprocess some ASM files which include C headers, then pass the generated ASM sources to the ASM generator. Now it doesn't work. Could be done with custom_targets but not clean.
Same here. I'm using gnu m4 and  [lemon](https://en.wikipedia.org/wiki/Lemon_(parser_generator)). Unfortunately

`lemongen.process(m4gen.process('myfile'))`

fails with

`ERROR: Generator arguments must be strings or files not <GeneratedListHolder: ['myfile']>.`

I stumbled upon this issue while trying to consume the output of `gperf` as input for `re2c` before finally using the generated sources to build an executable.
What I wanted to do:

```
gperf_gen.process(re2c_gen.process(re2c_src))
```

What I had to do thanks to Meson's limitation regarding non-chainable generators:

```
pp_state = custom_target(
    'bc2_state.cpp',
    output : 'bc2_state.cpp',
    input : re2c_gen.process('src/state.ypp'),
    command : [
        emilua_gperf,
        '--assign', 'GPERF_BIN=' + find_program('gperf').full_path(),
        '@INPUT@', '@OUTPUT@',
    ],
)
```

Full example: https://gitlab.com/emilua/emilua/-/blob/63683c851b1701ac3827f183624f13c341d03156/meson.build

Resolving patch:
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -2509,6 +2509,9 @@ def replace_paths(self, target, args, override_subdir=None):
         return args
 
     def generate_genlist_for_target(self, genlist: build.GeneratedList, target: build.BuildTarget) -> None:
+        for x in genlist.depends:
+            if isinstance(x, build.GeneratedList):
+                self.generate_genlist_for_target(x, target)
         generator = genlist.get_generator()
         subdir = genlist.subdir
         exe = generator.get_exe()
@@ -2520,7 +2523,7 @@ def generate_genlist_for_target(self, genlist: build.GeneratedList, target: buil
                 sole_output = os.path.join(self.get_target_private_dir(target), outfilelist[i])
             else:
                 sole_output = f'{curfile}'
-            infilename = curfile.rel_to_builddir(self.build_to_src)
+            infilename = curfile.rel_to_builddir(self.build_to_src, self.get_target_private_dir(target))
             base_args = generator.get_arglist(infilename)
             outfiles = genlist.get_outputs_for(curfile)
             outfiles = [os.path.join(self.get_target_private_dir(target), of) for of in outfiles]
diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py
--- a/mesonbuild/backend/vs2010backend.py
+++ b/mesonbuild/backend/vs2010backend.py
@@ -118,67 +118,74 @@ def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional[Inter
     def get_target_private_dir(self, target):
         return os.path.join(self.get_target_dir(target), target.get_id())
 
+    def generate_genlist_for_target(self, genlist: T.Union[build.GeneratedList, build.CustomTarget, build.CustomTargetIndex], target: build.BuildTarget, parent_node: ET.Element, generator_output_files: T.List[str], custom_target_include_dirs: T.List[str], custom_target_output_files: T.List[str]) -> None:
+        if isinstance(genlist, build.GeneratedList):
+            for x in genlist.depends:
+                self.generate_genlist_for_target(x, target, parent_node, [], [], [])
+        target_private_dir = self.relpath(self.get_target_private_dir(target), self.get_target_dir(target))
+        down = self.target_to_build_root(target)
+        if isinstance(genlist, (build.CustomTarget, build.CustomTargetIndex)):
+            for i in genlist.get_outputs():
+                # Path to the generated source from the current vcxproj dir via the build root
+                ipath = os.path.join(down, self.get_target_dir(genlist), i)
+                custom_target_output_files.append(ipath)
+            idir = self.relpath(self.get_target_dir(genlist), self.get_target_dir(target))
+            if idir not in custom_target_include_dirs:
+                custom_target_include_dirs.append(idir)
+        else:
+            generator = genlist.get_generator()
+            exe = generator.get_exe()
+            infilelist = genlist.get_inputs()
+            outfilelist = genlist.get_outputs()
+            source_dir = os.path.join(down, self.build_to_src, genlist.subdir)
+            idgroup = ET.SubElement(parent_node, 'ItemGroup')
+            samelen = len(infilelist) == len(outfilelist)
+            for i, curfile in enumerate(infilelist):
+                if samelen:
+                    sole_output = os.path.join(target_private_dir, outfilelist[i])
+                else:
+                    sole_output = ''
+                infilename = os.path.join(down, curfile.rel_to_builddir(self.build_to_src, target_private_dir))
+                deps = self.get_custom_target_depend_files(genlist, True)
+                base_args = generator.get_arglist(infilename)
+                outfiles_rel = genlist.get_outputs_for(curfile)
+                outfiles = [os.path.join(target_private_dir, of) for of in outfiles_rel]
+                generator_output_files += outfiles
+                args = [x.replace("@INPUT@", infilename).replace('@OUTPUT@', sole_output)
+                        for x in base_args]
+                args = self.replace_outputs(args, target_private_dir, outfiles_rel)
+                args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir())
+                        .replace("@BUILD_DIR@", target_private_dir)
+                        for x in args]
+                args = [x.replace("@CURRENT_SOURCE_DIR@", source_dir) for x in args]
+                args = [x.replace("@SOURCE_ROOT@", self.environment.get_source_dir())
+                        .replace("@BUILD_ROOT@", self.environment.get_build_dir())
+                        for x in args]
+                args = [x.replace('\\', '/') for x in args]
+                # Always use a wrapper because MSBuild eats random characters when
+                # there are many arguments.
+                tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+                cmd, _ = self.as_meson_exe_cmdline(
+                    exe,
+                    self.replace_extra_args(args, genlist),
+                    workdir=tdir_abs,
+                    capture=outfiles[0] if generator.capture else None,
+                    force_serialize=True
+                )
+                deps = cmd[-1:] + deps
+                abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+                os.makedirs(abs_pdir, exist_ok=True)
+                cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename)
+                ET.SubElement(cbs, 'Command').text = ' '.join(self.quote_arguments(cmd))
+                ET.SubElement(cbs, 'Outputs').text = ';'.join(outfiles)
+                ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps)
+
     def generate_custom_generator_commands(self, target, parent_node):
         generator_output_files = []
         custom_target_include_dirs = []
         custom_target_output_files = []
-        target_private_dir = self.relpath(self.get_target_private_dir(target), self.get_target_dir(target))
-        down = self.target_to_build_root(target)
         for genlist in target.get_generated_sources():
-            if isinstance(genlist, (build.CustomTarget, build.CustomTargetIndex)):
-                for i in genlist.get_outputs():
-                    # Path to the generated source from the current vcxproj dir via the build root
-                    ipath = os.path.join(down, self.get_target_dir(genlist), i)
-                    custom_target_output_files.append(ipath)
-                idir = self.relpath(self.get_target_dir(genlist), self.get_target_dir(target))
-                if idir not in custom_target_include_dirs:
-                    custom_target_include_dirs.append(idir)
-            else:
-                generator = genlist.get_generator()
-                exe = generator.get_exe()
-                infilelist = genlist.get_inputs()
-                outfilelist = genlist.get_outputs()
-                source_dir = os.path.join(down, self.build_to_src, genlist.subdir)
-                idgroup = ET.SubElement(parent_node, 'ItemGroup')
-                for i, curfile in enumerate(infilelist):
-                    if len(infilelist) == len(outfilelist):
-                        sole_output = os.path.join(target_private_dir, outfilelist[i])
-                    else:
-                        sole_output = ''
-                    infilename = os.path.join(down, curfile.rel_to_builddir(self.build_to_src))
-                    deps = self.get_custom_target_depend_files(genlist, True)
-                    base_args = generator.get_arglist(infilename)
-                    outfiles_rel = genlist.get_outputs_for(curfile)
-                    outfiles = [os.path.join(target_private_dir, of) for of in outfiles_rel]
-                    generator_output_files += outfiles
-                    args = [x.replace("@INPUT@", infilename).replace('@OUTPUT@', sole_output)
-                            for x in base_args]
-                    args = self.replace_outputs(args, target_private_dir, outfiles_rel)
-                    args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir())
-                             .replace("@BUILD_DIR@", target_private_dir)
-                            for x in args]
-                    args = [x.replace("@CURRENT_SOURCE_DIR@", source_dir) for x in args]
-                    args = [x.replace("@SOURCE_ROOT@", self.environment.get_source_dir())
-                             .replace("@BUILD_ROOT@", self.environment.get_build_dir())
-                            for x in args]
-                    args = [x.replace('\\', '/') for x in args]
-                    # Always use a wrapper because MSBuild eats random characters when
-                    # there are many arguments.
-                    tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
-                    cmd, _ = self.as_meson_exe_cmdline(
-                        exe,
-                        self.replace_extra_args(args, genlist),
-                        workdir=tdir_abs,
-                        capture=outfiles[0] if generator.capture else None,
-                        force_serialize=True
-                    )
-                    deps = cmd[-1:] + deps
-                    abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
-                    os.makedirs(abs_pdir, exist_ok=True)
-                    cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename)
-                    ET.SubElement(cbs, 'Command').text = ' '.join(self.quote_arguments(cmd))
-                    ET.SubElement(cbs, 'Outputs').text = ';'.join(outfiles)
-                    ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps)
+            self.generate_genlist_for_target(genlist, target, parent_node, generator_output_files, custom_target_include_dirs, custom_target_output_files)
         return generator_output_files, custom_target_output_files, custom_target_include_dirs
 
     def generate(self):
diff --git a/mesonbuild/build.py b/mesonbuild/build.py
--- a/mesonbuild/build.py
+++ b/mesonbuild/build.py
@@ -1668,6 +1668,34 @@ def check_module_linking(self):
                                      'use shared_library() with `override_options: [\'b_lundef=false\']` instead.')
                     link_target.force_soname = True
 
+class FileInTargetPrivateDir:
+    """Represents a file with the path '/path/to/build/target_private_dir/fname'.
+       target_private_dir is the return value of get_target_private_dir which is e.g. 'subdir/target.p'.
+    """
+
+    def __init__(self, fname: str):
+        self.fname = fname
+
+class FileMaybeInTargetPrivateDir:
+    """Union between 'File' and 'FileInTargetPrivateDir'"""
+
+    def __init__(self, inner: T.Union[File, FileInTargetPrivateDir]):
+        self.inner = inner
+
+    @property
+    def fname(self) -> str:
+        return self.inner.fname
+
+    def rel_to_builddir(self, build_to_src: str, target_private_dir: str) -> str:
+        if isinstance(self.inner, FileInTargetPrivateDir):
+            return os.path.join(target_private_dir, self.inner.fname)
+        return self.inner.rel_to_builddir(build_to_src)
+
+    def absolute_path(self, srcdir: str, builddir: str) -> str:
+        if isinstance(self.inner, FileInTargetPrivateDir):
+            raise RuntimeError('Unreachable code')
+        return self.inner.absolute_path(srcdir, builddir)
+
 class Generator(HoldableObject):
     def __init__(self, exe: T.Union['Executable', programs.ExternalProgram],
                  arguments: T.List[str],
@@ -1727,10 +1755,14 @@ def process_files(self, files: T.Iterable[T.Union[str, File, 'CustomTarget', 'Cu
                 output.depends.add(e)
             if isinstance(e, CustomTargetIndex):
                 output.depends.add(e.target)
-
-            if isinstance(e, (CustomTarget, CustomTargetIndex, GeneratedList)):
+            if isinstance(e, (CustomTarget, CustomTargetIndex)):
                 output.depends.add(e)
                 fs = [File.from_built_file(state.subdir, f) for f in e.get_outputs()]
+            elif isinstance(e, GeneratedList):
+                if preserve_path_from:
+                    raise InvalidArguments("generator.process: 'preserve_path_from' is not allowed if one input is a 'generated_list'.")
+                output.depends.add(e)
+                fs = [FileInTargetPrivateDir(f) for f in e.get_outputs()]
             elif isinstance(e, str):
                 fs = [File.from_source_file(state.environment.source_dir, state.subdir, e)]
             else:
@@ -1741,6 +1773,7 @@ def process_files(self, files: T.Iterable[T.Union[str, File, 'CustomTarget', 'Cu
                     abs_f = f.absolute_path(state.environment.source_dir, state.environment.build_dir)
                     if not self.is_parent_path(preserve_path_from, abs_f):
                         raise InvalidArguments('generator.process: When using preserve_path_from, all input files must be in a subdirectory of the given dir.')
+                f = FileMaybeInTargetPrivateDir(f)
                 output.add_file(f, state)
         return output
 
@@ -1758,9 +1791,9 @@ class GeneratedList(HoldableObject):
     def __post_init__(self) -> None:
         self.name = self.generator.exe
         self.depends: T.Set[GeneratedTypes] = set()
-        self.infilelist: T.List['File'] = []
+        self.infilelist: T.List[FileMaybeInTargetPrivateDir] = []
         self.outfilelist: T.List[str] = []
-        self.outmap: T.Dict[File, T.List[str]] = {}
+        self.outmap: T.Dict[FileMaybeInTargetPrivateDir, T.List[str]] = {}
         self.extra_depends = []  # XXX: Doesn't seem to be used?
         self.depend_files: T.List[File] = []
 
@@ -1776,7 +1809,7 @@ def __post_init__(self) -> None:
                 # know the absolute path of
                 self.depend_files.append(File.from_absolute_file(path))
 
-    def add_preserved_path_segment(self, infile: File, outfiles: T.List[str], state: T.Union['Interpreter', 'ModuleState']) -> T.List[str]:
+    def add_preserved_path_segment(self, infile: FileMaybeInTargetPrivateDir, outfiles: T.List[str], state: T.Union['Interpreter', 'ModuleState']) -> T.List[str]:
         result: T.List[str] = []
         in_abs = infile.absolute_path(state.environment.source_dir, state.environment.build_dir)
         assert os.path.isabs(self.preserve_path_from)
@@ -1786,7 +1819,7 @@ def add_preserved_path_segment(self, infile: File, outfiles: T.List[str], state:
             result.append(os.path.join(path_segment, of))
         return result
 
-    def add_file(self, newfile: File, state: T.Union['Interpreter', 'ModuleState']) -> None:
+    def add_file(self, newfile: FileMaybeInTargetPrivateDir, state: T.Union['Interpreter', 'ModuleState']) -> None:
         self.infilelist.append(newfile)
         outfiles = self.generator.get_base_outnames(newfile.fname)
         if self.preserve_path_from:
@@ -1794,13 +1827,13 @@ def add_file(self, newfile: File, state: T.Union['Interpreter', 'ModuleState'])
         self.outfilelist += outfiles
         self.outmap[newfile] = outfiles
 
-    def get_inputs(self) -> T.List['File']:
+    def get_inputs(self) -> T.List[FileMaybeInTargetPrivateDir]:
         return self.infilelist
 
     def get_outputs(self) -> T.List[str]:
         return self.outfilelist
 
-    def get_outputs_for(self, filename: 'File') -> T.List[str]:
+    def get_outputs_for(self, filename: FileMaybeInTargetPrivateDir) -> T.List[str]:
         return self.outmap[filename]
 
     def get_generator(self) -> 'Generator':


Test patch:
diff --git a/test cases/common/262 generator chain/data.txt b/test cases/common/262 generator chain/data.txt
new file mode 100644
--- /dev/null
+++ b/test cases/common/262 generator chain/data.txt	
@@ -0,0 +1 @@
+stage1
diff --git a/test cases/common/262 generator chain/meson.build b/test cases/common/262 generator chain/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/common/262 generator chain/meson.build	
@@ -0,0 +1,19 @@
+project('Generator Chain', 'c')
+
+stage1_exe = find_program('stage1.py')
+
+stage2_exe = find_program('stage2.py')
+
+stage1_gen = generator(stage1_exe,
+  output : '@PLAINNAME@.inter',
+  arguments : ['@INPUT@', '@OUTPUT@'])
+
+stage2_gen = generator(stage2_exe,
+  output : '@PLAINNAME@.c',
+  arguments : ['@INPUT@', '@OUTPUT@'])
+
+out = stage2_gen.process(stage1_gen.process('data.txt'))
+
+hello = executable('hello', out)
+
+test('basic', hello)
diff --git a/test cases/common/262 generator chain/stage1.py b/test cases/common/262 generator chain/stage1.py
new file mode 100644
--- /dev/null
+++ b/test cases/common/262 generator chain/stage1.py	
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+assert(Path(sys.argv[1]).read_text() == 'stage1\n')
+Path(sys.argv[2]).write_text('stage2\n')
diff --git a/test cases/common/262 generator chain/stage2.py b/test cases/common/262 generator chain/stage2.py
new file mode 100644
--- /dev/null
+++ b/test cases/common/262 generator chain/stage2.py	
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+assert(Path(sys.argv[1]).read_text() == 'stage2\n')
+Path(sys.argv[2]).write_text('int main(void){}\n')