Repository: pulumi/pulumi
Pull request #19500: [convert] Copy assets emitted from converter plugins to target directory of the generated project
Author: login: Zaid-Ajaj
Created: 2025-05-09T16:26:23Z | Merged: 2025-05-13T20:25:19Z

### Description

When `pulumi convert` talks to converter plugins, it tells them where they should emit PCL files from which we bind a program and convert it. If the converters happen to emit non-PCL files such as assets of the program, we just ignore them. This PR addresses the issue by copying non-PCL files from the source program directory (where PCL files were emitted) into the target generated project directory so that the assets stay part of the project. 

This was implemented for conformance tests (specifically for the `l2-resource-asset-archive` test) but not for `pulumi convert` so here we implement it. 

This might be used to support using local TF modules via `pulumi-converter-terraform` where we could copy the contents of the modules into the target project directory. 

### Implementation

Extending `Copy` and `CopyDir` with a filter function `func(afero.File) bool` to become `CopyWhen` and `CopyDirWhen`, then 
 - implementing `Copy` in terms of `CopyWhen` with a filter that always passes
 - implementing `CopyDir` in terms of `CopyDirWhen` with a filter that always passes


Diff:
diff --git a/changelog/pending/20250509--programgen--copy-assets-emitted-from-converter-plugins-to-target-directory-of-the-generated-projects.yaml b/changelog/pending/20250509--programgen--copy-assets-emitted-from-converter-plugins-to-target-directory-of-the-generated-projects.yaml
new file mode 100644
index 000000000000..2e8524ca658a
--- /dev/null
+++ b/changelog/pending/20250509--programgen--copy-assets-emitted-from-converter-plugins-to-target-directory-of-the-generated-projects.yaml
@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: programgen
+  description: Copy assets emitted from converter plugins to target directory of the generated projects
diff --git a/pkg/cmd/pulumi/convert/convert.go b/pkg/cmd/pulumi/convert/convert.go
index 02a1b373911d..26c36834f34f 100644
--- a/pkg/cmd/pulumi/convert/convert.go
+++ b/pkg/cmd/pulumi/convert/convert.go
@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -152,7 +153,7 @@ func pclGenerateProject(
 ) (hcl.Diagnostics, error) {
 	_, diagnostics, bindErr := safePclBindDirectory(sourceDirectory, loader, strict)
 	// We always try to copy the source directory to the target directory even if binding failed
-	copyErr := aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory)
+	copyErr := aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory, nil)
 	// And then we return the combined diagnostics and errors
 	var err error
 	if bindErr != nil || copyErr != nil {
@@ -255,6 +256,32 @@ func runConvert(
 				return nil, err
 			}
 
+			// copy all non-PCL files from the source directory to the target directory
+			// such that any assets are copied over, excluding the Pulumi.yaml project file
+			err = aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory,
+				func(file os.FileInfo) bool {
+					if file.IsDir() {
+						sourceAbsPath, err := filepath.Abs(filepath.Join(sourceDirectory, file.Name()))
+						if err != nil {
+							return false
+						}
+
+						targetAbsPath, err := filepath.Abs(targetDirectory)
+						if err != nil {
+							return false
+						}
+						// if the target directory is a subdirectory of the source directory,
+						// skip copying it over
+						return sourceAbsPath != targetAbsPath
+					}
+
+					return file.Name() != "Pulumi.yaml" &&
+						path.Ext(file.Name()) != ".pp"
+				})
+			if err != nil {
+				return nil, fmt.Errorf("copying files from source directory: %w", err)
+			}
+
 			packageBlockDescriptors, ds, err := getPackagesToGenerateSdks(sourceDirectory)
 			diags = append(diags, ds...)
 			if err != nil {
diff --git a/pkg/util/afero/afero.go b/pkg/util/afero/afero.go
index 66078d00cbb1..c64e839e0dbc 100644
--- a/pkg/util/afero/afero.go
+++ b/pkg/util/afero/afero.go
@@ -22,8 +22,8 @@ import (
 	"github.com/spf13/afero"
 )
 
-// Copies all file and directories from src to dst
-func CopyDir(fs afero.Fs, src, dst string) error {
+// Copies all file and directories from src to dst that match the filter
+func CopyDir(fs afero.Fs, src, dst string, filter func(os.FileInfo) bool) error {
 	entries, err := afero.ReadDir(fs, src)
 	if err != nil {
 		return err
@@ -36,8 +36,12 @@ func CopyDir(fs afero.Fs, src, dst string) error {
 		sourcePath := filepath.Join(src, entry.Name())
 		destPath := filepath.Join(dst, entry.Name())
 
+		if filter != nil && !filter(entry) {
+			continue
+		}
+
 		if entry.IsDir() {
-			if err := CopyDir(fs, sourcePath, destPath); err != nil {
+			if err := CopyDir(fs, sourcePath, destPath, filter); err != nil {
 				return err
 			}
 		} else {
diff --git a/pkg/util/afero/afero_test.go b/pkg/util/afero/afero_test.go
new file mode 100644
index 000000000000..fff1d8fac789
--- /dev/null
+++ b/pkg/util/afero/afero_test.go
@@ -0,0 +1,54 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package afero
+
+import (
+	"os"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCopyDirWorksWithFilters(t *testing.T) {
+	t.Parallel()
+	content := []byte("hello world")
+	fs := afero.NewMemMapFs()
+	err := afero.WriteFile(fs, "/src/file.txt", content, 0o644)
+	assert.NoError(t, err)
+	// filter always returns false so nothing is copied
+	err = CopyDir(fs, "/src", "/dst", func(f os.FileInfo) bool {
+		return false
+	})
+
+	assert.NoError(t, err)
+	// assert that the file was not copied and thus should not exist
+	_, err = fs.Open("/dst/file.txt")
+	assert.True(t, os.IsNotExist(err))
+
+	// now copy the file with a filter that always returns true
+	err = CopyDir(fs, "/src", "/dst", func(f os.FileInfo) bool {
+		return true
+	})
+
+	assert.NoError(t, err)
+	// check that the file was copied
+	file, err := fs.Open("/dst/file.txt")
+	assert.NoError(t, err)
+	defer file.Close()
+	fileContent, err := afero.ReadAll(file)
+	assert.NoError(t, err)
+	assert.Equal(t, string(content), string(fileContent))
+}


Reviews:
user: login: Frassle
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-09T19:53:24Z
user: login: Zaid-Ajaj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-10T16:37:42Z
user: login: Frassle
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T16:08:53Z
user: login: Zaid-Ajaj
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T16:33:04Z
user: login: Frassle
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-13T17:27:14Z

Review comments:
diff_hunk: @@ -37,31 +37,36 @@ func CopyDir(fs afero.Fs, src, dst string) error {
 		destPath := filepath.Join(dst, entry.Name())
 
 		if entry.IsDir() {
-			if err := CopyDir(fs, sourcePath, destPath); err != nil {
+			if err := CopyDirWhen(fs, sourcePath, destPath, filter); err != nil {
 				return err
 			}
 		} else {
-			if err := Copy(fs, sourcePath, destPath); err != nil {
+			if err := CopyWhen(fs, sourcePath, destPath, filter); err != nil {
 				return err
 			}
 		}
 	}
 	return nil
 }
 
-// Copies a file from src to dst
-func Copy(fs afero.Fs, src, dst string) error {
-	out, err := fs.Create(dst)
+// Copies a file from src to dst if the filter returns true
+func CopyWhen(fs afero.Fs, src, dst string, filter func(afero.File) bool) error {
path: pkg/util/afero/afero.go
user: login: Frassle
body: This feels a bit odd to me. Like it doesn't feel like filtering should be a concern of this function because it's only operating on a single file path, if you want to filter just don't call Copy.
created_at: 2025-05-09T19:52:44Z
author_association: MEMBER
diff_hunk: @@ -22,8 +22,8 @@ import (
 	"github.com/spf13/afero"
 )
 
-// Copies all file and directories from src to dst
-func CopyDir(fs afero.Fs, src, dst string) error {
+// Copies all file and directories from src to dst that match the filter
+func CopyDirWhen(fs afero.Fs, src, dst string, filter func(afero.File) bool) error {
path: pkg/util/afero/afero.go
user: login: Frassle
body: Does anything else use CopyDir outside pu/pu? Might be ok to just add filter to the CopyDir function and treat `nil` the same as `const true`
created_at: 2025-05-09T19:53:18Z
author_association: MEMBER
diff_hunk: @@ -37,31 +37,36 @@ func CopyDir(fs afero.Fs, src, dst string) error {
 		destPath := filepath.Join(dst, entry.Name())
 
 		if entry.IsDir() {
-			if err := CopyDir(fs, sourcePath, destPath); err != nil {
+			if err := CopyDirWhen(fs, sourcePath, destPath, filter); err != nil {
 				return err
 			}
 		} else {
-			if err := Copy(fs, sourcePath, destPath); err != nil {
+			if err := CopyWhen(fs, sourcePath, destPath, filter); err != nil {
 				return err
 			}
 		}
 	}
 	return nil
 }
 
-// Copies a file from src to dst
-func Copy(fs afero.Fs, src, dst string) error {
-	out, err := fs.Create(dst)
+// Copies a file from src to dst if the filter returns true
+func CopyWhen(fs afero.Fs, src, dst string, filter func(afero.File) bool) error {
path: pkg/util/afero/afero.go
user: login: Zaid-Ajaj
body: > if you want to filter just don't call Copy.

You are right, addressed it now such that `Copy` isn't changed and `CopyDir` takes the filter function ✅ 
created_at: 2025-05-10T16:37:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -152,7 +153,9 @@ func pclGenerateProject(
 ) (hcl.Diagnostics, error) {
 	_, diagnostics, bindErr := safePclBindDirectory(sourceDirectory, loader, strict)
 	// We always try to copy the source directory to the target directory even if binding failed
-	copyErr := aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory)
+	copyErr := aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory, func(fi os.FileInfo) bool {
path: pkg/cmd/pulumi/convert/convert.go
user: login: Frassle
body: Last arg can just be nil
created_at: 2025-05-13T16:08:35Z
author_association: MEMBER
diff_hunk: @@ -152,7 +153,9 @@ func pclGenerateProject(
 ) (hcl.Diagnostics, error) {
 	_, diagnostics, bindErr := safePclBindDirectory(sourceDirectory, loader, strict)
 	// We always try to copy the source directory to the target directory even if binding failed
-	copyErr := aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory)
+	copyErr := aferoUtil.CopyDir(afero.NewOsFs(), sourceDirectory, targetDirectory, func(fi os.FileInfo) bool {
path: pkg/cmd/pulumi/convert/convert.go
user: login: Zaid-Ajaj
body: Yeah changed it, I am always used to passing actual functions xD
created_at: 2025-05-13T16:33:04Z
author_association: CONTRIBUTOR

PR comments:
user: login: Zaid-Ajaj
created_at: 2025-05-09T16:52:10Z
body: While a lint check failed, also saw this failure in CI 
```
 integration_go_test.go:1420: STDERR: Converting from terraform...
        Converting to go...
        error: failed to load language plugin terraform: no language plugin 'pulumi-language-terraform' found in the workspace or on your $PATH
    integration_go_test.go:1420: Ran command pulumi args [convert --from terraform --language go --out godir] and expected success. Instead got failure.
```

author_association: CONTRIBUTOR
user: login: pulumi-bot
created_at: 2025-05-15T14:59:59Z
body: This PR has been shipped in [release v3.170.0](https://github.com/pulumi/pulumi/releases/tag/v3.170.0).
author_association: CONTRIBUTOR