Repository: pulumi/pulumi-terraform-bridge
Pull request #1063 (2023-05-01 18:08:19)

[Muxer] Allow non disjoint augmented provider
This is necessary to build `pulumi-gcp` due to backported resources.

This PR is based on https://github.com/pulumi/pulumi-terraform-bridge/pull/1062, but will  merge separately.

github-actions[bot]: Diff for [pulumi-azuread](https://github.com/pulumi-bot/pulumi-azuread/compare/107610a..06169e6) with merge commit dec3fc0e33b1dc5dba9025a960aae66d96b63e24
github-actions[bot]: Diff for [pulumi-random](https://github.com/pulumi-bot/pulumi-random/compare/74e026a..32af30d) with merge commit dec3fc0e33b1dc5dba9025a960aae66d96b63e24
github-actions[bot]: Diff for [pulumi-random](https://github.com/pulumi-bot/pulumi-random/compare/5eba7ab..43f8cba) with merge commit b8441c2e72d13958457b04af3325e666127d5694
github-actions[bot]: Diff for [pulumi-azuread](https://github.com/pulumi-bot/pulumi-azuread/compare/ee6a4dd..c092588) with merge commit b8441c2e72d13958457b04af3325e666127d5694
github-actions[bot]: Diff for [pulumi-gcp](https://github.com/pulumi-bot/pulumi-gcp/compare/d9183eed0..71b3b393c) with merge commit dec3fc0e33b1dc5dba9025a960aae66d96b63e24
github-actions[bot]: Diff for [pulumi-gcp](https://github.com/pulumi-bot/pulumi-gcp/compare/ae5855d33..de22a068f) with merge commit b8441c2e72d13958457b04af3325e666127d5694
github-actions[bot]: Diff for [pulumi-azure](https://github.com/pulumi-bot/pulumi-azure/compare/784f78f9e..db086f1a0) with merge commit dec3fc0e33b1dc5dba9025a960aae66d96b63e24
github-actions[bot]: Diff for [pulumi-azure](https://github.com/pulumi-bot/pulumi-azure/compare/56ae97a01..f952fdae8) with merge commit b8441c2e72d13958457b04af3325e666127d5694

Patch:
diff --git a/pf/internal/muxer/muxer.go b/pf/internal/muxer/muxer.go
index 41f68826..c0a3c30c 100644
--- a/pf/internal/muxer/muxer.go
+++ b/pf/internal/muxer/muxer.go
@@ -15,6 +15,7 @@
 package muxer
 
 import (
+	"errors"
 	"fmt"
 	"sort"
 	"strings"
@@ -37,8 +38,16 @@ func SchemaOnlyPluginFrameworkProvider(ctx context.Context, provider provider.Pr
 }
 
 // AugmentShimWithPF augments an existing shim with a PF provider.Provider.
+//
+// If there is overlap between shim and pf, shim will dominate.
 func AugmentShimWithPF(ctx context.Context, shim shim.Provider, pf provider.Provider) *ProviderShim {
+	p, _, _ := augmentShimWithPF(ctx, shim, pf)
+	return p
+}
 
+func augmentShimWithPF(
+	ctx context.Context, shim shim.Provider, pf provider.Provider,
+) (*ProviderShim, []string, []string) {
 	var p ProviderShim
 	if alreadyMerged, ok := shim.(*ProviderShim); ok {
 		p = *alreadyMerged
@@ -46,9 +55,26 @@ func AugmentShimWithPF(ctx context.Context, shim shim.Provider, pf provider.Prov
 		p = newProviderShim(shim)
 	}
 
-	err := p.extend(schemashim.ShimSchemaOnlyProvider(ctx, pf))
-	contract.AssertNoErrorf(err, "shim and pf provider are not disjoint")
-	return &p
+	r, d := p.extend(schemashim.ShimSchemaOnlyProvider(ctx, pf))
+	return &p, r, d
+}
+
+// AugmentShimWithDisjointPF augments an existing shim with a PF provider.Provider.
+//
+// This function asserts that there is no overlap between providers.
+func AugmentShimWithDisjointPF(ctx context.Context, shim shim.Provider, pf provider.Provider) *ProviderShim {
+	p, resources, datasources := augmentShimWithPF(ctx, shim, pf)
+
+	var rErr, dErr error
+	if len(datasources) > 0 {
+		dErr = fmt.Errorf("DataSourceMap is not disjoint: conflicting keys: %s", strings.Join(datasources, ", "))
+	}
+	if len(resources) > 0 {
+		rErr = fmt.Errorf("ResourcesMap is not disjoint: conflicting keys: %s", strings.Join(resources, ", "))
+	}
+	contract.AssertNoErrorf(errors.Join(rErr, dErr), "providers are not disjoint")
+
+	return p
 }
 
 // A merged `shim.Provider` that remembers which `shim.Provider`s it is composed of.
@@ -91,21 +117,15 @@ func (m *ProviderShim) DataSourceIsPF(token string) bool {
 // Extend the `ProviderShim` with another `shim.Provider`.
 //
 // `provider` will be the `len(m.MuxedProviders)` when mappings are computed.
-func (m *ProviderShim) extend(provider shim.Provider) error {
-	res, err := disjointUnion(m.resources, provider.ResourcesMap())
-	if err != nil {
-		return fmt.Errorf("ResourcesMap is not disjoint: %w", err)
-	}
+func (m *ProviderShim) extend(provider shim.Provider) ([]string, []string) {
+	res, conflictingResources := union(m.resources, provider.ResourcesMap())
 
-	data, err := disjointUnion(m.dataSources, provider.DataSourcesMap())
-	if err != nil {
-		return fmt.Errorf("DataSourcesMap is not disjoint: %w", err)
-	}
+	data, conflictingDataSources := union(m.dataSources, provider.DataSourcesMap())
 
 	m.resources = res
 	m.dataSources = data
 	m.MuxedProviders = append(m.MuxedProviders, provider)
-	return nil
+	return conflictingResources, conflictingDataSources
 }
 
 func newProviderShim(provider shim.Provider) ProviderShim {
@@ -230,12 +250,9 @@ func (p *simpleSchemaProvider) DataSourcesMap() shim.ResourceMap {
 
 var _ shim.Provider = (*simpleSchemaProvider)(nil)
 
-func disjointUnion(baseline, extension shim.ResourceMap) (shim.ResourceMap, error) {
-	union, err := disjointMapUnion(toResourceMap(baseline), toResourceMap(extension))
-	if err != nil {
-		return nil, err
-	}
-	return shimSchema.ResourceMap(union), nil
+func union(baseline, extension shim.ResourceMap) (shim.ResourceMap, []string) {
+	union, conflictingKeys := mapUnion(toResourceMap(baseline), toResourceMap(extension))
+	return shimSchema.ResourceMap(union), conflictingKeys
 }
 
 func toResourceMap(rmap shim.ResourceMap) shimSchema.ResourceMap {
@@ -247,7 +264,7 @@ func toResourceMap(rmap shim.ResourceMap) shimSchema.ResourceMap {
 	return m
 }
 
-func disjointMapUnion[T any](baseline, extension map[string]T) (map[string]T, error) {
+func mapUnion[T any](baseline, extension map[string]T) (map[string]T, []string) {
 	u := copyMap(baseline)
 
 	var conflictingKeys []string
@@ -260,12 +277,9 @@ func disjointMapUnion[T any](baseline, extension map[string]T) (map[string]T, er
 		u[k] = v
 	}
 
-	if len(conflictingKeys) > 0 {
-		sort.Strings(conflictingKeys)
-		return nil, fmt.Errorf("conflicting keys: %s", strings.Join(conflictingKeys, ", "))
-	}
+	sort.Strings(conflictingKeys)
 
-	return u, nil
+	return u, conflictingKeys
 }
 
 func copyMap[K comparable, V any](m map[K]V) map[K]V {
diff --git a/pf/tfbridge/extend.go b/pf/tfbridge/extend.go
index b948eeeb..923ac854 100644
--- a/pf/tfbridge/extend.go
+++ b/pf/tfbridge/extend.go
@@ -29,7 +29,18 @@ func SchemaOnlyPluginFrameworkProvider(ctx context.Context, p pfprovider.Provide
 
 // MuxShimWithPF initializes a shim.Provider that will server resources from both shim and p.
 //
+// If shim and p both define the same token, then the value from shim will be used.
+//
 // To create a muxed provider, ProviderInfo.P must be the result of this function.
 func MuxShimWithPF(ctx context.Context, shim shim.Provider, p pfprovider.Provider) shim.Provider {
 	return muxer.AugmentShimWithPF(ctx, shim, p)
 }
+
+// MuxShimWithDisjointgPF initializes a shim.Provider that will server resources from both shim and p.
+//
+// This function will panic if shim and p both define the same token.
+//
+// To create a muxed provider, ProviderInfo.P must be the result of this function.
+func MuxShimWithDisjointgPF(ctx context.Context, shim shim.Provider, p pfprovider.Provider) shim.Provider {
+	return muxer.AugmentShimWithDisjointPF(ctx, shim, p)
+}
diff --git a/pf/tfbridge/main.go b/pf/tfbridge/main.go
index 472d21b6..0e33b102 100644
--- a/pf/tfbridge/main.go
+++ b/pf/tfbridge/main.go
@@ -101,15 +101,15 @@ func handleFlags(
 // Implements main() or a bridged Pulumi plugin, complete with argument parsing.
 //
 // This is an experimental API.
-func MainWithMuxer(ctx context.Context, schema []byte, info tfbridge.ProviderInfo) {
+func MainWithMuxer(ctx context.Context, pkg string, info tfbridge.ProviderInfo, schema []byte) {
 	handleFlags(ctx, info.Version, func() (*tfbridge.MarshallableProviderInfo, error) {
 		info := info
 		return tfbridge.MarshalProviderInfo(&info), nil
 	})
 
-	f := MakeMuxedServer(ctx, schema, info)
+	f := MakeMuxedServer(ctx, pkg, info, schema)
 
-	err := rprovider.Main(info.Name, f)
+	err := rprovider.Main(pkg, f)
 	if err != nil {
 		cmdutil.ExitError(err.Error())
 	}
@@ -120,7 +120,7 @@ func MainWithMuxer(ctx context.Context, schema []byte, info tfbridge.ProviderInf
 // This function exposes implementation details for testing. It should not be used outside
 // of pulumi-terraform-bridge.  This is an experimental API.
 func MakeMuxedServer(
-	ctx context.Context, schema []byte, info tfbridge.ProviderInfo,
+	ctx context.Context, pkg string, info tfbridge.ProviderInfo, schema []byte,
 ) func(host *rprovider.HostClient) (pulumirpc.ResourceProviderServer, error) {
 
 	shim, ok := info.P.(*pfmuxer.ProviderShim)
@@ -142,7 +142,7 @@ func MakeMuxedServer(
 		marshalled := tfbridge.MarshalProviderInfo(&info)
 		data, err := json.Marshal(marshalled)
 		return muxer.GetMappingResponse{
-			Provider: info.Name,
+			Provider: pkg,
 			Data:     data,
 		}, err
 	}
@@ -170,10 +170,10 @@ func MakeMuxedServer(
 			default:
 				m.Servers = append(m.Servers, muxer.Endpoint{
 					Server: func(host *rprovider.HostClient) (pulumirpc.ResourceProviderServer, error) {
-						return tfbridge.NewProvider(ctx, host, info.Name, version, prov, info, schema), nil
+						return tfbridge.NewProvider(ctx, host, pkg, version, prov, info, schema), nil
 					}})
 			}
 		}
-		return m.Server(host, info.Name, version)
+		return m.Server(host, pkg, version)
 	}
 }
diff --git a/pf/tfgen/main.go b/pf/tfgen/main.go
index ba99a294..cc4a1af7 100644
--- a/pf/tfgen/main.go
+++ b/pf/tfgen/main.go
@@ -73,11 +73,11 @@ func Main(provider string, info tfbridge.ProviderInfo) {
 // This is an experimental API.
 //
 // [Pulumi Package Schema]: https://www.pulumi.com/docs/guides/pulumi-packages/schema/
-func MainWithMuxer(info sdkBridge.ProviderInfo) {
+func MainWithMuxer(provider string, info sdkBridge.ProviderInfo) {
 	shim, ok := info.P.(*pfmuxer.ProviderShim)
 	contract.Assertf(ok, "MainWithMuxer must have a ProviderInfo.P created with AugmentShimWithPF")
 
-	tfgen.MainWithCustomGenerate(info.Name, info.Version, info, func(opts tfgen.GeneratorOptions) error {
+	tfgen.MainWithCustomGenerate(provider, info.Version, info, func(opts tfgen.GeneratorOptions) error {
 
 		if info.MetadataInfo == nil {
 			return fmt.Errorf("ProviderInfo.MetadataInfo is required and cannot be nil")


Test patch:
diff --git a/pf/tests/internal/testprovider/cmd/pulumi-resource-muxedrandom/main.go b/pf/tests/internal/testprovider/cmd/pulumi-resource-muxedrandom/main.go
index b1ef8767..e1fda80e 100644
--- a/pf/tests/internal/testprovider/cmd/pulumi-resource-muxedrandom/main.go
+++ b/pf/tests/internal/testprovider/cmd/pulumi-resource-muxedrandom/main.go
@@ -27,6 +27,6 @@ import (
 var schema []byte
 
 func main() {
-	tfbridge.MainWithMuxer(context.Background(), schema,
-		testprovider.MuxedRandomProvider())
+	tfbridge.MainWithMuxer(context.Background(), "muxedrandom",
+		testprovider.MuxedRandomProvider(), schema)
 }
diff --git a/pf/tests/internal/testprovider/cmd/pulumi-tfgen-muxedrandom/main.go b/pf/tests/internal/testprovider/cmd/pulumi-tfgen-muxedrandom/main.go
index 24575c6a..492372f7 100644
--- a/pf/tests/internal/testprovider/cmd/pulumi-tfgen-muxedrandom/main.go
+++ b/pf/tests/internal/testprovider/cmd/pulumi-tfgen-muxedrandom/main.go
@@ -22,5 +22,5 @@ import (
 )
 
 func main() {
-	tfgen.MainWithMuxer(testprovider.MuxedRandomProvider())
+	tfgen.MainWithMuxer("muxedrandom", testprovider.MuxedRandomProvider())
 }
diff --git a/pf/tests/provider_create_test.go b/pf/tests/provider_create_test.go
index 2152bf3a..0e59c869 100644
--- a/pf/tests/provider_create_test.go
+++ b/pf/tests/provider_create_test.go
@@ -174,3 +174,57 @@ func TestCreateWithFirstClassSecrets(t *testing.T) {
 	}`
 	testutils.Replay(t, server, testCase)
 }
+
+func TestCreateWithSchemaBasedSecrets(t *testing.T) {
+	// Ensure that resources that mark output properties as secret in the schema return them as secrets.
+	// RandomPassword is a good example. Surprisingly this test requires a Configure call first, otherwise the
+	// plubming is confused about secrets bits and retursn the wrong result. The test represents production use.
+	server := newProviderServer(t, testprovider.RandomProvider())
+	testCase := `
+	[
+	  {
+	    "method": "/pulumirpc.ResourceProvider/Configure",
+	    "request": {
+	      "args": {},
+	      "acceptSecrets": true,
+	      "acceptResources": true
+	    },
+	    "response": "*"
+	  },
+	  {
+	    "method": "/pulumirpc.ResourceProvider/Create",
+	    "request": {
+	      "urn": "urn:pulumi:dev::secret-random-yaml::random:index/randomPassword:RandomPassword::param",
+	      "properties": {
+		"length": 10
+	      }
+	    },
+	    "response": {
+	      "id": "none",
+	      "properties": {
+		"__meta": "{\"schema_version\":\"3\"}",
+		"bcryptHash": {
+		  "4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
+		  "value": "*"
+		},
+		"id": "none",
+		"length": 10,
+		"lower": true,
+		"minLower": 0,
+		"minNumeric": 0,
+		"minSpecial": 0,
+		"minUpper": 0,
+		"number": true,
+		"numeric": true,
+		"result": {
+		  "4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
+		  "value": "*"
+		},
+		"special": true,
+		"upper": true
+	      }
+	    }
+	  }
+	]`
+	testutils.ReplaySequence(t, server, testCase)
+}
diff --git a/pf/tests/provider_get_mapping_test.go b/pf/tests/provider_get_mapping_test.go
index 19b9e510..2442e15f 100644
--- a/pf/tests/provider_get_mapping_test.go
+++ b/pf/tests/provider_get_mapping_test.go
@@ -64,7 +64,7 @@ func TestMuxedGetMapping(t *testing.T) {
 
 	info := testprovider.MuxedRandomProvider()
 
-	server, err := tfbridge.MakeMuxedServer(ctx, genSDKSchema(t, info), info)(nil)
+	server, err := tfbridge.MakeMuxedServer(ctx, "muxedrandom", info, genSDKSchema(t, info))(nil)
 	require.NoError(t, err)
 
 	req := func(key string) (context.Context, *pulumirpc.GetMappingRequest) {
diff --git a/pf/tests/util.go b/pf/tests/util.go
index 28164bdb..4259ec65 100644
--- a/pf/tests/util.go
+++ b/pf/tests/util.go
@@ -38,7 +38,7 @@ func newProviderServer(t *testing.T, info tfbridge.ProviderInfo) pulumirpc.Resou
 func newMuxedProviderServer(t *testing.T, info tfbridge0.ProviderInfo) pulumirpc.ResourceProviderServer {
 	ctx := context.Background()
 	meta := genSDKSchema(t, info)
-	p, err := tfbridge.MakeMuxedServer(ctx, meta, info)(nil)
+	p, err := tfbridge.MakeMuxedServer(ctx, info.Name, info, meta)(nil)
 	require.NoError(t, err)
 	return p
 }