Repository: bufbuild/buf
Pull request #1467 (2022-10-05 15:34:07)

update buf beta convert 
- Clarify that `<source>` is actually `<input>` 
- Adds ability to specify input format just like every other buf command: `buf convert -#format=json`
- changes current `input` flag to `payload` flag
- Adds more tests around `buf beta convert` 



Going back to the questions raised on #1432


- buf convert --input tmp.bin - this would be parsed as a binary file
  - buf convert --payload tmp.bin ✅ 
- buf convert --input tmp.json - this would be parsed as a JSON file ✅ 
  - buf convert --payload tmp.json ✅
- buf convert --input - - this would default to binary ✅ 
  - buf convert --payload - ✅
- buf convert --input -#format=json - this would be parsed as JSON ✅ 
  - buf convert --payload -#format=json ✅

Still outstanding: https://github.com/bufbuild/buf/issues/1434
Also:
`buf beta convert <(buf build -o -)#format=bin --payload=payload.json --type foo.Bar` will be addressed in https://github.com/bufbuild/buf/issues/1470


Patch:
diff --git a/private/buf/bufcli/bufcli.go b/private/buf/bufcli/bufcli.go
index 254e68a8b..a4a61b69e 100644
--- a/private/buf/bufcli/bufcli.go
+++ b/private/buf/bufcli/bufcli.go
@@ -802,26 +802,26 @@ func NewImageForSource(
 	return bufimage.MergeImages(images...)
 }
 
-// ParseSourceAndType returns the moduleReference and typeName from the source and type provided by the user.
-// When source is not provided, we assume the type is a fully qualified path to the type and try to parse it.
-// Otherwise, if both source and type are provided, the type must be a valid Protobuf identifier (e.g. weather.v1.Units).
-func ParseSourceAndType(
+// ParseInputAndType returns the moduleReference and typeName from the input and type provided by the user.
+// When input is not provided, we assume the type is a fully qualified path to the type and try to parse it.
+// Otherwise, if both input and type are provided, the type must be a valid Protobuf identifier (e.g. weather.v1.Units).
+func ParseInputAndType(
 	ctx context.Context,
-	source string,
+	input string,
 	typeName string,
 ) (string, string, error) {
-	if source != "" && typeName != "" {
+	if input != "" && typeName != "" {
 		if err := bufreflect.ValidateTypeName(typeName); err != nil {
 			return "", "", err
 		}
-		return source, typeName, nil
+		return input, typeName, nil
 	}
 	if typeName == "" {
 		return "", "", appcmd.NewInvalidArgumentError("type is required")
 	}
 	moduleReference, moduleTypeName, err := parseFullyQualifiedPath(typeName)
 	if err != nil {
-		return "", "", appcmd.NewInvalidArgumentErrorf("if a source isn't provided, the type needs to be a fully qualified path that includes the module reference; failed to parse the type: %v", err)
+		return "", "", appcmd.NewInvalidArgumentErrorf("if a input isn't provided, the type needs to be a fully qualified path that includes the module reference; failed to parse the type: %v", err)
 	}
 	return moduleReference, moduleTypeName, nil
 }
diff --git a/private/buf/cmd/buf/command/beta/convert/convert.go b/private/buf/cmd/buf/command/beta/convert/convert.go
index 481284550..005ad255d 100644
--- a/private/buf/cmd/buf/command/beta/convert/convert.go
+++ b/private/buf/cmd/buf/command/beta/convert/convert.go
@@ -31,7 +31,7 @@ import (
 const (
 	errorFormatFlagName = "error-format"
 	typeFlagName        = "type"
-	inputFlagName       = "input"
+	payloadFlagName     = "payload"
 	outputFlagName      = "output"
 	outputFlagShortName = "o"
 )
@@ -43,10 +43,10 @@ func NewCommand(
 ) *appcmd.Command {
 	flags := newFlags()
 	return &appcmd.Command{
-		Use:   name + " <source>",
-		Short: "Use a source reference to convert a binary or JSON serialized message supplied through stdin or the input flag.",
-		Long: `The first argument is the source that defines the serialized message (like buf.build/acme/weather).
-Alternatively, you can omit the source and specify a fully qualified path for the type using the --type option (like buf.build/acme/weather#acme.weather.v1.Units).`,
+		Use:   name + " <input>",
+		Short: "Use a input reference to convert a binary or JSON serialized message supplied through stdin or the payload flag.",
+		Long: `The first argument is the input that defines the serialized message (like buf.build/acme/weather).
+Alternatively, you can omit the input and specify a fully qualified path for the type using the --type option (like buf.build/acme/weather#acme.weather.v1.Units).`,
 		Args: cobra.MaximumNArgs(1),
 		Run: builder.NewRunFunc(
 			func(ctx context.Context, container appflag.Container) error {
@@ -61,8 +61,11 @@ Alternatively, you can omit the source and specify a fully qualified path for th
 type flags struct {
 	ErrorFormat string
 	Type        string
-	Input       string
+	Payload     string
 	Output      string
+
+	// special
+	InputHashtag string
 }
 
 func newFlags() *flags {
@@ -70,6 +73,7 @@ func newFlags() *flags {
 }
 
 func (f *flags) Bind(flagSet *pflag.FlagSet) {
+	bufcli.BindInputHashtag(flagSet, &f.InputHashtag)
 	flagSet.StringVar(
 		&f.ErrorFormat,
 		errorFormatFlagName,
@@ -83,15 +87,15 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) {
 		&f.Type,
 		typeFlagName,
 		"",
-		`The full type name of the serialized message (like acme.weather.v1.Units).
+		`The full type name of the serialized payload (like acme.weather.v1.Units) within the input.
 Alternatively, this can be a fully qualified path to the type without providing the source (like buf.build/acme/weather#acme.weather.v1.Units).`,
 	)
 	flagSet.StringVar(
-		&f.Input,
-		inputFlagName,
+		&f.Payload,
+		payloadFlagName,
 		"-",
 		fmt.Sprintf(
-			`The location to read the input message. Must be one of format %s.`,
+			`The location to read the payload. Must be one of format %s.`,
 			bufconvert.MessageEncodingFormatsString,
 		),
 	)
@@ -115,11 +119,11 @@ func run(
 	if err := bufcli.ValidateErrorFormatFlag(flags.ErrorFormat, errorFormatFlagName); err != nil {
 		return err
 	}
-	input, err := bufcli.GetInputValue(container, "", "")
+	input, err := bufcli.GetInputValue(container, flags.InputHashtag, ".")
 	if err != nil {
 		return err
 	}
-	source, typeName, err := bufcli.ParseSourceAndType(ctx, input, flags.Type)
+	source, typeName, err := bufcli.ParseInputAndType(ctx, input, flags.Type)
 	if err != nil {
 		return err
 	}
@@ -138,7 +142,7 @@ func run(
 	if err != nil {
 		return err
 	}
-	inputMessageRef, err := bufconvert.NewMessageEncodingRef(ctx, flags.Input, bufconvert.MessageEncodingBin)
+	payloadMessageRef, err := bufconvert.NewMessageEncodingRef(ctx, flags.Payload, bufconvert.MessageEncodingBin)
 	if err != nil {
 		return fmt.Errorf("--%s: %v", outputFlagName, err)
 	}
@@ -149,12 +153,12 @@ func run(
 		container,
 		image,
 		typeName,
-		inputMessageRef,
+		payloadMessageRef,
 	)
 	if err != nil {
 		return err
 	}
-	defaultOutputEncoding, err := inverseEncoding(inputMessageRef.MessageEncoding())
+	defaultOutputEncoding, err := inverseEncoding(payloadMessageRef.MessageEncoding())
 	if err != nil {
 		return err
 	}
@@ -174,7 +178,7 @@ func run(
 }
 
 // inverseEncoding returns the opposite encoding of the provided encoding,
-// which will be the default output encoding for a given input encoding.
+// which will be the default output encoding for a given payload encoding.
 func inverseEncoding(encoding bufconvert.MessageEncoding) (bufconvert.MessageEncoding, error) {
 	switch encoding {
 	case bufconvert.MessageEncodingBin:


Test patch:
diff --git a/private/buf/bufcli/bufcli_test.go b/private/buf/bufcli/bufcli_test.go
index 1e1173a05..fb4904ec9 100644
--- a/private/buf/bufcli/bufcli_test.go
+++ b/private/buf/bufcli/bufcli_test.go
@@ -22,33 +22,33 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestParseSourceAndType(t *testing.T) {
+func TestParseInputAndType(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()
 	t.Run("default", func(t *testing.T) {
-		moduleReference, typeName, err := bufcli.ParseSourceAndType(ctx, "", "buf.test/testuser/testrepo#buf.v1.Foo")
+		moduleReference, typeName, err := bufcli.ParseInputAndType(ctx, "", "buf.test/testuser/testrepo#buf.v1.Foo")
 		assert.NoError(t, err)
 		assert.Equal(t, "buf.test/testuser/testrepo", moduleReference)
 		assert.Equal(t, "buf.v1.Foo", typeName)
 	})
 	t.Run("main track", func(t *testing.T) {
-		moduleReference, typeName, err := bufcli.ParseSourceAndType(ctx, "", "buf.test/testuser/testrepo:main#buf.v1.Foo")
+		moduleReference, typeName, err := bufcli.ParseInputAndType(ctx, "", "buf.test/testuser/testrepo:main#buf.v1.Foo")
 		assert.NoError(t, err)
 		assert.Equal(t, "buf.test/testuser/testrepo", moduleReference)
 		assert.Equal(t, "buf.v1.Foo", typeName)
 	})
 	t.Run("dev track", func(t *testing.T) {
-		moduleReference, typeName, err := bufcli.ParseSourceAndType(ctx, "", "buf.test/testuser/testrepo:dev#buf.v1.Foo")
+		moduleReference, typeName, err := bufcli.ParseInputAndType(ctx, "", "buf.test/testuser/testrepo:dev#buf.v1.Foo")
 		assert.NoError(t, err)
 		assert.Equal(t, "buf.test/testuser/testrepo:dev", moduleReference)
 		assert.Equal(t, "buf.v1.Foo", typeName)
 	})
 	t.Run("fail with module name", func(t *testing.T) {
-		_, _, err := bufcli.ParseSourceAndType(ctx, "", "buf.test/testuser/testrepo")
-		assert.EqualError(t, err, `if a source isn't provided, the type needs to be a fully qualified path that includes the module reference; failed to parse the type: "buf.test/testuser/testrepo" is not a valid fully qualified path`)
+		_, _, err := bufcli.ParseInputAndType(ctx, "", "buf.test/testuser/testrepo")
+		assert.EqualError(t, err, `if a input isn't provided, the type needs to be a fully qualified path that includes the module reference; failed to parse the type: "buf.test/testuser/testrepo" is not a valid fully qualified path`)
 	})
 	t.Run("fail with type name", func(t *testing.T) {
-		_, _, err := bufcli.ParseSourceAndType(ctx, "", "buf.v1.Foo")
-		assert.EqualError(t, err, `if a source isn't provided, the type needs to be a fully qualified path that includes the module reference; failed to parse the type: "buf.v1.Foo" is not a valid fully qualified path`)
+		_, _, err := bufcli.ParseInputAndType(ctx, "", "buf.v1.Foo")
+		assert.EqualError(t, err, `if a input isn't provided, the type needs to be a fully qualified path that includes the module reference; failed to parse the type: "buf.v1.Foo" is not a valid fully qualified path`)
 	})
 }
diff --git a/private/buf/cmd/buf/buf_test.go b/private/buf/cmd/buf/buf_test.go
index fb0536ab8..12495b416 100644
--- a/private/buf/cmd/buf/buf_test.go
+++ b/private/buf/cmd/buf/buf_test.go
@@ -2127,7 +2127,7 @@ func TestConvertRoundTrip(t *testing.T) {
 			filepath.Join(tempDir, "image.bin"),
 			"--type",
 			"buf.Foo",
-			"--input",
+			"--payload",
 			"-#format=json",
 		)
 		testRun(
@@ -2157,7 +2157,7 @@ func TestConvertRoundTrip(t *testing.T) {
 			filepath.Join(tempDir, "image.bin"),
 			"--type",
 			"buf.Foo",
-			"--input",
+			"--payload",
 			"-#format=json",
 			"-o",
 			"-#format=bin",
@@ -2172,7 +2172,7 @@ func TestConvertRoundTrip(t *testing.T) {
 			filepath.Join(tempDir, "image.bin"),
 			"--type",
 			"buf.Foo",
-			"--input",
+			"--payload",
 			"-#format=bin",
 			"-o",
 			"-#format=json",
@@ -2192,7 +2192,7 @@ func TestConvertRoundTrip(t *testing.T) {
 			filepath.Join(tempDir, "image.bin"),
 			"--type",
 			"buf.Foo",
-			"--input",
+			"--payload",
 			"-#format=json",
 			"-o",
 			filepath.Join(tempDir, "decoded_message.bin"),
@@ -2207,7 +2207,7 @@ func TestConvertRoundTrip(t *testing.T) {
 			filepath.Join(tempDir, "image.bin"),
 			"--type",
 			"buf.Foo",
-			"--input",
+			"--payload",
 			filepath.Join(tempDir, "decoded_message.bin"),
 		)
 		assert.JSONEq(t, `{"one":"55"}`, decodedMessage.String())
diff --git a/private/buf/cmd/buf/command/beta/convert/convert_test.go b/private/buf/cmd/buf/command/beta/convert/convert_test.go
new file mode 100644
index 000000000..99d794454
--- /dev/null
+++ b/private/buf/cmd/buf/command/beta/convert/convert_test.go
@@ -0,0 +1,183 @@
+// Copyright 2020-2022 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package convert
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/bufbuild/buf/private/buf/cmd/buf/internal/internaltesting"
+	"github.com/bufbuild/buf/private/pkg/app/appcmd"
+	"github.com/bufbuild/buf/private/pkg/app/appcmd/appcmdtesting"
+	"github.com/bufbuild/buf/private/pkg/app/appflag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvert(t *testing.T) {
+	t.Run("bin-to-json-file-proto", func(t *testing.T) {
+		testConvert(
+			t,
+			nil,
+			"testdata/bin_json/payload.json",
+			"--type=buf.Foo",
+			"--payload=testdata/bin_json/payload.bin",
+			"testdata/bin_json/buf.proto",
+		)
+	})
+	t.Run("json-to-bin-file-proto", func(t *testing.T) {
+		testConvert(
+			t,
+			nil,
+			"testdata/bin_json/payload.bin",
+			"--type=buf.Foo",
+			"--payload=testdata/bin_json/payload.json",
+			"testdata/bin_json/buf.proto",
+		)
+	})
+	t.Run("stdin-json-to-bin-proto", func(t *testing.T) {
+		testConvert(
+			t,
+			strings.NewReader(`{"one":"55"}`),
+			"testdata/bin_json/payload.bin",
+			"--type=buf.Foo",
+			"--payload",
+			"-#format=json",
+			"testdata/bin_json/buf.proto",
+		)
+	})
+	t.Run("stdin-bin-to-json-proto", func(t *testing.T) {
+		file, err := os.Open("testdata/bin_json/payload.bin")
+		require.NoError(t, err)
+		testConvert(
+			t,
+			file,
+			"testdata/bin_json/payload.json",
+			"--type=buf.Foo",
+			"--payload",
+			"-#format=bin",
+			"testdata/bin_json/buf.proto",
+		)
+	})
+	t.Run("stdin-json-to-json-proto", func(t *testing.T) {
+		testConvert(
+			t,
+			strings.NewReader(`{"one":"55"}`),
+			"testdata/bin_json/payload.json",
+			"--type=buf.Foo",
+			"testdata/bin_json/buf.proto",
+			"--payload",
+			"-#format=json",
+			"-o",
+			"-#format=json",
+		)
+	})
+	t.Run("stdin-input-to-json-image", func(t *testing.T) {
+		file, err := os.Open("testdata/bin_json/image.bin")
+		require.NoError(t, err)
+		testConvert(
+			t,
+			file,
+			"testdata/bin_json/payload.json",
+			"--type=buf.Foo",
+			"-",
+			"--payload=testdata/bin_json/payload.bin",
+			"-o",
+			"-#format=json",
+		)
+	})
+	t.Run("stdin-json-to-json-image", func(t *testing.T) {
+		file, err := os.Open("testdata/bin_json/payload.bin")
+		require.NoError(t, err)
+		testConvert(
+			t,
+			file,
+			"testdata/bin_json/payload.json",
+			"--type=buf.Foo",
+			"testdata/bin_json/image.bin",
+			"--payload",
+			"-#format=bin",
+			"-o",
+			"-#format=json",
+		)
+	})
+	t.Run("stdin-bin-payload-to-json-with-image", func(t *testing.T) {
+		file, err := os.Open("testdata/bin_json/payload.bin")
+		require.NoError(t, err)
+		testConvert(
+			t,
+			file,
+			"testdata/bin_json/payload.json",
+			"--type=buf.Foo",
+			"testdata/bin_json/image.bin",
+			"-o",
+			"-#format=json",
+		)
+	})
+	t.Run("stdin-json-payload-to-bin-with-image", func(t *testing.T) {
+		file, err := os.Open("testdata/bin_json/payload.json")
+		require.NoError(t, err)
+		testConvert(
+			t,
+			file,
+			"testdata/bin_json/payload.bin",
+			"--type=buf.Foo",
+			"testdata/bin_json/image.bin",
+			"--payload",
+			"-#format=json",
+			"-o",
+			"-#format=bin",
+		)
+	})
+	t.Run("stdin-image-json-to-bin", func(t *testing.T) {
+		file, err := os.Open("testdata/bin_json/image.json")
+		require.NoError(t, err)
+		testConvert(
+			t,
+			file,
+			"testdata/bin_json/payload.bin",
+			"--type=buf.Foo",
+			"-#format=json",
+			"--payload=testdata/bin_json/payload.json",
+			"-o",
+			"-#format=bin",
+		)
+	})
+}
+
+func testConvert(t *testing.T, stdin io.Reader, want string, args ...string) {
+	var stdout bytes.Buffer
+	appcmdtesting.RunCommandSuccess(
+		t,
+		func(name string) *appcmd.Command {
+			return NewCommand(
+				name,
+				appflag.NewBuilder(name),
+			)
+		},
+		internaltesting.NewEnvFunc(t),
+		stdin,
+		&stdout,
+		args...,
+	)
+	wantReader, err := os.Open(want)
+	require.NoError(t, err)
+	wantBytes, err := io.ReadAll(wantReader)
+	require.NoError(t, err)
+	assert.Equal(t, string(wantBytes), stdout.String())
+}