Repository: elixir-protobuf/protobuf
Pull request #390 (2024-12-29 17:16:16)

Support json_name for proto2; Ensures proper field emission for JSON encoder
Fixes >500 conformance tests
- Makes proto2 also support json_name. This alone fixes >200 tests
- Ensures JSON encoder decides correctly when to emit or not emit fields (at the very least, more correctly than before :))


Patch:
diff --git a/conformance/exemptions.txt b/conformance/exemptions.txt
index 7c39f7e..7c29239 100644
--- a/conformance/exemptions.txt
+++ b/conformance/exemptions.txt
@@ -1,5 +1,4 @@
 Recommended.Proto2.JsonInput.FieldNameExtension.Validator
-Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInOptionalField.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput
diff --git a/lib/protobuf/json.ex b/lib/protobuf/json.ex
index 1a51704..63c7193 100644
--- a/lib/protobuf/json.ex
+++ b/lib/protobuf/json.ex
@@ -121,8 +121,8 @@ defmodule Protobuf.JSON do
   """
   @spec encode!(struct, [encode_opt]) :: String.t() | no_return
   def encode!(struct, opts \\ []) do
-    case encode(struct, opts) do
-      {:ok, json} -> json
+    case encode_to_iodata(struct, opts) do
+      {:ok, iodata} -> IO.iodata_to_binary(iodata)
       {:error, error} -> raise error
     end
   end
@@ -169,9 +169,41 @@ defmodule Protobuf.JSON do
   """
   @spec encode(struct, [encode_opt]) ::
           {:ok, String.t()} | {:error, EncodeError.t() | Exception.t()}
-  def encode(%_{} = struct, opts \\ []) when is_list(opts) do
+  def encode(struct, opts \\ []) when is_list(opts) do
+    case encode_to_iodata(struct, opts) do
+      {:ok, iodata} -> {:ok, IO.iodata_to_binary(iodata)}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @doc """
+  Similar to `encode!/2`, but returns iodata
+
+  See `encode_to_iodata/2` for more information about when this function should
+  be preferred over `encode!/2`.
+  """
+  @spec encode_to_iodata!(struct, [encode_opt]) :: iodata()
+  def encode_to_iodata!(struct, opts \\ []) when is_list(opts) do
+    case encode_to_iodata(struct, opts) do
+      {:ok, iodata} -> iodata
+      {:error, error} -> raise error
+    end
+  end
+
+  @doc """
+  Similar to `encode/2`, but returns iodata
+
+  This function should be preferred to encode/2, if the generated JSON will be
+  handed over to one of the IO functions or sent over the socket. The Erlang
+  runtime is able to leverage vectorised writes and avoid allocating a continuous
+  buffer for the whole resulting string, lowering memory use and increasing
+  performance.
+  """
+  @spec encode_to_iodata(struct, [encode_opt]) ::
+          {:ok, iodata()} | {:error, EncodeError.t() | Exception.t()}
+  def encode_to_iodata(%_{} = struct, opts \\ []) when is_list(opts) do
     if jason = load_jason() do
-      with {:ok, map} <- to_encodable(struct, opts), do: jason.encode(map)
+      with {:ok, map} <- to_encodable(struct, opts), do: jason.encode_to_iodata(map)
     else
       {:error, EncodeError.new(:no_json_lib)}
     end
diff --git a/lib/protobuf/json/encode.ex b/lib/protobuf/json/encode.ex
index 63f29fe..13e97e0 100644
--- a/lib/protobuf/json/encode.ex
+++ b/lib/protobuf/json/encode.ex
@@ -159,22 +159,14 @@ defmodule Protobuf.JSON.Encode do
     :maps.from_list(regular ++ oneofs)
   end
 
-  defp encode_regular_fields(struct, %{field_props: field_props}, opts) do
+  defp encode_regular_fields(struct, %{field_props: field_props, syntax: syntax}, opts) do
     for {_field_num, %{name_atom: name, oneof: nil} = prop} <- field_props,
         %{^name => value} = struct,
-        emit?(prop, value) || opts[:emit_unpopulated] do
+        emit?(syntax, prop, value) || opts[:emit_unpopulated] do
       encode_field(prop, value, opts)
     end
   end
 
-  defp emit?(_prop, nil) do
-    false
-  end
-
-  defp emit?(prop, value) do
-    if default?(prop, value), do: prop.proto3_optional?, else: true
-  end
-
   defp encode_oneof_fields(struct, message_props, opts) do
     %{field_tags: field_tags, field_props: field_props, oneof: oneofs} = message_props
 
@@ -309,15 +301,18 @@ defmodule Protobuf.JSON.Encode do
   defp maybe_repeat(%{repeated?: false}, val, fun), do: fun.(val)
   defp maybe_repeat(%{repeated?: true}, val, fun), do: Enum.map(val, fun)
 
-  defp default?(_prop, +0.0), do: true
-  defp default?(_prop, nil), do: true
-  defp default?(_prop, 0), do: true
-  defp default?(_prop, false), do: true
-  defp default?(_prop, []), do: true
-  defp default?(_prop, ""), do: true
-  defp default?(_prop, %{} = map) when map_size(map) == 0, do: true
-  defp default?(%{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) == 0
-  defp default?(_prop, _value), do: false
+  defp emit?(:proto2, %{default: value}, value), do: false
+  defp emit?(:proto2, %{optional?: true}, val), do: not is_nil(val)
+  defp emit?(:proto3, %{proto3_optional?: true}, val), do: not is_nil(val)
+  defp emit?(_syntax, _prop, +0.0), do: false
+  defp emit?(_syntax, _prop, nil), do: false
+  defp emit?(_syntax, _prop, 0), do: false
+  defp emit?(_syntax, _prop, false), do: false
+  defp emit?(_syntax, _prop, []), do: false
+  defp emit?(_syntax, _prop, ""), do: false
+  defp emit?(_syntax, _prop, %{} = map) when map_size(map) == 0, do: false
+  defp emit?(_syntax, %{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) != 0
+  defp emit?(_syntax, _prop, _value), do: true
 
   defp transform_module(message, module) do
     if transform_module = module.transform_module() do
diff --git a/lib/protobuf/protoc/generator/message.ex b/lib/protobuf/protoc/generator/message.ex
index 82c1dde..b3cefab 100644
--- a/lib/protobuf/protoc/generator/message.ex
+++ b/lib/protobuf/protoc/generator/message.ex
@@ -339,10 +339,10 @@ defmodule Protobuf.Protoc.Generator.Message do
   end
 
   # Omit `json_name` from the options list when it matches the original field
-  # name to keep the list small. Only Proto3 has JSON support for now.
-  defp add_json_name_to_opts(opts, :proto3, %{name: name, json_name: name}), do: opts
+  # name to keep the list small.
+  defp add_json_name_to_opts(opts, _, %{name: name, json_name: name}), do: opts
 
-  defp add_json_name_to_opts(opts, :proto3, %{json_name: json_name}),
+  defp add_json_name_to_opts(opts, _, %{json_name: json_name}),
     do: Keyword.put(opts, :json_name, json_name)
 
   defp add_json_name_to_opts(opts, _syntax, _props), do: opts


Test patch:
diff --git a/test/protobuf/json_test.exs b/test/protobuf/json_test.exs
index 5d10efd..9285f92 100644
--- a/test/protobuf/json_test.exs
+++ b/test/protobuf/json_test.exs
@@ -8,6 +8,12 @@ defmodule Protobuf.JSONTest do
     assert Protobuf.JSON.encode!(%TestMsg.Foo2{b: 10}) == ~S|{"b":"10"}|
   end
 
+  test "encode_to_iodata variants encode to iodata" do
+    assert iodata = Protobuf.JSON.encode_to_iodata!(%TestMsg.Foo2{b: 10})
+    assert {:ok, ^iodata} = Protobuf.JSON.encode_to_iodata(%TestMsg.Foo2{b: 10})
+    assert IO.iodata_to_binary(iodata) == ~S|{"b":"10"}|
+  end
+
   test "encoding string field with invalid UTF-8 data" do
     message = %Scalars{string: "   \xff   "}
     assert {:error, %Jason.EncodeError{}} = Protobuf.JSON.encode(message)
diff --git a/test/protobuf/protoc/generator/message_test.exs b/test/protobuf/protoc/generator/message_test.exs
index 3bef634..9bc642d 100644
--- a/test/protobuf/protoc/generator/message_test.exs
+++ b/test/protobuf/protoc/generator/message_test.exs
@@ -731,27 +731,6 @@ defmodule Protobuf.Protoc.Generator.MessageTest do
       assert msg =~ "field :simple, 1, type: :int32\n"
       assert msg =~ "field :the_field_name, 2, type: :string, json_name: \"theFieldName\"\n"
     end
-
-    test "is omitted when syntax is not proto3" do
-      ctx = %Context{}
-
-      desc = %Google.Protobuf.DescriptorProto{
-        name: "Foo",
-        field: [
-          %Google.Protobuf.FieldDescriptorProto{
-            name: "the_field_name",
-            json_name: "theFieldName",
-            number: 1,
-            type: :TYPE_STRING,
-            label: :LABEL_REQUIRED
-          }
-        ]
-      }
-
-      {[], [{_mod, msg}]} = Generator.generate(ctx, desc)
-
-      assert msg =~ "field :the_field_name, 1, required: true, type: :string\n"
-    end
   end
 
   test "generate/2 repeated enum field" do