Repository: tokio-rs/prost (language: rust)

Issue:
rustdoc does not emit Message impl for generated types
I am not sure if I am too dumb, but I couldn't figure out how to actually read a file using the generated code  an example would be really helpful.

Using prost, I've generated the rust bindings for the OpenStreetMap .pbf files: https://crates.io/crates/osm-proto-rs

And here is a small PBF file, for testing: https://github.com/sharazam/small-test-pbf-file

How do I read the file, for example the header block? Do I use bytes or just `File::open` or is there a hidden "Reader" somewhere? Thanks in advance.


Discussion / hints:
Hi @sharazam I left a comment on reddit here: https://www.reddit.com/r/rust/comments/6jyhmx/prost_a_protocol_buffers_implementation_for_the/djikvj9/, but I'll expand a bit on your options for decoding from a file.  The simplest way is, as I mentioned, to read the entire file into a `Vec<u8>` using something like [`Read::read_to_end`](https://doc.rust-lang.org/nightly/std/io/trait.Read.html#method.read_to_end).  If the file is really big and you want to do it more lazily, you could also read-as-you-go, since the OSM format appears to be length-prefixed.  Read the length-prefix, then read that much into the buffer.  Another option is to use the `memmap` crate to map the file to a slice, which can then be wrapped in a `Cursor` to get a `Buf` impl.
@danburkert Yes, I know how to read a file. What I don't get is how I should serialize the file into the structs. That's what protobuf is for, isn't it? I've published version 0.2 of the osm-pbf-rs library with the fileformat.proto file added: https://docs.rs/osm-proto-rs/0.2.0/osm_proto_rs/

But there is no "decode" method like you mentioned, nothing. How should I serialize the PBF file into structs, I thought that was what protobuf was for?
Interesting, I think there may be a bug in rustdoc.  The encode method comes from an impl of the [`Message`](https://docs.rs/prost/0.1.0/prost/trait.Message.html) trait.  Every Rust struct generated by `prost` has a derived impl of `Message`.  I do see the issue where the rustdocs aren't listing that, though, and I'll look into that.

Fix patch:
diff --git a/prost-derive/src/prost-derive.rs b/prost-derive/src/prost-derive.rs
index 8419e8243..e2a8ed746 100644
--- a/prost-derive/src/prost-derive.rs
+++ b/prost-derive/src/prost-derive.rs
@@ -73,7 +73,8 @@ fn try_message(input: TokenStream) -> Result<TokenStream> {
         bail!("message {} has fields with duplicate tags", ident);
     }
 
-    let dummy_const = Ident::new(format!("_IMPL_MESSAGE_FOR_{}", ident));
+    // Put impls in a special module, so that 'extern crate' can be used.
+    let module = Ident::new(format!("{}_MESSAGE", ident));
 
     let encoded_len = fields.iter()
                             .map(|&(ref field_ident, ref field)| {
@@ -107,6 +108,7 @@ fn try_message(input: TokenStream) -> Result<TokenStream> {
         quote!()
     } else {
         quote! {
+            #[allow(dead_code)]
             impl #ident {
                 #(#methods)*
             }
@@ -114,26 +116,17 @@ fn try_message(input: TokenStream) -> Result<TokenStream> {
     };
 
     let expanded = quote! {
-        #[allow(
-            non_upper_case_globals,
-            unused_attributes,
-            unused_imports,
-            unused_qualifications,
-            unused_variables
-        )]
-        const #dummy_const: () = {
-
+        #[allow(non_snake_case, unused_attributes)]
+        mod #module {
             extern crate prost as _prost;
             extern crate bytes as _bytes;
+            use super::*;
 
-            #[automatically_derived]
             impl _prost::Message for #ident {
-                #[inline]
                 fn encode_raw<B>(&self, buf: &mut B) where B: _bytes::BufMut {
                     #(#encode)*
                 }
 
-                #[inline]
                 fn merge_field<B>(&mut self, buf: &mut B) -> ::std::result::Result<(), _prost::DecodeError>
                 where B: _bytes::Buf {
                     let (tag, wire_type) = _prost::encoding::decode_key(buf)?;
@@ -143,13 +136,11 @@ fn try_message(input: TokenStream) -> Result<TokenStream> {
                     }
                 }
 
-                #[inline]
                 fn encoded_len(&self) -> usize {
                     0 #(+ #encoded_len)*
                 }
             }
 
-            #[automatically_derived]
             impl Default for #ident {
                 fn default() -> #ident {
                     #ident {
@@ -157,9 +148,9 @@ fn try_message(input: TokenStream) -> Result<TokenStream> {
                     }
                 }
             }
-        };
 
-        #methods
+            #methods
+        };
     };
 
     expanded.parse::<TokenStream>().map_err(|err| Error::from(format!("{:?}", err)))
@@ -204,7 +195,8 @@ pub fn enumeration(input: TokenStream) -> TokenStream {
 
     let default = variants[0].0.clone();
 
-    let dummy_const = Ident::new(format!("_IMPL_ENUMERATION_FOR_{}", ident));
+    // Put impls in a special module, so that 'extern crate' can be used.
+    let module = Ident::new(format!("{}_ENUMERATION", ident));
     let is_valid = variants.iter().map(|&(_, ref value)| quote!(#value => true));
     let from = variants.iter().map(|&(ref variant, ref value)| quote!(#value => ::std::option::Option::Some(#ident::#variant)));
 
@@ -212,12 +204,12 @@ pub fn enumeration(input: TokenStream) -> TokenStream {
     let from_i32_doc = format!("Converts an `i32` to a `{}`, or `None` if `value` is not a valid variant.", ident);
 
     let expanded = quote! {
-        #[allow(non_upper_case_globals, unused_attributes, unused_qualifications)]
-        const #dummy_const: () = {
+        #[allow(non_snake_case, unused_attributes)]
+        mod #module {
             extern crate bytes as _bytes;
             extern crate prost as _prost;
+            use super::*;
 
-            #[automatically_derived]
             impl #ident {
 
                 #[doc=#is_valid_doc]
@@ -237,14 +229,12 @@ pub fn enumeration(input: TokenStream) -> TokenStream {
                 }
             }
 
-            #[automatically_derived]
             impl ::std::default::Default for #ident {
                 fn default() -> #ident {
                     #ident::#default
                 }
             }
 
-            #[automatically_derived]
             impl ::std::convert::From<#ident> for i32 {
                 fn from(value: #ident) -> i32 {
                     value as i32
@@ -304,7 +294,8 @@ fn try_oneof(input: TokenStream) -> Result<TokenStream> {
         panic!("invalid oneof {}: variants have duplicate tags", ident);
     }
 
-    let dummy_const = Ident::new(format!("_IMPL_ONEOF_FOR_{}", ident));
+    // Put impls in a special module, so that 'extern crate' can be used.
+    let module = Ident::new(format!("{}_ONEOF", ident));
 
     let encode = fields.iter().map(|&(ref variant_ident, ref field)| {
         let encode = field.encode(&Ident::new("*value"));
@@ -328,16 +319,11 @@ fn try_oneof(input: TokenStream) -> Result<TokenStream> {
     });
 
     let expanded = quote! {
-        #[allow(
-            non_upper_case_globals,
-            unused_attributes,
-            unused_imports,
-            unused_qualifications,
-            unused_variables
-        )]
-        const #dummy_const: () = {
+        #[allow(non_snake_case, unused_attributes)]
+        mod #module {
             extern crate bytes as _bytes;
             extern crate prost as _prost;
+            use super::*;
 
             impl #ident {
                 pub fn encode<B>(&self, buf: &mut B) where B: _bytes::BufMut {
diff --git a/src/encoding.rs b/src/encoding.rs
index 6b3bca941..6cfee8adb 100644
--- a/src/encoding.rs
+++ b/src/encoding.rs
@@ -148,6 +148,29 @@ pub fn check_wire_type(expected: WireType, actual: WireType) -> Result<(), Decod
     Ok(())
 }
 
+/// Helper function which abstracts reading a length delimiter prefix followed
+/// by decoding values until the length of bytes is exhausted.
+pub fn merge_loop<T, M, B>(value: &mut T, buf: &mut B, mut merge: M) -> Result<(), DecodeError>
+where
+    M: FnMut(&mut T, &mut B) -> Result<(), DecodeError>,
+    B: Buf {
+        let len = decode_varint(buf)?;
+        let remaining = buf.remaining();
+        if len > remaining as u64 {
+            return Err(DecodeError::new("buffer underflow"))
+        }
+
+        let limit = remaining - len as usize;
+        while buf.remaining() > limit {
+            merge(value, buf)?;
+        }
+
+        if buf.remaining() != limit {
+            return Err(DecodeError::new("delimited length exceeded"))
+        }
+        Ok(())
+}
+
 pub fn skip_field<B>(wire_type: WireType, buf: &mut B) -> Result<(), DecodeError> where B: Buf {
     let len = match wire_type {
         WireType::Varint => decode_varint(buf).map(|_| 0)?,
@@ -186,31 +209,21 @@ macro_rules! merge_repeated_numeric {
                                   buf: &mut B)
                                   -> Result<(), DecodeError> where B: Buf {
             if wire_type == WireType::LengthDelimited {
-                let len = decode_varint(buf)?;
-                let remaining = buf.remaining();
-                if len > remaining as u64 {
-                    return Err(DecodeError::new("buffer underflow"));
-                }
-                let len = len as usize;
-                let limit = remaining - len as usize;
-
-                while buf.remaining() > limit {
+                // Packed.
+                merge_loop(values, buf, |values, buf| {
                     let mut value = Default::default();
                     $merge($wire_type, &mut value, buf)?;
                     values.push(value);
-                }
-
-                if buf.remaining() != limit {
-                    return Err(DecodeError::new("delimited length exceeded"))
-                }
-
+                    Ok(())
+                })
             } else {
+                // Unpacked.
                 check_wire_type($wire_type, wire_type)?;
                 let mut value = Default::default();
                 $merge(wire_type, &mut value, buf)?;
                 values.push(value);
+                Ok(())
             }
-            Ok(())
         }
     )
 }
@@ -562,21 +575,7 @@ pub mod message {
     where M: Message,
           B: Buf {
         check_wire_type(WireType::LengthDelimited, wire_type)?;
-        let len = decode_varint(buf)?;
-        let remaining = buf.remaining();
-        if len > remaining as u64 {
-            return Err(DecodeError::new("buffer underflow"))
-        }
-
-        let limit = remaining - len as usize;
-        while buf.remaining() > limit {
-            msg.merge_field(buf)?;
-        }
-
-        if buf.remaining() != limit {
-            return Err(DecodeError::new("delimited length exceeded"))
-        }
-        Ok(())
+        merge_loop(msg, buf, M::merge_field)
     }
 
     pub fn encode_repeated<M, B>(tag: u32, messages: &[M], buf: &mut B)
@@ -718,30 +717,19 @@ macro_rules! map {
               B: Buf,
               KM: Fn(WireType, &mut K, &mut B) -> Result<(), DecodeError>,
               VM: Fn(WireType, &mut V, &mut B) -> Result<(), DecodeError> {
-            let len = decode_varint(buf)?;
-            let remaining = buf.remaining();
-            if len > remaining as u64 {
-                return Err(DecodeError::new("buffer underflow"));
-            }
 
             let mut key = Default::default();
             let mut val = val_default;
-
-            let limit = remaining - len as usize;
-            while buf.remaining() > limit {
+            merge_loop(&mut (&mut key, &mut val), buf, |&mut (ref mut key, ref mut val), buf| {
                 let (tag, wire_type) = decode_key(buf)?;
                 match tag {
-                    1 => key_merge(wire_type, &mut key, buf)?,
-                    2 => val_merge(wire_type, &mut val, buf)?,
-                    _ => (),
+                    1 => key_merge(wire_type, key, buf),
+                    2 => val_merge(wire_type, val, buf),
+                    _ => Ok(()),
                 }
-            }
+            })?;
             values.insert(key, val);
 
-            if buf.remaining() != limit {
-                return Err(DecodeError::new("delimited length exceeded"))
-            }
-
             Ok(())
         }
 


Test patch: