Repository: elastic-rs/elastic (language: rust)

Issue:
Cannot use `_doc` type for bulk requests
I am using `v0.21.0-pre.1`

Maybe it is because Elastic's [removal of mapping types](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/removal-of-types.html) breaking change isn't fully supported in preparation for the `v0.21.0` release or maybe I don't understand how this library [handles types](https://docs.rs/elastic/0.21.0-pre.1/elastic/types/index.html), but 

```rust
#[derive(Serialize, Deserialize, GraphQLObject, ElasticType, Debug)]
#[elastic(index = "test")]
#[elastic(ty = "_doc")]
pub struct MyType {
    #[elastic(id)]
    id: String,
    timestamp: ChronoDateTime,
}

let ops = (0..1000).into_iter().map(|i| {
    bulk::<MyType>().index(MyType {
        id: i.to_string(),
        timestamp: "2019-02-18T00:00:00Z".parse().unwrap(),
    })
});

let response = client.bulk().extend(ops).send()?;
```

produces the following mapping:
```json
{
  "test": {
    "mappings": {
      "_doc": {
        "properties": {
          "doc": {
            "properties": {
              "id": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "timestamp": {
                "type": "date"
              }
            }
          }
        }
      }
    }
  }
}
```

whereas I would expect* it to produce:
```json
{
  "test": {
    "mappings": {
      "_doc": {
        "properties": {
          "timestamp": {
            "type": "date"
          }
        }
      }
    }
  }
}
```

*if you index it like:
```
curl -X PUT ":9200/test/_doc/1" -H 'Content-Type: application/json' -d'
{
     "timestamp" : "2009-11-15T14:12:12"
}
'
```


Discussion / hints:
Thanks for the report @mwilliammyers!

This looks like a bug.
So I am in a little over my head with the macros/custom derives but I think this is due to (at least partly) [this](https://github.com/elastic-rs/elastic/blob/dbb095a0ded5756b45e74271c149695483c41f8a/src/types_derive_internals/src/elastic_type/mod.rs#L192) line.
Ah yes that's where we use `doc` as a default type name if one hasn't been supplied. That path shouldn't be hit in your case though since you're supplying `_doc` as an explicit type name to use. If we run `cargo expand` on your sample then everything looks ok in the generated code for `#[derive(ElasticType)]`:

```rust
#[rustc_copy_clone_marker]
pub struct MyTypeMapping;
#[automatically_derived]
#[allow(unused_qualifications)]
impl ::std::default::Default for MyTypeMapping {
    #[inline]
    fn default() -> MyTypeMapping {
        MyTypeMapping
    }
}
#[automatically_derived]
#[allow(unused_qualifications)]
impl ::std::clone::Clone for MyTypeMapping {
    #[inline]
    fn clone(&self) -> MyTypeMapping {
        {
            *self
        }
    }
}
#[automatically_derived]
#[allow(unused_qualifications)]
impl ::std::marker::Copy for MyTypeMapping {}
#[automatically_derived]
#[allow(unused_qualifications)]
impl ::std::fmt::Debug for MyTypeMapping {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        match *self {
            MyTypeMapping => {
                let mut debug_trait_builder = f.debug_tuple("MyTypeMapping");
                debug_trait_builder.finish()
            }
        }
    }
}
#[allow(non_upper_case_globals, dead_code, unused_variables)]
const _IMPL_EASTIC_TYPE_FOR_MyType: () = {
    impl ::elastic::types::derive::ObjectMapping for MyTypeMapping {
        type Properties = MyType;
    }
    impl ::elastic::types::derive::ObjectFieldType for MyType {
        type Mapping = MyTypeMapping;
    }
    impl ::elastic::types::derive::DocumentType for MyType {
        fn index(&self) -> ::std::borrow::Cow<str> {
            ("test").into()
        }
        fn ty(&self) -> ::std::borrow::Cow<str> {
            ("_doc").into()
        }
        fn partial_id(&self) -> ::std::option::Option<::std::borrow::Cow<str>> {
            (Some((&self.id).into())).into()
        }
        fn partial_static_index() -> ::std::option::Option<&'static str> {
            Some("test")
        }
        fn partial_static_ty() -> ::std::option::Option<&'static str> {
            Some("_doc")
        }
    }
    impl ::elastic::types::derive::StaticIndex for MyType {}
    impl ::elastic::types::derive::StaticType for MyType {}
    impl ::elastic::types::derive::PropertiesMapping for MyType {
        fn props_len() -> usize {
            2usize
        }
        fn serialize_props<S>(state: &mut S) -> ::std::result::Result<(), S::Error>
        where
            S: ::elastic::types::derive::SerializeStruct,
        {
            match ::elastic::types::derive::field_ser::<String, _, _, _>(state, "id") {
                ::core::result::Result::Ok(val) => val,
                ::core::result::Result::Err(err) => {
                    return ::core::result::Result::Err(::core::convert::From::from(err));
                }
            };
            match ::elastic::types::derive::field_ser::<ChronoDateTime, _, _, _>(state, "timestamp")
            {
                ::core::result::Result::Ok(val) => val,
                ::core::result::Result::Err(err) => {
                    return ::core::result::Result::Err(::core::convert::From::from(err));
                }
            };
            Ok(())
        }
    }
};
```

Are you calling `client.document::<MyType>().put_mapping()` at all?
No I am not, I figured we didn't *have* to do that since you don't normally with Elasticsearch... Should I do that first?

Shouldn't `_doc` be the default type?
Ok, I added a `client.document::<MyType>().put_mapping()` (on a fresh index), and there is still a nested `doc` property, so it looks like:
```
"_source": {
    "doc": {
        "timestamp": "2000-01-01T00:00:00Z"
    }
}
```

instead of:
```
"_source": {
    "timestamp": "2000-01-01T00:00:00Z"
}
```

Oh my this is embarassing... I think the bug here is that the new bulk API is re-using a wrapper type from the document update API that's serializing the document you provide in a `"doc": {}` object.

Fix patch:
diff --git a/src/elastic/src/client/requests/bulk/operation.rs b/src/elastic/src/client/requests/bulk/operation.rs
index 9540bc0a24..1ca2be3493 100644
--- a/src/elastic/src/client/requests/bulk/operation.rs
+++ b/src/elastic/src/client/requests/bulk/operation.rs
@@ -189,7 +189,7 @@ where
         }
     }
 
-    pub fn index(self, doc: TDocument) -> BulkOperation<Doc<TDocument>> {
+    pub fn index(self, doc: TDocument) -> BulkOperation<TDocument> {
         BulkOperation {
             action: Action::Index,
             header: BulkHeader {
@@ -197,7 +197,7 @@ where
                 ty: Some(Type::from(doc.ty().into_owned())),
                 id: doc.partial_id().map(|id| Id::from(id.into_owned())),
             },
-            inner: Some(Doc::value(doc)),
+            inner: Some(doc),
         }
     }
 
@@ -256,7 +256,7 @@ where
         .script_fluent(builder)
     }
 
-    pub fn create(self, doc: TDocument) -> BulkOperation<Doc<TDocument>> {
+    pub fn create(self, doc: TDocument) -> BulkOperation<TDocument> {
         BulkOperation {
             action: Action::Create,
             header: BulkHeader {
@@ -264,7 +264,7 @@ where
                 ty: Some(Type::from(doc.ty().into_owned())),
                 id: doc.partial_id().map(|id| Id::from(id.into_owned())),
             },
-            inner: Some(Doc::value(doc)),
+            inner: Some(doc),
         }
     }
 


Test patch:
diff --git a/tests/run/Cargo.toml b/tests/run/Cargo.toml
index 5417fab788..a286251818 100644
--- a/tests/run/Cargo.toml
+++ b/tests/run/Cargo.toml
@@ -17,4 +17,5 @@ tokio-threadpool = "~0.1"
 tokio-timer = "~0.2"
 term-painter = "~0.2"
 clap = "~2"
+log = "~0.4"
 env_logger = "~0.6"
diff --git a/tests/run/src/bulk/index_get.rs b/tests/run/src/bulk/index_get.rs
new file mode 100644
index 0000000000..831fc5317c
--- /dev/null
+++ b/tests/run/src/bulk/index_get.rs
@@ -0,0 +1,58 @@
+use elastic::error::Error;
+use elastic::prelude::*;
+use futures::Future;
+use run_tests::IntegrationTest;
+
+#[derive(Debug, Clone, Copy)]
+pub struct IndexGet;
+
+#[derive(Debug, PartialEq, Serialize, Deserialize, ElasticType)]
+#[elastic(ty = "document", index = "bulk_index")]
+pub struct Doc {
+    #[elastic(id)]
+    id: String,
+    title: String,
+    timestamp: Date<DefaultDateMapping>,
+}
+
+impl IntegrationTest for IndexGet {
+    type Response = GetResponse<Doc>;
+
+    fn kind() -> &'static str {
+        "bulk"
+    }
+    fn name() -> &'static str {
+        "index"
+    }
+
+    // Ensure the index doesn't exist
+    fn prepare(&self, client: AsyncClient) -> Box<Future<Item = (), Error = Error>> {
+        let delete_res = client.index(Doc::static_index()).delete().send().map(|_| ());
+
+        Box::new(delete_res)
+    }
+
+    // Index some bulk documents
+    fn request(&self, client: AsyncClient) -> Box<Future<Item = Self::Response, Error = Error>> {
+        let ops = (0..10).into_iter().map(|i| bulk().index(Doc {
+            id: i.to_string(),
+            title: "A document title".to_owned(),
+            timestamp: Date::build(2017, 03, 24, 13, 44, 0, 0),
+        }));
+
+        let bulk_res = client
+            .bulk()
+            .extend(ops)
+            .params_fluent(|p| p.url_param("refresh", true))
+            .send();
+
+        let get_res = client.document().get("4").send();
+
+        Box::new(bulk_res.and_then(|_| get_res))
+    }
+
+    // Ensure the response contains the expected document
+    fn assert_ok(&self, res: &Self::Response) -> bool {
+        res.document().map(|doc| &*doc.id) == Some("4")
+    }
+}
diff --git a/tests/run/src/bulk/mod.rs b/tests/run/src/bulk/mod.rs
index 8d0707ed13..54b643c7a4 100644
--- a/tests/run/src/bulk/mod.rs
+++ b/tests/run/src/bulk/mod.rs
@@ -4,6 +4,7 @@ use run_tests::{
 };
 
 mod delete;
+mod index_get;
 mod index_create;
 mod stream;
 mod stream_tiny_size_limit;
@@ -13,6 +14,7 @@ mod stream_tiny_timeout;
 pub fn tests() -> Vec<Test> {
     vec![
         Box::new(|client| test(client, delete::Delete)),
+        Box::new(|client| test(client, index_get::IndexGet)),
         Box::new(|client| test(client, index_create::IndexCreate)),
         Box::new(|client| test(client, stream::BulkStream)),
         Box::new(|client| test(client, stream_tiny_size_limit::BulkStreamTinySize)),
diff --git a/tests/run/src/document/mod.rs b/tests/run/src/document/mod.rs
index a7ef546133..4ef0c9e718 100644
--- a/tests/run/src/document/mod.rs
+++ b/tests/run/src/document/mod.rs
@@ -7,6 +7,7 @@ mod compile_test;
 
 mod delete;
 mod simple_index_get;
+mod simple_mapping;
 mod update_no_index;
 mod update_with_doc;
 mod update_with_inline_script;
@@ -15,6 +16,7 @@ mod update_with_script;
 pub fn tests() -> Vec<Test> {
     vec![
         Box::new(|client| test(client, simple_index_get::SimpleIndexGet)),
+        Box::new(|client| test(client, simple_mapping::SimpleMapping)),
         Box::new(|client| test(client, update_with_doc::UpdateWithDoc)),
         Box::new(|client| test(client, update_with_script::UpdateWithScript)),
         Box::new(|client| test(client, update_with_inline_script::UpdateWithInlineScript)),
diff --git a/tests/run/src/document/simple_mapping.rs b/tests/run/src/document/simple_mapping.rs
new file mode 100644
index 0000000000..d07b59b40b
--- /dev/null
+++ b/tests/run/src/document/simple_mapping.rs
@@ -0,0 +1,85 @@
+use elastic::error::Error;
+use elastic::prelude::*;
+use futures::Future;
+use run_tests::IntegrationTest;
+
+use serde_json::Value;
+
+#[derive(Debug, Clone, Copy)]
+pub struct SimpleMapping;
+
+#[derive(Debug, PartialEq, Serialize, Deserialize, ElasticType)]
+#[elastic(ty = "document", index = "simple_mapping")]
+pub struct Doc {
+    #[elastic(id)]
+    id: String,
+    timestamp: Date<DefaultDateMapping>,
+}
+
+impl IntegrationTest for SimpleMapping {
+    type Response = Value;
+
+    fn kind() -> &'static str {
+        "document"
+    }
+    fn name() -> &'static str {
+        "simple mapping"
+    }
+
+    // Ensure the index doesn't exist
+    fn prepare(&self, client: AsyncClient) -> Box<Future<Item = (), Error = Error>> {
+        let delete_res = client
+            .index(Doc::static_index())
+            .delete()
+            .send()
+            .map(|_| ());
+
+        Box::new(delete_res)
+    }
+
+    // Put the document mapping, then get it back from Elasticsearch
+    fn request(&self, client: AsyncClient) -> Box<Future<Item = Self::Response, Error = Error>> {
+        let create_index = client.index(Doc::static_index()).create().send().map(|_| ());
+
+        let put_mapping = client
+            .document::<Doc>()
+            .put_mapping()
+            .send();
+
+        let get_mapping = client
+            .request(IndicesGetMappingRequest::for_index_ty(Doc::static_index(), Doc::static_ty()))
+            .send()
+            .and_then(|res| res.into_response::<Value>());
+
+        Box::new(create_index.and_then(|_| put_mapping).and_then(|_| get_mapping))
+    }
+
+    // Ensure the response contains the expected document
+    fn assert_ok(&self, res: &Self::Response) -> bool {
+        let expected = json!({
+            "simple_mapping": {
+                "mappings": {
+                    "document": {
+                        "properties": {
+                            "id": {
+                                "type": "text",
+                                "fields": {
+                                    "keyword": {
+                                        "type": "keyword",
+                                        "ignore_above": 256
+                                    }
+                                }
+                            },
+                            "timestamp": {
+                                "type": "date",
+                                "format": "basic_date_time"
+                            }
+                        }
+                    }
+                }
+            }
+        });
+
+        res.to_string() == expected.to_string()
+    }
+}