Repository: open-telemetry/opentelemetry-collector-contrib (Go)

Issue:
[exporter/elasticsearch] Regression: Duplicate Key in JSON After #31694
### Component(s)

exporter/elasticsearch

### What happened?

## Description
Previously, the request to ELK contained a well-structured JSON object with unique keys:

```
{
  ...
  "host": {
    "name": "hostname"
  },
  "os": {
    "description": "Windows 11 10.0",
    "type": "windows"
  },
  ...
}
```
However, after the change, the JSON format has been modified, resulting in a duplicate host key:
```
{
  ...
  "host": {
    "hostname": "hostname",
    "os": {
      "full": "Windows 11 10.0",
      "platform": "windows"
    }
  },
  ...
  "host": {  // Duplicate key
    "os": {
      "type": "windows"
    }
  },
  ...
}
```

This duplicate key renders the JSON invalid, leading to parsing errors from ELK:

```
{
  "error": {
    "reason": "failed to parse",
    "type": "mapper_parsing_exception"
  },
  "status": 400
}
```
## Steps to Reproduce
add the following resource attribute to OTLP request
host.name: Str(hostname)
os.description: Str(Windows 11 10.0)
os.type: Str(windows)

## Expected Result
ELK successfully parses the JSON data and processes it without errors.

## Actual Result
ELK encounters a parsing error due to the duplicate host key, and the data is not processed successfully.

### Collector version

0.101.0

### Environment information

## Environment
Not relevant

### OpenTelemetry Collector configuration

```yaml
exporters:
  elasticsearch/logs:
    ...
    mapping:
      mode: ecs
    ....
```


### Log output

```shell
2024-05-27T06:13:19.135Z	debug	elasticsearchexporter@v0.101.0/elasticsearch_bulk.go:45	Request roundtrip completed.	{"kind": "exporter", "data_type": "logs", "name": "elasticsearch/logs", "path": "/_bulk", "method": "POST", "duration": 0.057272404, "status": "200 OK"}
2024-05-27T06:13:19.136294606Z 2024-05-27T06:13:19.136Z	error	elasticsearchexporter@v0.101.0/elasticsearch_bulk.go:318	Drop docs: failed to index: struct { Type string "json:\"type\""; Reason string "json:\"reason\"" }{Type:"mapper_parsing_exception", Reason:"failed to parse"}	{"kind": "exporter", "data_type": "logs", "name": "elasticsearch/logs", "status": 400}
2024-05-27T06:13:19.136315906Z github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter.(*worker).flush
2024-05-27T06:13:19.136322306Z 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter@v0.101.0/elasticsearch_bulk.go:318
2024-05-27T06:13:19.136327106Z github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter.(*worker).run
2024-05-27T06:13:19.136332306Z 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter@v0.101.0/elasticsearch_bulk.go:304
2024-05-27T06:13:19.136337506Z github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter.newBulkIndexer.func1
2024-05-27T06:13:19.136342306Z 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter@v0.101.0/elasticsearch_bulk.go:220
```



### Additional context

https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/31694#issuecomment-2132746471


Discussion:
Pinging code owners:
- exporter/elasticsearch: @JaredTan95 @ycombinator @carsonip

 See [Adding Labels via Comments](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#adding-labels-via-comments) if you do not have permissions to add labels yourself.

Thanks for the detailed explanation and investigation, @BenEfrati!

_Removing `needs triage` as this is a valid bug, and has a clear reference to where the bug occurs._



Patch:
diff --git a/.chloggen/fix_esexp-mapping.yaml b/.chloggen/fix_esexp-mapping.yaml
new file mode 100644
index 0000000000000..917d63244a8a1
--- /dev/null
+++ b/.chloggen/fix_esexp-mapping.yaml
@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Duplicate Key in JSON 
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33454]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]
diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go
index b3e8829a3006d..1bb8fd2c4b437 100644
--- a/exporter/elasticsearchexporter/model.go
+++ b/exporter/elasticsearchexporter/model.go
@@ -82,6 +82,11 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord
 	}
 
 	var buf bytes.Buffer
+	if m.dedup {
+		document.Dedup()
+	} else if m.dedot {
+		document.Sort()
+	}
 	err := document.Serialize(&buf, m.dedot)
 	return buf.Bytes(), err
 }
@@ -104,12 +109,6 @@ func (m *encodeModel) encodeLogDefaultMode(resource pcommon.Resource, record plo
 	document.AddAttributes("Resource", resource.Attributes())
 	document.AddAttributes("Scope", scopeToAttributes(scope))
 
-	if m.dedup {
-		document.Dedup()
-	} else if m.dedot {
-		document.Sort()
-	}
-
 	return document
 
 }


Test patch:
diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go
index 0f2e512358c3a..eea08bdcdd0ea 100644
--- a/exporter/elasticsearchexporter/exporter_test.go
+++ b/exporter/elasticsearchexporter/exporter_test.go
@@ -43,15 +43,9 @@ func TestExporterLogs(t *testing.T) {
 		server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) {
 			rec.Record(docs)
 
-			var expectedDoc, actualDoc map[string]any
-			expected := []byte(`{"attrKey1":"abc","attrKey2":"def","application":"myapp","service":{"name":"myservice"},"error":{"stacktrace":"no no no no"},"agent":{"name":"otlp"},"@timestamp":"1970-01-01T00:00:00.000000000Z","message":"hello world"}`)
-			err := json.Unmarshal(expected, &expectedDoc)
-			require.NoError(t, err)
-
-			actual := docs[0].Document
-			err = json.Unmarshal(actual, &actualDoc)
-			require.NoError(t, err)
-			assert.Equal(t, expectedDoc, actualDoc)
+			expected := `{"@timestamp":"1970-01-01T00:00:00.000000000Z","agent":{"name":"otlp"},"application":"myapp","attrKey1":"abc","attrKey2":"def","error":{"stacktrace":"no no no no"},"message":"hello world","service":{"name":"myservice"}}`
+			actual := string(docs[0].Document)
+			assert.Equal(t, expected, actual)
 
 			return itemsAllOK(docs)
 		})
diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go
index 266c5dfecb596..a8c652a64485e 100644
--- a/exporter/elasticsearchexporter/model_test.go
+++ b/exporter/elasticsearchexporter/model_test.go
@@ -218,6 +218,48 @@ func TestEncodeEvents(t *testing.T) {
 	}
 }
 
+func TestEncodeLogECSModeDuplication(t *testing.T) {
+	resource := pcommon.NewResource()
+	err := resource.Attributes().FromRaw(map[string]any{
+		semconv.AttributeServiceName:    "foo.bar",
+		semconv.AttributeHostName:       "localhost",
+		semconv.AttributeServiceVersion: "1.1.0",
+		semconv.AttributeOSType:         "darwin",
+		semconv.AttributeOSDescription:  "Mac OS Mojave",
+		semconv.AttributeOSName:         "Mac OS X",
+		semconv.AttributeOSVersion:      "10.14.1",
+	})
+	require.NoError(t, err)
+
+	want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
+	require.NoError(t, err)
+
+	resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags)
+	err = resourceContainerImageTags.FromRaw([]any{"v3.4.0"})
+	require.NoError(t, err)
+
+	scope := pcommon.NewInstrumentationScope()
+
+	record := plog.NewLogRecord()
+	err = record.Attributes().FromRaw(map[string]any{
+		"event.name": "user-password-change",
+	})
+	require.NoError(t, err)
+	observedTimestamp := pcommon.Timestamp(1710273641123456789)
+	record.SetObservedTimestamp(observedTimestamp)
+
+	m := encodeModel{
+		mode:  MappingECS,
+		dedot: true,
+		dedup: true,
+	}
+	doc, err := m.encodeLog(resource, record, scope)
+	require.NoError(t, err)
+
+	assert.Equal(t, want, string(doc))
+
+}
+
 func TestEncodeLogECSMode(t *testing.T) {
 	resource := pcommon.NewResource()
 	err := resource.Attributes().FromRaw(map[string]any{