Repository: opensearch-project/opensearch-remote-metadata-sdk
Pull request #156: Make generated responses robust to URL encoded id and index values
Author: login: dbwiddis
Labels: name: backport 2.19
name: backport 3.0
description: 3.0.0 Release
Created: 2025-04-27T19:07:04Z | Merged: 2025-04-28T18:52:48Z

### Description

Properly handles URL-encoded strings for ID and Index passed through to response

### Issues Resolved

Part of #154.

Related: 
- [x] request length validation https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/157
- [x] search request body validation https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/158

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).



Diff:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0db5d3e..ef52d964 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,11 +3,12 @@ All notable changes to this project are documented in this file.
 
 Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 
-## [Unreleased 3.0](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/compare/2.x...HEAD)
+## [Unreleased 3.0](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/compare/3.0...HEAD)
 ### Features
 ### Enhancements
 ### Bug Fixes
 - Directly return responses from Local Cluster client ([#141](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/141))
+- Make generated responses robust to URL encoded id and index values ([#156](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/156))
 - Properly handle remote client search failures with status codes ([#158](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/158))
 
 ### Infrastructure
diff --git a/ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java b/ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java
index e1e7d636..1cecb075 100644
--- a/ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java
+++ b/ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java
@@ -8,6 +8,7 @@
  */
 package org.opensearch.remote.metadata.client.impl;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -73,6 +74,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -190,14 +192,14 @@ public CompletionStage<PutDataObjectResponse> putDataObjectAsync(
                 final PutItemRequest putItemRequest = builder.build();
 
                 return dynamoDbAsyncClient.putItem(putItemRequest).thenApply(putItemResponse -> {
-                    String simulatedIndexResponse = simulateOpenSearchResponse(
-                        request.index(),
-                        id,
-                        source,
-                        sequenceNumber,
-                        Map.of("result", "created")
-                    );
                     try {
+                        String simulatedIndexResponse = simulateOpenSearchResponse(
+                            request.index(),
+                            id,
+                            source,
+                            sequenceNumber,
+                            Map.of("result", "created")
+                        );
                         return PutDataObjectResponse.builder().id(id).parser(SdkClientUtils.createParser(simulatedIndexResponse)).build();
                     } catch (IOException e) {
                         throw new OpenSearchStatusException("Failed to create parser for response", RestStatus.INTERNAL_SERVER_ERROR, e);
@@ -611,40 +613,32 @@ private Long initOrIncrementSeqNo(GetItemResponse getItemResponse) {
         return sequenceNumber;
     }
 
-    private String simulateOpenSearchResponse(
+    // package private for testing
+    static String simulateOpenSearchResponse(
         String index,
         String id,
         String source,
         Long sequenceNumber,
         Map<String, Object> additionalFields
-    ) {
-        Long seqNo = UNASSIGNED_SEQ_NO;
-        Long primaryTerm = UNASSIGNED_PRIMARY_TERM;
-        if (sequenceNumber != null) {
-            seqNo = sequenceNumber;
-            primaryTerm = DEFAULT_PRIMARY_TERM;
+    ) throws JsonProcessingException {
+        ObjectMapper mapper = new ObjectMapper();
+        Map<String, Object> response = new LinkedHashMap<>();
+        response.put("_index", index);
+        response.put("_id", id);
+        if (sequenceNumber == null) {
+            response.put("_primary_term", UNASSIGNED_PRIMARY_TERM);
+            response.put("_seq_no", UNASSIGNED_SEQ_NO);
+        } else {
+            response.put("_primary_term", DEFAULT_PRIMARY_TERM);
+            response.put("_seq_no", sequenceNumber);
         }
-        StringBuilder sb = new StringBuilder("{");
-        // Fields with a DDB counterpart
-        sb.append("\"_index\":\"").append(index).append("\",");
-        sb.append("\"_id\":\"").append(id).append("\",");
-        // Fields we must simulate using default values
-        sb.append("\"_primary_term\":").append(primaryTerm).append(",");
-        sb.append("\"_seq_no\":").append(seqNo).append(",");
-        sb.append("\"_version\":").append(-1).append(",");
-        sb.append("\"_shards\":").append(Strings.toString(MediaTypeRegistry.JSON, new ShardInfo())).append(",");
-        // Finish up
-        additionalFields.entrySet()
-            .stream()
-            .forEach(
-                e -> sb.append("\"")
-                    .append(e.getKey())
-                    .append("\":")
-                    .append(e.getValue() instanceof String ? ("\"" + e.getValue() + "\"") : e.getValue())
-                    .append(",")
-            );
-        sb.append("\"_source\":").append(source).append("}");
-        return sb.toString();
+        response.put("_version", -1);
+        response.put("_shards", new ShardInfo());
+        response.putAll(additionalFields);
+        if (source != null) {
+            response.put("_source", mapper.readTree(source));
+        }
+        return mapper.writeValueAsString(response);
     }
 
     private static void validateAwsParams(String clientType, String remoteMetadataEndpoint, String region, String serviceName) {
diff --git a/ddb-client/src/test/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClientTests.java b/ddb-client/src/test/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClientTests.java
index 239b2b1c..235fcfac 100644
--- a/ddb-client/src/test/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClientTests.java
+++ b/ddb-client/src/test/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClientTests.java
@@ -52,6 +52,7 @@
 import org.opensearch.remote.metadata.client.UpdateDataObjectResponse;
 import org.opensearch.remote.metadata.common.CommonValue;
 import org.opensearch.remote.metadata.common.ComplexDataObject;
+import org.opensearch.remote.metadata.common.SdkClientUtils;
 import org.opensearch.remote.metadata.common.TestDataObject;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.threadpool.ScalingExecutorBuilder;
@@ -76,6 +77,7 @@
 import org.mockito.MockitoAnnotations;
 
 import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.opensearch.remote.metadata.client.impl.DDBOpenSearchClient.simulateOpenSearchResponse;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -837,4 +839,67 @@ private Map<String, AttributeValue> getComplexDataSource() {
             Map.entry("testObject", AttributeValue.builder().m(Map.of("data", AttributeValue.builder().s("foo").build())).build())
         );
     }
+
+    @Test
+    public void testDeleteResponseWithQuotes() throws IOException {
+        // Test with double quote in ID
+        String json = simulateOpenSearchResponse("test-index", "\"", null, 0L, Map.of("result", "deleted"));
+
+        XContentParser parser = SdkClientUtils.createParser(json);
+        DeleteResponse response = DeleteResponse.fromXContent(parser);
+
+        assertEquals("\"", response.getId());
+        assertEquals("test-index", response.getIndex());
+        assertEquals("deleted", response.getResult().toString().toLowerCase());
+    }
+
+    @Test
+    public void testGetResponseWithBackslash() throws IOException {
+        // Test with backslash in ID
+        String source = "{\"field\":\"value\"}";
+        String json = simulateOpenSearchResponse("test-index", "\\", source, 0L, Map.of("found", true));
+
+        XContentParser parser = SdkClientUtils.createParser(json);
+        GetResponse response = GetResponse.fromXContent(parser);
+
+        assertEquals("\\", response.getId());
+        assertEquals("test-index", response.getIndex());
+        assertTrue(response.isExists());
+        assertNotNull(response.getSourceAsString());
+        assertEquals("{\"field\":\"value\"}", response.getSourceAsString());
+    }
+
+    @Test
+    public void testUpdateResponseWithSpecialChars() throws IOException {
+        // Test with multiple special characters in ID
+        String source = "{\"updated_field\":\"new_value\"}";
+        String json = simulateOpenSearchResponse("test-index", "<>?:\"{}|+", source, 0L, Map.of("result", "updated"));
+
+        XContentParser parser = SdkClientUtils.createParser(json);
+        UpdateResponse response = UpdateResponse.fromXContent(parser);
+
+        assertEquals("<>?:\"{}|+", response.getId());
+        assertEquals("test-index", response.getIndex());
+        assertEquals("updated", response.getResult().toString().toLowerCase());
+    }
+
+    @Test
+    public void testIndexResponseWithUrlEncodedChars() throws IOException {
+        // Test with URL-encoded characters in both index and ID
+        String source = "{\"new_field\":\"test_value\"}";
+        String json = simulateOpenSearchResponse(
+            "test%20index",
+            "%22%3C%3E", // URL encoded "<>"
+            source,
+            0L,
+            Map.of("result", "created")
+        );
+
+        XContentParser parser = SdkClientUtils.createParser(json);
+        IndexResponse response = IndexResponse.fromXContent(parser);
+
+        assertEquals("%22%3C%3E", response.getId());
+        assertEquals("test%20index", response.getIndex());
+        assertEquals("created", response.getResult().toString().toLowerCase());
+    }
 }


Reviews:
user: login: dhrubo-os
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T16:23:52Z
user: login: dhrubo-os
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T16:24:57Z
user: login: dbwiddis
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T16:51:46Z
user: login: dhrubo-os
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T18:26:48Z

Review comments:
diff_hunk: @@ -626,8 +628,9 @@ private String simulateOpenSearchResponse(
         }
         StringBuilder sb = new StringBuilder("{");
path: ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java
user: login: dhrubo-os
body: What do you think about using an Objectmapper here: `ObjectMapper mapper = new ObjectMapper();` ?

Using StringBuilder for JSON construction is error-prone.
created_at: 2025-04-28T16:23:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -626,8 +628,9 @@ private String simulateOpenSearchResponse(
         }
         StringBuilder sb = new StringBuilder("{");
path: ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java
user: login: dhrubo-os
body: May be something like:

```
static String simulateOpenSearchResponse(
        String index,
        String id,
        String source,
        Long sequenceNumber,
        Map<String, Object> additionalFields
    ) {
    ObjectMapper mapper = new ObjectMapper();
    Map<String, Object> response = new LinkedHashMap<>();
    
    response.put("_index", index);
    response.put("_id", id);
    response.put("_primary_term", sequenceNumber != null ? DEFAULT_PRIMARY_TERM : UNASSIGNED_PRIMARY_TERM);
    response.put("_seq_no", sequenceNumber != null ? sequenceNumber : UNASSIGNED_SEQ_NO);
    response.put("_version", -1);
    response.put("_shards", new ShardInfo());
    response.putAll(additionalFields);
    response.put("_source", mapper.readTree(source));
    
    return mapper.writeValueAsString(response);
}
```

?
created_at: 2025-04-28T16:24:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -626,8 +628,9 @@ private String simulateOpenSearchResponse(
         }
         StringBuilder sb = new StringBuilder("{");
path: ddb-client/src/main/java/org/opensearch/remote/metadata/client/impl/DDBOpenSearchClient.java
user: login: dbwiddis
body: As long as it passes the test cases I'm fine with that!
created_at: 2025-04-28T16:51:46Z
author_association: MEMBER

PR comments:
user: login: codecov[bot]
created_at: 2025-04-27T19:11:55Z
body: ## [Codecov](https://app.codecov.io/gh/opensearch-project/opensearch-remote-metadata-sdk/pull/156?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=opensearch-project) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 83.17%. Comparing base [(`c7ae104`)](https://app.codecov.io/gh/opensearch-project/opensearch-remote-metadata-sdk/commit/c7ae10438c0f2acb3d08142e9a34a6a8d965f23a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=opensearch-project) to head [(`da3cc42`)](https://app.codecov.io/gh/opensearch-project/opensearch-remote-metadata-sdk/commit/da3cc4253ca43af126f0d6ec665c44c4f65be9e1?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=opensearch-project).
> Report is 7 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff              @@
##               main     #156      +/-   ##
============================================
- Coverage     83.25%   83.17%   -0.08%     
+ Complexity      389      388       -1     
============================================
  Files            25       25              
  Lines          1475     1468       -7     
  Branches        122      122              
============================================
- Hits           1228     1221       -7     
  Misses          188      188              
  Partials         59       59              
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/opensearch-project/opensearch-remote-metadata-sdk/pull/156?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=opensearch-project).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=opensearch-project).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: opensearch-trigger-bot[bot]
created_at: 2025-04-28T18:53:40Z
body: The backport to `3.0` failed:
```
The process '/usr/bin/git' failed with exit code 128
```
To backport manually, run these commands in your terminal:
```bash
# Navigate to the root of your repository
cd $(git rev-parse --show-toplevel)
# Fetch latest updates from GitHub
git fetch
# Create a new working tree
git worktree add ../.worktrees/opensearch-remote-metadata-sdk/backport-3.0 3.0
# Navigate to the new working tree
pushd ../.worktrees/opensearch-remote-metadata-sdk/backport-3.0
# Create a new branch
git switch --create backport/backport-156-to-3.0
# Cherry-pick the merged commit of this pull request and resolve the conflicts
git cherry-pick -x --mainline 1 3d24da321a036bc0a1094145244901cd2e555d61
# Push it to GitHub
git push --set-upstream origin backport/backport-156-to-3.0
# Go back to the original working tree
popd
# Delete the working tree
git worktree remove ../.worktrees/opensearch-remote-metadata-sdk/backport-3.0
```
Then, create a pull request where the `base` branch is `3.0` and the `compare`/`head` branch is `backport/backport-156-to-3.0`.
author_association: CONTRIBUTOR
user: login: opensearch-trigger-bot[bot]
created_at: 2025-05-05T19:09:05Z
body: The backport to `2.19` failed:
```
The process '/usr/bin/git' failed with exit code 128
```
To backport manually, run these commands in your terminal:
```bash
# Navigate to the root of your repository
cd $(git rev-parse --show-toplevel)
# Fetch latest updates from GitHub
git fetch
# Create a new working tree
git worktree add ../.worktrees/opensearch-remote-metadata-sdk/backport-2.19 2.19
# Navigate to the new working tree
pushd ../.worktrees/opensearch-remote-metadata-sdk/backport-2.19
# Create a new branch
git switch --create backport/backport-156-to-2.19
# Cherry-pick the merged commit of this pull request and resolve the conflicts
git cherry-pick -x --mainline 1 3d24da321a036bc0a1094145244901cd2e555d61
# Push it to GitHub
git push --set-upstream origin backport/backport-156-to-2.19
# Go back to the original working tree
popd
# Delete the working tree
git worktree remove ../.worktrees/opensearch-remote-metadata-sdk/backport-2.19
```
Then, create a pull request where the `base` branch is `2.19` and the `compare`/`head` branch is `backport/backport-156-to-2.19`.
author_association: CONTRIBUTOR