Repository: smithy-lang/smithy (java)

Issue:
triple-slash docs accidentally merged when put in the wrong position
When you do this:

```smithy
$version: "2"

namespace demo

structure Hello {
    @required
    /// totally a string
    s: String
    /// a number!
    i: Integer
}
```

The result will end up concatenating both doc strings and putting them on `i`:

```json
{
    "smithy": "2.0",
    "shapes": {
        "demo#Hello": {
            "type": "structure",
            "members": {
                "s": {
                    "target": "smithy.api#String",
                    "traits": {
                        "smithy.api#required": {}
                    }
                },
                "i": {
                    "target": "smithy.api#Integer",
                    "traits": {
                        "smithy.api#documentation": "totally a string\na number!"
                    }
                }
            }
        }
    }
}
```

whereas if `i` has no `///` on it, you'll rightfully get a warning about the wrong position:

<img width="894" alt="image" src="https://github.com/user-attachments/assets/400fb0fc-19e5-4a1d-a3bd-61184531ae5e">

I think the actual behavior in the original snippet should be:

- produce the same type of warning as in the screenshot
- don't concat the docs (ignore the comment from `totally a string` completely)

Pull request description:
Fix multiple doc comment parse issues
This commit fixes several issues where documentation comments were not emitting events individually, an issue where a valid documentation comment was being declared invalid, and an invalid documentation comment was being prepended to a valid documtation comment on the next member.

Resolves #2388 

---
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.


Patch:
diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java
index d30fe637b..786f5dba6 100644
--- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java
+++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java
@@ -149,7 +149,7 @@ final class IdlInternalTokenizer extends DefaultTokenizer {
         }
     }
 
-    private void clearDocCommentLinesForBr() {
+    void clearDocCommentLinesForBr() {
         if (!docCommentLines.isEmpty()) {
             validationEventListener.accept(LoaderUtils.emitBadDocComment(getCurrentTokenLocation(),
                                                                          removePendingDocCommentLines()));
diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java
index 562c07e8a..81fbda08e 100644
--- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java
+++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java
@@ -448,7 +448,7 @@ final class IdlModelLoader {
             try {
                 tokenizer.skipWsAndDocs();
                 String docLines = tokenizer.removePendingDocCommentLines();
-                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, false);
                 if (docLines != null) {
                     traits.add(new IdlTraitParser.Result(DocumentationTrait.ID.toString(),
                                                          new StringNode(docLines, possibleDocCommentLocation),
@@ -467,7 +467,7 @@ final class IdlModelLoader {
         while (tokenizer.hasNext()) {
             try {
                 boolean hasDocComment = tokenizer.getCurrentToken() == IdlToken.DOC_COMMENT;
-                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, false);
                 if (parseShapeDefinition(traits, hasDocComment)) {
                     parseShapeOrApply(traits);
                 }
@@ -672,7 +672,7 @@ final class IdlModelLoader {
 
         while (tokenizer.getCurrentToken() != IdlToken.EOF && tokenizer.getCurrentToken() != IdlToken.RBRACE) {
             List<IdlTraitParser.Result> memberTraits = IdlTraitParser
-                    .parseDocsAndTraitsBeforeShape(this);
+                    .parseDocsAndTraitsBeforeShape(this, true);
             SourceLocation memberLocation = tokenizer.getCurrentTokenLocation();
             tokenizer.expect(IdlToken.IDENTIFIER);
             String memberName = internString(tokenizer.getCurrentTokenLexeme());
@@ -723,6 +723,9 @@ final class IdlModelLoader {
         Set<String> definedMembers = new HashSet<>();
 
         tokenizer.skipWsAndDocs();
+        // Clear out any doc comments between the shape type or
+        // identifier and the opening LBRACE token.
+        tokenizer.clearDocCommentLinesForBr();
         tokenizer.expect(IdlToken.LBRACE);
         tokenizer.next();
         tokenizer.skipWs();
@@ -740,7 +743,7 @@ final class IdlModelLoader {
         ShapeId parent = operation.toShapeId();
 
         // Parse optional member traits.
-        List<IdlTraitParser.Result> memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+        List<IdlTraitParser.Result> memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, true);
         SourceLocation memberLocation = tokenizer.getCurrentTokenLocation();
 
         // Handle the case of a dangling documentation comment followed by "}".
@@ -988,7 +991,7 @@ final class IdlModelLoader {
     }
 
     private ShapeId parseInlineStructure(String name, IdlTraitParser.Result defaultTrait) {
-        List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+        List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, true);
         if (defaultTrait != null) {
             traits.add(defaultTrait);
         }
diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java
index 82b52de57..90aabfa3a 100644
--- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java
+++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java
@@ -69,7 +69,7 @@ final class IdlTraitParser {
      * @param loader IDL parser.
      * @return Return the parsed traits.
      */
-    static List<Result> parseDocsAndTraitsBeforeShape(IdlModelLoader loader) {
+    static List<Result> parseDocsAndTraitsBeforeShape(IdlModelLoader loader, boolean isParsingMember) {
         IdlInternalTokenizer tokenizer = loader.getTokenizer();
         tokenizer.skipWs();
 
@@ -91,6 +91,12 @@ final class IdlTraitParser {
             traits.add(docComment);
         }
         tokenizer.skipWsAndDocs();
+        // If there's a doc comment between traits and their target member,
+        // clear them so they're not prepended to a doc comment on the next
+        // member shape.
+        if (isParsingMember) {
+            tokenizer.clearDocCommentLinesForBr();
+        }
 
         return traits;
     }


Test patch:
diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-invalid-documentation-comments.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-invalid-documentation-comments.errors
index 56aced838..978716049 100644
--- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-invalid-documentation-comments.errors
+++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-invalid-documentation-comments.errors
@@ -1,7 +1,9 @@
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 1 (before control) | Model.BadDocumentationComment
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 2 (before namespace) | Model.BadDocumentationComment
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 3 (before use) | Model.BadDocumentationComment
-[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 4 (inside trait)\nInvalid 5 (not before traits)\nValid docs for Foo | Model.BadDocumentationComment
-[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 6 (not before traits)\nInvalid 7 (inside shape line)\nInvalid 8 (dangling) | Model.BadDocumentationComment
-[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 9 (not before traits)\nInvalid 10 (dangling) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 4 (inside trait)\nInvalid 5 (not before traits) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 6 (not before traits)\nInvalid 7 (inside shape line) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 8 (dangling) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 9 (not before traits) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 10 (dangling) | Model.BadDocumentationComment
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 11 (dangling) | Model.BadDocumentationComment
diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json
index ea06639a0..fa4fc6697 100644
--- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json
+++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json
@@ -30,6 +30,12 @@
                         "smithy.api#documentation": "Docs on another member!"
                     }
                 },
+                "boot": {
+                    "target": "smithy.api#String",
+                    "traits": {
+                        "smithy.api#documentation": "I'll only have this documentation."
+                    }
+                },
                 "bam": {
                     "target": "smithy.api#String"
                 }
diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy
index 2ded3aa19..3f96dc95c 100644
--- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy
+++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy
@@ -29,6 +29,9 @@ structure MyStructure {
     /// IMPORTANT: These docs are ignored since they come after traits!
     baz: String,
 
+    /// I'll only have this documentation.
+    boot: String
+
     // no docs.
     bam: String,
 }