Repository: awsdocs/aws-doc-sdk-examples-tools

Issue:
Add metadata to track "AI Level"
How much AI:

- None - no AI was used.
    - Claude, ChatGPT, or other "chat" interfaces may have been used for reference, but their output was not included directly in the code.
- Some - AI was consulted while writing this section, and some words or phrases may have been copied, but the bulk was written by the human author.
    - Copilot, CodeWhisperer, inline AI, or `ailly --edit` may have been used for some code.
- Most - a human directed an AI to create this section. While they may have applied some editing, they have reviewed and revised it for accuracy.
    - Significant portions of code were written by an LLM, under the direction of _and reviewed by_ a human.
    - Generating an entire file with Ailly, and then editing it.
- All - this content was created entirely by AI, and no human has reviewed it before publishing.

(The line between some and most is fuzzy- you'll know it when you do it.)

Discussion:

Patch:
diff --git a/aws_doc_sdk_examples_tools/metadata.py b/aws_doc_sdk_examples_tools/metadata.py
index 4e771a8..5769980 100755
--- a/aws_doc_sdk_examples_tools/metadata.py
+++ b/aws_doc_sdk_examples_tools/metadata.py
@@ -6,7 +6,7 @@ from __future__ import annotations
 
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Set, Union, Iterable, TypedDict
+from typing import Any, Dict, Literal, List, Optional, Set, Union, Iterable, TypedDict
 from os.path import splitext
 from .project_validator import ValidationConfig
 
@@ -51,13 +51,26 @@ class Excerpt:
     snippet_tags: List[str]
     # A path within the repo to extract the entire file as a snippet.
     snippet_files: List[str] = field(default_factory=list)
+    # The amount to which generative AI was used to create this content.
+    #
+    # none: No GenAI generated content is included in this excerpt, but it may have been consulted for reference.
+    # some: A human wrote this content, though some portions may be copied or inserted from a GenAI tool.
+    # most: The bulk of this content was written by a GenAI tool, though a human has edited and reviewed it for accuracy.
+    # all: This content was entirely written by GenAI, and has not been reviewed by a human.
+    genai: Literal["none", "some", "most", "all"] = "none"
 
     @classmethod
-    def from_yaml(cls, yaml: Any) -> "Excerpt":
+    def from_yaml(cls, yaml: Any) -> tuple["Excerpt", MetadataErrors]:
         description = yaml.get("description")
         snippet_files = [str(file) for file in yaml.get("snippet_files", [])]
         snippet_tags = [str(tag) for tag in yaml.get("snippet_tags", [])]
-        return cls(description, snippet_tags, snippet_files)
+        genai = yaml.get("genai", "none")
+
+        errors = MetadataErrors()
+        if genai not in {"none", "some", "most", "all"}:
+            errors.append(metadata_errors.FieldError(field="genai", value=genai))
+
+        return (cls(description, snippet_tags, snippet_files, genai), errors)
 
 
 @dataclass
@@ -106,7 +119,11 @@ class Version:
                     )
                 )
 
-        excerpts = [Excerpt.from_yaml(excerpt) for excerpt in yaml.get("excerpts", [])]
+        excerpts = []
+        for excerpt in yaml.get("excerpts", []):
+            parsed, parse_errors = Excerpt.from_yaml(excerpt)
+            excerpts.append(parsed)
+            errors.extend(parse_errors)
 
         if len(excerpts) == 0 and block_content is None:
             errors.append(metadata_errors.MissingBlockContentAndExcerpt())
@@ -129,6 +146,10 @@ class Version:
         if block_content is not None and block_content not in cross_content_blocks:
             errors.append(metadata_errors.MissingCrossContent(block=block_content))
 
+        for error in errors:
+            if hasattr(error, "sdk_version"):
+                error.sdk_version = sdk_version
+
         return (
             cls(
                 sdk_version,
@@ -463,10 +484,11 @@ def parse(
     sdks: Dict[str, Sdk],
     services: Dict[str, Service],
     blocks: Set[str],
-    validation: ValidationConfig,
+    validation: Optional[ValidationConfig],
 ) -> tuple[List[Example], MetadataErrors]:
     examples: List[Example] = []
     errors = MetadataErrors()
+    validation = ValidationConfig() if validation is None else validation
     for id in yaml:
         example, example_errors = Example.from_yaml(
             yaml[id], sdks, services, blocks, validation
@@ -521,7 +543,7 @@ def main():
     )
     with open(path) as file:
         meta = yaml.safe_load(file)
-    (examples, errors) = parse(path.name, meta, {}, {}, set())
+    (examples, errors) = parse(path.name, meta, {}, {}, set(), ValidationConfig())
     if len(errors) > 0:
         print(f"{errors}")
     else:


Test patch:
diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py
index 3180c01..0a4295b 100644
--- a/aws_doc_sdk_examples_tools/metadata_test.py
+++ b/aws_doc_sdk_examples_tools/metadata_test.py
@@ -454,6 +454,7 @@ def test_verify_load_successful():
                         description="Descriptive",
                         snippet_files=[],
                         snippet_tags=["javascript.snippet.tag"],
+                        genai="some",
                     )
                 ],
                 sdkguide=None,
@@ -554,18 +555,19 @@ def test_verify_load_successful():
                     id="sqs_WrongServiceSlug",
                     language="Perl",
                     guide="https://docs.aws.amazon.com/absolute/link-to-my-guide",
+                    sdk_version=1,
                 ),
                 metadata_errors.MissingBlockContentAndExcerpt(
                     file="errors_metadata.yaml",
                     id="sqs_WrongServiceSlug",
                     language="Perl",
-                    sdk_version=None,
+                    sdk_version=1,
                 ),
                 metadata_errors.APIExampleCannotAddService(
                     file="errors_metadata.yaml",
                     id="sqs_WrongServiceSlug",
                     language="Perl",
-                    sdk_version=None,
+                    sdk_version=1,
                 ),
                 metadata_errors.ServiceNameFormat(
                     file="errors_metadata.yaml",
@@ -583,7 +585,7 @@ def test_verify_load_successful():
                     file="errors_metadata.yaml",
                     id="sns_TestExample",
                     language="Java",
-                    sdk_version=None,
+                    sdk_version=2,
                 ),
                 # example_errors.MissingSnippetTag(
                 #     file="errors_metadata.yaml",
@@ -604,23 +606,32 @@ def test_verify_load_successful():
                     sdk_version=2,
                     link="github/link/to/README.md",
                 ),
+                metadata_errors.FieldError(
+                    file="errors_metadata.yaml",
+                    id="sns_TestExample2",
+                    field="genai",
+                    value="so much",
+                    language="Java",
+                    sdk_version=2,
+                ),
                 metadata_errors.BlockContentAndExcerptConflict(
                     file="errors_metadata.yaml",
                     id="cross_TestExample_Versions",
                     language="Java",
-                    sdk_version=None,
+                    sdk_version=2,
                 ),
                 metadata_errors.MissingCrossContent(
                     file="errors_metadata.yaml",
                     id="cross_TestExample_Missing",
                     language="Java",
-                    sdk_version=None,
+                    sdk_version=2,
                     block="missing_block_content.xml",
                 ),
                 metadata_errors.MissingBlockContentAndExcerpt(
                     file="errors_metadata.yaml",
                     id="snsBadFormat",
                     language="Java",
+                    sdk_version=2,
                 ),
                 metadata_errors.NameFormat(
                     file="errors_metadata.yaml",
@@ -639,6 +650,7 @@ def test_verify_load_successful():
                     file="formaterror_metadata.yaml",
                     id="cross_TestExample",
                     language="Java",
+                    sdk_version=2,
                     service="garbage",
                 ),
             ],
diff --git a/aws_doc_sdk_examples_tools/test_resources/errors_metadata.yaml b/aws_doc_sdk_examples_tools/test_resources/errors_metadata.yaml
index b876901..ca6b98d 100644
--- a/aws_doc_sdk_examples_tools/test_resources/errors_metadata.yaml
+++ b/aws_doc_sdk_examples_tools/test_resources/errors_metadata.yaml
@@ -35,7 +35,7 @@ sns_TestExample:
         - sdk_version: 2
           sdkguide: http://sdk/guide/url
   services:
-    sns: {TestAction}
+    sns: { TestAction }
 sns_TestExample2:
   title: Test title
   title_abbrev: Test title abbrev
@@ -47,11 +47,12 @@ sns_TestExample2:
         - sdk_version: 2
           github: github/link/to/README.md
           excerpts:
+            - genai: so much
             - snippet_tags:
                 - this.snippet.does.not.exist
   services:
-    garbled: {TestAction}
-    sns: {TestAction}
+    garbled: { TestAction }
+    sns: { TestAction }
 cross_TestExample_Versions:
   title: Test title
   title_abbrev: Test title abbrev
@@ -95,4 +96,4 @@ snsBadFormat:
         - sdk_version: 2
           github: github/link
   services:
-    sns: {TestAction}
+    sns: { TestAction }
diff --git a/aws_doc_sdk_examples_tools/test_resources/valid_metadata.yaml b/aws_doc_sdk_examples_tools/test_resources/valid_metadata.yaml
index 1b13257..f3ca640 100644
--- a/aws_doc_sdk_examples_tools/test_resources/valid_metadata.yaml
+++ b/aws_doc_sdk_examples_tools/test_resources/valid_metadata.yaml
@@ -20,6 +20,7 @@ sns_TestExample:
         - sdk_version: 3
           excerpts:
             - description: Descriptive
+              genai: some
               snippet_tags:
                 - javascript.snippet.tag
           add_services: