Repository: microsoft/kiota (csharp)

Issue:
As a developer, I want to have a sliced OpenAPI document with only the endpoints and components used by my solution when generating a plugin
### Is your feature request related to a problem? Please describe the problem.

Kiota produces large OpenAPI documents even when only one endpoint is selected because it doesn’t remove unused components.

### Client library/SDK language

None

### Describe the solution you'd like

When generating plugin and clients, Kiota should be able to trim unused components from the original OpenAPI document to generate a clean sliced OpenAPI document with only the information that is important for my solution.

For clients:
- Remove all unused components and keep all used components including schemas, parameters, responses.
- Remove all extensions (starts with x-) that don't match with our internal documentation of supported extensions in Kiota.

For plugins:
- Removes all unused components and keep only schemas and parameter of used components. For plugins we should also remove any responses components as this won't be used by orchestrators and can increase the size of the OpenAPI document.
- Remove all extensions (starts with x-) that don't match with our internal documentation of supported extensions in Kiota.

### Additional context

_No response_

Pull request description:
Fixes #4672

On plugin generation, this PR leverages the `OpenApi.Net` library to trim unused component in the document similar to Hidi using the `OpenApiFilterService.CreateFilteredDocument` operation.

It also replaces all responses in the input document with a single 2XX response to reduce the size.

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81dce71b69..96f37bf98b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed a bug where models would not be created when a multipart content schema existed with no encoding [#4734](https://github.com/microsoft/kiota/issues/4734)
 - Types generated by Kiota are now referenced with their full name to avoid namespace ambiguities [#4475](https://github.com/microsoft/kiota/issues/4475)
 - Fixes a bug where warnings about discriminator not being inherited were generated [#4761](https://github.com/microsoft/kiota/issues/4761)
+- Trims unused components from output openApi document when generating plugins [#4672](https://github.com/microsoft/kiota/issues/4672)
 
 ## [1.14.0] - 2024-05-02
 
diff --git a/src/Kiota.Builder/Plugins/PluginsGenerationService.cs b/src/Kiota.Builder/Plugins/PluginsGenerationService.cs
index 7a51fe4b2e..7b5daca469 100644
--- a/src/Kiota.Builder/Plugins/PluginsGenerationService.cs
+++ b/src/Kiota.Builder/Plugins/PluginsGenerationService.cs
@@ -51,7 +51,8 @@ public async Task GenerateManifestAsync(CancellationToken cancellationToken = de
         await using var fileWriter = new StreamWriter(descriptionStream);
 #pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task
         var descriptionWriter = new OpenApiYamlWriter(fileWriter);
-        OAIDocument.SerializeAsV3(descriptionWriter);
+        var trimmedPluginDocument = GetDocumentWithTrimmedComponentsAndResponses(OAIDocument);
+        trimmedPluginDocument.SerializeAsV3(descriptionWriter);
         descriptionWriter.Flush();
 
         // write the plugins
@@ -93,6 +94,55 @@ public async Task GenerateManifestAsync(CancellationToken cancellationToken = de
             await writer.FlushAsync(cancellationToken).ConfigureAwait(false);
         }
     }
+
+
+    private OpenApiDocument GetDocumentWithTrimmedComponentsAndResponses(OpenApiDocument doc)
+    {
+        // ensure the info and components are not null
+        doc.Info ??= new OpenApiInfo();
+        doc.Components ??= new OpenApiComponents();
+
+        if (string.IsNullOrEmpty(doc.Info?.Version)) // filtering fails if there's no version.
+            doc.Info!.Version = "1.0";
+
+        //empty out all the responses with a single empty 2XX
+        foreach (var operation in doc.Paths.SelectMany(static item => item.Value.Operations.Values))
+        {
+            operation.Responses = new OpenApiResponses()
+            {
+                {
+                    "2XX",new OpenApiResponse
+                    {
+                        Content = new Dictionary<string, OpenApiMediaType>
+                        {
+                            {
+                                "text/plain", new OpenApiMediaType
+                                {
+                                    Schema = new OpenApiSchema
+                                    {
+                                        Type = "string"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
+        // remove unused components using the OpenApi.Net
+        var requestUrls = new Dictionary<string, List<string>>();
+        var basePath = doc.GetAPIRootUrl(Configuration.OpenAPIFilePath);
+        foreach (var path in doc.Paths.Where(static path => path.Value.Operations.Count > 0))
+        {
+            var key = string.IsNullOrEmpty(basePath) ? path.Key : $"{basePath}/{path.Key.TrimStart(KiotaBuilder.ForwardSlash)}";
+            requestUrls[key] = path.Value.Operations.Keys.Select(static key => key.ToString().ToUpperInvariant()).ToList();
+        }
+
+        var predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source: doc);
+        return OpenApiFilterService.CreateFilteredDocument(doc, predicate);
+    }
+
     private PluginManifestDocument GetV1ManifestDocument(string openApiDocumentPath)
     {
         var descriptionForHuman = OAIDocument.Info?.Description.CleanupXMLString() is string d && !string.IsNullOrEmpty(d) ? d : $"Description for {OAIDocument.Info?.Title.CleanupXMLString()}";


Test patch:
diff --git a/tests/Kiota.Builder.Tests/Plugins/PluginsGenerationServiceTests.cs b/tests/Kiota.Builder.Tests/Plugins/PluginsGenerationServiceTests.cs
index 04e52db82c..d3b4e3cfbb 100644
--- a/tests/Kiota.Builder.Tests/Plugins/PluginsGenerationServiceTests.cs
+++ b/tests/Kiota.Builder.Tests/Plugins/PluginsGenerationServiceTests.cs
@@ -7,6 +7,8 @@
 using Kiota.Builder.Configuration;
 using Kiota.Builder.Plugins;
 using Microsoft.Extensions.Logging;
+using Microsoft.OpenApi.Models;
+using Microsoft.OpenApi.Readers;
 using Microsoft.OpenApi.Services;
 using Microsoft.Plugins.Manifest;
 using Moq;
@@ -109,4 +111,113 @@ public async Task GeneratesManifest()
     private const string ManifestFileName = "client-microsoft.json";
     private const string OpenAIPluginFileName = "openai-plugins.json";
     private const string OpenApiFileName = "client-openapi.yml";
+
+    [Fact]
+    public async Task GeneratesManifestAndCleansUpInputDescription()
+    {
+        var simpleDescriptionContent = @"openapi: 3.0.0
+info:
+  title: test
+  version: 1.0
+x-test-root-extension: test
+servers:
+  - url: http://localhost/
+    description: There's no place like home
+paths:
+  /test:
+    get:
+      description: description for test path
+      responses:
+        '200':
+          description: test
+        '400':
+          description: client error response
+  /test/{id}:
+    get:
+      description: description for test path with id
+      operationId: test.WithId
+      parameters:
+      - name: id
+        in: path
+        required: true
+        description: The id of the test
+        schema:
+          type: integer
+          format: int32
+      responses:
+        '200':
+          description: test
+        '500':
+          description: api error response
+components:
+  schemas:
+    microsoft.graph.entity:
+      title: entity
+      required:
+        - '@odata.type'
+      type: object
+      properties:
+        id:
+          type: string
+        '@odata.type':
+          type: string";
+        var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        var simpleDescriptionPath = Path.Combine(workingDirectory) + "description.yaml";
+        await File.WriteAllTextAsync(simpleDescriptionPath, simpleDescriptionContent);
+        var mockLogger = new Mock<ILogger<PluginsGenerationService>>();
+        var openAPIDocumentDS = new OpenApiDocumentDownloadService(_httpClient, mockLogger.Object);
+        var outputDirectory = Path.Combine(workingDirectory, "output");
+        var generationConfiguration = new GenerationConfiguration
+        {
+            OutputPath = outputDirectory,
+            OpenAPIFilePath = "openapiPath",
+            PluginTypes = [PluginType.Microsoft],
+            ClientClassName = "client",
+            ApiRootUrl = "http://localhost/", //Kiota builder would set this for us
+        };
+        var (openAPIDocumentStream, _) = await openAPIDocumentDS.LoadStreamAsync(simpleDescriptionPath, generationConfiguration, null, false);
+        var openApiDocument = await openAPIDocumentDS.GetDocumentFromStreamAsync(openAPIDocumentStream, generationConfiguration);
+        KiotaBuilder.CleanupOperationIdForPlugins(openApiDocument);
+        var urlTreeNode = OpenApiUrlTreeNode.Create(openApiDocument, Constants.DefaultOpenApiLabel);
+
+        var pluginsGenerationService = new PluginsGenerationService(openApiDocument, urlTreeNode, generationConfiguration, workingDirectory);
+        await pluginsGenerationService.GenerateManifestAsync();
+
+        Assert.True(File.Exists(Path.Combine(outputDirectory, ManifestFileName)));
+        Assert.True(File.Exists(Path.Combine(outputDirectory, OpenApiFileName)));
+
+        // Validate the v2 plugin
+        var manifestContent = await File.ReadAllTextAsync(Path.Combine(outputDirectory, ManifestFileName));
+        using var jsonDocument = JsonDocument.Parse(manifestContent);
+        var resultingManifest = PluginManifestDocument.Load(jsonDocument.RootElement);
+        Assert.NotNull(resultingManifest.Document);
+        Assert.Equal(OpenApiFileName, resultingManifest.Document.Runtimes.OfType<OpenApiRuntime>().First().Spec.Url);
+        Assert.Equal(2, resultingManifest.Document.Functions.Count);// all functions are generated despite missing operationIds
+        Assert.Empty(resultingManifest.Problems);// no problems are expected with names
+
+        var openApiReader = new OpenApiStreamReader();
+
+        // Validate the original file.
+        var originalOpenApiFile = File.OpenRead(simpleDescriptionPath);
+        var originalDocument = openApiReader.Read(originalOpenApiFile, out var originalDiagnostic);
+        Assert.Empty(originalDiagnostic.Errors);
+
+        Assert.Single(originalDocument.Components.Schemas);// one schema originally
+        Assert.Single(originalDocument.Extensions); // single unsupported extension at root
+        Assert.Equal(2, originalDocument.Paths.Count); // document has only two paths
+        Assert.Equal(2, originalDocument.Paths["/test"].Operations[OperationType.Get].Responses.Count); // 2 responses originally
+        Assert.Equal(2, originalDocument.Paths["/test/{id}"].Operations[OperationType.Get].Responses.Count); // 2 responses originally
+
+        // Validate the output open api file
+        var resultOpenApiFile = File.OpenRead(Path.Combine(outputDirectory, OpenApiFileName));
+        var resultDocument = openApiReader.Read(resultOpenApiFile, out var diagnostic);
+        Assert.Empty(diagnostic.Errors);
+
+        // Assertions / validations
+        Assert.Empty(resultDocument.Components.Schemas);// no schema is referenced. so ensure they are all removed
+        Assert.Empty(resultDocument.Extensions); // no extension at root (unsupported extension is removed)
+        Assert.Equal(2, resultDocument.Paths.Count); // document has only two paths
+        Assert.Single(resultDocument.Paths["/test"].Operations[OperationType.Get].Responses); // other responses are removed from the document
+        Assert.Single(resultDocument.Paths["/test/{id}"].Operations[OperationType.Get].Responses); // 2 responses originally
+    }
 }