Repository: Azure/azure-sdk-tools
Pull request #10437: [AI Triage] Prompt tweaks and Extra Payload attributes
Author: login: jeo02
Created: 2025-04-23T23:54:31Z | Merged: 2025-04-25T20:49:48Z

- Prompt tweaks matching the user prompts in the labeler for the answer service 
- Extra Payload attributes to match PR #10435 changes. 
- Added a new `EnableLabels` config
- Reduced logging noise from Azure Search


Diff:
diff --git a/tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs b/tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs
index 2f2d5968188..3a037f899d1 100644
--- a/tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs
+++ b/tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs
@@ -1,8 +1,5 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using System.ComponentModel;
+using Newtonsoft.Json;
 
 namespace IssueLabeler.Shared
 {
@@ -14,5 +11,13 @@ public class IssuePayload
         public string IssueUserLogin { get; set; }
         public string RepositoryName { get; set; }
         public string RepositoryOwnerName { get; set; }
+
+        [DefaultValue(true)]
+        [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate)]
+        public bool PredictLabels { get; set; }
+
+        [DefaultValue(true)]
+        [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate)]
+        public bool PredictAnswers { get; set; }
     }
 }
diff --git a/tools/issue-labeler/src/IssueLabelerService/Answers/OpenAiAnswerService.cs b/tools/issue-labeler/src/IssueLabelerService/Answers/OpenAiAnswerService.cs
index 0165e2e458c..2884e398e78 100644
--- a/tools/issue-labeler/src/IssueLabelerService/Answers/OpenAiAnswerService.cs
+++ b/tools/issue-labeler/src/IssueLabelerService/Answers/OpenAiAnswerService.cs
@@ -53,13 +53,18 @@ public async Task<AnswerOutput> AnswerQuery(IssuePayload issue, Dictionary<strin
 
             _logger.LogInformation($"Highest relevance score among the sources: {highestScore}");
 
-            // Makes it nicer for the model to read (Can probably be made more readable but oh well)
-            var printableIssues = string.Join("\n", issues.Select(r => JsonConvert.SerializeObject(r)));
-            var printableDocs = string.Join("\n", docs.Select(r => JsonConvert.SerializeObject(r)));
+            // Format issues 
+            var printableIssues = string.Join("\n\n", issues.Select(issue =>
+                $"Title: {issue.Title}\nDescription: {issue.chunk}\nURL: {issue.Url}\nScore: {issue.Score}"));
+
+            // Format documents 
+            var printableDocs = string.Join("\n\n", docs.Select(doc =>
+                $"Content: {doc.chunk}\nURL: {doc.Url}\nScore: {doc.Score}"));
 
             var replacementsUserPrompt = new Dictionary<string, string>
             {
-                { "Query", query },
+                { "Title", issue.Title },
+                { "Description", issue.Body },
                 { "PrintableDocs", printableDocs },
                 { "PrintableIssues", printableIssues }
             };
diff --git a/tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs b/tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs
index 04cbf60d62b..095c029ef5a 100644
--- a/tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs
+++ b/tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs
@@ -47,31 +47,42 @@ public async Task<ActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "P
             var config = _configurationService.GetForRepository($"{issue.RepositoryOwnerName}/{issue.RepositoryName}");
             Dictionary<string, string> labels;
 
-            try
+            // Enable labels if both the configuration enable labels and the issue predict labels are true
+            if(bool.Parse(config.EnableLabels) && issue.PredictLabels)
             {
-                // Get the labeler based on the configuration
-                var labeler = _labelers.GetLabeler(config);
-
-                // Predict labels for the issue
-                labels = await labeler.PredictLabels(issue);
-
-                // If no labels are returned, do not generate an answer
-                // Fixing the issue by replacing 'Length' with 'Count' for Dictionary
-                if (labels?.Count == 0)
+                try
                 {
-                    _logger.LogInformation($"No labels predicted for issue #{issue.IssueNumber} in repository {issue.RepositoryName}.");
+                    // Get the labeler based on the configuration
+                    var labeler = _labelers.GetLabeler(config);
+
+                    // Predict labels for the issue
+                    labels = await labeler.PredictLabels(issue);
+
+                    // If no labels are returned, do not generate an answer
+                    // Fixing the issue by replacing 'Length' with 'Count' for Dictionary
+                    if (labels?.Count == 0)
+                    {
+                        _logger.LogInformation($"No labels predicted for issue #{issue.IssueNumber} in repository {issue.RepositoryName}.");
+                        return EmptyResult;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError($"Error labeling issue #{issue.IssueNumber} in repository {issue.RepositoryName}: {ex.Message}{Environment.NewLine}\t{ex}{Environment.NewLine}");
                     return EmptyResult;
                 }
+
             }
-            catch (Exception ex)
+            else
             {
-                _logger.LogError($"Error labeling issue #{issue.IssueNumber} in repository {issue.RepositoryName}: {ex.Message}{Environment.NewLine}\t{ex}{Environment.NewLine}");
+                _logger.LogError($"Labeling is turned off for issue #{issue.IssueNumber} in repository {issue.RepositoryName}. Without Labels, answers will not be suggested.");
                 return EmptyResult;
             }
 
             TriageOutput result = new TriageOutput { Labels = labels.Values };
-
-            if(bool.Parse(config.EnableAnswers))
+            
+            // Enable answers if both the configuration enable answers and the issue predict answers are true
+            if(bool.Parse(config.EnableAnswers) && issue.PredictAnswers)
             {
                 try{
 
diff --git a/tools/issue-labeler/src/IssueLabelerService/Configuration/RepositoryConfiguration.cs b/tools/issue-labeler/src/IssueLabelerService/Configuration/RepositoryConfiguration.cs
index f115405b784..90613a35327 100644
--- a/tools/issue-labeler/src/IssueLabelerService/Configuration/RepositoryConfiguration.cs
+++ b/tools/issue-labeler/src/IssueLabelerService/Configuration/RepositoryConfiguration.cs
@@ -21,6 +21,7 @@ internal RepositoryConfiguration(IConfiguration config) =>
         public string DocumentIndexName => GetItem("DocumentIndexName");
         public string DocumentSemanticName => GetItem("DocumentSemanticName");
         public string EnableAnswers => GetItem("EnableAnswers");
+        public string EnableLabels => GetItem("EnableLabels");
         public string IssueIndexName => GetItem("IssueIndexName");
         public string IssueSemanticName => GetItem("IssueSemanticName");
         public string LabelPredictor => GetItem("LabelPredictor");
diff --git a/tools/issue-labeler/src/IssueLabelerService/Labelers/OpenAiLabeler.cs b/tools/issue-labeler/src/IssueLabelerService/Labelers/OpenAiLabeler.cs
index f26b4e01b9f..3d1a052c745 100644
--- a/tools/issue-labeler/src/IssueLabelerService/Labelers/OpenAiLabeler.cs
+++ b/tools/issue-labeler/src/IssueLabelerService/Labelers/OpenAiLabeler.cs
@@ -55,12 +55,12 @@ public async Task<Dictionary<string, string>> PredictLabels(IssuePayload issue)
             _logger.LogInformation($"Highest relevance score among the sources: {highestScore}");
 
             // Format issues 
-            var printableIssues = string.Join("\n", issues.Select(issue =>
-                $"Title: {issue.Title}\nDescription: {issue.chunk}\nService: {issue.Service}"));
+            var printableIssues = string.Join("\n\n", issues.Select(issue =>
+                $"Title: {issue.Title}\nDescription: {issue.chunk}\nService: {issue.Service}\nScore: {issue.Score}"));
 
             // Format documents 
-            var printableDocs = string.Join("\n", docs.Select(doc =>
-                $"Content: {doc.chunk}\nService: {doc.Service}"));
+            var printableDocs = string.Join("\n\n", docs.Select(doc =>
+                $"Content: {doc.chunk}\nService: {doc.Service}\nScore: {doc.Score}"));
 
             // Get labels for this repository
             var labels = await GetLabelsAsync(issue.RepositoryName);
@@ -180,7 +180,7 @@ private string GetCategoryLabelsForPrompt(IEnumerable<Label> labels, string repo
                 .Select(label => label.Name)
                 .ToList();
 
-            return "'" + string.Join("';'", categoryLabels) + "';";
+            return "['" + string.Join("', '", categoryLabels) + "'']";
         }
 
         private bool ValidateLabels(IEnumerable<Label> labels, Dictionary<string, string> labelsToValidate)
diff --git a/tools/issue-labeler/src/IssueLabelerService/TraigeRag.cs b/tools/issue-labeler/src/IssueLabelerService/TraigeRag.cs
index 0b6bbec6456..feaa5398b75 100644
--- a/tools/issue-labeler/src/IssueLabelerService/TraigeRag.cs
+++ b/tools/issue-labeler/src/IssueLabelerService/TraigeRag.cs
@@ -76,7 +76,6 @@ public TriageRag(ILogger<TriageRag> logger, AzureOpenAIClient openAiClient, Sear
             List<(T, double)> results = new List<(T, double)>();
             foreach (SearchResult<T> result in response.GetResults())
             {
-                _logger.LogInformation(result.SemanticSearch.RerankerScore.ToString());
                 results.Add((result.Document, result.SemanticSearch.RerankerScore ?? 0.0));
             }
 


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This pull request updates the issue labeler service to incorporate prompt tweaks and add extra payload attributes. Key changes include:
- Removal of verbose logging from the semantic search results.
- Enhancements to issue and document formatting with additional score information.
- Introduction of new configuration options (EnableLabels) and corresponding conditional logic in the labeling process.

### Reviewed Changes

Copilot reviewed 6 out of 6 changed files in this pull request and generated 1 comment.

<details>
<summary>Show a summary per file</summary>

| File | Description |
| ---- | ----------- |
| tools/issue-labeler/src/IssueLabelerService/TraigeRag.cs | Removed redundant logging to reduce noise. |
| tools/issue-labeler/src/IssueLabelerService/Labelers/OpenAiLabeler.cs | Updated issue and document formatting to include additional score data. |
| tools/issue-labeler/src/IssueLabelerService/Configuration/RepositoryConfiguration.cs | Added new configuration property EnableLabels. |
| tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs | Implemented conditional checks for enabling labels and improved error logging. |
| tools/issue-labeler/src/IssueLabelerService/Answers/OpenAiAnswerService.cs | Adjusted formatting for issues and documents in the answer output. |
| tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs | Added PredictLabels and PredictAnswers attributes with defaults. |
</details>





state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T23:54:53Z
user: login: jsquire
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-24T00:08:20Z
user: login: jeo02
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T17:33:59Z
user: login: jsquire
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-25T17:10:54Z

Review comments:
diff_hunk: @@ -47,31 +47,42 @@ public async Task<ActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "P
             var config = _configurationService.GetForRepository($"{issue.RepositoryOwnerName}/{issue.RepositoryName}");
             Dictionary<string, string> labels;
 
-            try
+            // Enable labels if both the configuration enable labels and the issue predict labels are true
+            if(bool.Parse(config.EnableLabels) && issue.PredictLabels)
             {
-                // Get the labeler based on the configuration
-                var labeler = _labelers.GetLabeler(config);
-
-                // Predict labels for the issue
-                labels = await labeler.PredictLabels(issue);
-
-                // If no labels are returned, do not generate an answer
-                // Fixing the issue by replacing 'Length' with 'Count' for Dictionary
-                if (labels?.Count == 0)
+                try
                 {
-                    _logger.LogInformation($"No labels predicted for issue #{issue.IssueNumber} in repository {issue.RepositoryName}.");
+                    // Get the labeler based on the configuration
+                    var labeler = _labelers.GetLabeler(config);
+
+                    // Predict labels for the issue
+                    labels = await labeler.PredictLabels(issue);
+
+                    // If no labels are returned, do not generate an answer
+                    // Fixing the issue by replacing 'Length' with 'Count' for Dictionary
+                    if (labels?.Count == 0)
+                    {
+                        _logger.LogInformation($"No labels predicted for issue #{issue.IssueNumber} in repository {issue.RepositoryName}.");
+                        return EmptyResult;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError($"Error labeling issue #{issue.IssueNumber} in repository {issue.RepositoryName}: {ex.Message}{Environment.NewLine}\t{ex}{Environment.NewLine}");
                     return EmptyResult;
                 }
+
             }
-            catch (Exception ex)
+            else
             {
-                _logger.LogError($"Error labeling issue #{issue.IssueNumber} in repository {issue.RepositoryName}: {ex.Message}{Environment.NewLine}\t{ex}{Environment.NewLine}");
+                _logger.LogError($"Labeling is turned off for issue #{issue.IssueNumber} in repository {issue.RepositoryName}. Wihtout Labels answer service will not be called.");
                 return EmptyResult;
path: tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs
user: login: Copilot
body: There is a spelling mistake in the error message. 'Wihtout' should be corrected to 'Without'.
```suggestion
                _logger.LogError($"Labeling is turned off for issue #{issue.IssueNumber} in repository {issue.RepositoryName}. Without Labels answer service will not be called.");
```
created_at: 2025-04-23T23:54:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -75,8 +75,7 @@ public async Task<ActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "P
             }
             else
             {
-                _logger.LogError($"Labeling is turned off for issue #{issue.IssueNumber} in repository {issue.RepositoryName}. Wihtout Labels answer service will not be called.");
-                return EmptyResult;
+                _logger.LogError($"Labeling is turned off for issue #{issue.IssueNumber} in repository {issue.RepositoryName}. Without Labels answer service will not be called.");
path: tools/issue-labeler/src/IssueLabelerService/AzureSdkIssueLabelerService.cs
user: login: jsquire
body: ```suggestion
                _logger.LogError($"Labeling is turned off for issue #{issue.IssueNumber} in repository {issue.RepositoryName}. Without Labels, answers will not be suggested.");
```
created_at: 2025-04-24T00:04:27Z
author_association: MEMBER
diff_hunk: @@ -14,5 +11,13 @@ public class IssuePayload
         public string IssueUserLogin { get; set; }
         public string RepositoryName { get; set; }
         public string RepositoryOwnerName { get; set; }
+
+        [DefaultValue(true)]
+        [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate)]
path: tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs
user: login: jsquire
body: nifty.   I didn't know this was a thing!
created_at: 2025-04-24T00:05:39Z
author_association: MEMBER
diff_hunk: @@ -14,5 +11,13 @@ public class IssuePayload
         public string IssueUserLogin { get; set; }
         public string RepositoryName { get; set; }
         public string RepositoryOwnerName { get; set; }
+
+        [DefaultValue(true)]
+        [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate)]
path: tools/issue-labeler/src/IssueLabeler.Shared/IssuePayload.cs
user: login: jeo02
body: Yea super cool got it from json.NET docs (after copilot failed to try something similar anyways)
created_at: 2025-04-24T17:33:58Z
author_association: MEMBER

PR comments:
user: login: jeo02
created_at: 2025-04-24T20:07:01Z
body: Just tweaked the label list we pass to be a bit more friendly for the prompt to read 
author_association: MEMBER
user: login: jeo02
created_at: 2025-04-25T20:49:20Z
body: /check-enforcer evaluate 
author_association: MEMBER