Repository: dotnet/BenchmarkDotNet

Issue:
Custom `SimpleJob` Id ignored when the `MinIterationTime` attribute is used.
Hello, just identified the issue that causes all Jobs declared as attributes to have their Id's replaced with random Job names in the summary table if the `MinIterationTime `attribute is used. 

Steps to reproduce:
Create benchmark class and add two `SimpleJob `attributes as well as `AllStatisticsColumn` to show the Job Id. Here I just use the default example from README. 

```
[AllStatisticsColumn]
[SimpleJob(RunStrategy.ColdStart, launchCount: 5, warmupCount: 1, iterationCount: 1, id: "Job A")]
[SimpleJob(RunStrategy.Throughput, launchCount:1, warmupCount:1, iterationCount: 5, id: "Job B")]
public class Md5VsSha256
{
    private SHA256 sha256 = SHA256.Create();
    private MD5 md5 = MD5.Create();
    private byte[] data;

    [Params(1000, 10000)]
    public int N;

    [GlobalSetup]
    public void Setup()
    {
        data = new byte[N];
        new Random(42).NextBytes(data);
    }

    [Benchmark]
    public byte[] Sha256() => sha256.ComputeHash(data);

    [Benchmark]
    public byte[] Md5() => md5.ComputeHash(data);
}
```
In this case the table shows correct Job Id in the table( included only 2 lines for brevity)

| Method | Job   | IterationCount | LaunchCount | RunStrategy | UnrollFactor | N     | Mean         | Error        | StdDev       | StdErr       | Min          | Q1           | Median       | Q3           | Max          | Op/s        |
|------- |------ |--------------- |------------ |------------ |------------- |------ |-------------:|-------------:|-------------:|-------------:|-------------:|-------------:|-------------:|-------------:|-------------:|------------:|
| Sha256 | Job A | 1              | 5           | ColdStart   | 1            | 1000  | 488,140.0 ns | 134,559.1 ns | 34,944.57 ns | 15,627.69 ns | 454,800.0 ns | 474,900.0 ns | 475,200.0 ns | 489,100.0 ns | 546,700.0 ns |     2,048.6 |
| Md5    | Job A | 1              | 5           | ColdStart   | 1            | 1000  | 512,220.0 ns | 101,163.3 ns | 26,271.79 ns | 11,749.10 ns | 483,100.0 ns | 487,600.0 ns | 518,800.0 ns | 527,100.0 ns | 544,500.0 ns |     1,952.3 |

In case when also the `MinIterationTIme `attribute is added, the table shows the randomly generated Job Id

```
[AllStatisticsColumn]
[MinIterationTime(25)]
[SimpleJob(RunStrategy.ColdStart, launchCount: 5, warmupCount: 1, iterationCount: 1, id: "Job A")]
[SimpleJob(RunStrategy.Throughput, launchCount:1, warmupCount:1, iterationCount: 5, id: "Job B")]
public class Md5VsSha256
{
   ...
}
```

| Method | Job        | IterationCount | LaunchCount | RunStrategy | UnrollFactor | N     | Mean       | Error       | StdDev     | StdErr     | Min        | Q1         | Median     | Q3         | Max        | Op/s      |
|------- |----------- |--------------- |------------ |------------ |------------- |------ |-----------:|------------:|-----------:|-----------:|-----------:|-----------:|-----------:|-----------:|-----------:|----------:|
| Sha256 | Job-UJDDOK | 1              | 5           | ColdStart   | 1            | 1000  | 498.600 us |  93.2607 us | 24.2195 us | 10.8313 us | 483.000 us | 483.500 us | 487.300 us | 498.800 us | 540.400 us |   2,005.6 |
| Md5    | Job-UJDDOK | 1              | 5           | ColdStart   | 1            | 1000  | 552.900 us | 314.1166 us | 81.5751 us | 36.4815 us | 485.900 us | 514.700 us | 526.400 us | 543.600 us | 693.900 us |   1,808.6 |



Hints:
Leaving some findings here to (hopefully) help the next person. 

I noticed that I can recreate the issue with other attributes like [IterationTime(100)] or [MinInvokeCount(2)] as long as it has .IsMutator set to true.

The ResolvedId is set in the Job class  `public string ResolvedId => HasValue(IdCharacteristic) ? Id : JobIdGenerator.GenerateRandomId(this);`

So I assume somewhere along the way IdCharacteristic loses it's value.

One method of interest with my comments added. Found in the ImmutableConfigBuilder class:

```
private static IReadOnlyList<Job> GetRunnableJobs(IEnumerable<Job> jobs)
{
    var unique = jobs.Distinct(JobComparer.Instance).ToArray();
    var result = new List<Job>();

    foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault))
        result.Add(standardJob);

    var customDefaultJob = unique.SingleOrDefault(job => job.Meta.IsDefault);
    var defaultJob = customDefaultJob ?? Job.Default;

    if (!result.Any())
        result.Add(defaultJob);

    // If no mutatorJob then the ResolvedId names are kept in place as the code will skip this foreach.

    // Here is where the ResolvedId changes to the randomly generated value
    foreach (var mutatorJob in unique.Where(job => job.Meta.IsMutator))
    {
        for (int i = 0; i < result.Count; i++)
        {
	    // As soon as UnfreezeCopy() finishes the resolvedId changes
            var copy = result[i].UnfreezeCopy();

            copy.Apply(mutatorJob);

            result[i] = copy.Freeze();
        }
    }

    return result;
}
```

I also kept testing old commits and the issue was there as well. I went as far as 7/26/23 d93490d1f and this was still an issue.



Patch:
diff --git a/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs b/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs
index a6d7b37b81..6cb4a983c3 100644
--- a/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs
+++ b/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs
@@ -402,6 +402,12 @@ protected CharacteristicObject UnfreezeCopyCore()
             var newRoot = (CharacteristicObject)Activator.CreateInstance(GetType());
             newRoot.ApplyCore(this);
 
+            // Preserve the IdCharacteristic of the original object
+            if (this.HasValue(IdCharacteristic))
+            {
+                newRoot.SetValue(IdCharacteristic, this.GetValue(IdCharacteristic));
+            }
+
             return newRoot;
         }
         #endregion


Test patch:
diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs
index 896f93925b..64e3889fcf 100644
--- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs
+++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs
@@ -233,7 +233,7 @@ public void UserCanSpecifyMultipleCoreRunPaths()
             Assert.Equal(2, jobs.Length);
             Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1));
             Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2));
-            Assert.Equal(2, jobs.Select(job => job.Id).Distinct().Count()); // each job must have a unique ID
+
         }
 
         [Fact]
diff --git a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs
index 96f671e60d..050b84de7c 100644
--- a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs
+++ b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs
@@ -60,11 +60,11 @@ public static void Test01Create()
             Assert.False(j.Environment.Gc.AllowVeryLargeObjects);
             Assert.Equal(Platform.AnyCpu, j.Environment.Platform);
             Assert.Equal(RunStrategy.Throughput, j.Run.RunStrategy); // set by default
-            Assert.Equal("Default", j.Id); // id reset
-            Assert.True(j.DisplayInfo == "DefaultJob", "DisplayInfo = " + j.DisplayInfo);
-            Assert.True(j.ResolvedId == "DefaultJob", "ResolvedId = " + j.ResolvedId);
+            Assert.Equal("CustomId", j.Id); // id remains after unfreeze
+            Assert.Equal("CustomId", j.DisplayInfo);
+            Assert.Equal("CustomId", j.ResolvedId);
             Assert.Equal(j.ResolvedId, j.FolderInfo);
-            Assert.Equal("Default", j.Environment.Id);
+            Assert.Equal("CustomId", j.Environment.Id); // id remains after unfreeze
 
             // new job
             j = new Job(j.Freeze());
@@ -160,7 +160,7 @@ public static void Test02Modify()
             // 4. Freeze-unfreeze:
             j = j.Freeze().UnfreezeCopy();
 
-            Assert.Equal("Platform=X86, LaunchCount=1", j.Id);
+            Assert.Equal("SomeId", j.Id); // id not lost
             Assert.Equal(Platform.X86, j.Environment.Platform);
             Assert.Equal(1, j.Run.LaunchCount);
 
@@ -204,15 +204,15 @@ public static void Test03IdDoesNotFlow()
             Assert.False(j.HasValue(CharacteristicObject.IdCharacteristic));
             Assert.False(j.Environment.HasValue(CharacteristicObject.IdCharacteristic));
 
-            Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow
-            Assert.False(j.HasValue(CharacteristicObject.IdCharacteristic));
-            Assert.False(j.Environment.HasValue(CharacteristicObject.IdCharacteristic));
+            Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will flow
+            Assert.True(j.HasValue(CharacteristicObject.IdCharacteristic));
+            Assert.True(j.Environment.HasValue(CharacteristicObject.IdCharacteristic));
 
             var c = new CharacteristicSet(EnvironmentMode.LegacyJitX64, RunMode.Long); // id will not flow, new CharacteristicSet
             Assert.False(c.HasValue(CharacteristicObject.IdCharacteristic));
 
-            Job.EnvironmentCharacteristic[c] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow
-            Assert.False(c.HasValue(CharacteristicObject.IdCharacteristic));
+            Job.EnvironmentCharacteristic[c] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will flow
+            Assert.True(c.HasValue(CharacteristicObject.IdCharacteristic));
 
             CharacteristicObject.IdCharacteristic[c] = "MyId"; // id set explicitly
             Assert.Equal("MyId", c.Id);
@@ -221,12 +221,12 @@ public static void Test03IdDoesNotFlow()
             Assert.Equal("MyId", j.Id);
             Assert.Equal("MyId", j.Environment.Id);
 
-            Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow
-            Assert.Equal("MyId", j.Id);
-            Assert.Equal("MyId", j.Environment.Id);
+            Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy();
+            Assert.Equal("LegacyJitX86", j.Id);
+            Assert.Equal("LegacyJitX86", j.Environment.Id);
 
-            j = j.WithJit(Jit.RyuJit);  // custom id will flow
-            Assert.Equal("MyId", j.Id);
+            j = j.WithJit(Jit.RyuJit);
+            Assert.Equal("LegacyJitX86", j.Id);
         }
 
         [Fact]
@@ -474,6 +474,20 @@ public static void WithNuGet()
             Assert.Equal(expected, j.Infrastructure.NuGetReferences); // ensure that the list's equality operator returns true when the contents are the same
         }
 
+        [Fact]
+        public static void UnfreezeCopy_PreservesIdCharacteristic()
+        {
+            // Arrange
+            var original = new Job();
+            original.SetValue(Job.IdCharacteristic, "TestID");
+
+            // Act
+            var copy = original.UnfreezeCopy();
+
+            // Assert
+            Assert.Equal("TestID", copy.GetValue(Job.IdCharacteristic));
+        }
+
         private static bool IsSubclassOfobModeOfItself(Type type)
         {
             Type jobModeOfT;