Repository: dotnet/BenchmarkDotNet

Issue:
Any combination of Params/ParamsSource/ParamsAllValues is broken
```C#
[DryJob]
public class Benchmarks
{
    [Params("a", "b", "c")]
    [ParamsSource(nameof(Source))]
    public string Input { get; set; }

    [Benchmark]
    public void Test()
    {
        if(Input is "ddd" or "eee") Thread.Sleep(700);
    }

    public static IEnumerable<string> Source()
    {
        yield return "ddd";
        yield return "eee";
    }
}
```

| Method | Input |     Mean |
|------- |------ |---------:|
|   Test |     a | 702.3 ms |
|   Test |     a | 710.1 ms |
|   Test |     b | 706.1 ms |
|   Test |     b | 714.9 ms |
|   Test |     c | 707.2 ms |
|   Test |     c | 707.1 ms |

### More interesting scenario is for Boolean:
```C#
[DryJob]
public class Benchmarks
{
    [Params(false)]
    [ParamsSource(nameof(Source))]
    [ParamsAllValues]
    public bool Input { get; set; }

    [Benchmark]
    public void Test()
    {
        if(Input) Thread.Sleep(700);
    }

    public static IEnumerable<bool> Source()
    {
        yield return true;
        yield return true;
        yield return true;
    }
}
```

| Method | Input |       Mean |
|------- |------ |-----------:|
|   Test | False |   1.085 ms |
|   Test | False |   1.134 ms |
|   Test | False |   1.261 ms |
|   Test | False | 705.045 ms |
|   Test | False | 707.033 ms |
|   Test | False | 713.798 ms |

Some notes:
- no `[Params]` or `[Params(true)]` changes all table's Input for true.
- `[Params(false, false)` doubles the number of rows in the table. (6 rows for ~1ms, 6 rows for ~700ms)

#### Change Source from `true, true, true` to `true, false, true`:

| Input |       Mean |
|------ |-----------:|
| False |   1.119 ms |
| False | 714.345 ms |
| False |   1.277 ms |
| False |   1.127 ms |
| False | 713.454 ms |
| False | 715.087 ms |

#### Change Source to `false, false, false`:

It equals to `true, true, true`.

| Input |       Mean |
|------ |-----------:|
| False |   1.175 ms |
| False |   1.456 ms |
| False |   1.232 ms |
| False | 715.423 ms |
| False | 713.352 ms |
| False | 707.733 ms |


##### Merry Christmas 🎄 


Hints:
Hi @YegorStepanov 

I am surprised that BDN is trying to make it work instead of just failing the benchmark validation.

@AndreyAkinshin what is your opinion on this? Should we add a mandatory validator that says it's not supported? Or try to make it work?

Patch:
diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs
index 2ee08c8169..9a8f0345f0 100644
--- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs
+++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs
@@ -68,6 +68,7 @@ public IEnumerable<IValidator> GetValidators()
             yield return GenericBenchmarksValidator.DontFailOnError;
             yield return DeferredExecutionValidator.FailOnError;
             yield return ParamsAllValuesValidator.FailOnError;
+            yield return ParamsValidator.FailOnError;
         }
 
         public IOrderer Orderer => null;
diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
index 56a1a6e8c4..ae4aa48b3f 100644
--- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
+++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
@@ -27,7 +27,8 @@ public static class ImmutableConfigBuilder
             ShadowCopyValidator.DontFailOnError,
             JitOptimizationsValidator.DontFailOnError,
             DeferredExecutionValidator.DontFailOnError,
-            ParamsAllValuesValidator.FailOnError
+            ParamsAllValuesValidator.FailOnError,
+            ParamsValidator.FailOnError
         };
 
         /// <summary>
diff --git a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs
index d3669d4029..b01d63d1be 100644
--- a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs
+++ b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs
@@ -3,33 +3,35 @@
 using System.Linq;
 using System.Reflection;
 using BenchmarkDotNet.Attributes;
-using JetBrains.Annotations;
 
 namespace BenchmarkDotNet.Extensions
 {
     internal static class ReflectionExtensions
     {
-        [PublicAPI]
-        internal static T ResolveAttribute<T>(this Type type) where T : Attribute =>
+        internal static T? ResolveAttribute<T>(this Type? type) where T : Attribute =>
             type?.GetTypeInfo().GetCustomAttributes(typeof(T), false).OfType<T>().FirstOrDefault();
 
-        [PublicAPI]
-        internal static T ResolveAttribute<T>(this MethodInfo methodInfo) where T : Attribute =>
-            methodInfo?.GetCustomAttributes(typeof(T), false).FirstOrDefault() as T;
+        internal static T? ResolveAttribute<T>(this MemberInfo? memberInfo) where T : Attribute =>
+            memberInfo?.GetCustomAttributes(typeof(T), false).FirstOrDefault() as T;
 
-        [PublicAPI]
-        internal static T ResolveAttribute<T>(this PropertyInfo propertyInfo) where T : Attribute =>
-            propertyInfo?.GetCustomAttributes(typeof(T), false).FirstOrDefault() as T;
+        internal static bool HasAttribute<T>(this MemberInfo? memberInfo) where T : Attribute =>
+            memberInfo.ResolveAttribute<T>() != null;
 
-        [PublicAPI]
-        internal static T ResolveAttribute<T>(this FieldInfo fieldInfo) where T : Attribute =>
-            fieldInfo?.GetCustomAttributes(typeof(T), false).FirstOrDefault() as T;
+        internal static bool IsNullable(this Type type) => Nullable.GetUnderlyingType(type) != null;
 
-        [PublicAPI]
-        internal static bool HasAttribute<T>(this MethodInfo methodInfo) where T : Attribute =>
-            methodInfo.ResolveAttribute<T>() != null;
+        public static bool IsInitOnly(this PropertyInfo propertyInfo)
+        {
+            var setMethodReturnParameter = propertyInfo.SetMethod?.ReturnParameter;
+            if (setMethodReturnParameter == null)
+                return false;
 
-        internal static bool IsNullable(this Type type) => Nullable.GetUnderlyingType(type) != null;
+            var isExternalInitType = typeof(System.Runtime.CompilerServices.Unsafe).Assembly
+                .GetType("System.Runtime.CompilerServices.IsExternalInit");
+            if (isExternalInitType == null)
+                return false;
+
+            return setMethodReturnParameter.GetRequiredCustomModifiers().Contains(isExternalInitType);
+        }
 
         /// <summary>
         /// returns type name which can be used in generated C# code
@@ -183,13 +185,13 @@ internal static (string Name, TAttribute Attribute, bool IsPrivate, bool IsStati
             return joined;
         }
 
-        internal static bool IsStackOnlyWithImplicitCast(this Type argumentType, object argumentInstance)
+        internal static bool IsStackOnlyWithImplicitCast(this Type argumentType, object? argumentInstance)
         {
             if (argumentInstance == null)
                 return false;
 
             // IsByRefLikeAttribute is not exposed for older runtimes, so we need to check it in an ugly way ;)
-            bool isByRefLike = argumentType.GetCustomAttributes().Any(attribute => attribute.ToString().Contains("IsByRefLike"));
+            bool isByRefLike = argumentType.GetCustomAttributes().Any(attribute => attribute.ToString()?.Contains("IsByRefLike") ?? false);
             if (!isByRefLike)
                 return false;
 
diff --git a/src/BenchmarkDotNet/Validators/ParamsValidator.cs b/src/BenchmarkDotNet/Validators/ParamsValidator.cs
new file mode 100644
index 0000000000..2a49dac434
--- /dev/null
+++ b/src/BenchmarkDotNet/Validators/ParamsValidator.cs
@@ -0,0 +1,58 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Extensions;
+
+namespace BenchmarkDotNet.Validators
+{
+    public class ParamsValidator : IValidator
+    {
+        public static readonly ParamsValidator FailOnError = new ();
+
+        public bool TreatsWarningsAsErrors => true;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters input) => input.Benchmarks
+            .Select(benchmark => benchmark.Descriptor.Type)
+            .Distinct()
+            .SelectMany(Validate);
+
+        private IEnumerable<ValidationError> Validate(Type type)
+        {
+            foreach (var memberInfo in type.GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.FlattenHierarchy))
+            {
+                var attributes = new Attribute[]
+                    {
+                        memberInfo.ResolveAttribute<ParamsAttribute>(),
+                        memberInfo.ResolveAttribute<ParamsAllValuesAttribute>(),
+                        memberInfo.ResolveAttribute<ParamsSourceAttribute>()
+                    }
+                    .Where(attribute => attribute != null)
+                    .ToList();
+                if (attributes.IsEmpty())
+                    continue;
+
+                string name = $"{type.Name}.{memberInfo.Name}";
+                string? attributeString = string.Join(", ", attributes.Select(attribute => $"[{attribute.GetType().Name.Replace(nameof(Attribute), "")}]"));
+
+                if (attributes.Count > 1)
+                    yield return new ValidationError(TreatsWarningsAsErrors,
+                        $"Unable to use {name} with {attributeString} at the same time. Please, use a single attribute.");
+
+                if (memberInfo is FieldInfo fieldInfo && (fieldInfo.IsLiteral || fieldInfo.IsInitOnly))
+                {
+                    string modifier = fieldInfo.IsInitOnly ? "readonly" : "constant";
+                    yield return new ValidationError(TreatsWarningsAsErrors,
+                        $"Unable to use {name} with {attributeString} because it's a {modifier} field. Please, remove the {modifier} modifier.");
+                }
+
+                if (memberInfo is PropertyInfo propertyInfo && propertyInfo.IsInitOnly())
+                {
+                    yield return new ValidationError(TreatsWarningsAsErrors,
+                        $"Unable to use {name} with {attributeString} because it's init-only. Please, provide a public setter.");
+                }
+            }
+        }
+    }
+}
\ No newline at end of file


Test patch:
diff --git a/tests/BenchmarkDotNet.Tests/Validators/ParamsValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ParamsValidatorTests.cs
new file mode 100644
index 0000000000..c33c0625e6
--- /dev/null
+++ b/tests/BenchmarkDotNet.Tests/Validators/ParamsValidatorTests.cs
@@ -0,0 +1,206 @@
+﻿using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Validators;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BenchmarkDotNet.Tests.Validators
+{
+    [SuppressMessage("ReSharper", "ClassNeverInstantiated.Global")]
+    [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")]
+    public class ParamsValidatorTests
+    {
+        private readonly ITestOutputHelper output;
+
+        public ParamsValidatorTests(ITestOutputHelper output)
+        {
+            this.output = output;
+        }
+
+        private void Check<T>(params string[] messageParts)
+        {
+            var typeToBenchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(T));
+            Assert.NotEmpty(typeToBenchmarks.BenchmarksCases);
+
+            var validationErrors = ParamsValidator.FailOnError.Validate(typeToBenchmarks).ToList();
+            output.WriteLine("Number of validation errors: " + validationErrors.Count);
+            foreach (var error in validationErrors)
+                output.WriteLine("* " + error.Message);
+
+            Assert.Single(validationErrors);
+            foreach (string messagePart in messageParts)
+                Assert.Contains(messagePart, validationErrors.Single().Message);
+        }
+
+        private const string P = "[Params]";
+        private const string Pa = "[ParamsAllValues]";
+        private const string Ps = "[ParamsSource]";
+
+        [Fact] public void Const1Test() => Check<Const1>(nameof(Const1.Input), "constant", P);
+        [Fact] public void Const2Test() => Check<Const2>(nameof(Const2.Input), "constant", Pa);
+        [Fact] public void Const3Test() => Check<Const3>(nameof(Const3.Input), "constant", Ps);
+        [Fact] public void StaticReadonly1Test() => Check<StaticReadonly1>(nameof(StaticReadonly1.Input), "readonly", P);
+        [Fact] public void StaticReadonly2Test() => Check<StaticReadonly2>(nameof(StaticReadonly2.Input), "readonly", Pa);
+        [Fact] public void StaticReadonly3Test() => Check<StaticReadonly3>(nameof(StaticReadonly3.Input), "readonly", Ps);
+        [Fact] public void NonStaticReadonly1Test() => Check<NonStaticReadonly1>(nameof(NonStaticReadonly1.Input), "readonly", P);
+        [Fact] public void NonStaticReadonly2Test() => Check<NonStaticReadonly2>(nameof(NonStaticReadonly2.Input), "readonly", Pa);
+        [Fact] public void NonStaticReadonly3Test() => Check<NonStaticReadonly3>(nameof(NonStaticReadonly3.Input), "readonly", Ps);
+        [Fact] public void FieldMultiple1Test() => Check<FieldMultiple1>(nameof(FieldMultiple1.Input), "single attribute", P, Pa);
+        [Fact] public void FieldMultiple2Test() => Check<FieldMultiple2>(nameof(FieldMultiple2.Input), "single attribute", P, Ps);
+        [Fact] public void FieldMultiple3Test() => Check<FieldMultiple3>(nameof(FieldMultiple3.Input), "single attribute", Pa, Ps);
+        [Fact] public void FieldMultiple4Test() => Check<FieldMultiple4>(nameof(FieldMultiple4.Input), "single attribute", P, Pa, Ps);
+        [Fact] public void PropMultiple1Test() => Check<PropMultiple1>(nameof(PropMultiple1.Input), "single attribute", P, Pa);
+        [Fact] public void PropMultiple2Test() => Check<PropMultiple2>(nameof(PropMultiple2.Input), "single attribute", P, Ps);
+        [Fact] public void PropMultiple3Test() => Check<PropMultiple3>(nameof(PropMultiple3.Input), "single attribute", Pa, Ps);
+        [Fact] public void PropMultiple4Test() => Check<PropMultiple4>(nameof(PropMultiple4.Input), "single attribute", P, Pa, Ps);
+
+        public class Base
+        {
+            [Benchmark]
+            public void Foo() { }
+
+            public static IEnumerable<bool> Source() => new[] { false, true };
+        }
+
+        public class Const1 : Base
+        {
+            [Params(false, true)]
+            public const bool Input = false;
+        }
+
+        public class Const2 : Base
+        {
+            [ParamsAllValues]
+            public const bool Input = false;
+        }
+
+        public class Const3 : Base
+        {
+            [ParamsSource(nameof(Source))]
+            public const bool Input = false;
+        }
+
+        public class StaticReadonly1 : Base
+        {
+            [Params(false, true)]
+            public static readonly bool Input = false;
+        }
+
+        public class StaticReadonly2 : Base
+        {
+            [ParamsAllValues]
+            public static readonly bool Input = false;
+        }
+
+        public class StaticReadonly3 : Base
+        {
+            [ParamsSource(nameof(Source))]
+            public static readonly bool Input = false;
+        }
+
+        public class NonStaticReadonly1 : Base
+        {
+            [Params(false, true)]
+            public readonly bool Input = false;
+        }
+
+        public class NonStaticReadonly2 : Base
+        {
+            [ParamsAllValues]
+            public readonly bool Input = false;
+        }
+
+        public class NonStaticReadonly3 : Base
+        {
+            [ParamsSource(nameof(Source))]
+            public readonly bool Input = false;
+        }
+
+        public class FieldMultiple1 : Base
+        {
+            [Params(false, true)]
+            [ParamsAllValues]
+            public bool Input = false;
+        }
+
+        public class FieldMultiple2 : Base
+        {
+            [Params(false, true)]
+            [ParamsSource(nameof(Source))]
+            public bool Input = false;
+        }
+
+        public class FieldMultiple3 : Base
+        {
+            [ParamsAllValues]
+            [ParamsSource(nameof(Source))]
+            public bool Input = false;
+        }
+
+        public class FieldMultiple4 : Base
+        {
+            [Params(false, true)]
+            [ParamsAllValues]
+            [ParamsSource(nameof(Source))]
+            public bool Input = false;
+        }
+
+        public class PropMultiple1 : Base
+        {
+            [Params(false, true)]
+            [ParamsAllValues]
+            public bool Input { get; set; }
+        }
+
+        public class PropMultiple2 : Base
+        {
+            [Params(false, true)]
+            [ParamsSource(nameof(Source))]
+            public bool Input { get; set; }
+        }
+
+        public class PropMultiple3 : Base
+        {
+            [ParamsAllValues]
+            [ParamsSource(nameof(Source))]
+            public bool Input { get; set; }
+        }
+
+        public class PropMultiple4 : Base
+        {
+            [Params(false, true)]
+            [ParamsAllValues]
+            [ParamsSource(nameof(Source))]
+            public bool Input { get; set; }
+        }
+
+#if NET5_0_OR_GREATER
+
+        [Fact] public void InitOnly1Test() => Check<InitOnly1>(nameof(InitOnly1.Input), "init-only", P);
+        [Fact] public void InitOnly2Test() => Check<InitOnly2>(nameof(InitOnly2.Input), "init-only", Pa);
+        [Fact] public void InitOnly3Test() => Check<InitOnly3>(nameof(InitOnly3.Input), "init-only", Ps);
+
+        public class InitOnly1 : Base
+        {
+            [Params(false, true)]
+            public bool Input { get; init; }
+        }
+
+        public class InitOnly2 : Base
+        {
+            [ParamsAllValues]
+            public bool Input { get; init; }
+        }
+
+        public class InitOnly3 : Base
+        {
+            [ParamsSource(nameof(Source))]
+            public bool Input { get; init; }
+        }
+
+#endif
+    }
+}
\ No newline at end of file