Repository: 7mind/izumi
Pull request #848 (2020-01-22 18:21:08)

distage-testkit: Enhance parallel test reporting, ensure all exceptions are caught
* ensure that all tests are reported
* ensure that late statuses are finalized in scalatest runner
* launch envs in parallel
* move testkit methods from private to protected or package-private


Patch:
diff --git a/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffect.scala b/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffect.scala
index 6b6e7dd92..3e5c3d658 100644
--- a/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffect.scala
+++ b/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffect.scala
@@ -24,11 +24,11 @@ trait DIEffect[F[_]] {
     * this will _also_ intercept Throwable defects in `ZIO`, not only typed errors */
   def definitelyRecover[A](action: => F[A])(recover: Throwable => F[A]): F[A]
 
-  /** `definitelyRecover`, but the Throwable is not guaranteed to be
-    * the original Throwable, it can be a Throwable enhanced with a
-    * Monad's own debugging information (e.g. in case of `ZIO` it's always `FiberFailure`)
+  /** `definitelyRecover`, but second Throwable is not the original Throwable,
+    * it can be a Throwable enhanced with an effect's own debugging information
+    * (e.g. in case of `ZIO` it's always `FiberFailure`)
     */
-  def definitelyRecoverCause[A](action: => F[A])(recoverCause: Throwable => F[A]): F[A]
+  def definitelyRecoverCause[A](action: => F[A])(recoverCause: (Throwable, Throwable) => F[A]): F[A]
 
   def fail[A](t: => Throwable): F[A]
 
@@ -74,8 +74,8 @@ object DIEffect extends LowPriorityDIEffectInstances {
         case t: Throwable => recover(t)
       }
     }
-    override def definitelyRecoverCause[A](action: => Identity[A])(recoverCause: Throwable => Identity[A]): Identity[A] = {
-      definitelyRecover(action)(recoverCause)
+    override def definitelyRecoverCause[A](action: => Identity[A])(recoverCause: (Throwable, Throwable) => Identity[A]): Identity[A] = {
+      definitelyRecover(action)(e => recoverCause(e, e))
     }
     override def bracket[A, B](acquire: => Identity[A])(release: A => Identity[Unit])(use: A => Identity[B]): Identity[B] = {
       val a = acquire
@@ -106,8 +106,8 @@ object DIEffect extends LowPriorityDIEffectInstances {
       override def definitelyRecover[A](action: => F[E, A])(recover: Throwable => F[E, A]): F[E, A] = {
         suspendF(action).sandbox.catchAll(recover apply _.toThrowable)
       }
-      override def definitelyRecoverCause[A](action: => F[Throwable, A])(recover: Throwable => F[Throwable, A]): F[Throwable, A] = {
-        suspendF(action).sandbox.catchAll(recover apply _.trace.toThrowable)
+      override def definitelyRecoverCause[A](action: => F[Throwable, A])(recover: (Throwable, Throwable) => F[Throwable, A]): F[Throwable, A] = {
+        suspendF(action).sandbox.catchAll(e => recover(e.toThrowable, e.trace.toThrowable))
       }
 
       override def fail[A](t: => Throwable): F[Throwable, A] = F.fail(t)
@@ -146,8 +146,8 @@ private[effect] sealed trait LowPriorityDIEffectInstances {
       override def definitelyRecover[A](action: => F[A])(recover: Throwable => F[A]): F[A] = {
         F.handleErrorWith(F.suspend(action))(recover)
       }
-      override def definitelyRecoverCause[A](action: => F[A])(recoverCause: Throwable => F[A]): F[A] = {
-        definitelyRecover(action)(recoverCause)
+      override def definitelyRecoverCause[A](action: => F[A])(recoverCause: (Throwable, Throwable) => F[A]): F[A] = {
+        definitelyRecover(action)(e => recoverCause(e, e))
       }
       override def fail[A](t: => Throwable): F[A] = F.suspend(F.raiseError(t))
       override def bracket[A, B](acquire: => F[A])(release: A => F[Unit])(use: A => F[B]): F[B] = {
diff --git a/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffectAsync.scala b/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffectAsync.scala
index 5cb27b728..5ad7e690f 100644
--- a/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffectAsync.scala
+++ b/distage/distage-core-api/src/main/scala/izumi/distage/model/effect/DIEffectAsync.scala
@@ -20,6 +20,8 @@ object DIEffectAsync extends LowPriorityDIEffectAsyncInstances {
 
   implicit val diEffectParIdentity: DIEffectAsync[Identity] = {
     new DIEffectAsync[Identity] {
+      final val DIEffectAsyncIdentityPool = ExecutionContext.fromExecutorService(null)
+
       override def parTraverse_[A](l: Iterable[A])(f: A => Unit): Unit = {
         parTraverse(l)(f)
         ()
@@ -29,9 +31,7 @@ object DIEffectAsync extends LowPriorityDIEffectAsyncInstances {
       }
 
       override def parTraverse[A, B](l: Iterable[A])(f: A => Identity[B]): Identity[List[B]] = {
-        implicit val ec = ExecutionContext.global
-        val future = Future.sequence(l.map(a => Future(f(a))))
-        Await.result(future, Duration.Inf).toList
+        parTraverseIdentity(DIEffectAsyncIdentityPool)(l)(f)
       }
     }
   }
@@ -50,6 +50,12 @@ object DIEffectAsync extends LowPriorityDIEffectAsyncInstances {
     }
   }
 
+  private[izumi] def parTraverseIdentity[A, B](ec0: ExecutionContext)(l: Iterable[A])(f: A => Identity[B]): Identity[List[B]] = {
+    implicit val ec = ec0
+    val future = Future.sequence(l.map(a => Future(scala.concurrent.blocking(f(a)))))
+    Await.result(future, Duration.Inf).toList
+  }
+
 }
 
 private[effect] sealed trait LowPriorityDIEffectAsyncInstances {
diff --git a/distage/distage-framework-docker/src/main/scala/izumi/distage/docker/DockerContainer.scala b/distage/distage-framework-docker/src/main/scala/izumi/distage/docker/DockerContainer.scala
index 233a19898..ac8da99e2 100644
--- a/distage/distage-framework-docker/src/main/scala/izumi/distage/docker/DockerContainer.scala
+++ b/distage/distage-framework-docker/src/main/scala/izumi/distage/docker/DockerContainer.scala
@@ -123,7 +123,7 @@ object DockerContainer {
         for {
           containers <- DIEffect[F]
             .maybeSuspend {
-              // FIXME: temporary hack to allow missing containers to skip tests
+              // FIXME: temporary hack to allow missing containers to skip tests (happens when both DockerWrapper & integration check that depends on Docker.Container are memoized)
               try {
                 client.listContainersCmd()
                   .withAncestorFilter(List(config.image).asJava)
diff --git a/distage/distage-framework/src/main/scala/izumi/distage/framework/services/RoleAppPlanner.scala b/distage/distage-framework/src/main/scala/izumi/distage/framework/services/RoleAppPlanner.scala
index 4c67f01f5..c2f8ae90f 100644
--- a/distage/distage-framework/src/main/scala/izumi/distage/framework/services/RoleAppPlanner.scala
+++ b/distage/distage-framework/src/main/scala/izumi/distage/framework/services/RoleAppPlanner.scala
@@ -26,7 +26,7 @@ object RoleAppPlanner {
                           bsModule: BootstrapModule,
                           logger: IzLogger,
                         ) extends RoleAppPlanner[F] { self =>
-    private[this] val injector = Injector.Standard(bsModule)
+    private[this] val injector = Injector(bsModule)
 
     override def reboot(bsOverride: BootstrapModule): RoleAppPlanner[F] = {
       new RoleAppPlanner.Impl[F](options, bsModule overridenBy bsOverride, logger)


Test patch:
diff --git a/distage/distage-core/.jvm/src/test/scala/izumi/distage/injector/AnimalModel.scala b/distage/distage-core/.jvm/src/test/scala/izumi/distage/injector/AnimalModel.scala
index f86aa09da..75d7f1cff 100644
--- a/distage/distage-core/.jvm/src/test/scala/izumi/distage/injector/AnimalModel.scala
+++ b/distage/distage-core/.jvm/src/test/scala/izumi/distage/injector/AnimalModel.scala
@@ -24,9 +24,9 @@ class AnimalModel extends AnyWordSpec with MkInjector {
       val debug = false
 
       val injector = if (debug) {
-        Injector.Standard(GraphDumpBootstrapModule())
+        Injector(GraphDumpBootstrapModule())
       } else {
-        Injector.Standard()
+        Injector()
       }
 
       val plan = injector.plan(definition)
diff --git a/distage/distage-core/src/test/scala/izumi/distage/fixtures/ResourceCases.scala b/distage/distage-core/src/test/scala/izumi/distage/fixtures/ResourceCases.scala
index c9b810025..b512bb66f 100644
--- a/distage/distage-core/src/test/scala/izumi/distage/fixtures/ResourceCases.scala
+++ b/distage/distage-core/src/test/scala/izumi/distage/fixtures/ResourceCases.scala
@@ -167,8 +167,8 @@ object ResourceCases {
           case Right(value) => Right(value)
         })
       }
-      override def definitelyRecoverCause[A](action: => Suspend2[E, A])(recoverCause: Throwable => Suspend2[E, A]): Suspend2[E, A] = {
-        definitelyRecover(action)(recoverCause)
+      override def definitelyRecoverCause[A](action: => Suspend2[E, A])(recoverCause: (Throwable, Throwable) => Suspend2[E, A]): Suspend2[E, A] = {
+        definitelyRecover(action)(e => recoverCause(e, e))
       }
 
       override def bracket[A, B](acquire: => Suspend2[E, A])(release: A => Suspend2[E, Unit])(use: A => Suspend2[E, B]): Suspend2[E, B] =
diff --git a/distage/distage-core/src/test/scala/izumi/distage/injector/AutoSetTest.scala b/distage/distage-core/src/test/scala/izumi/distage/injector/AutoSetTest.scala
index d45881637..9ddc26c07 100644
--- a/distage/distage-core/src/test/scala/izumi/distage/injector/AutoSetTest.scala
+++ b/distage/distage-core/src/test/scala/izumi/distage/injector/AutoSetTest.scala
@@ -19,7 +19,7 @@ class AutoSetTest extends AnyWordSpec with MkInjector {
       make[ServiceD]
     }
 
-    val injector = Injector.Standard(new BootstrapModuleDef {
+    val injector = Injector(new BootstrapModuleDef {
       many[PlanningHook]
         .add(new AutoSetHook[Ordered, Ordered](identity))
     })
@@ -39,7 +39,7 @@ class AutoSetTest extends AnyWordSpec with MkInjector {
         .addValue(5)
     }
 
-    val injector = Injector.Standard(new BootstrapModuleDef {
+    val injector = Injector(new BootstrapModuleDef {
       many[PlanningHook]
         .add(new AutoSetHook[Int, Int](identity))
     })
diff --git a/distage/distage-extension-logstage/src/test/scala/izumi/logstage/distage/LoggerInjectionTest.scala b/distage/distage-extension-logstage/src/test/scala/izumi/logstage/distage/LoggerInjectionTest.scala
index b95296467..af026fb34 100644
--- a/distage/distage-extension-logstage/src/test/scala/izumi/logstage/distage/LoggerInjectionTest.scala
+++ b/distage/distage-extension-logstage/src/test/scala/izumi/logstage/distage/LoggerInjectionTest.scala
@@ -33,7 +33,7 @@ class LoggerInjectionTest extends AnyWordSpec {
 
       val loggerModule = new LogstageModule(router, false)
 
-      val injector = Injector.Standard(loggerModule)
+      val injector = Injector(loggerModule)
       val plan = injector.plan(definition)
       val context = injector.produceUnsafe(plan)
       assert(context.get[ExampleApp].test == 265)
diff --git a/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/DistageTestDockerBIO.scala b/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/DistageTestDockerBIO.scala
index 7176b4ca1..861d324ba 100644
--- a/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/DistageTestDockerBIO.scala
+++ b/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/DistageTestDockerBIO.scala
@@ -1,7 +1,6 @@
 package izumi.distage.testkit.docker
 
 import distage.DIKey
-import izumi.distage.docker.examples.{DynamoDocker, PostgresDocker}
 import izumi.distage.testkit.TestConfig
 import izumi.distage.testkit.docker.fixtures.PgSvcExample
 import izumi.distage.testkit.scalatest.DistageBIOSpecScalatest
@@ -32,8 +31,9 @@ final class DistageTestDockerBIO extends DistageBIOSpecScalatest[IO] {
   override protected def config: TestConfig = {
     super.config.copy(
       memoizationRoots = Set(
-        DIKey.get[DynamoDocker.Container],
-        DIKey.get[PostgresDocker.Container],
+        DIKey.get[PgSvcExample],
+//        DIKey.get[DynamoDocker.Container],
+//        DIKey.get[PostgresDocker.Container],
       ))
   }
 }
diff --git a/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/fixtures/DockerPlugin.scala b/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/fixtures/DockerPlugin.scala
index 6032d85f8..60712a206 100644
--- a/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/fixtures/DockerPlugin.scala
+++ b/distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/fixtures/DockerPlugin.scala
@@ -8,9 +8,18 @@ import izumi.distage.plugins.PluginDef
 import izumi.distage.docker.Docker.AvailablePort
 import izumi.distage.docker.examples.{DynamoDocker, PostgresDocker}
 import izumi.distage.docker.modules.DockerContainerModule
+import izumi.distage.framework.model.IntegrationCheck
+import izumi.fundamentals.platform.integration.{PortCheck, ResourceCheck}
 import zio.Task
 
-class PgSvcExample(val pg: AvailablePort @Id("pg"), val ddb: AvailablePort @Id("ddb"))
+class PgSvcExample(
+                    val pg: AvailablePort @Id("pg"),
+                    val ddb: AvailablePort @Id("ddb"),
+                  ) extends IntegrationCheck {
+  override def resourcesAvailable(): ResourceCheck = {
+    new PortCheck(10).checkPort(pg.hostV4, pg.port)
+  }
+}
 
 object MonadPlugin extends PluginDef
   with CatsDIEffectModule
diff --git a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestEnv.scala b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestEnv.scala
index 11257edd9..31cfce980 100644
--- a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestEnv.scala
+++ b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestEnv.scala
@@ -32,7 +32,7 @@ trait DistageTestEnv {
     }
   }
 
-  private def makeEnv(logger: IzLogger, testConfig: TestConfig, pluginLoader: PluginLoader, roles: RolesInfo, mergeStrategy: PluginMergeStrategy): TestEnvironment = {
+  protected[distage] def makeEnv(logger: IzLogger, testConfig: TestConfig, pluginLoader: PluginLoader, roles: RolesInfo, mergeStrategy: PluginMergeStrategy): TestEnvironment = {
     val appPlugins = pluginLoader.load(testConfig.pluginConfig)
     val bsPlugins = pluginLoader.load(testConfig.bootstrapPluginConfig)
     val appModule = mergeStrategy.merge(appPlugins) overridenBy testConfig.moduleOverrides
@@ -72,8 +72,7 @@ trait DistageTestEnv {
 }
 
 object DistageTestEnv {
-  private lazy val cache = new SyncCache[EnvCacheKey, TestEnvironment]
-
-  case class EnvCacheKey(config: TestConfig, rolesInfo: RolesInfo, mergeStrategy: PluginMergeStrategy)
+  private[distage] lazy val cache = new SyncCache[EnvCacheKey, TestEnvironment]
 
+  final case class EnvCacheKey(config: TestConfig, rolesInfo: RolesInfo, mergeStrategy: PluginMergeStrategy)
 }
diff --git a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala
index df5238798..8a9b41573 100644
--- a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala
+++ b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala
@@ -15,15 +15,16 @@ import izumi.distage.model.exceptions.ProvisioningException
 import izumi.distage.model.plan.{OrderedPlan, TriSplittedPlan}
 import izumi.distage.model.providers.ProviderMagnet
 import izumi.distage.testkit.DebugProperties
-import izumi.distage.testkit.services.dstest.DistageTestRunner.{DistageTest, SuiteData, TestReporter, TestStatus}
+import izumi.distage.testkit.services.dstest.DistageTestRunner._
 import izumi.fundamentals.platform.console.TrivialLogger
 import izumi.fundamentals.platform.functional.Identity
 import izumi.fundamentals.platform.integration.ResourceCheck
 import izumi.fundamentals.platform.language.CodePosition
+import izumi.fundamentals.platform.strings.IzString._
 import izumi.fundamentals.reflection.Tags.TagK
 
-import scala.concurrent.duration.FiniteDuration
-import izumi.fundamentals.platform.strings.IzString._
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.{Duration, FiniteDuration}
 
 class DistageTestRunner[F[_]: TagK]
 (
@@ -32,7 +33,8 @@ class DistageTestRunner[F[_]: TagK]
   runnerEnvironment: SpecEnvironment,
   tests: Seq[DistageTest[F]],
   isTestSkipException: Throwable => Boolean,
-  parallelTests: Boolean
+  parallelTests: Boolean,
+  parallelEnvs: Boolean,
 ) {
   def run(): Unit = {
     val groups = tests.groupBy(_.environment)
@@ -50,15 +52,14 @@ class DistageTestRunner[F[_]: TagK]
          |test environments=${groups.keys.niceList()}
          |tests=${groups.values.map(_.map(_.meta.id).niceList()).zipWithIndex.map(_.swap).niceList()}""".stripMargin)
 
-    groups.foreach {
-      case (env, tests) =>
-
+    try configuredForeach(groups) {
+      case (env, tests) => try {
         // here we scan our classpath to enumerate of our components (we have "bootstrap" components - injector plugins, and app components)
         val provider = runnerEnvironment.makeModuleProvider(options, config, logger, env.roles, env.activationInfo, env.activation)
         val bsModule = provider.bootstrapModules().merge overridenBy env.bsModule overridenBy runnerEnvironment.bootstrapOverrides
         val appModule = provider.appModules().merge overridenBy env.appModule overridenBy runnerEnvironment.moduleOverrides
 
-        val injector = Injector.Standard(bsModule)
+        val injector = Injector(bsModule)
 
         // first we need to plan runtime for our monad. Identity is also supported
         val runtimeGcRoots: Set[DIKey] = Set(
@@ -115,13 +116,14 @@ class DistageTestRunner[F[_]: TagK]
             implicit val F: DIEffect[F] = runtimeLocator.get[DIEffect[F]]
             implicit val P: DIEffectAsync[F] = runtimeLocator.get[DIEffectAsync[F]]
 
-            try {
-              runner.run {
-                // now we produce integration components for our shared plan
-                checker.verify(shared.side)
+            runner.run {
+              // now we produce integration components for our shared plan
+              checker.verify(shared.shared)
 
+              F.definitelyRecoverCause {
                 Injector.inherit(runtimeLocator).produceF[F](shared.shared).use {
                   sharedLocator =>
+                    checker.verify(shared.side)
 
                     Injector.inherit(sharedLocator).produceF[F](shared.side).use {
                       sharedIntegrationLocator =>
@@ -130,19 +132,26 @@ class DistageTestRunner[F[_]: TagK]
                         }
                     }
                 }
+              } {
+                case (ProvisioningIntegrationException(integrations), _) =>
+                  // FIXME: temporary hack to allow missing containers to skip tests (happens when both DockerWrapper & integration check that depends on Docker.Container are memoized)
+                  F.maybeSuspend(ignoreIntegrationCheckFailedTests(tests, integrations))
+                case (_, cause) =>
+                  // fail all tests (if an exception reached here, it must have happened before the individual test runs)
+                  F.maybeSuspend(failAllTests(tests, cause))
               }
-            } catch {
-              case p: ProvisioningException =>
-                val integrations = p.getSuppressed.collect { case i: IntegrationCheckException => i.failures }.toSeq
-                if (integrations.nonEmpty) {
-                  ignoreIntegrationCheckFailedTests(tests, integrations.flatten)
-                } else throw p
             }
         }
-    }
+      } catch {
+        case t: Throwable =>
+          // fail all tests (if an exception reaches here, it must have happened before the runtime was successfully produced)
+          failAllTests(tests, t)
+          reporter.onFailure(t)
+      }
+    } finally reporter.endAll()
   }
 
-  private def ifIntegChecksOk(integLocator: Locator)(testplans: Seq[DistageTest[F]], plans: TriSplittedPlan)(onSuccess: => F[Unit])(implicit F: DIEffect[F]): F[Unit] = {
+  protected def ifIntegChecksOk(integLocator: Locator)(testplans: Seq[DistageTest[F]], plans: TriSplittedPlan)(onSuccess: => F[Unit])(implicit F: DIEffect[F]): F[Unit] = {
     integrationChecker.collectFailures(plans.side.declaredRoots, integLocator).flatMap {
       case Left(failures) =>
         F.maybeSuspend(ignoreIntegrationCheckFailedTests(testplans, failures))
@@ -152,36 +161,46 @@ class DistageTestRunner[F[_]: TagK]
     }
   }
 
-  private def ignoreIntegrationCheckFailedTests(tests: Seq[DistageTest[F]], failures: Seq[ResourceCheck.Failure]): Unit = {
+  protected def ignoreIntegrationCheckFailedTests(tests: Seq[DistageTest[F]], failures: Seq[ResourceCheck.Failure]): Unit = {
     tests.foreach {
       test =>
         reporter.testStatus(test.meta, TestStatus.Ignored(failures))
     }
   }
 
-  private def proceed(appmodule: ModuleBase,
-                      checker: PlanCircularDependencyCheck,
-                      testplans: Seq[(DistageTest[F], OrderedPlan)],
-                      shared: TriSplittedPlan,
-                      parent: Locator,
-                     )(implicit F: DIEffect[F], P: DIEffectAsync[F]): F[Unit] = {
+  protected def failAllTests(tests: Seq[DistageTest[F]], cause: Throwable): Unit = {
+    tests.foreach {
+      test =>
+        reporter.testStatus(test.meta, TestStatus.Failed(cause, Duration.Zero))
+    }
+  }
+
+  protected def proceed(appmodule: ModuleBase,
+                        checker: PlanCircularDependencyCheck,
+                        testplans: Seq[(DistageTest[F], OrderedPlan)],
+                        shared: TriSplittedPlan,
+                        parent: Locator,
+                       )(implicit F: DIEffect[F], P: DIEffectAsync[F]): F[Unit] = {
     // here we produce our shared plan
     checker.verify(shared.primary)
+
     Injector.inherit(parent).produceF[F](shared.primary).use {
       mainSharedLocator =>
         val testInjector = Injector.inherit(mainSharedLocator)
 
+        val testsBySuite = testplans.groupBy {
+          t =>
+            val testId = t._1.meta.id
+            SuiteData(testId.suiteName, testId.suiteId, testId.suiteClassName)
+        }
         // now we are ready to run each individual test
         // note: scheduling here is custom also and tests may automatically run in parallel for any non-trivial monad
-        val tests = configuredTraverse_(testplans.groupBy {
-          t =>
-            val id = t._1.meta.id
-            SuiteData(id.suiteName, id.suiteId, id.suiteClassName)
-        }) {
-          case (id, plans) =>
-            for {
-              _ <- F.maybeSuspend(reporter.beginSuite(id))
-              _ <- configuredTraverse_(plans) {
+        configuredTraverse_(testsBySuite) {
+          case (suiteData, plans) =>
+            F.bracket(
+              acquire = F.maybeSuspend(reporter.beginSuite(suiteData))
+            )(release = _ => F.maybeSuspend(reporter.endSuite(suiteData))) { _ =>
+              configuredTraverse_(plans) {
                 case (test, testplan) =>
                   val allSharedKeys = mainSharedLocator.allInstances.map(_.key).toSet
 
@@ -205,58 +224,47 @@ class DistageTestRunner[F[_]: TagK]
                       }
                   }
               }
-              _ <- F.maybeSuspend(reporter.endSuite(id))
-            } yield ()
-        }
-
-        F.definitelyRecover(tests.flatMap(_ => F.maybeSuspend(reporter.endAll()))) {
-          f =>
-            F.maybeSuspend(reporter.onFailure(f)).flatMap(_ => F.fail(f))
+            }
         }
-
     }
   }
 
-  private def proceedIndividual(test: DistageTest[F], newtestplan: TriSplittedPlan, parent: Locator)(implicit F: DIEffect[F]): F[Unit] = {
-    Injector.inherit(parent)
-      .produceF[F](newtestplan.primary).use {
-      testLocator =>
-        def doRun(before: Long): F[Unit] = {
-          for {
-            _ <- F.definitelyRecover(testLocator.run(test.test).flatMap {
-              _ =>
-                F.maybeSuspend {
-                  val after = System.nanoTime()
-                  val testDuration = FiniteDuration(after - before, TimeUnit.NANOSECONDS)
-                  reporter.testStatus(test.meta, TestStatus.Succeed(testDuration))
-                }
-            }) {
-              case s if isTestSkipException(s) =>
-                F.maybeSuspend {
-                  val after = System.nanoTime()
-                  val testDuration = FiniteDuration(after - before, TimeUnit.NANOSECONDS)
-                  reporter.testStatus(test.meta, TestStatus.Cancelled(s.getMessage, testDuration))
-                }
-              case o =>
-                F.fail(o)
-            }
-          } yield ()
-        }
+  protected def proceedIndividual(test: DistageTest[F], newtestplan: TriSplittedPlan, parent: Locator)(implicit F: DIEffect[F]): F[Unit] = {
+    def testDuration(before: Option[Long]): FiniteDuration = {
+      before.fold(Duration.Zero) {
+        before =>
+          val after = System.nanoTime()
+          FiniteDuration(after - before, TimeUnit.NANOSECONDS)
+      }
+    }
+
+    def doRecover(before: Option[Long])(action: => F[Unit]): F[Unit] = {
+      F.definitelyRecoverCause(action) {
+        case (s, _) if isTestSkipException(s) =>
+          F.maybeSuspend {
+            reporter.testStatus(test.meta, TestStatus.Cancelled(s.getMessage, testDuration(before)))
+          }
+        case (_, cause) =>
+          F.maybeSuspend {
+            reporter.testStatus(test.meta, TestStatus.Failed(cause, testDuration(before)))
+          }
+      }
+    }
 
-        def doRecover(before: Long): Throwable => F[Unit] = {
-          // TODO: here we may also ignore individual tests
-          throwable =>
-            F.maybeSuspend {
-              val after = System.nanoTime()
-              reporter.testStatus(test.meta, TestStatus.Failed(throwable, FiniteDuration(after - before, TimeUnit.NANOSECONDS)))
+    doRecover(None) {
+      Injector.inherit(parent).produceF[F](newtestplan.primary).use {
+        testLocator =>
+          F.suspendF {
+            val before = System.nanoTime()
+            reporter.testStatus(test.meta, TestStatus.Running)
+
+            doRecover(Some(before)) {
+              testLocator
+                .run(test.test)
+                .flatMap(_ => F.maybeSuspend(reporter.testStatus(test.meta, TestStatus.Succeed(testDuration(Some(before))))))
             }
+          }
         }
-
-        for {
-          before <- F.maybeSuspend(System.nanoTime())
-          _ <- F.maybeSuspend(reporter.testStatus(test.meta, TestStatus.Running))
-          _ <- F.definitelyRecoverCause(doRun(before))(doRecover(before))
-        } yield ()
     }
   }
 
@@ -268,6 +276,17 @@ class DistageTestRunner[F[_]: TagK]
     }
   }
 
+  protected def configuredForeach[A](environments: Iterable[A])(f: A => Unit): Unit = {
+    if (parallelEnvs && environments.size > 1) {
+      val ec = ExecutionContext.fromExecutorService(null)
+      try {
+        DIEffectAsync.diEffectParIdentity.parTraverse_(environments)(f)
+      } finally ec.shutdown()
+    } else {
+      environments.foreach(f)
+    }
+  }
+
   lazy val debugLogger: TrivialLogger = TrivialLogger.make[DistageTestRunner[F]](DebugProperties.`izumi.distage.testkit.debug`)
 }
 
@@ -283,12 +302,10 @@ object DistageTestRunner {
 
   sealed trait TestStatus
   object TestStatus {
-    //    case object Scheduled extends TestStatus
     case object Running extends TestStatus
 
     sealed trait Done extends TestStatus
     final case class Ignored(checks: Seq[ResourceCheck.Failure]) extends Done
-
     sealed trait Finished extends Done
     final case class Cancelled(clue: String, duration: FiniteDuration) extends Finished
     final case class Succeed(duration: FiniteDuration) extends Finished
@@ -303,4 +320,12 @@ object DistageTestRunner {
     def testStatus(test: TestMeta, testStatus: TestStatus): Unit
   }
 
+  object ProvisioningIntegrationException {
+    def unapply(arg: ProvisioningException): Option[Seq[ResourceCheck.Failure]] = {
+      Some(arg.getSuppressed.collect { case i: IntegrationCheckException => i.failures }.toSeq)
+        .filter(_.nonEmpty)
+        .map(_.flatten)
+    }
+  }
+
 }
diff --git a/distage/distage-testkit-legacy/src/main/scala/izumi/distage/testkit/services/scalatest/adapter/DistagePluginTestSupport.scala b/distage/distage-testkit-legacy/src/main/scala/izumi/distage/testkit/services/scalatest/adapter/DistagePluginTestSupport.scala
index b135ea8b9..7121e0ba8 100644
--- a/distage/distage-testkit-legacy/src/main/scala/izumi/distage/testkit/services/scalatest/adapter/DistagePluginTestSupport.scala
+++ b/distage/distage-testkit-legacy/src/main/scala/izumi/distage/testkit/services/scalatest/adapter/DistagePluginTestSupport.scala
@@ -4,10 +4,8 @@ import distage.SafeType
 import izumi.distage.model.definition.Activation
 import izumi.distage.model.definition.StandardAxis._
 import izumi.distage.plugins.PluginConfig
-import izumi.distage.plugins.load.{PluginLoader, PluginLoaderDefaultImpl}
 import izumi.distage.testkit.TestConfig
 import izumi.distage.testkit.services.dstest.{DistageTestEnv, TestEnvironment}
-import izumi.fundamentals.platform.language.unused
 import izumi.fundamentals.reflection.Tags.TagK
 import izumi.logstage.api.IzLogger
 
diff --git a/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageAbstractScalatestSpec.scala b/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageAbstractScalatestSpec.scala
index 9e899022a..e3f915d33 100644
--- a/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageAbstractScalatestSpec.scala
+++ b/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageAbstractScalatestSpec.scala
@@ -3,7 +3,6 @@ package izumi.distage.testkit.services.scalatest.dstest
 import distage.{Tag, TagK, TagKK}
 import izumi.distage.model.effect.DIEffect
 import izumi.distage.model.providers.ProviderMagnet
-import izumi.distage.plugins.load.PluginLoader
 import izumi.distage.testkit.TestConfig
 import izumi.distage.testkit.services.dstest.DistageTestRunner.{DistageTest, TestId, TestMeta}
 import izumi.distage.testkit.services.dstest._
diff --git a/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageTestsRegistrySingleton.scala b/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageTestsRegistrySingleton.scala
index 08cd04ab5..190fd7fbb 100644
--- a/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageTestsRegistrySingleton.scala
+++ b/distage/distage-testkit-scalatest/src/main/scala/izumi/distage/testkit/services/scalatest/dstest/DistageTestsRegistrySingleton.scala
@@ -5,6 +5,7 @@ import java.util.concurrent.atomic.AtomicBoolean
 
 import distage.{SafeType, TagK}
 import izumi.distage.testkit.services.dstest.DistageTestRunner.DistageTest
+import izumi.functional.Value
 import izumi.fundamentals.platform.language.Quirks._
 import org.scalatest.{StatefulStatus, Tracker}
 
@@ -14,7 +15,7 @@ object DistageTestsRegistrySingleton {
   protected[this] type Fake[T]
   private[this] object Fake
   private[this] val registry = new mutable.HashMap[SafeType, mutable.ArrayBuffer[DistageTest[Fake]]]()
-  private[this] val statuses = new mutable.HashMap[SafeType, mutable.HashMap[String, StatefulStatus]]()
+  private[this] val statuses = new mutable.HashMap[SafeType, Option[mutable.HashMap[String, StatefulStatus]]]()
   private[this] val trackers = new mutable.HashMap[String, Either[mutable.ArrayBuffer[Tracker => Unit], Tracker]]()
   private[this] val runTracker = new ConcurrentHashMap[SafeType, Fake.type]()
   private[this] val knownSuites = new ConcurrentHashMap[(SafeType, String), Fake.type]()
@@ -31,7 +32,9 @@ object DistageTestsRegistrySingleton {
 
   def register[F[_]: TagK](t: DistageTest[F]): Unit = synchronized {
     if (registrationOpen.get()) {
-      registry.getOrElseUpdate(SafeType.getK[F], mutable.ArrayBuffer.empty).append(t.asInstanceOf[DistageTest[Fake]]).discard()
+      registry
+        .getOrElseUpdate(SafeType.getK[F], mutable.ArrayBuffer.empty)
+        .append(castTest(t))
     }
   }
 
@@ -45,24 +48,33 @@ object DistageTestsRegistrySingleton {
   }
 
   def registeredTests[F[_]: TagK]: Seq[DistageTest[F]] = synchronized {
-    registry.getOrElseUpdate(SafeType.getK[F], mutable.ArrayBuffer.empty).map(_.asInstanceOf[DistageTest[F]]).toSeq
+    val arr = registry.getOrElseUpdate(SafeType.getK[F], mutable.ArrayBuffer.empty)
+    castArray(arr).toSeq
   }
 
   def registerStatus[F[_]: TagK](suiteId: String): StatefulStatus = synchronized {
     statuses
-      .getOrElseUpdate(SafeType.getK[F], mutable.HashMap.empty)
-      .getOrElseUpdate(suiteId, new StatefulStatus)
+      .getOrElseUpdate(SafeType.getK[F], Some(mutable.HashMap.empty))
+      .fold {
+        // return completed test if the runner has already ran before this test got registered
+        Value(new StatefulStatus).eff(_.setCompleted()).get
+      } {
+        _.getOrElseUpdate(suiteId, new StatefulStatus)
+      }
   }
 
   def completeStatuses[F[_]: TagK](): Unit = synchronized {
-    statuses
-      .getOrElseUpdate(SafeType.getK[F], mutable.HashMap.empty)
-      .valuesIterator.foreach {
-        status =>
-          if (!status.isCompleted) {
-            status.setCompleted()
-          }
+    statuses.get(SafeType.getK[F])
+      .flatten
+      .foreach {
+        _.valuesIterator.foreach {
+          status =>
+            if (!status.isCompleted) {
+              status.setCompleted()
+            }
+        }
       }
+    statuses.put(SafeType.getK[F], None).discard()
   }
 
   def runReport(suiteId: String)(f: Tracker => Unit): Unit = synchronized {
@@ -82,4 +94,7 @@ object DistageTestsRegistrySingleton {
       case Right(_) =>
     }
   }
+
+  @inline private[this] def castTest[F[_]](t: DistageTest[F]): DistageTest[Fake] = t.asInstanceOf[DistageTest[Fake]]
+  @inline private[this] def castArray[C[_], F[_]](a: C[DistageTest[Fake]]): C[DistageTest[F]] = a.asInstanceOf[C[DistageTest[F]]]
 }
diff --git a/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala b/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala
index 91478f6aa..698e6aa4b 100644
--- a/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala
+++ b/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala
@@ -195,7 +195,7 @@ abstract class DistageScalatestTestSuiteRunner[F[_]](implicit override val tagMo
         val runner = {
           val logger = IzLogger(Log.Level.Debug)("phase" -> "test")
           val checker = new IntegrationChecker.Impl(logger)
-          new DistageTestRunner[F](testReporter, checker, specEnv, toRun, _.isInstanceOf[TestCanceledException], parallelTestsAlways)
+          new DistageTestRunner[F](testReporter, checker, specEnv, toRun, _.isInstanceOf[TestCanceledException], parallelTestsAlways, parallelEnvs)
         }
         runner.run()
       }
@@ -205,6 +205,7 @@ abstract class DistageScalatestTestSuiteRunner[F[_]](implicit override val tagMo
   }
 
   protected def parallelTestsAlways: Boolean = true
+  protected def parallelEnvs: Boolean = true
 
   private def mkTestReporter(args: Args): TestReporter = {
     val scalatestReporter = new ScalatestReporter(args.reporter)
diff --git a/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/SafeTestReporter.scala b/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/SafeTestReporter.scala
index 707e3407f..b973ef0e9 100644
--- a/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/SafeTestReporter.scala
+++ b/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/SafeTestReporter.scala
@@ -1,15 +1,14 @@
 package org.scalatest
 
-import izumi.distage.testkit.services.dstest.DistageTestRunner.TestStatus.{Done, Finished, Ignored, Running}
+import izumi.distage.testkit.services.dstest.DistageTestRunner.TestStatus.{Done, Running}
 import izumi.distage.testkit.services.dstest.DistageTestRunner.{SuiteData, TestMeta, TestReporter, TestStatus}
-import izumi.fundamentals.platform.language.Quirks._
 import izumi.fundamentals.platform.language.unused
 
 import scala.collection.mutable
 
 class SafeTestReporter(underlying: TestReporter) extends TestReporter {
   private val delayedReports = new mutable.LinkedHashMap[TestMeta, TestStatus]()
-  private var runningTest: Option[TestMeta] = None
+  private val runningTests = new mutable.HashMap[String, TestMeta]()
 
   override def onFailure(f: Throwable): Unit = synchronized {
     endAll()
@@ -17,7 +16,7 @@ class SafeTestReporter(underlying: TestReporter) extends TestReporter {
   }
 
   override def endAll(): Unit = synchronized {
-    finish()
+    finish(_ => true)
     underlying.endAll()
   }
 
@@ -27,35 +26,32 @@ class SafeTestReporter(underlying: TestReporter) extends TestReporter {
 
   override def testStatus(test: TestMeta, testStatus: TestStatus): Unit = synchronized {
     testStatus match {
-//      case TestStatus.Scheduled => reportStatus(test, testStatus)
-
       case TestStatus.Running =>
-        runningTest match {
+        runningTests.get(test.id.suiteId) match {
           case Some(_) =>
+            // a test is running in this suite, delay new test
             delayedReports.put(test, testStatus)
           case None =>
+            // no test is running in this suite, report this one and set it as running
             reportStatus(test, testStatus)
-            runningTest = Some(test)
+            runningTests(test.id.suiteId) = test
         }
-
-      case _: Ignored =>
-        runningTest match {
-          case None =>
-            reportStatus(test, TestStatus.Running)
+      case _: Done =>
+        runningTests.get(test.id.suiteId) match {
+          case Some(value) if value == test =>
+            // finished current running test in suite,
+            // report delayed reports in suite &
+            // switch to another currently running test in suite if exists
+            runningTests.remove(test.id.suiteId)
             reportStatus(test, testStatus)
+            finish(_.id.suiteId == test.id.suiteId)
           case Some(_) =>
-            // Ignores happen before running, so an ignored test cannot be the same as `signalled`
-            delayedReports.put(test, testStatus)
-        }
-
-      case _: Finished =>
-        runningTest match {
-          case Some(value) if value == test =>
-            finishTest(test, testStatus)
-            runningTest = None
-            finish()
-          case _ =>
+            // another test is running in this suite, delay finish report
             delayedReports.put(test, testStatus)
+          case None =>
+            // no other test is running in this suite, report directly
+            reportStatus(test, TestStatus.Running)
+            reportStatus(test, testStatus)
         }
     }
     ()
@@ -65,26 +61,24 @@ class SafeTestReporter(underlying: TestReporter) extends TestReporter {
     underlying.testStatus(test, testStatus)
   }
 
-  private def finish(): Unit = {
-    val finishedTests = delayedReports.collect { case (t, s: Done) => (t, s) }
+  private def finish(predicate: TestMeta => Boolean): Unit = synchronized {
+    val finishedTests = delayedReports.collect { case (t, s: Done) if predicate(t) => (t, s) }
 
     // report all finished tests
     finishedTests
       .foreach {
         case (t, s) =>
           reportStatus(t, TestStatus.Running)
-          finishTest(t, s)
+          reportStatus(t, s)
+          delayedReports.remove(t)
       }
     // switch to another test if it's already running
-    delayedReports.collectFirst {
-      case (t, Running) =>
+    delayedReports.toList.foreach {
+      case (t, Running) if predicate(t) && !runningTests.contains(t.id.suiteId) =>
+        runningTests(t.id.suiteId) = t
         reportStatus(t, TestStatus.Running)
-        runningTest = Some(t)
-    }.discard()
-  }
-
-  private def finishTest(test: TestMeta, testStatus: TestStatus): Option[TestStatus] = {
-    reportStatus(test, testStatus)
-    delayedReports.remove(test)
+        delayedReports.remove(t)
+      case _ =>
+    }
   }
 }