Repository: 7mind/izumi
Pull request #807 (2020-01-02 10:40:44)

Feature/weak reference memoization

codecov-io: # [Codecov](https://codecov.io/gh/7mind/izumi/pull/807?src=pr&el=h1) Report
> :exclamation: No coverage uploaded for pull request base (`series/zio-RC16@3f8a4a6`). [Click here to learn what that means](https://docs.codecov.io/docs/error-reference#section-missing-base-commit).
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/7mind/izumi/pull/807/graphs/tree.svg?width=650&token=77pa3UU1Je&height=150&src=pr)](https://codecov.io/gh/7mind/izumi/pull/807?src=pr&el=tree)

```diff
@@                Coverage Diff                 @@
##             series/zio-RC16     #807   +/-   ##
==================================================
  Coverage                   ?   70.41%           
==================================================
  Files                      ?      550           
  Lines                      ?    14029           
  Branches                   ?      740           
==================================================
  Hits                       ?     9879           
  Misses                     ?     4150           
  Partials                   ?        0
```


| [Impacted Files](https://codecov.io/gh/7mind/izumi/pull/807?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [...ge/testkit/services/dstest/DistageTestRunner.scala](https://codecov.io/gh/7mind/izumi/pull/807/diff?src=pr&el=tree#diff-ZGlzdGFnZS9kaXN0YWdlLXRlc3RraXQtY29yZS9zcmMvbWFpbi9zY2FsYS9penVtaS9kaXN0YWdlL3Rlc3RraXQvc2VydmljZXMvZHN0ZXN0L0Rpc3RhZ2VUZXN0UnVubmVyLnNjYWxh) | `88.63% <ø> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/7mind/izumi/pull/807?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/7mind/izumi/pull/807?src=pr&el=footer). Last update [3f8a4a6...099238e](https://codecov.io/gh/7mind/izumi/pull/807?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).

neko-kai: @Caparow Could you please rebase on top of `develop`, not `series/zio-RC16` ?
Caparow: @neko-kai I suppose that all done ?

Patch:
diff --git a/distage/distage-core-api/src/main/scala/izumi/distage/model/definition/ModuleBase.scala b/distage/distage-core-api/src/main/scala/izumi/distage/model/definition/ModuleBase.scala
index 4d62a75cc..c9f8c368e 100644
--- a/distage/distage-core-api/src/main/scala/izumi/distage/model/definition/ModuleBase.scala
+++ b/distage/distage-core-api/src/main/scala/izumi/distage/model/definition/ModuleBase.scala
@@ -59,6 +59,21 @@ object ModuleBase {
       T.make(module.bindings.map(f))
     }
 
+    def foldLeft[B](b: B)(f: (B, Binding) => B): B = {
+      module.bindings.foldLeft(b) {
+        case (acc, bind) => f(acc, bind)
+      }
+    }
+
+    def foldLeftWith[B](b: B)(f: (B, Binding) => (B, Binding)): (B, T) = {
+      val (bindings, fold) = foldLeft(List.empty[Binding] -> b) {
+        case ((list, acc), bind) =>
+          val (acc1, el) = f(acc, bind)
+          (el :: list) -> acc1
+      }
+      fold -> T.make(bindings.toSet)
+    }
+
     def flatMap(f: Binding => Iterable[Binding]): T = {
       T.make(module.bindings.flatMap(f))
     }
diff --git a/distage/distage-framework/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala b/distage/distage-framework/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala
index d425596c9..1c4813e60 100644
--- a/distage/distage-framework/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala
+++ b/distage/distage-framework/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala
@@ -22,12 +22,14 @@ trait ConfigLoader {
 object ConfigLoader {
 
   /**
-    * default config locations:
-    *   - common-reference.conf
-    *   - common-reference-dev.conf
-    *   - $roleName-reference.conf
-    *   - $roleName-reference-dev.conf
-    */
+   * default config locations:
+   *   - common.conf
+   *   - common-reference.conf
+   *   - common-reference-dev.conf
+   *   - $roleName.conf
+   *   - $roleName-reference.conf
+   *   - $roleName-reference-dev.conf
+   */
   class LocalFSImpl(
     logger: IzLogger,
     primaryConfig: Option[File],
@@ -98,6 +100,7 @@ object ConfigLoader {
 
     protected def defaultConfigReferences(name: String): Seq[ConfigSource] = {
       Seq(
+        ConfigSource.Resource(s"$name.conf", ResourceConfigKind.Primary),
         ConfigSource.Resource(s"$name-reference.conf", ResourceConfigKind.Primary),
         ConfigSource.Resource(s"$name-reference-dev.conf", ResourceConfigKind.Development),
       )


Test patch:
diff --git a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/SpecConfig.scala b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/SpecConfig.scala
index 6d49cdbeb..5384394c4 100644
--- a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/SpecConfig.scala
+++ b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/SpecConfig.scala
@@ -1,6 +1,7 @@
 package izumi.distage.testkit
 
 import distage.{BootstrapModule, ModuleBase}
+import izumi.distage.config.model.AppConfig
 import izumi.distage.framework.config.PlanningOptions
 import izumi.logstage.api.Log
 
@@ -9,4 +10,5 @@ final case class SpecConfig(
                              bootstrapOverrides: BootstrapModule = BootstrapModule.empty,
                              moduleOverrides: ModuleBase = ModuleBase.empty,
                              bootstrapLogLevel: Log.Level = Log.Level.Info,
+                             configOverrides: AppConfig => AppConfig = identity
                            )
diff --git a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala
index 445e5a95d..ecaa639fe 100644
--- a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala
+++ b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/DistageTestRunner.scala
@@ -7,7 +7,8 @@ import izumi.distage.framework.model.IntegrationCheck
 import izumi.distage.framework.model.exceptions.IntegrationCheckException
 import izumi.distage.framework.services.{IntegrationChecker, PlanCircularDependencyCheck}
 import izumi.distage.model.Locator
-import izumi.distage.model.definition.ModuleBase
+import izumi.distage.model.definition.Binding.SetElementBinding
+import izumi.distage.model.definition.{ImplDef, ModuleBase}
 import izumi.distage.model.effect.DIEffect.syntax._
 import izumi.distage.model.effect.{DIEffect, DIEffectAsync, DIEffectRunner}
 import izumi.distage.model.exceptions.ProvisioningException
@@ -21,7 +22,7 @@ import izumi.fundamentals.reflection.Tags.TagK
 
 import scala.concurrent.duration.FiniteDuration
 
-class DistageTestRunner[F[_]: TagK]
+class DistageTestRunner[F[_] : TagK]
 (
   reporter: TestReporter,
   integrationChecker: IntegrationChecker[F],
@@ -70,15 +71,23 @@ class DistageTestRunner[F[_]: TagK]
             distageTest -> injector.plan(PlannerInput(appModule, keys))
         }
 
+        val wholeEnvKeys = testPlans.flatMap(_._2.steps).map(_.target).toSet
+
         // here we find all the shared components in each of our individual tests
-        val sharedKeys = testPlans.map(_._2).flatMap {
-          plan =>
-            plan.steps.filter(env memoizationRoots _.target).map(_.target)
-        }.toSet -- runtimeGcRoots
+        val sharedKeys = wholeEnvKeys.filter(env.memoizationRoots) -- runtimeGcRoots
 
         logger.info(s"Memoized components in env: $sharedKeys")
 
-        val shared = injector.trisectByKeys(appModule.drop(runtimeGcRoots), sharedKeys) {
+        val (strengthenKeys, strengthen) = appModule.drop(runtimeGcRoots).foldLeftWith(List.empty[DIKey]) {
+          case (acc, b@SetElementBinding(key, r: ImplDef.ReferenceImpl, _, _)) if r.weak && wholeEnvKeys(key) =>
+            (key :: acc) -> b.copy(implementation = r.copy(weak = false))
+          case (acc, b) =>
+            acc -> b
+        }
+
+        logger.info(s"Strengthen weak components in env: $strengthenKeys")
+
+        val shared = injector.trisectByKeys(strengthen, sharedKeys) {
           _.collectChildren[IntegrationCheck].map(_.target).toSet
         }
 
@@ -258,7 +267,7 @@ object DistageTestRunner {
 
   sealed trait TestStatus
   object TestStatus {
-//    case object Scheduled extends TestStatus
+    //    case object Scheduled extends TestStatus
     case object Running extends TestStatus
 
     sealed trait Done extends TestStatus
diff --git a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/SpecEnvironment.scala b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/SpecEnvironment.scala
index c84e777da..a5f671b0c 100644
--- a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/SpecEnvironment.scala
+++ b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/services/dstest/SpecEnvironment.scala
@@ -20,6 +20,7 @@ trait SpecEnvironment {
 
   def contextOptions: PlanningOptions
 
+  def configOverrides: AppConfig => AppConfig
   def makeConfigLoader(logger: IzLogger): ConfigLoader
   def makeModuleProvider(options: PlanningOptions, config: AppConfig, lateLogger: IzLogger, roles: RolesInfo, activationInfo: ActivationInfo, activation: Activation): ModuleProvider
 }
@@ -33,6 +34,7 @@ object SpecEnvironment {
     override val bootstrapOverrides: BootstrapModule,
     override val moduleOverrides: ModuleBase,
     override val bootstrapLogLevel: Log.Level,
+    override val configOverrides: AppConfig => AppConfig
   ) extends SpecEnvironment {
 
     override def makeLogger(): IzLogger = {
@@ -47,8 +49,9 @@ object SpecEnvironment {
       val moreConfigs = Map(
         s"$lastPackage-test" -> None,
         s"$classname-test" -> None,
+        "application" -> None,
       )
-      new ConfigLoader.LocalFSImpl(logger, None, moreConfigs)
+      new ConfigLoader.LocalFSImpl(logger, None, moreConfigs) map configOverrides
     }
 
     override def makeModuleProvider(options: PlanningOptions, config: AppConfig, lateLogger: IzLogger, roles: RolesInfo, activationInfo: ActivationInfo, activation: Activation): ModuleProvider = {
diff --git a/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala b/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala
index ba928c776..111863618 100644
--- a/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala
+++ b/distage/distage-testkit-scalatest/src/main/scala/org/scalatest/DistageScalatestTestSuiteRunner.scala
@@ -80,6 +80,7 @@ trait DistageScalatestTestSuiteRunner[F[_]] extends Suite with AbstractDistageSp
       bootstrapOverrides = c.bootstrapOverrides,
       moduleOverrides = c.moduleOverrides,
       bootstrapLogLevel = c.bootstrapLogLevel,
+      configOverrides = c.configOverrides
     )
   }
 
diff --git a/distage/distage-testkit-scalatest/src/test/scala/izumi/distage/testkit/distagesuite/DistageTestExample.scala b/distage/distage-testkit-scalatest/src/test/scala/izumi/distage/testkit/distagesuite/DistageTestExample.scala
index d6b112f52..f30d93a7f 100644
--- a/distage/distage-testkit-scalatest/src/test/scala/izumi/distage/testkit/distagesuite/DistageTestExample.scala
+++ b/distage/distage-testkit-scalatest/src/test/scala/izumi/distage/testkit/distagesuite/DistageTestExample.scala
@@ -1,6 +1,6 @@
 package izumi.distage.testkit.distagesuite
 
-import java.util.concurrent.atomic.AtomicReference
+import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
 
 import cats.effect.{IO => CIO}
 import distage._
@@ -9,10 +9,12 @@ import izumi.distage.model.effect.DIEffect
 import izumi.distage.model.effect.DIEffect.syntax._
 import izumi.distage.plugins.PluginConfig
 import izumi.distage.testkit.TestConfig
+import izumi.distage.testkit.distagesuite.DistageTestExampleBase.{SetCounter, SetElement, SetElement1, SetElement2, SetElement3}
 import izumi.distage.testkit.distagesuite.fixtures.{ApplePaymentProvider, MockCache, MockCachedUserService, MockUserRepository}
 import izumi.distage.testkit.scalatest.{DistageBIOSpecScalatest, DistageSpecScalatest}
 import izumi.distage.testkit.services.scalatest.dstest.DistageAbstractScalatestSpec
 import izumi.fundamentals.platform.functional.Identity
+import izumi.fundamentals.platform.language.Quirks
 import zio.Task
 
 trait DistageMemoizeExample[F[_]] extends DistageAbstractScalatestSpec[F] {
@@ -20,6 +22,8 @@ trait DistageMemoizeExample[F[_]] extends DistageAbstractScalatestSpec[F] {
     super.config.copy(
       memoizationRoots = Set(
         DIKey.get[MockCache[F]],
+        DIKey.get[Set[SetElement]],
+        DIKey.get[SetCounter],
       ))
   }
 }
@@ -37,15 +41,79 @@ class DistageTestExampleBIO extends DistageBIOSpecScalatest[zio.IO] with Distage
 
 }
 
-abstract class DistageTestExampleBase[F[_]: TagK](implicit F: DIEffect[F]) extends DistageSpecScalatest[F] with DistageMemoizeExample[F] {
+object DistageTestExampleBase {
+  class SetCounter {
+    import Quirks._
+    private[this] val c: AtomicInteger = new AtomicInteger(0)
+    def inc(): Unit = c.incrementAndGet().discard()
+    def get: Int = c.get()
+  }
+  sealed trait SetElement {
+    Quirks.discard(counter.inc())
+    def counter: SetCounter
+    def v: Int
+  }
+  final case class SetElement1(v: Int, counter: SetCounter) extends SetElement
+  final case class SetElement2(v: Int, counter: SetCounter) extends SetElement
+  final case class SetElement3(v: Int, counter: SetCounter) extends SetElement
+}
+
+abstract class DistageTestExampleBase[F[_] : TagK](implicit F: DIEffect[F]) extends DistageSpecScalatest[F] with DistageMemoizeExample[F] {
 
   override protected def config: TestConfig = super.config.copy(
-      pluginSource = super.config.pluginSource ++ PluginSource(PluginConfig.cached(Seq("xxx")))
-    )
+    pluginSource = super.config.pluginSource ++ PluginSource(PluginConfig.cached(Seq("xxx"))) ++ new ModuleDef {
+      make[SetCounter]
+
+      make[SetElement1].from(SetElement1(1, _))
+      make[SetElement2].from(SetElement2(2, _))
+      make[SetElement3].from(SetElement3(3, _))
+
+      many[SetElement]
+        .weak[SetElement1]
+        .weak[SetElement2]
+        .weak[SetElement3]
+
+      many[SetElement].named("set-id")
+        .weak[SetElement1]
+        .weak[SetElement2]
+        .weak[SetElement3]
+    }
+  )
 
   val XXX_Whitebox_memoizedMockCache = new AtomicReference[MockCache[F]]
 
   "distage test custom runner" should {
+    "support memoized weak sets" in {
+      (
+        set: Set[SetElement],
+        s1: SetElement1,
+        s2: SetElement2,
+        s3: SetElement3,
+      ) =>
+        Quirks.discard(s1, s2, s3)
+        F.maybeSuspend(assert(set.size == 3))
+    }
+
+    "support memoized named weak sets" in {
+      (
+        set: Set[SetElement]@Id("set-id"),
+        s1: SetElement1,
+        s2: SetElement2,
+        s3: SetElement3,
+      ) =>
+        Quirks.discard(s1, s2, s3)
+        F.maybeSuspend(assert(set.size == 3))
+    }
+
+    "return memoized weak set with have whole list of members even if test does not depends on them" in {
+      (
+        set: Set[SetElement],
+        c: SetCounter
+      ) =>
+        assume(c.get != 0)
+        F.maybeSuspend(assert(set.size == 3))
+    }
+
     "test 1" in {
       service: MockUserRepository[F] =>
         for {