Repository: scala-steward-org/scala-steward
Pull request #2913 (2023-01-11 22:06:11)

Implement HttpJsonClient#getAll as fs2.Stream
This implements `HttpJsonClient#getAll` as `fs2.Stream`. The nice thing about this is that the consumer of the stream has control over how many requests are made independent of how many pages exists in total.

`GitHubAppApiAlg` still accumulates all values into a list, so this change does not make a difference in practice but I think it is a nicer foundation.

I should also note that the values returned by `getAll` are now in the same order is in the pages. This means Scala Steward as GitHub App will work on repos in the same order as in the `/installation/repositories` responses.


Patch:
diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/HttpJsonClient.scala b/modules/core/src/main/scala/org/scalasteward/core/util/HttpJsonClient.scala
index cc708237..5cfe2d6e 100644
--- a/modules/core/src/main/scala/org/scalasteward/core/util/HttpJsonClient.scala
+++ b/modules/core/src/main/scala/org/scalasteward/core/util/HttpJsonClient.scala
@@ -18,6 +18,7 @@ package org.scalasteward.core.util
 
 import cats.effect.Concurrent
 import cats.syntax.all._
+import fs2.Stream
 import io.circe.{Decoder, Encoder}
 import org.http4s.Method.{GET, PATCH, POST, PUT}
 import org.http4s.Status.Successful
@@ -36,19 +37,16 @@ final class HttpJsonClient[F[_]](implicit
   def get[A: Decoder](uri: Uri, modify: ModReq): F[A] =
     request[A](GET, uri, modify)
 
-  def getAll[A: Decoder](uri: Uri, modify: ModReq): F[List[A]] =
-    all[A](GET, uri, modify, Nil)
-
-  private def all[A: Decoder](
-      method: Method,
-      uri: Uri,
-      modify: ModReq,
-      xs: List[A]
-  ): F[List[A]] =
-    requestWithHeaders[A](method, uri, modify)(jsonOf).flatMap { case (a, headers) =>
-      headers.get[Link].flatMap(_.values.find(_.rel.contains("next"))) match {
-        case Some(linkValue) => all(method, linkValue.uri, modify, a :: xs)
-        case None            => F.pure(a :: xs)
+  /** Retrieves all values via pagination.
+    * @see
+    *   [[https://tools.ietf.org/html/rfc8288]]
+    *   [[https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api]]
+    */
+  def getAll[A: Decoder](uri: Uri, modify: ModReq): Stream[F, A] =
+    Stream.unfoldLoopEval(uri) { curr =>
+      requestWithHeaders[A](GET, curr, modify)(jsonOf).map { case (a, headers) =>
+        val next = headers.get[Link].flatMap(_.values.find(_.rel.contains("next"))).map(_.uri)
+        (a, next)
       }
     }
 
diff --git a/modules/core/src/main/scala/org/scalasteward/core/vcs/github/GitHubAppApiAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/vcs/github/GitHubAppApiAlg.scala
index 274394e9..08f971cb 100644
--- a/modules/core/src/main/scala/org/scalasteward/core/vcs/github/GitHubAppApiAlg.scala
+++ b/modules/core/src/main/scala/org/scalasteward/core/vcs/github/GitHubAppApiAlg.scala
@@ -16,16 +16,17 @@
 
 package org.scalasteward.core.vcs.github
 
-import cats.Applicative
-import cats.syntax.functor._
+import cats.effect.Concurrent
+import cats.syntax.all._
 import org.http4s.{Header, Uri}
 import org.scalasteward.core.util.HttpJsonClient
 import org.typelevel.ci._
 
-class GitHubAppApiAlg[F[_]: Applicative](
+class GitHubAppApiAlg[F[_]](
     gitHubApiHost: Uri
 )(implicit
-    client: HttpJsonClient[F]
+    client: HttpJsonClient[F],
+    F: Concurrent[F]
 ) {
 
   private[this] val acceptHeader =
@@ -33,7 +34,7 @@ class GitHubAppApiAlg[F[_]: Applicative](
 
   private[this] def addHeaders(jwt: String): client.ModReq =
     req =>
-      Applicative[F].point(
+      F.point(
         req.putHeaders(
           Header.Raw(ci"Authorization", s"Bearer $jwt"),
           acceptHeader
@@ -48,7 +49,8 @@ class GitHubAppApiAlg[F[_]: Applicative](
         (gitHubApiHost / "app" / "installations").withQueryParam("per_page", 100),
         addHeaders(jwt)
       )
-      .map(_.flatten)
+      .compile
+      .foldMonoid
 
   /** [[https://docs.github.com/en/free-pro-team@latest/rest/reference/apps#create-an-installation-access-token-for-an-app]]
     */
@@ -65,13 +67,15 @@ class GitHubAppApiAlg[F[_]: Applicative](
       .getAll[RepositoriesOut](
         (gitHubApiHost / "installation" / "repositories").withQueryParam("per_page", 100),
         req =>
-          Applicative[F].point(
+          F.point(
             req.putHeaders(
               Header.Raw(ci"Authorization", s"token $token"),
               acceptHeader
             )
           )
       )
+      .compile
+      .toList
       .map(values =>
         RepositoriesOut(
           values.flatMap(_.repositories)


Test patch:
diff --git a/modules/core/src/test/scala/org/scalasteward/core/util/HttpJsonClientTest.scala b/modules/core/src/test/scala/org/scalasteward/core/util/HttpJsonClientTest.scala
index 3b475f58..0fe80002 100644
--- a/modules/core/src/test/scala/org/scalasteward/core/util/HttpJsonClientTest.scala
+++ b/modules/core/src/test/scala/org/scalasteward/core/util/HttpJsonClientTest.scala
@@ -18,8 +18,8 @@ class HttpJsonClientTest extends CatsEffectSuite with Http4sDsl[MockEff] {
       case GET -> Root / "2" => Ok("2", Link(LinkValue(url1, Some("prev"))))
       case _                 => NotFound()
     })
-    val obtained = httpJsonClient.getAll[Int](url1, _.pure[MockEff]).runA(state)
-    assertIO(obtained, List(2, 1))
+    val obtained = httpJsonClient.getAll[Int](url1, _.pure[MockEff]).compile.toList.runA(state)
+    assertIO(obtained, List(1, 2))
   }
 
   test("get with malformed JSON") {