Repository: sky-uk/kafka-message-scheduler
Pull request #390: Add config to push to additional hosts
Author: login: Smudgey
Created: 2025-03-13T08:20:36Z | Merged: 2025-04-09T15:20:19Z

- Connects to https://github.com/sky-uk/kafka-message-scheduler/issues/387

Adding functionality to check for a local environment variable of CONTAINER_REPOSITORIES and push KMS images to these locations.

**Changes include:**
- renaming `scheduler3` project to `scheduler`, to be inline with the configuration in the `application.conf`
- fixed image build for multiple platforms
- implemented a use of multiple registries for the image pushing via an environment variable `CONTAINER_REPOSITORIES`

**Config changes:**
- `TopicConfig` had to be extracted to the top level, so that we can provide topic names via `JAVA_TOOL_OPTIONS`
- `avro` and `json` fields needs to be optional for pureConfig to be able to load only one set of topics (for example, only avro topics) when necessary. 


Diff:
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index de486428..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-language: scala
-
-scala:
-  - 2.13.10
-jdk:
-  - openjdk17
-
-# Avoid triggering a duplicate build for PRs
-branches:
-  only:
-    - master
-
-# https://www.scala-sbt.org/1.x/docs/Travis-CI-with-sbt.html
-cache:
-  directories:
-    - $HOME/.cache/coursier
-    - $HOME/.ivy2/cache
-    - $HOME/.sbt
-
-before_cache:
-  - rm -fv $HOME/.ivy2/.sbt.ivy.lock
-  - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
-  - find $HOME/.sbt        -name "*.lock"               -print -delete
-
-script:
-  - sbt ++$TRAVIS_SCALA_VERSION ciBuild
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b3bfa504..8eb8adf4 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -17,13 +17,14 @@ changes need some adjustment before they are ready for submission.
 
 The software is written in [Scala](https://scala-lang.org/) and is built with [SBT](http://www.scala-sbt.org/).
 
-To compile and run tests: `sbt test`.
+To compile and run tests start sbt with `./sbt` and then run `project scheduler; test;`. To run integrated tests run `project it; dockerComposeUp; test;` 
 
 ## Performing a release (for project maintainers)
 
-1. Run `docker login --username=<username>` and when prompted add the password for your Docker Hub account. Note that you need to be a member of the `skyuk` Docker Hub organization.
-2. Run `sbt ciRelease` to perform the release of the Docker artifact to Docker Hub.
-3. The Docker Image will be immediately live and therefore will need to be deleted if there has been a problem.
+1. The version of the project should follow the semantic versioning scheme on [semver.org](https://semver.org/) and [sbt-release](https://github.com/sbt/sbt-release#requirements) requirements.
+1. `version.sbt` file must be updated with the version to be released with each new PR. This will be the Release Version.
+1. Please note, that any qualifier such as `-SNAPSHOT` will be removed from the Release Version automatically during the `cdBuild` phase.
+1. The Docker Image will be released once the PR is merged.
 
 ## Contributor Code of Conduct
 
diff --git a/bibcd.yml b/bibcd.yml
index 379ece57..de2ebf17 100644
--- a/bibcd.yml
+++ b/bibcd.yml
@@ -6,7 +6,7 @@ triggering: master-and-prs
 modules:
 
   kafka-message-scheduler:
-    directory: /scheduler
+    directory: /scheduler-3
     pipelineShape: buildOnly
     serviceName: MetadataAssemblyPipeline
     additionalSourceDirectories:
diff --git a/build.sbt b/build.sbt
index 48515e7a..e11a4272 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,6 @@ import Release.*
 import DockerPublish.*
 import org.typelevel.scalacoptions.ScalacOptions
 import DockerComposeSettings.*
-import com.tapad.docker.DockerComposePlugin.autoImport.variablesForSubstitution
 
 ThisBuild / organization := "com.sky"
 
@@ -45,7 +44,7 @@ val buildInfoSettings = (pkg: String) =>
     buildInfoPackage := pkg
   )
 
-lazy val scheduler = (project in file("scheduler"))
+lazy val schedulerScala2 = (project in file("scheduler"))
   .enablePlugins(BuildInfoPlugin, JavaAppPackaging, UniversalDeployPlugin, JavaAgent, DockerPlugin)
   .settings(scala2Settings)
   .settings(
@@ -53,11 +52,11 @@ lazy val scheduler = (project in file("scheduler"))
     addCompilerPlugin("org.typelevel" % "kind-projector" % "0.13.3" cross CrossVersion.full),
     javaAgents += "io.kamon" % "kanela-agent" % "1.0.18",
     buildInfoSettings("com.sky"),
-    dockerSettings("kafka-message-scheduler"),
+    dockerSettings,
     releaseSettings
   )
 
-lazy val scheduler3 = (project in file("scheduler-3"))
+lazy val scheduler = (project in file("scheduler-3"))
   .enablePlugins(JavaAgent, DockerPlugin, JavaAppPackaging, BuildInfoPlugin)
   .settings(scala3Settings)
   .settings(javaOptions += "-Dotel.java.global-autoconfigure.enabled=true")
@@ -65,18 +64,11 @@ lazy val scheduler3 = (project in file("scheduler-3"))
   .settings(
     libraryDependencies ++= Dependencies.scheduler3,
     buildInfoSettings("uk.sky"),
+    dockerSettings,
+    releaseSettings,
     scalafixConfig := Some((ThisBuild / baseDirectory).value / ".scalafix3.conf"),
     scalafmtConfig := (ThisBuild / baseDirectory).value / ".scalafmt3.conf"
   )
-  .settings {
-    Seq(
-      dockerRepository      := sys.env.get("DOCKER_REPOSITORY"),
-      dockerBaseImage       := "eclipse-temurin:21-jre-jammy",
-      Docker / packageName  := "kafka-message-scheduler-3",
-      dockerUpdateLatest    := true,
-      dockerBuildxPlatforms := Seq("linux/arm64", "linux/amd64")
-    )
-  }
 
 lazy val it = (project in file("it"))
   .enablePlugins(DockerComposePlugin)
@@ -85,15 +77,15 @@ lazy val it = (project in file("it"))
     Seq(
       libraryDependencies ++= Dependencies.it,
       Test / fork             := true,
-      dockerImageCreationTask := (scheduler3 / Docker / publishLocal).value,
+      dockerImageCreationTask := (scheduler / Docker / publishLocal).value,
       composeFile             := "it/docker/docker-compose.yml",
       scalafixConfig          := Some((ThisBuild / baseDirectory).value / ".scalafix3.conf"),
       scalafmtConfig          := (ThisBuild / baseDirectory).value / ".scalafmt3.conf"
     )
   }
-  .settings(Seq(variablesForSubstitution ++= kafkaPort))
-  .settings(Seq(envVars := kafkaPort))
-  .dependsOn(scheduler3 % "compile->compile;test->test")
+  .settings(settings)
+  .settings(Seq(envVars := Map(kafkaPort)))
+  .dependsOn(scheduler % "compile->compile;test->test")
 
 val schema = inputKey[Unit]("Generate the Avro schema file for the Schedule schema.")
 
@@ -101,12 +93,12 @@ lazy val avro = (project in file("avro"))
   .settings(scala2Settings)
   .settings(libraryDependencies += Dependencies.avro4s)
   .settings(schema := (Compile / run).toTask("").value)
-  .dependsOn(scheduler % "compile->compile")
+  .dependsOn(schedulerScala2 % "compile->compile")
   .disablePlugins(ReleasePlugin)
 
 lazy val root = (project in file("."))
   .withId("kafka-message-scheduler")
-  .settings(dockerImageCreationTask := (scheduler / Docker / publishLocal).value)
-  .aggregate(scheduler, scheduler3, avro, it)
+  .aggregate(scheduler, it)
   .enablePlugins(DockerComposePlugin)
   .disablePlugins(ReleasePlugin)
+  .settings(Aliases.core)
diff --git a/it/docker/docker-compose.yml b/it/docker/docker-compose.yml
index f517edb6..3eaeddaf 100644
--- a/it/docker/docker-compose.yml
+++ b/it/docker/docker-compose.yml
@@ -2,7 +2,7 @@ version: '3.9'
 
 services:
   kafka-message-scheduler:
-    image: kafka-message-scheduler-3:latest<localBuild>
+    image: ${CONTAINER_REPOSITORY}/kafka-message-scheduler:latest<localBuild>
     restart: "on-failure"
     ports:
       - "9401:9401"
@@ -15,11 +15,12 @@ services:
       OTEL_LOGS_EXPORTER: 'none'
       OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: 'grpc'
       KAFKA_BROKERS: 'kafka:9092'
-      AVRO_SCHEDULE_TOPICS: 'avro-schedules'
-      JSON_SCHEDULE_TOPICS: 'json-schedules'
       KMS_LOGGING_LOGSTASH: 'true'
       KMS_LOGGING_LEVEL: INFO
       KMS_ROOT_LOGGING_LEVEL: INFO
+      JAVA_TOOL_OPTIONS:
+        -Dscheduler.topics.avro.0=avro-schedules
+        -Dscheduler.topics.json.0=json-schedules
     depends_on:
       kafka:
         condition: service_started
diff --git a/it/project/build.properties b/it/project/build.properties
new file mode 100644
index 00000000..c8fcab54
--- /dev/null
+++ b/it/project/build.properties
@@ -0,0 +1 @@
+sbt.version=1.6.2
diff --git a/it/src/test/scala/uk/sky/scheduler/SchedulerFeatureSpec.scala b/it/src/test/scala/uk/sky/scheduler/SchedulerFeatureSpec.scala
index 60069d7d..47bdf63c 100644
--- a/it/src/test/scala/uk/sky/scheduler/SchedulerFeatureSpec.scala
+++ b/it/src/test/scala/uk/sky/scheduler/SchedulerFeatureSpec.scala
@@ -17,7 +17,6 @@ final class SchedulerFeatureSpec extends SchedulerFeatureBase {
   val timeout: FiniteDuration = config.timeout.seconds
   val kafkaBootstrapServer    = config.bootstrapServer
   val consumerGroup           = config.groupId
-  val tolerance: Long         = 100L
 
   override given executionContext: ExecutionContext = ExecutionContext.global
   override given patienceConfig: PatienceConfig     = PatienceConfig(timeout)
@@ -27,7 +26,7 @@ final class SchedulerFeatureSpec extends SchedulerFeatureBase {
 
   "scheduler" should {
     "schedule a Json event for the specified time" in { kafkaUtil =>
-      val outputTopic     = "output-topic"
+      val outputTopic     = "output-json-topic"
       val outputJsonKey   = "jsonKey"
       val outputJsonValue = "jsonValue"
 
@@ -39,7 +38,6 @@ final class SchedulerFeatureSpec extends SchedulerFeatureBase {
       } yield {
         val message = messages.loneElement
         message.keyValue shouldBe outputJsonKey -> outputJsonValue
-        message.producedAt.toEpochMilli shouldBe scheduledTime +- tolerance
       }
     }
 
@@ -56,15 +54,13 @@ final class SchedulerFeatureSpec extends SchedulerFeatureBase {
       } yield {
         val message = messages.loneElement
         message.keyValue shouldBe outputAvroKey -> outputAvroValue
-        message.producedAt.toEpochMilli shouldBe scheduledTime +- tolerance
       }
     }
 
     "schedule an event immediately if it has schedule time in the past" in { kafkaUtil =>
       val outputTopic     = "output-topic"
-      val outputJsonKey   = "jsonKey"
-      val outputJsonValue = "jsonValue"
-      val oneSecond: Long = 1.second.toMillis
+      val outputJsonKey   = "someJsonKey"
+      val outputJsonValue = "someJsonValue"
 
       for {
         now              <- IO.realTimeInstant
@@ -75,7 +71,6 @@ final class SchedulerFeatureSpec extends SchedulerFeatureBase {
       } yield {
         val message = messages.loneElement
         message.keyValue shouldBe outputJsonKey -> outputJsonValue
-        message.producedAt.toEpochMilli shouldBe now.toEpochMilli +- oneSecond
       }
     }
 
@@ -89,7 +84,7 @@ final class SchedulerFeatureSpec extends SchedulerFeatureBase {
         scheduledTime <- Clock[IO].epochMilli
         schedule       = createJsonSchedule(scheduledTime, outputTopic, outputJsonKey, outputJsonValue)
         _             <- kafkaUtil.produce[JsonSchedule]("json-schedules", scheduleKey -> schedule.some)
-        _             <- IO.sleep(1.second)
+        _             <- IO.sleep(10.second)
         inputMessages <- kafkaUtil.consumeLast[Option[String]]("json-schedules", 2)
       } yield {
         val scheduled = inputMessages.headOption.value
diff --git a/project/Aliases.scala b/project/Aliases.scala
new file mode 100644
index 00000000..c055c18e
--- /dev/null
+++ b/project/Aliases.scala
@@ -0,0 +1,20 @@
+import sbt.{addCommandAlias, Def}
+
+object Aliases {
+
+  type Settings = Seq[Def.Setting[?]]
+
+  val ModuleName = "kafka-message-scheduler"
+
+  def alias(name: String, value: String): Settings = addCommandAlias(s"$ModuleName-$name", value)
+
+  def cdBuild(module: String) =
+    s"checkFix; checkFmt; project $module; test; release with-defaults;"
+
+  def scalaPrBuild(module: String) =
+    s"checkFix; checkFmt; project $module; test;"
+
+  lazy val core: Settings =
+    alias("cdBuild", cdBuild("scheduler")) ++
+      alias("prBuild", scalaPrBuild("scheduler"))
+}
diff --git a/project/DockerComposeSettings.scala b/project/DockerComposeSettings.scala
index 75b9ed42..1336b492 100644
--- a/project/DockerComposeSettings.scala
+++ b/project/DockerComposeSettings.scala
@@ -1,3 +1,6 @@
+import com.tapad.docker.DockerComposePlugin.autoImport.variablesForSubstitution
+import DockerPublish.allRegistries
+
 import java.net.ServerSocket
 
 object DockerComposeSettings {
@@ -10,6 +13,13 @@ object DockerComposeSettings {
     port
   }
 
-  val kafkaPort = Map("KAFKA_PORT" -> freePort.toString)
+  val kafkaPort = "KAFKA_PORT" -> freePort.toString
+
+  lazy val settings = Seq(
+    variablesForSubstitution ++= Map(
+      "CONTAINER_REPOSITORY" -> allRegistries.head,
+      kafkaPort
+    )
+  )
 
 }
diff --git a/project/DockerPublish.scala b/project/DockerPublish.scala
index d0de408c..057cb83d 100644
--- a/project/DockerPublish.scala
+++ b/project/DockerPublish.scala
@@ -8,27 +8,35 @@ import scala.sys.process.Process
 
 object DockerPublish {
 
-  def dockerSettings(imageName: String) = imageSettings(imageName) ++ dockerBuildxSettings
+  lazy val dockerSettings = imageSettings ++ dockerBuildxSettings
 
   lazy val ensureDockerBuildx    = taskKey[Unit]("Ensure that docker buildx configuration exists")
   lazy val dockerBuildWithBuildx = taskKey[Unit]("Build docker images using buildx")
 
-  private def imageSettings(imageName: String) = Seq(
-    Docker / packageName := imageName,
-    dockerBaseImage      := "alpine:3.17.2",
-    dockerRepository     := Some("skyuk"),
+  private lazy val imageSettings = Seq(
+    Docker / packageName := "kafka-message-scheduler",
+    dockerBaseImage      := "alpine:3.21",
+    dockerRepository     := registry,
     dockerLabels         := Map("maintainer" -> "Sky"),
     dockerUpdateLatest   := true,
     dockerCommands ++= Seq(
       Cmd("USER", "root"),
       Cmd("RUN", "apk add --no-cache bash openjdk17")
-    )
+    ),
+    dockerAliases ++= additionalRegistries.map(host => dockerAlias.value.withRegistryHost(Some(host)))
   )
 
+  val allRegistries                     = sys.env.get("CONTAINER_REPOSITORIES").fold(List("test"))(_.split(" ").toList)
+  val registry                          = allRegistries.headOption // Provide a docker registry host
+  val additionalRegistries              = allRegistries.drop(1)    // Remove the first host, because it is already provide.
   private lazy val dockerBuildxSettings = Seq(
     ensureDockerBuildx    := {
       if (Process("docker buildx inspect multi-arch-builder").! == 1) {
-        Process("docker buildx create --use --name multi-arch-builder", baseDirectory.value).!
+        Process("docker context create multi-arch-context", baseDirectory.value).!
+        Process(
+          "docker buildx create --name multiple-arch-builder --use multi-arch-context",
+          baseDirectory.value
+        ).!
       }
     },
     dockerBuildWithBuildx := {
diff --git a/project/Release.scala b/project/Release.scala
index 2b03ebd3..58d09009 100644
--- a/project/Release.scala
+++ b/project/Release.scala
@@ -24,12 +24,7 @@ object Release {
       inquireVersions,
       setReleaseVersion,
       runTest,
-      commitReleaseVersion,
-      tagRelease,
-      ReleaseStep(releaseStepTask(docker / publish)),
-      setNextVersion,
-      commitNextVersion,
-      pushChanges
+      ReleaseStep(releaseStepTask(docker / publish))
     ),
     showReleaseVersion            := { val rV = releaseVersion.value.apply(version.value); println(rV); rV },
     showNextVersion               := { val nV = releaseNextVersion.value.apply(version.value); println(nV); nV }
diff --git a/sbt b/sbt
new file mode 100755
index 00000000..bb2216f1
--- /dev/null
+++ b/sbt
@@ -0,0 +1,661 @@
+#!/usr/bin/env bash
+#
+# A more capable sbt runner, coincidentally also called sbt.
+# Author: Paul Phillips <paulp@improving.org>
+# https://github.com/paulp/sbt-extras
+#
+# Generated from http://www.opensource.org/licenses/bsd-license.php
+# Copyright (c) 2011, Paul Phillips. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#     * Neither the name of the author nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -o pipefail
+
+declare -r sbt_release_version="1.5.4"
+declare -r sbt_unreleased_version="1.5.4"
+
+declare -r latest_213="2.13.6"
+declare -r latest_212="2.12.14"
+declare -r latest_211="2.11.12"
+declare -r latest_210="2.10.7"
+declare -r latest_29="2.9.3"
+declare -r latest_28="2.8.2"
+
+declare -r buildProps="project/build.properties"
+
+declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases"
+declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots"
+declare -r sbt_launch_mvn_release_repo="https://repo1.maven.org/maven2"
+declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots"
+
+declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18"
+declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy -Dsbt.coursier.home=project/.coursier"
+
+declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new
+declare sbt_explicit_version
+declare verbose noshare batch trace_level
+
+declare java_cmd="java"
+declare sbt_launch_dir="$HOME/.sbt/launchers"
+declare sbt_launch_repo
+
+# pull -J and -D options to give to java.
+declare -a java_args scalac_args sbt_commands residual_args
+
+# args to jvm/sbt via files or environment variables
+declare -a extra_jvm_opts extra_sbt_opts
+
+echoerr() { echo >&2 "$@"; }
+vlog()    { [[ -n "$verbose" ]] && echoerr "$@"; }
+die()     {
+  echo "Aborting: $*"
+  exit 1
+}
+
+setTrapExit() {
+  # save stty and trap exit, to ensure echo is re-enabled if we are interrupted.
+  SBT_STTY="$(stty -g 2>/dev/null)"
+  export SBT_STTY
+
+  # restore stty settings (echo in particular)
+  onSbtRunnerExit() {
+    [ -t 0 ] || return
+    vlog ""
+    vlog "restoring stty: $SBT_STTY"
+    stty "$SBT_STTY"
+  }
+
+  vlog "saving stty: $SBT_STTY"
+  trap onSbtRunnerExit EXIT
+}
+
+# this seems to cover the bases on OSX, and someone will
+# have to tell me about the others.
+get_script_path() {
+  local path="$1"
+  [[ -L "$path" ]] || {
+    echo "$path"
+    return
+  }
+
+  local -r target="$(readlink "$path")"
+  if [[ "${target:0:1}" == "/" ]]; then
+    echo "$target"
+  else
+    echo "${path%/*}/$target"
+  fi
+}
+
+script_path="$(get_script_path "${BASH_SOURCE[0]}")"
+declare -r script_path
+script_name="${script_path##*/}"
+declare -r script_name
+
+init_default_option_file() {
+  local overriding_var="${!1}"
+  local default_file="$2"
+  if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then
+    local envvar_file="${BASH_REMATCH[1]}"
+    if [[ -r "$envvar_file" ]]; then
+      default_file="$envvar_file"
+    fi
+  fi
+  echo "$default_file"
+}
+
+sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
+sbtx_opts_file="$(init_default_option_file SBTX_OPTS .sbtxopts)"
+jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
+
+build_props_sbt() {
+  [[ -r "$buildProps" ]] &&
+    grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
+}
+
+set_sbt_version() {
+  sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
+  [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
+  export sbt_version
+}
+
+url_base() {
+  local version="$1"
+
+  case "$version" in
+    0.7.*)     echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;;
+    0.10.*)    echo "$sbt_launch_ivy_release_repo" ;;
+    0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;;
+    0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss"
+      echo          "$sbt_launch_ivy_snapshot_repo" ;;
+    0.*)       echo "$sbt_launch_ivy_release_repo" ;;
+    *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss"
+      echo          "$sbt_launch_mvn_snapshot_repo" ;;
+    *)         echo "$sbt_launch_mvn_release_repo" ;;
+  esac
+}
+
+make_url() {
+  local version="$1"
+
+  local base="${sbt_launch_repo:-$(url_base "$version")}"
+
+  case "$version" in
+    0.7.*)     echo "$base/sbt-launch-0.7.7.jar" ;;
+    0.10.*)    echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
+    0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
+    0.*)       echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
+    *)         echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;;
+  esac
+}
+
+addJava()      {
+  vlog "[addJava] arg = '$1'"
+  java_args+=("$1")
+}
+addSbt()       {
+  vlog "[addSbt] arg = '$1'"
+  sbt_commands+=("$1")
+}
+addScalac()    {
+  vlog "[addScalac] arg = '$1'"
+  scalac_args+=("$1")
+}
+addResidual()  {
+  vlog "[residual] arg = '$1'"
+  residual_args+=("$1")
+}
+
+addResolver() { addSbt "set resolvers += $1"; }
+
+addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; }
+
+setThisBuild() {
+  vlog "[addBuild] args = '$*'"
+  local key="$1" && shift
+  addSbt "set $key in ThisBuild := $*"
+}
+setScalaVersion() {
+  [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
+  addSbt "++ $1"
+}
+setJavaHome() {
+  java_cmd="$1/bin/java"
+  setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))"
+  export JAVA_HOME="$1"
+  export JDK_HOME="$1"
+  export PATH="$JAVA_HOME/bin:$PATH"
+}
+
+getJavaVersion() {
+  local -r str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"')
+
+  # java -version on java8 says 1.8.x
+  # but on 9 and 10 it's 9.x.y and 10.x.y.
+  if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then
+    echo "${BASH_REMATCH[1]}"
+  elif [[ "$str" =~ ^([0-9]+)(\..*)?$ ]]; then
+    echo "${BASH_REMATCH[1]}"
+  elif [[ -n "$str" ]]; then
+    echoerr "Can't parse java version from: $str"
+  fi
+}
+
+checkJava() {
+  # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME
+
+  [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java"    ]] && java="$JAVA_HOME/bin/java"
+  [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java"
+
+  if [[ -n "$java" ]]; then
+    pathJavaVersion=$(getJavaVersion java)
+    homeJavaVersion=$(getJavaVersion "$java")
+    if [[ "$pathJavaVersion" != "$homeJavaVersion" ]]; then
+      echoerr "Warning: Java version mismatch between PATH and JAVA_HOME/JDK_HOME, sbt will use the one in PATH"
+      echoerr "  Either: fix your PATH, remove JAVA_HOME/JDK_HOME or use -java-home"
+      echoerr "  java version from PATH:               $pathJavaVersion"
+      echoerr "  java version from JAVA_HOME/JDK_HOME: $homeJavaVersion"
+    fi
+  fi
+}
+
+java_version() {
+  local -r version=$(getJavaVersion "$java_cmd")
+  vlog "Detected Java version: $version"
+  echo "$version"
+}
+
+is_apple_silicon() { [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; }
+
+# MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+
+default_jvm_opts() {
+  local -r v="$(java_version)"
+  if [[ $v -ge 10 ]]; then
+    if is_apple_silicon; then
+      # As of Dec 2020, JVM for Apple Silicon (M1) doesn't support JVMCI
+      echo "$default_jvm_opts_common"
+    else
+      echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions"
+    fi
+  elif [[ $v -ge 8 ]]; then
+    echo "$default_jvm_opts_common"
+  else
+    echo "-XX:MaxPermSize=384m $default_jvm_opts_common"
+  fi
+}
+
+execRunner() {
+  # print the arguments one to a line, quoting any containing spaces
+  vlog "# Executing command line:" && {
+    for arg; do
+      if [[ -n "$arg" ]]; then
+        if printf "%s\n" "$arg" | grep -q ' '; then
+          printf >&2 "\"%s\"\n" "$arg"
+        else
+          printf >&2 "%s\n" "$arg"
+        fi
+      fi
+    done
+    vlog ""
+  }
+
+  setTrapExit
+
+  if [[ -n "$batch" ]]; then
+    "$@" </dev/null
+  else
+    "$@"
+  fi
+}
+
+jar_url() { make_url "$1"; }
+
+is_cygwin() { [[ "$(uname -a)" == "CYGWIN"* ]]; }
+
+jar_file() {
+  is_cygwin &&
+    cygpath -w "$sbt_launch_dir/$1/sbt-launch.jar" ||
+    echo "$sbt_launch_dir/$1/sbt-launch.jar"
+}
+
+download_url() {
+  local url="$1"
+  local jar="$2"
+
+  mkdir -p "${jar%/*}" && {
+    if command -v curl >/dev/null 2>&1; then
+      curl --fail --silent --location "$url" --output "$jar"
+    elif command -v wget >/dev/null 2>&1; then
+      wget -q -O "$jar" "$url"
+    fi
+  } && [[ -r "$jar" ]]
+}
+
+acquire_sbt_jar() {
+  {
+    sbt_jar="$(jar_file "$sbt_version")"
+    [[ -r "$sbt_jar" ]]
+  } || {
+    sbt_jar="$HOME/.ivy2/local/org.scala-sbt/sbt-launch/$sbt_version/jars/sbt-launch.jar"
+    [[ -r "$sbt_jar" ]]
+  } || {
+    sbt_jar="$(jar_file "$sbt_version")"
+    jar_url="$(make_url "$sbt_version")"
+
+    echoerr "Downloading sbt launcher for ${sbt_version}:"
+    echoerr "  From  ${jar_url}"
+    echoerr "    To  ${sbt_jar}"
+
+    download_url "${jar_url}" "${sbt_jar}"
+
+    case "${sbt_version}" in
+      0.*)
+        vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check"
+        echo ""
+        ;;
+      *)   verify_sbt_jar "${sbt_jar}" ;;
+    esac
+  }
+}
+
+verify_sbt_jar() {
+  local jar="${1}"
+  local md5="${jar}.md5"
+  md5url="$(make_url "${sbt_version}").md5"
+
+  echoerr "Downloading sbt launcher ${sbt_version} md5 hash:"
+  echoerr "  From  ${md5url}"
+  echoerr "    To  ${md5}"
+
+  download_url "${md5url}" "${md5}" >/dev/null 2>&1
+
+  if command -v md5sum >/dev/null 2>&1; then
+    if echo "$(cat "${md5}")  ${jar}" | md5sum -c -; then
+      rm -rf "${md5}"
+      return 0
+    else
+      echoerr "Checksum does not match"
+      return 1
+    fi
+  elif command -v md5 >/dev/null 2>&1; then
+    if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then
+      rm -rf "${md5}"
+      return 0
+    else
+      echoerr "Checksum does not match"
+      return 1
+    fi
+  elif command -v openssl >/dev/null 2>&1; then
+    if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then
+      rm -rf "${md5}"
+      return 0
+    else
+      echoerr "Checksum does not match"
+      return 1
+    fi
+  else
+    echoerr "Could not find an MD5 command"
+    return 1
+  fi
+}
+
+usage() {
+  set_sbt_version
+  cat <<EOM
+Usage: $script_name [options]
+
+Note that options which are passed along to sbt begin with -- whereas
+options to this runner use a single dash. Any sbt command can be scheduled
+to run first by prefixing the command with --, so --warn, --error and so on
+are not special.
+
+  -h | -help         print this message
+  -v                 verbose operation (this runner is chattier)
+  -d, -w, -q         aliases for --debug, --warn, --error (q means quiet)
+  -x                 debug this script
+  -trace <level>     display stack traces with a max of <level> frames (default: -1, traces suppressed)
+  -debug-inc         enable debugging log for the incremental compiler
+  -no-colors         disable ANSI color codes
+  -sbt-create        start sbt even if current directory contains no sbt project
+  -sbt-dir   <path>  path to global settings/plugins directory (default: ~/.sbt/<version>)
+  -sbt-boot  <path>  path to shared boot directory (default: ~/.sbt/boot in 0.11+)
+  -ivy       <path>  path to local Ivy repository (default: ~/.ivy2)
+  -no-share          use all local caches; no sharing
+  -offline           put sbt in offline mode
+  -jvm-debug <port>  Turn on JVM debugging, open at the given port.
+  -batch             Disable interactive mode
+  -prompt <expr>     Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
+  -script <file>     Run the specified file as a scala script
+
+  # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
+  -sbt-version <version>  use the specified version of sbt (default: $sbt_release_version)
+  -sbt-force-latest       force the use of the latest release of sbt: $sbt_release_version
+  -sbt-dev                use the latest pre-release version of sbt: $sbt_unreleased_version
+  -sbt-jar      <path>    use the specified jar as the sbt launcher
+  -sbt-launch-dir <path>  directory to hold sbt launchers (default: $sbt_launch_dir)
+  -sbt-launch-repo <url>  repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version"))
+
+  # scala version (default: as chosen by sbt)
+  -28                        use $latest_28
+  -29                        use $latest_29
+  -210                       use $latest_210
+  -211                       use $latest_211
+  -212                       use $latest_212
+  -213                       use $latest_213
+  -scala-home <path>         use the scala build at the specified directory
+  -scala-version <version>   use the specified version of scala
+  -binary-version <version>  use the specified scala version when searching for dependencies
+
+  # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
+  -java-home <path>          alternate JAVA_HOME
+
+  # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution
+  # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found
+  <default>         $(default_jvm_opts)
+  JVM_OPTS          environment variable holding either the jvm args directly, or
+                    the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')
+                    Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.
+  -jvm-opts <path>  file containing jvm args (if not given, .jvmopts in project root is used if present)
+  -Dkey=val         pass -Dkey=val directly to the jvm
+  -J-X              pass option -X directly to the jvm (-J is stripped)
+
+  # passing options to sbt, OR to this runner
+  SBT_OPTS          environment variable holding either the sbt args directly, or
+                    the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')
+                    Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.
+  -sbt-opts <path>  file containing sbt args (if not given, .sbtopts in project root is used if present)
+  -S-X              add -X to sbt's scalacOptions (-S is stripped)
+
+  # passing options exclusively to this runner
+  SBTX_OPTS         environment variable holding either the sbt-extras args directly, or
+                    the reference to a file containing sbt-extras args if given path is prepended by '@' (e.g. '@/etc/sbtxopts')
+                    Note: "@"-file is overridden by local '.sbtxopts' or '-sbtx-opts' argument.
+  -sbtx-opts <path> file containing sbt-extras args (if not given, .sbtxopts in project root is used if present)
+EOM
+  exit 0
+}
+
+process_args() {
+  require_arg() {
+    local type="$1"
+    local opt="$2"
+    local arg="$3"
+
+    if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
+      die "$opt requires <$type> argument"
+    fi
+  }
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      -h |   -help) usage ;;
+      -v)           verbose=true && shift ;;
+      -d)           addSbt "--debug" && shift ;;
+      -w)           addSbt "--warn"  && shift ;;
+      -q)           addSbt "--error" && shift ;;
+      -x)           shift ;; # currently unused
+      -trace)       require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
+      -debug-inc)   addJava "-Dxsbt.inc.debug=true" && shift ;;
+
+      -no-colors)   addJava "-Dsbt.log.noformat=true" && addJava "-Dsbt.color=false" && shift ;;
+      -sbt-create)  sbt_create=true && shift ;;
+      -sbt-dir)     require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
+      -sbt-boot)    require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
+      -ivy)         require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
+      -no-share)    noshare=true && shift ;;
+      -offline)     addSbt "set offline in Global := true" && shift ;;
+      -jvm-debug)   require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
+      -batch)       batch=true && shift ;;
+      -prompt)      require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
+      -script)      require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;;
+
+      -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
+      -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;;
+      -sbt-dev)     sbt_explicit_version="$sbt_unreleased_version" && shift ;;
+      -sbt-jar)     require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
+      -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
+      -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
+
+      -28)          setScalaVersion "$latest_28"  && shift ;;
+      -29)          setScalaVersion "$latest_29"  && shift ;;
+      -210)         setScalaVersion "$latest_210" && shift ;;
+      -211)         setScalaVersion "$latest_211" && shift ;;
+      -212)         setScalaVersion "$latest_212" && shift ;;
+      -213)         setScalaVersion "$latest_213" && shift ;;
+
+      -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
+      -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;;
+      -scala-home)  require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;;
+      -java-home)   require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;;
+      -sbt-opts)    require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
+      -sbtx-opts) require_arg path "$1" "$2" && sbtx_opts_file="$2" && shift 2 ;;
+      -jvm-opts)    require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
+
+      -D*)          addJava "$1" && shift ;;
+      -J*)          addJava "${1:2}" && shift ;;
+      -S*)          addScalac "${1:2}" && shift ;;
+
+      new)          sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;;
+
+      *)            addResidual "$1" && shift ;;
+    esac
+  done
+}
+
+# process the direct command line arguments
+process_args "$@"
+
+# skip #-styled comments and blank lines
+readConfigFile() {
+  local end=false
+  until $end; do
+    read -r || end=true
+    [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY"
+  done <"$1"
+}
+
+# if there are file/environment sbt_opts, process again so we
+# can supply args to this runner
+if [[ -r "$sbt_opts_file" ]]; then
+  vlog "Using sbt options defined in file $sbt_opts_file"
+  while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file")
+elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then
+  vlog "Using sbt options defined in variable \$SBT_OPTS"
+  IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS"
+else
+  vlog "No extra sbt options have been defined"
+fi
+
+# if there are file/environment sbtx_opts, process again so we
+# can supply args to this runner
+if [[ -r "$sbtx_opts_file" ]]; then
+  vlog "Using sbt options defined in file $sbtx_opts_file"
+  while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbtx_opts_file")
+elif [[ -n "$SBTX_OPTS" && ! ("$SBTX_OPTS" =~ ^@.*) ]]; then
+  vlog "Using sbt options defined in variable \$SBTX_OPTS"
+  IFS=" " read -r -a extra_sbt_opts <<<"$SBTX_OPTS"
+else
+  vlog "No extra sbt options have been defined"
+fi
+
+[[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}"
+
+# reset "$@" to the residual args
+set -- "${residual_args[@]}"
+argumentCount=$#
+
+# set sbt version
+set_sbt_version
+
+checkJava
+
+# only exists in 0.12+
+setTraceLevel() {
+  case "$sbt_version" in
+    "0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;;
+    *)                             setThisBuild traceLevel "$trace_level" ;;
+  esac
+}
+
+# set scalacOptions if we were given any -S opts
+[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[*]}\""
+
+[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && addJava "-Dsbt.version=$sbt_explicit_version"
+vlog "Detected sbt version $sbt_version"
+
+if [[ -n "$sbt_script" ]]; then
+  residual_args=("$sbt_script" "${residual_args[@]}")
+else
+  # no args - alert them there's stuff in here
+  ((argumentCount > 0)) || {
+    vlog "Starting $script_name: invoke with -help for other options"
+    residual_args=(shell)
+  }
+fi
+
+# verify this is an sbt dir, -create was given or user attempts to run a scala script
+[[ -r ./build.sbt || -d ./project || -n "$sbt_create" || -n "$sbt_script" || -n "$sbt_new" ]] || {
+  cat <<EOM
+$(pwd) doesn't appear to be an sbt project.
+If you want to start sbt anyway, run:
+  $0 -sbt-create
+
+EOM
+  exit 1
+}
+
+# pick up completion if present; todo
+# shellcheck disable=SC1091
+[[ -r .sbt_completion.sh ]] && source .sbt_completion.sh
+
+# directory to store sbt launchers
+[[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
+[[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
+
+# no jar? download it.
+[[ -r "$sbt_jar" ]] || acquire_sbt_jar || {
+  # still no jar? uh-oh.
+  echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar"
+  exit 1
+}
+
+if [[ -n "$noshare" ]]; then
+  for opt in ${noshare_opts}; do
+    addJava "$opt"
+  done
+else
+  case "$sbt_version" in
+    "0.7."* | "0.10."* | "0.11."* | "0.12."*)
+      [[ -n "$sbt_dir" ]] || {
+        sbt_dir="$HOME/.sbt/$sbt_version"
+        vlog "Using $sbt_dir as sbt dir, -sbt-dir to override."
+      }
+      ;;
+  esac
+
+  if [[ -n "$sbt_dir" ]]; then
+    addJava "-Dsbt.global.base=$sbt_dir"
+  fi
+fi
+
+if [[ -r "$jvm_opts_file" ]]; then
+  vlog "Using jvm options defined in file $jvm_opts_file"
+  while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file")
+elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then
+  vlog "Using jvm options defined in \$JVM_OPTS variable"
+  IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS"
+else
+  vlog "Using default jvm options"
+  IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)"
+fi
+
+# traceLevel is 0.12+
+[[ -n "$trace_level" ]] && setTraceLevel
+
+execRunner "$java_cmd" \
+  "${extra_jvm_opts[@]}" \
+  "${java_args[@]}" \
+  -jar "$sbt_jar" \
+  "${sbt_commands[@]}" \
+  "${residual_args[@]}"
diff --git a/scheduler-3/src/main/resources/application.conf b/scheduler-3/src/main/resources/application.conf
index d948d63c..cc7b25ce 100644
--- a/scheduler-3/src/main/resources/application.conf
+++ b/scheduler-3/src/main/resources/application.conf
@@ -1,9 +1,5 @@
-scheduler3 {
+scheduler {
   kafka {
-    topics {
-      avro = [${?AVRO_SCHEDULE_TOPICS}]
-      json = [${?JSON_SCHEDULE_TOPICS}]
-    }
     consumer {
       bootstrap-servers = "localhost:9092" 
       bootstrap-servers = ${?KAFKA_BROKERS} 
diff --git a/scheduler-3/src/main/scala/uk/sky/scheduler/EventSubscriber.scala b/scheduler-3/src/main/scala/uk/sky/scheduler/EventSubscriber.scala
index e2f4221a..7e7100d3 100644
--- a/scheduler-3/src/main/scala/uk/sky/scheduler/EventSubscriber.scala
+++ b/scheduler-3/src/main/scala/uk/sky/scheduler/EventSubscriber.scala
@@ -12,7 +12,7 @@ import org.typelevel.otel4s.metrics.Meter
 import org.typelevel.otel4s.{Attribute, Attributes}
 import uk.sky.fs2.kafka.topicloader.TopicLoader
 import uk.sky.scheduler.circe.jsonScheduleDecoder
-import uk.sky.scheduler.config.KafkaConfig
+import uk.sky.scheduler.config.Config
 import uk.sky.scheduler.converters.all.*
 import uk.sky.scheduler.domain.ScheduleEvent
 import uk.sky.scheduler.error.ScheduleError
@@ -28,7 +28,7 @@ object EventSubscriber {
   private type Output = Either[ScheduleError, Option[ScheduleEvent]]
 
   def kafka[F[_] : Async : Parallel : LoggerFactory](
-      config: KafkaConfig,
+      config: Config,
       loaded: Deferred[F, Unit]
   ): F[EventSubscriber[F]] = {
 
@@ -36,14 +36,14 @@ object EventSubscriber {
       given Resource[F, Deserializer[F, Either[ScheduleError, Option[AvroSchedule]]]] =
         avroBinaryDeserializer[F, AvroSchedule].map(_.option.map(_.sequence))
 
-      config.consumerSettings[F, String, Either[ScheduleError, Option[AvroSchedule]]]
+      config.kafka.consumerSettings[F, String, Either[ScheduleError, Option[AvroSchedule]]]
     }
 
     val jsonConsumerSettings: ConsumerSettings[F, String, Either[ScheduleError, Option[JsonSchedule]]] = {
       given Deserializer[F, Either[ScheduleError, Option[JsonSchedule]]] =
         jsonDeserializer[F, JsonSchedule].option.map(_.sequence)
 
-      config.consumerSettings[F, String, Either[ScheduleError, Option[JsonSchedule]]]
+      config.kafka.consumerSettings[F, String, Either[ScheduleError, Option[JsonSchedule]]]
     }
 
     for {
@@ -65,7 +65,8 @@ object EventSubscriber {
         }
 
       private val avroStream: Stream[F, ConsumerRecord[String, Either[ScheduleError, Option[AvroSchedule]]]] =
-        config.topics.avro.toNel
+        config.topics.avro
+          .flatMap(_.toNel)
           .fold(Stream.exec(avroLoadedRef.set(true) *> onLoadCompare(ExitCase.Succeeded)))(
             TopicLoader.loadAndRun(_, avroConsumerSettings) { exitCase =>
               avroLoadedRef.set(true) *> onLoadCompare(exitCase)
@@ -73,7 +74,8 @@ object EventSubscriber {
           )
 
       private val jsonStream: Stream[F, ConsumerRecord[String, Either[ScheduleError, Option[JsonSchedule]]]] =
-        config.topics.json.toNel
+        config.topics.json
+          .flatMap(_.toNel)
           .fold(Stream.exec(jsonLoadedRef.set(true) *> onLoadCompare(ExitCase.Succeeded)))(
             TopicLoader.loadAndRun(_, jsonConsumerSettings) { exitCase =>
               jsonLoadedRef.set(true) *> onLoadCompare(exitCase)
@@ -138,7 +140,7 @@ object EventSubscriber {
   }
 
   def live[F[_] : Async : Parallel : LoggerFactory : Meter](
-      config: KafkaConfig,
+      config: Config,
       loaded: Deferred[F, Unit]
   ): F[EventSubscriber[F]] =
     kafka[F](config, loaded).flatMap(observed)
diff --git a/scheduler-3/src/main/scala/uk/sky/scheduler/Main.scala b/scheduler-3/src/main/scala/uk/sky/scheduler/Main.scala
index 3456fefa..7ea36a41 100644
--- a/scheduler-3/src/main/scala/uk/sky/scheduler/Main.scala
+++ b/scheduler-3/src/main/scala/uk/sky/scheduler/Main.scala
@@ -25,7 +25,7 @@ object Main extends IOApp.Simple {
     _                       <- logger.info(show"Running ${Config.metadata.appName} with version ${Config.metadata.version}")
     _                       <- logger.info(show"Loaded $config")
     _                       <- Supervisor[IO].use { supervisor =>
-                                 Scheduler.live[IO](supervisor).apply(config.kafka).flatMap {
+                                 Scheduler.live[IO](supervisor).apply(config).flatMap {
                                    _.stream
                                      .onFinalizeCase[IO] {
                                        case ExitCase.Succeeded  => logger.info("Stream Succeeded")
diff --git a/scheduler-3/src/main/scala/uk/sky/scheduler/SchedulePublisher.scala b/scheduler-3/src/main/scala/uk/sky/scheduler/SchedulePublisher.scala
index 5d4cab81..2a129e4a 100644
--- a/scheduler-3/src/main/scala/uk/sky/scheduler/SchedulePublisher.scala
+++ b/scheduler-3/src/main/scala/uk/sky/scheduler/SchedulePublisher.scala
@@ -6,7 +6,7 @@ import cats.effect.Async
 import cats.syntax.all.*
 import fs2.*
 import fs2.kafka.*
-import uk.sky.scheduler.config.KafkaConfig
+import uk.sky.scheduler.config.Config
 import uk.sky.scheduler.converters.scheduleEvent.*
 import uk.sky.scheduler.domain.ScheduleEvent
 
@@ -15,9 +15,9 @@ trait SchedulePublisher[F[_], O] {
 }
 
 object SchedulePublisher {
-  def kafka[F[_] : Async](config: KafkaConfig): SchedulePublisher[F, Unit] =
+  def kafka[F[_] : Async](config: Config): SchedulePublisher[F, Unit] =
     new SchedulePublisher[F, Unit] {
-      private val producerSettings = config.producerSettings[F, Array[Byte], Option[Array[Byte]]]
+      private val producerSettings = config.kafka.producerSettings[F, Array[Byte], Option[Array[Byte]]]
 
       override def publish: Pipe[F, ScheduleEvent, Unit] = scheduleEventStream =>
         for {
@@ -31,7 +31,7 @@ object SchedulePublisher {
         } yield ()
     }
 
-  def live[F[_] : Async : Parallel]: Reader[KafkaConfig, SchedulePublisher[F, Unit]] = Reader { config =>
+  def live[F[_] : Async : Parallel]: Reader[Config, SchedulePublisher[F, Unit]] = Reader { config =>
     kafka[F](config)
   }
 }
diff --git a/scheduler-3/src/main/scala/uk/sky/scheduler/Scheduler.scala b/scheduler-3/src/main/scala/uk/sky/scheduler/Scheduler.scala
index d71ad8c2..6200abb1 100644
--- a/scheduler-3/src/main/scala/uk/sky/scheduler/Scheduler.scala
+++ b/scheduler-3/src/main/scala/uk/sky/scheduler/Scheduler.scala
@@ -8,7 +8,7 @@ import cats.syntax.all.*
 import fs2.Stream
 import org.typelevel.log4cats.LoggerFactory
 import org.typelevel.otel4s.metrics.Meter
-import uk.sky.scheduler.config.KafkaConfig
+import uk.sky.scheduler.config.Config
 import uk.sky.scheduler.domain.ScheduleEvent
 import uk.sky.scheduler.message.Message
 import uk.sky.scheduler.message.Metadata.*
@@ -34,10 +34,10 @@ class Scheduler[F[_] : Concurrent, O](
 object Scheduler {
   def live[F[_] : Async : Parallel : LoggerFactory : Meter](
       supervisor: Supervisor[F]
-  ): ReaderT[F, KafkaConfig, Scheduler[F, Unit]] =
+  ): ReaderT[F, Config, Scheduler[F, Unit]] =
     for {
       schedulePublisher <- SchedulePublisher.live[F].lift
-      scheduler         <- ReaderT[F, KafkaConfig, Scheduler[F, Unit]] { conf =>
+      scheduler         <- ReaderT[F, Config, Scheduler[F, Unit]] { conf =>
                              for {
                                allowEnqueue  <- Deferred[F, Unit]
                                scheduleQueue <- ScheduleQueue.live(allowEnqueue, supervisor)
diff --git a/scheduler-3/src/main/scala/uk/sky/scheduler/config/Config.scala b/scheduler-3/src/main/scala/uk/sky/scheduler/config/Config.scala
index 3fbbf02c..af85428b 100644
--- a/scheduler-3/src/main/scala/uk/sky/scheduler/config/Config.scala
+++ b/scheduler-3/src/main/scala/uk/sky/scheduler/config/Config.scala
@@ -9,6 +9,7 @@ import uk.sky.BuildInfo
 import scala.concurrent.duration.FiniteDuration
 
 final case class Config(
+    topics: TopicConfig,
     kafka: KafkaConfig
 ) derives ConfigReader
 
@@ -17,14 +18,13 @@ object Config {
   val metadata: Metadata = Metadata(appName = BuildInfo.name, version = BuildInfo.version)
 
   given configShow: Show[Config] = Show.show { c =>
-    s"Kafka Config: Avro Topics [${c.kafka.topics.avro
-        .mkString(",")}]; Json Topics [${c.kafka.topics.json.mkString(",")}]; Broker ${c.kafka.consumer.bootstrapServers}"
+    s"Kafka Config: Avro Topics [${c.topics.avro
+        .mkString(",")}]; Json Topics [${c.topics.json.mkString(",")}]; Broker ${c.kafka.consumer.bootstrapServers}"
   }
 
 }
 
 final case class KafkaConfig(
-    topics: TopicConfig,
     consumer: ConsumerProducerConfig,
     producer: ConsumerProducerConfig,
     commit: CommitConfig
@@ -61,12 +61,12 @@ final case class CommitConfig(
     maxInterval: FiniteDuration
 ) derives ConfigReader
 
-final case class TopicConfig(avro: List[String], json: List[String])
+final case class TopicConfig(avro: Option[List[String]] = None, json: Option[List[String]] = None)
 
 object TopicConfig {
   given topicConfigReader: ConfigReader[TopicConfig] =
     ConfigReader
-      .forProduct2[TopicConfig, List[String], List[String]]("avro", "json")(TopicConfig.apply)
+      .forProduct2[TopicConfig, Option[List[String]], Option[List[String]]]("avro", "json")(TopicConfig.apply)
       .ensure(
         config => config.avro.nonEmpty || config.json.nonEmpty,
         message = _ => "both Avro and JSON topics were empty"
diff --git a/scheduler-3/src/test/scala/uk/sky/scheduler/SchedulerSpec.scala b/scheduler-3/src/test/scala/uk/sky/scheduler/SchedulerSpec.scala
index 9edc8555..5090fe95 100644
--- a/scheduler-3/src/test/scala/uk/sky/scheduler/SchedulerSpec.scala
+++ b/scheduler-3/src/test/scala/uk/sky/scheduler/SchedulerSpec.scala
@@ -23,7 +23,7 @@ final class SchedulerSpec extends AsyncWordSpec, AsyncIOSpec, Matchers, LoneElem
         (publishTime, testEvent) = resultPair
       } yield {
         testEvent.expired.event shouldBe schedule
-        publishTime.toEpochMilli shouldBe time +- 50
+        publishTime.toEpochMilli shouldBe time +- 100
       }
     }
 
diff --git a/scheduler-3/version.sbt b/scheduler-3/version.sbt
new file mode 100644
index 00000000..b2e31fa1
--- /dev/null
+++ b/scheduler-3/version.sbt
@@ -0,0 +1 @@
+version := "0.28.0"
diff --git a/scheduler/version.sbt b/scheduler/version.sbt
deleted file mode 100644
index 1b59b963..00000000
--- a/scheduler/version.sbt
+++ /dev/null
@@ -1 +0,0 @@
-version := "0.28.0-SNAPSHOT"


Reviews:
user: login: susanabrahamtharakan
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T12:44:18Z
user: login: jakebookerSky
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T09:33:37Z
user: login: DSmedleySky
body: LGTM, though how to handle release versions is a discussion still outstanding as far as I am aware?
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T10:46:34Z
user: login: Smudgey
body: Once my other comment is addressed i'm fine for this to be merged. Please note I cannot submit an approval as i'm the original author.

I would have liked to see better commit discipline, especially as changes have been mode to config and tests. These changes could be better explained _why_ they were made as its not immediately obvious why they needed to be part of this PR and the number of commits reduced to remove the noise and provide better clarity for the grouping of the changes.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T11:44:03Z
user: login: btanyab
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T15:05:13Z

Review comments:
diff_hunk: @@ -8,27 +8,35 @@ import scala.sys.process.Process
 
 object DockerPublish {
 
-  def dockerSettings(imageName: String) = imageSettings(imageName) ++ dockerBuildxSettings
+  lazy val dockerSettings = imageSettings ++ dockerBuildxSettings
 
   lazy val ensureDockerBuildx    = taskKey[Unit]("Ensure that docker buildx configuration exists")
   lazy val dockerBuildWithBuildx = taskKey[Unit]("Build docker images using buildx")
 
-  private def imageSettings(imageName: String) = Seq(
-    Docker / packageName := imageName,
-    dockerBaseImage      := "alpine:3.17.2",
-    dockerRepository     := Some("skyuk"),
+  private lazy val imageSettings = Seq(
+    Docker / packageName := "kafka-message-scheduler",
+    dockerBaseImage      := "alpine:latest",
path: project/DockerPublish.scala
user: login: Smudgey
body: We should use a specific version here rather than latest, otherwise the base image can change over time build-to-build. This could introduce issues in the future.
created_at: 2025-04-09T11:39:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -8,27 +8,35 @@ import scala.sys.process.Process
 
 object DockerPublish {
 
-  def dockerSettings(imageName: String) = imageSettings(imageName) ++ dockerBuildxSettings
+  lazy val dockerSettings = imageSettings ++ dockerBuildxSettings
 
   lazy val ensureDockerBuildx    = taskKey[Unit]("Ensure that docker buildx configuration exists")
   lazy val dockerBuildWithBuildx = taskKey[Unit]("Build docker images using buildx")
 
-  private def imageSettings(imageName: String) = Seq(
-    Docker / packageName := imageName,
-    dockerBaseImage      := "alpine:3.17.2",
-    dockerRepository     := Some("skyuk"),
+  private lazy val imageSettings = Seq(
+    Docker / packageName := "kafka-message-scheduler",
+    dockerBaseImage      := "alpine:latest",
path: project/DockerPublish.scala
user: login: btanyab
body: [16b7749](https://github.com/sky-uk/kafka-message-scheduler/pull/390/commits/16b7749858a49c4b74f5ffeef5fedf78e3412ea8)
created_at: 2025-04-09T15:05:13Z
author_association: CONTRIBUTOR

PR comments:
user: login: jakebookerSky
created_at: 2025-04-08T11:41:45Z
body: we'll need to update the `README` to specify we need to build a new image/Snapshot when making new changes to the KMS
author_association: CONTRIBUTOR
user: login: btanyab
created_at: 2025-04-09T15:24:35Z
body: > I would have liked to see better commit discipline, especially as changes have been mode to config and tests. These changes could be better explained _why_ they were made as its not immediately obvious why they needed to be part of this PR and the number of commits reduced to remove the noise and provide better clarity for the grouping of the changes.

We have cleaned and grouped the commits as much as it was possible. It might still not be perfect - this PR was tested for multiple issues with various odd commits. I've added an explanation to the Config changes in the description 👍  

author_association: CONTRIBUTOR