Repository: AbsaOSS/pramen (language: python)

Issue:
#420 Fix Hive DDL escaping for complex schema.
Closes #420
Add support for incremental ingestion
## Background
Currently, incremental updates are made by overwriting the latest info date partitions multiple times a day.
This can be inefficient, especially for big tables with many events.

If the source table has a monotonically increasing field (a timestamp, a number, etc), it can be used for updates to the latest partition.

## Feature
Add support for incremental ingestion (in Pramen).

## Example
```hocon
offset.column {
  name = "created_at"
  type = "datetime"
}
```

## Proposed Solution
This PR adds 'incremental' as a schedule type, and mechanisms for managing offsets (experimental).

Pramen `version 1.10` introduces the concept of incremental ingestion. It allows running a pipeline multiple times a day
without reprocessing data that was already processed. In order to enable it, use `incremental` schedule when defining your
ingestion operation:
```hocon
schedule = "incremental"
```

In order for the incremental ingestion to work you need to define a monotonically increasing field, called an offset.
Usually, this incremental field can be a counter, or a record creation timestamp. You need to define the offset field in
your source. The source should support incremental ingestion in order to use this mode.
```hocon
offset.column {
  name = "created_at"
  type = "datetime"
}
```

Offset types available at the moment:

| Type     | Description                                |
|----------|--------------------------------------------|
| integral | Any integral type (`short`, `int`, `long`) |
| datetime | A `datetime `or `timestamp` fields         |
| string   | Only `string` / `varchar(n)` types.        |

Only ingestion jobs support incremental schedule at the moment. Incremental transformations and sinks are planned to be
available soon.

Pramen PR: https://github.com/AbsaOSS/pramen/pull/487

After the completion of the issue an epic will be created for Aqueduct.


Discussion / hints:
### Unit Test Coverage
|File|Coverage [94.6%]|:green_apple:|
|:-|:-:|:-:|
|[SparkUtils.scala](https://github.com/AbsaOSS/pramen/blob/fa48cbf81004e25308540e66e3d27fd259ff31b1/pramen%2Fcore%2Fsrc%2Fmain%2Fscala%2Fza%2Fco%2Fabsa%2Fpramen%2Fcore%2Futils%2FSparkUtils.scala)|94.6%|:green_apple:|

|Total Project Coverage|83.65%|:green_apple:|
|:-|:-:|:-:|


Fix patch:
diff --git a/.github/workflows/jacoco.yml b/.github/workflows/jacoco.yml
index cb00d46db..6dcb6bdde 100644
--- a/.github/workflows/jacoco.yml
+++ b/.github/workflows/jacoco.yml
@@ -48,7 +48,7 @@ jobs:
         run: sbt -DSPARK_VERSION=${{matrix.spark}} ++${{matrix.scala}} jacoco
       - name: Add coverage to PR
         id: jacoco
-        uses: madrapps/jacoco-report@v1.3
+        uses: madrapps/jacoco-report@v1.7.1
         with:
           paths: >
             ${{ github.workspace }}/pramen/core/target/scala-${{ matrix.scala_short }}/jacoco/report/jacoco.xml,
@@ -58,6 +58,7 @@ jobs:
           min-coverage-changed-files: ${{ matrix.changed }}
           title: Unit Test Coverage
           update-comment: true
+          #debug-mode: true
       - name: Get the Coverage info
         run: |
           echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}"
diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml
index 1cf8cd1c0..6e50ea496 100644
--- a/.github/workflows/scala.yml
+++ b/.github/workflows/scala.yml
@@ -2,12 +2,16 @@ name: ScalaCI
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - "main"
+      - "support/*"
     paths:
       - "pramen/**"
       - ".github/workflows/scala.yml"
   pull_request:
-    branches: [ main ]
+    branches:
+      - "main"
+      - "support/*"
     paths:
       - "pramen/**"
       - ".github/workflows/scala.yml"
@@ -18,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        scala: [2.11.12, 2.12.19, 2.13.13]
+        scala: [2.11.12, 2.12.20, 2.13.14]
         spark: [2.4.8, 3.3.4, 3.4.2, 3.5.1]
         exclude:
           - scala: 2.11.12
@@ -27,9 +31,9 @@ jobs:
             spark: 3.4.2
           - scala: 2.11.12
             spark: 3.5.1
-          - scala: 2.12.19
+          - scala: 2.12.20
             spark: 2.4.8
-          - scala: 2.13.13
+          - scala: 2.13.14
             spark: 2.4.8
     name: Test Spark ${{matrix.spark}} on Scala ${{matrix.scala}}
     steps:
@@ -42,9 +46,19 @@ jobs:
           distribution: temurin
           java-version: 8
           cache: sbt
+      - name: Install sbt
+        run: |
+          sudo apt-get update
+          sudo apt-get install apt-transport-https curl gnupg -yqq
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list
+          curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo -H gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import
+          sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+          sudo apt-get update
+          sudo apt-get install sbt
       - name: Build and run unit tests
         working-directory: ./pramen
-        run: sbt ++${{matrix.scala}} test -DSPARK_VERSION=${{matrix.spark}}
+        run: sbt ++${{matrix.scala}} unit:test -DSPARK_VERSION=${{matrix.spark}}
       - name: Run integration tests
         working-directory: ./pramen
         run: sbt ++${{matrix.scala}} integration:test -DSPARK_VERSION=${{matrix.spark}}
diff --git a/README.md b/README.md
index e4bf267ba..b573b8713 100644
--- a/README.md
+++ b/README.md
@@ -115,6 +115,24 @@ In addition to basic error notification, typical operational warnings are genera
 
 Pramen is built using SBT.
 
+**Note** By default `sbt test` runs unit tests and integration tests. In order to run just unit tests, please use
+`sbt t` alias.
+
+- `sbt +t` - runs unit tests only, for all Scala versions
+- `sbt test` - runs all tests (unit and integration)
+- `sbt unit:test` - runs unit tests only
+- `sbt integration:test` - runs integration tests only
+
+Install locally for `sbt` projects:
+```
+sbt +publishLocal
+```
+
+Install locally for `Maven` projects:
+```
+sbt +publishM2
+```
+
 ## Project structure
 Pramen consists of a few components:
 - `pramen-api` - contains traits (interfaces) for defining custom transformations, sources and sinks. 
@@ -188,8 +206,8 @@ dependencies in an uber jar that you can build for your Scala version. You can d
 Creating an uber jar for Pramen is very easy. Just clone the repository and run one of the following commands:
 ```sh
 sbt ++2.11.12 assembly 
-sbt ++2.12.18 assembly
-sbt ++2.13.12 assembly
+sbt ++2.12.20 assembly
+sbt ++2.13.14 assembly
 ```
 
 You can collect the uber jar of Pramen either at
@@ -201,8 +219,8 @@ Spark distributions. This makes the runner independent of Spark version. But if
 in your bundle, use one of example commands specifying your Spark version:
 ```sh
 sbt -DSPARK_VERSION="2.4.8" -Dassembly.features="includeDelta" ++2.11.12 assembly 
-sbt -DSPARK_VERSION="3.3.3" -Dassembly.features="includeDelta" ++2.12.18 assembly
-sbt -DSPARK_VERSION="3.4.1" -Dassembly.features="includeDelta" ++2.13.12 assembly
+sbt -DSPARK_VERSION="3.3.4" -Dassembly.features="includeDelta" ++2.12.20 assembly
+sbt -DSPARK_VERSION="3.5.2" -Dassembly.features="includeDelta" ++2.13.14 assembly
 ```
 
 Then, run `spark-shell` or `spark-submit` adding the fat jar as the option.
@@ -602,6 +620,10 @@ is determined by the pipeline configuration.
     # Specifies the maximum number of records to fetch. Good for testing purposes.
     #limit.records = 100
 
+    # Specify the timezone of the database server, if it is different from the default timezone.
+    # It is needed for incremental ingestion based on offset field that has a timestamp or datetime data type.
+    #server.timezone = "Africa/Johannesburg"
+
     # Optionally, you can specify a class for a custom SQL generator for your RDMS engine.
     # The class whould extend 'za.co.absa.pramen.api.sql.SqlGenerator'
     #sql.generator.class = "com.example.MySqlGenerator"
@@ -786,6 +808,34 @@ pramen.operations = [
 ]
 ```
 
+### Incremental Ingestion (experimental)
+Pramen `version 1.10` introduces the concept of incremental ingestion. It allows running a pipeline multiple times a day
+without reprocessing data that was already processed. In order to enable it, use `incremental` schedule when defining your
+ingestion operation:
+```hocon
+schedule = "incremental"
+```
+
+In order for the incremental ingestion to work you need to define a monotonically increasing field, called an offset.
+Usually, this incremental field can be a counter, or a record creation timestamp. You need to define the offset field in
+your source. The source should support incremental ingestion in order to use this mode.
+```hocon
+offset.column {
+  name = "created_at"
+  type = "datetime"
+}
+```
+
+Offset types available at the moment:
+
+| Type     | Description                                |
+|----------|--------------------------------------------|
+| integral | Any integral type (`short`, `int`, `long`) |
+| datetime | A `datetime `or `timestamp` fields         |
+| string   | Only `string` / `varchar(n)` types.        |
+
+Only ingestion jobs support incremental schedule at the moment. Incremental transformations and sinks are planned to be
+available soon.
 
 ### Sinks
 Sinks define a way data needs to be sent to a target system. Built-in sinks include:
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/MetaTableDef.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/MetaTableDef.scala
index 5400b1d59..832149ec0 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/MetaTableDef.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/MetaTableDef.scala
@@ -26,6 +26,7 @@ import java.time.LocalDate
   * @param format          The format of the table.
   * @param infoDateColumn  The name of the column that contains the information date (partitioned by).
   * @param infoDateFormat  The format of the information date.
+  * @param batchIdColumn   The name of the column that contains the batch id.
   * @param hiveTable       The name of the Hive table.
   * @param hivePath        The path of the Hive table (if it differs from the path in the underlying format).
   * @param infoDateStart   The start date of the information date.
@@ -38,6 +39,7 @@ case class MetaTableDef(
                          format: DataFormat,
                          infoDateColumn: String,
                          infoDateFormat: String,
+                         batchIdColumn: String,
                          hiveTable: Option[String],
                          hivePath: Option[String],
                          infoDateStart: LocalDate,
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/MetastoreReader.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/MetastoreReader.scala
index b64824716..d2a28313d 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/MetastoreReader.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/MetastoreReader.scala
@@ -17,6 +17,8 @@
 package za.co.absa.pramen.api
 
 import org.apache.spark.sql.DataFrame
+import za.co.absa.pramen.api.offset.DataOffset
+import za.co.absa.pramen.api.status.TaskRunReason
 
 import java.time.LocalDate
 
@@ -27,7 +29,7 @@ import java.time.LocalDate
 trait MetastoreReader {
 
   /**
-    * Reads a table given th range of information dates, and returns back the dataframe.
+    * Reads a table given the range of information dates, and returns back the dataframe.
     *
     * In order to read a table it is not sufficient the table to be registered in the metastore. It also
     * should be defined as input tables of the job. Otherwise, a runtime exception will be thrown.
@@ -41,6 +43,29 @@ trait MetastoreReader {
                infoDateFrom: Option[LocalDate] = None,
                infoDateTo: Option[LocalDate] = None): DataFrame
 
+  /**
+    * Reads the 'current batch' of the table to be processed incrementally.
+    *
+    * For incremental processing this method returns the current chunk being processed.
+    * It may include multiple chunks from non-processed data if transformer has failed previously.
+    *
+    * For non-incremental processing the call to this method is equivalent to:
+    * {{{
+    *   val df = getTable(tableName)
+    * }}}
+    *
+    * which returns all data for the current information date being processed.
+    *
+    * This method is the method to use for transformers that would use 'incremental' schedule.
+    *
+    * In order to read a table it is not sufficient the table to be registered in the metastore. It also
+    * should be defined as input tables of the job. Otherwise, a runtime exception will be thrown.
+    *
+    * @param tableName    The name of the table to read.
+    * @return The dataframe containing data from the table.
+    */
+  def getCurrentBatch(tableName: String): DataFrame
+
   /**
     * Reads the latest partition of a given table.
     *
@@ -66,7 +91,6 @@ trait MetastoreReader {
     */
   def getLatestAvailableDate(tableName: String, until: Option[LocalDate] = None): Option[LocalDate]
 
-
   /**
     * Returns true if data for the specified table is available for the specified range.
     *
@@ -79,6 +103,15 @@ trait MetastoreReader {
     */
   def isDataAvailable(tableName: String, from: Option[LocalDate], until: Option[LocalDate]): Boolean
 
+  /**
+    * Returns offsets for an information date (both committed and uncommitted).
+    *
+    * This info can be used by transformers and sinks to decide if actions need to be taken depending on the
+    * current micro batch. For example, adding partitions to Hive needs to happen only once per info date,
+    * so a sink that does this can check if micro-batches have been ran for the current day.
+    */
+  def getOffsets(table: String, infoDate: LocalDate): Array[DataOffset]
+
   /**
     * Gets definition of a metastore table. Please, use with caution and do not write to the underlying path
     * from transformers.
@@ -99,6 +132,12 @@ trait MetastoreReader {
     */
   def getTableRunInfo(tableName: String, infoDate: LocalDate): Option[MetaTableRunInfo]
 
+  /**
+    * Returns the reason of running the task. This helps transformers and sinks to determine logic based on whether
+    * thr run is a normal run or a force re-run.
+    */
+  def getRunReason: TaskRunReason
+
   /**
     * Returns an object that allows accessing metadata of metastore tables.
     */
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/Source.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/Source.scala
index ea7133b38..5557adb2c 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/Source.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/Source.scala
@@ -16,6 +16,8 @@
 
 package za.co.absa.pramen.api
 
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
+
 import java.time.LocalDate
 
 /**
@@ -42,6 +44,13 @@ trait Source extends ExternalChannel {
     */
   def hasInfoDateColumn(query: Query): Boolean = true
 
+  /**
+    * If non-empty, the source is configured for incremental ingestion, returns minimum value with type
+    *
+    * If empty, the source can't be used for incremental ingestion.
+    */
+  def getOffsetInfo: Option[OffsetInfo] = None
+
   /**
     * Validates if the source is okay and the ingestion can proceed.
     */
@@ -57,6 +66,51 @@ trait Source extends ExternalChannel {
     */
   def getData(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String]): SourceResult
 
+  /**
+    * Returns the incremental data between specified offsets. The offset intervals could be half open,
+    * e.g. only offsetFrom or offsetTo is specified.
+    *
+    * If an information date is provided and available at the source, the query will be limited to that date.
+    *
+    * <ul>
+    * <li> When both `offsetFrom` from and `offsetTo` are passed the source should return offsets using an inclusive interval
+    *   (offsetFrom <= offset <= offsetTo) </li>
+    * <li> When only `offsetFrom` is present the source should return offsets using an exclusive interval interval
+    *   (offset > offsetFrom)</li>
+    * <li> When only `offsetTo` is present the source should return offsets using an inclusive interval
+    *   (offset <= offsetTo)</li>
+    *</ul>
+    *
+    * The method will be used in incremental ingestion like this. When the framework queries new data the caller would
+    * specify only `offsetFrom`, and the query is going to look like:
+    *
+    * {{{
+    * SELECT * FROM table WHERE offset > offsetFrom
+    * (exclusive)
+    * }}}
+    *
+    * When a rerun is happening for a day, the caller provided both minimum and maximum offsets for that day and runs:
+    *
+    * {{{
+    * SELECT * FROM table WHERE offset >= offsetFrom offset <= offsetTo
+    * (inclusive)
+    * }}}
+    *
+    * The last case, when only `offsetTo` is available might not be used in practice. Added it for completion.
+    * Potentially it can be used to query the old database for all data that was already loaded:
+    *
+    * {{{
+    * SELECT * FROM table WHERE offset <= offsetTo
+    * (inclusive)
+    * }}}
+    *
+    * @param offsetFromOpt   This is an exclusive parameter the query will be SELECT ... WHERE offset_col > min_offset
+    * @param offsetToOpt     This is an exclusive parameter the query will be SELECT ... WHERE offset_col <= min_offset
+    * @param onlyForInfoDate An information date to get data for. Can be empty if the source table doesn't have such a column.
+    * @param columns         Select only specified columns. Selects all if an empty Seq is passed.
+    */
+  def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): SourceResult
+
   /**
     * This method is called after the ingestion is finished. You can query the output table form the output information
     * data and the data should be there.
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/TableReader.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/TableReader.scala
index ce447d94d..0453b14ab 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/TableReader.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/TableReader.scala
@@ -17,6 +17,7 @@
 package za.co.absa.pramen.api
 
 import org.apache.spark.sql.DataFrame
+import za.co.absa.pramen.api.offset.OffsetValue
 
 import java.time.LocalDate
 
@@ -24,4 +25,6 @@ trait TableReader {
   def getRecordCount(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long
 
   def getData(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String]): DataFrame
+
+  def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): DataFrame
 }
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/DataOffset.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/DataOffset.scala
new file mode 100644
index 000000000..c5f1f410a
--- /dev/null
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/DataOffset.scala
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.api.offset
+
+import java.time.{Instant, LocalDate}
+
+sealed trait DataOffset {
+  val tableName: String
+
+  val infoDate: LocalDate
+
+  val batchId: Long
+
+  val createdAt: Instant
+
+  val isCommitted: Boolean
+}
+
+object DataOffset {
+  case class CommittedOffset(
+                              tableName: String,
+                              infoDate: LocalDate,
+                              batchId: Long,
+                              minOffset: OffsetValue,
+                              maxOffset: OffsetValue,
+                              createdAt: Instant,
+                              committedAt: Instant
+                            ) extends DataOffset {
+
+    override val isCommitted: Boolean = true
+  }
+
+  case class UncommittedOffset(
+                                tableName: String,
+                                infoDate: LocalDate,
+                                batchId: Long,
+                                offsetType: OffsetType,
+                                createdAt: Instant
+                              ) extends DataOffset {
+
+    override val isCommitted: Boolean = false
+  }
+}
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetInfo.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetInfo.scala
new file mode 100644
index 000000000..f5ae0c701
--- /dev/null
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetInfo.scala
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.api.offset
+
+case class OffsetInfo(offsetColumn: String, offsetType: OffsetType)
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetType.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetType.scala
new file mode 100644
index 000000000..ae39d4107
--- /dev/null
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetType.scala
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.api.offset
+
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{LongType, StringType => SparkStringType}
+
+sealed trait OffsetType {
+  def dataTypeString: String
+
+  def getSparkCol(col: Column): Column
+}
+
+object OffsetType {
+  val DATETIME_TYPE_STR = "datetime"
+  val INTEGRAL_TYPE_STR = "integral"
+  val STRING_TYPE_STR = "string"
+
+  val MINIMUM_TIMESTAMP_EPOCH_MILLI: Long = -62135596800000L
+
+  case object DateTimeType extends OffsetType {
+    override val dataTypeString: String = DATETIME_TYPE_STR
+
+    override def getSparkCol(c: Column): Column = concat(unix_timestamp(c), date_format(c, "SSS")).cast(LongType)
+  }
+
+  case object IntegralType extends OffsetType {
+    override val dataTypeString: String = INTEGRAL_TYPE_STR
+
+    override def getSparkCol(c: Column): Column = c.cast(LongType)
+  }
+
+  case object StringType extends OffsetType {
+    override val dataTypeString: String = STRING_TYPE_STR
+
+    override def getSparkCol(c: Column): Column = c.cast(SparkStringType)
+  }
+
+  def fromString(dataType: String): OffsetType = dataType match {
+    case DATETIME_TYPE_STR => DateTimeType
+    case INTEGRAL_TYPE_STR => IntegralType
+    case STRING_TYPE_STR => StringType
+    case _ => throw new IllegalArgumentException(s"Unknown offset data type: $dataType")
+  }
+}
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetValue.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetValue.scala
new file mode 100644
index 000000000..dd3a3642a
--- /dev/null
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetValue.scala
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.api.offset
+
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.functions._
+import za.co.absa.pramen.api.offset.OffsetType.{DATETIME_TYPE_STR, INTEGRAL_TYPE_STR, STRING_TYPE_STR}
+
+import java.time.Instant
+
+sealed trait OffsetValue extends Comparable[OffsetValue] {
+  def dataType: OffsetType
+
+  def valueString: String
+
+  def getSparkLit: Column
+}
+
+object OffsetValue {
+  case class DateTimeValue(t: Instant) extends OffsetValue {
+    override val dataType: OffsetType = OffsetType.DateTimeType
+
+    override def valueString: String = t.toEpochMilli.toString
+
+    override def getSparkLit: Column = lit(t.toEpochMilli)
+
+    override def compareTo(other: OffsetValue): Int = {
+      other match {
+        case DateTimeValue(otherValue) => t.compareTo(otherValue)
+        case _ => throw new IllegalArgumentException(s"Cannot compare ${dataType.dataTypeString} with ${other.dataType.dataTypeString}")
+      }
+    }
+  }
+
+  case class IntegralValue(value: Long) extends OffsetValue {
+    override val dataType: OffsetType = OffsetType.IntegralType
+
+    override def valueString: String = value.toString
+
+    override def getSparkLit: Column = lit(value)
+
+    override def compareTo(other: OffsetValue): Int = {
+      other match {
+        case IntegralValue(otherValue) => value.compareTo(otherValue)
+        case _ => throw new IllegalArgumentException(s"Cannot compare ${dataType.dataTypeString} with ${other.dataType.dataTypeString}")
+      }
+    }
+  }
+
+  case class StringValue(s: String) extends OffsetValue {
+    override val dataType: OffsetType = OffsetType.StringType
+
+    override def valueString: String = s
+
+    override def getSparkLit: Column = lit(s)
+
+    override def compareTo(other: OffsetValue): Int = {
+      other match {
+        case StringValue(otherValue) => s.compareTo(otherValue)
+        case _ => throw new IllegalArgumentException(s"Cannot compare ${dataType.dataTypeString} with ${other.dataType.dataTypeString}")
+      }
+    }
+  }
+
+  def fromString(dataType: String, value: String): Option[OffsetValue] = {
+    if (value.isEmpty)
+      None
+    else
+      dataType match {
+        case DATETIME_TYPE_STR => Some(DateTimeValue(Instant.ofEpochMilli(value.toLong)))
+        case INTEGRAL_TYPE_STR => Some(IntegralValue(value.toLong))
+        case STRING_TYPE_STR => Some(StringValue(value))
+        case _ => throw new IllegalArgumentException(s"Unknown offset data type: $dataType")
+      }
+  }
+}
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala
index b89d73f3d..8852b3b16 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala
@@ -44,4 +44,20 @@ object SqlColumnType {
       case _ => None
     }
   }
+
+  def fromStringStrict(s: String, configPath: String = ""): SqlColumnType = {
+    s.toLowerCase() match {
+      case DATE.toString     => DATE
+      case DATETIME.toString => DATETIME
+      case STRING.toString   => STRING
+      case NUMBER.toString   => NUMBER
+      case _ =>
+        val allowedTypes = Seq(DATE.toString, DATETIME.toString, STRING.toString, NUMBER.toString).mkString(", ")
+        if (configPath.nonEmpty) {
+          throw new IllegalArgumentException(s"Unknown information type '$s' configured at $configPath. Allowed valued: $allowedTypes.")
+        } else {
+          throw new IllegalArgumentException(s"Unknown information type '$s'. Allowed valued: $allowedTypes.")
+        }
+    }
+  }
 }
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala
index eba62e797..74990da29 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala
@@ -17,11 +17,16 @@
 package za.co.absa.pramen.api.sql
 
 import com.typesafe.config.Config
+import za.co.absa.pramen.api.offset.OffsetInfo
+
+import java.time.ZoneId
 
 case class SqlConfig(
                       infoDateColumn: String,
                       infoDateType: SqlColumnType,
                       dateFormatApp: String,
+                      offsetInfo: Option[OffsetInfo],
+                      serverTimeZone: ZoneId,
                       identifierQuotingPolicy: QuotingPolicy,
                       sqlGeneratorClass: Option[String],
                       extraConfig: Config
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala
index 947e331ed..e38009542 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala
@@ -16,6 +16,8 @@
 
 package za.co.absa.pramen.api.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
+
 import java.sql.Connection
 import java.time.LocalDate
 
@@ -51,6 +53,24 @@ trait SqlGenerator {
     */
   def getDataQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String], limit: Option[Int]): String
 
+  /**
+    * Generates a query for incremental ingestion, the result can be restricted by an information column, if present, but also by offset range.
+    *
+    * <ul>
+    * <li> When both `offsetFrom` from and `offsetTo` are passed the generator should return offsets using an inclusive interval
+    *   (offsetFrom <= offset <= offsetTo) </li>
+    * <li> When only `offsetFrom` is present the generator should return offsets using an exclusive interval interval
+    *   (offset > offsetFrom)</li>
+    * <li> When only `offsetTo` is present the generator should return offsets using an inclusive interval
+    *   (offset <= offsetTo)</li>
+    *</ul>
+    */
+  def getDataQueryIncremental(tableName: String,
+                              onlyForInfoDate: Option[LocalDate],
+                              offsetFrom: Option[OffsetValue],
+                              offsetTo: Option[OffsetValue],
+                              columns: Seq[String]): String
+
   /**
     * Returns WHERE condition for table that has the information date field given the time period.
     */
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala
index 15945a1c8..8d40f7114 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala
@@ -16,6 +16,10 @@
 
 package za.co.absa.pramen.api.sql
 
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
+
+import java.time.LocalDate
+import java.time.format.DateTimeFormatter
 import scala.collection.mutable.ListBuffer
 
 /**
@@ -26,6 +30,8 @@ import scala.collection.mutable.ListBuffer
 abstract class SqlGeneratorBase(sqlConfig: SqlConfig) extends SqlGenerator {
   import SqlGeneratorBase._
 
+  protected val timestampGenericDbFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+
   /**
     * This returns characters used for escaping into a mode that allows special characters in identifiers.
     * For example,
@@ -54,21 +60,23 @@ abstract class SqlGeneratorBase(sqlConfig: SqlConfig) extends SqlGenerator {
     }
   }
 
+  def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String
+
   override def getAliasExpression(expression: String, alias: String): String = {
     s"$expression AS ${escape(alias)}"
   }
 
-  override final def quote(identifier: String): String = {
+  override def quote(identifier: String): String = {
     validateIdentifier(identifier)
     splitComplexIdentifier(identifier).map(quoteSingleIdentifier).mkString(".")
   }
 
-  override final def unquote(identifier: String): String = {
+  override def unquote(identifier: String): String = {
     validateIdentifier(identifier)
     splitComplexIdentifier(identifier).map(unquoteSingleIdentifier).mkString(".")
   }
 
-  override final def escape(identifier: String): String = {  
+  override def escape(identifier: String): String = {
     if (needsEscaping(sqlConfig.identifierQuotingPolicy, identifier)) {
       quote(identifier)
     } else {
@@ -76,6 +84,51 @@ abstract class SqlGeneratorBase(sqlConfig: SqlConfig) extends SqlGenerator {
     }
   }
 
+  override def getDataQueryIncremental(tableName: String,
+                                       onlyForInfoDate: Option[LocalDate],
+                                       offsetFromOpt: Option[OffsetValue],
+                                       offsetToOpt: Option[OffsetValue],
+                                       columns: Seq[String]): String = {
+    if (sqlConfig.offsetInfo.isEmpty)
+      throw new IllegalArgumentException(s"Offset information is not configured for database table: $tableName.")
+
+    val dataQuery = onlyForInfoDate match {
+      case Some(infoDate) => getDataQuery(tableName, infoDate, infoDate, columns, None)
+      case None => getDataQuery(tableName, columns, None)
+    }
+
+    val offsetWhere = getOffsetWhereClause(sqlConfig.offsetInfo.get, offsetFromOpt, offsetToOpt)
+
+    if (offsetWhere.nonEmpty) {
+      if (onlyForInfoDate.isEmpty) {
+        s"$dataQuery WHERE $offsetWhere"
+      } else {
+        s"$dataQuery AND $offsetWhere"
+      }
+    } else {
+      dataQuery
+    }
+  }
+
+  def getOffsetWhereClause(offsetInfo: OffsetInfo, offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue]): String = {
+    val offsetColumn = escape(offsetInfo.offsetColumn)
+
+    (offsetFromOpt, offsetToOpt) match {
+      case (Some(offsetFrom), Some(offsetTo)) =>
+        validateOffsetValue(offsetFrom)
+        validateOffsetValue(offsetTo)
+        s"${getOffsetWhereCondition(offsetColumn, ">=", offsetFrom)} AND ${getOffsetWhereCondition(offsetColumn, "<=", offsetTo)}"
+      case (Some(offsetFrom), None) =>
+        validateOffsetValue(offsetFrom)
+        s"${getOffsetWhereCondition(offsetColumn, ">", offsetFrom)}"
+      case (None, Some(offsetTo)) =>
+        validateOffsetValue(offsetTo)
+        s"${getOffsetWhereCondition(offsetColumn, "<=", offsetTo)}"
+      case (None, None) =>
+        ""
+    }
+  }
+
   /**
     * An expression for the list of configured columns.
     * @return A part of SQL expression listing column names.
@@ -92,7 +145,7 @@ abstract class SqlGeneratorBase(sqlConfig: SqlConfig) extends SqlGenerator {
     val trimmedIdentifier = identifier.trim
 
     if (trimmedIdentifier.isEmpty) {
-      throw new IllegalArgumentException(f"Found an empty table name or column name ('$identifier').")
+      throw new IllegalArgumentException(s"Found an empty table name or column name ('$identifier').")
     }
 
     val (escapeBegin, escapeEnd) = beginEndEscapeChars
@@ -157,6 +210,8 @@ abstract class SqlGeneratorBase(sqlConfig: SqlConfig) extends SqlGenerator {
 }
 
 object SqlGeneratorBase {
+  val MAX_STRING_OFFSET_CHARACTERS = 64
+
   val forbiddenCharacters = ";'\\"
   val normalCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_."
 
@@ -174,4 +229,16 @@ object SqlGeneratorBase {
       case QuotingPolicy.Auto => !identifier.forall(normalCharacters.contains(_))
     }
   }
+
+  final def validateOffsetValue(offset: OffsetValue): Unit = {
+    offset match {
+      case OffsetValue.StringValue(s) =>
+        if (s.contains('\''))
+          throw new IllegalArgumentException(s"Offset value '$s' contains a single quote character, which is not supported.")
+        if (s.length > MAX_STRING_OFFSET_CHARACTERS)
+          throw new IllegalArgumentException(s"Offset value '$s' is bigger than $MAX_STRING_OFFSET_CHARACTERS bytes")
+      case _ =>
+      // Any value is okay
+    }
+  }
 }
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/status/PipelineStateSnapshot.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/status/PipelineStateSnapshot.scala
index 1f98496b2..3a89186b3 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/status/PipelineStateSnapshot.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/status/PipelineStateSnapshot.scala
@@ -20,6 +20,7 @@ import za.co.absa.pramen.api.PipelineInfo
 
 case class PipelineStateSnapshot(
                                   pipelineInfo: PipelineInfo,
+                                  batchId: Long,
                                   isFinished: Boolean,
                                   warningFlag: Boolean,
                                   exitedNormally: Boolean,
diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/status/RunStatus.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/status/RunStatus.scala
index 15d579494..970817a4a 100644
--- a/pramen/api/src/main/scala/za/co/absa/pramen/api/status/RunStatus.scala
+++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/status/RunStatus.scala
@@ -24,7 +24,8 @@ sealed trait RunStatus {
 
 object RunStatus {
   case class Succeeded(recordCountOld: Option[Long],
-                       recordCount: Long,
+                       recordCount: Option[Long],
+                       recordsAppended: Option[Long],
                        sizeBytes: Option[Long],
                        reason: TaskRunReason,
                        filesRead: Seq[String],
diff --git a/pramen/build.sbt b/pramen/build.sbt
index 2eb1fc058..4b7231984 100644
--- a/pramen/build.sbt
+++ b/pramen/build.sbt
@@ -20,12 +20,12 @@ import BuildInfoTemplateSettings._
 import com.github.sbt.jacoco.report.JacocoReportSettings
 
 val scala211 = "2.11.12"
-val scala212 = "2.12.19"
-val scala213 = "2.13.13"
+val scala212 = "2.12.20"
+val scala213 = "2.13.14"
 
 ThisBuild / organization := "za.co.absa.pramen"
 
-ThisBuild / scalaVersion := scala211
+ThisBuild / scalaVersion := scala212
 ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213)
 
 ThisBuild / scalacOptions := Seq("-unchecked", "-deprecation")
@@ -52,6 +52,7 @@ val shadeBase = "za.co.absa.pramen.shaded"
 
 def itFilter(name: String): Boolean = name endsWith "LongSuite"
 def unitFilter(name: String): Boolean = (name endsWith "Suite") && !itFilter(name)
+def allFilter(name: String): Boolean = (name endsWith "Suite")
 def shade(pkg: String): (String, String) = s"$pkg.**" -> s"$shadeBase.$pkg.@1"
 
 /* This is so that Pramen uber jar has the name compatible to versions where 'pramen-runner' module existed */
@@ -70,6 +71,7 @@ def runnerSparkVersionSuffix(moduleName: String, scalaVersion: String, includeDe
 
 val assemblyFeatures = settingKey[Seq[String]]("Define assembly scope")
 
+lazy val UnitTest = config("unit") extend Test
 lazy val IntegrationTest = config("integration") extend Test
 
 lazy val pramen = (project in file("."))
@@ -86,6 +88,10 @@ lazy val pramen = (project in file("."))
 
 lazy val api = (project in file("api"))
   .disablePlugins(sbtassembly.AssemblyPlugin)
+  .configs( UnitTest )
+  .configs( IntegrationTest )
+  .settings( inConfig(UnitTest)(Defaults.testTasks) : _*)
+  .settings( inConfig(IntegrationTest)(Defaults.testTasks) : _*)
   .settings(
     name := "pramen-api",
     printSparkVersion := {
@@ -95,12 +101,17 @@ lazy val api = (project in file("api"))
     },
     (Compile / compile) := ((Compile / compile) dependsOn printSparkVersion).value,
     libraryDependencies ++= ApiDependencies(scalaVersion.value) :+ getScalaDependency(scalaVersion.value),
+    (Test / testOptions) := Seq(Tests.Filter(allFilter)),
+    (UnitTest / testOptions) := Seq(Tests.Filter(unitFilter)),
+    (IntegrationTest / testOptions) := Seq(Tests.Filter(itFilter)),
     releasePublishArtifactsAction := PgpKeys.publishSigned.value
   )
   .enablePlugins(AutomateHeaderPlugin)
 
 lazy val core = (project in file("core"))
+  .configs( UnitTest )
   .configs( IntegrationTest )
+  .settings( inConfig(UnitTest)(Defaults.testTasks) : _*)
   .settings( inConfig(IntegrationTest)(Defaults.testTasks) : _*)
   .settings(
     name := "pramen-core",
@@ -114,7 +125,8 @@ lazy val core = (project in file("core"))
     libraryDependencies ++= CoreDependencies(scalaVersion.value, assemblyFeatures.value.contains("includeDelta"))  ++
       getSparkVersionRelatedDeps(sparkVersion(scalaVersion.value)) :+
       getScalaDependency(scalaVersion.value),
-    (Test / testOptions) := Seq(Tests.Filter(unitFilter)),
+    (Test / testOptions) := Seq(Tests.Filter(allFilter)),
+    (UnitTest / testOptions) := Seq(Tests.Filter(unitFilter)),
     (IntegrationTest / testOptions) := Seq(Tests.Filter(itFilter)),
     Test / fork := true,
     populateBuildInfoTemplate,
@@ -127,6 +139,10 @@ lazy val core = (project in file("core"))
   .enablePlugins(AutomateHeaderPlugin)
 
 lazy val extras = (project in file("extras"))
+  .configs( UnitTest )
+  .configs( IntegrationTest )
+  .settings( inConfig(UnitTest)(Defaults.testTasks) : _*)
+  .settings( inConfig(IntegrationTest)(Defaults.testTasks) : _*)
   .settings(
     name := "pramen-extras",
     printSparkVersion := {
@@ -140,6 +156,9 @@ lazy val extras = (project in file("extras"))
       getSparkVersionRelatedDeps(sparkVersion(scalaVersion.value)) :+
       getScalaDependency(scalaVersion.value),
     resolvers += "confluent" at "https://packages.confluent.io/maven/",
+    (Test / testOptions) := Seq(Tests.Filter(allFilter)),
+    (UnitTest / testOptions) := Seq(Tests.Filter(unitFilter)),
+    (IntegrationTest / testOptions) := Seq(Tests.Filter(itFilter)),
     Test / fork := true,
     releasePublishArtifactsAction := PgpKeys.publishSigned.value,
     jacocoReportSettings := commonJacocoReportSettings.withTitle("pramen-extras Jacoco Report"),
@@ -274,5 +293,7 @@ lazy val assemblySettingsRunner = assemblySettingsCommon ++ Seq(assembly / assem
 )
 
 addCommandAlias("releaseNow", ";set releaseVersionBump := sbtrelease.Version.Bump.Bugfix; release with-defaults")
+addCommandAlias("t", "unit:test")
+addCommandAlias("utest", "unit:test")
 addCommandAlias("itTest", "integration:test")
-addCommandAlias("xcoverage", "clean;coverage;test;coverageReport")
+addCommandAlias("xcoverage", "jacoco")
diff --git a/pramen/core/src/main/resources/reference.conf b/pramen/core/src/main/resources/reference.conf
index bf79f7d6d..4a7112bb1 100644
--- a/pramen/core/src/main/resources/reference.conf
+++ b/pramen/core/src/main/resources/reference.conf
@@ -104,6 +104,13 @@ pramen {
   # If this is set the current date will overridden by the specified value.
   #current.date =
 
+  # Optionally, you can specify which dates to run historical pipeline for a date range:
+  #load.date.from = "2022-01-01"
+  #load.date.to = "2022-01-15"
+
+  # Specify one of run modes for historical run: fill_gaps, check_updates (default), force.
+  #runtime.run.mode = force
+
   #spark.conf = {
     # Pass arbitrary Spark Configuration when initializing Spark Session
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/PramenImpl.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/PramenImpl.scala
index f70c08806..48765f908 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/PramenImpl.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/PramenImpl.scala
@@ -44,7 +44,7 @@ class PramenImpl extends Pramen {
       throw new IllegalStateException("Pipeline state is not available at the context.")
     )
 
-    pipelineState.getState().pipelineInfo
+    pipelineState.getState.pipelineInfo
   }
 
   override def pipelineState: PipelineStateSnapshot = {
@@ -52,7 +52,7 @@ class PramenImpl extends Pramen {
       throw new IllegalStateException("Pipeline state is not available at the context.")
     )
 
-    pipelineState.getState()
+    pipelineState.getState
   }
 
   override def notificationBuilder: NotificationBuilder = notificationBuilderImpl
@@ -66,7 +66,7 @@ class PramenImpl extends Pramen {
       throw new IllegalStateException("Pipeline state is not available at the context.")
     )
 
-    val state = pipelineState.getState()
+    val state = pipelineState.getState
 
     state.taskResults
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/app/AppContextImpl.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/app/AppContextImpl.scala
index 618d38c80..91970eed0 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/app/AppContextImpl.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/app/AppContextImpl.scala
@@ -51,13 +51,13 @@ class AppContextImpl(val appConfig: AppConfig,
 }
 
 object AppContextImpl {
-  def apply(conf: Config)(implicit spark: SparkSession): AppContextImpl = {
+  def apply(conf: Config, batchId: Long)(implicit spark: SparkSession): AppContextImpl = {
 
     val appConfig = AppConfig.fromConfig(conf)
 
-    val (bookkeeper, tokenLockFactory, journal, metadataManager, closable) = Bookkeeper.fromConfig(appConfig.bookkeepingConfig, appConfig.runtimeConfig)
+    val (bookkeeper, tokenLockFactory, journal, metadataManager, closable) = Bookkeeper.fromConfig(appConfig.bookkeepingConfig, appConfig.runtimeConfig, batchId)
 
-    val metastore: Metastore = MetastoreImpl.fromConfig(conf, appConfig.infoDateDefaults, bookkeeper, metadataManager)
+    val metastore: Metastore = MetastoreImpl.fromConfig(conf, appConfig.infoDateDefaults, bookkeeper, metadataManager, batchId)
 
     PramenImpl.instance.asInstanceOf[PramenImpl].setMetadataManager(metadataManager)
     PramenImpl.instance.asInstanceOf[PramenImpl].setWorkflowConfig(conf)
@@ -84,7 +84,7 @@ object AppContextImpl {
 
     val metadataManager = new MetadataManagerNull(isPersistenceEnabled = false)
 
-    val metastore: Metastore = MetastoreImpl.fromConfig(conf, infoDateConfig, bookkeeper, metadataManager)
+    val metastore: Metastore = MetastoreImpl.fromConfig(conf, infoDateConfig, bookkeeper, metadataManager, 0L)
 
     val appContext = new AppContextImpl(
       appConfig,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/app/config/InfoDateConfig.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/app/config/InfoDateConfig.scala
index e55155bdd..761c4a0da 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/app/config/InfoDateConfig.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/app/config/InfoDateConfig.scala
@@ -39,6 +39,8 @@ case class InfoDateConfig(
 object InfoDateConfig {
   val DEFAULT_DATE_FORMAT = "yyyy-MM-dd"
 
+  val DEFAULT_INCREMENTAL_INFO_DATE_EXPR = "@runDate"
+
   val INFORMATION_DATE_COLUMN_KEY = "pramen.information.date.column"
   val INFORMATION_DATE_FORMAT_KEY = "pramen.information.date.format"
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/Bookkeeper.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/Bookkeeper.scala
index 933cc4b70..8320497fd 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/Bookkeeper.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/Bookkeeper.scala
@@ -57,12 +57,16 @@ trait Bookkeeper {
   def getLatestSchema(table: String, until: LocalDate): Option[(StructType, LocalDate)]
 
   private[pramen] def saveSchema(table: String, infoDate: LocalDate, schema: StructType): Unit
+
+  private[pramen] def getOffsetManager: OffsetManager
 }
 
 object Bookkeeper {
   private val log = LoggerFactory.getLogger(this.getClass)
 
-  def fromConfig(bookkeepingConfig: BookkeeperConfig, runtimeConfig: RuntimeConfig)
+  def fromConfig(bookkeepingConfig: BookkeeperConfig,
+                 runtimeConfig: RuntimeConfig,
+                 batchId: Long)
                 (implicit spark: SparkSession): (Bookkeeper, TokenLockFactory, Journal, MetadataManager, AutoCloseable) = {
     val mongoDbConnection = bookkeepingConfig.bookkeepingConnectionString.map { url =>
       MongoDbConnection.getConnection(url, bookkeepingConfig.bookkeepingDbName.get)
@@ -106,7 +110,7 @@ object Bookkeeper {
       log.info(s"Bookkeeping is DISABLED. Updates won't be tracked")
       new BookkeeperNull()
     } else if (hasBookkeepingJdbc) {
-      new BookkeeperJdbc(dbOpt.get.slickDb)
+      new BookkeeperJdbc(dbOpt.get.slickDb, batchId)
     } else {
       mongoDbConnection match {
         case Some(connection) =>
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperBase.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperBase.scala
index 15aeedf9e..4169663eb 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperBase.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperBase.scala
@@ -112,6 +112,11 @@ abstract class BookkeeperBase(isBookkeepingEnabled: Boolean) extends Bookkeeper
     }
   }
 
+  private[pramen] override def getOffsetManager: OffsetManager = {
+    throw new IllegalArgumentException(s"This implementation of bookeeping does not support offset management and incremental pipelines. " +
+      "Please, use JDBC for bookkeeping to enable this.")
+  }
+
   private def getLatestTransientDate(table: String, from: Option[LocalDate], until: Option[LocalDate]): Option[LocalDate] = {
     val chunks = getTransientDataChunks(table, from, until)
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperJdbc.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperJdbc.scala
index 75aba8f70..de82d0ea3 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperJdbc.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/BookkeeperJdbc.scala
@@ -16,7 +16,6 @@
 
 package za.co.absa.pramen.core.bookkeeper
 
-import com.typesafe.config.Config
 import org.apache.spark.sql.types.StructType
 import org.slf4j.LoggerFactory
 import slick.jdbc.H2Profile.api._
@@ -30,11 +29,11 @@ import za.co.absa.pramen.core.utils.SlickUtils
 import java.time.LocalDate
 import scala.util.control.NonFatal
 
-class BookkeeperJdbc(db: Database) extends BookkeeperBase(true) {
-
+class BookkeeperJdbc(db: Database, batchId: Long) extends BookkeeperBase(true) {
   import za.co.absa.pramen.core.utils.FutureImplicits._
 
   private val log = LoggerFactory.getLogger(this.getClass)
+  private val offsetManagement = new OffsetManagerJdbc(db, batchId)
 
   override val bookkeepingEnabled: Boolean = true
 
@@ -121,6 +120,10 @@ class BookkeeperJdbc(db: Database) extends BookkeeperBase(true) {
     }
   }
 
+  private[pramen] override def getOffsetManager: OffsetManager = {
+    offsetManagement
+  }
+
   private def toChunk(r: BookkeepingRecord): DataChunk = {
     DataChunk(
       r.pramenTableName, r.infoDate, r.infoDateBegin, r.infoDateEnd, r.inputRecordCount, r.outputRecordCount, r.jobStarted, r.jobFinished)
@@ -201,7 +204,7 @@ class BookkeeperJdbc(db: Database) extends BookkeeperBase(true) {
 }
 
 object BookkeeperJdbc {
-  def fromJdbcConfig(jdbcConfig: JdbcConfig): BookkeeperJdbc = {
+  def fromJdbcConfig(jdbcConfig: JdbcConfig, batchId: Long): BookkeeperJdbc = {
     val selector = JdbcUrlSelector(jdbcConfig)
     val url = selector.getWorkingUrl(DEFAULT_RETRIES)
     val prop = selector.getProperties
@@ -211,12 +214,7 @@ object BookkeeperJdbc {
     } else {
       Database.forURL(url = url, driver = jdbcConfig.driver, prop = prop)
     }
-    new BookkeeperJdbc(db)
-  }
-
-  def fromConfig(conf: Config, path: String): BookkeeperJdbc = {
-    val db = Database.forConfig(path, conf)
-    new BookkeeperJdbc(db)
+    new BookkeeperJdbc(db, batchId)
   }
 
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/OffsetManager.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/OffsetManager.scala
new file mode 100644
index 000000000..fbbb0d486
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/OffsetManager.scala
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper
+
+import za.co.absa.pramen.api.offset.DataOffset.UncommittedOffset
+import za.co.absa.pramen.api.offset.{DataOffset, OffsetType, OffsetValue}
+import za.co.absa.pramen.core.bookkeeper.model.{DataOffsetAggregated, DataOffsetRequest}
+
+import java.time.LocalDate
+
+/**
+  * The offset manager allows managing offsets of incremental operations.
+  *
+  * It provides correctness guarantees only if not running in parallel with other jobs outputting to the same
+  * Pramen table.
+  *
+  * Locks are used to ensure no jobs outputting to the same incremental table is happening in parallel.
+  *
+  * The startWriteOffsets() together with commitOffsets() and rollbackOffsets() provide mechanisms to ensure consistency
+  * with data.
+  */
+trait OffsetManager {
+  /**
+    * Returns offsets for an information date.
+    *
+    * If there are uncommitted offsets, they should be handled this way:
+    * - maximum offset should be derived from data,
+    * - the latest uncommitted offset should be committed to the latest offset
+    * - previous uncommitted offsets should be rolled back.
+    */
+  def getOffsets(table: String, infoDate: LocalDate): Array[DataOffset]
+
+  /**
+    * Returns only uncommitted offsets for a give table.
+    */
+  def getUncommittedOffsets(table: String, onlyForInfoDate: Option[LocalDate]): Array[UncommittedOffset]
+
+  /**
+    * Returns the maximum information date the bookkeeping has offsets for.
+    *
+    * If onlyForInfoDate is not empty, offset management is being processed for each info date individually, e.g.
+    * info_date + offset column is monotonic.
+    *
+    * if onlyForInfoDate is empty
+    * offset must be a monotonically increasing field.
+    */
+  def getMaxInfoDateAndOffset(table: String, onlyForInfoDate: Option[LocalDate]): Option[DataOffsetAggregated]
+
+  /**
+    * Starts an uncommitted offset for an incremental ingestion for a day.
+    * This can only be done for the latest information date.
+    */
+  def startWriteOffsets(table: String, infoDate: LocalDate, offsetType: OffsetType): DataOffsetRequest
+
+  /**
+    * Commits changes to the table. If maxOffset is
+    * - greater than or equal to minOffset, a new entry is created.
+    * - less than minOffset - an exception will be thrown.
+    */
+  def commitOffsets(request: DataOffsetRequest, minOffset: OffsetValue, maxOffset: OffsetValue): Unit
+
+  /**
+    * Commits changes to the table as the result of a re-run. This replaces all batch ids
+    * and offsets for that day with new batch id.
+    */
+  def commitRerun(request: DataOffsetRequest, minOffset: OffsetValue, maxOffset: OffsetValue): Unit
+
+  /**
+    * Rolls back an offset request
+    */
+  def rollbackOffsets(request: DataOffsetRequest): Unit
+}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/OffsetManagerJdbc.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/OffsetManagerJdbc.scala
new file mode 100644
index 000000000..9d9ef6f68
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/OffsetManagerJdbc.scala
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper
+
+import org.slf4j.LoggerFactory
+import slick.jdbc.H2Profile.api._
+import za.co.absa.pramen.api.offset.DataOffset.UncommittedOffset
+import za.co.absa.pramen.api.offset.{DataOffset, OffsetType, OffsetValue}
+import za.co.absa.pramen.core.bookkeeper.model._
+import za.co.absa.pramen.core.utils.SlickUtils
+
+import java.time.{Instant, LocalDate}
+import scala.util.control.NonFatal
+
+class OffsetManagerJdbc(db: Database, batchId: Long) extends OffsetManager {
+  import za.co.absa.pramen.core.utils.FutureImplicits._
+
+  private val log = LoggerFactory.getLogger(this.getClass)
+
+  override def getOffsets(table: String, infoDate: LocalDate): Array[DataOffset] = {
+    val offsets = getOffsetRecords(table, infoDate)
+
+    if (offsets.isEmpty) {
+      return Array.empty
+    }
+
+    offsets.map(OffsetRecordConverter.toDataOffset)
+  }
+
+  override def getUncommittedOffsets(table: String, onlyForInfoDate: Option[LocalDate]): Array[UncommittedOffset] = {
+    val query = onlyForInfoDate match {
+      case Some(infoDate) =>
+        val infoDateStr = infoDate.toString
+        OffsetRecords.records
+          .filter(r => r.pramenTableName === table && r.infoDate === infoDateStr && r.committedAt.isEmpty)
+          .sorted(r => r.infoDate)
+      case None =>
+        OffsetRecords.records
+          .filter(r => r.pramenTableName === table && r.committedAt.isEmpty)
+          .sorted(r => r.infoDate)
+    }
+
+    SlickUtils.executeQuery[OffsetRecords, OffsetRecord](db, query)
+      .toArray[OffsetRecord]
+      .map(record => OffsetRecordConverter.toDataOffset(record).asInstanceOf[UncommittedOffset])
+  }
+
+  override def getMaxInfoDateAndOffset(table: String, onlyForInfoDate: Option[LocalDate]): Option[DataOffsetAggregated] = {
+    val maxInfoDateOpt = onlyForInfoDate.orElse(getMaximumInfoDate(table))
+
+    try {
+      maxInfoDateOpt.flatMap { infoDate =>
+        getMinMaxOffsets(table, infoDate)
+      }
+    } catch {
+      case NonFatal(ex) => throw new RuntimeException(s"Unable to read from the offset table.", ex)
+    }
+  }
+
+  override def startWriteOffsets(table: String, infoDate: LocalDate, offsetType: OffsetType): DataOffsetRequest = {
+    val createdAt = Instant.now()
+
+    val record = OffsetRecord(table, infoDate.toString, offsetType.dataTypeString, "", "", batchId, createdAt.toEpochMilli, None)
+
+    db.run(
+      OffsetRecords.records += record
+    ).execute()
+
+    DataOffsetRequest(table, infoDate, batchId, createdAt)
+  }
+
+  override def commitOffsets(request: DataOffsetRequest, minOffset: OffsetValue, maxOffset: OffsetValue): Unit = {
+    val committedAt = Instant.now().toEpochMilli
+
+    db.run(
+      OffsetRecords.records
+        .filter(r => r.pramenTableName === request.tableName && r.infoDate === request.infoDate.toString && r.createdAt === request.createdAt.toEpochMilli)
+        .map(r => (r.minOffset, r.maxOffset, r.committedAt))
+        .update((minOffset.valueString, maxOffset.valueString, Some(committedAt)))
+    ).execute()
+  }
+
+  override def commitRerun(request: DataOffsetRequest, minOffset: OffsetValue, maxOffset: OffsetValue): Unit = {
+    if (minOffset.compareTo(maxOffset) > 0) {
+      throw new IllegalArgumentException(s"minOffset is greater than maxOffset: ${minOffset.valueString} > ${maxOffset.valueString}")
+    }
+
+    val committedAt = Instant.now().toEpochMilli
+
+    db.run(
+      OffsetRecords.records
+        .filter(r => r.pramenTableName === request.tableName && r.infoDate === request.infoDate.toString && r.createdAt === request.createdAt.toEpochMilli)
+        .map(r => (r.minOffset, r.maxOffset, r.committedAt))
+        .update((minOffset.valueString, maxOffset.valueString, Some(committedAt)))
+    ).execute()
+
+    // Cleaning up previous batches
+    db.run(
+      OffsetRecords.records
+        .filter(r => r.pramenTableName === request.tableName && r.infoDate === request.infoDate.toString && r.createdAt =!= request.createdAt.toEpochMilli)
+        .delete
+    ).execute()
+  }
+
+  override def rollbackOffsets(request: DataOffsetRequest): Unit = {
+    db.run(
+      OffsetRecords.records
+        .filter(r => r.pramenTableName === request.tableName && r.infoDate === request.infoDate.toString && r.createdAt === request.createdAt.toEpochMilli)
+        .delete
+    ).execute()
+  }
+
+  private[core] def getMaximumInfoDate(table: String): Option[LocalDate] = {
+    val query = OffsetRecords.records
+      .filter(r => r.pramenTableName === table)
+      .map(_.infoDate).max
+
+    try {
+      SlickUtils.executeMaxString(db, query)
+        .map(LocalDate.parse)
+    } catch {
+      case NonFatal(ex) => throw new RuntimeException(s"Unable to read from the offset table.", ex)
+    }
+  }
+
+  private[core] def getOffsetRecords(table: String, infoDate: LocalDate): Array[OffsetRecord] = {
+    val infoDateStr = infoDate.toString
+    val query = OffsetRecords.records
+      .filter(r => r.pramenTableName === table && r.infoDate === infoDateStr)
+
+    SlickUtils.executeQuery[OffsetRecords, OffsetRecord](db, query)
+      .toArray[OffsetRecord]
+  }
+
+  private[core] def getMinMaxOffsets(table: String, infoDate: LocalDate): Option[DataOffsetAggregated] = {
+    val offsets = getOffsetRecords(table, infoDate).filter(_.committedAtMilli.nonEmpty)
+
+    if (offsets.isEmpty) {
+      return None
+    }
+
+    validateOffsets(table, infoDate, offsets)
+
+    val offsetDataType =  offsets.head.dataType
+    val minOffset = offsets.flatMap(or => OffsetValue.fromString(offsetDataType, or.minOffset)).min
+    val maxOffset = offsets.flatMap(or => OffsetValue.fromString(offsetDataType, or.maxOffset)).max
+
+    Some(DataOffsetAggregated(table, infoDate, minOffset, maxOffset, offsets.map(OffsetRecordConverter.toDataOffset)))
+  }
+
+  /**
+    * Checks offsets for inconsistencies. They include:
+    * - inconsistent offset value types
+    *
+    * @param offsets An array of offset records
+    */
+  private[core] def validateOffsets(table: String, infoDate: LocalDate, offsets: Array[OffsetRecord]): Unit = {
+    val inconsistentOffsets = offsets.groupBy(_.dataType).keys.toArray.sorted
+    if (inconsistentOffsets.length > 1) {
+      throw new RuntimeException(s"Inconsistent offset value types found for $table at $infoDate: ${inconsistentOffsets.mkString(", ")}")
+    }
+  }
+
+}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetAggregated.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetAggregated.scala
new file mode 100644
index 000000000..81854057c
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetAggregated.scala
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper.model
+
+import za.co.absa.pramen.api.offset.{DataOffset, OffsetValue}
+
+import java.time.LocalDate
+
+case class DataOffsetAggregated(
+                                 tableName: String,
+                                 maximumInfoDate: LocalDate,
+                                 minimumOffset: OffsetValue,
+                                 maximumOffset: OffsetValue,
+                                 offsetsForTheDay: Array[DataOffset]
+                               )
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetRequest.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetRequest.scala
new file mode 100644
index 000000000..7cc6e7162
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetRequest.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper.model
+
+import java.time.{Instant, LocalDate}
+
+case class DataOffsetRequest(
+                              tableName: String,
+                              infoDate: LocalDate,
+                              batchId: Long,
+                              createdAt: Instant
+                            )
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecord.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecord.scala
new file mode 100644
index 000000000..68928d779
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecord.scala
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper.model
+
+case class OffsetRecord(
+                         pramenTableName: String,
+                         infoDate: String,              /* Use String to workaround serialization issues */
+                         dataType: String,              /* Converts to OffsetDataType */
+                         minOffset: String,
+                         maxOffset: String,
+                         batchId: Long,
+                         createdAtMilli: Long,          /* Use epoch milli */
+                         committedAtMilli: Option[Long] /* Use epoch milli, can be empty for uncommitted offsets */
+                       )
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecordConverter.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecordConverter.scala
new file mode 100644
index 000000000..d6ae4db23
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecordConverter.scala
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper.model
+
+import za.co.absa.pramen.api.offset.{DataOffset, OffsetType, OffsetValue}
+
+import java.time.{Instant, LocalDate}
+
+object OffsetRecordConverter {
+  def toDataOffset(r: OffsetRecord): DataOffset = {
+    val minOffsetOpt = OffsetValue.fromString(r.dataType, r.minOffset)
+    val maxOffsetOpt = OffsetValue.fromString(r.dataType, r.maxOffset)
+
+    if (r.committedAtMilli.nonEmpty && minOffsetOpt.isDefined && maxOffsetOpt.isDefined) {
+      DataOffset.CommittedOffset(
+        r.pramenTableName,
+        LocalDate.parse(r.infoDate),
+        r.batchId,
+        minOffsetOpt.get,
+        maxOffsetOpt.get,
+        Instant.ofEpochMilli(r.createdAtMilli),
+        Instant.ofEpochMilli(r.committedAtMilli.get)
+      )
+    } else {
+      val dataType = OffsetType.fromString(r.dataType)
+
+      DataOffset.UncommittedOffset(
+        r.pramenTableName,
+        LocalDate.parse(r.infoDate),
+        r.batchId,
+        dataType,
+        Instant.ofEpochMilli(r.createdAtMilli)
+      )
+    }
+  }
+}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecords.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecords.scala
new file mode 100644
index 000000000..4f7b56fdc
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetRecords.scala
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.bookkeeper.model
+
+import slick.jdbc.H2Profile.api._
+
+class OffsetRecords(tag: Tag) extends Table[OffsetRecord](tag, "offsets") {
+  def pramenTableName = column[String]("table_name", O.Length(128))
+  def infoDate = column[String]("info_date", O.Length(20))
+  def dataType = column[String]("data_type", O.Length(20))
+  def minOffset = column[String]("min_offset", O.Length(64))
+  def maxOffset = column[String]("max_offset", O.Length(64))
+  def batchId = column[Long]("batch_id")
+  def createdAt = column[Long]("created_at")
+  def committedAt = column[Option[Long]]("committed_at")
+  def * = (pramenTableName, infoDate, dataType, minOffset, maxOffset, batchId, createdAt, committedAt) <> (OffsetRecord.tupled, OffsetRecord.unapply)
+  def idx1 = index("offset_idx_1", (pramenTableName, infoDate, createdAt), unique = true)
+  def idx2 = index("offset_idx_2", (pramenTableName, committedAt), unique = false)
+}
+
+object OffsetRecords {
+  lazy val records = TableQuery[OffsetRecords]
+}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalHadoopCsv.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalHadoopCsv.scala
index f31509ca9..f0b4cc872 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalHadoopCsv.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalHadoopCsv.scala
@@ -77,6 +77,7 @@ class JournalHadoopCsv(journalPath: String)
         inputRecordCountOld = v.inputRecordCountOld,
         outputRecordCount = v.outputRecordCount,
         outputRecordCountOld = v.outputRecordCountOld,
+        appendedRecordCount = v.appendedRecordCount,
         outputSize = v.outputSize,
         startedAt = v.startedAt,
         finishedAt = v.finishedAt,
@@ -95,8 +96,11 @@ class JournalHadoopCsv(journalPath: String)
     val periodEnd = t.periodEnd.format(dateFormatter)
     val infoDate = t.informationDate.format(dateFormatter)
 
+    val inputRecordCount = t.inputRecordCount.map(_.toString).getOrElse("")
+    val inputRecordCountOld = t.inputRecordCountOld.map(_.toString).getOrElse("")
     val outputRecordCount = t.outputRecordCount.map(_.toString).getOrElse("")
     val outputRecordCountOld = t.outputRecordCountOld.map(_.toString).getOrElse("")
+    val appendedRecordCount = t.appendedRecordCount.map(_.toString).getOrElse("")
     val outputSize = t.outputSize.map(_.toString).getOrElse("")
 
     val record = removeSeparators(t.jobName) ::
@@ -104,10 +108,11 @@ class JournalHadoopCsv(journalPath: String)
       periodBegin ::
       periodEnd ::
       infoDate ::
-      t.inputRecordCount ::
-      t.inputRecordCountOld ::
+      inputRecordCount ::
+      inputRecordCountOld ::
       outputRecordCount ::
       outputRecordCountOld ::
+      appendedRecordCount ::
       outputSize ::
       t.startedAt ::
       t.finishedAt ::
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalJdbc.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalJdbc.scala
index 7346581be..37834d8f9 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalJdbc.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalJdbc.scala
@@ -41,10 +41,11 @@ class JournalJdbc(db: Database) extends Journal {
       periodBegin,
       periodEnd,
       infoDate,
-      entry.inputRecordCount,
-      entry.inputRecordCountOld,
+      entry.inputRecordCount.getOrElse(-1L),
+      entry.inputRecordCountOld.getOrElse(-1L),
       entry.outputRecordCount,
       entry.outputRecordCountOld,
+      entry.appendedRecordCount,
       entry.outputSize,
       entry.startedAt,
       entry.finishedAt,
@@ -72,16 +73,19 @@ class JournalJdbc(db: Database) extends Journal {
     val entries = SlickUtils.executeQuery(db, JournalTasks.journalTasks.filter(d => d.finishedAt >= fromSec && d.finishedAt <= toSec ))
 
     entries.map(v => {
+      val recordCountOpt = if (v.inputRecordCount < 0) None else Option(v.inputRecordCount)
+      val recordCountOldOpt = if (v.inputRecordCountOld < 0) None else Option(v.inputRecordCountOld)
       model.TaskCompleted(
         jobName = v.jobName,
         tableName = v.pramenTableName,
         periodBegin = LocalDate.parse(v.periodBegin, dateFormatter),
         periodEnd = LocalDate.parse(v.periodEnd, dateFormatter),
         informationDate = LocalDate.parse(v.informationDate, dateFormatter),
-        inputRecordCount = v.inputRecordCount,
-        inputRecordCountOld = v.inputRecordCountOld,
+        inputRecordCount = recordCountOpt,
+        inputRecordCountOld = recordCountOldOpt,
         outputRecordCount = v.outputRecordCount,
         outputRecordCountOld = v.outputRecordCountOld,
+        appendedRecordCount = v.appendedRecordCount,
         outputSize = v.outputSize,
         startedAt = v.startedAt,
         finishedAt = v.finishedAt,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTask.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTask.scala
index a0bb1d22d..ff06d22a8 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTask.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTask.scala
@@ -26,6 +26,7 @@ case class JournalTask(
                           inputRecordCountOld: Long,
                           outputRecordCount: Option[Long],
                           outputRecordCountOld: Option[Long],
+                          appendedRecordCount: Option[Long],
                           outputSize: Option[Long],
                           startedAt: Long,
                           finishedAt: Long,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTasks.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTasks.scala
index 0e76da77c..a04d37e5c 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTasks.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/JournalTasks.scala
@@ -29,6 +29,7 @@ class JournalTasks(tag: Tag) extends Table[JournalTask](tag, "journal") {
   def inputRecordCountOld = column[Long]("input_record_count_old")
   def outputRecordCount = column[Option[Long]]("output_record_count")
   def outputRecordCountOld = column[Option[Long]]("output_record_count_old")
+  def appendedRecordCount = column[Option[Long]]("appended_record_count")
   def outputSize = column[Option[Long]]("output_size")
   def startedAt = column[Long]("started_at")
   def finishedAt = column[Long]("finished_at")
@@ -41,7 +42,7 @@ class JournalTasks(tag: Tag) extends Table[JournalTask](tag, "journal") {
   def tenant = column[Option[String]]("tenant", O.Length(200))
   def * = (jobName, pramenTableName, periodBegin, periodEnd,
     informationDate, inputRecordCount, inputRecordCountOld, outputRecordCount,
-    outputRecordCountOld, outputSize, startedAt, finishedAt, status, failureReason, sparkApplicationId, pipelineId, pipelineName, environmentName, tenant) <> (JournalTask.tupled, JournalTask.unapply)
+    outputRecordCountOld, appendedRecordCount, outputSize, startedAt, finishedAt, status, failureReason, sparkApplicationId, pipelineId, pipelineName, environmentName, tenant) <> (JournalTask.tupled, JournalTask.unapply)
   def idx1 = index("idx_started_at", startedAt, unique = false)
   def idx2 = index("idx_finished_at", finishedAt, unique = false)
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompleted.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompleted.scala
index a39fec579..47857281e 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompleted.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompleted.scala
@@ -29,10 +29,11 @@ case class TaskCompleted(
                           periodBegin: LocalDate,
                           periodEnd: LocalDate,
                           informationDate: LocalDate,
-                          inputRecordCount: Long,
-                          inputRecordCountOld: Long,
+                          inputRecordCount: Option[Long],
+                          inputRecordCountOld: Option[Long],
                           outputRecordCount: Option[Long],
                           outputRecordCountOld: Option[Long],
+                          appendedRecordCount: Option[Long],
                           outputSize: Option[Long],
                           startedAt: Long,
                           finishedAt: Long,
@@ -58,9 +59,9 @@ object TaskCompleted {
     val failureReason = taskResult.runStatus.getReason
     val sparkApplicationId = Option(taskResult.applicationId)
 
-    val (recordCountOld, inputRecordCount, outputRecordCount, sizeBytes) = taskResult.runStatus match {
-      case s: Succeeded => (s.recordCountOld, s.recordCount, Some(s.recordCount), s.sizeBytes)
-      case _            => (None, 0L, None, None)
+    val (recordCountOld, inputRecordCount, outputRecordCount, sizeBytes, appendedRecords) = taskResult.runStatus match {
+      case s: Succeeded => (s.recordCountOld, s.recordCount, s.recordCount, s.sizeBytes, s.recordsAppended)
+      case _            => (None, None, None, None, None)
     }
 
     TaskCompleted(
@@ -70,9 +71,10 @@ object TaskCompleted {
       task.infoDate,
       task.infoDate,
       inputRecordCount,
-      recordCountOld.getOrElse(0L),
+      recordCountOld,
       outputRecordCount,
       recordCountOld,
+      appendedRecords,
       sizeBytes,
       taskStarted,
       taskFinished,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompletedCsv.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompletedCsv.scala
index 3aa1bd128..8ac82f9a2 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompletedCsv.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/journal/model/TaskCompletedCsv.scala
@@ -22,11 +22,12 @@ case class TaskCompletedCsv(
                              periodBegin: String,
                              periodEnd: String,
                              informationDate: String,
-                             inputRecordCount: Long,
-                             inputRecordCountOld: Long,
+                             inputRecordCount: Option[Long],
+                             inputRecordCountOld: Option[Long],
                              outputRecordCount: Option[Long],
                              outputRecordCountOld: Option[Long],
                              outputSize: Option[Long],
+                             appendedRecordCount: Option[Long],
                              startedAt: Long,
                              finishedAt: Long,
                              status: String,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetaTableStats.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetaTableStats.scala
index 4e0482296..a18f9359b 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetaTableStats.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetaTableStats.scala
@@ -17,6 +17,7 @@
 package za.co.absa.pramen.core.metastore
 
 case class MetaTableStats(
-                           recordCount: Long,
+                           recordCount: Option[Long],
+                           recordCountAppended: Option[Long],
                            dataSizeBytes: Option[Long]
                          )
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/Metastore.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/Metastore.scala
index f3c095ede..1209c882f 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/Metastore.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/Metastore.scala
@@ -16,9 +16,10 @@
 
 package za.co.absa.pramen.core.metastore
 
-import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.{DataFrame, SaveMode}
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.status.TaskRunReason
 import za.co.absa.pramen.core.metastore.model.MetaTable
 import za.co.absa.pramen.core.utils.hive.HiveHelper
 
@@ -37,9 +38,11 @@ trait Metastore {
 
   def getTable(tableName: String, infoDateFrom: Option[LocalDate], infoDateTo: Option[LocalDate]): DataFrame
 
+  def getBatch(tableName: String, infoDate: LocalDate, batchIdOpt: Option[Long]): DataFrame
+
   def getLatest(tableName: String, until: Option[LocalDate]): DataFrame
 
-  def saveTable(tableName: String, infoDate: LocalDate, df: DataFrame, inputRecordCount: Option[Long] = None): MetaTableStats
+  def saveTable(tableName: String, infoDate: LocalDate, df: DataFrame, inputRecordCount: Option[Long] = None, saveModeOverride: Option[SaveMode] = None): MetaTableStats
 
   def getHiveHelper(tableName: String): HiveHelper
 
@@ -51,5 +54,5 @@ trait Metastore {
 
   def getStats(tableName: String, infoDate: LocalDate): MetaTableStats
 
-  def getMetastoreReader(tables: Seq[String], infoDate: LocalDate): MetastoreReader
+  def getMetastoreReader(tables: Seq[String], infoDate: LocalDate, runReason: TaskRunReason, isIncremental: Boolean): MetastoreReader
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetastoreImpl.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetastoreImpl.scala
index 5b5651372..092e5dee6 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetastoreImpl.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetastoreImpl.scala
@@ -18,10 +18,13 @@ package za.co.absa.pramen.core.metastore
 
 import com.typesafe.config.Config
 import org.apache.hadoop.fs.Path
+import org.apache.spark.sql.functions.{col, lit}
 import org.apache.spark.sql.types.{DateType, StringType, StructField, StructType}
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.offset.DataOffset
+import za.co.absa.pramen.api.status.TaskRunReason
 import za.co.absa.pramen.core.app.config.InfoDateConfig
 import za.co.absa.pramen.core.app.config.InfoDateConfig.DEFAULT_DATE_FORMAT
 import za.co.absa.pramen.core.app.config.RuntimeConfig.UNDERCOVER
@@ -38,6 +41,7 @@ class MetastoreImpl(appConfig: Config,
                     tableDefs: Seq[MetaTable],
                     bookkeeper: Bookkeeper,
                     metadata: MetadataManager,
+                    batchId: Long,
                     skipBookKeepingUpdates: Boolean)(implicit spark: SparkSession) extends Metastore {
   import MetastoreImpl._
 
@@ -75,14 +79,27 @@ class MetastoreImpl(appConfig: Config,
   override def getTable(tableName: String, infoDateFrom: Option[LocalDate], infoDateTo: Option[LocalDate]): DataFrame = {
     val mt = getTableDef(tableName)
 
-    MetastorePersistence.fromMetaTable(mt, appConfig).loadTable(infoDateFrom, infoDateTo)
+    MetastorePersistence.fromMetaTable(mt, appConfig, batchId = batchId).loadTable(infoDateFrom, infoDateTo)
+  }
+
+  override def getBatch(tableName: String, infoDate: LocalDate, batchIdOpt: Option[Long]): DataFrame = {
+    val mt = getTableDef(tableName)
+    val effectiveBatchId = batchIdOpt.getOrElse(batchId)
+
+    val df = MetastorePersistence.fromMetaTable(mt, appConfig, batchId = effectiveBatchId).loadTable(Option(infoDate), Option(infoDate))
+
+    if (df.schema.fields.exists(_.name.equalsIgnoreCase(mt.batchIdColumn))) {
+      df.filter(col(mt.batchIdColumn) === lit(batchId))
+    } else {
+      df
+    }
   }
 
   override def getLatest(tableName: String, until: Option[LocalDate]): DataFrame = {
     val mt = getTableDef(tableName)
     val isLazy = mt.format.isLazy
     if (isLazy) {
-      MetastorePersistence.fromMetaTable(mt, appConfig).loadTable(None, until)
+      MetastorePersistence.fromMetaTable(mt, appConfig, batchId = batchId).loadTable(None, until)
     } else {
       bookkeeper.getLatestProcessedDate(tableName, until) match {
         case Some(infoDate) => getTable(tableName, Some(infoDate), Some(infoDate))
@@ -91,21 +108,25 @@ class MetastoreImpl(appConfig: Config,
     }
   }
 
-  override def saveTable(tableName: String, infoDate: LocalDate, df: DataFrame, inputRecordCount: Option[Long]): MetaTableStats = {
+  override def saveTable(tableName: String, infoDate: LocalDate, df: DataFrame, inputRecordCount: Option[Long], saveModeOverride: Option[SaveMode]): MetaTableStats = {
     val mt = getTableDef(tableName)
     val isTransient = mt.format.isTransient
     val start = Instant.now.getEpochSecond
 
-    var stats = MetaTableStats(0, None)
+    var stats = MetaTableStats(Some(0), None, None)
 
     withSparkConfig(mt.sparkConfig) {
-      stats = MetastorePersistence.fromMetaTable(mt, appConfig).saveTable(infoDate, df, inputRecordCount)
+      stats = MetastorePersistence.fromMetaTable(mt, appConfig, saveModeOverride, batchId).saveTable(infoDate, df, inputRecordCount)
     }
 
     val finish = Instant.now.getEpochSecond
 
-    if (!skipBookKeepingUpdates) {
-      bookkeeper.setRecordCount(tableName, infoDate, infoDate, infoDate, inputRecordCount.getOrElse(stats.recordCount), stats.recordCount, start, finish, isTransient)
+    val nothingAppended = stats.recordCountAppended.contains(0)
+
+    stats.recordCount.foreach{recordCount =>
+      if (!skipBookKeepingUpdates && !nothingAppended) {
+        bookkeeper.setRecordCount(tableName, infoDate, infoDate, infoDate, inputRecordCount.getOrElse(recordCount), recordCount, start, finish, isTransient)
+      }
     }
 
     stats
@@ -177,10 +198,10 @@ class MetastoreImpl(appConfig: Config,
   override def getStats(tableName: String, infoDate: LocalDate): MetaTableStats = {
     val mt = getTableDef(tableName)
 
-    MetastorePersistence.fromMetaTable(mt, appConfig).getStats(infoDate)
+    MetastorePersistence.fromMetaTable(mt, appConfig, batchId = batchId).getStats(infoDate, onlyForCurrentBatchId = false)
   }
 
-  override def getMetastoreReader(tables: Seq[String], infoDate: LocalDate): MetastoreReader = {
+  override def getMetastoreReader(tables: Seq[String], infoDate: LocalDate, runReason: TaskRunReason, isIncremental: Boolean): MetastoreReader = {
     val metastore = this
 
     new MetastoreReader {
@@ -191,6 +212,14 @@ class MetastoreImpl(appConfig: Config,
         metastore.getTable(tableName, from, to)
       }
 
+      override def getCurrentBatch(tableName: String): DataFrame = {
+        validateTable(tableName)
+        if (isIncremental)
+          metastore.getBatch(tableName, infoDate, None)
+        else
+          metastore.getTable(tableName, Option(infoDate), Option(infoDate))
+      }
+
       override def getLatest(tableName: String, until: Option[LocalDate] = None): DataFrame = {
         validateTable(tableName)
         val untilDate = until.orElse(Option(infoDate))
@@ -210,8 +239,14 @@ class MetastoreImpl(appConfig: Config,
         metastore.isDataAvailable(tableName, fromDate, untilDate)
       }
 
+      override def getOffsets(table: String, infoDate: LocalDate): Array[DataOffset] = {
+        val om = bookkeeper.getOffsetManager
+
+        om.getOffsets(table, infoDate)
+      }
+
       override def getTableDef(tableName: String): MetaTableDef = {
-        validateTable(tableName)
+        validateTable(tableName) // ToDo Consider removing
 
         MetaTable.getMetaTableDef(metastore.getTableDef(tableName))
       }
@@ -223,6 +258,8 @@ class MetastoreImpl(appConfig: Config,
           )
       }
 
+      override def getRunReason: TaskRunReason = runReason
+
       override def metadataManager: MetadataManager = metadata
 
       private def validateTable(tableName: String): Unit = {
@@ -253,12 +290,13 @@ object MetastoreImpl {
   def fromConfig(conf: Config,
                  infoDateConfig: InfoDateConfig,
                  bookkeeper: Bookkeeper,
-                 metadataManager: MetadataManager)(implicit spark: SparkSession): MetastoreImpl = {
+                 metadataManager: MetadataManager,
+                 batchId: Long)(implicit spark: SparkSession): MetastoreImpl = {
     val tableDefs = MetaTable.fromConfig(conf, infoDateConfig, METASTORE_KEY)
 
     val isUndercover = ConfigUtils.getOptionBoolean(conf, UNDERCOVER).getOrElse(false)
 
-    new MetastoreImpl(conf, tableDefs, bookkeeper, metadataManager, isUndercover)
+    new MetastoreImpl(conf, tableDefs, bookkeeper, metadataManager, batchId, isUndercover)
   }
 
   private[core] def withSparkConfig(sparkConfig: Map[String, String])
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/model/MetaTable.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/model/MetaTable.scala
index da4f9b8f6..d8bcddc7a 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/model/MetaTable.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/model/MetaTable.scala
@@ -35,6 +35,7 @@ import scala.util.{Failure, Success, Try}
   * @param format                 The format of the table.
   * @param infoDateColumn         The name of the column that contains the information date (partitioned by).
   * @param infoDateFormat         The format of the information date.
+  * @param batchIdColumn          The name of the column that contains the batch id.
   * @param hiveConfig             The effective Hive configuration to use for Hive operations.
   * @param hiveTable              The name of the Hive table.
   * @param hivePath               The path of the Hive table (if it differs from the path in the underlying format).
@@ -53,6 +54,7 @@ case class MetaTable(
                       format: DataFormat,
                       infoDateColumn: String,
                       infoDateFormat: String,
+                      batchIdColumn: String,
                       hiveConfig: HiveConfig,
                       hiveTable: Option[String],
                       hivePath: Option[String],
@@ -82,6 +84,9 @@ object MetaTable {
   val TABLE_HIVE_CONFIG_PREFIX = "hive"
   val DEFAULT_HIVE_CONFIG_PREFIX = "pramen.hive"
   val SPARK_CONFIG_PREFIX = "spark.conf"
+  val BATCH_ID_COLUMN_KEY = "batchid.column"
+  val DEFAULT_BATCH_ID_COLUMN_KEY = "pramen.batchid.column.default"
+  val DEFAULT_BATCH_ID_COLUMN_NAME = "pramen_batchid"
 
   def fromConfig(conf: Config, infoDateConfig: InfoDateConfig, key: String): Seq[MetaTable] = {
     val defaultInfoDateColumnName = infoDateConfig.columnName
@@ -90,6 +95,7 @@ object MetaTable {
     val defaultTrackDays = infoDateConfig.defaultTrackDays
     val defaultHiveConfig = HiveDefaultConfig.fromConfig(ConfigUtils.getOptionConfig(conf, DEFAULT_HIVE_CONFIG_PREFIX))
     val defaultPreferAddPartition = conf.getBoolean(s"pramen.$HIVE_PREFER_ADD_PARTITION_KEY")
+    val defaultBatchIdColumnName = ConfigUtils.getOptionString(conf, DEFAULT_BATCH_ID_COLUMN_KEY).getOrElse(DEFAULT_BATCH_ID_COLUMN_NAME)
 
     val tableConfigs = ConfigUtils.getOptionConfigList(conf, key)
 
@@ -98,7 +104,7 @@ object MetaTable {
     }
 
     val metatables = tableConfigs
-      .map(tableConfig => fromConfigSingleEntity(tableConfig, conf, defaultInfoDateColumnName, defaultInfoDateFormat, defaultStartDate, defaultTrackDays, defaultHiveConfig, defaultPreferAddPartition))
+      .map(tableConfig => fromConfigSingleEntity(tableConfig, conf, defaultInfoDateColumnName, defaultInfoDateFormat, defaultStartDate, defaultTrackDays, defaultHiveConfig, defaultPreferAddPartition, defaultBatchIdColumnName))
       .toSeq
 
     val duplicates = AlgorithmUtils.findDuplicates(metatables.map(_.name))
@@ -115,7 +121,8 @@ object MetaTable {
                              defaultStartDate: LocalDate,
                              defaultTrackDays: Int,
                              defaultHiveConfig: HiveDefaultConfig,
-                             defaultPreferAddPartition: Boolean): MetaTable = {
+                             defaultPreferAddPartition: Boolean,
+                             defaultBatchIdColumn: String): MetaTable = {
     val name = ConfigUtils.getOptionString(conf, NAME_KEY).getOrElse(throw new IllegalArgumentException(s"Mandatory option missing: $NAME_KEY"))
     val description = ConfigUtils.getOptionString(conf, NAME_DESCRIPTION).getOrElse("")
     val infoDateOverride = InfoDateOverride.fromConfig(conf)
@@ -125,6 +132,7 @@ object MetaTable {
     val startDate = infoDateOverride.startDate.getOrElse(defaultStartDate)
     val trackDays = ConfigUtils.getOptionInt(conf, TRACK_DAYS_KEY).getOrElse(defaultTrackDays)
     val trackDaysExplicitlySet = conf.hasPath(TRACK_DAYS_KEY)
+    val batchIdColumn = ConfigUtils.getOptionString(conf, BATCH_ID_COLUMN_KEY).getOrElse(defaultBatchIdColumn)
 
     val format = Try {
       DataFormatParser.fromConfig(conf, appConf)
@@ -153,6 +161,7 @@ object MetaTable {
       format,
       infoDateColumn,
       infoDateFormat,
+      batchIdColumn,
       hiveConfig,
       hiveTable,
       hivePath,
@@ -174,6 +183,7 @@ object MetaTable {
       table.format,
       table.infoDateColumn,
       table.infoDateFormat,
+      table.batchIdColumn,
       table.hiveTable,
       table.hivePath,
       table.infoDateStart,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistence.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistence.scala
index 8a858ad08..2feb18b66 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistence.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistence.scala
@@ -17,7 +17,7 @@
 package za.co.absa.pramen.core.metastore.peristence
 
 import com.typesafe.config.Config
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
 import za.co.absa.pramen.api.DataFormat
 import za.co.absa.pramen.core.metastore.MetaTableStats
 import za.co.absa.pramen.core.metastore.model.{HiveConfig, MetaTable}
@@ -30,7 +30,7 @@ trait MetastorePersistence {
 
   def saveTable(infoDate: LocalDate, df: DataFrame, numberOfRecordsEstimate: Option[Long]): MetaTableStats
 
-  def getStats(infoDate: LocalDate): MetaTableStats
+  def getStats(infoDate: LocalDate, onlyForCurrentBatchId: Boolean): MetaTableStats
 
   def createOrUpdateHiveTable(infoDate: LocalDate,
                               hiveTableName: String,
@@ -43,18 +43,20 @@ trait MetastorePersistence {
 }
 
 object MetastorePersistence {
-  def fromMetaTable(metaTable: MetaTable, conf: Config)(implicit spark: SparkSession): MetastorePersistence = {
+  def fromMetaTable(metaTable: MetaTable, conf: Config, saveModeOverride: Option[SaveMode] = None, batchId: Long)(implicit spark: SparkSession): MetastorePersistence = {
+    val saveModeOpt = saveModeOverride.orElse(metaTable.saveModeOpt)
+
     metaTable.format match {
       case DataFormat.Parquet(path, recordsPerPartition) =>
         new MetastorePersistenceParquet(
-          path, metaTable.infoDateColumn, metaTable.infoDateFormat, recordsPerPartition, metaTable.saveModeOpt, metaTable.readOptions, metaTable.writeOptions
+          path, metaTable.infoDateColumn, metaTable.infoDateFormat, metaTable.batchIdColumn, batchId, recordsPerPartition, saveModeOpt, metaTable.readOptions, metaTable.writeOptions
         )
       case DataFormat.Delta(query, recordsPerPartition)  =>
         new MetastorePersistenceDelta(
-          query, metaTable.infoDateColumn, metaTable.infoDateFormat, recordsPerPartition, metaTable.saveModeOpt, metaTable.readOptions, metaTable.writeOptions
+          query, metaTable.infoDateColumn, metaTable.infoDateFormat, metaTable.batchIdColumn, batchId, recordsPerPartition, saveModeOpt, metaTable.readOptions, metaTable.writeOptions
         )
       case DataFormat.Raw(path)                          =>
-        new MetastorePersistenceRaw(path, metaTable.infoDateColumn, metaTable.infoDateFormat, metaTable.saveModeOpt)
+        new MetastorePersistenceRaw(path, metaTable.infoDateColumn, metaTable.infoDateFormat, saveModeOpt)
       case DataFormat.TransientEager(cachePolicy)             =>
         new MetastorePersistenceTransientEager(TransientTableManager.getTempDirectory(cachePolicy, conf), metaTable.name, cachePolicy)
       case DataFormat.Transient(cachePolicy) =>
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceDelta.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceDelta.scala
index 9fdfce062..273b18b9c 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceDelta.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceDelta.scala
@@ -35,6 +35,8 @@ import scala.util.Try
 class MetastorePersistenceDelta(query: Query,
                                 infoDateColumn: String,
                                 infoDateFormat: String,
+                                batchIdColumn: String,
+                                batchId: Long,
                                 recordsPerPartition: Option[Long],
                                 saveModeOpt: Option[SaveMode],
                                 readOptions: Map[String, String],
@@ -66,18 +68,22 @@ class MetastorePersistenceDelta(query: Query,
 
     val whereCondition = s"$infoDateColumn='$infoDateStr'"
 
-    val recordCount = numberOfRecordsEstimate match {
-      case Some(count) => count
-      case None => dfIn.count()
-    }
+    val dfRepartitioned = if (recordsPerPartition.nonEmpty) {
+      val recordCount = numberOfRecordsEstimate match {
+        case Some(count) => count
+        case None => dfIn.count()
+      }
 
-    val dfRepartitioned = applyRepartitioning(dfIn, recordCount)
+      applyRepartitioning(dfIn, recordCount)
+    } else {
+      dfIn
+    }
 
     val saveMode = saveModeOpt.getOrElse(SaveMode.Overwrite)
 
-    val operationStr = saveMode match {
-      case SaveMode.Append => "Appending to"
-      case _               => "Writing to"
+    val (isAppend, operationStr) = saveMode match {
+      case SaveMode.Append => (true, "Appending to")
+      case _               => (false, "Writing to")
     }
 
     if (log.isDebugEnabled) {
@@ -108,19 +114,32 @@ class MetastorePersistenceDelta(query: Query,
         throw new IllegalStateException(s"The '${q.name}' option is not supported as a write target for Delta.")
     }
 
-    val stats = getStats(infoDate)
+    val stats = getStats(infoDate, isAppend)
 
     stats.dataSizeBytes match {
-      case Some(size) => log.info(s"$SUCCESS Successfully saved ${stats.recordCount} records (${StringUtils.prettySize(size)}) to ${query.query}")
-      case None => log.info(s"$SUCCESS Successfully saved ${stats.recordCount} records to ${query.query}")
+      case Some(size) =>
+        stats.recordCountAppended match {
+          case Some(recordsAppended) =>
+            log.info(s"$SUCCESS Successfully saved $recordsAppended records (new count: ${stats.recordCount.get}, " +
+              s"new size: ${StringUtils.prettySize(size)}) to ${query.query}")
+          case None =>
+            log.info(s"$SUCCESS Successfully saved ${stats.recordCount.get} records " +
+              s"(${StringUtils.prettySize(size)}) to ${query.query}")
+        }
+      case None =>
+        stats.recordCountAppended match {
+          case Some(recordsAppended) =>
+            log.info(s"$SUCCESS Successfully saved $recordsAppended records (new count: ${stats.recordCount.get} to ${query.query}")
+          case None =>
+            log.info(s"$SUCCESS Successfully saved ${stats.recordCount.get} records  to ${query.query}")
+        }
     }
 
     stats
   }
 
-  override def getStats(infoDate: LocalDate): MetaTableStats = {
+  override def getStats(infoDate: LocalDate, onlyForCurrentBatchId: Boolean): MetaTableStats = {
     val df = loadTable(Option(infoDate), Option(infoDate))
-    val recordCount = df.count()
 
     val sizeOpt = query match {
       case Query.Path(path)   =>
@@ -137,7 +156,16 @@ class MetastorePersistenceDelta(query: Query,
         None
     }
 
-    MetaTableStats(recordCount, sizeOpt)
+    if (onlyForCurrentBatchId && df.schema.exists(_.name.equalsIgnoreCase(batchIdColumn))) {
+      val batchCount = df.filter(col(batchIdColumn) === batchId).count()
+      val countAll = df.count()
+
+      MetaTableStats(Option(countAll), Option(batchCount), sizeOpt)
+    } else {
+      val countAll = df.count()
+
+      MetaTableStats(Option(countAll), None, sizeOpt)
+    }
   }
 
   override def createOrUpdateHiveTable(infoDate: LocalDate,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceParquet.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceParquet.scala
index ac082b6b0..e5e5fd40f 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceParquet.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceParquet.scala
@@ -36,6 +36,8 @@ import scala.util.control.NonFatal
 class MetastorePersistenceParquet(path: String,
                                   infoDateColumn: String,
                                   infoDateFormat: String,
+                                  batchIdColumn: String,
+                                  batchId: Long,
                                   recordsPerPartition: Option[Long],
                                   saveModeOpt: Option[SaveMode],
                                   readOptions: Map[String, String],
@@ -64,45 +66,69 @@ class MetastorePersistenceParquet(path: String,
 
     fsUtils.createDirectoryRecursive(new Path(path))
 
+    val saveMode = saveModeOpt.getOrElse(SaveMode.Overwrite)
+
+    val isAppend = saveMode match {
+      case SaveMode.Append =>
+        log.info(s"Appending to '$outputDirStr'...")
+        true
+      case _               =>
+        log.info(s"Writing to '$outputDirStr'...")
+        false
+    }
+
     val dfIn = if (df.schema.exists(_.name.equalsIgnoreCase(infoDateColumn))) {
       df.drop(infoDateColumn)
     } else {
       df
     }
 
-    val saveMode = saveModeOpt.getOrElse(SaveMode.Overwrite)
-
-    saveMode match {
-      case SaveMode.Append => log.info(s"Appending to '$outputDirStr'...")
-      case _               => log.info(s"Writing to '$outputDirStr'...")
-    }
+    val dfRepartitioned = if (recordsPerPartition.nonEmpty) {
+      val recordCount = numberOfRecordsEstimate match {
+        case Some(count) => count
+        case None => dfIn.count()
+      }
 
-    val recordCount = numberOfRecordsEstimate match {
-      case Some(count) => count
-      case None => dfIn.count()
+      applyRepartitioning(dfIn, recordCount)
+    } else {
+      dfIn
     }
 
-    val dfRepartitioned = applyRepartitioning(dfIn, recordCount)
-
     writeAndCleanOnFailure(dfRepartitioned, outputDirStr, fsUtils, saveMode)
 
-    val stats = getStats(infoDate)
+    val stats = getStats(infoDate, isAppend)
 
-    log.info(s"$SUCCESS Successfully saved ${stats.recordCount} records (${StringUtils.prettySize(stats.dataSizeBytes.get)}) to $outputDir")
+    stats.recordCountAppended match {
+      case Some(recordsAppended) =>
+        log.info(s"$SUCCESS Successfully saved $recordsAppended records (new count: ${stats.recordCount.get}, " +
+          s"new size: ${StringUtils.prettySize(stats.dataSizeBytes.get)}) to $outputDir")
+      case None =>
+        log.info(s"$SUCCESS Successfully saved ${stats.recordCount.get} records " +
+          s"(${StringUtils.prettySize(stats.dataSizeBytes.get)}) to $outputDir")
+    }
 
     stats
   }
 
-  override def getStats(infoDate: LocalDate): MetaTableStats = {
+  override def getStats(infoDate: LocalDate, onlyForCurrentBatchId: Boolean): MetaTableStats = {
     val outputDirStr = SparkUtils.getPartitionPath(infoDate, infoDateColumn, infoDateFormat, path).toUri.toString
 
     val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, path)
 
-    val actualCount = spark.read.parquet(outputDirStr).count()
+    val df = spark.read.parquet(outputDirStr)
 
     val size = fsUtils.getDirectorySize(outputDirStr)
 
-    MetaTableStats(actualCount, Option(size))
+    if (onlyForCurrentBatchId && df.schema.exists(_.name.equalsIgnoreCase(batchIdColumn))) {
+      val batchCount = df.filter(col(batchIdColumn) === batchId).count()
+      val countAll = df.count()
+
+      MetaTableStats(Option(countAll), Option(batchCount), Option(size))
+    } else {
+      val countAll = df.count()
+
+      MetaTableStats(Option(countAll), None, Option(size))
+    }
   }
 
   override def createOrUpdateHiveTable(infoDate: LocalDate,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceRaw.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceRaw.scala
index 9266b8bef..ed95b41a8 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceRaw.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceRaw.scala
@@ -88,12 +88,13 @@ class MetastorePersistenceRaw(path: String,
     }
 
     MetaTableStats(
-      totalSize,
+      Option(totalSize),
+      None,
       Some(totalSize)
     )
   }
 
-  override def getStats(infoDate: LocalDate): MetaTableStats = {
+  override def getStats(infoDate: LocalDate, onlyForCurrentBatchId: Boolean): MetaTableStats = {
     val partitionDir = SparkUtils.getPartitionPath(infoDate, infoDateColumn, infoDateFormat, path)
 
     val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, path)
@@ -107,7 +108,8 @@ class MetastorePersistenceRaw(path: String,
     })
 
     MetaTableStats(
-      files.length,
+      Option(files.length),
+      None,
       Some(totalSize)
     )
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransient.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransient.scala
index a57dc5024..35c1f1adc 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransient.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransient.scala
@@ -49,7 +49,7 @@ class MetastorePersistenceTransient(tempPath: Option[String],
     transientPersistence.saveTable(infoDate, df, numberOfRecordsEstimate)
   }
 
-  override def getStats(infoDate: LocalDate): MetaTableStats = {
+  override def getStats(infoDate: LocalDate, onlyForCurrentBatchId: Boolean): MetaTableStats = {
     throw new UnsupportedOperationException("The 'transient' format does not support getting record count and size statistics.")
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransientEager.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransientEager.scala
index 1f7c8710f..961b1133d 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransientEager.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePersistenceTransientEager.scala
@@ -54,17 +54,23 @@ class MetastorePersistenceTransientEager(tempPathOpt: Option[String],
     }
 
     val recordCount = numberOfRecordsEstimate match {
-      case Some(n) => n
-      case None => dfOut.count()
+      case Some(n) => Option(n)
+      case None =>
+        cachePolicy match {
+          case CachePolicy.Cache => Option(dfOut.count())
+          case CachePolicy.Persist => Option(dfOut.count())
+          case _ => None
+        }
     }
 
     MetaTableStats(
       recordCount,
+      None,
       sizeBytesOpt
     )
   }
 
-  override def getStats(infoDate: LocalDate): MetaTableStats = {
+  override def getStats(infoDate: LocalDate, onlyForCurrentBatchId: Boolean): MetaTableStats = {
     throw new UnsupportedOperationException("Transient format does not support getting record count and size statistics.")
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineNotificationBuilderHtml.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineNotificationBuilderHtml.scala
index d5bc35ab4..3e7f6bef9 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineNotificationBuilderHtml.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineNotificationBuilderHtml.scala
@@ -25,7 +25,6 @@ import za.co.absa.pramen.api.{FieldChange, SchemaDifference}
 import za.co.absa.pramen.core.config.Keys.TIMEZONE
 import za.co.absa.pramen.core.exceptions.{CmdFailedException, ProcessFailedException}
 import za.co.absa.pramen.core.notify.message._
-import za.co.absa.pramen.core.notify.pipeline.PipelineNotificationBuilderHtml.{MIN_MEGABYTES, MIN_RPS_JOB_DURATION_SECONDS, MIN_RPS_RECORDS}
 import za.co.absa.pramen.core.utils.JvmUtils.getShortExceptionDescription
 import za.co.absa.pramen.core.utils.StringUtils.renderThrowable
 import za.co.absa.pramen.core.utils.{BuildPropertyUtils, ConfigUtils, StringUtils, TimeUtils}
@@ -40,6 +39,7 @@ object PipelineNotificationBuilderHtml {
   val MIN_MEGABYTES = 10
   val NOTIFICATION_REASON_MAX_LENGTH_KEY = "pramen.notifications.reason.max.length"
   val NOTIFICATION_EXCEPTION_MAX_LENGTH_KEY = "pramen.notifications.exception.max.length"
+  val SUPPRESS_WARNING_STARTING_WITH = "Based on outdated tables: "
 }
 
 class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNotificationBuilder {
@@ -266,9 +266,17 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
 
   private[core] def hasWarnings: Boolean = {
     completedTasks.exists{task =>
+      val hasTaskWarnings = task.runStatus match {
+        case succeeded: Succeeded =>
+          val warnings = succeeded.warnings
+            .filterNot(_.startsWith(SUPPRESS_WARNING_STARTING_WITH))
+          warnings.nonEmpty
+        case _ =>
+          false
+      }
+
       val hasDependencyWarnings = task.dependencyWarnings.nonEmpty
       val hasNotificationErrors = task.notificationTargetErrors.nonEmpty
-      val hasTaskWarnings = task.runStatus.isInstanceOf[RunStatus.Succeeded] && task.runStatus.asInstanceOf[RunStatus.Succeeded].warnings.nonEmpty
       val hasSkippedWithWarnings = task.runStatus.isInstanceOf[RunStatus.Skipped] && task.runStatus.asInstanceOf[RunStatus.Skipped].isWarning
       val hasSchemaChanges = task.schemaChanges.nonEmpty
       val hasPipelineNotificationFailures = pipelineNotificationFailures.nonEmpty
@@ -499,10 +507,14 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
     builder.withTable(tableBuilder)
   }
 
-  private[core] def getThroughputRps(task: TaskResult): TextElement = {
+  def getThroughputRps(task: TaskResult): TextElement = {
     val recordCount = task.runStatus match {
-      case s: Succeeded => s.recordCount
-      case _            => 0
+      case s: Succeeded =>
+        s.recordsAppended match {
+          case Some(appended) => appended
+          case None => s.recordCount.getOrElse(0L)
+        }
+      case _ => 0L
     }
 
     task.runInfo match {
@@ -544,8 +556,8 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
     }
   }
 
-  private[core] def getRecordCountText(task: TaskResult): String = {
-    def renderDifference(numRecords: Long, numRecordsOld: Option[Long]): String = {
+  def getRecordCountText(task: TaskResult): String = {
+    def renderDifference(numRecords: Long, numRecordsOld: Option[Long], numRecordsAppended: Option[Long]): String = {
       numRecordsOld match {
         case Some(old) if old > 0 =>
           val diff = numRecords - old
@@ -556,7 +568,11 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
           else {
             numRecords.toString
           }
-        case _ => numRecords.toString
+        case _ =>
+          numRecordsAppended match {
+            case Some(n) => s"$numRecords (+$n)"
+            case None => numRecords.toString
+          }
       }
     }
 
@@ -564,14 +580,18 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
       "-"
     } else {
       task.runStatus match {
-        case s: Succeeded        => renderDifference(s.recordCount, s.recordCountOld)
-        case d: InsufficientData => renderDifference(d.actual, d.recordCountOld)
-        case _                   => ""
+        case s: Succeeded        =>
+          s.recordCount match {
+            case Some(recordCount) => renderDifference(recordCount, s.recordCountOld, s.recordsAppended)
+            case None => "-"
+          }
+        case d: InsufficientData => renderDifference(d.actual, d.recordCountOld, None)
+        case _                   => "-"
       }
     }
   }
 
-  private[core] def getSizeText(task: TaskResult): String = {
+  def getSizeText(task: TaskResult): String = {
     def renderDifferenceSize(numBytes: Long, numBytesOld: Option[Long]): String = {
       numBytesOld match {
         case Some(old) if old > 0 =>
@@ -588,7 +608,11 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
     }
 
     task.runStatus match {
-      case s: Succeeded        => renderDifferenceSize(s.recordCount, s.recordCountOld)
+      case s: Succeeded        =>
+        s.recordCount match {
+          case Some(recordCount) => renderDifferenceSize(recordCount, s.recordCountOld)
+          case None => ""
+        }
       case d: InsufficientData => renderDifferenceSize(d.actual, d.recordCountOld)
       case _                   => ""
     }
@@ -673,8 +697,12 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
     } else {
       if (status.warnings.nonEmpty)
         TextElement("Warning", style)
-      else
-        TextElement("Success", style)
+      else {
+        if (status.recordsAppended.contains(0))
+          TextElement("No new data", style)
+        else
+          TextElement("Success", style)
+      }
     }
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IncrementalIngestionJob.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IncrementalIngestionJob.scala
new file mode 100644
index 000000000..a6a6e3afb
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IncrementalIngestionJob.scala
@@ -0,0 +1,327 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.pipeline
+
+import com.typesafe.config.Config
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SparkSession}
+import za.co.absa.pramen.api.offset.DataOffset.UncommittedOffset
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetType, OffsetValue}
+import za.co.absa.pramen.api.sql.SqlGeneratorBase
+import za.co.absa.pramen.api.status.{DependencyWarning, TaskRunReason}
+import za.co.absa.pramen.api.{Reason, Source}
+import za.co.absa.pramen.core.bookkeeper.model.{DataOffsetAggregated, DataOffsetRequest}
+import za.co.absa.pramen.core.bookkeeper.{Bookkeeper, OffsetManager}
+import za.co.absa.pramen.core.metastore.Metastore
+import za.co.absa.pramen.core.metastore.model.MetaTable
+import za.co.absa.pramen.core.runner.splitter.{ScheduleStrategy, ScheduleStrategyIncremental}
+import za.co.absa.pramen.core.utils.SparkUtils._
+
+import java.time.{Instant, LocalDate}
+
+class IncrementalIngestionJob(operationDef: OperationDef,
+                              metastore: Metastore,
+                              bookkeeper: Bookkeeper,
+                              notificationTargets: Seq[JobNotificationTarget],
+                              latestOffsetIn: Option[DataOffsetAggregated],
+                              batchId: Long,
+                              sourceName: String,
+                              source: Source,
+                              sourceTable: SourceTable,
+                              outputTable: MetaTable,
+                              specialCharacters: String)
+                             (implicit spark: SparkSession)
+  extends IngestionJob(operationDef, metastore, bookkeeper, notificationTargets, sourceName, source, sourceTable, outputTable, specialCharacters, None, false) {
+
+  override val scheduleStrategy: ScheduleStrategy = new ScheduleStrategyIncremental(latestOffsetIn.map(_.maximumInfoDate), source.hasInfoDateColumn(sourceTable.query))
+
+  override def trackDays: Int = 0
+
+  override def preRunCheckJob(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config, dependencyWarnings: Seq[DependencyWarning]): JobPreRunResult = {
+    JobPreRunResult(JobPreRunStatus.Ready, None, dependencyWarnings, Nil)
+  }
+
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config): Reason = {
+    val om = bookkeeper.getOffsetManager
+    val sourceHasInfoDate = source.hasInfoDateColumn(sourceTable.query)
+    val isReRun = runReason == TaskRunReason.Rerun
+
+    val onlyForInfoDate = if (sourceHasInfoDate)
+      Some(infoDate)
+    else
+      None
+
+    if (source.getOffsetInfo.isEmpty) {
+      return Reason.NotReady(s"Offset column is not configured for source '$sourceName' of '${operationDef.name}'")
+    }
+
+    validateUncommittedOffsets(onlyForInfoDate, om)
+
+    validateReadiness(infoDate, om, sourceHasInfoDate, isReRun)
+  }
+
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
+    val columns = if (sourceTable.transformations.isEmpty && sourceTable.filters.isEmpty) {
+      sourceTable.columns
+    } else {
+      Seq.empty[String]
+    }
+
+    val om = bookkeeper.getOffsetManager
+    val sourceHasInfoDate = source.hasInfoDateColumn(sourceTable.query)
+    val isReRun = runReason == TaskRunReason.Rerun
+
+    val sourceResult = (sourceHasInfoDate, isReRun) match {
+      case (false, false) =>
+        om.getMaxInfoDateAndOffset(outputTable.name, None) match {
+          case Some(maxOffset) =>
+            source.getDataIncremental(sourceTable.query, None, Option(maxOffset.maximumOffset), None, columns)
+          case None =>
+            source.getData(sourceTable.query, infoDate, infoDate, columns)
+        }
+      case (false, true) =>
+        om.getMaxInfoDateAndOffset(outputTable.name, Option(infoDate)) match {
+          case Some(offsets) =>
+            source.getDataIncremental(sourceTable.query, None, Option(offsets.minimumOffset), Option(offsets.maximumOffset), columns)
+          case None =>
+            throw new IllegalStateException(s"No offsets for '${outputTable.name}' for '$infoDate'. Cannot rerun.")
+        }
+      case (true, false) =>
+        val om = bookkeeper.getOffsetManager
+        val infoDateLatestOffset = om.getMaxInfoDateAndOffset(outputTable.name, Some(infoDate))
+        infoDateLatestOffset match {
+          case Some(maxOffset) =>
+            log.info(s"Running ingestion to '${outputTable.name}' at '$infoDate' for offset > ${maxOffset.maximumOffset.valueString}.")
+            source.getDataIncremental(sourceTable.query, Option(infoDate), Option(maxOffset.maximumOffset), None, columns)
+          case None =>
+            log.info(s"Running ingestion to '${outputTable.name}' at '$infoDate' for all data available at the day.")
+            source.getData(sourceTable.query, infoDate, infoDate, columns)
+        }
+      case (true, true) =>
+        source.getData(sourceTable.query, infoDate, infoDate, columns)
+    }
+
+    val sanitizedDf = sanitizeDfColumns(sourceResult.data, specialCharacters)
+
+    val result = sourceResult.copy(data = sanitizedDf)
+
+    RunResult(result.data, result.filesRead, result.warnings)
+  }
+
+  override def save(df: DataFrame,
+                    infoDate: LocalDate,
+                    runReason: TaskRunReason,
+                    conf: Config,
+                    jobStarted: Instant,
+                    inputRecordCount: Option[Long]): SaveResult = {
+    val isRerun = runReason == TaskRunReason.Rerun
+
+    val dfToSave = df.withColumn(outputTable.batchIdColumn, lit(batchId))
+
+    val om = bookkeeper.getOffsetManager
+
+    val offsetInfo = source.getOffsetInfo.getOrElse(
+      throw new IllegalArgumentException(s"Offset type is not configured for the source '$sourceName' outputting to '${outputTable.name}''")
+    )
+
+    validateOffsetColumn(df, offsetInfo)
+
+    val req = om.startWriteOffsets(outputTable.name, infoDate, offsetInfo.offsetType)
+
+    val stats = try {
+      val statsToReturn = if (isRerun) {
+        metastore.saveTable(outputTable.name, infoDate, dfToSave, inputRecordCount, saveModeOverride = Some(SaveMode.Overwrite))
+      } else {
+        metastore.saveTable(outputTable.name, infoDate, dfToSave, inputRecordCount, saveModeOverride = Some(SaveMode.Append))
+      }
+
+      val updatedDf = metastore.getBatch(outputTable.name, infoDate, None)
+
+      if (updatedDf.isEmpty) {
+        om.rollbackOffsets(req)
+      } else {
+        val (minOffset, maxOffset) = getMinMaxOffsetFromDf(df, offsetInfo)
+
+        if (isRerun) {
+          om.commitRerun(req, minOffset, maxOffset)
+        } else {
+          om.commitOffsets(req, minOffset, maxOffset)
+        }
+      }
+      statsToReturn
+    } catch {
+      case ex: Throwable =>
+        om.rollbackOffsets(req)
+        throw ex
+    }
+
+    try {
+      source.postProcess(
+        sourceTable.query,
+        outputTable.name,
+        metastore.getMetastoreReader(Seq(outputTable.name), infoDate, runReason, isIncremental = true),
+        infoDate,
+        operationDef.extraOptions
+      )
+    } catch {
+      case _: AbstractMethodError => log.warn(s"Sources were built using old version of Pramen that does not support post processing. Ignoring...")
+    }
+
+    source.close()
+
+    val jobFinished = Instant.now
+    val tooLongWarnings = getTookTooLongWarnings(jobStarted, jobFinished, sourceTable.warnMaxExecutionTimeSeconds)
+
+    SaveResult(stats, warnings = tooLongWarnings)
+  }
+
+  private[core] def validateReadiness(infoDate: LocalDate, om: OffsetManager, hasInfoDate: Boolean, isRerun: Boolean): Reason = {
+    (hasInfoDate, isRerun) match {
+      case (false, false) =>
+        om.getMaxInfoDateAndOffset(outputTable.name, None) match {
+          case Some(offset) if offset.maximumInfoDate.isAfter(infoDate) =>
+            log.warn(s"Cannot run '${outputTable.name}' for '$infoDate' since offsets exists for ${offset.maximumInfoDate}.")
+            Reason.Skip("Incremental ingestion cannot be retrospective")
+          case _ =>
+            Reason.Ready
+        }
+      case (false, true) =>
+        om.getMaxInfoDateAndOffset(outputTable.name, Option(infoDate)) match {
+          case Some(offsets) =>
+            log.info(s"Rerunning ingestion to '${outputTable.name}' at '$infoDate' for ${offsets.minimumOffset.valueString} < offsets <= ${offsets.maximumOffset.valueString}.")
+            Reason.Ready
+          case None =>
+            log.info(s"Offsets not found for '${outputTable.name}' at '$infoDate'.")
+            Reason.SkipOnce("No offsets registered")
+        }
+      case (true, false) =>
+        Reason.Ready
+      case (true, true) =>
+        log.info(s"Rerunning ingestion to '${outputTable.name}' at '$infoDate'.")
+        Reason.Ready
+    }
+  }
+
+  private[core] def validateUncommittedOffsets(onlyForInfoDate: Option[LocalDate], om: OffsetManager): Unit = {
+    val uncommittedOffsets = om.getUncommittedOffsets(outputTable.name, onlyForInfoDate)
+
+    if (uncommittedOffsets.nonEmpty) {
+      onlyForInfoDate match {
+        case Some(date) => log.warn(s"Found uncommitted offsets for ${outputTable.name} at $date. Fixing...")
+        case None => log.warn(s"Found uncommitted offsets for ${outputTable.name}. Fixing...")
+      }
+
+      handleUncommittedOffsets(om, metastore, uncommittedOffsets)
+    }
+  }
+
+  private[core] def handleUncommittedOffsets(om: OffsetManager, mt: Metastore, uncommittedOffsets: Array[UncommittedOffset]): Unit = {
+    import za.co.absa.pramen.core.utils.DateUtils._
+
+    val offsetInfo = source.getOffsetInfo.getOrElse(throw new IllegalArgumentException(s"Offset column not defined for the ingestion job '${operationDef.name}', " +
+      s"query: '${sourceTable.query.query}''"))
+
+    val infoDates = uncommittedOffsets.map(_.infoDate).distinct.sorted
+
+    infoDates.foreach { infoDate =>
+      handleUncommittedOffsetsForDay(om, mt, uncommittedOffsets.filter(_.infoDate == infoDate), infoDate, offsetInfo)
+    }
+  }
+
+  private[core] def handleUncommittedOffsetsForDay(om: OffsetManager, mt: Metastore, uncommittedOffsets: Array[UncommittedOffset], infoDate: LocalDate, offsetInfo: OffsetInfo): Unit = {
+    val df = try {
+      mt.getTable(outputTable.name, Option(infoDate), Option(infoDate))
+    } catch {
+      case _: AnalysisException =>
+        log.warn(s"Table ${outputTable.name} has empty partition for $infoDate. Rolling back uncommitted offsets..")
+        rollbackOffsets(infoDate, om, uncommittedOffsets)
+        return
+    }
+
+    if (df.isEmpty) {
+      log.warn(s"Table ${outputTable.name} is empty for $infoDate. Rolling back uncommitted offsets...")
+      rollbackOffsets(infoDate, om, uncommittedOffsets)
+      return
+    }
+
+    if (!df.schema.fields.exists(_.name.equalsIgnoreCase(offsetInfo.offsetColumn))) {
+      throw new IllegalArgumentException(s"Offset column '${offsetInfo.offsetColumn}' not found in the output table '${outputTable.name}'. Cannot update uncommitted offsets.")
+    }
+
+    val (newMinOffset, newMaxOffset) = getMinMaxOffsetFromDf(df, offsetInfo)
+
+    log.warn(s"Fixing uncommitted offsets. New offset to commit for ${outputTable.name} at $infoDate: " +
+      s"min offset: ${newMinOffset.valueString}, max offset: ${newMaxOffset.valueString}.")
+
+    val req = om.startWriteOffsets(outputTable.name, infoDate, offsetInfo.offsetType)
+    om.commitOffsets(req, newMinOffset, newMaxOffset)
+
+    uncommittedOffsets.foreach { of =>
+      log.warn(s"Cleaning uncommitted offset: $of...")
+      om.rollbackOffsets(DataOffsetRequest(outputTable.name, infoDate, of.batchId, of.createdAt))
+    }
+  }
+
+  private[core] def rollbackOffsets(infoDate: LocalDate, om: OffsetManager, uncommittedOffsets: Array[UncommittedOffset]): Unit = {
+    log.warn(s"No data found for ${outputTable.name}. Rolling back uncommitted offsets...")
+
+    uncommittedOffsets.foreach { of =>
+      log.warn(s"Cleaning uncommitted offset: $of...")
+      om.rollbackOffsets(DataOffsetRequest(outputTable.name, infoDate, of.batchId, of.createdAt))
+    }
+  }
+
+  private[core] def getMinMaxOffsetFromDf(df: DataFrame, offsetInfo: OffsetInfo): (OffsetValue, OffsetValue) = {
+    val row = df.agg(min(offsetInfo.offsetType.getSparkCol(col(offsetInfo.offsetColumn)).cast(StringType)),
+        max(offsetInfo.offsetType.getSparkCol(col(offsetInfo.offsetColumn))).cast(StringType))
+      .collect()(0)
+    val minValue = OffsetValue.fromString(offsetInfo.offsetType.dataTypeString, row(0).asInstanceOf[String]).get
+    val maxValue = OffsetValue.fromString(offsetInfo.offsetType.dataTypeString, row(1).asInstanceOf[String]).get
+
+    SqlGeneratorBase.validateOffsetValue(minValue)
+    SqlGeneratorBase.validateOffsetValue(maxValue)
+
+    (minValue, maxValue)
+  }
+
+  private[core] def validateOffsetColumn(df: DataFrame, offsetInfo: OffsetInfo): Unit = {
+    if (!df.schema.fields.exists(_.name.equalsIgnoreCase(offsetInfo.offsetColumn))) {
+      throw new IllegalArgumentException(s"Offset column '${offsetInfo.offsetColumn}' not found in the output table '${outputTable.name}'.")
+    }
+
+    val field = df.schema.fields.find(_.name.equalsIgnoreCase(offsetInfo.offsetColumn)).get
+
+    offsetInfo.offsetType match {
+      case OffsetType.DateTimeType =>
+        if (!field.dataType.isInstanceOf[TimestampType]) {
+          throw new IllegalArgumentException(s"Offset column '${offsetInfo.offsetColumn}' has type '${field.dataType}'. " +
+            s"But only '${TimestampType.typeName}' is supported for offset type '${offsetInfo.offsetType.dataTypeString}'.")
+        }
+      case OffsetType.IntegralType =>
+        if (!field.dataType.isInstanceOf[ShortType] && !field.dataType.isInstanceOf[IntegerType] && !field.dataType.isInstanceOf[LongType]) {
+          throw new IllegalArgumentException(s"Offset column '${offsetInfo.offsetColumn}' has type '${field.dataType}'. " +
+            s"But only integral types are supported for offset type '${offsetInfo.offsetType.dataTypeString}'.")
+        }
+      case OffsetType.StringType =>
+        if (!field.dataType.isInstanceOf[StringType]) {
+          throw new IllegalArgumentException(s"Offset column '${offsetInfo.offsetColumn}' has type '${field.dataType}'. " +
+            s"But only string type is supported for offset type '${offsetInfo.offsetType.dataTypeString}'.")
+        }
+    }
+  }
+}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IngestionJob.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IngestionJob.scala
index 7b0cc0b8f..5d8b14e41 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IngestionJob.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/IngestionJob.scala
@@ -131,11 +131,11 @@ class IngestionJob(operationDef: OperationDef,
     }
   }
 
-  override def validate(infoDate: LocalDate, jobConfig: Config): Reason = {
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config): Reason = {
     Reason.Ready
   }
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = {
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
     val result = getSourcingResult(infoDate)
 
     RunResult(result.data, result.filesRead, result.warnings)
@@ -160,6 +160,7 @@ class IngestionJob(operationDef: OperationDef,
 
   override def save(df: DataFrame,
                     infoDate: LocalDate,
+                    runReason: TaskRunReason,
                     conf: Config,
                     jobStarted: Instant,
                     inputRecordCount: Option[Long]): SaveResult = {
@@ -169,7 +170,7 @@ class IngestionJob(operationDef: OperationDef,
       source.postProcess(
         sourceTable.query,
         outputTable.name,
-        metastore.getMetastoreReader(Seq(outputTable.name), infoDate),
+        metastore.getMetastoreReader(Seq(outputTable.name), infoDate, runReason, isIncremental = false),
         infoDate,
         operationDef.extraOptions
       )
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/Job.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/Job.scala
index 04ede107e..c28a404ea 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/Job.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/Job.scala
@@ -19,8 +19,8 @@ package za.co.absa.pramen.core.pipeline
 import com.typesafe.config.Config
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.types.StructType
+import za.co.absa.pramen.api.Reason
 import za.co.absa.pramen.api.status.TaskRunReason
-import za.co.absa.pramen.api.{Query, Reason}
 import za.co.absa.pramen.core.metastore.model.MetaTable
 import za.co.absa.pramen.core.runner.splitter.ScheduleStrategy
 
@@ -52,12 +52,14 @@ trait Job {
     * Validates if the job can run. This is the business side of the validation.
     */
   def validate(infoDate: LocalDate,
+               runReason: TaskRunReason,
                conf: Config): Reason
 
   /**
     * Runs the business logic of the job, resulting in a DataFrame.
     */
   def run(infoDate: LocalDate,
+          runReason: TaskRunReason,
           conf: Config): RunResult
 
   /**
@@ -81,6 +83,7 @@ trait Job {
     */
   def save(df: DataFrame,
            infoDate: LocalDate,
+           runReason: TaskRunReason,
            conf: Config,
            jobStarted: Instant,
            inputRecordCount: Option[Long]): SaveResult
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobBase.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobBase.scala
index ad14fa81f..ffb4f5a90 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobBase.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobBase.scala
@@ -24,8 +24,9 @@ import za.co.absa.pramen.core.bookkeeper.Bookkeeper
 import za.co.absa.pramen.core.expr.DateExprEvaluator
 import za.co.absa.pramen.core.metastore.Metastore
 import za.co.absa.pramen.core.metastore.model.MetaTable
+import za.co.absa.pramen.core.schedule.Schedule
 import za.co.absa.pramen.core.utils.Emoji._
-import za.co.absa.pramen.core.utils.TimeUtils
+import za.co.absa.pramen.core.utils.{Emoji, TimeUtils}
 
 import java.time.{Instant, LocalDate}
 import scala.util.{Failure, Success, Try}
@@ -52,6 +53,8 @@ abstract class JobBase(operationDef: OperationDef,
 
   def preRunCheckJob(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config, dependencyWarnings: Seq[DependencyWarning]): JobPreRunResult
 
+  def isIncremental: Boolean = operationDef.schedule == Schedule.Incremental
+
   final override def preRunCheck(infoDate: LocalDate,
                                  runReason: TaskRunReason,
                                  conf: Config): JobPreRunResult = {
@@ -106,23 +109,52 @@ abstract class JobBase(operationDef: OperationDef,
     }
   }
 
-  protected def preRunTransformationCheck(infoDate: LocalDate, dependencyWarnings: Seq[DependencyWarning]): JobPreRunResult = {
-    validateTransformationAlreadyRanCases(infoDate, dependencyWarnings) match {
-      case Some(result) => result
-      case None => JobPreRunResult(JobPreRunStatus.Ready, None, dependencyWarnings, Seq.empty[String])
+  protected def preRunTransformationCheck(infoDate: LocalDate, runReason: TaskRunReason, dependencyWarnings: Seq[DependencyWarning]): JobPreRunResult = {
+    if (isIncremental || runReason == TaskRunReason.Rerun) {
+      JobPreRunResult(JobPreRunStatus.Ready, None, dependencyWarnings, Seq.empty[String])
+    } else {
+      validateTransformationAlreadyRanCases(infoDate, dependencyWarnings) match {
+        case Some(result) => result
+        case None => JobPreRunResult(JobPreRunStatus.Ready, None, dependencyWarnings, Seq.empty[String])
+      }
     }
   }
 
   protected def validateTransformationAlreadyRanCases(infoDate: LocalDate, dependencyWarnings: Seq[DependencyWarning]): Option[JobPreRunResult] = {
-    if (bookkeeper.getLatestDataChunk(outputTableDef.name, infoDate, infoDate).isDefined) {
-      log.info(s"Job for table ${outputTableDef.name} as already ran for $infoDate.")
-      Some(JobPreRunResult(JobPreRunStatus.AlreadyRan, None, dependencyWarnings, Seq.empty[String]))
-    } else {
-      log.info(s"Job for table ${outputTableDef.name} has not yet ran $infoDate.")
-      None
+    bookkeeper.getLatestDataChunk(outputTableDef.name, infoDate, infoDate) match {
+      case Some(chunk) =>
+        val outOfDateTables = getOutdatedTables(infoDate, chunk.jobFinished)
+        if (outOfDateTables.nonEmpty) {
+          log.info(s"Job for table ${outputTableDef.name} as already ran for $infoDate, but has outdated tables: ${outOfDateTables.mkString(", ")}")
+          val warning = s"Based on outdated tables: ${outOfDateTables.mkString(", ")}"
+          Some(JobPreRunResult(JobPreRunStatus.NeedsUpdate, None, dependencyWarnings, Seq(warning)))
+        } else {
+          log.info(s"Job for table ${outputTableDef.name} as already ran for $infoDate.")
+          Some(JobPreRunResult(JobPreRunStatus.AlreadyRan, None, dependencyWarnings, Seq.empty[String]))
+        }
+      case None =>
+        log.info(s"Job for table ${outputTableDef.name} has not yet ran $infoDate.")
+        None
     }
   }
 
+  private def getOutdatedTables(infoDate: LocalDate, targetJobFinishedSeconds: Long): Seq[String] = {
+    operationDef.dependencies
+      .filter(d => !d.isOptional && !d.isPassive)
+      .flatMap(_.tables)
+      .distinct
+      .filter { table =>
+        bookkeeper.getLatestDataChunk(table, infoDate, infoDate) match {
+          case Some(chunk) if chunk.jobFinished >= targetJobFinishedSeconds =>
+            log.warn(s"${Emoji.WARNING} The dependent table '$table' has been updated at ${Instant.ofEpochSecond(chunk.jobFinished)} retrospectively " +
+              s"after the transformation at ${Instant.ofEpochSecond(targetJobFinishedSeconds)} .")
+            true
+          case _ =>
+            false
+        }
+      }
+  }
+
   protected def checkDependency(dep: MetastoreDependency, infoDate: LocalDate): Option[DependencyFailure] = {
     val evaluator = new DateExprEvaluator
     evaluator.setValue("infoDate", infoDate)
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationDef.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationDef.scala
index 8e3485452..70f92f63d 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationDef.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationDef.scala
@@ -20,6 +20,7 @@ import com.typesafe.config.Config
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api.status.MetastoreDependency
 import za.co.absa.pramen.core.app.config.InfoDateConfig
+import za.co.absa.pramen.core.app.config.InfoDateConfig.DEFAULT_INCREMENTAL_INFO_DATE_EXPR
 import za.co.absa.pramen.core.config.Keys
 import za.co.absa.pramen.core.metastore.model.MetastoreDependencyFactory
 import za.co.absa.pramen.core.schedule.Schedule
@@ -113,6 +114,7 @@ object OperationDef {
       case Some(expr) => expr
       case None       =>
         schedule match {
+          case Schedule.Incremental => DEFAULT_INCREMENTAL_INFO_DATE_EXPR
           case _: Schedule.EveryDay => infoDateConfig.expressionDaily
           case _: Schedule.Weekly   => infoDateConfig.expressionWeekly
           case _: Schedule.Monthly  => infoDateConfig.expressionMonthly
@@ -123,9 +125,10 @@ object OperationDef {
       case Some(expr) => expr
       case None       =>
         schedule match {
-          case _: Schedule.EveryDay => infoDateConfig.initialSourcingDateExprDaily
-          case _: Schedule.Weekly   => infoDateConfig.initialSourcingDateExprWeekly
-          case _: Schedule.Monthly  => infoDateConfig.initialSourcingDateExprMonthly
+          case Schedule.Incremental    => DEFAULT_INCREMENTAL_INFO_DATE_EXPR
+          case _: Schedule.EveryDay    => infoDateConfig.initialSourcingDateExprDaily
+          case _: Schedule.Weekly      => infoDateConfig.initialSourcingDateExprWeekly
+          case _: Schedule.Monthly     => infoDateConfig.initialSourcingDateExprMonthly
         }
     }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationSplitter.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationSplitter.scala
index dc839cd4f..a89a0c8c1 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationSplitter.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/OperationSplitter.scala
@@ -30,13 +30,15 @@ import za.co.absa.pramen.core.pipeline.OperationSplitter.{DISABLE_COUNT_QUERY, g
 import za.co.absa.pramen.core.pipeline.OperationType._
 import za.co.absa.pramen.core.pipeline.PythonTransformationJob._
 import za.co.absa.pramen.core.process.ProcessRunner
+import za.co.absa.pramen.core.schedule.Schedule
 import za.co.absa.pramen.core.sink.SinkManager
 import za.co.absa.pramen.core.source.SourceManager
 import za.co.absa.pramen.core.utils.{ClassLoaderUtils, ConfigUtils}
 
 class OperationSplitter(conf: Config,
                         metastore: Metastore,
-                        bookkeeper: Bookkeeper)(implicit spark: SparkSession) {
+                        bookkeeper: Bookkeeper,
+                        batchId: Long)(implicit spark: SparkSession) {
   private val log = LoggerFactory.getLogger(this.getClass)
 
   def createJobs(operationDef: OperationDef): Seq[Job] = {
@@ -68,7 +70,12 @@ class OperationSplitter(conf: Config,
       val notificationTargets = operationDef.notificationTargets
         .map(targetName => getNotificationTarget(conf, targetName, sourceTable.conf))
 
-      new IngestionJob(operationDef, metastore, bookkeeper, notificationTargets, sourceName, source, sourceTable, outputTable, specialCharacters, temporaryDirectory, disableCountQuery)
+      if (operationDef.schedule == Schedule.Incremental) {
+        val latestOffsets = bookkeeper.getOffsetManager.getMaxInfoDateAndOffset(outputTable.name, None)
+        new IncrementalIngestionJob(operationDef, metastore, bookkeeper, notificationTargets, latestOffsets, batchId, sourceName, source, sourceTable, outputTable, specialCharacters)
+      } else {
+        new IngestionJob(operationDef, metastore, bookkeeper, notificationTargets, sourceName, source, sourceTable, outputTable, specialCharacters, temporaryDirectory, disableCountQuery)
+      }
     })
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJob.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJob.scala
index 7dfbfefdd..33d6ebb2b 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJob.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJob.scala
@@ -78,7 +78,7 @@ class PythonTransformationJob(operationDef: OperationDef,
     }
   }
 
-  override def validate(infoDate: LocalDate, jobConfig: Config): Reason = {
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config): Reason = {
     if (outputTable.format.isTransient) {
       Reason.NotReady(s"Python transformations cannot output to transient tables. Please, change teh format of ${outputTable.name}")
     } else {
@@ -86,7 +86,7 @@ class PythonTransformationJob(operationDef: OperationDef,
     }
   }
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = {
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
     if (pramenPyCmdConfigOpt.isDefined)
       runPythonCmdLine(pramenPyCmdConfigOpt.get, infoDate, conf)
     else if (databricksClientOpt.isDefined)
@@ -109,6 +109,7 @@ class PythonTransformationJob(operationDef: OperationDef,
 
   override def save(df: DataFrame,
                     infoDate: LocalDate,
+                    runReason: TaskRunReason,
                     conf: Config,
                     jobStarted: Instant,
                     inputRecordCount: Option[Long]): SaveResult = {
@@ -119,25 +120,29 @@ class PythonTransformationJob(operationDef: OperationDef,
       case ex: AnalysisException => throw new RuntimeException(s"Output data not found in the metastore for $infoDate", ex)
     }
 
-    if (stats.recordCount == 0 && minimumRecords > 0) {
+    val recordCount = stats.recordCount.getOrElse(0L)
+
+    if (recordCount == 0 && minimumRecords > 0) {
       throw new RuntimeException(s"Output table is empty in the metastore for $infoDate")
     }
 
-    if (stats.recordCount < minimumRecords) {
-      throw new RuntimeException(s"The transformation returned too few records (${stats.recordCount} < $minimumRecords).")
+    if (recordCount < minimumRecords) {
+      throw new RuntimeException(s"The transformation returned too few records ($recordCount < $minimumRecords).")
     }
 
     val jobFinished = Instant.now()
 
-    bookkeeper.setRecordCount(outputTable.name,
-      infoDate,
-      infoDate,
-      infoDate,
-      stats.recordCount,
-      stats.recordCount,
-      jobStarted.getEpochSecond,
-      jobFinished.getEpochSecond,
-      isTableTransient = false)
+    stats.recordCount.foreach{ recordCount =>
+      bookkeeper.setRecordCount(outputTable.name,
+        infoDate,
+        infoDate,
+        infoDate,
+        recordCount,
+        recordCount,
+        jobStarted.getEpochSecond,
+        jobFinished.getEpochSecond,
+        isTableTransient = false)
+    }
 
     SaveResult(stats)
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/SinkJob.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/SinkJob.scala
index a055f1aaf..7d3913474 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/SinkJob.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/SinkJob.scala
@@ -48,7 +48,7 @@ class SinkJob(operationDef: OperationDef,
   override val scheduleStrategy: ScheduleStrategy = new ScheduleStrategySourcing
 
   override def preRunCheckJob(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config, dependencyWarnings: Seq[DependencyWarning]): JobPreRunResult = {
-    val alreadyRanStatus = preRunTransformationCheck(infoDate, dependencyWarnings)
+    val alreadyRanStatus = preRunTransformationCheck(infoDate, runReason, dependencyWarnings)
 
     alreadyRanStatus.status match {
       case JobPreRunStatus.Ready => JobPreRunResult(Ready, Some(getDataDf(infoDate).count()), dependencyWarnings, alreadyRanStatus.warnings)
@@ -56,7 +56,7 @@ class SinkJob(operationDef: OperationDef,
     }
   }
 
-  override def validate(infoDate: LocalDate, jobConfig: Config): Reason = {
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config): Reason = {
     val minimumRecordsOpt = ConfigUtils.getOptionInt(sink.config, MINIMUM_RECORDS_KEY)
 
     minimumRecordsOpt.foreach(n => log.info(s"Minimum records to send: $n"))
@@ -81,7 +81,7 @@ class SinkJob(operationDef: OperationDef,
     }
   }
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = {
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
     RunResult(getDataDf(infoDate))
   }
 
@@ -108,6 +108,7 @@ class SinkJob(operationDef: OperationDef,
 
   override def save(df: DataFrame,
                     infoDate: LocalDate,
+                    runReason: TaskRunReason,
                     conf: Config,
                     jobStarted: Instant,
                     inputRecordCount: Option[Long]): SaveResult = {
@@ -120,7 +121,7 @@ class SinkJob(operationDef: OperationDef,
     try {
       val sinkResult = sink.send(df,
         sinkTable.metaTableName,
-        metastore.getMetastoreReader(List(sinkTable.metaTableName) ++ inputTables, infoDate),
+        metastore.getMetastoreReader(List(sinkTable.metaTableName) ++ inputTables, infoDate, runReason, isIncremental),
         infoDate,
         sinkTable.options
       )
@@ -141,7 +142,7 @@ class SinkJob(operationDef: OperationDef,
         isTransient
       )
 
-      val stats = MetaTableStats(sinkResult.recordsSent, None)
+      val stats = MetaTableStats(Option(sinkResult.recordsSent), None, None)
       SaveResult(stats, sinkResult.filesSent, sinkResult.hiveTables, sinkResult.warnings ++ tooLongWarnings)
     } catch {
       case NonFatal(ex) => throw new IllegalStateException("Unable to write to the sink.", ex)
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferJob.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferJob.scala
index 869414b2b..250115a07 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferJob.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferJob.scala
@@ -51,12 +51,12 @@ class TransferJob(operationDef: OperationDef,
     ingestionJob.preRunCheckJob(infoDate, runReason, jobConfig, dependencyWarnings)
   }
 
-  override def validate(infoDate: LocalDate, jobConfig: Config): Reason = {
-    ingestionJob.validate(infoDate, jobConfig)
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config): Reason = {
+    ingestionJob.validate(infoDate, runReason, jobConfig)
   }
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = {
-    ingestionJob.run(infoDate, conf)
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
+    ingestionJob.run(infoDate, runReason, conf)
   }
 
   def postProcessing(df: DataFrame,
@@ -67,9 +67,10 @@ class TransferJob(operationDef: OperationDef,
 
   override def save(df: DataFrame,
                     infoDate: LocalDate,
+                    runReason: TaskRunReason,
                     conf: Config,
                     jobStarted: Instant,
                     inputRecordCount: Option[Long]): SaveResult = {
-    sinkJob.save(df, infoDate, conf, jobStarted, inputRecordCount)
+    sinkJob.save(df, infoDate, runReason, conf, jobStarted, inputRecordCount)
   }
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferTable.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferTable.scala
index 888cfda90..f76810f46 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferTable.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransferTable.scala
@@ -57,7 +57,7 @@ case class TransferTable(
   }
 
   def getMetaTable: MetaTable = {
-    MetaTable(jobMetaTableName, "", DataFormat.Null(), "", "", HiveConfig.getNullConfig, None, None, hivePreferAddPartition = true, None, infoDateStart, trackDays, trackDaysExplicitlySet = trackDaysExplicitlySet, None, readOptions, writeOptions, sparkConfig)
+    MetaTable(jobMetaTableName, "", DataFormat.Null(), "", "", "", HiveConfig.getNullConfig, None, None, hivePreferAddPartition = true, None, infoDateStart, trackDays, trackDaysExplicitlySet = trackDaysExplicitlySet, None, readOptions, writeOptions, sparkConfig)
   }
 }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransformationJob.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransformationJob.scala
index 7c0750d2b..18d710f79 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransformationJob.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TransformationJob.scala
@@ -17,13 +17,13 @@
 package za.co.absa.pramen.core.pipeline
 
 import com.typesafe.config.Config
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
 import za.co.absa.pramen.api.status.{DependencyWarning, TaskRunReason}
 import za.co.absa.pramen.api.{Reason, Transformer}
 import za.co.absa.pramen.core.bookkeeper.Bookkeeper
 import za.co.absa.pramen.core.metastore.Metastore
 import za.co.absa.pramen.core.metastore.model.MetaTable
-import za.co.absa.pramen.core.runner.splitter.{ScheduleStrategy, ScheduleStrategySourcing}
+import za.co.absa.pramen.core.runner.splitter.{ScheduleStrategy, ScheduleStrategyIncremental, ScheduleStrategySourcing}
 
 import java.time.{Instant, LocalDate}
 
@@ -38,18 +38,23 @@ class TransformationJob(operationDef: OperationDef,
 
   private val inputTables = operationDef.dependencies.flatMap(_.tables).distinct
 
-  override val scheduleStrategy: ScheduleStrategy = new ScheduleStrategySourcing
+  override val scheduleStrategy: ScheduleStrategy = {
+    if (isIncremental)
+      new ScheduleStrategyIncremental(None, true)
+    else
+      new ScheduleStrategySourcing
+  }
 
   override def preRunCheckJob(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config, dependencyWarnings: Seq[DependencyWarning]): JobPreRunResult = {
-    preRunTransformationCheck(infoDate, dependencyWarnings)
+    preRunTransformationCheck(infoDate, runReason, dependencyWarnings)
   }
 
-  override def validate(infoDate: LocalDate, jobConfig: Config): Reason = {
-    transformer.validate(metastore.getMetastoreReader(inputTables, infoDate), infoDate, operationDef.extraOptions)
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config): Reason = {
+    transformer.validate(metastore.getMetastoreReader(inputTables, infoDate, runReason, isIncremental), infoDate, operationDef.extraOptions)
   }
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = {
-    RunResult(transformer.run(metastore.getMetastoreReader(inputTables, infoDate), infoDate, operationDef.extraOptions))
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
+    RunResult(transformer.run(metastore.getMetastoreReader(inputTables, infoDate, runReason, isIncremental), infoDate, operationDef.extraOptions))
   }
 
   def postProcessing(df: DataFrame,
@@ -60,15 +65,19 @@ class TransformationJob(operationDef: OperationDef,
 
   override def save(df: DataFrame,
                     infoDate: LocalDate,
+                    runReason: TaskRunReason,
                     conf: Config,
                     jobStarted: Instant,
                     inputRecordCount: Option[Long]): SaveResult = {
-    val saveResults = SaveResult(metastore.saveTable(outputTable.name, infoDate, df, None))
+    val saveResults = if (isIncremental && runReason != TaskRunReason.Rerun)
+      SaveResult(metastore.saveTable(outputTable.name, infoDate, df, None, saveModeOverride = Some(SaveMode.Append)))
+    else
+      SaveResult(metastore.saveTable(outputTable.name, infoDate, df, None))
 
     try {
       transformer.postProcess(
         outputTable.name,
-        metastore.getMetastoreReader(inputTables :+ outputTable.name, infoDate),
+        metastore.getMetastoreReader(inputTables :+ outputTable.name, infoDate, runReason, isIncremental),
         infoDate, operationDef.extraOptions
       )
     } catch {
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/rdb/PramenDb.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/rdb/PramenDb.scala
index 914a2a997..765232d4b 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/rdb/PramenDb.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/rdb/PramenDb.scala
@@ -19,7 +19,7 @@ package za.co.absa.pramen.core.rdb
 import org.slf4j.LoggerFactory
 import slick.jdbc.H2Profile
 import slick.jdbc.H2Profile.api._
-import za.co.absa.pramen.core.bookkeeper.model.{BookkeepingRecords, MetadataRecords, SchemaRecords}
+import za.co.absa.pramen.core.bookkeeper.model.{BookkeepingRecords, MetadataRecords, OffsetRecords, SchemaRecords}
 import za.co.absa.pramen.core.journal.model.JournalTasks
 import za.co.absa.pramen.core.lock.model.LockTickets
 import za.co.absa.pramen.core.rdb.PramenDb.MODEL_VERSION
@@ -71,6 +71,14 @@ class PramenDb(val jdbcConfig: JdbcConfig,
       addColumn(JournalTasks.journalTasks.baseTableRow.tableName, "environmentName", "varchar(128)")
       addColumn(JournalTasks.journalTasks.baseTableRow.tableName, "tenant", "varchar(200)")
     }
+
+    if (dbVersion < 5) {
+      initTable(OffsetRecords.records.schema)
+    }
+
+    if (0 < dbVersion && dbVersion < 6) {
+      addColumn(JournalTasks.journalTasks.baseTableRow.tableName, "appended_record_count", "bigint")
+    }
   }
 
   def initTable(schema: H2Profile.SchemaDescription): Unit = {
@@ -106,7 +114,7 @@ class PramenDb(val jdbcConfig: JdbcConfig,
 }
 
 object PramenDb {
-  val MODEL_VERSION = 4
+  val MODEL_VERSION = 6
   val DEFAULT_RETRIES = 3
 
   def apply(jdbcConfig: JdbcConfig): PramenDb = {
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderDelta.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderDelta.scala
deleted file mode 100644
index 651a9afd0..000000000
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderDelta.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright 2022 ABSA Group Limited
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package za.co.absa.pramen.core.reader
-
-import org.apache.spark.sql.functions.{col, lit}
-import org.apache.spark.sql.{DataFrame, SparkSession}
-import org.slf4j.LoggerFactory
-import za.co.absa.pramen.api.{Query, TableReader}
-
-import java.time.LocalDate
-import java.time.format.DateTimeFormatter
-
-class TableReaderDelta(infoDateColumn: String,
-                       infoDateFormat: String = "yyyy-MM-dd"
-                      )(implicit spark: SparkSession) extends TableReader {
-
-  private val log = LoggerFactory.getLogger(this.getClass)
-  private val dateFormatter = DateTimeFormatter.ofPattern(infoDateFormat)
-
-  override def getRecordCount(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long = {
-    if (infoDateBegin.equals(infoDateEnd)) {
-      log.info(s"Reading COUNT(*) FROM ${query.query} WHERE $infoDateColumn='${dateFormatter.format(infoDateBegin)}'")
-    } else {
-      log.info(s"Reading COUNT(*) FROM ${query.query} WHERE $infoDateColumn BETWEEN '${dateFormatter.format(infoDateBegin)}' AND '${dateFormatter.format(infoDateEnd)}'")
-    }
-    getFilteredDataFrame(query, infoDateBegin, infoDateEnd).count()
-  }
-
-  override def getData(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String]): DataFrame = {
-    if (infoDateBegin.equals(infoDateEnd)) {
-      log.info(s"Reading * FROM ${query.query} WHERE $infoDateColumn='${dateFormatter.format(infoDateEnd)}'")
-    } else {
-      log.info(s"Reading * FROM ${query.query} WHERE $infoDateColumn BETWEEN '${dateFormatter.format(infoDateBegin)}' AND '${dateFormatter.format(infoDateEnd)}'")
-    }
-    getFilteredDataFrame(query, infoDateBegin, infoDateEnd)
-  }
-
-  private def getFilteredDataFrame(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): DataFrame = {
-    val infoDateBeginStr = dateFormatter.format(infoDateBegin)
-    val infoDateEndStr = dateFormatter.format(infoDateEnd)
-
-    val reader = query match {
-      case Query.Path(path)   =>
-        spark
-          .read
-          .format("delta")
-          .load(path)
-      case Query.Table(table) =>
-        spark
-          .table(table)
-      case _                  => throw new IllegalArgumentException(s"Arguments of type ${query.getClass} are not supported for Delta format.")
-    }
-
-    if (infoDateBegin.equals(infoDateEnd)) {
-      reader.filter(col(s"$infoDateColumn") === lit(infoDateBeginStr))
-    } else {
-      reader.filter(col(s"$infoDateColumn") >= lit(infoDateBeginStr) && col(s"$infoDateColumn") <= lit(infoDateEndStr))
-    }
-  }
-}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala
index d80f77ec6..429d6b212 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala
@@ -20,8 +20,9 @@ import com.typesafe.config.Config
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api.Query
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
 import za.co.absa.pramen.core.config.Keys
-import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig
+import za.co.absa.pramen.core.reader.model.{OffsetInfoParser, TableReaderJdbcConfig}
 import za.co.absa.pramen.core.utils.{ConfigUtils, JdbcNativeUtils, JdbcSparkUtils, TimeUtils}
 
 import java.time.format.DateTimeFormatter
@@ -35,8 +36,6 @@ class TableReaderJdbc(jdbcReaderConfig: TableReaderJdbcConfig,
                      )(implicit spark: SparkSession) extends TableReaderJdbcBase(jdbcReaderConfig, jdbcUrlSelector, conf) {
   private val log = LoggerFactory.getLogger(this.getClass)
 
-  private val infoDateFormatter = DateTimeFormatter.ofPattern(jdbcReaderConfig.infoDateFormat)
-
   logConfiguration()
 
   override def getRecordCount(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long = {
@@ -67,6 +66,15 @@ class TableReaderJdbc(jdbcReaderConfig: TableReaderJdbcConfig,
     }
   }
 
+  override def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): DataFrame = {
+    query match {
+      case Query.Table(tableName) =>
+        getDataForTableIncremental(tableName, onlyForInfoDate, offsetFrom, offsetTo, columns)
+      case other =>
+        throw new IllegalArgumentException(s"'${other.name}' incremental ingestion is not supported by the JDBC reader. Use 'table' instead.")
+    }
+  }
+
   private[core] def getJdbcConfig: TableReaderJdbcConfig = jdbcReaderConfig
 
   @tailrec
@@ -146,6 +154,24 @@ class TableReaderJdbc(jdbcReaderConfig: TableReaderJdbcConfig,
     df
   }
 
+  private[core] def getDataForTableIncremental(tableName: String,
+                                               onlyForInfoDate: Option[LocalDate],
+                                               offsetFrom: Option[OffsetValue],
+                                               offsetTo: Option[OffsetValue],
+                                               columns: Seq[String]): DataFrame = {
+    val sql = sqlGen.getDataQueryIncremental(tableName, onlyForInfoDate, offsetFrom, offsetTo, columns)
+
+    log.info(s"JDBC Query: $sql")
+
+    val df = getWithRetry[DataFrame](sql, isDataQuery = true, jdbcRetries, Option(tableName))(df => {
+      // Make sure connection to the server is made without fetching the data
+      log.debug(df.schema.treeString)
+      df
+    })
+
+    df
+  }
+
   private[core] def getDataForSql(sql: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String]): DataFrame = {
     val filteredSql = TableReaderJdbcNative.getFilteredSql(sql, infoDateBegin, infoDateEnd)
     getWithRetry[DataFrame](filteredSql, isDataQuery = true, jdbcRetries, None)(df => filterDfColumns(df, columns))
@@ -222,8 +248,8 @@ class TableReaderJdbc(jdbcReaderConfig: TableReaderJdbcConfig,
 }
 
 object TableReaderJdbc {
-  def apply(conf: Config, parent: String)(implicit spark: SparkSession): TableReaderJdbc = {
-    val jdbcTableReaderConfig = TableReaderJdbcConfig.load(conf, parent)
+  def apply(conf: Config, workflowConf: Config, parent: String)(implicit spark: SparkSession): TableReaderJdbc = {
+    val jdbcTableReaderConfig = TableReaderJdbcConfig.load(conf, workflowConf, parent)
 
     val urlSelector = JdbcUrlSelector(jdbcTableReaderConfig.jdbcConfig)
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcBase.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcBase.scala
index 06550fd0e..938baa8bf 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcBase.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcBase.scala
@@ -19,6 +19,7 @@ package za.co.absa.pramen.core.reader
 import com.typesafe.config.Config
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api.TableReader
+import za.co.absa.pramen.api.offset.OffsetInfo
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig}
 import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig
 import za.co.absa.pramen.core.sql.SqlGeneratorLoader
@@ -44,18 +45,14 @@ abstract class TableReaderJdbcBase(jdbcReaderConfig: TableReaderJdbcConfig,
   }
 
   private[core] def getSqlConfig: SqlConfig = {
-    val dateFieldType = SqlColumnType.fromString(jdbcReaderConfig.infoDateType)
-    dateFieldType match {
-      case Some(infoDateType) =>
-        SqlConfig(jdbcReaderConfig.infoDateColumn,
-          infoDateType,
-          jdbcReaderConfig.infoDateFormat,
-          jdbcReaderConfig.identifierQuotingPolicy,
-          jdbcReaderConfig.sqlGeneratorClass,
-          ConfigUtils.getExtraConfig(conf, "sql"))
-      case None => throw new IllegalArgumentException(s"Unknown info date type specified (${jdbcReaderConfig.infoDateType}). " +
-        s"It should be one of: date, string, number")
-    }
+    SqlConfig(jdbcReaderConfig.infoDateColumn,
+      jdbcReaderConfig.infoDateType,
+      jdbcReaderConfig.infoDateFormat,
+      jdbcReaderConfig.offsetInfoOpt,
+      jdbcReaderConfig.serverTimeZone,
+      jdbcReaderConfig.identifierQuotingPolicy,
+      jdbcReaderConfig.sqlGeneratorClass,
+      ConfigUtils.getExtraConfig(conf, "sql"))
   }
 
   protected def logConfiguration(): Unit = {
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcNative.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcNative.scala
index e9f597b24..632764cbb 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcNative.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbcNative.scala
@@ -20,6 +20,7 @@ import com.typesafe.config.Config
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api.Query
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.core.expr.DateExprEvaluator
 import za.co.absa.pramen.core.reader.model.{JdbcConfig, TableReaderJdbcConfig}
 import za.co.absa.pramen.core.utils.{JdbcNativeUtils, JdbcSparkUtils, StringUtils, TimeUtils}
@@ -64,6 +65,15 @@ class TableReaderJdbcNative(jdbcReaderConfig: TableReaderJdbcConfig,
     }
   }
 
+  override def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): DataFrame = {
+    query match {
+      case Query.Table(tableName) =>
+        getDataForTableIncremental(tableName, onlyForInfoDate, offsetFrom, offsetTo, columns)
+      case other =>
+        throw new IllegalArgumentException(s"'${other.name}' incremental ingestion is not supported by the JDBC Native reader. Use 'table' instead.")
+    }
+  }
+
   private[core] def getSqlExpression(query: Query): String = {
     query match {
       case Query.Sql(sql) => sql
@@ -103,15 +113,41 @@ class TableReaderJdbcNative(jdbcReaderConfig: TableReaderJdbcConfig,
 
     df
   }
+
+  private[core] def getDataForTableIncremental(tableName: String,
+                                               onlyForInfoDate: Option[LocalDate],
+                                               offsetFrom: Option[OffsetValue],
+                                               offsetTo: Option[OffsetValue],
+                                               columns: Seq[String]): DataFrame = {
+    val sql = sqlGen.getDataQueryIncremental(tableName, onlyForInfoDate, offsetFrom, offsetTo, columns)
+    log.info(s"JDBC Query: $sql")
+
+    var df = JdbcNativeUtils.getJdbcNativeDataFrame(jdbcConfig, url, sql)
+
+    if (log.isDebugEnabled) {
+      log.debug(df.schema.treeString)
+    }
+
+    if (jdbcReaderConfig.enableSchemaMetadata) {
+      JdbcSparkUtils.withJdbcMetadata(jdbcReaderConfig.jdbcConfig, sql) { (connection, _) =>
+        log.info(s"Reading JDBC metadata descriptions the table: $tableName")
+        df = spark.createDataFrame(df.rdd,
+          JdbcSparkUtils.addColumnDescriptionsFromJdbc(df.schema, sqlGen.unquote(tableName), connection))
+      }
+    }
+
+    df
+  }
 }
 
 object TableReaderJdbcNative {
   val FETCH_SIZE_KEY = "option.fetchsize"
 
   def apply(conf: Config,
+            workflowConf: Config,
             parent: String = "")
            (implicit spark: SparkSession): TableReaderJdbcNative = {
-    val tableReaderJdbcOrig = TableReaderJdbcConfig.load(conf, parent)
+    val tableReaderJdbcOrig = TableReaderJdbcConfig.load(conf, workflowConf, parent)
     val jdbcConfig = getJdbcConfig(tableReaderJdbcOrig, conf)
     val tableReaderJdbc = tableReaderJdbcOrig.copy(jdbcConfig = jdbcConfig)
     val urlSelector = JdbcUrlSelector(tableReaderJdbc.jdbcConfig)
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderSpark.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderSpark.scala
index 6e3d22214..49562a27a 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderSpark.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderSpark.scala
@@ -18,10 +18,14 @@ package za.co.absa.pramen.core.reader
 
 import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.functions.{col, lit}
+import org.apache.spark.sql.types.DateType
 import org.apache.spark.sql.{DataFrame, DataFrameReader, SparkSession}
 import org.slf4j.LoggerFactory
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
+import za.co.absa.pramen.api.sql.SqlColumnType
 import za.co.absa.pramen.api.{Query, TableReader}
 
+import java.sql.Date
 import java.time.LocalDate
 import java.time.format.DateTimeFormatter
 
@@ -29,7 +33,9 @@ class TableReaderSpark(formatOpt: Option[String],
                        schemaOpt: Option[String],
                        hasInfoDateColumn: Boolean,
                        infoDateColumn: String,
+                       infoDateDataType: SqlColumnType,
                        infoDateFormat: String = "yyyy-MM-dd",
+                       offsetInfoOpt: Option[OffsetInfo],
                        options: Map[String, String] = Map.empty[String, String]
                       )(implicit spark: SparkSession) extends TableReader {
 
@@ -64,6 +70,50 @@ class TableReaderSpark(formatOpt: Option[String],
     }
   }
 
+  override def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): DataFrame = {
+    val offsetInfo = offsetInfoOpt.getOrElse(throw new IllegalArgumentException(s"Offset column and type is not defined for ${query.query}."))
+    val offsetCol = offsetInfo.offsetType.getSparkCol(col(offsetInfo.offsetColumn))
+
+    onlyForInfoDate match {
+      case Some(infoDate) if hasInfoDateColumn =>
+        (offsetFromOpt, offsetToOpt) match {
+          case (Some(offsetFrom), Some(offsetTo)) =>
+            log.info(s"Reading * FROM ${query.query} WHERE $infoDateColumn='$infoDate' AND ${offsetInfo.offsetColumn} >= ${offsetFrom.valueString} AND ${offsetInfo.offsetColumn} <= ${offsetTo.valueString}")
+            getData(query, infoDate, infoDate, columns)
+              .filter(offsetCol >= offsetFrom.getSparkLit && offsetCol <= offsetTo.getSparkLit)
+          case (Some(offsetFrom), None) =>
+            log.info(s"Reading * FROM ${query.query} WHERE $infoDateColumn='$infoDate' AND ${offsetInfo.offsetColumn} > ${offsetFrom.valueString}")
+            getData(query, infoDate, infoDate, columns)
+              .filter(offsetCol > offsetFrom.getSparkLit)
+          case (None, Some(offsetTo)) =>
+            log.info(s"Reading * FROM ${query.query} WHERE $infoDateColumn='$infoDate' AND ${offsetInfo.offsetColumn} <= ${offsetTo.valueString}")
+            getData(query, infoDate, infoDate, columns)
+              .filter(offsetCol <= offsetTo.getSparkLit)
+          case (None, None) =>
+            log.info(s"Reading * FROM ${query.query} WHERE $infoDateColumn='$infoDate'")
+            getData(query, infoDate, infoDate, columns)
+        }
+      case _ =>
+        (offsetFromOpt, offsetToOpt) match {
+          case (Some(offsetFrom), Some(offsetTo)) =>
+            log.info(s"Reading * FROM ${query.query} WHERE ${offsetInfo.offsetColumn} >= ${offsetFrom.valueString} AND ${offsetInfo.offsetColumn} <= ${offsetTo.valueString}")
+            getBaseDataFrame(query)
+              .filter(offsetCol >= offsetFrom.getSparkLit && offsetCol <= offsetTo.getSparkLit)
+          case (Some(offsetFrom), None) =>
+            log.info(s"Reading * FROM ${query.query} WHERE ${offsetInfo.offsetColumn} > ${offsetFrom.valueString}")
+            getBaseDataFrame(query)
+              .filter(offsetCol > offsetFrom.getSparkLit)
+          case (None, Some(offsetTo)) =>
+            log.info(s"Reading * FROM ${query.query} WHERE ${offsetInfo.offsetColumn} <= ${offsetTo.valueString}")
+            getBaseDataFrame(query)
+              .filter(offsetCol <= offsetTo.getSparkLit)
+          case (None, None) =>
+            log.info(s"Reading * FROM ${query.query}")
+            getBaseDataFrame(query)
+        }
+    }
+  }
+
   private[core] def getDailyDataFrame(query: Query, infoDate: LocalDate): DataFrame = {
     val dateStr = dateFormatter.format(infoDate)
 
@@ -99,15 +149,26 @@ class TableReaderSpark(formatOpt: Option[String],
   }
 
   private[core] def getFilteredDataFrame(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): DataFrame = {
-    val infoDateBeginStr = dateFormatter.format(infoDateBegin)
-    val infoDateEndStr = dateFormatter.format(infoDateEnd)
-
-    if (infoDateBegin.equals(infoDateEnd)) {
-      getBaseDataFrame(query)
-        .filter(col(s"$infoDateColumn") === lit(infoDateBeginStr))
-    } else {
-      getBaseDataFrame(query)
-        .filter(col(s"$infoDateColumn") >= lit(infoDateBeginStr) && col(s"$infoDateColumn") <= lit(infoDateEndStr))
+    infoDateDataType match {
+      case SqlColumnType.DATETIME =>
+        if (infoDateBegin.equals(infoDateEnd)) {
+          getBaseDataFrame(query)
+            .filter(col(infoDateColumn).cast(DateType) === lit(Date.valueOf(infoDateBegin)))
+        } else {
+          getBaseDataFrame(query)
+            .filter(col(infoDateColumn).cast(DateType) >= lit(Date.valueOf(infoDateBegin)) && col(infoDateColumn).cast(DateType) <= lit(Date.valueOf(infoDateEnd)))
+        }
+      case _ =>
+        val infoDateBeginStr = dateFormatter.format(infoDateBegin)
+        val infoDateEndStr = dateFormatter.format(infoDateEnd)
+
+        if (infoDateBegin.equals(infoDateEnd)) {
+          getBaseDataFrame(query)
+            .filter(col(infoDateColumn) === lit(infoDateBeginStr))
+        } else {
+          getBaseDataFrame(query)
+            .filter(col(infoDateColumn) >= lit(infoDateBeginStr) && col(infoDateColumn) <= lit(infoDateEndStr))
+        }
     }
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/OffsetInfoParser.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/OffsetInfoParser.scala
new file mode 100644
index 000000000..a7cc0d135
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/OffsetInfoParser.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.reader.model
+
+import com.typesafe.config.Config
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetType}
+import za.co.absa.pramen.core.utils.ConfigUtils
+
+object OffsetInfoParser {
+  val OFFSET_COLUMN_NAME_KEY = "offset.column.name"
+  val OFFSET_COLUMN_TYPE_KEY = "offset.column.type"
+
+  def fromConfig(conf: Config): Option[OffsetInfo] = {
+    for {
+      columnName <- ConfigUtils.getOptionString(conf, OFFSET_COLUMN_NAME_KEY)
+      columnType <- ConfigUtils.getOptionString(conf, OFFSET_COLUMN_TYPE_KEY)
+    } yield OffsetInfo(columnName, OffsetType.fromString(columnType))
+  }
+
+}
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala
index 529b9311c..3ecffd2b0 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala
@@ -18,21 +18,27 @@ package za.co.absa.pramen.core.reader.model
 
 import com.typesafe.config.Config
 import org.slf4j.LoggerFactory
-import za.co.absa.pramen.api.sql.QuotingPolicy
+import za.co.absa.pramen.api.offset.OffsetInfo
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType}
+import za.co.absa.pramen.core.config.Keys
 import za.co.absa.pramen.core.utils.ConfigUtils
 
+import java.time.ZoneId
+
 case class TableReaderJdbcConfig(
                                   jdbcConfig: JdbcConfig,
                                   hasInfoDate: Boolean,
                                   infoDateColumn: String,
-                                  infoDateType: String,
+                                  infoDateType: SqlColumnType,
                                   infoDateFormat: String = "yyyy-MM-dd",
+                                  offsetInfoOpt: Option[OffsetInfo],
                                   limitRecords: Option[Int] = None,
                                   saveTimestampsAsDates: Boolean = false,
                                   correctDecimalsInSchema: Boolean = false,
                                   correctDecimalsFixPrecision: Boolean = false,
                                   enableSchemaMetadata: Boolean = false,
                                   useJdbcNative: Boolean = false,
+                                  serverTimeZone: ZoneId = ZoneId.systemDefault(),
                                   identifierQuotingPolicy: QuotingPolicy = QuotingPolicy.Auto,
                                   sqlGeneratorClass: Option[String] = None
                                 )
@@ -52,10 +58,11 @@ object TableReaderJdbcConfig {
   val CORRECT_DECIMALS_FIX_PRECISION = "correct.decimals.fix.precision"
   val ENABLE_SCHEMA_METADATA_KEY = "enable.schema.metadata"
   val USE_JDBC_NATIVE = "use.jdbc.native"
+  val SERVER_TIMEZONE = "server.timezone"
   val IDENTIFIER_QUOTING_POLICY = "identifier.quoting.policy"
   val SQL_GENERATOR_CLASS_KEY = "sql.generator.class"
 
-  def load(conf: Config, parent: String = ""): TableReaderJdbcConfig = {
+  def load(conf: Config, workflowConf: Config, parent: String = ""): TableReaderJdbcConfig = {
     ConfigUtils.validatePathsExistence(conf, parent, HAS_INFO_DATE :: Nil)
 
     val hasInformationDate = conf.getBoolean(HAS_INFO_DATE)
@@ -66,6 +73,10 @@ object TableReaderJdbcConfig {
         INFORMATION_DATE_COLUMN :: INFORMATION_DATE_TYPE :: Nil)
     }
 
+    val infoDateTypeStr = ConfigUtils.getOptionString(conf, INFORMATION_DATE_TYPE).getOrElse("date")
+
+    val infoDateType = SqlColumnType.fromStringStrict(infoDateTypeStr, parent)
+
     val saveTimestampsAsDates = ConfigUtils.getOptionBoolean(conf, JDBC_TIMESTAMPS_AS_DATES).getOrElse(false)
 
     if (saveTimestampsAsDates) {
@@ -74,6 +85,11 @@ object TableReaderJdbcConfig {
 
     val infoDateFormat = getInfoDateFormat(conf)
 
+    val offsetInfoOpt = OffsetInfoParser.fromConfig(conf)
+
+    val defaultTimezone = ConfigUtils.getOptionString(workflowConf, Keys.TIMEZONE).getOrElse("Africa/Johannesburg")
+    val serverTimezone = ZoneId.of(ConfigUtils.getOptionString(conf, SERVER_TIMEZONE).getOrElse(defaultTimezone))
+
     val identifierQuotingPolicy = ConfigUtils.getOptionString(conf, IDENTIFIER_QUOTING_POLICY)
       .map(s => QuotingPolicy.fromString(s))
       .getOrElse(QuotingPolicy.Auto)
@@ -82,14 +98,16 @@ object TableReaderJdbcConfig {
       jdbcConfig = JdbcConfig.load(conf, parent),
       hasInfoDate = conf.getBoolean(HAS_INFO_DATE),
       infoDateColumn = ConfigUtils.getOptionString(conf, INFORMATION_DATE_COLUMN).getOrElse(""),
-      infoDateType = ConfigUtils.getOptionString(conf, INFORMATION_DATE_TYPE).getOrElse("date"),
+      infoDateType = infoDateType,
       infoDateFormat,
+      offsetInfoOpt,
       limitRecords = ConfigUtils.getOptionInt(conf, JDBC_SYNC_LIMIT_RECORDS),
       saveTimestampsAsDates,
       correctDecimalsInSchema = ConfigUtils.getOptionBoolean(conf, CORRECT_DECIMALS_IN_SCHEMA).getOrElse(false),
       correctDecimalsFixPrecision = ConfigUtils.getOptionBoolean(conf, CORRECT_DECIMALS_FIX_PRECISION).getOrElse(false),
       enableSchemaMetadata = ConfigUtils.getOptionBoolean(conf, ENABLE_SCHEMA_METADATA_KEY).getOrElse(false),
       useJdbcNative = ConfigUtils.getOptionBoolean(conf, USE_JDBC_NATIVE).getOrElse(false),
+      serverTimezone,
       identifierQuotingPolicy = identifierQuotingPolicy,
       sqlGeneratorClass = ConfigUtils.getOptionString(conf, SQL_GENERATOR_CLASS_KEY)
     )
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/AppRunner.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/AppRunner.scala
index 20faa72b0..d700a6280 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/AppRunner.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/AppRunner.scala
@@ -96,7 +96,9 @@ object AppRunner {
 
   private[core] def createPipelineState(implicit conf: Config): Try[PipelineState] = {
     Try {
-      new PipelineStateImpl()(conf, PramenImpl.instance.notificationBuilder)
+      val state = new PipelineStateImpl()(conf, PramenImpl.instance.notificationBuilder)
+      log.info(s"Starting pipeline, batchId=${state.getBatchId}")
+      state
     }
   }
 
@@ -105,7 +107,7 @@ object AppRunner {
                                      spark: SparkSession): Try[AppContext] = {
     handleFailure(Try {
       PramenImpl.instance.asInstanceOf[PramenImpl].setPipelineState(state)
-      AppContextImpl(conf)
+      AppContextImpl(conf, state.getBatchId)
     }, state, "initialization of the pipeline")
   }
 
@@ -183,7 +185,7 @@ object AppRunner {
                               spark: SparkSession): Try[Seq[Job]] = {
     handleFailure(Try {
       val isHistoricalRun = appContext.appConfig.runtimeConfig.runDateTo.nonEmpty
-      val splitter = new OperationSplitter(conf, appContext.metastore, appContext.bookkeeper)
+      val splitter = new OperationSplitter(conf, appContext.metastore, appContext.bookkeeper, state.getBatchId)
 
       if (isHistoricalRun)
         log.info("This is a historical run. Making all dependencies 'passive' for all jobs...")
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/jobrunner/ConcurrentJobRunnerImpl.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/jobrunner/ConcurrentJobRunnerImpl.scala
index 4122bea5c..8703c7295 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/jobrunner/ConcurrentJobRunnerImpl.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/jobrunner/ConcurrentJobRunnerImpl.scala
@@ -92,7 +92,7 @@ class ConcurrentJobRunnerImpl(runtimeConfig: RuntimeConfig,
         completedJobsChannel.send((job, Nil, isSucceeded))
       } catch {
         case ex: FatalErrorWrapper if ex.cause != null => onFatalException(ex.cause, job, isTransient)
-        case NonFatal(ex)                              => sendFailure(ex, job, isTransient)
+        case NonFatal(ex)                              => onNonFatalException(ex, job, isTransient)
         case ex: Throwable                             => onFatalException(ex, job, isTransient)
       }
     }
@@ -105,6 +105,11 @@ class ConcurrentJobRunnerImpl(runtimeConfig: RuntimeConfig,
     sendFailure(fatalEx, job, isTransient)
   }
 
+  private[core] def onNonFatalException(ex: Throwable, job: Job, isTransient: Boolean): Unit = {
+    log.error(s"${Emoji.FAILURE} Job '${job.name}' outputting to '${job.outputTable.name}' has thrown an error", ex)
+    sendFailure(ex, job, isTransient)
+  }
+
   private[core] def sendFailure(ex: Throwable, job: Job, isTransient: Boolean): Unit = {
     completedJobsChannel.send((job,
       TaskResult(job.name,
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyIncremental.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyIncremental.scala
new file mode 100644
index 000000000..97ac5c7eb
--- /dev/null
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyIncremental.scala
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.runner.splitter
+
+import za.co.absa.pramen.api.status.{MetastoreDependency, TaskRunReason}
+import za.co.absa.pramen.core.bookkeeper.Bookkeeper
+import za.co.absa.pramen.core.pipeline
+import za.co.absa.pramen.core.pipeline.TaskPreDef
+import za.co.absa.pramen.core.runner.splitter.ScheduleStrategyUtils._
+import za.co.absa.pramen.core.schedule.Schedule
+
+import java.time.LocalDate
+
+class ScheduleStrategyIncremental(lastInfoDateProcessedOpt: Option[LocalDate], hasInfoDateColumn: Boolean) extends ScheduleStrategy {
+  private val log = org.slf4j.LoggerFactory.getLogger(this.getClass)
+
+  override def getDaysToRun(
+                             outputTable: String,
+                             dependencies: Seq[MetastoreDependency],
+                             bookkeeper: Bookkeeper,
+                             infoDateExpression: String,
+                             schedule: Schedule,
+                             params: ScheduleParams,
+                             initialSourcingDateExpr: String,
+                             minimumDate: LocalDate
+                           ): Seq[TaskPreDef] = {
+    val dates = params match {
+      case ScheduleParams.Normal(runDate, trackDays, _, newOnly, lateOnly) =>
+        val infoDate = evaluateRunDate(runDate, infoDateExpression)
+        log.info(s"Normal run strategy: runDate=$runDate, infoDate=$infoDate")
+
+        val runInfoDays = if (hasInfoDateColumn) {
+          lastInfoDateProcessedOpt match {
+            case Some(lastInfoDate) =>
+              val newDays = if (lastInfoDate.isBefore(infoDate))
+                Seq(TaskPreDef(infoDate.minusDays(1), TaskRunReason.New), TaskPreDef(infoDate, TaskRunReason.New))
+              else
+                Seq(TaskPreDef(infoDate, TaskRunReason.New))
+
+              val lateDays = getLateDays(infoDate, lastInfoDate, trackDays)
+
+              if (newOnly) {
+                newDays
+              } else if (lateOnly) {
+                lateDays
+              } else {
+                lateDays ++ newDays
+              }
+            case None => Seq(TaskPreDef(infoDate, TaskRunReason.New))
+          }
+        } else {
+          lastInfoDateProcessedOpt match {
+            case Some(lastInfoDate) if lastInfoDate.isAfter(infoDate) => Seq.empty
+            case _ => Seq(TaskPreDef(infoDate, TaskRunReason.New))
+          }
+        }
+
+        if (runInfoDays.nonEmpty) {
+          log.info(s"Days to run:  ${runInfoDays.map(_.infoDate).mkString(", ")}")
+        } else {
+          log.info(s"Days to run: no days have been selected to run by the scheduler.")
+        }
+
+        runInfoDays.toList
+      case ScheduleParams.Rerun(runDate) =>
+        log.info(s"Rerun strategy for a single day: $runDate")
+        val infoDate = evaluateRunDate(runDate, infoDateExpression)
+
+        log.info(s"Rerunning '$outputTable' for date $runDate. Info date = '$infoDateExpression' = $infoDate.")
+        List(pipeline.TaskPreDef(infoDate, TaskRunReason.Rerun))
+      case ScheduleParams.Historical(dateFrom, dateTo, inverseDateOrder, mode) =>
+        log.info(s"Ranged strategy: from $dateFrom to $dateTo, mode = '${mode.toString}', minimumDate = $minimumDate")
+        getHistorical(outputTable, dateFrom, dateTo, schedule, mode, infoDateExpression, minimumDate, inverseDateOrder, bookkeeper)
+    }
+
+    filterOutPastMinimumDates(dates, minimumDate)
+  }
+
+  private[core] def getLateDays(infoDate: LocalDate, lastInfoDate: LocalDate, trackDays: Int): Seq[TaskPreDef] = {
+    val lastNewDate = infoDate.minusDays(1)
+
+    if (lastInfoDate.isBefore(lastNewDate)) {
+      val startDate = if (trackDays > 1) {
+        val trackDate = lastNewDate.minusDays(trackDays - 1)
+        val date = if (trackDate.isAfter(lastInfoDate))
+          trackDate
+        else
+          lastInfoDate
+        log.warn(s"Last ran day: $lastInfoDate. Tracking days = '$trackDate'. Catching up from '$date' until '$infoDate'.")
+        date
+      } else {
+        if (trackDays < 0) {
+          log.warn(s"Last ran day: $lastInfoDate. Catching up data until '$infoDate'.")
+          lastInfoDate
+        } else {
+          log.warn(s"Last ran day: $lastInfoDate. Not catching up since 'track.days=$trackDays'. Set it to the number of days or '-1' for infinite catching up.")
+          lastNewDate
+        }
+      }
+
+      val potentialDates = getInfoDateRange(startDate, lastNewDate.minusDays(1), "@runDate", Schedule.Incremental)
+      potentialDates.map(date => {
+        TaskPreDef(date, TaskRunReason.Late)
+      })
+    } else {
+      Seq.empty
+    }
+  }
+}
+
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyUtils.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyUtils.scala
index a62c300dd..070c7d7b5 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyUtils.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/splitter/ScheduleStrategyUtils.scala
@@ -214,7 +214,7 @@ object ScheduleStrategyUtils {
         dependency.tables.foldLeft(false)((acc, table) => {
           bookkeeper.getLatestDataChunk(table, dateFrom, dateTo) match {
             case Some(dependencyUpdated) =>
-              val isUpdatedRetrospectively = dependencyUpdated.jobFinished > lastUpdated.jobFinished
+              val isUpdatedRetrospectively = dependencyUpdated.jobFinished >= lastUpdated.jobFinished
               if (isUpdatedRetrospectively) {
                 log.warn(s"Input table '$table' has updated retrospectively${renderPeriod(Option(dateFrom), Option(dateTo))}. " +
                   s"Adding '$outputTable' to rerun for $infoDate.")
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/task/TaskRunnerBase.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/task/TaskRunnerBase.scala
index de2683a82..e3c747c4e 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/task/TaskRunnerBase.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/runner/task/TaskRunnerBase.scala
@@ -35,8 +35,8 @@ import za.co.absa.pramen.core.pipeline._
 import za.co.absa.pramen.core.state.PipelineState
 import za.co.absa.pramen.core.utils.Emoji._
 import za.co.absa.pramen.core.utils.SparkUtils._
-import za.co.absa.pramen.core.utils.{ThreadUtils, TimeUtils}
 import za.co.absa.pramen.core.utils.hive.HiveHelper
+import za.co.absa.pramen.core.utils.{ThreadUtils, TimeUtils}
 
 import java.sql.Date
 import java.time.{Instant, LocalDate}
@@ -282,7 +282,7 @@ abstract class TaskRunnerBase(conf: Config,
         Left(result)
       case Right(status) =>
         Try {
-          task.job.validate(task.infoDate, conf)
+          task.job.validate(task.infoDate, task.reason, conf)
         } match {
           case Success(validationResult) =>
             validationResult match {
@@ -335,7 +335,7 @@ abstract class TaskRunnerBase(conf: Config,
 
         val recordCountOldOpt = bookkeeper.getLatestDataChunk(task.job.outputTable.name, task.infoDate, task.infoDate).map(_.outputRecordCount)
 
-        val runResult = task.job.run(task.infoDate, conf)
+        val runResult = task.job.run(task.infoDate, task.reason, conf)
 
         val schemaChangesBeforeTransform = handleSchemaChange(runResult.data, task.job.outputTable, task.infoDate)
 
@@ -369,9 +369,9 @@ abstract class TaskRunnerBase(conf: Config,
 
         val saveResult = if (runtimeConfig.isDryRun) {
           log.warn(s"$WARNING DRY RUN mode, no actual writes to ${task.job.outputTable.name} for ${task.infoDate} will be performed.")
-          SaveResult(MetaTableStats(dfTransformed.count(), None))
+          SaveResult(MetaTableStats(Option(dfTransformed.count()), None, None))
         } else {
-          task.job.save(dfTransformed, task.infoDate, conf, started, validationResult.inputRecordsCount)
+          task.job.save(dfTransformed, task.infoDate, task.reason, conf, started, validationResult.inputRecordsCount)
         }
 
         val hiveWarnings = if (task.job.outputTable.hiveTable.nonEmpty) {
@@ -397,6 +397,7 @@ abstract class TaskRunnerBase(conf: Config,
           MetaTable.getMetaTableDef(task.job.outputTable),
           RunStatus.Succeeded(recordCountOldOpt,
             stats.recordCount,
+            stats.recordCountAppended,
             stats.dataSizeBytes,
             completionReason,
             runResult.filesRead,
@@ -453,7 +454,8 @@ abstract class TaskRunnerBase(conf: Config,
       log.warn("Skipping the interrupted exception of the killed task.")
     } else {
       pipelineState.addTaskCompletion(Seq(updatedResult))
-      addJournalEntry(task, updatedResult, pipelineState.getState().pipelineInfo)
+      if (taskResult.runStatus != RunStatus.NotRan)
+        addJournalEntry(task, updatedResult, pipelineState.getState.pipelineInfo)
     }
 
     updatedResult.runStatus
@@ -491,7 +493,7 @@ abstract class TaskRunnerBase(conf: Config,
   }
 
   private def sendNotifications(task: Task, result: TaskResult): Seq[NotificationFailure] = {
-    val pipelineInfo = pipelineState.getState().pipelineInfo
+    val pipelineInfo = pipelineState.getState.pipelineInfo
     task.job.notificationTargets.flatMap(notificationTarget => sendNotifications(task, result, notificationTarget, pipelineInfo))
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/schedule/Schedule.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/schedule/Schedule.scala
index 3d625bc1c..702846fd9 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/schedule/Schedule.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/schedule/Schedule.scala
@@ -31,6 +31,12 @@ object Schedule {
   val SCHEDULE_DAYS_OF_WEEK_KEY = "schedule.days.of.week"
   val SCHEDULE_DAYS_OF_MONTH_KEY = "schedule.days.of.month"
 
+  case object Incremental extends Schedule {
+    def isEnabled(day: LocalDate): Boolean = true
+
+    override def toString: String = "incremental"
+  }
+
   case class EveryDay() extends Schedule {
     def isEnabled(day: LocalDate): Boolean = true
 
@@ -58,10 +64,11 @@ object Schedule {
 
   def fromConfig(conf: Config): Schedule = {
     conf.getString(SCHEDULE_TYPE_KEY) match {
-      case "daily"   => EveryDay()
-      case "weekly"  => Weekly(getDaysOfWeek(conf))
-      case "monthly" => Monthly(getDaysOfMonth(conf))
-      case s         => throw new IllegalArgumentException(s"Unknown schedule type: $s")
+      case "incremental"   => Incremental
+      case "daily"         => EveryDay()
+      case "weekly"        => Weekly(getDaysOfWeek(conf))
+      case "monthly"       => Monthly(getDaysOfMonth(conf))
+      case s               => throw new IllegalArgumentException(s"Unknown schedule type: $s")
     }
   }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/JdbcSource.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/JdbcSource.scala
index 545219308..b54404921 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/JdbcSource.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/JdbcSource.scala
@@ -20,6 +20,7 @@ import com.typesafe.config.Config
 import org.apache.spark.sql.SparkSession
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
 import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig
 import za.co.absa.pramen.core.reader.{JdbcUrlSelector, TableReaderJdbc, TableReaderJdbcNative}
 
@@ -34,6 +35,10 @@ class JdbcSource(sourceConfig: Config,
 
   override def hasInfoDateColumn(query: Query): Boolean = jdbcReaderConfig.hasInfoDate
 
+  override def getOffsetInfo: Option[OffsetInfo] = {
+    jdbcReaderConfig.offsetInfoOpt
+  }
+
   override def getRecordCount(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long = {
     val reader = getReader(query, isCountQuery = true)
 
@@ -48,6 +53,14 @@ class JdbcSource(sourceConfig: Config,
     SourceResult(df)
   }
 
+  override def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): SourceResult = {
+    val reader = getReader(query, isCountQuery = false)
+
+    val df = reader.getIncrementalData(query, onlyForInfoDate, offsetFrom, offsetTo, columns)
+
+    SourceResult(df)
+  }
+
   private[core] def getReader(query: Query, isCountQuery: Boolean): TableReader = {
     val jdbcConfig = TableReaderJdbcNative.getJdbcConfig(jdbcReaderConfig, sourceConfig)
     val jdbcReaderConfigNative = jdbcReaderConfig.copy(jdbcConfig = jdbcConfig)
@@ -76,9 +89,9 @@ class JdbcSource(sourceConfig: Config,
   }
 }
 
-object JdbcSource extends ExternalChannelFactory[JdbcSource] {
-  override def apply(conf: Config, parentPath: String, spark: SparkSession): JdbcSource = {
-    val tableReaderJdbc = TableReaderJdbcConfig.load(conf)
+object JdbcSource extends ExternalChannelFactoryV2[JdbcSource] {
+  override def apply(conf: Config, workflowConf: Config, parentPath: String, spark: SparkSession): JdbcSource = {
+    val tableReaderJdbc = TableReaderJdbcConfig.load(conf, workflowConf)
 
     new JdbcSource(conf, parentPath, tableReaderJdbc)(spark)
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/LocalSparkSource.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/LocalSparkSource.scala
index 37e8f95f3..c81b80b7c 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/LocalSparkSource.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/LocalSparkSource.scala
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.{Path => HadoopPath}
 import org.apache.spark.sql.SparkSession
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.core.utils.traverser.{FsTraverser, FsTraverserLocal}
 import za.co.absa.pramen.core.utils.{ConfigUtils, FsUtils}
 
@@ -99,6 +100,8 @@ class LocalSparkSource(sparkSource: SparkSource,
 
     tempPath.toString
   }
+
+  override def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): SourceResult = ???
 }
 
 object LocalSparkSource extends ExternalChannelFactoryV2[LocalSparkSource] {
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/RawFileSource.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/RawFileSource.scala
index 119c11ced..910c2709f 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/RawFileSource.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/RawFileSource.scala
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
 import org.apache.spark.sql.SparkSession
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.core.utils.{ConfigUtils, FsUtils}
 
 import java.io.FileNotFoundException
@@ -138,6 +139,8 @@ class RawFileSource(val sourceConfig: Config,
       files.toSeq
     }
   }
+
+  override def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): SourceResult = ???
 }
 
 object RawFileSource extends ExternalChannelFactory[RawFileSource] {
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/SparkSource.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/SparkSource.scala
index 49f8f9a19..308be2af1 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/source/SparkSource.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/source/SparkSource.scala
@@ -18,12 +18,15 @@ package za.co.absa.pramen.core.source
 
 import com.typesafe.config.Config
 import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.slf4j.LoggerFactory
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
+import za.co.absa.pramen.api.sql.SqlColumnType
 import za.co.absa.pramen.core.config.Keys.KEYS_TO_REDACT
 import za.co.absa.pramen.core.reader.TableReaderSpark
-import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig.{HAS_INFO_DATE, INFORMATION_DATE_COLUMN, getInfoDateFormat}
+import za.co.absa.pramen.core.reader.model.OffsetInfoParser
+import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig._
 import za.co.absa.pramen.core.utils.{ConfigUtils, FsUtils}
 
 import java.time.LocalDate
@@ -32,7 +35,9 @@ class SparkSource(val format: Option[String],
                   val schema: Option[String],
                   val hasInfoDateCol: Boolean,
                   val infoDateColumn: String,
+                  val infoDateType: SqlColumnType,
                   val infoDateFormat: String,
+                  val offsetInfo: Option[OffsetInfo],
                   val sourceConfig: Config,
                   val options: Map[String, String])(implicit spark: SparkSession) extends Source {
   private val log = LoggerFactory.getLogger(this.getClass)
@@ -41,6 +46,10 @@ class SparkSource(val format: Option[String],
 
   override def hasInfoDateColumn(query: Query): Boolean = hasInfoDateCol
 
+  override def getOffsetInfo: Option[OffsetInfo] = {
+    offsetInfo
+  }
+
   override def getRecordCount(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long = {
     val reader = getReader(query)
 
@@ -52,14 +61,7 @@ class SparkSource(val format: Option[String],
 
     val df = reader.getData(query, infoDateBegin, infoDateEnd, columns)
 
-    val filesRead = query match {
-      case _: Query.Table   => df.inputFiles
-      case _: Query.Sql     => Array.empty[String]
-      case Query.Path(path) =>
-        val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, path)
-        fsUtils.getHadoopFiles(new Path(path)).map(_.getPath.toString).sorted
-      case other            => throw new IllegalArgumentException(s"'${other.name}' is not supported by the Spark source. Use 'path', 'table' or 'sql' instead.")
-    }
+    val filesRead = getFilesRead(query, df)
 
     SourceResult(df, filesRead)
   }
@@ -68,13 +70,13 @@ class SparkSource(val format: Option[String],
     val tableReader = query match {
       case Query.Table(table) =>
         log.info(s"Using TableReaderSpark to read table: $table")
-        new TableReaderSpark(format, schema, hasInfoDateCol, infoDateColumn, infoDateFormat, options)
+        new TableReaderSpark(format, schema, hasInfoDateCol, infoDateColumn, infoDateType, infoDateFormat, getOffsetInfo, options)
       case Query.Sql(sql)     =>
         log.info(s"Using TableReaderSpark to read SQL for: $sql")
-        new TableReaderSpark(format, schema, hasInfoDateCol, infoDateColumn, infoDateFormat, options)
+        new TableReaderSpark(format, schema, hasInfoDateCol, infoDateColumn, infoDateType, infoDateFormat, getOffsetInfo, options)
       case Query.Path(path)   =>
         log.info(s"Using TableReaderSpark to read '$format' from: $path")
-        new TableReaderSpark(format, schema, hasInfoDateCol, infoDateColumn, infoDateFormat, options)
+        new TableReaderSpark(format, schema, hasInfoDateCol, infoDateColumn, infoDateType, infoDateFormat, getOffsetInfo, options)
       case other              => throw new IllegalArgumentException(s"'${other.name}' is not supported by the Spark source. Use 'path', 'table' or 'sql' instead.")
     }
 
@@ -86,6 +88,27 @@ class SparkSource(val format: Option[String],
     schema.foreach(s => log.info(s"Using schema: $s"))
     tableReader
   }
+
+  override def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): SourceResult = {
+    val reader = getReader(query)
+
+    val df = reader.getIncrementalData(query, onlyForInfoDate, offsetFrom, offsetTo, columns)
+
+    val filesRead = getFilesRead(query, df)
+
+    SourceResult(df, filesRead)
+  }
+
+  private def getFilesRead(query: Query, df: DataFrame): Seq[String] = {
+    query match {
+      case _: Query.Table   => df.inputFiles
+      case _: Query.Sql     => Array.empty[String]
+      case Query.Path(path) =>
+        val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, path)
+        fsUtils.getHadoopFiles(new Path(path)).map(_.getPath.toString).sorted
+      case other            => throw new IllegalArgumentException(s"'${other.name}' is not supported by the Spark source. Use 'path', 'table' or 'sql' instead.")
+    }
+  }
 }
 
 object SparkSource extends ExternalChannelFactory[SparkSource] {
@@ -97,14 +120,18 @@ object SparkSource extends ExternalChannelFactory[SparkSource] {
     val schema = ConfigUtils.getOptionString(conf, SCHEMA)
 
     val hasInfoDate = conf.hasPath(HAS_INFO_DATE) && conf.getBoolean(HAS_INFO_DATE)
-    val (infoDateColumn, infoDateFormat) = if (hasInfoDate) {
-      (conf.getString(INFORMATION_DATE_COLUMN), getInfoDateFormat(conf))
+    val (infoDateColumn, infoDateType, infoDateFormat) = if (hasInfoDate) {
+      val infoDateTypeStr = ConfigUtils.getOptionString(conf, INFORMATION_DATE_TYPE).getOrElse("date")
+      val infoDateType = SqlColumnType.fromString(infoDateTypeStr).getOrElse(throw new IllegalArgumentException(s"Unknown info date type: $infoDateTypeStr"))
+      (conf.getString(INFORMATION_DATE_COLUMN), infoDateType, getInfoDateFormat(conf))
     } else {
-      ("", "")
+      ("", SqlColumnType.DATE, "")
     }
 
+    val offsetInfoOpt = OffsetInfoParser.fromConfig(conf)
+
     val options = ConfigUtils.getExtraOptions(conf, "option")
 
-    new SparkSource(format, schema, hasInfoDate, infoDateColumn, infoDateFormat, conf, options)(spark)
+    new SparkSource(format, schema, hasInfoDate, infoDateColumn, infoDateType, infoDateFormat, offsetInfoOpt, conf, options)(spark)
   }
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala
index 21a9223d9..f51fc2042 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 class SqlGeneratorDb2(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
@@ -93,6 +94,19 @@ class SqlGeneratorDb2(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig)
     }
   }
 
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition TIMESTAMP('$tsLiteral')"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala
index fb232f928..bfa205b70 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala
@@ -18,11 +18,12 @@ package za.co.absa.pramen.core.sql
 
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.slf4j.LoggerFactory
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 import za.co.absa.pramen.core.sql.dialects.DenodoDialect
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 object SqlGeneratorDenodo {
   private val log = LoggerFactory.getLogger(this.getClass)
@@ -94,7 +95,7 @@ class SqlGeneratorDenodo(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
     }
   }
 
-  def getDateLiteral(date: LocalDate): String = {
+  override def getDateLiteral(date: LocalDate): String = {
     sqlConfig.infoDateType match {
       case SqlColumnType.DATE =>
         val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
@@ -111,6 +112,19 @@ class SqlGeneratorDenodo(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
     }
   }
 
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition TIMESTAMP '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     // limit.map(n => s" LIMIT $n").getOrElse("")
     // Adding limits to Denodo queries seems is not supported
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala
index 700d718f3..d15d278af 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 class SqlGeneratorGeneric(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
@@ -93,6 +94,19 @@ class SqlGeneratorGeneric(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConf
     }
   }
 
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala
index 7cdc84e71..aa30e9f6a 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala
@@ -18,11 +18,12 @@ package za.co.absa.pramen.core.sql
 
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.slf4j.LoggerFactory
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 import za.co.absa.pramen.core.sql.dialects.HiveDialect
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 object SqlGeneratorHive {
   private val log = LoggerFactory.getLogger(this.getClass)
@@ -107,6 +108,19 @@ class SqlGeneratorHive(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig)
     }
   }
 
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala
index d5478c715..e8f7e2b4f 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 class SqlGeneratorHsqlDb(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
@@ -93,6 +94,19 @@ class SqlGeneratorHsqlDb(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
     }
   }
 
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition TIMESTAMP '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala
index 1d5064454..92603b707 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala
@@ -16,16 +16,18 @@
 
 package za.co.absa.pramen.core.sql
 
-import za.co.absa.pramen.api.sql.SqlGeneratorBase.{needsEscaping, validateIdentifier}
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetValue}
+import za.co.absa.pramen.api.sql.SqlGeneratorBase.{needsEscaping, validateIdentifier, validateOffsetValue}
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGenerator}
 import za.co.absa.pramen.core.utils.MutableStack
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 import scala.collection.mutable.ListBuffer
 
 class SqlGeneratorMicrosoft(sqlConfig: SqlConfig) extends SqlGenerator {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
+  private val timestampMsDbFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
   private val isIso = sqlConfig.dateFormatApp.toLowerCase.startsWith("yyyy-mm-dd")
 
   // 23 is "yyyy-MM-dd", see https://www.mssqltips.com/sqlservertip/1145/date-and-time-conversions-using-sql-server/
@@ -127,6 +129,64 @@ class SqlGeneratorMicrosoft(sqlConfig: SqlConfig) extends SqlGenerator {
     }
   }
 
+  override def getDataQueryIncremental(tableName: String,
+                                       onlyForInfoDate: Option[LocalDate],
+                                       offsetFromOpt: Option[OffsetValue],
+                                       offsetToOpt: Option[OffsetValue],
+                                       columns: Seq[String]): String = {
+    if (sqlConfig.offsetInfo.isEmpty)
+      throw new IllegalArgumentException(s"Offset information is not configured for database table: $tableName.")
+
+    val dataQuery = onlyForInfoDate match {
+      case Some(infoDate) => getDataQuery(tableName, infoDate, infoDate, columns, None)
+      case None => getDataQuery(tableName, columns, None)
+    }
+
+    val offsetWhere = getOffsetWhereClause(sqlConfig.offsetInfo.get, offsetFromOpt, offsetToOpt)
+
+    if (offsetWhere.nonEmpty) {
+      if (onlyForInfoDate.isEmpty) {
+        s"$dataQuery WHERE $offsetWhere"
+      } else {
+        s"$dataQuery AND $offsetWhere"
+      }
+    } else {
+      dataQuery
+    }
+  }
+
+  private[core] def getOffsetWhereClause(offsetInfo: OffsetInfo, offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue]): String = {
+    val offsetColumn = escape(offsetInfo.offsetColumn)
+
+    (offsetFromOpt, offsetToOpt) match {
+      case (Some(offsetFrom), Some(offsetTo)) =>
+        validateOffsetValue(offsetFrom)
+        validateOffsetValue(offsetTo)
+        s"${getOffsetWhereCondition(offsetColumn, ">=", offsetFrom)} AND ${getOffsetWhereCondition(offsetColumn, "<=", offsetTo)}"
+      case (Some(offsetFrom), None) =>
+        validateOffsetValue(offsetFrom)
+        s"${getOffsetWhereCondition(offsetColumn, ">", offsetFrom)}"
+      case (None, Some(offsetTo)) =>
+        validateOffsetValue(offsetTo)
+        s"${getOffsetWhereCondition(offsetColumn, "<=", offsetTo)}"
+      case (None, None) =>
+        ""
+    }
+  }
+
+  private[core] def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampMsDbFormatter.format(ldt)
+        s"$column $condition '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s"TOP $n ").getOrElse("")
   }
@@ -232,4 +292,5 @@ class SqlGeneratorMicrosoft(sqlConfig: SqlConfig) extends SqlGenerator {
 
     output.toSeq
   }
+
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMySQL.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMySQL.scala
index f0c74c2ba..f7dab7195 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMySQL.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMySQL.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 class SqlGeneratorMySQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
@@ -57,22 +58,17 @@ class SqlGeneratorMySQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig
   }
 
   override def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String = {
-    val dateBeginLit = getDateLiteral(dateBegin)
-    val dateEndLit = getDateLiteral(dateEnd)
-
-    val dateTypes: Array[SqlColumnType] = Array(SqlColumnType.DATETIME)
+    if (sqlConfig.infoDateType == SqlColumnType.DATETIME) {
+      s"$infoDateColumn >= '$dateBegin 00:00:00' AND $infoDateColumn < '${dateEnd.plusDays(1)} 00:00:00'"
+    } else {
+      val dateBeginLit = getDateLiteral(dateBegin)
+      val dateEndLit = getDateLiteral(dateEnd)
 
-    val infoDateColumnAdjusted =
-      if (dateTypes.contains(sqlConfig.infoDateType)) {
-        s"DATE($infoDateColumn)"
+      if (dateBeginLit == dateEndLit) {
+        s"$infoDateColumn = $dateBeginLit"
       } else {
-        infoDateColumn
+        s"$infoDateColumn >= $dateBeginLit AND $infoDateColumn <= $dateEndLit"
       }
-
-    if (dateBeginLit == dateEndLit) {
-      s"$infoDateColumnAdjusted = $dateBeginLit"
-    } else {
-      s"$infoDateColumnAdjusted >= $dateBeginLit AND $infoDateColumnAdjusted <= $dateEndLit"
     }
   }
 
@@ -93,6 +89,20 @@ class SqlGeneratorMySQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig
     }
   }
 
+
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala
index 5fd9b4a6b..0e4fac22d 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 class SqlGeneratorOracle(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
@@ -97,6 +98,19 @@ class SqlGeneratorOracle(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
     }
   }
 
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition TIMESTAMP '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int], hasWhere: Boolean): String = {
     if (hasWhere) {
       limit.map(n => s" AND ROWNUM <= $n").getOrElse("")
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala
index 79c58d965..444aa0087 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.sql
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
 
 class SqlGeneratorPostgreSQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
@@ -57,22 +58,17 @@ class SqlGeneratorPostgreSQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlC
   }
 
   override def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String = {
-    val dateBeginLit = getDateLiteral(dateBegin)
-    val dateEndLit = getDateLiteral(dateEnd)
-
-    val dateTypes: Array[SqlColumnType] = Array(SqlColumnType.DATETIME)
+    if (sqlConfig.infoDateType == SqlColumnType.DATETIME) {
+      s"$infoDateColumn >= '$dateBegin' AND $infoDateColumn < '${dateEnd.plusDays(1)}'"
+    } else {
+      val dateBeginLit = getDateLiteral(dateBegin)
+      val dateEndLit = getDateLiteral(dateEnd)
 
-    val infoDateColumnAdjusted =
-      if (dateTypes.contains(sqlConfig.infoDateType)) {
-        s"CAST($infoDateColumn AS DATE)"
+      if (dateBeginLit == dateEndLit) {
+        s"$infoDateColumn = $dateBeginLit"
       } else {
-        infoDateColumn
+        s"$infoDateColumn >= $dateBeginLit AND $infoDateColumn <= $dateEndLit"
       }
-
-    if (dateBeginLit == dateEndLit) {
-      s"$infoDateColumnAdjusted = $dateBeginLit"
-    } else {
-      s"$infoDateColumnAdjusted >= $dateBeginLit AND $infoDateColumnAdjusted <= $dateEndLit"
     }
   }
 
@@ -83,7 +79,7 @@ class SqlGeneratorPostgreSQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlC
         s"date'$dateStr'"
       case SqlColumnType.DATETIME =>
         val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
-        s"date'$dateStr'"
+        s"'$dateStr'"
       case SqlColumnType.STRING =>
         val dateStr = dateFormatterApp.format(date)
         s"'$dateStr'"
@@ -93,6 +89,20 @@ class SqlGeneratorPostgreSQL(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlC
     }
   }
 
+
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampGenericDbFormatter.format(ldt)
+        s"$column $condition '$tsLiteral'"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
+
   private def getLimit(limit: Option[Int]): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala
index 24c6e1236..7b878157c 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala
@@ -18,12 +18,14 @@ package za.co.absa.pramen.core.sql
 
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.slf4j.LoggerFactory
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
 import za.co.absa.pramen.core.sql.dialects.SasDialect
 
 import java.sql.{Connection, ResultSet}
-import java.time.LocalDate
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDate, LocalDateTime}
+import java.util.Locale
 import scala.collection.mutable.ListBuffer
 
 object SqlGeneratorSas {
@@ -41,6 +43,7 @@ object SqlGeneratorSas {
 
 class SqlGeneratorSas(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
   private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
+  private val timestampSasDbFormatter = DateTimeFormatter.ofPattern("ddMMMyyyy:HH:mm:ss.SSS", Locale.ENGLISH)
 
   override val beginEndEscapeChars: (Char, Char) = ('\'', '\'')
 
@@ -153,14 +156,49 @@ class SqlGeneratorSas(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig)
   }
 
   override final def quoteSingleIdentifier(identifier: String): String = {
-    if (identifier.startsWith(".") && identifier.toLowerCase.endsWith("n")) {
+    if (identifier.startsWith("'") && identifier.toLowerCase.endsWith("'n")) {
       identifier
     } else {
       s"'$identifier'n"
     }
   }
 
+  override final def unquoteSingleIdentifier(identifier: String): String = {
+    val (escapeBegin, escapeEnd) = beginEndEscapeChars
+
+    if (identifier.startsWith(s"$escapeBegin") && identifier.endsWith(s"${escapeEnd}n") && identifier.length > 3) {
+      identifier.substring(1, identifier.length - 2)
+    } else {
+      identifier
+    }
+  }
+
+  override def quote(identifier: String): String = {
+    splitComplexSasIdentifier(identifier).map(quoteSingleIdentifier).mkString(".")
+  }
+
+  override def unquote(identifier: String): String = {
+    splitComplexSasIdentifier(identifier).map(unquoteSingleIdentifier).mkString(".")
+  }
+
+  private def splitComplexSasIdentifier(identifier: String): Seq[String] = {
+    identifier.split('.')
+  }
+
   private def getLimit(limit: Option[Int], hasWhere: Boolean): String = {
     limit.map(n => s" LIMIT $n").getOrElse("")
   }
+
+  override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
+    offset match {
+      case OffsetValue.DateTimeValue(ts) =>
+        val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
+        val tsLiteral = timestampSasDbFormatter.format(ldt)
+        s"$column $condition '$tsLiteral'dt"
+      case OffsetValue.IntegralValue(value) =>
+        s"$column $condition $value"
+      case OffsetValue.StringValue(value) =>
+        s"$column $condition '$value'"
+    }
+  }
 }
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineState.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineState.scala
index 075c6dd08..cd61e3efe 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineState.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineState.scala
@@ -19,7 +19,9 @@ package za.co.absa.pramen.core.state
 import za.co.absa.pramen.api.status.{PipelineStateSnapshot, TaskResult}
 
 trait PipelineState extends AutoCloseable {
-  def getState(): PipelineStateSnapshot
+  def getState: PipelineStateSnapshot
+
+  def getBatchId: Long
 
   def setShutdownHookCanRun(): Unit
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineStateImpl.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineStateImpl.scala
index fccc62af7..5ec62bc21 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineStateImpl.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineStateImpl.scala
@@ -49,6 +49,7 @@ class PipelineStateImpl(implicit conf: Config, notificationBuilder: Notification
   private val sendEmailIfNoNewData: Boolean = conf.getBoolean(EMAIL_IF_NO_CHANGES)
   private val hookConfig = HookConfig.fromConfig(conf)
   private var pipelineNotificationTargets: Seq[PipelineNotificationTarget] = Seq.empty
+  private val batchId = Instant.now().toEpochMilli
 
   // State
   private val startedInstant = Instant.now
@@ -79,7 +80,7 @@ class PipelineStateImpl(implicit conf: Config, notificationBuilder: Notification
     pipelineNotificationTargets = PipelineNotificationTargetFactory.fromConfig(conf)
   }
 
-  override def getState(): PipelineStateSnapshot = synchronized {
+  override def getState: PipelineStateSnapshot = synchronized {
     val appException =  if (!exitedNormally && failureException.isEmpty && signalException.isDefined) {
       signalException
     } else
@@ -114,6 +115,7 @@ class PipelineStateImpl(implicit conf: Config, notificationBuilder: Notification
         pipelineId,
         tenant
       ),
+      batchId,
       isFinished,
       warningFlag,
       exitedNormally,
@@ -125,6 +127,8 @@ class PipelineStateImpl(implicit conf: Config, notificationBuilder: Notification
     )
   }
 
+  override def getBatchId: Long = batchId
+
   override def setShutdownHookCanRun(): Unit = synchronized {
     customShutdownHookCanRun = true
   }
@@ -228,7 +232,7 @@ class PipelineStateImpl(implicit conf: Config, notificationBuilder: Notification
   }
 
   private[state] def sendPipelineNotifications(): Unit = {
-    pipelineNotificationTargets.foreach(notificationTarget => sendCustomNotification(notificationTarget, getState(), taskResults.toSeq))
+    pipelineNotificationTargets.foreach(notificationTarget => sendCustomNotification(notificationTarget, getState, taskResults.toSeq))
   }
 
   private[state] def sendCustomNotification(pipelineNotificationTarget: PipelineNotificationTarget, pipelineStateSnapshot: PipelineStateSnapshot, taskResults: Seq[TaskResult]): Unit = {
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala
index 3bb3f911b..6e63ebb61 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/transformers/IdentityTransformer.scala
@@ -32,7 +32,7 @@ class IdentityTransformer extends Transformer {
 
     val tableName = options.getOrElse(INPUT_TABLE_KEY, options(INPUT_TABLE_LEGACY_KEY))
 
-    val df = metastore.getTable(tableName, Option(infoDate), Option(infoDate))
+    val df = metastore.getCurrentBatch(tableName)
 
     if (emptyAllowed || !df.isEmpty) {
       Reason.Ready
@@ -46,7 +46,7 @@ class IdentityTransformer extends Transformer {
                    options: Map[String, String]): DataFrame = {
     val tableName = options.getOrElse(INPUT_TABLE_KEY, options(INPUT_TABLE_LEGACY_KEY))
 
-    metastore.getTable(tableName, Option(infoDate), Option(infoDate))
+    metastore.getCurrentBatch(tableName)
   }
 }
 
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SlickUtils.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SlickUtils.scala
index 9ea66bdb7..ba1fdae6a 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SlickUtils.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SlickUtils.scala
@@ -27,6 +27,8 @@ object SlickUtils {
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
+  private val WARN_IF_LONGER_MS = 1000L
+
   /**
     * Synchronously executes a query against a JDBC connection.
     * - Handles exceptions
@@ -47,7 +49,7 @@ object SlickUtils {
       val finish = Instant.now
 
       val elapsedTime = TimeUtils.prettyPrintElapsedTimeShort(finish.toEpochMilli - start.toEpochMilli)
-      if (Duration.between(start, finish).toMillis > 1000L) {
+      if (Duration.between(start, finish).toMillis > WARN_IF_LONGER_MS) {
         log.warn(s"Query execution time: $elapsedTime. SQL: $sql")
       } else {
         log.debug(s"Query execution time: $elapsedTime. SQL: $sql")
@@ -79,7 +81,39 @@ object SlickUtils {
       val finish = Instant.now
 
       val elapsedTime = TimeUtils.prettyPrintElapsedTimeShort(finish.toEpochMilli - start.toEpochMilli)
-      if (Duration.between(start, finish).toMillis > 1000L) {
+      if (Duration.between(start, finish).toMillis > WARN_IF_LONGER_MS) {
+        log.warn(s"Count execution time: $elapsedTime. SQL: $sql")
+      } else {
+        log.debug(s"Count execution time: $elapsedTime. SQL: $sql")
+      }
+
+      result
+    } catch {
+      case NonFatal(ex) => throw new RuntimeException(s"Error executing an SQL query: $sql", ex)
+    }
+  }
+
+  /**
+    * Synchronously executes a MAX query against a string column and JDBC connection.
+    * - Handles exceptions
+    * - Logs SQL statements generated by the framework
+    * - Measures execution time
+    *
+    * @param db A database
+    * @param rep A result of query
+    * @return The result of the query.
+    */
+  def executeMaxString(db: Database, rep: Rep[Option[String]]): Option[String] = {
+    val action = rep.result
+    val sql = action.statements.mkString("; ")
+
+    try {
+      val start = Instant.now
+      val result = db.run(action).execute()
+      val finish = Instant.now
+
+      val elapsedTime = TimeUtils.prettyPrintElapsedTimeShort(finish.toEpochMilli - start.toEpochMilli)
+      if (Duration.between(start, finish).toMillis > WARN_IF_LONGER_MS) {
         log.warn(s"Count execution time: $elapsedTime. SQL: $sql")
       } else {
         log.debug(s"Count execution time: $elapsedTime. SQL: $sql")
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SparkUtils.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SparkUtils.scala
index 08288ecc9..0b7bf2cac 100644
--- a/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SparkUtils.scala
+++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/SparkUtils.scala
@@ -31,7 +31,6 @@ import za.co.absa.pramen.core.pipeline.TransformExpression
 import java.io.ByteArrayOutputStream
 import java.time.format.DateTimeFormatter
 import java.time.{Instant, LocalDate}
-import scala.annotation.tailrec
 import scala.collection.mutable.ArrayBuffer
 import scala.reflect.runtime.universe._
 import scala.util.{Failure, Success, Try}
@@ -128,13 +127,7 @@ object SparkUtils {
     */
   def compareSchemas(schema1: StructType, schema2: StructType): List[FieldChange] = {
     def dataTypeToString(dt: DataType, metadata: Metadata): String = {
-      val maxLength = if (metadata.contains(MAX_LENGTH_METADATA_KEY)) {
-        Try {
-          metadata.getLong(MAX_LENGTH_METADATA_KEY).toInt
-        }.getOrElse(0)
-      } else {
-        0
-      }
+      val maxLength = getLengthFromMetadata(metadata).getOrElse(0)
 
       dt match {
         case _: StructType | _: ArrayType    => dt.simpleString
@@ -240,11 +233,9 @@ object SparkUtils {
     def transformPrimitive(dataType: DataType, field: StructField): DataType = {
       dataType match {
         case _: StringType =>
-          if (field.metadata.contains(MAX_LENGTH_METADATA_KEY)) {
-            val length = field.metadata.getLong(MAX_LENGTH_METADATA_KEY).toInt
-            VarcharType(length)
-          } else {
-            StringType
+          getLengthFromMetadata(field.metadata) match {
+            case Some(n) => VarcharType(n)
+            case None => StringType
           }
         case _ =>
           dataType
@@ -266,6 +257,26 @@ object SparkUtils {
     transformStruct(schema)
   }
 
+  def getLengthFromMetadata(metadata: Metadata): Option[Int] = {
+    if (metadata.contains(MAX_LENGTH_METADATA_KEY)) {
+      val try1 = Try {
+        val length = metadata.getLong(MAX_LENGTH_METADATA_KEY).toInt
+        Option(length)
+      }
+      val try2 = if (try1.isFailure) {
+        Try {
+          val length = metadata.getString(MAX_LENGTH_METADATA_KEY).toInt
+          Option(length)
+        }
+      } else {
+        try1
+      }
+      try2.getOrElse(None)
+    } else {
+      None
+    }
+  }
+
   /**
     * Removes metadata of nested fields to make DDL compatible with some Hive-like catalogs.
     * In addition, removes the nullability flag for all fields.
diff --git a/pramen/project/plugins.sbt b/pramen/project/plugins.sbt
index 217f82fad..247057853 100644
--- a/pramen/project/plugins.sbt
+++ b/pramen/project/plugins.sbt
@@ -23,8 +23,8 @@ addDependencyTreePlugin
 
 // sbt-jacoco - workaround related dependencies required to download
 val ow2Version = "9.5"
-val jacocoVersion = "0.8.10-absa.1"
-val sbtJacocoVersion = "3.4.1-absa.3"
+val jacocoVersion = "0.8.11-absa.1"
+val sbtJacocoVersion = "3.4.1-absa.4"
 val scalaArmVersion = "2.0"
 
 def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar"
diff --git a/pramen/version.sbt b/pramen/version.sbt
index b5d8d385c..ad2ffeddb 100644
--- a/pramen/version.sbt
+++ b/pramen/version.sbt
@@ -1,1 +1,1 @@
-ThisBuild / version := "1.9.10-SNAPSHOT"
+ThisBuild / version := "1.10.0-SNAPSHOT"


Test patch:
diff --git a/pramen/api/src/test/scala/za/co/absa/pramen/api/offset/OffsetTypeSuite.scala b/pramen/api/src/test/scala/za/co/absa/pramen/api/offset/OffsetTypeSuite.scala
new file mode 100644
index 000000000..8edb2e61c
--- /dev/null
+++ b/pramen/api/src/test/scala/za/co/absa/pramen/api/offset/OffsetTypeSuite.scala
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.api.offset
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{LongType, StringType}
+import org.scalatest.wordspec.AnyWordSpec
+
+class OffsetTypeSuite extends AnyWordSpec {
+  "OffsetType" should {
+    "be able to create a DateTimeType instance" in {
+      val offsetValue = OffsetType.DateTimeType
+      assert(offsetValue.dataTypeString == "datetime")
+      assert(offsetValue.getSparkCol(col("a")) == concat(unix_timestamp(col("a")), date_format(col("a"), "SSS")).cast(LongType))
+    }
+
+    "be able to create a IntegralType instance" in {
+      val offsetValue = OffsetType.IntegralType
+      assert(offsetValue.dataTypeString == "integral")
+      assert(offsetValue.getSparkCol(col("a")) == col("a").cast(LongType))
+    }
+
+    "be able to create a StringType instance" in {
+      val offsetValue = OffsetType.StringType
+      assert(offsetValue.dataTypeString == "string")
+      assert(offsetValue.getSparkCol(col("a")) == col("a").cast(StringType))
+    }
+  }
+
+  "fromString" should {
+    "be able to create a DateTimeType instance from a string" in {
+      val offsetValue = OffsetType.fromString("datetime")
+      assert(offsetValue.dataTypeString == "datetime")
+    }
+
+    "be able to create a IntegralType instance from a string" in {
+      val offsetValue = OffsetType.fromString("integral")
+      assert(offsetValue.dataTypeString == "integral")
+    }
+
+    "be able to create a StringType instance from a string" in {
+      val offsetValue = OffsetType.fromString("string")
+      assert(offsetValue.dataTypeString == "string")
+    }
+
+    "throw an exception when trying to create an instance from a string with an unknown type" in {
+      assertThrows[IllegalArgumentException] {
+        OffsetType.fromString("unknown")
+      }
+    }
+  }
+}
diff --git a/pramen/api/src/test/scala/za/co/absa/pramen/api/offset/OffsetValueSuite.scala b/pramen/api/src/test/scala/za/co/absa/pramen/api/offset/OffsetValueSuite.scala
new file mode 100644
index 000000000..9eb873b7f
--- /dev/null
+++ b/pramen/api/src/test/scala/za/co/absa/pramen/api/offset/OffsetValueSuite.scala
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.api.offset
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{LongType, StringType}
+import org.scalatest.wordspec.AnyWordSpec
+
+import java.time.Instant
+
+class OffsetValueSuite extends AnyWordSpec {
+  "OffsetValue" should {
+    "be able to create a DateTimeType instance" in {
+      val offsetValue = OffsetValue.DateTimeValue(Instant.ofEpochMilli(1726564198000L))
+      assert(offsetValue.dataType == OffsetType.DateTimeType)
+      assert(offsetValue.valueString == "1726564198000")
+      assert(offsetValue.getSparkLit == lit(1726564198000L))
+      assert(offsetValue.dataType.getSparkCol(col("a")) == concat(unix_timestamp(col("a")), date_format(col("a"), "SSS")).cast(LongType))
+    }
+
+    "be able to create a IntegralValue instance" in {
+      val offsetValue = OffsetValue.IntegralValue(42)
+      assert(offsetValue.dataType == OffsetType.IntegralType)
+      assert(offsetValue.valueString == "42")
+      assert(offsetValue.getSparkLit == lit(42L))
+      assert(offsetValue.dataType.getSparkCol(col("a")) == col("a").cast(LongType))
+    }
+
+    "be able to create a StringValue instance" in {
+      val offsetValue = OffsetValue.StringValue("foo")
+      assert(offsetValue.dataType == OffsetType.StringType)
+      assert(offsetValue.valueString == "foo")
+      assert(offsetValue.getSparkLit == lit("foo"))
+      assert(offsetValue.dataType.getSparkCol(col("a")) == col("a").cast(StringType))
+    }
+  }
+
+  "fromString" should {
+    "return None for an empty string" in {
+      val offsetValue = OffsetValue.fromString("datetime", "")
+      assert(offsetValue.isEmpty)
+    }
+
+    "be able to create a DateTimeType instance from a string" in {
+      val offsetValue = OffsetValue.fromString("datetime", "1726552310000")
+      assert(offsetValue.get.dataType == OffsetType.DateTimeType)
+      assert(offsetValue.get.valueString == "1726552310000")
+    }
+
+    "be able to create a IntegralType instance from a string" in {
+      val offsetValue = OffsetValue.fromString("integral", "42")
+      assert(offsetValue.get.dataType == OffsetType.IntegralType)
+      assert(offsetValue.get.valueString == "42")
+    }
+
+    "be able to create a StringType instance from a string" in {
+      val offsetValue = OffsetValue.fromString("string", "foo")
+      assert(offsetValue.get.dataType == OffsetType.StringType)
+      assert(offsetValue.get.valueString == "foo")
+    }
+
+    "throw an exception when trying to create an instance from a string with an unknown type" in {
+      assertThrows[IllegalArgumentException] {
+        OffsetValue.fromString("unknown", "42")
+      }
+    }
+  }
+
+  "compareTo" should {
+    "compare 2 datetime values" in {
+      val offsetValue1 = OffsetValue.DateTimeValue(Instant.ofEpochMilli(1726564198000L))
+      val offsetValue2 = OffsetValue.DateTimeValue(Instant.ofEpochMilli(1726564198001L))
+
+      assert(offsetValue1.compareTo(offsetValue2) < 0)
+      assert(offsetValue2.compareTo(offsetValue1) > 0)
+      assert(offsetValue2.compareTo(offsetValue2) == 0)
+    }
+
+    "throw an exception when attempting to compare a datetime value with value of some other type" in {
+      val offsetValue1 = OffsetValue.DateTimeValue(Instant.ofEpochMilli(1726564198000L))
+      val offsetValue2 = OffsetValue.IntegralValue(42)
+
+      assertThrows[IllegalArgumentException] {
+        offsetValue1.compareTo(offsetValue2)
+      }
+    }
+
+    "compare 2 integral values" in {
+      val offsetValue1 = OffsetValue.IntegralValue(42)
+      val offsetValue2 = OffsetValue.IntegralValue(43)
+
+      assert(offsetValue1.compareTo(offsetValue2) < 0)
+      assert(offsetValue2.compareTo(offsetValue1) > 0)
+      assert(offsetValue2.compareTo(offsetValue2) == 0)
+    }
+
+    "throw an exception when attempting to compare an integral value with value of some other type" in {
+      val offsetValue1 = OffsetValue.IntegralValue(42)
+      val offsetValue2 = OffsetValue.StringValue("foo")
+
+      assertThrows[IllegalArgumentException] {
+        offsetValue1.compareTo(offsetValue2)
+      }
+    }
+
+    "compare 2 string values" in {
+      val offsetValue1 = OffsetValue.StringValue("bar")
+      val offsetValue2 = OffsetValue.StringValue("foo")
+
+      assert(offsetValue1.compareTo(offsetValue2) < 0)
+      assert(offsetValue2.compareTo(offsetValue1) > 0)
+      assert(offsetValue2.compareTo(offsetValue2) == 0)
+    }
+
+    "throw an exception when attempting to compare a string value with value of some other type" in {
+      val offsetValue1 = OffsetValue.StringValue("foo")
+      val offsetValue2 = OffsetValue.DateTimeValue(Instant.ofEpochMilli(1726564198000L))
+
+      assertThrows[IllegalArgumentException] {
+        offsetValue1.compareTo(offsetValue2)
+      }
+    }
+  }
+}
diff --git a/pramen/core/src/test/resources/log4j.properties b/pramen/core/src/test/resources/log4j.properties
index 2ff2c9f3a..6d2eae20d 100644
--- a/pramen/core/src/test/resources/log4j.properties
+++ b/pramen/core/src/test/resources/log4j.properties
@@ -24,6 +24,7 @@ log4j.logger.za.co.absa.pramen.core.mocks.job.JobBaseDummy=OFF
 log4j.logger.za.co.absa.pramen.core.notify.SyncNotificationEmail=OFF
 log4j.logger.za.co.absa.pramen.core.reader.TableReaderJdbc=OFF
 log4j.logger.za.co.absa.pramen.core.runner.AppRunner$=OFF
+log4j.logger.za.co.absa.pramen.core.runner.jobrunner.ConcurrentJobRunnerImpl=OFF
 log4j.logger.za.co.absa.pramen.core.runner.task.TaskRunnerBase=OFF
 log4j.logger.za.co.absa.pramen.core.runner.task.TaskRunnerMultithreaded=OFF
 log4j.logger.za.co.absa.pramen.core.runner.task.TaskRunnerParallel=OFF
diff --git a/pramen/core/src/test/resources/log4j2.properties b/pramen/core/src/test/resources/log4j2.properties
index 7282300c6..c8c58ca0a 100644
--- a/pramen/core/src/test/resources/log4j2.properties
+++ b/pramen/core/src/test/resources/log4j2.properties
@@ -49,6 +49,9 @@ logger.jdbcnativeutils.level = OFF
 logger.apprunner.name = za.co.absa.pramen.core.runner.AppRunner$
 logger.apprunner.level = OFF
 
+logger.concurrentjobrunner.name = za.co.absa.pramen.core.runner.jobrunner.ConcurrentJobRunnerImpl
+logger.concurrentjobrunner.level = OFF
+
 logger.taskrunnermultithreaded.name = za.co.absa.pramen.core.runner.task.TaskRunnerMultithreaded
 logger.taskrunnermultithreaded.level = OFF
 
diff --git a/pramen/core/src/test/resources/test/config/incremental_pipeline.conf b/pramen/core/src/test/resources/test/config/incremental_pipeline.conf
new file mode 100644
index 000000000..350697cd5
--- /dev/null
+++ b/pramen/core/src/test/resources/test/config/incremental_pipeline.conf
@@ -0,0 +1,104 @@
+# Copyright 2022 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This variable is expected to be set up by the test suite
+#base.path = "/tmp"
+
+pramen {
+  pipeline.name = "Incremental ingestion with a file-based source"
+
+  bookkeeping.enabled = true
+  stop.spark.session = false
+
+  track.days = 2 # Previous day and today
+}
+
+pramen.metastore {
+  tables = [
+    {
+      name = "table1"
+      format = ${metastore.format}
+      path = ${base.path}/table1
+    },
+    {
+      name = "table2"
+      format = ${metastore.format}
+      path = ${base.path}/table2
+    }
+  ]
+}
+
+pramen.sources.1 = [
+  {
+    name = "spark_source"
+    factory.class = "za.co.absa.pramen.core.source.SparkSource"
+
+    has.information.date.column = ${has.information.date.column}
+    information.date.column = "info_date"
+    information.date.type = "date"
+    information.date.format = "yyyy-MM-dd"
+
+    offset.column {
+      name = "id"
+      type = "integral"
+    }
+
+    format = "csv"
+
+    schema = ${csv.schema}
+
+    option {
+      header = true
+      inferSchema = ${infer.schema}
+    }
+  }
+]
+
+pramen.operations = [
+  {
+    name = "Sourcing from a folder"
+    type = "ingestion"
+    schedule.type = "incremental"
+
+    source = "spark_source"
+
+    info.date.expr = "@runDate"
+
+    tables = [
+      {
+        input.path = ${base.path}/landing
+        output.metastore.table = table1
+      }
+    ]
+  },
+  {
+    name = "Running a transformer"
+    type = "transformation"
+
+    class = "za.co.absa.pramen.core.transformers.IdentityTransformer"
+    schedule.type = ${transformer.schedule}
+
+    output.table = "table2"
+
+    dependencies = [
+      {
+        tables = [ table1 ]
+        date.from = "@infoDate"
+      }
+    ]
+
+    option {
+      input.table = "table1"
+    }
+  }
+]
diff --git a/pramen/core/src/test/resources/test/config/incremental_pipeline_jdbc.conf b/pramen/core/src/test/resources/test/config/incremental_pipeline_jdbc.conf
new file mode 100644
index 000000000..75c375ed3
--- /dev/null
+++ b/pramen/core/src/test/resources/test/config/incremental_pipeline_jdbc.conf
@@ -0,0 +1,77 @@
+# Copyright 2022 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This variable is expected to be set up by the test suite
+#base.path = "/tmp"
+
+pramen {
+  pipeline.name = "Incremental ingestion with a JDBC-based source"
+
+  bookkeeping.enabled = true
+  stop.spark.session = false
+
+  track.days = 2 # Previous day and today
+}
+
+pramen.metastore {
+  tables = [
+    {
+      name = "table1"
+      format = "delta"
+      path = ${base.path}/table1
+    }
+  ]
+}
+
+pramen.sources.1 = [
+  {
+    name = "jdbc_source"
+    factory.class = "za.co.absa.pramen.core.source.JdbcSource"
+    jdbc {
+      driver = "org.hsqldb.jdbc.JDBCDriver"
+      connection.string = ${pramen.bookkeeping.jdbc.url}
+      user = ${pramen.bookkeeping.jdbc.user}
+      password = ${pramen.bookkeeping.jdbc.password}
+    }
+
+    has.information.date.column = ${has.information.date.column}
+    information.date.column = "info_date"
+    information.date.type = "string"
+
+    offset.column {
+      name = "last_updated"
+      type = "datetime"
+    }
+
+    format = "parquet"
+  }
+]
+
+pramen.operations = [
+  {
+    name = "Sourcing from a JDBC table"
+    type = "ingestion"
+    schedule.type = "incremental"
+
+    source = "jdbc_source"
+
+    info.date.expr = "@runDate"
+
+    tables = [
+      {
+        input.table = "incremental_table"
+        output.metastore.table = table1
+      }
+    ]
+  }
+]
diff --git a/pramen/core/src/test/resources/test/config/incremental_pipeline_ts.conf b/pramen/core/src/test/resources/test/config/incremental_pipeline_ts.conf
new file mode 100644
index 000000000..3bae0cb6b
--- /dev/null
+++ b/pramen/core/src/test/resources/test/config/incremental_pipeline_ts.conf
@@ -0,0 +1,96 @@
+# Copyright 2022 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This variable is expected to be set up by the test suite
+#base.path = "/tmp"
+
+pramen {
+  pipeline.name = "Incremental ingestion with a file-based source"
+
+  bookkeeping.enabled = true
+  stop.spark.session = false
+
+  track.days = 2 # Previous day and today
+}
+
+pramen.metastore {
+  tables = [
+    {
+      name = "table1"
+      format = ${metastore.format}
+      path = ${base.path}/table1
+    },
+    {
+      name = "table2"
+      format = ${metastore.format}
+      path = ${base.path}/table2
+    }
+  ]
+}
+
+pramen.sources.1 = [
+  {
+    name = "spark_source"
+    factory.class = "za.co.absa.pramen.core.source.SparkSource"
+
+    has.information.date.column = ${has.information.date.column}
+    information.date.column = "ts"
+    information.date.type = "datetime"
+
+    offset.column {
+      name = "ts"
+      type = "datetime"
+    }
+
+    format = "parquet"
+  }
+]
+
+pramen.operations = [
+  {
+    name = "Sourcing from a folder"
+    type = "ingestion"
+    schedule.type = "incremental"
+
+    source = "spark_source"
+
+    info.date.expr = "@runDate"
+
+    tables = [
+      {
+        input.path = ${base.path}/landing
+        output.metastore.table = table1
+      }
+    ]
+  },
+  {
+    name = "Running a transformer"
+    type = "transformation"
+
+    class = "za.co.absa.pramen.core.transformers.IdentityTransformer"
+    schedule.type = ${transformer.schedule}
+
+    output.table = "table2"
+
+    dependencies = [
+      {
+        tables = [ table1 ]
+        date.from = "@infoDate"
+      }
+    ]
+
+    option {
+      input.table = "table1"
+    }
+  }
+]
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/MetaTableDefFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/MetaTableDefFactory.scala
index c4c12ce30..f863c68b4 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/MetaTableDefFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/MetaTableDefFactory.scala
@@ -28,6 +28,7 @@ object MetaTableDefFactory {
                             format: DataFormat  =DataFormat.Null(),
                             infoDateColumn: String = "info_date",
                             infoDateFormat: String = "yyyy-MM-dd",
+                            batchIdColumn: String = "pramen_batchid",
                             hiveTable: Option[String] = None,
                             hivePath: Option[String] = None,
                             infoDateStart: LocalDate = LocalDate.of(2022, 1, 1),
@@ -40,6 +41,7 @@ object MetaTableDefFactory {
       format,
       infoDateColumn,
       infoDateFormat,
+      batchIdColumn,
       hiveTable,
       hivePath,
       infoDateStart,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/PramenImplSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/PramenImplSuite.scala
index 62bc9f6e4..590db7ab1 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/PramenImplSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/PramenImplSuite.scala
@@ -119,7 +119,7 @@ class PramenImplSuite extends AnyWordSpec {
       )
 
       val pipelineState = mock(classOf[PipelineState])
-      when(pipelineState.getState()).thenReturn(PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(taskResults = taskResults))
+      when(pipelineState.getState).thenReturn(PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(taskResults = taskResults))
 
       pramen.setPipelineState(pipelineState)
 
@@ -153,7 +153,7 @@ class PramenImplSuite extends AnyWordSpec {
       )
 
       val pipelineState = mock(classOf[PipelineState])
-      when(pipelineState.getState()).thenReturn(PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(taskResults = taskResults))
+      when(pipelineState.getState).thenReturn(PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(taskResults = taskResults))
 
       pramen.setPipelineState(pipelineState)
 
@@ -188,7 +188,7 @@ class PramenImplSuite extends AnyWordSpec {
       )
 
       val pipelineState = mock(classOf[PipelineState])
-      when(pipelineState.getState()).thenReturn(PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(taskResults = taskResults))
+      when(pipelineState.getState).thenReturn(PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(taskResults = taskResults))
 
       pramen.setPipelineState(pipelineState)
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/TaskNotificationFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/TaskNotificationFactory.scala
index 1e8920325..1bc2fe436 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/TaskNotificationFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/TaskNotificationFactory.scala
@@ -29,7 +29,7 @@ object TaskNotificationFactory {
                                  Instant.ofEpochMilli(1613600000000L),
                                  Instant.ofEpochMilli(1672759508000L)
                                )),
-                               status: RunStatus = RunStatus.Succeeded(None, 100, None, TaskRunReason.New, Seq.empty, Seq.empty, Seq.empty, Seq.empty),
+                               status: RunStatus = RunStatus.Succeeded(None, Some(100), None, None, TaskRunReason.New, Seq.empty, Seq.empty, Seq.empty, Seq.empty),
                                applicationId: String = "app_12345",
                                isTransient: Boolean = false,
                                isRawFilesJob: Boolean = false,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/app/AppContextSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/app/AppContextSuite.scala
index 1b330cea5..0dc7abecf 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/app/AppContextSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/app/AppContextSuite.scala
@@ -49,7 +49,7 @@ class AppContextSuite extends AnyWordSpec with SparkTestBase{
       .resolve()
 
     "be able to create app context from config" in {
-      val context = AppContextImpl.apply(conf)
+      val context = AppContextImpl.apply(conf, 0L)
 
       assert(context.bookkeeper.isInstanceOf[BookkeeperNull])
       assert(context.tokenLockFactory.isInstanceOf[TokenLockFactoryAllow])
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/app/config/RuntimeConfigSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/app/config/RuntimeConfigSuite.scala
index 7e7cf21f6..49489a7f4 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/app/config/RuntimeConfigSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/app/config/RuntimeConfigSuite.scala
@@ -34,7 +34,6 @@ class RuntimeConfigSuite extends AnyWordSpec {
            |  }
            |  undercover = true
            |  use.lock = false
-           |  track.updates = false
            |  check.only.late.data = true
            |  check.only.new.data = true
            |  email.if.no.changes = false
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/base/SparkTestBase.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/base/SparkTestBase.scala
index 892d62a31..34114145c 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/base/SparkTestBase.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/base/SparkTestBase.scala
@@ -33,6 +33,7 @@ trait SparkTestBase {
     .config("spark.sql.session.timeZone", "Africa/Johannesburg")
     .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
     .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+    .config("spark.sql.legacy.timeParserPolicy", "CORRECTED")
     .config("spark.sql.shuffle.partitions", "1")
     .getOrCreate()
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/fixtures/RelationalDbFixture.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/fixtures/RelationalDbFixture.scala
index 159b4e206..e1e88c807 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/fixtures/RelationalDbFixture.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/fixtures/RelationalDbFixture.scala
@@ -19,6 +19,7 @@ package za.co.absa.pramen.core.fixtures
 import org.scalatest.{BeforeAndAfterAll, Suite}
 
 import java.sql._
+import java.util.TimeZone
 import scala.collection.mutable.ListBuffer
 
 trait RelationalDbFixture extends BeforeAndAfterAll {
@@ -35,6 +36,8 @@ trait RelationalDbFixture extends BeforeAndAfterAll {
   def getConnection: Connection = DriverManager.getConnection(url, user, password)
 
   override protected def beforeAll(): Unit = {
+    TimeZone.setDefault(TimeZone.getTimeZone("Africa/Johannesburg"))
+
     super.beforeAll()
     Class.forName(driver)
   }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineDeltaLongSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineDeltaLongSuite.scala
new file mode 100644
index 000000000..dffd3f161
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineDeltaLongSuite.scala
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.integration
+
+class IncrementalPipelineDeltaLongSuite extends IncrementalPipelineLongFixture {
+  private val format = "delta"
+
+  "For inputs without information date the pipeline" should {
+    "work end to end as a normal run" in {
+      testOffsetOnlyNormalRun(format)
+    }
+
+    "work for offsets out of order" in {
+      testOffsetOnlyRunningOutOfOrderOffsets(format)
+    }
+
+    "work for uncommitted late offsets" in {
+      testOffsetOnlyUncommittedLateOffsets(format)
+    }
+
+    "work with incremental ingestion and normal transformer" in {
+      testOffsetOnlyIncrementalIngestionNormalTransformer(format)
+    }
+
+    "work end to end as rerun" in {
+      testOffsetOnlyRerun(format)
+    }
+
+    "work end to end as rerun with deletion of records" in {
+      testOffsetOnlyRerunWithRecordsDeletion(format)
+    }
+
+    "work end to end as rerun with deletion of records with previous data present" in {
+      testOffsetOnlyRerunWithRecordsDeletionAndPreviousDataPresent(format)
+    }
+
+    "skip when rerunning for a day which does not have offsets" in {
+      testOffsetOnlySkipRerunWithoutOffsets(format)
+    }
+
+    "run for a historical date range with force update" in {
+      testOffsetOnlyHistoricalDateRangeWithForceUpdate(format)
+    }
+
+    "run for a historical date range with fill gaps update" in {
+      testOffsetOnlyHistoricalDateRangeWithFillGaps(format)
+    }
+
+    "deal with uncommitted offsets when no path" in {
+      testOffsetOnlyDealWithUncommittedOffsetsWithNoPath(format)
+    }
+
+    "deal with uncommitted offsets when no data" in {
+      testOffsetOnlyDealWithUncommittedOffsetsWithNoData(format)
+    }
+
+    "deal with uncommitted changes when there is data" in {
+      testOffsetOnlyDealWithUncommittedOffsetsWithData(format)
+    }
+
+    "fail is the input data type does not conform" in {
+      testOffsetOnlyFailWhenInputDataDoesNotConform(format)
+    }
+
+    "fail if the output table does not have the offset field" in {
+      testOffsetOnlyFailWhenInputTableDoestHaveOffsetField(format)
+    }
+  }
+
+  "For inputs with information date the pipeline" should {
+    "work end to end as a normal run" in {
+      testWithInfoDateNormalRun(format)
+    }
+
+    "work with incremental ingestion and normal transformer" in {
+      testWithInfoDateIncrementalIngestionNormalTransformer(format)
+    }
+
+    "work end to end as rerun" in {
+      testWithInfoDateRerun(format)
+    }
+
+    "work for historical runs" in {
+      testWithInfoDateHistoricalDateRange(format)
+    }
+  }
+
+  "When the input column is timestamp" should {
+    "work end to end as a normal run" in {
+      testWithTimestampNormalRun(format)
+    }
+  }
+
+  "Edge cases" should {
+    "offsets cross info days" in {
+      testOffsetCrossInfoDateEdgeCase(format)
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineJdbcLongSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineJdbcLongSuite.scala
new file mode 100644
index 000000000..b1a577584
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineJdbcLongSuite.scala
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.integration
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.hadoop.fs.Path
+import org.apache.spark.sql.functions.col
+import org.scalatest.wordspec.AnyWordSpec
+import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
+import za.co.absa.pramen.core.base.SparkTestBase
+import za.co.absa.pramen.core.fixtures.{RelationalDbFixture, TempDirFixture, TextComparisonFixture}
+import za.co.absa.pramen.core.rdb.PramenDb
+import za.co.absa.pramen.core.reader.JdbcUrlSelectorImpl
+import za.co.absa.pramen.core.reader.model.JdbcConfig
+import za.co.absa.pramen.core.runner.AppRunner
+import za.co.absa.pramen.core.samples.RdbExampleTable
+import za.co.absa.pramen.core.utils.{JdbcNativeUtils, ResourceUtils}
+
+import java.sql.Date
+import java.time.LocalDate
+
+class IncrementalPipelineJdbcLongSuite extends AnyWordSpec
+  with SparkTestBase
+  with RelationalDbFixture
+  with BeforeAndAfter
+  with BeforeAndAfterAll
+  with TempDirFixture
+  with TextComparisonFixture {
+
+  val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
+  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+
+  private val infoDate = LocalDate.of(2021, 2, 18)
+
+  private val BATCH_ID_COLUMN = "pramen_batchid"
+  private val INFO_DATE_COLUMN = "pramen_info_date"
+
+  before {
+    pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
+    pramenDb.setupDatabase()
+    RdbExampleTable.IncrementalTable.initTable(getConnection)
+  }
+
+  override def afterAll(): Unit = {
+    pramenDb.close()
+    super.afterAll()
+  }
+
+  "end to end normal mode" in {
+    val expectedOffsetOnly1: String =
+      """{"id":1,"name":"John"}
+        |{"id":2,"name":"Jack"}
+        |{"id":3,"name":"Jill"}
+        |""".stripMargin
+
+    val expectedOffsetOnlyAll: String =
+      """{"id":1,"name":"John"}
+        |{"id":2,"name":"Jack"}
+        |{"id":3,"name":"Jill"}
+        |{"id":4,"name":"Mary"}
+        |{"id":5,"name":"Jane"}
+        |{"id":6,"name":"Kate"}
+        |""".stripMargin
+
+    withTempDirectory("incremental1") { tempDir =>
+      val conf = getConfig(tempDir)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val dfTable1Before = spark.read.format("delta").load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+
+      val statement = getConnection.createStatement
+      RdbExampleTable.IncrementalTable.inserts2.foreach(sql => statement.executeUpdate(sql))
+      getConnection.commit()
+
+      val exitCode2 = AppRunner.runPipeline(conf)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format("delta").load(table1Path.toString)
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1After, expectedOffsetOnlyAll)
+    }
+    succeed
+  }
+
+  def getConfig(basePath: String,
+                isRerun: Boolean = false,
+                isHistoricalRun: Boolean = false,
+                historyRunMode: String = "force",
+                hasInfoDate: Boolean = false,
+                useInfoDate: LocalDate = infoDate,
+                resource: String = "/test/config/incremental_pipeline_jdbc.conf"): Config = {
+    val configContents = ResourceUtils.getResourceString(resource)
+    val basePathEscaped = basePath.replace("\\", "\\\\")
+    val historicalConfigStr = if (isHistoricalRun) {
+      s"""pramen.load.date.from = "${useInfoDate.minusDays(1)}"
+         |pramen.load.date.to = "$useInfoDate"
+         |pramen.runtime.run.mode = "$historyRunMode"
+         |""".stripMargin
+    } else {
+      ""
+    }
+
+    val conf = ConfigFactory.parseString(
+        s"""base.path = "$basePathEscaped"
+           |pramen.runtime.is.rerun = $isRerun
+           |pramen.current.date = "$useInfoDate"
+           |$historicalConfigStr
+           |has.information.date.column = $hasInfoDate
+           |
+           |pramen.bookkeeping.jdbc {
+           |  driver = "$driver"
+           |  url = "$url"
+           |  user = "$user"
+           |  password = "$password"
+           |}
+           |$configContents
+           |""".stripMargin
+      ).withFallback(ConfigFactory.load())
+      .resolve()
+
+    conf
+  }
+
+  private def debugSql(sql: String): Unit = {
+    JdbcNativeUtils.withResultSet(new JdbcUrlSelectorImpl(jdbcConfig), sql, 1) { rs =>
+      val mt = rs.getMetaData
+
+      for (i <- 1 to mt.getColumnCount) {
+        print(mt.getColumnName(i) + "\t")
+      }
+      println("")
+
+      while (rs.next()) {
+        for (i <- 1 to mt.getColumnCount) {
+          print(rs.getString(i) + "\t")
+        }
+        println("")
+      }
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineLongFixture.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineLongFixture.scala
new file mode 100644
index 000000000..14fe1f0eb
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineLongFixture.scala
@@ -0,0 +1,1278 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.integration
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.hadoop.fs.Path
+import org.apache.spark.sql.SaveMode
+import org.apache.spark.sql.functions.{col, lit}
+import org.apache.spark.sql.types.TimestampType
+import org.scalatest.wordspec.AnyWordSpec
+import org.scalatest.{Assertion, BeforeAndAfter, BeforeAndAfterAll}
+import za.co.absa.pramen.api.offset.DataOffset.CommittedOffset
+import za.co.absa.pramen.api.offset.OffsetType
+import za.co.absa.pramen.core.base.SparkTestBase
+import za.co.absa.pramen.core.bookkeeper.OffsetManagerJdbc
+import za.co.absa.pramen.core.fixtures.{RelationalDbFixture, TempDirFixture, TextComparisonFixture}
+import za.co.absa.pramen.core.rdb.PramenDb
+import za.co.absa.pramen.core.reader.JdbcUrlSelectorImpl
+import za.co.absa.pramen.core.reader.model.JdbcConfig
+import za.co.absa.pramen.core.runner.AppRunner
+import za.co.absa.pramen.core.utils.{FsUtils, JdbcNativeUtils, ResourceUtils}
+
+import java.sql.Date
+import java.time.LocalDate
+
+class IncrementalPipelineLongFixture extends AnyWordSpec
+  with SparkTestBase
+  with RelationalDbFixture
+  with BeforeAndAfter
+  with BeforeAndAfterAll
+  with TempDirFixture
+  with TextComparisonFixture {
+
+  val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
+  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+
+  before {
+    pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
+    pramenDb.setupDatabase()
+  }
+
+  override def afterAll(): Unit = {
+    pramenDb.close()
+    super.afterAll()
+  }
+
+  private val infoDate = LocalDate.of(2021, 2, 18)
+
+  private val BATCH_ID_COLUMN = "pramen_batchid"
+  private val INFO_DATE_COLUMN = "pramen_info_date"
+
+  val expectedOffsetOnly1: String =
+    """{"id":1,"name":"John"}
+      |{"id":2,"name":"Jack"}
+      |{"id":3,"name":"Jill"}
+      |""".stripMargin
+
+  val expectedOffsetOnly2: String =
+    """{"id":4,"name":"Mary"}
+      |{"id":5,"name":"Jane"}
+      |{"id":6,"name":"Kate"}
+      |""".stripMargin
+
+  val expectedOffsetOnlyAll: String =
+    """{"id":1,"name":"John"}
+      |{"id":2,"name":"Jack"}
+      |{"id":3,"name":"Jill"}
+      |{"id":4,"name":"Mary"}
+      |{"id":5,"name":"Jane"}
+      |{"id":6,"name":"Kate"}
+      |""".stripMargin
+
+  val csv1WithInfoDateStr = s"id,name,info_date\n0,Old,${infoDate.minusDays(1)}\n1,John,$infoDate\n2,Jack,$infoDate\n3,Jill,$infoDate\n99,New,${infoDate.plusDays(1)}\n"
+  val csv2WithInfoDateStr = s"id,name,info_date\n1,John,${infoDate.minusDays(1)}\n4,Mary,$infoDate\n5,Jane,$infoDate\n6,Kate,$infoDate\n999,New2,${infoDate.plusDays(1)}\n"
+  val csvWithInfoDateSchema = "id int,name string,info_date date"
+
+  val expectedWithInfoDate1: String =
+    """{"id":1,"name":"John"}
+      |{"id":2,"name":"Jack"}
+      |{"id":3,"name":"Jill"}
+      |""".stripMargin
+
+  val expectedWithInfoDate2: String =
+    """{"id":4,"name":"Mary"}
+      |{"id":5,"name":"Jane"}
+      |{"id":6,"name":"Kate"}
+      |""".stripMargin
+
+  val expectedWithInfoDateAll: String =
+    """{"id":1,"name":"John"}
+      |{"id":2,"name":"Jack"}
+      |{"id":3,"name":"Jill"}
+      |{"id":4,"name":"Mary"}
+      |{"id":5,"name":"Jane"}
+      |{"id":6,"name":"Kate"}
+      |""".stripMargin
+
+  val csv1WithTimestampStr = s"tss,name\n1613563930000,Old\n1613639398123,John\n1613639398124,Jack\n1613639399123,Jill\n1613740330000,New\n"
+
+  val expectedWithTimestamp1: String =
+    """{"ts":"2021-02-18T11:09:58.123+02:00","name":"John","pramen_info_date":"2021-02-18"}
+      |{"ts":"2021-02-18T11:09:58.124+02:00","name":"Jack","pramen_info_date":"2021-02-18"}
+      |{"ts":"2021-02-18T11:09:59.123+02:00","name":"Jill","pramen_info_date":"2021-02-18"}
+      |""".stripMargin
+
+  val expectedWithTimestamp2: String =
+    """{"ts":"2021-02-18T11:09:58.123+02:00","name":"John","pramen_info_date":"2021-02-18"}
+      |{"ts":"2021-02-18T11:09:58.124+02:00","name":"Jack","pramen_info_date":"2021-02-18"}
+      |{"ts":"2021-02-18T11:09:59.123+02:00","name":"Jill","pramen_info_date":"2021-02-18"}
+      |{"ts":"2021-02-19T15:12:10.000+02:00","name":"New","pramen_info_date":"2021-02-19"}
+      |""".stripMargin
+
+  def testOffsetOnlyNormalRun(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val exitCode2 = AppRunner.runPipeline(conf)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString)
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString)
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1After, expectedOffsetOnlyAll)
+      compareText(actualTable2After, expectedOffsetOnlyAll)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyRunningOutOfOrderOffsets(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, useInfoDate = infoDate.minusDays(1))
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After0 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(2)))
+      val dfTable2After0 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(2)))
+      val dfTable1After1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable2After1 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable1After2 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After2 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1After2 = dfTable1After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After2 = dfTable2After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      val batchIds = dfTable1After1.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.isEmpty)
+
+      val om = new OffsetManagerJdbc(pramenDb.db, 123L)
+      val offsets0 = om.getOffsets("table1", infoDate.minusDays(2))
+      val offsets1 = om.getOffsets("table1", infoDate.minusDays(1))
+      val offsets2 = om.getOffsets("table1", infoDate).map(_.asInstanceOf[CommittedOffset])
+
+      assert(offsets0.isEmpty)
+      assert(offsets1.isEmpty)
+      assert(offsets2.nonEmpty)
+
+      assert(offsets2.head.maxOffset.valueString == "3")
+
+      // Expecting empty records
+      assert(dfTable1After0.isEmpty)
+      assert(dfTable2After0.isEmpty)
+      assert(dfTable1After1.isEmpty)
+      assert(dfTable2After1.isEmpty)
+      compareText(actualTable1After2, expectedOffsetOnly1)
+      compareText(actualTable2After2, expectedOffsetOnly1)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyUncommittedLateOffsets(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat, useInfoDate = infoDate.minusDays(2))
+
+      // Running a job that ingests initial data for offsets 1..3 at 2021-02-16
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      // Adding an uncommitted offset for 2021-02-17
+      val om = new OffsetManagerJdbc(pramenDb.db, 123L)
+      om.startWriteOffsets("table1", infoDate.minusDays(1), OffsetType.IntegralType)
+      Thread.sleep(10)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      // Writing data for the metastore table for the uncommitted offset at 2021-02-17
+      val dfIn = spark.read
+        .option("header", "true")
+        .option("inferSchema", "true")
+        .csv(path2.toString)
+        .withColumn(INFO_DATE_COLUMN, lit(Date.valueOf(infoDate.minusDays(1))))
+
+      dfIn.write
+        .mode(SaveMode.Append)
+        .partitionBy(INFO_DATE_COLUMN)
+        .format(metastoreFormat)
+        .save(table1Path.toString)
+
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(2)))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(2)))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      // Checking expected offset table
+      val offsetsBefore0 = om.getOffsets("table1", infoDate.minusDays(2))
+      val offsetsBefore1 = om.getOffsets("table1", infoDate.minusDays(1))
+      val offsetsBefore2 = om.getOffsets("table1", infoDate)
+
+      assert(offsetsBefore0.length == 1)
+      assert(offsetsBefore1.length == 1)
+      assert(offsetsBefore2.isEmpty)
+
+      assert(offsetsBefore0.head.isCommitted)
+      assert(!offsetsBefore1.head.isCommitted)
+      assert(offsetsBefore0.head.asInstanceOf[CommittedOffset].minOffset.valueString == "1")
+      assert(offsetsBefore0.head.asInstanceOf[CommittedOffset].maxOffset.valueString == "3")
+
+      // Running the job for 2021-02-18. It should reconcile uncommitted offsets and return with 'No data' since no data arrived since 2021-02-17
+      val conf2 = getConfig(tempDir, metastoreFormat, useInfoDate = infoDate)
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 2)
+
+      // Checking expected offset table
+      val offsetsAfter0 = om.getOffsets("table1", infoDate.minusDays(2))
+      val offsetsAfter1 = om.getOffsets("table1", infoDate.minusDays(1))
+      val offsetsAfter2 = om.getOffsets("table1", infoDate)
+
+      assert(offsetsAfter0.length == 1)
+      assert(offsetsAfter1.length == 1)
+      assert(offsetsAfter2.isEmpty)
+
+      assert(offsetsAfter0.head.isCommitted)
+      assert(offsetsAfter1.head.isCommitted)
+      assert(offsetsAfter0.head.asInstanceOf[CommittedOffset].minOffset.valueString == "1")
+      assert(offsetsAfter0.head.asInstanceOf[CommittedOffset].maxOffset.valueString == "3")
+      assert(offsetsAfter1.head.asInstanceOf[CommittedOffset].minOffset.valueString == "4")
+      assert(offsetsAfter1.head.asInstanceOf[CommittedOffset].maxOffset.valueString == "6")
+
+      // Checking expected data
+      val dfTable1After0 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(2)))
+      val dfTable2After0 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(2)))
+      val dfTable1After1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable2After1 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable1After2 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After2 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1After0 = dfTable1After0.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After0 = dfTable2After0.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable1After1 = dfTable1After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      /* This will be enabled when incremental processing is implemented.
+      val actualTable2After1 = dfTable2After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")*/
+
+      // Expecting empty records
+      compareText(actualTable1After0, expectedOffsetOnly1)
+      compareText(actualTable2After0, expectedOffsetOnly1)
+      compareText(actualTable1After1, expectedOffsetOnly2)
+      /*compareText(actualTable2After1, expectedOffsetOnly2) This will be enabled when incremental processing is implemented. */
+      assert(dfTable1After2.isEmpty)
+      assert(dfTable2After2.isEmpty)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyIncrementalIngestionNormalTransformer(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat, isTransformerIncremental = false)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.deleteFile(path1)
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val exitCode2 = AppRunner.runPipeline(conf)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString)
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString)
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1After, expectedOffsetOnlyAll)
+      compareText(actualTable2After, expectedOffsetOnlyAll)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyRerun(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isRerun = true)
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString)
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString)
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 1)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      // Expecting original records
+      compareText(actualTable1After, expectedOffsetOnly1)
+      compareText(actualTable2After, expectedOffsetOnly1)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyRerunWithRecordsDeletion(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.deleteFile(path1)
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isRerun = true)
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString)
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString)
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.isEmpty)
+
+      // Expecting empty records
+      assert(dfTable1After.isEmpty)
+      assert(dfTable2After.isEmpty)
+    }
+    succeed
+  }
+
+  def testOffsetOnlySkipRerunWithoutOffsets(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isRerun = true, useInfoDate = infoDate.minusDays(1))
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable2After1 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable1After2 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After2 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1After2 = dfTable1After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After2 = dfTable2After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      val batchIds = dfTable1After1.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.isEmpty)
+
+      // Expecting empty records
+      assert(dfTable1After1.isEmpty)
+      assert(dfTable2After1.isEmpty)
+      compareText(actualTable1After2, expectedOffsetOnly1)
+      compareText(actualTable2After2, expectedOffsetOnly1)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyRerunWithRecordsDeletionAndPreviousDataPresent(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path0 = new Path(tempDir, new Path("landing", "landing_file0.csv"))
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path0, "id,name\n0,Old\n")
+
+      val conf0 = getConfig(tempDir, metastoreFormat, useInfoDate = infoDate.minusDays(1))
+
+      val exitCode0 = AppRunner.runPipeline(conf0)
+      fsUtils.deleteFile(path1)
+
+      assert(exitCode0 == 0)
+
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf1 = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf1)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.deleteFile(path1)
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isRerun = true)
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.isEmpty)
+
+      // Expecting empty records
+      assert(dfTable1After.isEmpty)
+      assert(dfTable2After.isEmpty)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyHistoricalDateRangeWithForceUpdate(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf1 = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf1)
+      assert(exitCode1 == 0)
+
+      val table1Path1 = new Path(tempDir, "table1")
+      val table2Path1 = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isHistoricalRun = true, useInfoDate = infoDate.plusDays(1))
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val table1Path2 = new Path(tempDir, "table1")
+      val table2Path2 = new Path(tempDir, "table2")
+      val dfTable1After1 = spark.read.format(metastoreFormat).load(table1Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After1 = spark.read.format(metastoreFormat).load(table2Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable1After2 = spark.read.format(metastoreFormat).load(table1Path2.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.plusDays(1)))
+      val dfTable2After2 = spark.read.format(metastoreFormat).load(table2Path2.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.plusDays(1)))
+      val actualTable1After1 = dfTable1After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After1 = dfTable2After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable1After2 = dfTable1After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After2 = dfTable2After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      val batchIdsOld = dfTable1After1.select(BATCH_ID_COLUMN).distinct().collect().map(_.getLong(0))
+      val batchIdsNew = dfTable1After2.select(BATCH_ID_COLUMN).distinct().collect().map(_.getLong(0))
+
+      assert(batchIdsOld.length == 1)
+      assert(batchIdsNew.length == 1)
+
+      // The batch id for all info dates in range should be the same since they were re-ran
+      assert(batchIdsOld.head == batchIdsNew.head)
+
+      // Expecting empty records
+      compareText(actualTable1After1, expectedOffsetOnly1)
+      compareText(actualTable2After1, expectedOffsetOnly1)
+      compareText(actualTable1After2, expectedOffsetOnly2)
+      compareText(actualTable2After2, expectedOffsetOnly2)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyHistoricalDateRangeWithFillGaps(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf1 = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf1)
+      assert(exitCode1 == 0)
+
+      val table1Path1 = new Path(tempDir, "table1")
+      val table2Path1 = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedOffsetOnly1)
+      compareText(actualTable2Before, expectedOffsetOnly1)
+
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isHistoricalRun = true, historyRunMode = "fill_gaps", useInfoDate = infoDate.plusDays(1))
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val table1Path2 = new Path(tempDir, "table1")
+      val table2Path2 = new Path(tempDir, "table2")
+      val dfTable1After1 = spark.read.format(metastoreFormat).load(table1Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After1 = spark.read.format(metastoreFormat).load(table2Path1.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable1After2 = spark.read.format(metastoreFormat).load(table1Path2.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.plusDays(1)))
+      val dfTable2After2 = spark.read.format(metastoreFormat).load(table2Path2.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.plusDays(1)))
+      val actualTable1After1 = dfTable1After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After1 = dfTable2After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable1After2 = dfTable1After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After2 = dfTable2After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      val batchIdsOld = dfTable1After1.select(BATCH_ID_COLUMN).distinct().collect().map(_.getLong(0))
+      val batchIdsNew = dfTable1After2.select(BATCH_ID_COLUMN).distinct().collect().map(_.getLong(0))
+
+      assert(batchIdsOld.length == 1)
+      assert(batchIdsNew.length == 1)
+
+      // The batch id for all info dates in range should not be the same since they we ran only for missing data
+      assert(batchIdsOld.head != batchIdsNew.head)
+
+      // Expecting empty records
+      compareText(actualTable1After1, expectedOffsetOnly1)
+      compareText(actualTable2After1, expectedOffsetOnly1)
+      compareText(actualTable1After2, expectedOffsetOnly2)
+      compareText(actualTable2After2, expectedOffsetOnly2)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyDealWithUncommittedOffsetsWithNoPath(metastoreFormat: String): Assertion = {
+    val om = new OffsetManagerJdbc(pramenDb.db, 123L)
+
+    om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+
+    Thread.sleep(10)
+
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val table1Path = new Path(tempDir, "table1")
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val dfTable1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1 = dfTable1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1, expectedOffsetOnly1)
+
+      val batchIds = dfTable1.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 1)
+
+      val offsets = om.getOffsets("table1", infoDate).map(_.asInstanceOf[CommittedOffset]).sortBy(_.createdAt)
+
+      assert(offsets.length == 1)
+
+      assert(offsets.head.minOffset.valueString.toLong == 1)
+      assert(offsets.head.maxOffset.valueString.toLong == 3)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyDealWithUncommittedOffsetsWithNoData(metastoreFormat: String): Assertion = {
+    val om = new OffsetManagerJdbc(pramenDb.db, 123L)
+
+    om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+
+    Thread.sleep(10)
+
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val table1Path = new Path(tempDir, "table1")
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val df = spark.read
+        .option("header", "true")
+        .option("inferSchema", "true")
+        .csv(path1.toString)
+        .withColumn(BATCH_ID_COLUMN, lit(123L))
+        .withColumn(INFO_DATE_COLUMN, lit(Date.valueOf(infoDate)))
+
+      df.filter(col("id") < 0)
+        .write
+        .format(metastoreFormat)
+        .partitionBy(INFO_DATE_COLUMN)
+        .save(table1Path.toString)
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val dfTable1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1 = dfTable1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1, expectedOffsetOnly1)
+
+      val batchIds = dfTable1.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 1)
+
+      val offsets = om.getOffsets("table1", infoDate).map(_.asInstanceOf[CommittedOffset]).sortBy(_.createdAt)
+
+      assert(offsets.length == 1)
+
+      assert(offsets.head.minOffset.valueString.toLong == 1)
+      assert(offsets.head.maxOffset.valueString.toLong == 3)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyDealWithUncommittedOffsetsWithData(metastoreFormat: String): Assertion = {
+    val om1 = new OffsetManagerJdbc(pramenDb.db, 123L)
+    om1.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+
+    Thread.sleep(10)
+
+    val om2 = new OffsetManagerJdbc(pramenDb.db, 123L)
+    om2.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+
+    Thread.sleep(10)
+
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+      fsUtils.writeFile(path2, "id,name\n4,Mary\n5,Jane\n6,Kate\n")
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+
+      val df = spark.read
+        .option("header", "true")
+        .option("inferSchema", "true")
+        .csv(path1.toString)
+        .withColumn(BATCH_ID_COLUMN, lit(123L))
+        .withColumn(INFO_DATE_COLUMN, lit(Date.valueOf(infoDate)))
+
+      df.write
+        .format(metastoreFormat)
+        .partitionBy(INFO_DATE_COLUMN)
+        .save(table1Path.toString)
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val dfTable1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1 = dfTable1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2 = dfTable2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1, expectedOffsetOnlyAll)
+      compareText(actualTable2, expectedOffsetOnly2) // ToDo This logic is to be changed when incremental transformations are supported
+
+      val batchIds = dfTable1.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val offsets = om2.getOffsets("table1", infoDate).map(_.asInstanceOf[CommittedOffset]).sortBy(_.createdAt)
+
+      assert(offsets.length == 2)
+
+      assert(offsets.head.minOffset.valueString.toLong == 1)
+      assert(offsets.head.maxOffset.valueString.toLong == 3)
+      assert(offsets(1).minOffset.valueString.toLong == 4)
+      assert(offsets(1).maxOffset.valueString.toLong == 6)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyFailWhenInputDataDoesNotConform(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      fsUtils.writeFile(path1, "id,name\n1,John\n2,Jack\n3,Jill\n")
+
+      val conf = getConfig(tempDir, metastoreFormat, inferSchema = false, csvSchema = "id string,name string")
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 2)
+    }
+    succeed
+  }
+
+  def testOffsetOnlyFailWhenInputTableDoestHaveOffsetField(metastoreFormat: String): Assertion = {
+    val om1 = new OffsetManagerJdbc(pramenDb.db, 123L)
+    om1.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+
+    Thread.sleep(10)
+
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, "name\nJohn\nJack\nJill\n")
+      fsUtils.writeFile(path2, "name\nMary\nJane\nKate\n")
+
+      val table1Path = new Path(tempDir, "table1")
+
+      val df = spark.read
+        .option("header", "true")
+        .option("inferSchema", "true")
+        .csv(path1.toString)
+        .withColumn(BATCH_ID_COLUMN, lit(123L))
+        .withColumn(INFO_DATE_COLUMN, lit(Date.valueOf(infoDate)))
+
+      df.write.format(metastoreFormat)
+        .partitionBy(INFO_DATE_COLUMN)
+        .save(table1Path.toString)
+
+      val conf = getConfig(tempDir, metastoreFormat)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 2)
+    }
+    succeed
+  }
+
+  def testWithInfoDateNormalRun(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, csv1WithInfoDateStr)
+
+      val conf = getConfig(tempDir, metastoreFormat, hasInfoDate = true, inferSchema = false, csvSchema = csvWithInfoDateSchema)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedWithInfoDate1)
+      compareText(actualTable2Before, expectedWithInfoDate1)
+
+      fsUtils.deleteFile(path1)
+      fsUtils.writeFile(path2, csv2WithInfoDateStr)
+
+      val exitCode2 = AppRunner.runPipeline(conf)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.parquet(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After = spark.read.parquet(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1After, expectedWithInfoDateAll)
+      compareText(actualTable2After, expectedWithInfoDateAll)
+    }
+    succeed
+  }
+
+  def testWithInfoDateIncrementalIngestionNormalTransformer(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, csv1WithInfoDateStr)
+
+      val conf = getConfig(tempDir, metastoreFormat, hasInfoDate = true, isTransformerIncremental = false, inferSchema = false, csvSchema = csvWithInfoDateSchema)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedWithInfoDate1)
+      compareText(actualTable2Before, expectedWithInfoDate1)
+
+      fsUtils.deleteFile(path1)
+      fsUtils.writeFile(path2, csv2WithInfoDateStr)
+
+      val exitCode2 = AppRunner.runPipeline(conf)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1After, expectedWithInfoDateAll)
+      compareText(actualTable2After, expectedWithInfoDateAll)
+    }
+    succeed
+  }
+
+  def testWithInfoDateRerun(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+      fsUtils.writeFile(path1, csv1WithInfoDateStr)
+
+      val conf = getConfig(tempDir, metastoreFormat, hasInfoDate = true, inferSchema = false, csvSchema = csvWithInfoDateSchema)
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedWithInfoDate1)
+      compareText(actualTable2Before, expectedWithInfoDate1)
+
+      fsUtils.deleteFile(path1)
+      fsUtils.writeFile(path2, csv2WithInfoDateStr)
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isRerun = true, hasInfoDate = true, inferSchema = false, csvSchema = csvWithInfoDateSchema)
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 1)
+
+      val actualTable1After = dfTable1After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      // Expecting original records
+      compareText(actualTable1After, expectedWithInfoDate2)
+      compareText(actualTable2After, expectedWithInfoDate2)
+    }
+    succeed
+  }
+
+  def testWithInfoDateHistoricalDateRange(metastoreFormat: String): Assertion = {
+    val csv1Str = s"id,name,info_date\n0,Old,${infoDate.minusDays(1)}\n1,John,$infoDate\n2,Jack,$infoDate\n3,Jill,$infoDate\n99,New,${infoDate.plusDays(2)}\n"
+    val csv2Str = s"id,name,info_date\n4,Mary,${infoDate.plusDays(1)}\n5,Jane,${infoDate.plusDays(1)}\n6,Kate,${infoDate.plusDays(1)}\n999,New2,${infoDate.plusDays(2)}\n"
+
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+      val path2 = new Path(tempDir, new Path("landing", "landing_file2.csv"))
+
+      fsUtils.writeFile(path1, csv1Str)
+      fsUtils.writeFile(path2, csv2Str)
+
+      val conf1 = getConfig(tempDir, metastoreFormat, hasInfoDate = true, inferSchema = false, csvSchema = csvWithInfoDateSchema)
+
+      val exitCode1 = AppRunner.runPipeline(conf1)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val actualTable1Before = dfTable1Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedWithInfoDate1)
+      compareText(actualTable2Before, expectedWithInfoDate1)
+
+      val conf2 = getConfig(tempDir, metastoreFormat, hasInfoDate = true, isHistoricalRun = true, historyRunMode = "fill_gaps", useInfoDate = infoDate.plusDays(1), inferSchema = false, csvSchema = csvWithInfoDateSchema)
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2After1 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable1After2 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.plusDays(1)))
+      val dfTable2After2 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.plusDays(1)))
+      val actualTable1After1 = dfTable1After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After1 = dfTable2After1.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable1After2 = dfTable1After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2After2 = dfTable2After2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      val batchIdsOld = dfTable1After1.select(BATCH_ID_COLUMN).distinct().collect().map(_.getLong(0))
+      val batchIdsNew = dfTable1After2.select(BATCH_ID_COLUMN).distinct().collect().map(_.getLong(0))
+
+      assert(batchIdsOld.length == 1)
+      assert(batchIdsNew.length == 1)
+
+      // The batch id for all info dates in range should not be the same since they we ran only for missing data
+      assert(batchIdsOld.head != batchIdsNew.head)
+
+      // Expecting empty records
+      compareText(actualTable1After1, expectedWithInfoDate1)
+      compareText(actualTable2After1, expectedWithInfoDate1)
+      compareText(actualTable1After2, expectedWithInfoDate2)
+      compareText(actualTable2After2, expectedWithInfoDate2)
+    }
+    succeed
+  }
+
+  def testWithTimestampNormalRun(metastoreFormat: String): Assertion = {
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1Csv = new Path(tempDir, new Path("landing_csv", "landing_file1.csv"))
+      val path1Parquet = new Path(tempDir, "landing")
+      fsUtils.writeFile(path1Csv, csv1WithTimestampStr)
+
+      val df = spark.read
+        .option("header", "true")
+        .option("inferSchema", "true")
+        .csv(path1Csv.toString)
+        .withColumn("ts", (col("tss") / 1000).cast(TimestampType))
+
+      df.write.parquet(path1Parquet.toString)
+
+      val conf = getConfig(tempDir, metastoreFormat, hasInfoDate = true, resource = "/test/config/incremental_pipeline_ts.conf")
+
+      val exitCode1 = AppRunner.runPipeline(conf)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1Before = spark.read.format(metastoreFormat).load(table1Path.toString)
+      val dfTable2Before = spark.read.format(metastoreFormat).load(table2Path.toString)
+      val actualTable1Before = dfTable1Before.select("ts", "name", INFO_DATE_COLUMN).orderBy("ts").toJSON.collect().mkString("\n")
+      val actualTable2Before = dfTable2Before.select("ts", "name", INFO_DATE_COLUMN).orderBy("ts").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1Before, expectedWithTimestamp1)
+      compareText(actualTable2Before, expectedWithTimestamp1)
+
+      val conf2 = getConfig(tempDir, metastoreFormat, isTransformerIncremental = false, hasInfoDate = true, useInfoDate = infoDate.plusDays(1), resource = "/test/config/incremental_pipeline_ts.conf")
+      val exitCode2 = AppRunner.runPipeline(conf2)
+      assert(exitCode2 == 0)
+
+      val dfTable1After = spark.read.format(metastoreFormat).load(table1Path.toString)
+      val dfTable2After = spark.read.format(metastoreFormat).load(table2Path.toString)
+
+      val batchIds = dfTable1After.select(BATCH_ID_COLUMN).distinct().collect()
+
+      assert(batchIds.length == 2)
+
+      val actualTable1After = dfTable1After.select("ts", "name", INFO_DATE_COLUMN).orderBy("ts").toJSON.collect().mkString("\n")
+      val actualTable2After = dfTable2After.select("ts", "name", INFO_DATE_COLUMN).orderBy("ts").toJSON.collect().mkString("\n")
+
+      compareText(actualTable1After, expectedWithTimestamp2)
+      compareText(actualTable2After, expectedWithTimestamp2)
+
+      val om = new OffsetManagerJdbc(pramenDb.db, 123L)
+
+      val offsets1 = om.getOffsets("table1", infoDate.minusDays(1)).map(_.asInstanceOf[CommittedOffset])
+      assert(offsets1.isEmpty)
+      assert(offsets1.isEmpty)
+
+      val offsets2 = om.getOffsets("table1", infoDate).map(_.asInstanceOf[CommittedOffset])
+      assert(offsets2.length == 1)
+      assert(offsets2.head.minOffset.valueString.toLong == 1613639398123L)
+      assert(offsets2.head.maxOffset.valueString.toLong == 1613639399123L)
+
+      val offsets3 = om.getOffsets("table1", infoDate.plusDays(1)).map(_.asInstanceOf[CommittedOffset])
+      assert(offsets3.length == 1)
+      assert(offsets3.head.minOffset.valueString.toLong == 1613740330000L)
+      assert(offsets3.head.maxOffset.valueString.toLong == 1613740330000L)
+    }
+    succeed
+  }
+
+  def testOffsetCrossInfoDateEdgeCase(metastoreFormat: String): Assertion = {
+    val expected =
+      """{"id":3,"name":"Jill"}
+        |{"id":4,"name":"Mary"}
+        |""".stripMargin
+
+    val csv1Str = s"id,name,info_date\n0,Old,${infoDate.minusDays(2)}\n1,John,${infoDate.minusDays(1)}\n2,Jack,${infoDate.minusDays(1)}\n3,Jill,$infoDate\n4,Mary,$infoDate\n"
+
+    withTempDirectory("incremental1") { tempDir =>
+      val fsUtils = new FsUtils(spark.sparkContext.hadoopConfiguration, tempDir)
+
+      val path1 = new Path(tempDir, new Path("landing", "landing_file1.csv"))
+
+      fsUtils.writeFile(path1, csv1Str)
+
+      val conf1 = getConfig(tempDir, metastoreFormat, hasInfoDate = true, inferSchema = false, csvSchema = "id int,name string,info_date date")
+
+      val exitCode1 = AppRunner.runPipeline(conf1)
+      assert(exitCode1 == 0)
+
+      val table1Path = new Path(tempDir, "table1")
+      val table2Path = new Path(tempDir, "table2")
+      val dfTable1_1 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable2_1 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate.minusDays(1)))
+      val dfTable1_2 = spark.read.format(metastoreFormat).load(table1Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+      val dfTable2_2 = spark.read.format(metastoreFormat).load(table2Path.toString).filter(col(INFO_DATE_COLUMN) === Date.valueOf(infoDate))
+
+      val actualTable1_2 = dfTable1_2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+      val actualTable2_2 = dfTable2_2.select("id", "name").orderBy("id").toJSON.collect().mkString("\n")
+
+      assert(dfTable1_1.isEmpty)
+      assert(dfTable2_1.isEmpty)
+      compareText(actualTable1_2, expected)
+      compareText(actualTable2_2, expected)
+
+      val om = new OffsetManagerJdbc(pramenDb.db, 123L)
+
+      val offsets1 = om.getOffsets("table1", infoDate.minusDays(1)).map(_.asInstanceOf[CommittedOffset])
+      assert(offsets1.isEmpty)
+
+      val offsets2 = om.getOffsets("table1", infoDate).map(_.asInstanceOf[CommittedOffset])
+      assert(offsets2.length == 1)
+      assert(offsets2.head.minOffset.valueString.toLong == 3)
+      assert(offsets2.head.maxOffset.valueString.toLong == 4)
+    }
+    succeed
+  }
+
+  def getConfig(basePath: String,
+                metastoreFormat: String,
+                isRerun: Boolean = false,
+                useDataFrame: Boolean = false,
+                isTransformerIncremental: Boolean = true,
+                isHistoricalRun: Boolean = false,
+                historyRunMode: String = "force",
+                inferSchema: Boolean = true,
+                csvSchema: String = "id int,name string",
+                hasInfoDate: Boolean = false,
+                useInfoDate: LocalDate = infoDate,
+                resource: String = "/test/config/incremental_pipeline.conf"): Config = {
+    val configContents = ResourceUtils.getResourceString(resource)
+    val basePathEscaped = basePath.replace("\\", "\\\\")
+    val transformerSchedule = if (isTransformerIncremental) "incremental" else "daily"
+    val historicalConfigStr = if (isHistoricalRun) {
+      s"""pramen.load.date.from = "${useInfoDate.minusDays(1)}"
+         |pramen.load.date.to = "$useInfoDate"
+         |pramen.runtime.run.mode = "$historyRunMode"
+         |""".stripMargin
+    } else {
+      ""
+    }
+
+    val conf = ConfigFactory.parseString(
+        s"""base.path = "$basePathEscaped"
+           |use.dataframe = $useDataFrame
+           |pramen.runtime.is.rerun = $isRerun
+           |pramen.current.date = "$useInfoDate"
+           |transformer.schedule = "$transformerSchedule"
+           |infer.schema = $inferSchema
+           |$historicalConfigStr
+           |has.information.date.column = $hasInfoDate
+           |metastore.format = $metastoreFormat
+           |csv.schema = "$csvSchema"
+           |
+           |pramen.bookkeeping.jdbc {
+           |  driver = "$driver"
+           |  url = "$url"
+           |  user = "$user"
+           |  password = "$password"
+           |}
+           |$configContents
+           |""".stripMargin
+      ).withFallback(ConfigFactory.load())
+      .resolve()
+
+    conf
+  }
+
+  /* This method is used to inspect offsets after operations */
+  private def debugOffsets(): Unit = {
+    JdbcNativeUtils.withResultSet(new JdbcUrlSelectorImpl(jdbcConfig), "SELECT * FROM \"offsets\"", 1) { rs =>
+      val mt = rs.getMetaData
+
+      for (i <- 1 to mt.getColumnCount) {
+        print(mt.getColumnName(i) + "\t")
+      }
+      println("")
+
+      while (rs.next()) {
+        for (i <- 1 to mt.getColumnCount) {
+          print(rs.getString(i) + "\t")
+        }
+        println("")
+      }
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineParquetLongSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineParquetLongSuite.scala
new file mode 100644
index 000000000..b8226be3f
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineParquetLongSuite.scala
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.integration
+
+class IncrementalPipelineParquetLongSuite extends IncrementalPipelineLongFixture {
+  private val format = "parquet"
+
+  "For inputs without information date the pipeline" should {
+    "work end to end as a normal run" in {
+      testOffsetOnlyNormalRun(format)
+    }
+
+    "work for offsets out of order" in {
+      testOffsetOnlyRunningOutOfOrderOffsets(format)
+    }
+
+    "work for uncommitted late offsets" in {
+      testOffsetOnlyUncommittedLateOffsets(format)
+    }
+
+    "work with incremental ingestion and normal transformer" in {
+      testOffsetOnlyIncrementalIngestionNormalTransformer(format)
+    }
+
+    "work end to end as rerun" in {
+      testOffsetOnlyRerun(format)
+    }
+
+    "work end to end as rerun with deletion of records" in {
+      testOffsetOnlyRerunWithRecordsDeletion(format)
+    }
+
+    "work end to end as rerun with deletion of records with previous data present" in {
+      testOffsetOnlyRerunWithRecordsDeletionAndPreviousDataPresent(format)
+    }
+
+    "skip when rerunning for a day which does not have offsets" in {
+      testOffsetOnlySkipRerunWithoutOffsets(format)
+    }
+
+    "run for a historical date range with force update" in {
+      testOffsetOnlyHistoricalDateRangeWithForceUpdate(format)
+    }
+
+    "run for a historical date range with fill gaps update" in {
+      testOffsetOnlyHistoricalDateRangeWithFillGaps(format)
+    }
+
+    "deal with uncommitted offsets when no path" in {
+      testOffsetOnlyDealWithUncommittedOffsetsWithNoPath(format)
+    }
+
+    "deal with uncommitted offsets when no data" in {
+      testOffsetOnlyDealWithUncommittedOffsetsWithNoData(format)
+    }
+
+    "deal with uncommitted changes when there is data" in {
+      testOffsetOnlyDealWithUncommittedOffsetsWithData(format)
+    }
+
+    "fail is the input data type does not conform" in {
+      testOffsetOnlyFailWhenInputDataDoesNotConform(format)
+    }
+
+    "fail if the output table does not have the offset field" in {
+      testOffsetOnlyFailWhenInputTableDoestHaveOffsetField(format)
+    }
+  }
+
+  "For inputs with information date the pipeline" should {
+    "work end to end as a normal run" in {
+      testWithInfoDateNormalRun(format)
+    }
+
+    "work with incremental ingestion and normal transformer" in {
+      testWithInfoDateIncrementalIngestionNormalTransformer(format)
+    }
+
+    "work end to end as rerun" in {
+      testWithInfoDateRerun(format)
+    }
+
+    "work for historical runs" in {
+      testWithInfoDateHistoricalDateRange(format)
+    }
+  }
+
+  "When the input column is timestamp" should {
+    "work end to end as a normal run" in {
+      testWithTimestampNormalRun(format)
+    }
+  }
+
+  "Edge cases" should {
+    "offsets cross info days" in {
+      testOffsetCrossInfoDateEdgeCase(format)
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/MetastoreSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/MetastoreSuite.scala
index 079470b5f..488bab536 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/MetastoreSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/MetastoreSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.functions.lit
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.{AnalysisException, DataFrame}
 import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.status.TaskRunReason
 import za.co.absa.pramen.api.{CachePolicy, DataFormat}
 import za.co.absa.pramen.core.OperationDefFactory
 import za.co.absa.pramen.core.app.config.InfoDateConfig
@@ -376,7 +377,7 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
 
         val stats = m.getStats("table1", infoDate)
 
-        assert(stats.recordCount == 3)
+        assert(stats.recordCount.contains(3))
         assert(stats.dataSizeBytes.exists(_ > 0))
       }
     }
@@ -389,7 +390,7 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
 
         m.saveTable("table1", infoDate, getDf)
 
-        val reader = m.getMetastoreReader("table1" :: Nil, infoDate)
+        val reader = m.getMetastoreReader("table1" :: Nil, infoDate, TaskRunReason.New, isIncremental = false)
 
         val df1 = reader.getTable("table1", Some(infoDate), Some(infoDate))
 
@@ -403,7 +404,7 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
 
         m.saveTable("table1", infoDate, getDf)
 
-        val reader = m.getMetastoreReader("table2" :: Nil, infoDate)
+        val reader = m.getMetastoreReader("table2" :: Nil, infoDate, TaskRunReason.New, isIncremental = false)
 
         val ex = intercept[TableNotConfigured] {
           reader.getTable("table1", Some(infoDate), Some(infoDate))
@@ -419,7 +420,7 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
 
         m.saveTable("table1", infoDate, getDf)
 
-        val reader = m.getMetastoreReader("table1" :: Nil, infoDate)
+        val reader = m.getMetastoreReader("table1" :: Nil, infoDate, TaskRunReason.New, isIncremental = false)
         val runInfo1 = reader.getTableRunInfo("table1", infoDate)
         val runInfo2 = reader.getTableRunInfo("table1", infoDate.plusDays(1))
 
@@ -437,7 +438,7 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
 
         m.saveTable("table1", infoDate, getDf)
 
-        val reader = m.getMetastoreReader("table1" :: Nil, infoDate)
+        val reader = m.getMetastoreReader("table1" :: Nil, infoDate, TaskRunReason.New, isIncremental = false)
         val metadataManager = reader.metadataManager
 
         metadataManager.setMetadata("table1", infoDate, "key1", "value1")
@@ -455,7 +456,7 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
           m.saveTable("table1", infoDate, getDf)
           m.saveTable("table1", infoDate.plusDays(1), getDf)
 
-          val reader = m.getMetastoreReader("table1" :: "table2" :: Nil, infoDate.plusDays(10))
+          val reader = m.getMetastoreReader("table1" :: "table2" :: Nil, infoDate.plusDays(10), TaskRunReason.New, isIncremental = false)
 
           val date1 = reader.getLatestAvailableDate("table1")
           val date2 = reader.getLatestAvailableDate("table1", Some(infoDate))
@@ -581,6 +582,6 @@ class MetastoreSuite extends AnyWordSpec with SparkTestBase with TextComparisonF
     val infoDateConfig = InfoDateConfig.fromConfig(conf)
     val bk = new SyncBookkeeperMock
     val mm = new MetadataManagerNull(isPersistenceEnabled = false)
-    (MetastoreImpl.fromConfig(conf, infoDateConfig, bk, mm), bk)
+    (MetastoreImpl.fromConfig(conf, infoDateConfig, bk, mm, 0L), bk)
   }
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/model/MetaTableSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/model/MetaTableSuite.scala
index 4be6b2e1d..15803ba92 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/model/MetaTableSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/model/MetaTableSuite.scala
@@ -159,7 +159,7 @@ class MetaTableSuite extends AnyWordSpec {
 
       val defaultHiveConfig = HiveDefaultConfig.getNullConfig
 
-      val metaTable = MetaTable.fromConfigSingleEntity(conf, conf, "INFO_DATE", "dd-MM-yyyy", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true)
+      val metaTable = MetaTable.fromConfigSingleEntity(conf, conf, "INFO_DATE", "dd-MM-yyyy", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
 
       assert(metaTable.name == "my_table")
       assert(metaTable.format.name == "delta")
@@ -212,7 +212,7 @@ class MetaTableSuite extends AnyWordSpec {
 
       val appConf = ConfigFactory.parseString("pramen.default.records.per.partition = 100")
 
-      val metaTable = MetaTable.fromConfigSingleEntity(conf, appConf, "INFO_DATE", "dd-MM-yyyy", LocalDate.parse("2020-01-31"), 1, defaultHiveConfig, defaultPreferAddPartition = true)
+      val metaTable = MetaTable.fromConfigSingleEntity(conf, appConf, "INFO_DATE", "dd-MM-yyyy", LocalDate.parse("2020-01-31"), 1, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
 
       assert(metaTable.name == "my_table")
       assert(metaTable.hiveConfig.hiveApi == HiveApi.Sql)
@@ -278,7 +278,7 @@ class MetaTableSuite extends AnyWordSpec {
 
       val appConf = ConfigFactory.parseString("pramen.default.records.per.partition = 100")
 
-      val metaTable = MetaTable.fromConfigSingleEntity(conf, appConf, "INFO_DATE", "dd-MM-yyyy", LocalDate.parse("2020-01-31"), 1, defaultHiveConfig, defaultPreferAddPartition = true)
+      val metaTable = MetaTable.fromConfigSingleEntity(conf, appConf, "INFO_DATE", "dd-MM-yyyy", LocalDate.parse("2020-01-31"), 1, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
 
       assert(metaTable.name == "my_table")
       assert(metaTable.hiveConfig.hiveApi == HiveApi.SparkCatalog)
@@ -304,7 +304,7 @@ class MetaTableSuite extends AnyWordSpec {
       val defaultHiveConfig = HiveDefaultConfig.getNullConfig
 
       val ex = intercept[IllegalArgumentException] {
-        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true)
+        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
       }
 
       assert(ex.getMessage.contains("Mandatory option missing: name"))
@@ -322,7 +322,7 @@ class MetaTableSuite extends AnyWordSpec {
       val defaultHiveConfig = HiveDefaultConfig.getNullConfig
 
       val ex = intercept[IllegalArgumentException] {
-        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true)
+        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
       }
 
       assert(ex.getMessage.contains("Unable to read data format from config for the metastore table: table1"))
@@ -340,7 +340,7 @@ class MetaTableSuite extends AnyWordSpec {
       val defaultHiveConfig = HiveDefaultConfig.getNullConfig
 
       val ex = intercept[IllegalArgumentException] {
-        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true)
+        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
       }
 
       assert(ex.getMessage.contains("Invalid or unsupported save mode: 'ignore' for table 'table1'."))
@@ -358,7 +358,7 @@ class MetaTableSuite extends AnyWordSpec {
       val defaultHiveConfig = HiveDefaultConfig.getNullConfig
 
       val ex = intercept[IllegalArgumentException] {
-        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true)
+        MetaTable.fromConfigSingleEntity(conf, conf, "", "", LocalDate.parse("2020-01-31"), 0, defaultHiveConfig, defaultPreferAddPartition = true, "batchid")
       }
 
       assert(ex.getMessage.contains("Invalid or unsupported save mode: 'test' for table 'table1'."))
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceParquetSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceParquetSuite.scala
index 600afba44..1c321f173 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceParquetSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceParquetSuite.scala
@@ -39,7 +39,7 @@ class MetastorePersistenceParquetSuite extends AnyWordSpec with SparkTestBase wi
       withTempDirectory("metastore_parquet") { tempDir =>
         val outputPath = new Path(tempDir, "partition=10")
 
-        val persistence = new MetastorePersistenceParquet(tempDir, "ignore", "yyyy-MM-dd", None, None, Map.empty, Map.empty)
+        val persistence = new MetastorePersistenceParquet(tempDir, "ignore", "yyyy-MM-dd", "batchid", 0, None, None, Map.empty, Map.empty)
 
         persistence.writeAndCleanOnFailure(exampleDf, outputPath.toString, fsUtils, SaveMode.Overwrite)
 
@@ -52,7 +52,7 @@ class MetastorePersistenceParquetSuite extends AnyWordSpec with SparkTestBase wi
       withTempDirectory("metastore_parquet") { tempDir =>
         val outputPath = new Path(tempDir, "partition=10")
 
-        val persistence = new MetastorePersistenceParquet(tempDir, "ignore", "yyyy-MM-dd", None, Some(SaveMode.Append), Map.empty, Map.empty)
+        val persistence = new MetastorePersistenceParquet(tempDir, "ignore", "yyyy-MM-dd", "batchid", 0, None, Some(SaveMode.Append), Map.empty, Map.empty)
 
         persistence.writeAndCleanOnFailure(exampleDf, outputPath.toString, fsUtils, SaveMode.Append)
         persistence.writeAndCleanOnFailure(exampleDf, outputPath.toString, fsUtils, SaveMode.Append)
@@ -71,7 +71,7 @@ class MetastorePersistenceParquetSuite extends AnyWordSpec with SparkTestBase wi
 
       when(df.write).thenThrow(new RuntimeException("test exception"))
 
-      val persistence = new MetastorePersistenceParquet("dummy", "ignore", "yyyy-MM-dd", None, None, Map.empty, Map.empty)
+      val persistence = new MetastorePersistenceParquet("dummy", "ignore", "yyyy-MM-dd", "batchid", 0, None, None, Map.empty, Map.empty)
 
       assertThrows[RuntimeException] {
         persistence.writeAndCleanOnFailure(df, "dummy", fsUtils, SaveMode.Overwrite)
@@ -88,7 +88,7 @@ class MetastorePersistenceParquetSuite extends AnyWordSpec with SparkTestBase wi
 
         fsUtils.createDirectoryRecursive(outputPath)
 
-        val persistence = new MetastorePersistenceParquet("dummy", "ignore", "yyyy-MM-dd", None, None, Map.empty, Map.empty)
+        val persistence = new MetastorePersistenceParquet("dummy", "ignore", "yyyy-MM-dd", "batchid", 0, None, None, Map.empty, Map.empty)
 
         val ex = intercept[RuntimeException] {
           persistence.writeAndCleanOnFailure(df, outputPath.toString, fsUtils, SaveMode.Overwrite)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceRawSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceRawSuite.scala
index 5bb8e124f..c01c05bba 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceRawSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceRawSuite.scala
@@ -230,9 +230,9 @@ class MetastorePersistenceRawSuite extends AnyWordSpec with SparkTestBase with T
 
         val persistence = new MetastorePersistenceRaw(tempDir, infoDateColumn, infoDateFormat, None)
 
-        val stats = persistence.getStats(infoDate)
+        val stats = persistence.getStats(infoDate, onlyForCurrentBatchId = false)
 
-        assert(stats.recordCount == 2)
+        assert(stats.recordCount.contains(2))
         assert(stats.dataSizeBytes.contains(7L))
       }
     }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceSuite.scala
index f35234590..007d9f566 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceSuite.scala
@@ -215,24 +215,24 @@ class MetastorePersistenceSuite extends AnyWordSpec with SparkTestBase with Temp
   def testStatsAvailable(mtp: MetastorePersistence): Assertion = {
     mtp.saveTable(infoDate, getDf, Some(4))
 
-    val stats = mtp.getStats(infoDate)
+    val stats = mtp.getStats(infoDate, onlyForCurrentBatchId = false)
 
-    assert(stats.recordCount == 3)
+    assert(stats.recordCount.contains(3))
     assert(stats.dataSizeBytes.exists(_ > 0))
   }
 
   def testStatsAvailableForEmptyTable(mtp: MetastorePersistence): Assertion = {
     mtp.saveTable(infoDate, getDf.filter(col("b") > 10), Some(0))
 
-    val stats = mtp.getStats(infoDate)
+    val stats = mtp.getStats(infoDate, onlyForCurrentBatchId = false)
 
-    assert(stats.recordCount == 0)
+    assert(stats.recordCount.contains(0))
     assert(stats.dataSizeBytes.contains(0L))
   }
 
   def testStatsNotAvailable(mtp: MetastorePersistence): Assertion = {
     val ex = intercept[AnalysisException] {
-      mtp.getStats(infoDate)
+      mtp.getStats(infoDate, onlyForCurrentBatchId = false)
     }
 
     assert(ex.getMessage.contains("Path does not exist") ||
@@ -571,7 +571,7 @@ class MetastorePersistenceSuite extends AnyWordSpec with SparkTestBase with Temp
         infoDateFormat = infoDateFormat
       )
 
-      val persistence = MetastorePersistence.fromMetaTable(mt, conf)
+      val persistence = MetastorePersistence.fromMetaTable(mt, conf, batchId = 0)
 
       assert(persistence.isInstanceOf[MetastorePersistenceTransientEager])
     }
@@ -589,7 +589,7 @@ class MetastorePersistenceSuite extends AnyWordSpec with SparkTestBase with Temp
       saveModeOpt = saveModeOpt
     )
 
-    MetastorePersistence.fromMetaTable(mt, null)
+    MetastorePersistence.fromMetaTable(mt, null, batchId = 0)
   }
 
   def getDeltaMtPersistence(tempDir: String,
@@ -605,7 +605,7 @@ class MetastorePersistenceSuite extends AnyWordSpec with SparkTestBase with Temp
       writeOptions = writeOptions
     )
 
-    MetastorePersistence.fromMetaTable(mt, null)
+    MetastorePersistence.fromMetaTable(mt, null, batchId = 0)
   }
 
   private def getDf: DataFrame = {
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceTransientEagerSuiteEager.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceTransientEagerSuiteEager.scala
index d83da9e6d..9a12edecc 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceTransientEagerSuiteEager.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/MetastorePersistenceTransientEagerSuiteEager.scala
@@ -117,7 +117,7 @@ class MetastorePersistenceTransientEagerSuiteEager extends AnyWordSpec with Befo
 
       val saveResult = persistor.saveTable(infoDate, exampleDf, Some(10))
 
-      assert(saveResult.recordCount == 10)
+      assert(saveResult.recordCount.contains(10))
       assert(saveResult.dataSizeBytes.isEmpty)
 
       TransientTableManager.reset()
@@ -128,7 +128,7 @@ class MetastorePersistenceTransientEagerSuiteEager extends AnyWordSpec with Befo
 
       val saveResult = persistor.saveTable(infoDate, exampleDf, None)
 
-      assert(saveResult.recordCount == 3)
+      assert(saveResult.recordCount.contains(3))
       assert(saveResult.dataSizeBytes.isEmpty)
 
       TransientTableManager.reset()
@@ -139,7 +139,7 @@ class MetastorePersistenceTransientEagerSuiteEager extends AnyWordSpec with Befo
 
       val saveResult = persistor.saveTable(infoDate, exampleDf, None)
 
-      assert(saveResult.recordCount == 3)
+      assert(saveResult.recordCount.contains(3))
       assert(saveResult.dataSizeBytes.isDefined)
       assert(saveResult.dataSizeBytes.exists(_ > 100))
 
@@ -152,7 +152,7 @@ class MetastorePersistenceTransientEagerSuiteEager extends AnyWordSpec with Befo
       val persistor = new MetastorePersistenceTransientEager(null, null, null)
 
       assertThrows[UnsupportedOperationException] {
-        persistor.getStats(null)
+        persistor.getStats(null, onlyForCurrentBatchId = false)
       }
     }
   }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/TransientJobManagerSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/TransientJobManagerSuite.scala
index f5f130105..e1ebcbf65 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/TransientJobManagerSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/metastore/persistence/TransientJobManagerSuite.scala
@@ -166,7 +166,7 @@ class TransientJobManagerSuite extends AnyWordSpec with BeforeAndAfterAll with S
     "runs a job via teh task runner and return the dataframe" in {
       val taskRunner = mock(classOf[TaskRunner])
       val job = mock(classOf[Job])
-      val successStatus = RunStatus.Succeeded(None, 1, None, TaskRunReason.OnRequest, Nil, Nil, Nil, Nil)
+      val successStatus = RunStatus.Succeeded(None, Some(1), None, None, TaskRunReason.OnRequest, Nil, Nil, Nil, Nil)
 
       whenMock(taskRunner.runLazyTask(job, infoDate)).thenReturn(successStatus)
       whenMock(job.outputTable).thenReturn(MetaTableFactory.getDummyMetaTable("table1", format = DataFormat.Transient(CachePolicy.NoCache)))
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala
index 42d8bfdf0..4728505b0 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala
@@ -17,18 +17,25 @@
 package za.co.absa.pramen.core.mocks
 
 import com.typesafe.config.ConfigFactory
+import za.co.absa.pramen.api.offset.OffsetInfo
 import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlConfig}
 
+import java.time.ZoneId
+
 object DummySqlConfigFactory {
   def getDummyConfig(infoDateColumn: String = "col",
                      infoDateType: SqlColumnType = SqlColumnType.DATE,
                      dateFormatApp: String = "yyyy-MM-dd",
+                     offsetInfo: Option[OffsetInfo] = None,
+                     serverTimeZone: ZoneId = ZoneId.of("Africa/Johannesburg"),
                      identifierQuotingPolicy: QuotingPolicy = QuotingPolicy.Auto,
                      sqlGeneratorClass: Option[String] = None
                     ): SqlConfig = SqlConfig(
     infoDateColumn = infoDateColumn,
     infoDateType = infoDateType,
     dateFormatApp = dateFormatApp,
+    offsetInfo = offsetInfo,
+    serverTimeZone,
     identifierQuotingPolicy = identifierQuotingPolicy,
     sqlGeneratorClass = sqlGeneratorClass,
     ConfigFactory.empty())
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/MetaTableFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/MetaTableFactory.scala
index 2151794ea..61f4f1404 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/MetaTableFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/MetaTableFactory.scala
@@ -28,6 +28,7 @@ object MetaTableFactory {
                         format: DataFormat = DataFormat.Parquet("/tmp/dummy", None),
                         infoDateColumn: String = "INFO_DATE",
                         infoDateFormat: String = "yyyy-MM-dd",
+                        batchIdColumn: String = "pramen_batchid",
                         hiveConfig: HiveConfig = HiveConfig.getNullConfig,
                         hiveTable: Option[String] = None,
                         hivePath: Option[String] = None,
@@ -46,6 +47,7 @@ object MetaTableFactory {
       format,
       infoDateColumn,
       infoDateFormat,
+      batchIdColumn,
       hiveConfig,
       hiveTable,
       hivePath,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/PipelineStateSnapshotFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/PipelineStateSnapshotFactory.scala
index de1999762..9acfdf0a0 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/PipelineStateSnapshotFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/PipelineStateSnapshotFactory.scala
@@ -21,6 +21,7 @@ import za.co.absa.pramen.api.status.{CustomNotification, PipelineNotificationFai
 
 object PipelineStateSnapshotFactory {
   def getDummyPipelineStateSnapshot(pipelineInfo: PipelineInfo = PipelineInfoFactory.getDummyPipelineInfo(),
+                                    batchId: Long = 0L,
                                     isFinished: Boolean = false,
                                     warningFlag: Boolean = false,
                                     exitedNormally: Boolean = false,
@@ -30,6 +31,7 @@ object PipelineStateSnapshotFactory {
                                     pipelineNotificationFailures: Seq[PipelineNotificationFailure] = Seq.empty,
                                     customNotification: CustomNotification = CustomNotification(Seq.empty, Seq.empty)): PipelineStateSnapshot = {
     PipelineStateSnapshot(pipelineInfo,
+      batchId,
       isFinished,
       warningFlag,
       exitedNormally,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/RunStatusFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/RunStatusFactory.scala
index 784f4840c..a89fce7ad 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/RunStatusFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/RunStatusFactory.scala
@@ -20,7 +20,8 @@ import za.co.absa.pramen.api.status.{RunStatus, TaskRunReason}
 
 object RunStatusFactory {
   def getDummySuccess(recordCountOld: Option[Long] = None,
-                      recordCount: Long = 1000,
+                      recordCount: Option[Long] = Some(1000),
+                      recordsAppended: Option[Long] = None,
                       sizeBytes: Option[Long] = None,
                       reason: TaskRunReason = TaskRunReason.New,
                       filesRead: Seq[String] = Nil,
@@ -29,6 +30,7 @@ object RunStatusFactory {
                       warnings: Seq[String] = Nil): RunStatus.Succeeded = {
     RunStatus.Succeeded(recordCountOld,
       recordCount,
+      recordsAppended,
       sizeBytes,
       reason,
       filesRead,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala
index 2aaea5ad3..cf94fc24f 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala
@@ -16,6 +16,7 @@
 
 package za.co.absa.pramen.core.mocks
 
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.sql.{SqlConfig, SqlGenerator}
 
 import java.time.LocalDate
@@ -35,6 +36,8 @@ class SqlGeneratorDummy(sqlConfig: SqlConfig) extends SqlGenerator {
 
   override def getDataQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String], limit: Option[Int]): String = null
 
+  override def getDataQueryIncremental(tableName: String, onlyForInfoDate: Option[LocalDate], offsetFrom: Option[OffsetValue], offsetTo: Option[OffsetValue], columns: Seq[String]): String = null
+
   override def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String = null
 
   override def getAliasExpression(expression: String, alias: String): String = null
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskCompletedFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskCompletedFactory.scala
index 7c9f43cf9..ef9f39fe2 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskCompletedFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskCompletedFactory.scala
@@ -28,10 +28,11 @@ object TaskCompletedFactory {
                        periodBegin: LocalDate = LocalDate.of(2020, 12, 9),
                        periodEnd: LocalDate = LocalDate.of(2020, 12, 9),
                        informationDate: LocalDate = LocalDate.of(2020, 12, 9),
-                       inputRecordCount: Long = 1000L,
-                       inputRecordCountOld: Long = 1000L,
+                       inputRecordCount: Option[Long] = Some(1000L),
+                       inputRecordCountOld: Option[Long] = Some(1000L),
                        outputRecordCount: Option[Long] = Some(1000L),
                        outputRecordCountOld: Option[Long] = Some(1000L),
+                       appendedRecordCount: Option[Long] = None,
                        outputSize: Option[Long] = None,
                        startedAt: Long = 1234567L,
                        finishedAt: Long = 1234568L,
@@ -52,6 +53,7 @@ object TaskCompletedFactory {
       inputRecordCountOld,
       outputRecordCount,
       outputRecordCountOld,
+      appendedRecordCount,
       outputSize,
       startedAt,
       finishedAt,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskResultFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskResultFactory.scala
index c740e5d8b..3b334963b 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskResultFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TaskResultFactory.scala
@@ -25,7 +25,7 @@ import java.time.{Instant, LocalDate}
 object TaskResultFactory {
   def getDummyTaskResult(jobName: String = "DummyJob",
                          outputTable: MetaTableDef = MetaTable.getMetaTableDef(MetaTableFactory.getDummyMetaTable(name = "table_out")),
-                         runStatus: RunStatus = RunStatus.Succeeded(Some(100), 200, Some(1000), TaskRunReason.New, Nil, Nil, Nil, Nil),
+                         runStatus: RunStatus = RunStatus.Succeeded(Some(100), Some(200), None, Some(1000), TaskRunReason.New, Nil, Nil, Nil, Nil),
                          runInfo: Option[RunInfo] = Some(RunInfo(LocalDate.of(2022, 2, 18), Instant.ofEpochSecond(1234), Instant.ofEpochSecond(5678))),
                          applicationId: String = "app_123",
                          isTransient: Boolean = false,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TestPrototypes.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TestPrototypes.scala
index 4764b9ff4..6df2a2041 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TestPrototypes.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/TestPrototypes.scala
@@ -20,10 +20,10 @@ import za.co.absa.pramen.api.status.{RunStatus, TaskRunReason}
 
 object TestPrototypes {
 
-  val runStatusSuccess: RunStatus = RunStatus.Succeeded(Some(100), 200, Some(1000), TaskRunReason.New, Seq.empty, Seq.empty, Seq.empty, Seq.empty)
+  val runStatusSuccess: RunStatus = RunStatus.Succeeded(Some(100), Some(200), None, Some(1000), TaskRunReason.New, Seq.empty, Seq.empty, Seq.empty, Seq.empty)
 
   val runStatusWarning: RunStatus = RunStatus.Succeeded(
-    Some(10000), 20000, Some(100000), TaskRunReason.New, Seq("file1.txt", "file1.ctl"),
+    Some(10000), Some(20000), None, Some(100000), TaskRunReason.New, Seq("file1.txt", "file1.ctl"),
     Seq("file1.csv", "file2.csv"), Seq("`db`.`table1`"), Seq("Test warning")
   )
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/bookkeeper/SyncBookkeeperMock.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/bookkeeper/SyncBookkeeperMock.scala
index 9fc74f0d5..fb8073903 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/bookkeeper/SyncBookkeeperMock.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/bookkeeper/SyncBookkeeperMock.scala
@@ -17,7 +17,7 @@
 package za.co.absa.pramen.core.mocks.bookkeeper
 
 import org.apache.spark.sql.types.{DataType, StructType}
-import za.co.absa.pramen.core.bookkeeper.Bookkeeper
+import za.co.absa.pramen.core.bookkeeper.{Bookkeeper, OffsetManager}
 import za.co.absa.pramen.core.model.{DataChunk, TableSchema}
 
 import java.time.LocalDate
@@ -131,4 +131,6 @@ class SyncBookkeeperMock extends Bookkeeper {
 
     schemas += table -> (infoDate, tableSchema)
   }
+
+  override private[pramen] def getOffsetManager: OffsetManager = null
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobBaseDummy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobBaseDummy.scala
index e19cf6acc..a764c6aa1 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobBaseDummy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobBaseDummy.scala
@@ -41,11 +41,11 @@ class JobBaseDummy(operationDef: OperationDef,
 
   override val scheduleStrategy: ScheduleStrategy = null
 
-  override def validate(infoDate: LocalDate, conf: Config): Reason = null
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): Reason = null
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = null
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = null
 
   override def postProcessing(df: DataFrame, infoDate: LocalDate, conf: Config): DataFrame = null
 
-  override def save(df: DataFrame, infoDate: LocalDate, conf: Config, jobStarted: Instant, inputRecordCount: Option[Long]): SaveResult = null
+  override def save(df: DataFrame, infoDate: LocalDate, runReason: TaskRunReason, conf: Config, jobStarted: Instant, inputRecordCount: Option[Long]): SaveResult = null
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobSpy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobSpy.scala
index dd7599496..192800ec6 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobSpy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/JobSpy.scala
@@ -40,7 +40,7 @@ class JobSpy(jobName: String = "DummyJob",
              runFunction: () => RunResult = () => null,
              scheduleStrategyIn: ScheduleStrategy = new ScheduleStrategySourcing,
              allowParallel: Boolean = true,
-             saveStats: MetaTableStats = MetaTableStats(0, None),
+             saveStats: MetaTableStats = MetaTableStats(Some(0), None, None),
              jobNotificationTargets: Seq[JobNotificationTarget] = Seq.empty,
              jobTrackDays: Int = 0
             ) extends Job {
@@ -74,12 +74,12 @@ class JobSpy(jobName: String = "DummyJob",
     preRunCheckFunction()
   }
 
-  override def validate(infoDate: LocalDate, conf: Config): Reason = {
+  override def validate(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): Reason = {
     validateCount += 1
     validationFunction()
   }
 
-  override def run(infoDate: LocalDate, conf: Config): RunResult = {
+  override def run(infoDate: LocalDate, runReason: TaskRunReason, conf: Config): RunResult = {
     runCount += 1
     runFunction()
   }
@@ -89,7 +89,7 @@ class JobSpy(jobName: String = "DummyJob",
     df
   }
 
-  override def save(df: DataFrame, infoDate: LocalDate, jobConfig: Config, jobStarted: Instant, inputRecordCount: Option[Long]): SaveResult = {
+  override def save(df: DataFrame, infoDate: LocalDate, runReason: TaskRunReason, jobConfig: Config, jobStarted: Instant, inputRecordCount: Option[Long]): SaveResult = {
     saveDf = df
     saveCount += 1
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/SourceSpy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/SourceSpy.scala
index 713fcbe3f..048095a1b 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/SourceSpy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/job/SourceSpy.scala
@@ -18,8 +18,8 @@ package za.co.absa.pramen.core.mocks.job
 
 import com.typesafe.config.{Config, ConfigFactory}
 import org.apache.spark.sql.{DataFrame, SparkSession}
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.{ExternalChannelFactory, Query, Source, SourceResult}
-import za.co.absa.pramen.core.mocks.reader.ReaderSpy
 
 import java.time.LocalDate
 import scala.collection.mutable.ListBuffer
@@ -57,6 +57,8 @@ class SourceSpy(sourceConfig: Config = ConfigFactory.empty(),
 
     SourceResult(df)
   }
+
+  override def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): SourceResult = ???
 }
 
 object SourceSpy extends ExternalChannelFactory[SourceSpy] {
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreReaderMock.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreReaderMock.scala
index f01b759ba..d4bd0e8ca 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreReaderMock.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreReaderMock.scala
@@ -17,7 +17,9 @@
 package za.co.absa.pramen.core.mocks.metastore
 
 import org.apache.spark.sql.DataFrame
-import za.co.absa.pramen.api.{DataFormat, MetaTableDef, MetaTableRunInfo, MetadataManager, MetastoreReader}
+import za.co.absa.pramen.api.offset.DataOffset
+import za.co.absa.pramen.api.status.TaskRunReason
+import za.co.absa.pramen.api._
 import za.co.absa.pramen.core.metadata.MetadataManagerNull
 
 import java.time.LocalDate
@@ -32,6 +34,8 @@ class MetastoreReaderMock(tables: Seq[(String, DataFrame)], infoDate: LocalDate)
     }
   }
 
+  override def getCurrentBatch(tableName: String): DataFrame = getTable(tableName, Option(infoDate), Option(infoDate))
+
   override def getLatest(tableName: String, until: Option[LocalDate]): DataFrame = {
     tables.find(_._1 == tableName) match {
       case Some((_, df)) => df
@@ -50,9 +54,11 @@ class MetastoreReaderMock(tables: Seq[(String, DataFrame)], infoDate: LocalDate)
     tables.exists(_._1 == tableName)
   }
 
+  override def getOffsets(table: String, infoDate: LocalDate): Array[DataOffset] = Array.empty
+
   override def getTableDef(tableName: String): MetaTableDef = {
     tables.find(_._1 == tableName) match {
-      case Some((name, _)) => MetaTableDef(name, "", DataFormat.Null(), "pramen_info_date", "yyyy-MM-dd", None, None, null, Map.empty[String, String], Map.empty[String, String])
+      case Some((name, _)) => MetaTableDef(name, "", DataFormat.Null(), "pramen_info_date", "yyyy-MM-dd", "pramen_batchid", None, None, null, Map.empty[String, String], Map.empty[String, String])
       case None          => throw new IllegalArgumentException(s"Table $tableName not found")
     }
   }
@@ -60,4 +66,6 @@ class MetastoreReaderMock(tables: Seq[(String, DataFrame)], infoDate: LocalDate)
   override def getTableRunInfo(tableName: String, infoDate: LocalDate): Option[MetaTableRunInfo] = None
 
   override def metadataManager: MetadataManager = metadata
+
+  override def getRunReason: TaskRunReason = TaskRunReason.New
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreSpy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreSpy.scala
index 582884163..d43749be3 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreSpy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/metastore/MetastoreSpy.scala
@@ -16,8 +16,10 @@
 
 package za.co.absa.pramen.core.mocks.metastore
 
-import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.{DataFrame, SaveMode}
+import za.co.absa.pramen.api.offset.DataOffset
+import za.co.absa.pramen.api.status.TaskRunReason
 import za.co.absa.pramen.api.{MetaTableDef, MetaTableRunInfo, MetadataManager, MetastoreReader}
 import za.co.absa.pramen.core.metadata.MetadataManagerNull
 import za.co.absa.pramen.core.metastore.model.MetaTable
@@ -33,7 +35,7 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
                    availableDates: Seq[LocalDate] = Seq(LocalDate.of(2022, 2, 17)),
                    tableDf: DataFrame = null,
                    tableException: Throwable = null,
-                   stats: MetaTableStats = MetaTableStats(0, None),
+                   stats: MetaTableStats = MetaTableStats(Some(0), None, None),
                    statsException: Throwable = null,
                    isTableAvailable: Boolean = true,
                    isTableEmpty: Boolean = false,
@@ -69,11 +71,15 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
     tableDf
   }
 
+  override def getBatch(tableName: String, infoDate: LocalDate, batchIdOpt: Option[Long]): DataFrame = {
+    getTable(tableName, Option(infoDate), Option(infoDate))
+  }
+
   override def getLatest(tableName: String, until: Option[LocalDate]): DataFrame = null
 
-  override def saveTable(tableName: String, infoDate: LocalDate, df: DataFrame, inputRecordCount: Option[Long]): MetaTableStats = {
+  override def saveTable(tableName: String, infoDate: LocalDate, df: DataFrame, inputRecordCount: Option[Long], saveModeOverride: Option[SaveMode]): MetaTableStats = {
     saveTableInvocations.append((tableName, infoDate, df))
-    MetaTableStats(df.count(), None)
+    MetaTableStats(Option(df.count()), None, None)
   }
 
   def getHiveHelper(tableName: String): HiveHelper = {
@@ -99,7 +105,7 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
     stats
   }
 
-  override def getMetastoreReader(tables: Seq[String], infoDate: LocalDate): MetastoreReader = {
+  override def getMetastoreReader(tables: Seq[String], infoDate: LocalDate, taskRunReason: TaskRunReason, isIncremental: Boolean): MetastoreReader = {
     val metastore = this
 
     new MetastoreReader {
@@ -110,6 +116,10 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
         metastore.getTable(tableName, from, to)
       }
 
+      override def getCurrentBatch(tableName: String): DataFrame = {
+        getTable(tableName, Option(infoDate), Option(infoDate))
+      }
+
       override def getLatest(tableName: String, until: Option[LocalDate] = None): DataFrame = {
         validateTable(tableName)
         val untilDate = until.orElse(Option(infoDate))
@@ -128,6 +138,8 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
         metastore.isDataAvailable(tableName, fromDate, untilDate)
       }
 
+      override def getOffsets(table: String, infoDate: LocalDate): Array[DataOffset] = Array.empty
+
       override def getTableDef(tableName: String): MetaTableDef = {
         validateTable(tableName)
 
@@ -136,6 +148,7 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
         MetaTableDef(table.name,
           table.description,
           table.format,
+          table.batchIdColumn,
           table.infoDateColumn,
           table.infoDateFormat,
           table.hiveTable,
@@ -147,6 +160,8 @@ class MetastoreSpy(registeredTables: Seq[String] = Seq("table1", "table2"),
 
       override def getTableRunInfo(tableName: String, infoDate: LocalDate): Option[MetaTableRunInfo] = None
 
+      override def getRunReason: TaskRunReason = TaskRunReason.New
+
       override def metadataManager: MetadataManager = metadataManagerMock
 
       private def validateTable(tableName: String): Unit = {
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderSpy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderSpy.scala
index 94af1090d..49c74d368 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderSpy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderSpy.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.core.mocks.reader
 
-import java.time.LocalDate
 import org.apache.spark.sql.{DataFrame, SparkSession}
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.{Query, TableReader}
 
+import java.time.LocalDate
 import scala.collection.mutable.ListBuffer
 
 class ReaderSpy(numRecords: Long,
@@ -50,4 +51,6 @@ class ReaderSpy(numRecords: Long,
     val df = generatedList.toDF("v")
     df
   }
+
+  override def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): DataFrame = ???
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderStub.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderStub.scala
index b5fe8b240..b5343e800 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderStub.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/ReaderStub.scala
@@ -16,12 +16,16 @@
 
 package za.co.absa.pramen.core.mocks.reader
 
-import java.time.LocalDate
 import org.apache.spark.sql.DataFrame
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.{Query, TableReader}
 
+import java.time.LocalDate
+
 class ReaderStub extends TableReader {
   override def getRecordCount(query: Query,  infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long = 0
 
   override def getData(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String]): DataFrame = null
+
+  override def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): DataFrame = ???
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/TableReaderSparkFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/TableReaderSparkFactory.scala
index 622265e6d..728a6d5a6 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/TableReaderSparkFactory.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/reader/TableReaderSparkFactory.scala
@@ -17,6 +17,8 @@
 package za.co.absa.pramen.core.mocks.reader
 
 import org.apache.spark.sql.SparkSession
+import za.co.absa.pramen.api.offset.OffsetInfo
+import za.co.absa.pramen.api.sql.SqlColumnType
 import za.co.absa.pramen.core.reader.TableReaderSpark
 
 object TableReaderSparkFactory {
@@ -24,9 +26,11 @@ object TableReaderSparkFactory {
                      schemaOpt: Option[String] = None,
                      hasInfoDateColumn: Boolean = false,
                      infoDateColumn: String = "info_date",
+                     infoDateType: SqlColumnType = SqlColumnType.DATE,
                      infoDateFormat: String = "yyyy-MM-dd",
+                     offsetInfoOpt: Option[OffsetInfo] = None,
                      options: Map[String, String] = Map.empty[String, String]
                     )(implicit spark: SparkSession): TableReaderSpark = {
-    new TableReaderSpark(formatOpt, schemaOpt, hasInfoDateColumn, infoDateColumn, infoDateFormat, options)
+    new TableReaderSpark(formatOpt, schemaOpt, hasInfoDateColumn, infoDateColumn, infoDateType, infoDateFormat, offsetInfoOpt, options)
   }
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/runner/ConcurrentJobRunnerSpy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/runner/ConcurrentJobRunnerSpy.scala
index 52464e331..d84172d3c 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/runner/ConcurrentJobRunnerSpy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/runner/ConcurrentJobRunnerSpy.scala
@@ -68,7 +68,7 @@ class ConcurrentJobRunnerSpy(includeFails: Boolean = false,
     var idx = 0
     incomingJobs.foreach(job => {
       val status = if (!includeFails || idx % 3 == 0) {
-        RunStatus.Succeeded(Some(1000), 500, Some(10000), TaskRunReason.New, Nil, Nil, Nil, Nil)
+        RunStatus.Succeeded(Some(1000), Some(500), None, Some(10000), TaskRunReason.New, Nil, Nil, Nil, Nil)
       } else if (idx % 3 == 1) {
         RunStatus.Failed(new RuntimeException("Dummy exception"))
       } else {
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/source/SourceMock.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/source/SourceMock.scala
index 7bac6902e..a2ff96228 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/source/SourceMock.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/source/SourceMock.scala
@@ -19,6 +19,7 @@ package za.co.absa.pramen.core.mocks.source
 import com.typesafe.config.{Config, ConfigFactory}
 import org.apache.spark.sql.SparkSession
 import za.co.absa.pramen.api._
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.core.utils.ConfigUtils
 
 import java.time.LocalDate
@@ -73,6 +74,8 @@ class SourceMock(
     val df = List(("A", 1), ("B", 2), ("C", 3)).toDF("a", "b")
     SourceResult(df, Nil, Nil)
   }
+
+  override def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): SourceResult = ???
 }
 
 object SourceMock extends ExternalChannelFactory[SourceMock] {
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/state/PipelineStateSpy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/state/PipelineStateSpy.scala
index 96c0d0b4e..fd68398bd 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/state/PipelineStateSpy.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/state/PipelineStateSpy.scala
@@ -33,13 +33,15 @@ class PipelineStateSpy extends PipelineState {
   var closeCalled = 0
   var sparkAppId: Option[String] = None
 
-  override def getState(): PipelineStateSnapshot = {
+  override def getState: PipelineStateSnapshot = {
     PipelineStateSnapshotFactory.getDummyPipelineStateSnapshot(PipelineInfoFactory.getDummyPipelineInfo(sparkApplicationId = sparkAppId),
       customShutdownHookCanRun = setShutdownHookCanRunCount > 0,
       taskResults = completedStatuses.toList
     )
   }
 
+  override def getBatchId: Long = 0L
+
   override def setShutdownHookCanRun(): Unit = synchronized {
     setShutdownHookCanRunCount += 1
   }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/IngestionJobSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/IngestionJobSuite.scala
index 080bd80b3..026ef5a42 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/IngestionJobSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/IngestionJobSuite.scala
@@ -258,7 +258,7 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
     "return Ready" in {
       val (_, _, job) = getUseCase(sourceTable = "empty")
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason: TaskRunReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -268,7 +268,7 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
     "get the source data frame" in {
       val (_, _, job) = getUseCase()
 
-      val runResult = job.run(infoDate, conf)
+      val runResult = job.run(infoDate, runReason: TaskRunReason, conf)
 
       val df = runResult.data
 
@@ -291,7 +291,7 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
         assert(preRunCheck.inputRecordsCount.contains(4))
         assert(TransientTableManager.hasDataForTheDate("source_cache://jdbc|company|2022-02-18", infoDate))
 
-        val runResult = job.run(infoDate, conf)
+        val runResult = job.run(infoDate, runReason: TaskRunReason, conf)
 
         val df = runResult.data
 
@@ -311,7 +311,7 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
       val (_, _, job) = getUseCase(disableCountQuery = true)
 
       val ex = intercept[IllegalArgumentException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason: TaskRunReason, conf)
       }
 
       assert(ex.getMessage.contains("a temporary directory in Hadoop (HDFS, S3, etc) should be set at 'pramen.temporary.directory'"))
@@ -321,7 +321,7 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
       val (_, _, job) = getUseCase(sourceTable = "noSuchTable")
 
       val ex = intercept[SQLSyntaxErrorException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason: TaskRunReason, conf)
       }
 
       assert(ex.getMessage.contains("NOSUCHTABLE"))
@@ -349,7 +349,7 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
           |} ]""".stripMargin
       val (_, _, job) = getUseCase()
 
-      val runResult = job.run(infoDate, conf)
+      val runResult = job.run(infoDate, runReason: TaskRunReason, conf)
 
       val dfIn = runResult.data
 
@@ -365,11 +365,11 @@ class IngestionJobSuite extends AnyWordSpec with SparkTestBase with TextComparis
     "save the dataframe to the metastore" in {
       val (_, mt, job) = getUseCase()
 
-      val saveResult = job.save(exampleDf, infoDate, conf, Instant.now(), Some(150))
+      val saveResult = job.save(exampleDf, infoDate, runReason: TaskRunReason, conf, Instant.now(), Some(150))
 
       val stats = saveResult.stats
 
-      assert(stats.recordCount == 3)
+      assert(stats.recordCount.contains(3))
       assert(mt.saveTableInvocations.length == 1)
       assert(mt.saveTableInvocations.head._1 == "table1")
       assert(mt.saveTableInvocations.head._2 == infoDate)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/OperationSplitterSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/OperationSplitterSuite.scala
index baffc32be..f5b6089ba 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/OperationSplitterSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/OperationSplitterSuite.scala
@@ -326,7 +326,7 @@ class OperationSplitterSuite extends AnyWordSpec with SparkTestBase {
   }
 
   def getUseCase(conf: Config = appConfig): (OperationSplitter, OperationDef) = {
-    val splitter = new OperationSplitter(conf, new MetastoreSpy(trackDays = 2), new SyncBookkeeperMock)
+    val splitter = new OperationSplitter(conf, new MetastoreSpy(trackDays = 2), new SyncBookkeeperMock, 0L)
     val operation = OperationDefFactory.getDummyOperationDef()
 
     (splitter, operation)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJobSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJobSuite.scala
index e268f076b..763299783 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJobSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/PythonTransformationJobSuite.scala
@@ -132,7 +132,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
     "return Ready" in {
       val (job, _, _, _) = getUseCase()
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -142,7 +142,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
     "run the command line script" in {
       val (job, runner, _, _) = getUseCase(tableDf = exampleDf)
 
-      val runResult = job.run(infoDate, conf)
+      val runResult = job.run(infoDate, runReason, conf)
 
       val df = runResult.data
 
@@ -159,7 +159,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
       )
       val databricksClient = databricksClientOpt.get
 
-      job.run(infoDate, conf)
+      job.run(infoDate, runReason, conf)
 
       val (contents, filename, overwrite) = databricksClient.createFileInvocations.head
       val expectedSubmittedJob = Map("job_setting_1" -> "spark-3.3.1")
@@ -181,7 +181,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
       )
 
       val ex = intercept[RuntimeException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason, conf)
       }
 
       assert(ex.getMessage == "Neither command line options nor databricks client configured correctly for Pramen-Py.")
@@ -191,7 +191,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
       val (job, _, _, _) = getUseCase(tableDf = exampleDf, runException = new IllegalStateException(s"Dummy exception"))
 
       val ex = intercept[RuntimeException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason, conf)
       }
 
       assert(ex.getCause.getMessage == "Dummy exception")
@@ -201,7 +201,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
       val (job, _, _, _) = getUseCase(tableDf = exampleDf, tableException = new NoSuchDatabaseException("Dummy"))
 
       val ex = intercept[RuntimeException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason, conf)
       }
 
       assert(ex.getMessage.contains("Output data not found in the metastore"))
@@ -220,13 +220,13 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
 
   "save" should {
     "update the bookkeeper" in {
-      val statsIn = MetaTableStats(100, None)
+      val statsIn = MetaTableStats(Some(100), None, None)
 
       val (job, _, _, _) = getUseCase(stats = statsIn)
 
       val started = Instant.ofEpochSecond(12345678L)
 
-      val statsOut = job.save(exampleDf, infoDate, conf, started, None).stats
+      val statsOut = job.save(exampleDf, infoDate, runReason, conf, started, None).stats
 
       assert(statsOut == statsIn)
     }
@@ -237,47 +237,47 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
       val started = Instant.ofEpochSecond(12345678L)
 
       val ex = intercept[RuntimeException] {
-        job.save(exampleDf, infoDate, conf, started, None)
+        job.save(exampleDf, infoDate, runReason, conf, started, None)
       }
 
       assert(ex.getMessage.contains("Output data not found in the metastore"))
     }
 
     "allow no records in the output table" in {
-      val statsIn = MetaTableStats(0, None)
+      val statsIn = MetaTableStats(Some(0), None, None)
 
       val (job, _, _, _) = getUseCase(stats = statsIn)
 
       val started = Instant.ofEpochSecond(12345678L)
 
-      val statsOut = job.save(exampleDf, infoDate, conf, started, None).stats
+      val statsOut = job.save(exampleDf, infoDate, runReason, conf, started, None).stats
 
-      assert(statsOut.recordCount == 0)
+      assert(statsOut.recordCount.contains(0))
     }
 
     "throw an exception if no records in the output table" in {
-      val statsIn = MetaTableStats(0, None)
+      val statsIn = MetaTableStats(Some(0), None, None)
 
       val (job, _, _, _) = getUseCase(stats = statsIn, extraOptions = Map("minimum.records" -> "1"))
 
       val started = Instant.ofEpochSecond(12345678L)
 
       val ex = intercept[RuntimeException] {
-        job.save(exampleDf, infoDate, conf, started, None)
+        job.save(exampleDf, infoDate, runReason, conf, started, None)
       }
 
       assert(ex.getMessage.contains("Output table is empty in the metastore"))
     }
 
     "throw an exception if the number of records is less then expected" in {
-      val statsIn = MetaTableStats(9, None)
+      val statsIn = MetaTableStats(Some(9), None, None)
 
       val (job, _, _, _) = getUseCase(stats = statsIn, extraOptions = Map("minimum.records" -> "10"))
 
       val started = Instant.ofEpochSecond(12345678L)
 
       val ex = intercept[RuntimeException] {
-        job.save(exampleDf, infoDate, conf, started, None)
+        job.save(exampleDf, infoDate, runReason, conf, started, None)
       }
 
       assert(ex.getMessage.contains("The transformation returned too few records (9 < 10)"))
@@ -289,7 +289,7 @@ class PythonTransformationJobSuite extends AnyWordSpec with BeforeAndAfterAll wi
       val started = Instant.ofEpochSecond(12345678L)
 
       val ex = intercept[RuntimeException] {
-        job.save(exampleDf, infoDate, conf, started, None)
+        job.save(exampleDf, infoDate, runReason, conf, started, None)
       }
 
       assert(ex.getMessage.contains("Dummy"))
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/SinkJobSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/SinkJobSuite.scala
index 8ad30223d..118d3d40d 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/SinkJobSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/SinkJobSuite.scala
@@ -76,7 +76,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
     "return Ready when the data frame is not empty" in {
       val (job, _) = getUseCase(tableDf = exampleDf)
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -84,7 +84,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
     "return Skip when the data frame is empty" in {
       val (job, _) = getUseCase(tableDf = exampleDf.filter(col("b") > 10))
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result.isInstanceOf[Reason.Skip])
     }
@@ -95,7 +95,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
 
       val (job, _) = getUseCase(sink = sink, tableDf = exampleDf)
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -106,7 +106,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
 
       val (job, _) = getUseCase(sink = sink, tableDf = exampleDf)
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result.isInstanceOf[Reason.NotReady])
     }
@@ -116,7 +116,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
     "returns a dataframe when it is available" in {
       val (job, _) = getUseCase(tableDf = exampleDf)
 
-      val actual = job.run(infoDate, conf).data
+      val actual = job.run(infoDate, runReason, conf).data
 
       assert(actual.count() == 3)
     }
@@ -125,7 +125,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
       val (job, _) = getUseCase(tableException = new RuntimeException("Dummy Exception"))
 
       val ex = intercept[IllegalStateException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason, conf)
       }
 
       assert(ex.getMessage == "Unable to read input table table1 for 2022-01-18.")
@@ -152,7 +152,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
 
       val (job, _) = getUseCase(sinkTable = sinkTable, tableDf = exampleDf)
 
-      val dfIn = job.run(infoDate, conf).data
+      val dfIn = job.run(infoDate, runReason, conf).data
 
       val dfOut = job.postProcessing(dfIn, infoDate, conf).orderBy("b1")
 
@@ -170,7 +170,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
 
       val (job, _) = getUseCase(sinkTable = sinkTable, tableDf = exampleDf)
 
-      val dfIn = job.run(infoDate, conf).data
+      val dfIn = job.run(infoDate, runReason, conf).data
 
       val ex = intercept[IllegalStateException] {
         job.postProcessing(dfIn, infoDate, conf).orderBy("b1")
@@ -188,7 +188,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
 
       val (job, bk) = getUseCase(tableDf = df, sink = sink)
 
-      job.save(df, infoDate, conf, Instant.now(), Some(10L))
+      job.save(df, infoDate, runReason, conf, Instant.now(), Some(10L))
 
       val outputTable = "table1->mysink"
 
@@ -210,7 +210,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
       val (job, _) = getUseCase(tableDf = exampleDf, sink = sink)
 
       val ex = intercept[IllegalStateException] {
-        job.save(exampleDf, infoDate, conf, Instant.now(), Some(10L))
+        job.save(exampleDf, infoDate, runReason, conf, Instant.now(), Some(10L))
       }
 
       assert(sink.connectCalled == 1)
@@ -226,7 +226,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
       val (job, _) = getUseCase(tableDf = exampleDf, sink = sink)
 
       val ex = intercept[IllegalStateException] {
-        job.save(exampleDf, infoDate, conf, Instant.now(), Some(10L))
+        job.save(exampleDf, infoDate, runReason, conf, Instant.now(), Some(10L))
       }
 
       assert(sink.connectCalled == 1)
@@ -241,7 +241,7 @@ class SinkJobSuite extends AnyWordSpec with SparkTestBase with TextComparisonFix
 
       val (job, _) = getUseCase(tableDf = exampleDf, sink = sink)
 
-      job.save(exampleDf, infoDate, conf, Instant.now(), Some(10L))
+      job.save(exampleDf, infoDate, runReason, conf, Instant.now(), Some(10L))
 
       assert(sink.connectCalled == 1)
       assert(sink.writeCalled == 1)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransferJobSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransferJobSuite.scala
index 0acd9e204..5217ebef2 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransferJobSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransferJobSuite.scala
@@ -103,7 +103,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
     "return Ready when there is some data" in {
       val (job, _) = getUseCase()
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -111,7 +111,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
     "return Ready when the data frame is empty" in {
       val (job, _) = getUseCase(numberOfRecords = 0)
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -119,7 +119,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
     "return Ready even when the reader throws" in {
       val (job, _) = getUseCase(getDataException = new RuntimeException("Dummy Exception"))
 
-      val result = job.validate(infoDate, conf)
+      val result = job.validate(infoDate, runReason, conf)
 
       assert(result == Reason.Ready)
     }
@@ -129,7 +129,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
     "returns a dataframe when it is available" in {
       val (job, _) = getUseCase()
 
-      val actual = job.run(infoDate, conf).data
+      val actual = job.run(infoDate, runReason, conf).data
 
       assert(actual.count() == 5)
     }
@@ -143,7 +143,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
         assert(preRunCheck.inputRecordsCount.contains(5))
         assert(TransientTableManager.hasDataForTheDate("source_cache://testsource|table1|2022-01-18", infoDate))
 
-        val runResult = job.run(infoDate, conf)
+        val runResult = job.run(infoDate, runReason, conf)
 
         val df = runResult.data
 
@@ -158,7 +158,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
       val (job, _) = getUseCase(disableCountQuery = true)
 
       val ex = intercept[IllegalArgumentException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason, conf)
       }
 
       assert(ex.getMessage.contains("a temporary directory in Hadoop (HDFS, S3, etc) should be set at 'pramen.temporary.directory'"))
@@ -168,7 +168,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
       val (job, _) = getUseCase(getDataException = new RuntimeException("Dummy Exception"))
 
       val ex = intercept[RuntimeException] {
-        job.run(infoDate, conf)
+        job.run(infoDate, runReason, conf)
       }
 
       assert(ex.getMessage == "Dummy Exception")
@@ -201,7 +201,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
 
       val (job, _) = getUseCase(transferTable = transferTable)
 
-      val dfIn = job.run(infoDate, conf).data
+      val dfIn = job.run(infoDate, runReason, conf).data
 
       val dfOut = job.postProcessing(dfIn, infoDate, conf).orderBy("b1")
 
@@ -219,7 +219,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
 
       val (job, _) = getUseCase(transferTable = transferTable)
 
-      val dfIn = job.run(infoDate, conf).data
+      val dfIn = job.run(infoDate, runReason, conf).data
 
       assertThrows[AnalysisException] {
         job.postProcessing(dfIn, infoDate, conf)
@@ -231,7 +231,7 @@ class TransferJobSuite extends AnyWordSpec with SparkTestBase with TextCompariso
     "update the bookkeeper after the save" in {
       val (job, bk) = getUseCase()
 
-      job.save(exampleDf, infoDate, conf, Instant.now(), Some(10L))
+      job.save(exampleDf, infoDate, runReason, conf, Instant.now(), Some(10L))
 
       val outputTable = "table1->sink"
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransformationJobSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransformationJobSuite.scala
index 7b9cb1f5f..5cf43cf21 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransformationJobSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/pipeline/TransformationJobSuite.scala
@@ -52,7 +52,7 @@ class TransformationJobSuite extends AnyWordSpec with SparkTestBase {
     "return Ready when the underlying validation returns ready" in {
       val (job, _) = getUseCase(validateFunction = () => Reason.Ready)
 
-      val actual = job.validate(infoDate, conf)
+      val actual = job.validate(infoDate, runReason, conf)
 
       assert(actual == Reason.Ready)
     }
@@ -60,7 +60,7 @@ class TransformationJobSuite extends AnyWordSpec with SparkTestBase {
     "return NotReady when the underlying validation returns not ready" in {
       val (job, _) = getUseCase(validateFunction = () => Reason.NotReady("DummyNotReady"))
 
-      val actual = job.validate(infoDate, conf)
+      val actual = job.validate(infoDate, runReason, conf)
 
       assert(actual == Reason.NotReady("DummyNotReady"))
     }
@@ -68,7 +68,7 @@ class TransformationJobSuite extends AnyWordSpec with SparkTestBase {
     "return Skip when the underlying validation returns skip" in {
       val (job, _) = getUseCase(validateFunction = () => Reason.Skip("DummySkip"))
 
-      val actual = job.validate(infoDate, conf)
+      val actual = job.validate(infoDate, runReason, conf)
 
       assert(actual == Reason.Skip("DummySkip"))
     }
@@ -77,7 +77,7 @@ class TransformationJobSuite extends AnyWordSpec with SparkTestBase {
       val (job, _) = getUseCase(validateFunction = () => throw new IllegalStateException("DummyMsg"))
 
       val ex = intercept[IllegalStateException] {
-        job.validate(infoDate, conf)
+        job.validate(infoDate, runReason, conf)
       }
 
       assert(ex.getMessage == "DummyMsg")
@@ -88,7 +88,7 @@ class TransformationJobSuite extends AnyWordSpec with SparkTestBase {
     "invoke run() method of the transformer" in {
       val (job, _) = getUseCase(runFunction = () => exampleDf)
 
-      val actual = job.run(infoDate, conf).data
+      val actual = job.run(infoDate, runReason, conf).data
 
       assert(actual.count() == 3)
     }
@@ -116,7 +116,7 @@ class TransformationJobSuite extends AnyWordSpec with SparkTestBase {
     "invoke save() method of teh metastore" in {
       val (job, mt) = getUseCase()
 
-      job.save(exampleDf, infoDate, conf, Instant.now(), None)
+      job.save(exampleDf, infoDate, runReason, conf, Instant.now(), None)
 
       assert(mt.saveTableInvocations.length == 1)
       assert(mt.saveTableInvocations.head._1 == "table1")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/samples/RdbExampleTable.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/samples/RdbExampleTable.scala
index 9eebbdb9e..73e94c0be 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/samples/RdbExampleTable.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/samples/RdbExampleTable.scala
@@ -82,6 +82,36 @@ object RdbExampleTable {
     )
   }
 
+  object IncrementalTable extends RdbExampleTable {
+    val tableName: String = "INCREMENTAL_TABLE"
+    val schemaName: String = "PUBLIC"
+    val databaseName: String = "PUBLIC"
+
+    val ddl: String =
+      s"""
+         |CREATE TABLE $tableName (
+         |  id INT NOT NULL,
+         |  name VARCHAR(50) NOT NULL,
+         |  last_updated TIMESTAMP NOT NULL,
+         |  info_date VARCHAR(10) NOT NULL,
+         |  PRIMARY KEY (id))
+         |""".stripMargin
+
+    override val comments: Seq[String] = Seq.empty
+
+    val inserts: Seq[String] = Seq(
+      s"INSERT INTO $tableName VALUES (1,'John', TIMESTAMP '2022-02-18 10:11:00', '2022-02-18')",
+      s"INSERT INTO $tableName VALUES (2,'Jack', TIMESTAMP '2022-02-18 10:22:33', '2022-02-18')",
+      s"INSERT INTO $tableName VALUES (3,'Jill', TIMESTAMP '2022-02-18 10:33:59', '2022-02-18')"
+    )
+
+    val inserts2: Seq[String] = Seq(
+      s"INSERT INTO $tableName VALUES (4,'Mary', TIMESTAMP '2022-02-18 11:11:00', '2022-02-18')",
+      s"INSERT INTO $tableName VALUES (5,'Jane', TIMESTAMP '2022-02-18 11:22:33', '2022-02-18')",
+      s"INSERT INTO $tableName VALUES (6,'Kate', TIMESTAMP '2022-02-18 11:33:59', '2022-02-18')"
+    )
+  }
+
   object Empty extends RdbExampleTable {
     val tableName: String = "empty"
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/schedule/ScheduleSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/schedule/ScheduleSuite.scala
index 46447b841..576b09bbe 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/schedule/ScheduleSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/schedule/ScheduleSuite.scala
@@ -26,6 +26,15 @@ import java.time.{DayOfWeek, LocalDate}
 class ScheduleSuite extends AnyWordSpec {
 
   "Schedule.fromConfig" should {
+    "Deserialize incremental jobs" when {
+      "a normal incremental job is provided" in {
+        val config = ConfigFactory.parseString(s"$SCHEDULE_TYPE_KEY = incremental")
+        val schedule = fromConfig(config)
+
+        assert(schedule == Schedule.Incremental)
+      }
+    }
+
     "Deserialize daily jobs" when {
       "a normal daily job is provided" in {
         val config = ConfigFactory.parseString(s"$SCHEDULE_TYPE_KEY = daily")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/state/PipelineStateSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/state/PipelineStateSuite.scala
index f8d014e2a..6c1dac961 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/state/PipelineStateSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/state/PipelineStateSuite.scala
@@ -38,11 +38,11 @@ class PipelineStateSuite extends AnyWordSpec {
     "set the flag to true" in {
       val stateManager = new PipelineStateImpl()(conf, PramenImpl.instance.notificationBuilder)
 
-      assert(!stateManager.getState().customShutdownHookCanRun)
+      assert(!stateManager.getState.customShutdownHookCanRun)
 
       stateManager.setShutdownHookCanRun()
 
-      assert(stateManager.getState().customShutdownHookCanRun)
+      assert(stateManager.getState.customShutdownHookCanRun)
     }
   }
 
@@ -50,14 +50,14 @@ class PipelineStateSuite extends AnyWordSpec {
     "set the success flag to true" in {
       val stateManager = getMockPipelineState()
 
-      assert(!stateManager.getState().isFinished)
-      assert(!stateManager.getState().exitedNormally)
+      assert(!stateManager.getState.isFinished)
+      assert(!stateManager.getState.exitedNormally)
 
       stateManager.setSuccess()
 
-      assert(stateManager.getState().isFinished)
-      assert(stateManager.getState().exitedNormally)
-      assert(stateManager.getState().pipelineInfo.failureException.isEmpty)
+      assert(stateManager.getState.isFinished)
+      assert(stateManager.getState.exitedNormally)
+      assert(stateManager.getState.pipelineInfo.failureException.isEmpty)
     }
   }
 
@@ -65,14 +65,14 @@ class PipelineStateSuite extends AnyWordSpec {
     "set the failure flag to true" in {
       val stateManager = getMockPipelineState()
 
-      assert(!stateManager.getState().isFinished)
-      assert(!stateManager.getState().exitedNormally)
+      assert(!stateManager.getState.isFinished)
+      assert(!stateManager.getState.exitedNormally)
 
       stateManager.setFailure("test", new RuntimeException("test"))
 
-      assert(stateManager.getState().isFinished)
-      assert(!stateManager.getState().exitedNormally)
-      assert(stateManager.getState().pipelineInfo.failureException.exists(_.isInstanceOf[RuntimeException]))
+      assert(stateManager.getState.isFinished)
+      assert(!stateManager.getState.exitedNormally)
+      assert(stateManager.getState.pipelineInfo.failureException.exists(_.isInstanceOf[RuntimeException]))
     }
   }
 
@@ -80,14 +80,14 @@ class PipelineStateSuite extends AnyWordSpec {
     "add the task completion statuses" in {
       val stateManager = getMockPipelineState()
 
-      assert(stateManager.getState().taskResults.isEmpty)
+      assert(stateManager.getState.taskResults.isEmpty)
 
       stateManager.addTaskCompletion(Seq(
         TaskResultFactory.getDummyTaskResult(runStatus = RunStatus.Failed(new RuntimeException("test")))
       ))
 
-      assert(stateManager.getState().taskResults.size == 1)
-      assert(stateManager.getState().exitCode == 2)
+      assert(stateManager.getState.taskResults.size == 1)
+      assert(stateManager.getState.exitCode == 2)
     }
   }
 
@@ -127,7 +127,7 @@ class PipelineStateSuite extends AnyWordSpec {
       stateManager.runCustomShutdownHook()
 
       assert(ShutdownHookFailureMock.ranTimes > 0)
-      assert(stateManager.getState().pipelineInfo.failureException.exists(_.isInstanceOf[LinkageError]))
+      assert(stateManager.getState.pipelineInfo.failureException.exists(_.isInstanceOf[LinkageError]))
     }
 
     "handle class does not exists errors" in {
@@ -137,9 +137,8 @@ class PipelineStateSuite extends AnyWordSpec {
       stateManager.runCustomShutdownHook()
 
       assert(ShutdownHookFailureMock.ranTimes > 0)
-      assert(stateManager.getState().pipelineInfo.failureException.isDefined)
-      assert(stateManager.getState().pipelineInfo.failureException.exists(_.isInstanceOf[ClassNotFoundException]))
-
+      assert(stateManager.getState.pipelineInfo.failureException.isDefined)
+      assert(stateManager.getState.pipelineInfo.failureException.exists(_.isInstanceOf[ClassNotFoundException]))
     }
   }
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperJdbcSuite.scala
index 2a39ec42a..47d6c87e3 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperJdbcSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperJdbcSuite.scala
@@ -38,7 +38,7 @@ class BookkeeperJdbcSuite extends BookkeeperCommonSuite with RelationalDbFixture
   }
 
   def getBookkeeper: Bookkeeper = {
-    new BookkeeperJdbc(pramenDb.slickDb)
+    new BookkeeperJdbc(pramenDb.slickDb, 0L)
   }
 
   "BookkeeperJdbc" when {
@@ -49,6 +49,7 @@ class BookkeeperJdbcSuite extends BookkeeperCommonSuite with RelationalDbFixture
         assert(getTables.exists(_.equalsIgnoreCase("bookkeeping")))
         assert(getTables.exists(_.equalsIgnoreCase("schemas")))
         assert(getTables.exists(_.equalsIgnoreCase("metadata")))
+        assert(getTables.exists(_.equalsIgnoreCase("offsets")))
       }
     }
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperSuite.scala
index 7fbbc505e..99345ca2b 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperSuite.scala
@@ -18,16 +18,16 @@ package za.co.absa.pramen.core.tests.bookkeeper
 
 import org.scalatest.wordspec.AnyWordSpec
 import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
-import za.co.absa.pramen.core.{BookkeepingConfigFactory, RuntimeConfigFactory}
-import za.co.absa.pramen.core.app.config.{BookkeeperConfig, HadoopFormat, RuntimeConfig}
+import za.co.absa.pramen.core.app.config.{HadoopFormat, RuntimeConfig}
 import za.co.absa.pramen.core.base.SparkTestBase
-import za.co.absa.pramen.core.bookkeeper.{Bookkeeper, BookkeeperDeltaPath, BookkeeperJdbc, BookkeeperMongoDb, BookkeeperText}
+import za.co.absa.pramen.core.bookkeeper._
 import za.co.absa.pramen.core.fixtures.{MongoDbFixture, RelationalDbFixture, TempDirFixture}
 import za.co.absa.pramen.core.journal.{JournalHadoopCsv, JournalHadoopDeltaPath, JournalJdbc, JournalMongoDb}
 import za.co.absa.pramen.core.lock.{TokenLockFactoryHadoopPath, TokenLockFactoryJdbc, TokenLockFactoryMongoDb}
 import za.co.absa.pramen.core.metadata.{MetadataManagerJdbc, MetadataManagerNull}
 import za.co.absa.pramen.core.rdb.PramenDb
 import za.co.absa.pramen.core.reader.model.JdbcConfig
+import za.co.absa.pramen.core.{BookkeepingConfigFactory, RuntimeConfigFactory}
 
 class BookkeeperSuite extends AnyWordSpec
   with MongoDbFixture
@@ -68,7 +68,7 @@ class BookkeeperSuite extends AnyWordSpec
         bookkeepingJdbcConfig = Some(jdbcConfig)
       )
 
-      val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig)
+      val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig, 0L)
 
       assert(bk.isInstanceOf[BookkeeperJdbc])
       assert(tf.isInstanceOf[TokenLockFactoryJdbc])
@@ -85,7 +85,7 @@ class BookkeeperSuite extends AnyWordSpec
           bookkeepingDbName = Some(dbName)
         )
 
-        val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig)
+        val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig, 0L)
 
         assert(bk.isInstanceOf[BookkeeperMongoDb])
         assert(tf.isInstanceOf[TokenLockFactoryMongoDb])
@@ -106,7 +106,7 @@ class BookkeeperSuite extends AnyWordSpec
           bookkeepingLocation = Some(tempDir)
         )
 
-        val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig)
+        val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig, 0L)
 
         assert(bk.isInstanceOf[BookkeeperText])
         assert(tf.isInstanceOf[TokenLockFactoryHadoopPath])
@@ -124,7 +124,7 @@ class BookkeeperSuite extends AnyWordSpec
           bookkeepingHadoopFormat = HadoopFormat.Delta
         )
 
-        val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig)
+        val (bk, tf, journal, metadataManager, closable) = Bookkeeper.fromConfig(bookkeepingConfig, runtimeConfig, 0L)
 
         assert(bk.isInstanceOf[BookkeeperDeltaPath])
         assert(tf.isInstanceOf[TokenLockFactoryHadoopPath])
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/OffsetManagerJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/OffsetManagerJdbcSuite.scala
new file mode 100644
index 000000000..e0896d60b
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/OffsetManagerJdbcSuite.scala
@@ -0,0 +1,411 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.bookkeeper
+
+import org.scalatest.wordspec.AnyWordSpec
+import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
+import za.co.absa.pramen.api.offset.DataOffset.{CommittedOffset, UncommittedOffset}
+import za.co.absa.pramen.api.offset.{OffsetType, OffsetValue}
+import za.co.absa.pramen.core.bookkeeper.{OffsetManager, OffsetManagerJdbc}
+import za.co.absa.pramen.core.fixtures.RelationalDbFixture
+import za.co.absa.pramen.core.rdb.PramenDb
+import za.co.absa.pramen.core.reader.model.JdbcConfig
+
+import java.time.{Instant, LocalDate}
+
+class OffsetManagerJdbcSuite extends AnyWordSpec with RelationalDbFixture with BeforeAndAfter with BeforeAndAfterAll {
+  val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
+  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+
+  private val infoDate = LocalDate.of(2023, 8, 25)
+
+  before {
+    pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
+    pramenDb.setupDatabase()
+  }
+
+  override def afterAll(): Unit = {
+    pramenDb.close()
+    super.afterAll()
+  }
+
+  def getOffsetManager: OffsetManager = {
+    new OffsetManagerJdbc(pramenDb.slickDb, 123L)
+  }
+
+  "getOffsets" should {
+    "should return an empty array if nothing to return " in {
+      val om = getOffsetManager
+
+      val actual = om.getOffsets("table1", infoDate)
+
+      assert(actual.isEmpty)
+    }
+
+    "return uncommitted offsets" in {
+      val now = Instant.now()
+      val nextHour = now.plusSeconds(3600)
+      val om = getOffsetManager
+
+      om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+
+      val actualNonEmpty = om.getOffsets("table1", infoDate)
+      val actualEmpty1 = om.getOffsets("table1", infoDate.plusDays(1))
+      val actualEmpty2 = om.getOffsets("table1", infoDate.minusDays(1))
+
+      assert(actualEmpty1.isEmpty)
+      assert(actualEmpty2.isEmpty)
+
+      assert(actualNonEmpty.nonEmpty)
+      assert(actualNonEmpty.length == 1)
+
+      val offset = actualNonEmpty.head.asInstanceOf[UncommittedOffset]
+
+      assert(offset.infoDate == infoDate)
+      assert(!offset.createdAt.isBefore(now))
+      assert(offset.createdAt.isBefore(nextHour))
+    }
+
+    "return committed offsets" in {
+      val om = getOffsetManager
+
+      val transactionReference = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      Thread.sleep(10)
+      om.commitOffsets(transactionReference, OffsetValue.IntegralValue(1), OffsetValue.IntegralValue(100))
+
+      val actualNonEmpty = om.getOffsets("table1", infoDate)
+      val actualEmpty1 = om.getOffsets("table1", infoDate.plusDays(1))
+      val actualEmpty2 = om.getOffsets("table1", infoDate.minusDays(1))
+
+      assert(actualEmpty1.isEmpty)
+      assert(actualEmpty2.isEmpty)
+
+      assert(actualNonEmpty.nonEmpty)
+      assert(actualNonEmpty.length == 1)
+
+      val offset = actualNonEmpty.head.asInstanceOf[CommittedOffset]
+
+      val now = Instant.now()
+      val nextHour = now.plusSeconds(3600)
+
+      assert(offset.infoDate == infoDate)
+      assert(!offset.createdAt.isAfter(now))
+      assert(offset.createdAt.isBefore(nextHour))
+      assert(offset.committedAt.isBefore(now))
+      assert(offset.committedAt.isBefore(nextHour))
+      assert(offset.committedAt.isAfter(actualNonEmpty.head.createdAt))
+      assert(offset.minOffset == OffsetValue.IntegralValue(1))
+      assert(offset.maxOffset == OffsetValue.IntegralValue(100))
+    }
+  }
+
+  "getMaxInfoDateAndOffset" should {
+    "return None when no offsets have loaded yet" in {
+      val om = getOffsetManager
+
+      val actual = om.getMaxInfoDateAndOffset("table1", None)
+
+      assert(actual.isEmpty)
+    }
+
+    "return None when no offsets have loaded yet for the info date" in {
+      val om = getOffsetManager
+
+      val actual = om.getMaxInfoDateAndOffset("table1", Some(infoDate))
+
+      assert(actual.isEmpty)
+    }
+
+    "return the maximum info date and offsets " in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(1), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(101), OffsetValue.IntegralValue(200))
+      Thread.sleep(10)
+
+      val t3 = om.startWriteOffsets("table1", infoDate.plusDays(1), OffsetType.IntegralType)
+      om.commitOffsets(t3, OffsetValue.IntegralValue(201), OffsetValue.IntegralValue(300))
+
+      val summaryMultiDay = om.getMaxInfoDateAndOffset("table1", None)
+      val summarySingleDay = om.getMaxInfoDateAndOffset("table1", Some(infoDate))
+
+      assert(summaryMultiDay.isDefined)
+      assert(summarySingleDay.isDefined)
+
+      val actualM = summaryMultiDay.get
+      val actual1 = summarySingleDay.get
+
+      assert(actualM.tableName == "table1")
+      assert(actualM.maximumInfoDate == infoDate.plusDays(1))
+      assert(actualM.minimumOffset == OffsetValue.IntegralValue(201))
+      assert(actualM.maximumOffset == OffsetValue.IntegralValue(300))
+      assert(actualM.offsetsForTheDay.length == 1)
+
+      assert(actual1.tableName == "table1")
+      assert(actual1.maximumInfoDate == infoDate)
+      assert(actual1.minimumOffset == OffsetValue.IntegralValue(1))
+      assert(actual1.maximumOffset == OffsetValue.IntegralValue(200))
+      assert(actual1.offsetsForTheDay.length == 2)
+    }
+  }
+
+  "committing offsets" should {
+    "work for empty offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(10), OffsetValue.IntegralValue(100))
+
+      val offsets = om.getOffsets("table1", infoDate)
+
+      assert(offsets.length == 1)
+
+      val offset = offsets.head.asInstanceOf[CommittedOffset]
+      assert(offset.tableName == "table1")
+      assert(offset.infoDate == infoDate)
+      assert(offset.minOffset == OffsetValue.IntegralValue(10))
+      assert(offset.maxOffset == OffsetValue.IntegralValue(100))
+    }
+
+    "work for non-empty offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(10), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(101), OffsetValue.IntegralValue(200))
+
+      val offsets = om.getOffsets("table1", infoDate)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+
+      assert(offsets.length == 2)
+
+      val offset1 = offsets.head
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset == OffsetValue.IntegralValue(10))
+      assert(offset1.maxOffset == OffsetValue.IntegralValue(100))
+
+      val offset2 = offsets(1)
+      assert(offset2.tableName == "table1")
+      assert(offset2.infoDate == infoDate)
+      assert(offset2.minOffset == OffsetValue.IntegralValue(101))
+      assert(offset2.maxOffset == OffsetValue.IntegralValue(200))
+    }
+  }
+
+  "committing reruns" should {
+    "work with reruns that do not change offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(0), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(101), OffsetValue.IntegralValue(200))
+      Thread.sleep(10)
+
+      val t3 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitRerun(t3, OffsetValue.IntegralValue(0), OffsetValue.IntegralValue(200))
+
+      val offsets = om.getOffsets("table1", infoDate)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+
+      assert(offsets.length == 1)
+
+      val offset1 = offsets.head
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset.valueString == "0")
+      assert(offset1.maxOffset.valueString == "200")
+    }
+
+    "work with reruns that do change offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate.minusDays(1), OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(0), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(101), OffsetValue.IntegralValue(200))
+      Thread.sleep(10)
+
+      val t3 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitRerun(t3, OffsetValue.IntegralValue(201), OffsetValue.IntegralValue(300))
+
+      val offsets = om.getOffsets("table1", infoDate)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+
+      assert(offsets.length == 1)
+
+      val offset1 = offsets.head
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset.valueString == "201")
+      assert(offset1.maxOffset.valueString == "300")
+    }
+
+    "work with reruns that deletes all data and no previous offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(0), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(101), OffsetValue.IntegralValue(200))
+      Thread.sleep(10)
+
+      val t3 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitRerun(t3, OffsetValue.IntegralValue(-1), OffsetValue.IntegralValue(-1))
+
+      val offsets = om.getOffsets("table1", infoDate)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+
+      assert(offsets.length == 1)
+
+      val offset1 = offsets.head
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset.valueString.toLong == -1)
+      assert(offset1.maxOffset.valueString.toLong == -1)
+    }
+
+    "work with reruns that deletes all data and there are previous offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate.minusDays(1), OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(Long.MinValue), OffsetValue.IntegralValue(0))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(1), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t3 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitRerun(t3, OffsetValue.IntegralValue(0), OffsetValue.IntegralValue(0))
+
+      val offsets = om.getOffsets("table1", infoDate)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+
+      assert(offsets.length == 1)
+
+      val offset1 = offsets.head
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset.valueString.toLong == 0)
+      assert(offset1.maxOffset.valueString.toLong == 0)
+    }
+
+    "throw an exception when offsets are incorrect" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(Long.MinValue), OffsetValue.IntegralValue(0))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t2, OffsetValue.IntegralValue(1), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t3 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      val ex = intercept[IllegalArgumentException] {
+        om.commitRerun(t3, OffsetValue.IntegralValue(1), OffsetValue.IntegralValue(-1))
+      }
+
+      assert(ex.getMessage == "minOffset is greater than maxOffset: 1 > -1")
+
+      val offsets0 = om.getOffsets("table1", infoDate)
+        .filter(_.isCommitted)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+      val offsets1 = om.getOffsets("table1", infoDate)
+        .filter(!_.isCommitted)
+        .map(_.asInstanceOf[UncommittedOffset])
+
+      assert(offsets0.length == 2)
+      assert(offsets1.length == 1)
+
+      val offset1 = offsets0.head
+      val offset2 = offsets0(1)
+      val offset3 = offsets1.head
+
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset.valueString.toLong == Long.MinValue)
+      assert(offset1.maxOffset.valueString.toLong == 0)
+
+      assert(offset2.tableName == "table1")
+      assert(offset2.infoDate == infoDate)
+      assert(offset2.minOffset.valueString.toLong == 1)
+      assert(offset2.maxOffset.valueString.toLong == 100)
+
+      assert(offset3.tableName == "table1")
+      assert(offset3.infoDate == infoDate)
+    }
+  }
+
+  "rollback offsets" should {
+    "work for empty offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.rollbackOffsets(t1)
+
+      val offsets = om.getOffsets("table1", infoDate)
+
+      assert(offsets.isEmpty)
+    }
+
+    "work for non-empty offsets" in {
+      val om = getOffsetManager
+
+      val t1 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.commitOffsets(t1, OffsetValue.IntegralValue(0), OffsetValue.IntegralValue(100))
+      Thread.sleep(10)
+
+      val t2 = om.startWriteOffsets("table1", infoDate, OffsetType.IntegralType)
+      om.rollbackOffsets(t2)
+
+      val offsets = om.getOffsets("table1", infoDate)
+        .map(_.asInstanceOf[CommittedOffset])
+        .sortBy(_.minOffset.valueString.toLong)
+
+      assert(offsets.length == 1)
+
+      val offset1 = offsets.head
+      assert(offset1.tableName == "table1")
+      assert(offset1.infoDate == infoDate)
+      assert(offset1.minOffset == OffsetValue.IntegralValue(0))
+      assert(offset1.maxOffset == OffsetValue.IntegralValue(100))
+    }
+  }
+
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TaskCompletedSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TaskCompletedSuite.scala
index d799e2f42..0e53a3af7 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TaskCompletedSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TaskCompletedSuite.scala
@@ -50,7 +50,7 @@ class TaskCompletedSuite extends AnyWordSpec {
       val taskResult = TaskResult(
         job.name,
         MetaTable.getMetaTableDef(job.outputTable),
-        RunStatus.Succeeded(Some(1000), 2000, Some(3000), runReason, Nil, Nil, Nil, Nil),
+        RunStatus.Succeeded(Some(1000), Some(2000), None, Some(3000), runReason, Nil, Nil, Nil, Nil),
         Some(RunInfo(infoDate, now.minusSeconds(10), now)),
         "app_123",
         isTransient = false,
@@ -66,8 +66,8 @@ class TaskCompletedSuite extends AnyWordSpec {
       assert(taskCompleted.periodBegin == infoDate)
       assert(taskCompleted.periodEnd == infoDate)
       assert(taskCompleted.informationDate == infoDate)
-      assert(taskCompleted.inputRecordCount == 2000)
-      assert(taskCompleted.inputRecordCountOld == 1000)
+      assert(taskCompleted.inputRecordCount.contains(2000))
+      assert(taskCompleted.inputRecordCountOld.contains(1000))
       assert(taskCompleted.outputRecordCount.contains(2000))
       assert(taskCompleted.outputRecordCountOld.contains(1000))
       assert(taskCompleted.outputSize.contains(3000))
@@ -106,8 +106,8 @@ class TaskCompletedSuite extends AnyWordSpec {
       assert(taskCompleted.periodBegin == infoDate)
       assert(taskCompleted.periodEnd == infoDate)
       assert(taskCompleted.informationDate == infoDate)
-      assert(taskCompleted.inputRecordCount == 0)
-      assert(taskCompleted.inputRecordCountOld == 0)
+      assert(taskCompleted.inputRecordCount.isEmpty)
+      assert(taskCompleted.inputRecordCountOld.isEmpty)
       assert(taskCompleted.outputRecordCount.isEmpty)
       assert(taskCompleted.outputRecordCountOld.isEmpty)
       assert(taskCompleted.outputSize.isEmpty)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TestCases.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TestCases.scala
index a2672320d..306bc15c8 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TestCases.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/TestCases.scala
@@ -30,7 +30,7 @@ object TestCases {
   val instant2: Instant = Instant.ofEpochSecond(1597318835)
   val instant3: Instant = Instant.ofEpochSecond(1597318839)
 
-  val task1: TaskCompleted = model.TaskCompleted("job1", "table1", infoDate1, infoDate1, infoDate1, 100, 0, Some(100), None, None, 597318830, 1597318830, "New", Some("Test1"), Some("abc123"), Some("p_id_1"), Some("p_1"), Some("TEST"), Some("T1"))
-  val task2: TaskCompleted = model.TaskCompleted("job1", "table1", infoDate2, infoDate2, infoDate2, 100, 0, Some(100), None, None, 1597318835, 1597318835, "Late", Some("Test2"), Some("abc123"), Some("p_id_2"), Some("p_2"), Some("TEST"), Some("T2"))
-  val task3: TaskCompleted = model.TaskCompleted("job2", "table2", infoDate3, infoDate3, infoDate3, 100, 0, Some(100), None, None, 1597318839, 1597318839, "Fail", Some("Test3"), Some("abc123"), Some("p_id_3"), Some("p_2"), Some("TEST"), Some("T2"))
+  val task1: TaskCompleted = model.TaskCompleted("job1", "table1", infoDate1, infoDate1, infoDate1, Some(100), Some(0), Some(100), None, None, None, 597318830, 1597318830, "New", Some("Test1"), Some("abc123"), Some("p_id_1"), Some("p_1"), Some("TEST"), Some("T1"))
+  val task2: TaskCompleted = model.TaskCompleted("job1", "table1", infoDate2, infoDate2, infoDate2, Some(100), Some(0), Some(100), None, None, None, 1597318835, 1597318835, "Late", Some("Test2"), Some("abc123"), Some("p_id_2"), Some("p_2"), Some("TEST"), Some("T2"))
+  val task3: TaskCompleted = model.TaskCompleted("job2", "table2", infoDate3, infoDate3, infoDate3, Some(100), Some(0), Some(100), None, None, None, 1597318839, 1597318839, "Fail", Some("Test3"), Some("abc123"), Some("p_id_3"), Some("p_2"), Some("TEST"), Some("T2"))
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/notify/pipeline/PipelineNotificationBuilderHtmlSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/notify/pipeline/PipelineNotificationBuilderHtmlSuite.scala
index b3210dcad..0cc1ddec6 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/notify/pipeline/PipelineNotificationBuilderHtmlSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/notify/pipeline/PipelineNotificationBuilderHtmlSuite.scala
@@ -402,7 +402,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for a task without a run info" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(None, 1000000, reason = TaskRunReason.New)
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(1000000), reason = TaskRunReason.New)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, runInfo = None)
 
       val actual = builder.getThroughputRps(task)
@@ -413,7 +413,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for a normal successful task" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(None, 1000000, reason = TaskRunReason.New)
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(1000000), reason = TaskRunReason.New)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
 
       val actual = builder.getThroughputRps(task)
@@ -421,10 +421,21 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
       assert(actual.text == "225 r/s")
     }
 
+    "work for an incremental successful task" in {
+      val builder = getBuilder()
+
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(1000000), recordsAppended = Some(500000), reason = TaskRunReason.New)
+      val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
+
+      val actual = builder.getThroughputRps(task)
+
+      assert(actual.text == "112 r/s")
+    }
+
     "work for a raw file task" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(None, 1000 * megabyte, reason = TaskRunReason.New)
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(1000 * megabyte), reason = TaskRunReason.New)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, isRawFilesJob = true)
 
       val actual = builder.getThroughputRps(task)
@@ -460,13 +471,13 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
 
       val actual = builder.getRecordCountText(task)
 
-      assert(actual.isEmpty)
+      assert(actual == "-")
     }
 
     "work for success file based job" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100), 100, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100), Some(100), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, isRawFilesJob = true)
 
       val actual = builder.getRecordCountText(task)
@@ -477,7 +488,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success new" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(None, 100, reason = TaskRunReason.New)
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(100), reason = TaskRunReason.New)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
 
       val actual = builder.getRecordCountText(task)
@@ -488,7 +499,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success unchanged" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100), 100, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100), Some(100), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
 
       val actual = builder.getRecordCountText(task)
@@ -499,7 +510,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success increased" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100), 110, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100), Some(110), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
 
       val actual = builder.getRecordCountText(task)
@@ -510,7 +521,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success decreased" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100), 90, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100), Some(90), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
 
       val actual = builder.getRecordCountText(task)
@@ -518,6 +529,17 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
       assert(actual == "90 (-10)")
     }
 
+    "work for successful appends" in {
+      val builder = getBuilder()
+
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(110), Some(10), reason = TaskRunReason.Update)
+      val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus)
+
+      val actual = builder.getRecordCountText(task)
+
+      assert(actual == "110 (+10)")
+    }
+
     "work for insufficient data" in {
       val builder = getBuilder()
 
@@ -544,7 +566,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success new" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(None, 100 * megabyte, reason = TaskRunReason.New)
+      val runStatus = RunStatusFactory.getDummySuccess(None, Some(100 * megabyte), reason = TaskRunReason.New)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, isRawFilesJob = true)
 
       val actual = builder.getSizeText(task)
@@ -555,7 +577,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success unchanged" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100 * megabyte), 100 * megabyte, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100 * megabyte), Some(100 * megabyte), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, isRawFilesJob = true)
 
       val actual = builder.getSizeText(task)
@@ -566,7 +588,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success increased" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100 * megabyte), 110 * megabyte, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100 * megabyte), Some(110 * megabyte), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, isRawFilesJob = true)
 
       val actual = builder.getSizeText(task)
@@ -577,7 +599,7 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     "work for success decreased" in {
       val builder = getBuilder()
 
-      val runStatus = RunStatusFactory.getDummySuccess(Some(100 * megabyte), 90 * megabyte, reason = TaskRunReason.Update)
+      val runStatus = RunStatusFactory.getDummySuccess(Some(100 * megabyte), Some(90 * megabyte), reason = TaskRunReason.Update)
       val task = TaskResultFactory.getDummyTaskResult(runStatus = runStatus, isRawFilesJob = true)
 
       val actual = builder.getSizeText(task)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcNativeSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcNativeSuite.scala
index 165681494..a9e079774 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcNativeSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcNativeSuite.scala
@@ -97,7 +97,7 @@ class TableReaderJdbcNativeSuite extends AnyWordSpec with RelationalDbFixture wi
   }
 
   private def getReader: TableReaderJdbcNative =
-    TableReaderJdbcNative(conf.getConfig("reader"), "reader")
+    TableReaderJdbcNative(conf.getConfig("reader"), conf.getConfig("reader"), "reader")
 
   "TableReaderJdbcNative factory" should {
     "construct a reader object" in {
@@ -109,13 +109,13 @@ class TableReaderJdbcNativeSuite extends AnyWordSpec with RelationalDbFixture wi
     }
 
     "work with legacy config" in {
-      val reader = TableReaderJdbcNative(conf.getConfig("reader_legacy"), "reader_legacy")
+      val reader = TableReaderJdbcNative(conf.getConfig("reader_legacy"), conf.getConfig("reader_legacy"), "reader_legacy")
       assert(reader.getJdbcReaderConfig.infoDateFormat == "yyyy-MM-DD")
       assert(!reader.getJdbcReaderConfig.jdbcConfig.sanitizeDateTime)
     }
 
     "work with minimal config" in {
-      val reader = TableReaderJdbcNative(conf.getConfig("reader_minimal"), "reader_minimal")
+      val reader = TableReaderJdbcNative(conf.getConfig("reader_minimal"), conf.getConfig("reader_minimal"), "reader_minimal")
       assert(reader.getJdbcReaderConfig.infoDateFormat == "yyyy-MM-dd")
       assert(reader.getJdbcReaderConfig.jdbcConfig.sanitizeDateTime)
       assert(!reader.getJdbcReaderConfig.jdbcConfig.autoCommit)
@@ -124,7 +124,7 @@ class TableReaderJdbcNativeSuite extends AnyWordSpec with RelationalDbFixture wi
 
     "throw an exception if config is missing" in {
       intercept[IllegalArgumentException] {
-        TableReaderJdbcNative(conf)
+        TableReaderJdbcNative(conf, conf)
       }
     }
   }
@@ -293,7 +293,7 @@ class TableReaderJdbcNativeSuite extends AnyWordSpec with RelationalDbFixture wi
     }
 
     "return a query without info date if it is disabled" in {
-      val reader = TableReaderJdbcNative(conf.getConfig("reader_minimal"), "reader_minimal")
+      val reader = TableReaderJdbcNative(conf.getConfig("reader_minimal"), conf.getConfig("reader_minimal"), "reader_minimal")
 
       val actual = reader.getSqlDataQuery("table1", infoDateBegin, infoDateEnd, Nil)
 
@@ -301,7 +301,7 @@ class TableReaderJdbcNativeSuite extends AnyWordSpec with RelationalDbFixture wi
     }
 
     "return a query without with limits" in {
-      val reader = TableReaderJdbcNative(conf.getConfig("reader_limit"), "reader_limit")
+      val reader = TableReaderJdbcNative(conf.getConfig("reader_limit"), conf.getConfig("reader_limit"), "reader_limit")
 
       val actual = reader.getSqlDataQuery("table1", infoDateBegin, infoDateEnd, Nil)
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala
index 77aad6f17..8cfed0f1b 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala
@@ -21,7 +21,7 @@ import org.mockito.Mockito.{mock, when => whenMock}
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.wordspec.AnyWordSpec
 import za.co.absa.pramen.api.Query
-import za.co.absa.pramen.api.sql.QuotingPolicy
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType}
 import za.co.absa.pramen.core.base.SparkTestBase
 import za.co.absa.pramen.core.fixtures.RelationalDbFixture
 import za.co.absa.pramen.core.mocks.SqlGeneratorDummy
@@ -62,6 +62,11 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
          |  information.date.type = "number"
          |  information.date.format = "yyyy-MM-DD"
          |
+         |  offset.column {
+         |    name = "ts"
+         |    type = "datetime"
+         |  }
+         |
          |  identifier.quoting.policy = "never"
          |}
          |reader_legacy {
@@ -94,7 +99,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
          |}""".stripMargin)
 
     "be able to be constructed properly from config" in {
-      val reader = TableReaderJdbc(conf.getConfig("reader"), "reader")
+      val reader = TableReaderJdbc(conf.getConfig("reader"), conf.getConfig("reader"), "reader")
 
       val jdbc = reader.getJdbcConfig
 
@@ -104,8 +109,11 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
       assert(jdbc.jdbcConfig.user.contains(user))
       assert(jdbc.jdbcConfig.password.contains(password))
       assert(jdbc.infoDateColumn == "INFO_DATE")
-      assert(jdbc.infoDateType == "number")
+      assert(jdbc.infoDateType == SqlColumnType.NUMBER)
       assert(jdbc.infoDateFormat == "yyyy-MM-DD")
+      assert(jdbc.offsetInfoOpt.nonEmpty)
+      assert(jdbc.offsetInfoOpt.get.offsetColumn == "ts")
+      assert(jdbc.offsetInfoOpt.get.offsetType.dataTypeString == "datetime")
       assert(jdbc.identifierQuotingPolicy == QuotingPolicy.Never)
       assert(jdbc.sqlGeneratorClass.isEmpty)
       assert(!jdbc.hasInfoDate)
@@ -113,7 +121,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
     }
 
     "be able to be constructed properly from legacy config" in {
-      val reader = TableReaderJdbc(conf.getConfig("reader_legacy"), "reader_legacy")
+      val reader = TableReaderJdbc(conf.getConfig("reader_legacy"), conf.getConfig("reader_legacy"), "reader_legacy")
 
       val jdbc = reader.getJdbcConfig
 
@@ -123,7 +131,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
       assert(jdbc.jdbcConfig.user.contains(user))
       assert(jdbc.jdbcConfig.password.contains(password))
       assert(jdbc.infoDateColumn == "INFO_DATE")
-      assert(jdbc.infoDateType == "date")
+      assert(jdbc.infoDateType == SqlColumnType.DATE)
       assert(jdbc.infoDateFormat == "YYYY-MM-dd")
       assert(jdbc.identifierQuotingPolicy == QuotingPolicy.Auto)
       assert(jdbc.sqlGeneratorClass.contains("za.co.absa.pramen.core.mocks.SqlGeneratorDummy"))
@@ -132,7 +140,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
     }
 
     "be able to be constructed properly from minimal config" in {
-      val reader = TableReaderJdbc(conf.getConfig("reader_minimal"), "reader_minimal")
+      val reader = TableReaderJdbc(conf.getConfig("reader_minimal"), conf.getConfig("reader_minimal"), "reader_minimal")
 
       val jdbc = reader.getJdbcConfig
 
@@ -142,8 +150,9 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
       assert(jdbc.jdbcConfig.user.contains(user))
       assert(jdbc.jdbcConfig.password.contains(password))
       assert(jdbc.infoDateColumn == "INFO_DATE")
-      assert(jdbc.infoDateType == "date")
+      assert(jdbc.infoDateType == SqlColumnType.DATE)
       assert(jdbc.infoDateFormat == "yyyy-MM-dd")
+      assert(jdbc.offsetInfoOpt.isEmpty)
       assert(jdbc.hasInfoDate)
       assert(!jdbc.saveTimestampsAsDates)
       assert(jdbc.identifierQuotingPolicy == QuotingPolicy.Auto)
@@ -151,13 +160,13 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
     }
 
     "ensure sql query generator is properly selected 1" in {
-      val reader = TableReaderJdbc(conf.getConfig("reader"), "reader")
+      val reader = TableReaderJdbc(conf.getConfig("reader"), conf.getConfig("reader"), "reader")
 
       assert(reader.sqlGen.isInstanceOf[SqlGeneratorHsqlDb])
     }
 
     "ensure sql query generator is properly selected 2" in {
-      val reader = TableReaderJdbc(conf.getConfig("reader_legacy"), "reader_legacy")
+      val reader = TableReaderJdbc(conf.getConfig("reader_legacy"), conf.getConfig("reader_legacy"), "reader_legacy")
 
       assert(reader.sqlGen.isInstanceOf[SqlGeneratorDummy])
     }
@@ -166,7 +175,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
       val testConfig = conf
         .withValue("reader.save.timestamps.as.dates", ConfigValueFactory.fromAnyRef(true))
         .withValue("reader.correct.decimals.in.schema", ConfigValueFactory.fromAnyRef(true))
-      val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+      val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
       val jdbc = reader.getJdbcConfig
 
@@ -186,7 +195,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
            |
            |  has.information.date.column = false
            |}""".stripMargin)
-      val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+      val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
       val jdbc = reader.getJdbcConfig
 
@@ -207,19 +216,19 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
            |  information.date.column = "sync_date"
            |  information.date.type = "date"
            |}""".stripMargin)
-      val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+      val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
       val jdbc = reader.getJdbcConfig
 
       assert(jdbc.hasInfoDate)
       assert(jdbc.infoDateColumn == "sync_date")
-      assert(jdbc.infoDateType == "date")
+      assert(jdbc.infoDateType == SqlColumnType.DATE)
     }
 
     "getWithRetry" should {
       "return the successful dataframe on the second try" in {
         val readerConfig = conf.getConfig("reader")
-        val jdbcTableReaderConfig = TableReaderJdbcConfig.load(readerConfig, "reader")
+        val jdbcTableReaderConfig = TableReaderJdbcConfig.load(readerConfig, readerConfig, "reader")
 
         val urlSelector = mock(classOf[JdbcUrlSelector])
 
@@ -238,7 +247,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
 
       "pass the exception when out of retries" in {
         val readerConfig = conf.getConfig("reader")
-        val jdbcTableReaderConfig = TableReaderJdbcConfig.load(readerConfig, "reader")
+        val jdbcTableReaderConfig = TableReaderJdbcConfig.load(readerConfig, readerConfig, "reader")
 
         val urlSelector = mock(classOf[JdbcUrlSelector])
 
@@ -264,7 +273,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("reader.correct.decimals.in.schema", ConfigValueFactory.fromAnyRef(true))
           .withValue("enable.schema.metadata", ConfigValueFactory.fromAnyRef(true))
 
-        val jdbcTableReaderConfig = TableReaderJdbcConfig.load(readerConfig, "reader")
+        val jdbcTableReaderConfig = TableReaderJdbcConfig.load(readerConfig, readerConfig, "reader")
         val urlSelector = JdbcUrlSelector(jdbcTableReaderConfig.jdbcConfig)
 
         val reader = new TableReaderJdbc(jdbcTableReaderConfig, urlSelector, readerConfig)
@@ -284,7 +293,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
     "getCount()" should {
       "return count for a table snapshot-like query" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val count = reader.getRecordCount(Query.Table("company"), null, null)
 
@@ -293,7 +302,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
 
       "return count for a sql snapshot-like query" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val count = reader.getRecordCount(Query.Sql("SELECT * FROM company"), null, null)
 
@@ -307,7 +316,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.type", ConfigValueFactory.fromAnyRef("string"))
           .withValue("information.date.format", ConfigValueFactory.fromAnyRef("yyyy-MM-dd"))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
+        val reader = TableReaderJdbc(testConfig, testConfig, "reader")
 
         val count = reader.getRecordCount(Query.Table("company"), infoDate,  infoDate)
 
@@ -316,7 +325,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
 
       "return count for a snapshot-like SQL" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val count = reader.getRecordCount(Query.Sql("SELECT id FROM company"), infoDate, infoDate)
 
@@ -330,7 +339,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.type", ConfigValueFactory.fromAnyRef("string"))
           .withValue("information.date.format", ConfigValueFactory.fromAnyRef("yyyy-MM-dd"))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
+        val reader = TableReaderJdbc(testConfig, testConfig, "reader")
 
         val count = reader.getRecordCount(Query.Sql("SELECT id, info_date FROM company WHERE info_date BETWEEN '@dateFrom' AND '@dateTo'"), infoDate, infoDate)
 
@@ -341,7 +350,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
     "getCountSqlQuery" should {
       "return a count query for a table snapshot-like query" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val sql = reader.getCountSqlQuery("SELECT * FROM COMPANY", infoDate, infoDate)
 
@@ -355,7 +364,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.type", ConfigValueFactory.fromAnyRef("string"))
           .withValue("information.date.format", ConfigValueFactory.fromAnyRef("yyyy-MM-dd"))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
+        val reader = TableReaderJdbc(testConfig, testConfig, "reader")
 
         val sql = reader.getCountSqlQuery("SELECT * FROM COMPANY WHERE info_date BETWEEN '@dateFrom' AND '@dateTo'", infoDate, infoDate)
 
@@ -370,7 +379,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.type", ConfigValueFactory.fromAnyRef("string"))
           .withValue("information.date.format", ConfigValueFactory.fromAnyRef("yyyy-MM-dd"))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
+        val reader = TableReaderJdbc(testConfig, testConfig, "reader")
 
         val sql = reader.getCountSqlQuery("SELECT * FROM my_db.my_table WHERE info_date = CAST(REPLACE(CAST(CAST('@infoDate' AS DATE) AS VARCHAR(10)), '-', '') AS INTEGER)", infoDate, infoDate)
 
@@ -381,7 +390,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
     "getData()" should {
       "return data for a table snapshot-like query" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val df = reader.getData(Query.Table("company"), null, null, Seq.empty[String])
 
@@ -391,7 +400,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
 
       "return selected column for a table snapshot-like query" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val df = reader.getData(Query.Table("company"), null, null, Seq("id", "name"))
 
@@ -407,7 +416,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.format", ConfigValueFactory.fromAnyRef("yyyy-MM-dd"))
           .withValue("correct.decimals.in.schema", ConfigValueFactory.fromAnyRef(true))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
+        val reader = TableReaderJdbc(testConfig, testConfig, "reader")
 
         val df = reader.getData(Query.Table("company"), infoDate, infoDate, Seq.empty[String])
 
@@ -416,7 +425,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
 
       "return data for a snapshot-like SQL" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val df = reader.getData(Query.Sql("SELECT id FROM company"), infoDate, infoDate, Seq.empty[String])
 
@@ -426,7 +435,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
 
       "return selected columns for a snapshot-like SQL" in {
         val testConfig = conf
-        val reader = TableReaderJdbc(testConfig.getConfig("reader"), "reader")
+        val reader = TableReaderJdbc(testConfig.getConfig("reader"), testConfig.getConfig("reader"), "reader")
 
         val df = reader.getData(Query.Sql("SELECT * FROM company"), infoDate, infoDate, Seq("id", "name"))
 
@@ -441,7 +450,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.type", ConfigValueFactory.fromAnyRef("string"))
           .withValue("information.date.format", ConfigValueFactory.fromAnyRef("yyyy-MM-dd"))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
+        val reader = TableReaderJdbc(testConfig, testConfig, "reader")
 
         val df = reader.getData(Query.Sql("SELECT id, info_date FROM company WHERE info_date BETWEEN '@dateFrom' AND '@dateTo'"), infoDate, infoDate, Seq.empty[String])
 
@@ -456,10 +465,8 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark
           .withValue("information.date.column", ConfigValueFactory.fromAnyRef("info_date"))
           .withValue("information.date.type", ConfigValueFactory.fromAnyRef("not_exist"))
 
-        val reader = TableReaderJdbc(testConfig, "reader")
-
         assertThrows[IllegalArgumentException] {
-          reader.getSqlConfig
+          TableReaderJdbc(testConfig, testConfig, "reader")
         }
       }
     }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderSparkSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderSparkSuite.scala
index 284b9063c..74b592b47 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderSparkSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderSparkSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.types.{DateType, IntegerType, StringType}
 import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.sql.SqlColumnType
 import za.co.absa.pramen.api.{Query, TableReader}
 import za.co.absa.pramen.core.base.SparkTestBase
 import za.co.absa.pramen.core.fixtures.TempDirFixture
@@ -327,7 +328,7 @@ class TableReaderSparkSuite extends AnyWordSpec with SparkTestBase with TempDirF
 
     val query = Query.Path(pathBase.toString)
 
-    (new TableReaderSpark(formatOpt, schemaOpt, hasInfoDate, "info_date", options = effectiveOptions), query)
+    (new TableReaderSpark(formatOpt, schemaOpt, hasInfoDate, "info_date", SqlColumnType.DATE, offsetInfoOpt = None, options = effectiveOptions), query)
   }
 
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/jobrunner/TaskRunnerMultithreadedSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/jobrunner/TaskRunnerMultithreadedSuite.scala
index bbc664450..a736f079b 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/jobrunner/TaskRunnerMultithreadedSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/jobrunner/TaskRunnerMultithreadedSuite.scala
@@ -190,7 +190,7 @@ class TaskRunnerMultithreadedSuite extends AnyWordSpec with SparkTestBase {
 
     bookkeeper.setRecordCount("table_out", runDate.minusDays(1), runDate.minusDays(1), runDate.minusDays(1), 1, 1, 0, 0, isTableTransient = false)
 
-    val stats = MetaTableStats(2, Some(100))
+    val stats = MetaTableStats(Some(2), None, Some(100))
 
     val operationDef = OperationDefFactory.getDummyOperationDef(consumeThreads = consumeThreads)
     val job = new JobSpy(runFunction = runFunction, saveStats = stats, operationDef = operationDef, allowParallel = allowParallel)
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/splitter/ScheduleStrategyIncrementalSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/splitter/ScheduleStrategyIncrementalSuite.scala
new file mode 100644
index 000000000..808f07f29
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/splitter/ScheduleStrategyIncrementalSuite.scala
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.runner.splitter
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.status.TaskRunReason
+import za.co.absa.pramen.core.bookkeeper.BookkeeperNull
+import za.co.absa.pramen.core.runner.splitter.{RunMode, ScheduleParams, ScheduleStrategyIncremental}
+import za.co.absa.pramen.core.schedule.Schedule
+
+import java.time.LocalDate
+
+class ScheduleStrategyIncrementalSuite extends AnyWordSpec {
+  private val minimumDate = LocalDate.of(2021, 1, 1)
+  private val infoDate = LocalDate.of(2021, 2, 18)
+
+  "normal run" when {
+    "info date is defined" should {
+      "run for the current info date when the job have never ran" in {
+        val strategy = new ScheduleStrategyIncremental(None, true)
+        val params = ScheduleParams.Normal(infoDate, 0, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 1)
+        assert(dates.head.infoDate == infoDate)
+        assert(dates.head.reason == TaskRunReason.New)
+      }
+
+      "run for the current info date when the job ran today before" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate), true)
+        val params = ScheduleParams.Normal(infoDate, 0, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 1)
+        assert(dates.head.infoDate == infoDate)
+        assert(dates.head.reason == TaskRunReason.New)
+      }
+
+      "run for the current info date when the job ran several before track days = 2" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate.minusDays(10)), true)
+        val params = ScheduleParams.Normal(infoDate, 2, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 3)
+        assert(dates.head.infoDate == infoDate.minusDays(2))
+        assert(dates.head.reason == TaskRunReason.Late)
+        assert(dates(1).infoDate == infoDate.minusDays(1))
+        assert(dates(1).reason == TaskRunReason.New)
+        assert(dates(2).infoDate == infoDate)
+        assert(dates(2).reason == TaskRunReason.New)
+      }
+
+      "run for the current info date when the job ran several before track days = -1" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate.minusDays(3)), true)
+        val params = ScheduleParams.Normal(infoDate, -1, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 4)
+        assert(dates.head.infoDate == infoDate.minusDays(3))
+        assert(dates.head.reason == TaskRunReason.Late)
+        assert(dates(1).infoDate == infoDate.minusDays(2))
+        assert(dates(1).reason == TaskRunReason.Late)
+        assert(dates(2).infoDate == infoDate.minusDays(1))
+        assert(dates(2).reason == TaskRunReason.New)
+        assert(dates(3).infoDate == infoDate)
+        assert(dates(3).reason == TaskRunReason.New)
+      }
+
+      "run for the current info date new only" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate.minusDays(3)), true)
+        val params = ScheduleParams.Normal(infoDate, -1, 0, newOnly = true, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 2)
+        assert(dates.head.infoDate == infoDate.minusDays(1))
+        assert(dates.head.reason == TaskRunReason.New)
+        assert(dates(1).infoDate == infoDate)
+        assert(dates(1).reason == TaskRunReason.New)
+      }
+
+      "run for the current info date late only" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate.minusDays(3)), true)
+        val params = ScheduleParams.Normal(infoDate, -1, 0, newOnly = false, lateOnly = true)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 2)
+        assert(dates.head.infoDate == infoDate.minusDays(3))
+        assert(dates.head.reason == TaskRunReason.Late)
+        assert(dates(1).infoDate == infoDate.minusDays(2))
+        assert(dates(1).reason == TaskRunReason.Late)
+      }
+    }
+
+    "info date is not defined" should {
+      "run for the current info date when the job have never ran" in {
+        val strategy = new ScheduleStrategyIncremental(None, false)
+        val params = ScheduleParams.Normal(infoDate, 0, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 1)
+        assert(dates.head.infoDate == infoDate)
+        assert(dates.head.reason == TaskRunReason.New)
+      }
+
+      "run for the current info date when the job ran today before" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate), false)
+        val params = ScheduleParams.Normal(infoDate, 0, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 1)
+        assert(dates.head.infoDate == infoDate)
+        assert(dates.head.reason == TaskRunReason.New)
+      }
+
+      "run for the current info date when the job ran some time ago" in {
+        val strategy = new ScheduleStrategyIncremental(Some(infoDate.minusDays(5)), false)
+        val params = ScheduleParams.Normal(infoDate, 0, 0, newOnly = false, lateOnly = false)
+
+        val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+        assert(dates.length == 1)
+        assert(dates.head.infoDate == infoDate)
+        assert(dates.head.reason == TaskRunReason.New)
+      }
+    }
+  }
+
+  "re-run" when {
+    "info date is defined" in {
+      val strategy = new ScheduleStrategyIncremental(Some(infoDate.plusDays(5)), true)
+      val params = ScheduleParams.Rerun(infoDate)
+
+      val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+      assert(dates.length == 1)
+      assert(dates.head.infoDate == infoDate)
+      assert(dates.head.reason == TaskRunReason.Rerun)
+    }
+
+    "info date is not defined" in {
+      val strategy = new ScheduleStrategyIncremental(Some(infoDate.plusDays(5)), false)
+      val params = ScheduleParams.Rerun(infoDate)
+
+      val dates = strategy.getDaysToRun("table1", Seq.empty, null, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+      assert(dates.length == 1)
+      assert(dates.head.infoDate == infoDate)
+      assert(dates.head.reason == TaskRunReason.Rerun)
+    }
+  }
+
+  "historical run" when {
+    val bk = new BookkeeperNull
+
+    "info date is defined" in {
+      val strategy = new ScheduleStrategyIncremental(Some(infoDate.plusDays(5)), true)
+      val params = ScheduleParams.Historical(infoDate.minusDays(2), infoDate, inverseDateOrder = false, RunMode.ForceRun)
+
+      val dates = strategy.getDaysToRun("table1", Seq.empty, bk, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+      dates.foreach(println)
+
+      assert(dates.length == 3)
+      assert(dates.head.infoDate == infoDate.minusDays(2))
+      assert(dates.head.reason == TaskRunReason.New)
+      assert(dates(1).infoDate == infoDate.minusDays(1))
+      assert(dates(1).reason == TaskRunReason.New)
+      assert(dates(2).infoDate == infoDate)
+      assert(dates(2).reason == TaskRunReason.New)
+    }
+
+    "info date is not defined" in {
+      val strategy = new ScheduleStrategyIncremental(Some(infoDate.plusDays(5)), false)
+      val params = ScheduleParams.Historical(infoDate.minusDays(2), infoDate, inverseDateOrder = true, RunMode.ForceRun)
+
+      val dates = strategy.getDaysToRun("table1", Seq.empty, bk, "@runDate", Schedule.Incremental, params, null, minimumDate)
+
+      assert(dates.length == 3)
+      assert(dates.head.infoDate == infoDate)
+      assert(dates.head.reason == TaskRunReason.New)
+      assert(dates(1).infoDate == infoDate.minusDays(1))
+      assert(dates(1).reason == TaskRunReason.New)
+      assert(dates(2).infoDate == infoDate.minusDays(2))
+      assert(dates(2).reason == TaskRunReason.New)
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/RunStatusSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/RunStatusSuite.scala
index 30d4595d0..0e22094b1 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/RunStatusSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/RunStatusSuite.scala
@@ -23,12 +23,12 @@ class RunStatusSuite extends AnyWordSpec {
   "toString" should {
     "Succeeded" when {
       "New" in {
-        val status = RunStatus.Succeeded(None, 0, None, TaskRunReason.New, Nil, Nil, Nil, Nil)
+        val status = RunStatus.Succeeded(None, Some(0), None, None, TaskRunReason.New, Nil, Nil, Nil, Nil)
 
         assert(status.toString == "New")
       }
       "Update" in {
-        val status = RunStatus.Succeeded(None, 0, None, TaskRunReason.Update, Nil, Nil, Nil, Nil)
+        val status = RunStatus.Succeeded(None, Some(0), None, None, TaskRunReason.Update, Nil, Nil, Nil, Nil)
 
         assert(status.toString == "Update")
       }
@@ -85,7 +85,7 @@ class RunStatusSuite extends AnyWordSpec {
 
   "isFailure" should {
     "Succeeded" in {
-       val status = RunStatus.Succeeded(None, 0, None, TaskRunReason.New, Nil, Nil, Nil, Nil)
+       val status = RunStatus.Succeeded(None, Some(0), None, None, TaskRunReason.New, Nil, Nil, Nil, Nil)
 
        assert(!status.isFailure)
     }
@@ -165,7 +165,7 @@ class RunStatusSuite extends AnyWordSpec {
 
   "getReason" should {
     "Succeeded" in {
-      val status = RunStatus.Succeeded(None, 0, None, TaskRunReason.New, Nil, Nil, Nil, Nil)
+      val status = RunStatus.Succeeded(None, Some(0), None, None, TaskRunReason.New, Nil, Nil, Nil, Nil)
 
       assert(status.getReason().isEmpty)
     }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/TaskRunnerBaseSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/TaskRunnerBaseSuite.scala
index 7a918de9f..ad897f1ca 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/TaskRunnerBaseSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/runner/task/TaskRunnerBaseSuite.scala
@@ -477,7 +477,7 @@ class TaskRunnerBaseSuite extends AnyWordSpec with SparkTestBase with TextCompar
       val success = result.runStatus.asInstanceOf[Succeeded]
 
       assert(success.recordCountOld.isEmpty)
-      assert(success.recordCount == 2)
+      assert(success.recordCount.contains(2))
       assert(success.sizeBytes.contains(100))
 
       val actualData = SparkUtils.convertDataFrameToPrettyJSON(job.saveDf)
@@ -511,7 +511,7 @@ class TaskRunnerBaseSuite extends AnyWordSpec with SparkTestBase with TextCompar
 
       val success = result.runStatus.asInstanceOf[Succeeded]
 
-      assert(success.recordCount == 2)
+      assert(success.recordCount.contains(2))
       assert(success.sizeBytes.isEmpty)
       assert(bk.getDataChunks("table_out", infoDate, infoDate).isEmpty)
     }
@@ -638,7 +638,7 @@ class TaskRunnerBaseSuite extends AnyWordSpec with SparkTestBase with TextCompar
       killMaxExecutionTimeSeconds = killTimer
     )
 
-    val stats = MetaTableStats(2, Some(100))
+    val stats = MetaTableStats(Some(2), None, Some(100))
 
     val job = new JobSpy(preRunCheckFunction = preRunCheckFunction,
       validationFunction = validationFunction,
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala
index c689fa684..fb6a29b0d 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala
@@ -41,4 +41,37 @@ class SqlColumnTypeSuite extends AnyWordSpec {
     }
   }
 
+  "fromStringStrict" should {
+    "return corresponding type" in {
+      assert(fromStringStrict("date") == DATE)
+      assert(fromStringStrict("datetime") == DATETIME)
+      assert(fromStringStrict("string") == STRING)
+      assert(fromStringStrict("number") == NUMBER)
+    }
+
+    "support mixed case values" in {
+      assert(fromStringStrict("Date") == DATE)
+      assert(fromStringStrict("DateTimE") == DATETIME)
+      assert(fromStringStrict("STRING") == STRING)
+      assert(fromStringStrict("nUmbeR") == NUMBER)
+    }
+
+    "throw IllegalArgumentException for unknown type with parent" in {
+      val ex = intercept[IllegalArgumentException] {
+        fromStringStrict("Hello", "parent")
+      }
+
+      assert(ex.getMessage.contains("Unknown information type 'Hello' configured at parent. Allowed valued: date, datetime, string, number."))
+    }
+
+    "throw IllegalArgumentException for unknown type without parent" in {
+      val ex = intercept[IllegalArgumentException] {
+        fromStringStrict("Hello")
+      }
+
+      assert(ex.getMessage.contains("Unknown information type 'Hello'. Allowed valued: date, datetime, string, number."))
+    }
+
+  }
+
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDb2Suite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDb2Suite.scala
new file mode 100644
index 000000000..ff354bd67
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDb2Suite.scala
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorDb2Suite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "com.ibm.db2.jcc.DB2Driver"
+
+  val genDate: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+
+  "generate count queries without date ranges" in {
+    assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = DATE '2020-08-17'")
+      assert(genDate.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= DATE '2020-08-17' AND D <= DATE '2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) = DATE '2020-08-17'")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) >= DATE '2020-08-17' AND CAST(D AS DATE) <= DATE '2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) AS \"CNT\" FROM \"Input Table\" WHERE \"Info date\" = DATE '2020-08-17'")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) AS \"CNT\" FROM \"Input Table\" WHERE \"Info date\" >= DATE '2020-08-17' AND \"Info date\" <= DATE '2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = DATE '2020-08-17'")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= DATE '2020-08-17' AND D <= DATE '2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) = DATE '2020-08-17'")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) >= DATE '2020-08-17' AND CAST(D AS DATE) <= DATE '2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = DATE '2020-08-17' LIMIT 100")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= DATE '2020-08-17' AND D <= DATE '2020-08-30' LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(genDate.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) AS T")
+    }
+  }
+
+  "quote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.quote("System User.\"Table Name\"")
+
+      assert(actual == "\"System User\".\"Table Name\"")
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.unquote("System User.\"Table Name\"")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > TIMESTAMP('1970-01-21 01:56:01.000')")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDenodoSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDenodoSuite.scala
new file mode 100644
index 000000000..33e0b690a
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDenodoSuite.scala
@@ -0,0 +1,197 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorDenodoSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "com.denodo.vdp.jdbc.Driver"
+
+  val gen: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+  val genEscapedAuto: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape.copy(identifierQuotingPolicy = QuotingPolicy.Auto))
+
+  "generate count queries without date ranges" in {
+    assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(gen.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified with escaping" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
+  }
+
+  "generate data queries when list of columns is specified with auto escaping" in {
+    assert(genEscapedAuto.getDataQuery("A", columns, None) == "SELECT A, D, \"Column with spaces\" FROM A")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(gen.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(gen.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
+      assert(gen.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
+      assert(genEscapedAuto.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17'")
+      assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17'")
+      assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(gen.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B) tbl")
+    }
+  }
+
+  "quote" should {
+    "quote each subfields separately" in {
+      val actual = gen.quote("System User.\"Table Name\"")
+
+      assert(actual == "\"System User\".\"Table Name\"")
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately" in {
+      val actual = gen.unquote("System User.\"Table Name\"")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > TIMESTAMP '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorGenericSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorGenericSuite.scala
new file mode 100644
index 000000000..f5bd44c02
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorGenericSuite.scala
@@ -0,0 +1,437 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetType, OffsetValue}
+import za.co.absa.pramen.api.sql.SqlGeneratorBase.MAX_STRING_OFFSET_CHARACTERS
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+import za.co.absa.pramen.core.sql.SqlGeneratorGeneric
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorGenericSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D", offsetInfo = Some(OffsetInfo("offset", OffsetType.IntegralType)))
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val genDate: SqlGenerator = getSqlGenerator("generic", sqlConfigDate)
+  val genDateTime: SqlGenerator = getSqlGenerator("generic", sqlConfigDateTime)
+  val genStr: SqlGenerator = getSqlGenerator("generic", sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator("generic", sqlConfigNumber)
+  val genEscaped: SqlGenerator = getSqlGenerator("generic", sqlConfigEscape)
+
+  "generate count queries without date ranges" in {
+    assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = date'2020-08-17'")
+      assert(genDate.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("SELECT", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM \"SELECT\" WHERE \"Info date\" = date'2020-08-17'")
+      assert(genEscaped.getCountQuery("SELECT", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM \"SELECT\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17'")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17' LIMIT 100")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(genDate.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) AS t")
+    }
+  }
+
+  "quote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.quote("System User.\"Table Name\"")
+
+      assert(actual == "\"System User\".\"Table Name\"")
+    }
+
+    "escape should not escape if turned off by config" in {
+      val sql = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D", identifierQuotingPolicy = QuotingPolicy.Never)
+      val genDate = getSqlGenerator("generic", sql)
+
+      val actual = genDate.asInstanceOf[SqlGeneratorBase].escape("System User.\"Table Name\"")
+
+      assert(actual == "System User.\"Table Name\"")
+    }
+
+    "throw an exception if a column contains a single quote" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.quote("ABC ' DEF")
+      }
+    }
+
+    "throw an exception if a column contains a semicolon" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.quote("ABC ; DEF")
+      }
+
+      assert(ex.getMessage == "The character ';' (0x3B) cannot be used as part of column name in 'ABC ; DEF'.")
+    }
+
+    "throw an exception if a column contains a back slash" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.quote("ABC \\n DEF")
+      }
+    }
+
+    "throw an exception if a column contains a new line" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.quote("ABC \n DEF")
+      }
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.unquote("System User.\"Table Name\"")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+
+  "getDataQueryIncremental" should {
+    "info date is absent" when {
+      "work without offsets" in {
+        val sql = genDate.getDataQueryIncremental("table1", None, None, None, Seq.empty)
+
+        assert(sql == "SELECT * FROM table1")
+      }
+
+      "work with only from offset" in {
+        val sql = genDate.getDataQueryIncremental("table1", None, Some(OffsetValue.IntegralValue(1)), None, Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE offset > 1")
+      }
+
+      "work with only to offset" in {
+        val sql = genDate.getDataQueryIncremental("table1", None, None, Some(OffsetValue.IntegralValue(1)), Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE offset <= 1")
+      }
+
+      "work with from and to offsets" in {
+        val sql = genDate.getDataQueryIncremental("table1", None, Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE offset >= 1 AND offset <= 2")
+      }
+
+      "work with from and to offsets, column projection" in {
+        val sql = genDate.getDataQueryIncremental("table1", None, Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), columns)
+
+        assert(sql == "SELECT A, D, \"Column with spaces\" FROM table1 WHERE offset >= 1 AND offset <= 2")
+      }
+    }
+    "info date is present" when {
+      "work without offsets" in {
+        val sql = genDate.getDataQueryIncremental("table1", Some(date1), None, None, Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE D = date'2020-08-17'")
+      }
+
+      "work with only from offset" in {
+        val sql = genDate.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.IntegralValue(1)), None, Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE D = date'2020-08-17' AND offset > 1")
+      }
+
+      "work with only to offset" in {
+        val sql = genDate.getDataQueryIncremental("table1", Some(date1), None, Some(OffsetValue.IntegralValue(1)), Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE D = date'2020-08-17' AND offset <= 1")
+      }
+
+      "work with from and to offsets" in {
+        val sql = genDate.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), Seq.empty)
+
+        assert(sql == "SELECT * FROM table1 WHERE D = date'2020-08-17' AND offset >= 1 AND offset <= 2")
+      }
+
+      "work with from and to offsets, column projection" in {
+        val sql = genDate.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), columns)
+
+        assert(sql == "SELECT A, D, \"Column with spaces\" FROM table1 WHERE D = date'2020-08-17' AND offset >= 1 AND offset <= 2")
+      }
+    }
+  }
+
+  "splitComplexIdentifier" should {
+    "throw on an empty identifier" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier(" ")
+      }
+    }
+
+    "keep original column name as is" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User")
+
+      assert(actual == Seq("System User"))
+    }
+
+    "split a complex column" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User.\"Table Name\"")
+
+      assert(actual == Seq("System User", "\"Table Name\""))
+    }
+
+    "handle escaped dots" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System User.Table Name\"")
+
+      assert(actual == Seq("\"System User.Table Name\""))
+    }
+
+    "handle first identifier escaped" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System User\".Table Name")
+
+      assert(actual == Seq("\"System User\"", "Table Name"))
+    }
+
+    "handle multiple level of identifiers" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.My Schema.\"My Table\".My column")
+
+      assert(actual == Seq("My Catalog", "My Schema", "\"My Table\"", "My column"))
+    }
+
+    "when opening and closing  characters are different" when {
+      val gen2 = new SqlGeneratorGeneric(sqlConfigEscape) {
+        override val beginEndEscapeChars: (Char, Char) = ('[', ']')
+      }
+
+      "handle the situation with multiple identifiers" in {
+        val actual = gen2.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.My Schema.[My Table].My column")
+
+        assert(actual == Seq("My Catalog", "My Schema", "[My Table]", "My column"))
+      }
+
+      "handle not matching open bracket" in {
+        val ex = intercept[IllegalArgumentException] {
+          gen2.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.[My Schema].[[My Table].My column")
+        }
+
+        assert(ex.getMessage.contains("Invalid character '[' in the identifier"))
+      }
+
+      "handle not matching close bracket" in {
+        val ex = intercept[IllegalArgumentException] {
+          gen2.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.My Schema.[My Table]].My column")
+        }
+
+        assert(ex.getMessage.contains("Found not matching ']' in the identifier"))
+      }
+    }
+
+    "handle escaped 2 quotes (maybe support this eventually)" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System \"\"User\"\".Table Name\"")
+      }
+
+      assert(ex.getMessage == "Invalid character '\"' in the identifier '\"System \"\"User\"\".Table Name\"', position 8.")
+    }
+
+    "handle escaped 3 quotes (this is never supported)" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System \"\"\"User\"\"\".Table Name\"")
+      }
+
+      assert(ex.getMessage == "Invalid character '\"' in the identifier '\"System \"\"\"User\"\"\".Table Name\"', position 8.")
+    }
+
+    "throw an exception if quotes are found inside an identifier" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System Use\"r.T\"able Name")
+      }
+
+      assert(ex.getMessage.contains("Invalid character '\"' in the identifier 'System Use\"r.T\"able Name'"))
+    }
+
+    "handle multiple level of identifiers of multiple levels" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.My Schema.\"My Tab\"le.My column")
+      }
+
+      assert(ex.getMessage == "Invalid character '\"' in the identifier 'My Catalog.My Schema.\"My Tab\"le.My column', position 28.")
+    }
+
+    "throw on unmatched open bracket" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User.\"Table Name")
+      }
+
+      assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.\"Table Name'."))
+    }
+
+    "throw on unmatched closing bracket" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User.Table Name\"")
+      }
+
+      assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.Table Name\"'."))
+    }
+
+    "throw on only whitespace identifier" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("   ")
+      }
+
+      assert(ex.getMessage.contains("Found an empty table name or column name ('   ')."))
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+
+  "validateOffsetValue" should {
+    "just return for a non-string offset type" in {
+      SqlGeneratorBase.validateOffsetValue(OffsetValue.IntegralValue(1))
+    }
+
+    "just return for a conforming string offset balue" in {
+      SqlGeneratorBase.validateOffsetValue(OffsetValue.StringValue("AAA"))
+    }
+
+    "throw an exception on a string value with quote character" in {
+      assertThrows[IllegalArgumentException] {
+        SqlGeneratorBase.validateOffsetValue(OffsetValue.StringValue("AAA'"))
+      }
+    }
+
+    "throw an exception on a string longer than maximum allowed" in {
+      val s = "A" * (MAX_STRING_OFFSET_CHARACTERS + 1)
+      assertThrows[IllegalArgumentException] {
+        SqlGeneratorBase.validateOffsetValue(OffsetValue.StringValue(s))
+      }
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorHiveSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorHiveSuite.scala
new file mode 100644
index 000000000..32185d991
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorHiveSuite.scala
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorHiveSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "com.cloudera.hive.jdbc41.HS2Driver"
+
+  val gen: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+  val genEscaped2: SqlGenerator = getSqlGenerator(driver, DummySqlConfigFactory.getDummyConfig(infoDateColumn = "`Info date`", identifierQuotingPolicy = QuotingPolicy.Auto))
+
+  "generate count queries without date ranges" in {
+    assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(gen.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT `A`, `D`, `Column with spaces` FROM `A`")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(gen.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(gen.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = to_date('2020-08-17')")
+      assert(gen.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= to_date('2020-08-17') AND D <= to_date('2020-08-30')")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) = to_date('2020-08-17')")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) >= to_date('2020-08-17') AND CAST(D AS DATE) <= to_date('2020-08-30')")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` = to_date('2020-08-17')")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` >= to_date('2020-08-17') AND `Info date` <= to_date('2020-08-30')")
+    }
+
+    "the table name and column name already escaped" in {
+      assert(genEscaped2.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` = to_date('2020-08-17')")
+      assert(genEscaped2.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` >= to_date('2020-08-17') AND `Info date` <= to_date('2020-08-30')")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = to_date('2020-08-17')")
+      assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= to_date('2020-08-17') AND D <= to_date('2020-08-30')")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) = to_date('2020-08-17')")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) >= to_date('2020-08-17') AND CAST(D AS DATE) <= to_date('2020-08-30')")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = to_date('2020-08-17') LIMIT 100")
+      assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= to_date('2020-08-17') AND D <= to_date('2020-08-30') LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(gen.getDtable("A") == "(A) tbl")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B) tbl")
+    }
+  }
+
+  "quote" should {
+    "escape each subfields separately" in {
+      val actual = gen.quote("System User.`Table Name`")
+
+      assert(actual == "`System User`.`Table Name`")
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorHsqlSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorHsqlSuite.scala
new file mode 100644
index 000000000..ac0da5ca6
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorHsqlSuite.scala
@@ -0,0 +1,196 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorHsqlSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "org.hsqldb.jdbc.JDBCDriver"
+
+  val genDate: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+
+  "generate count queries without date ranges" in {
+    assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
+      assert(genDate.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND D <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND CAST(D AS DATE) <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("SELECT", date1, date1) ==
+        "SELECT COUNT(*) AS \"CNT\" FROM \"SELECT\" WHERE \"Info date\" = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
+      assert(genEscaped.getCountQuery("SELECT", date1, date2) ==
+        "SELECT COUNT(*) AS \"CNT\" FROM \"SELECT\" WHERE \"Info date\" >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND \"Info date\" <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND D <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE CAST(D AS DATE) >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND CAST(D AS DATE) <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = TO_DATE('2020-08-17', 'YYYY-MM-DD') LIMIT 100")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND D <= TO_DATE('2020-08-30', 'YYYY-MM-DD') LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B)")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(genDate.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) t")
+    }
+  }
+
+  "quote" should {
+    "escape each subfields separately" in {
+      val actual = genDate.quote("System User.\"Table Name\"")
+
+      assert(actual == "\"System User\".\"Table Name\"")
+    }
+
+    "throw an exception if a column contains a single quote" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.quote("ABC ' DEF")
+      }
+    }
+
+    "throw an exception if a column contains a semicolon" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.quote("ABC ; DEF")
+      }
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > TIMESTAMP '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala
index c420a1423..5325cea7e 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala
@@ -72,1556 +72,7 @@ class SqlGeneratorLoaderSuite extends AnyWordSpec with RelationalDbFixture {
     }
   }
 
-  "Oracle SQL generator" should {
-    val gen = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigDate)
-    val genStr = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigString)
-    val genNum = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigNumber)
-    val genDateTime = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigDateTime)
-    val genEscaped = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigEscape)
-
-    "generate count queries without date ranges" in {
-      assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(gen.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-   "generate data queries when list of columns is specified" in {
-      assert(gen.getDataQuery("A", columns, None) == "SELECT A, D, \"Column with spaces\" FROM A")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(gen.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A WHERE ROWNUM <= 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(gen.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
-        assert(gen.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17'")
-        assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE TO_DATE(D, 'YYYY-MM-DD') = date'2020-08-17'")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE TO_DATE(D, 'YYYY-MM-DD') >= date'2020-08-17' AND TO_DATE(D, 'YYYY-MM-DD') <= date'2020-08-30'")
-      }
-
-      "with limit records" in {
-        assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17' AND ROWNUM <= 100")
-        assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' AND ROWNUM <= 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(gen.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B)")
-      }
-    }
-
-    "quote" should {
-      "quote each subfields separately" in {
-        val actual = gen.quote("System User.\"Table Name\"")
-
-        assert(actual == "\"System User\".\"Table Name\"")
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately" in {
-        val actual = gen.unquote("System User.\"Table Name\"")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
-  }
-
-  "Microsoft SQL generator" should {
-    val genDate = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigDate)
-    val genDateTime = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigDateTime)
-    val genStr = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigString)
-    val genStr2 = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, dateFormatApp = "yyyyMMdd",  infoDateColumn = "D"))
-    val genNum = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigNumber)
-    val genEscaped = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigEscape)
-    val genEscaped2 = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver",  DummySqlConfigFactory.getDummyConfig(infoDateColumn = "[Info date]", identifierQuotingPolicy = QuotingPolicy.Auto))
-
-    "generate count queries without date ranges" in {
-      assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK)")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A WITH (NOLOCK)")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT [A], [D], [Column with spaces] FROM [A] WITH (NOLOCK)")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT TOP 100 * FROM A WITH (NOLOCK)")
-    }
-
-    "generate data queries with limit clause date ranges when table is quoted" in {
-      assert(genDate.getDataQuery("\"A\"", Nil, Some(100)) == "SELECT TOP 100 * FROM \"A\" WITH (NOLOCK)")
-    }
-
-    "generate data queries with limit clause date ranges when table is escaped in brackets" in {
-      assert(genDate.getDataQuery("[A]", Nil, Some(100)) == "SELECT TOP 100 * FROM [A] WITH (NOLOCK)")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genDate.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) AND CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
-    }
-
-      "date is in STRING ISO format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) " +
-            "AND TRY_CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-
-      "date is in STRING non ISO format" in {
-        assert(genStr2.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = '20200817'")
-        assert(genStr2.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= '20200817' AND D <= '20200830'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) AS [CNT] FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) AS [CNT] FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17', 23) AND [Info date] <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-
-      "the table name and column name already escaped" in {
-        assert(genEscaped2.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genEscaped2.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17', 23) AND [Info date] <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) AND CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-
-      "date is in STRING ISO format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) " +
-            "AND TRY_CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-
-      "date is in STRING non-ISO format" in {
-        assert(genStr2.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE D = '20200817'")
-        assert(genStr2.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE D >= '20200817' AND D <= '20200830'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WITH (NOLOCK) WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(genDate.getDtable("A") == "A")
-      }
-
-      "wrapped query with alias for SQL queries " in {
-        assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) AS tbl")
-      }
-    }
-
-    "quote" should {
-      "escape each subfields separately" in {
-        val actual = genDate.quote("System User.[Table Name]")
-
-        assert(actual == "[System User].[Table Name]")
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately using quotes" in {
-        val actual = genDate.unquote("System User.\"Table Name\"")
-
-        assert(actual == "System User.Table Name")
-      }
-
-      "quote each subfields separately using brackets" in {
-        val actual = genDate.unquote("[System User].[Table Name]")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
-
-    "splitComplexIdentifier" should {
-      "throw on an empty identifier" in {
-        assertThrows[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier(" ")
-        }
-      }
-
-      "keep original column name as is" in {
-        val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User")
-
-        assert(actual == Seq("System User"))
-      }
-
-      "split a complex column with brackets" in {
-        val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.[Table Name]")
-
-        assert(actual == Seq("System User", "[Table Name]"))
-      }
-
-      "split a complex column with quotes 1" in {
-        val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System User\".Table Name")
-
-        assert(actual == Seq("\"System User\"", "Table Name"))
-      }
-
-      "split a complex column with quotes 2" in {
-        val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("[System User].\"Table Name\"")
-
-        assert(actual == Seq("[System User]", "\"Table Name\""))
-      }
-
-      "handle escaped dots with brackets" in {
-        val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("[System User.Table Name]")
-
-        assert(actual == Seq("[System User.Table Name]"))
-      }
-
-      "handle escaped dots with quotes" in {
-        val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System User.Table Name\"")
-
-        assert(actual == Seq("\"System User.Table Name\""))
-      }
-
-      "throw an exception if brackets are found inside the identifier" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System Use[r.T]able Name")
-        }
-
-        assert(ex.getMessage.contains("Invalid character '[' in the identifier 'System Use[r.T]able Name', position 10."))
-      }
-
-      "throw an exception if quotes are found inside the identifier" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System Use\"r.T\"able Name")
-        }
-
-        assert(ex.getMessage.contains("Invalid character '\"' in the identifier 'System Use\"r.T\"able Name', position 10."))
-      }
-
-      "throw on unmatched open bracket" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.[Table Name")
-        }
-
-        assert(ex.getMessage.contains("Found not matching '[' in the identifier 'System User.[Table Name'"))
-      }
-
-      "throw on unmatched open double quote" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.\"Table Name")
-        }
-
-        assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.\"Table Name'"))
-      }
-
-      "throw on unmatched closing bracket" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.Table Name]")
-        }
-
-        assert(ex.getMessage.contains("Found not matching ']' in the identifier 'System User.Table Name]'"))
-      }
-
-      "throw on unmatched double quote at the and" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.Table Name\"")
-        }
-
-        assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.Table Name\"'"))
-      }
-
-      "throw on invalid open double-quote" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System User.\"Table Name")
-        }
-
-        assert(ex.getMessage.contains("Invalid character '\"' in the identifier '\"System User.\"Table Name'"))
-      }
-
-      "throw on invalid close double-quote" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System Use\"r.Table Name")
-        }
-
-        assert(ex.getMessage.contains("Invalid character '\"' in the identifier '\"System Use\"r.Table Name'"))
-      }
-    }
-  }
-
-  "Denodo SQL generator" should {
-    val gen = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigDate)
-    val genStr = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigString)
-    val genNum = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigNumber)
-    val genDateTime = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigDateTime)
-    val genEscaped = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigEscape)
-    val genEscapedAuto = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigEscape.copy(identifierQuotingPolicy = QuotingPolicy.Auto))
-
-    "generate count queries without date ranges" in {
-      assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(gen.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified with escaping" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
-    }
-
-    "generate data queries when list of columns is specified with auto escaping" in {
-      assert(genEscapedAuto.getDataQuery("A", columns, None) == "SELECT A, D, \"Column with spaces\" FROM A")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(gen.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(gen.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
-        assert(gen.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
-        assert(genEscapedAuto.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17'")
-        assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17'")
-        assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(gen.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B) tbl")
-      }
-    }
-
-    "quote" should {
-      "quote each subfields separately" in {
-        val actual = gen.quote("System User.\"Table Name\"")
-
-        assert(actual == "\"System User\".\"Table Name\"")
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately" in {
-        val actual = gen.unquote("System User.\"Table Name\"")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
-
-    "splitComplexIdentifier" should {
-      "throw on an empty identifier" in {
-        assertThrows[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier(" ")
-        }
-      }
-
-      "keep original column name as is" in {
-        val actual = gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User")
-
-        assert(actual == Seq("System User"))
-      }
-
-      "split a complex column" in {
-        val actual = gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User.\"Table Name\"")
-
-        assert(actual == Seq("System User", "\"Table Name\""))
-      }
-
-      "handle escaped dots" in {
-        val actual = gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System User.Table Name\"")
-
-        assert(actual == Seq("\"System User.Table Name\""))
-      }
-
-      "handle first identifier escaped" in {
-        val actual = gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System User\".Table Name")
-
-        assert(actual == Seq("\"System User\"", "Table Name"))
-      }
-
-      "handle multiple level of identifiers" in {
-        val actual = gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.My Schema.\"My Table\".My column")
-
-        assert(actual == Seq("My Catalog", "My Schema", "\"My Table\"", "My column"))
-      }
-
-      "handle escaped 2 quotes (maybe support this eventually)" in {
-        val ex = intercept[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System \"\"User\"\".Table Name\"")
-        }
-
-        assert(ex.getMessage == "Invalid character '\"' in the identifier '\"System \"\"User\"\".Table Name\"', position 8.")
-      }
-
-      "handle escaped 3 quotes (this is never supported)" in {
-        val ex = intercept[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("\"System \"\"\"User\"\"\".Table Name\"")
-        }
-
-        assert(ex.getMessage == "Invalid character '\"' in the identifier '\"System \"\"\"User\"\"\".Table Name\"', position 8.")
-      }
-
-      "throw an exception if quotes are found inside an identifier" in {
-        val ex = intercept[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System Use\"r.T\"able Name")
-        }
-
-        assert(ex.getMessage.contains("Invalid character '\"' in the identifier 'System Use\"r.T\"able Name'"))
-      }
-
-      "handle multiple level of identifiers of multiple levels" in {
-        val ex = intercept[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("My Catalog.My Schema.\"My Tab\"le.My column")
-        }
-
-        assert(ex.getMessage == "Invalid character '\"' in the identifier 'My Catalog.My Schema.\"My Tab\"le.My column', position 28.")
-      }
-
-      "throw on unmatched open bracket" in {
-        val ex = intercept[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User.\"Table Name")
-        }
-
-        assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.\"Table Name'."))
-      }
-
-      "throw on unmatched closing bracket" in {
-        val ex = intercept[IllegalArgumentException] {
-          gen.asInstanceOf[SqlGeneratorBase].splitComplexIdentifier("System User.Table Name\"")
-        }
-
-        assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.Table Name\"'."))
-      }
-    }
-  }
-
-  "SAS SQL generator" should {
-    val connection = getConnection
-
-    val gen = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigDate)
-    val genStr = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigString)
-    val genNum = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigNumber)
-    val genDateTime = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigDateTime)
-    val genEscaped = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigEscape)
-
-    gen.setConnection(connection)
-    genStr.setConnection(connection)
-    genNum.setConnection(connection)
-
-    "generate count queries without date ranges" in {
-      assert(gen.getCountQuery("A") == "SELECT COUNT(*) AS cnt 'cnt' FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(gen.getDataQuery("company", Nil, None) == "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("company", columns, None) == "SELECT 'A'n, 'D'n, 'Column with spaces'n FROM 'company'n")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(gen.getDataQuery("company", Nil, Some(100)) == "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(gen.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D = date'2020-08-17'")
-        assert(gen.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "date is in DATETIME format" in {
-        assertThrows[UnsupportedOperationException] {
-          genDateTime.getDataQuery("A", date1, date1, Nil, None)
-        }
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM 'Input Table'n WHERE 'Info date'n = date'2020-08-17'")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) AS cnt 'cnt' FROM 'Input Table'n WHERE 'Info date'n >= date'2020-08-17' AND 'Info date'n <= date'2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(gen.getDataQuery("company", date1, date1, Nil, None) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = date'2020-08-17'")
-        assert(gen.getDataQuery("company", date1, date2, Nil, None) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("company", date1, date1, Nil, None) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("company", date1, date2, Nil, None) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("company", date1, date1, Nil, None) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = 20200817")
-        assert(genNum.getDataQuery("company", date1, date2, Nil, None) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(gen.getDataQuery("company", date1, date1, Nil, Some(100)) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = date'2020-08-17' LIMIT 100")
-        assert(gen.getDataQuery("company", date1, date2, Nil, Some(100)) ==
-          "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(gen.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B)")
-      }
-    }
-  }
-
-  "Hive SQL generator" should {
-    val gen = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigDate)
-    val genStr = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigString)
-    val genNum = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigNumber)
-    val genDateTime = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigDateTime)
-    val genEscaped = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigEscape)
-    val genEscaped2 = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", DummySqlConfigFactory.getDummyConfig(infoDateColumn = "`Info date`", identifierQuotingPolicy = QuotingPolicy.Auto))
-
-    "generate count queries without date ranges" in {
-      assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(gen.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT `A`, `D`, `Column with spaces` FROM `A`")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(gen.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(gen.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = to_date('2020-08-17')")
-        assert(gen.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= to_date('2020-08-17') AND D <= to_date('2020-08-30')")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) = to_date('2020-08-17')")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) >= to_date('2020-08-17') AND CAST(D AS DATE) <= to_date('2020-08-30')")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` = to_date('2020-08-17')")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` >= to_date('2020-08-17') AND `Info date` <= to_date('2020-08-30')")
-      }
-
-      "the table name and column name already escaped" in {
-        assert(genEscaped2.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` = to_date('2020-08-17')")
-        assert(genEscaped2.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` >= to_date('2020-08-17') AND `Info date` <= to_date('2020-08-30')")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = to_date('2020-08-17')")
-        assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= to_date('2020-08-17') AND D <= to_date('2020-08-30')")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) = to_date('2020-08-17')")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) >= to_date('2020-08-17') AND CAST(D AS DATE) <= to_date('2020-08-30')")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = to_date('2020-08-17') LIMIT 100")
-        assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= to_date('2020-08-17') AND D <= to_date('2020-08-30') LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(gen.getDtable("A") == "(A) tbl")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B) tbl")
-      }
-    }
-
-    "quote" should {
-      "escape each subfields separately" in {
-        val actual = gen.quote("System User.`Table Name`")
-
-        assert(actual == "`System User`.`Table Name`")
-      }
-    }
-  }
-
-  "PostgreSQL SQL generator" should {
-    val genDate = getSqlGenerator("org.postgresql.Driver", sqlConfigDate)
-    val genDateTime = getSqlGenerator("org.postgresql.Driver", sqlConfigDateTime)
-    val genStr = getSqlGenerator("org.postgresql.Driver", sqlConfigString)
-    val genNum = getSqlGenerator("org.postgresql.Driver", sqlConfigNumber)
-    val genEscaped = getSqlGenerator("org.postgresql.Driver", sqlConfigEscape)
-
-    "generate count queries without date ranges" in {
-      assert(genDate.getCountQuery("A") == "SELECT COUNT(*) FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
-        assert(genDate.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17'")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17' LIMIT 100")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(genDate.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) t")
-      }
-    }
-
-    "quote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.quote("System User.\"Table Name\"")
-
-        assert(actual == "\"System User\".\"Table Name\"")
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.unquote("System User.\"Table Name\"")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
-  }
-
-  "MySQL SQL generator" should {
-    val genDate = getSqlGenerator("com.mysql.cj.jdbc.Driver", sqlConfigDate)
-    val genDateTime = getSqlGenerator("com.mysql.cj.jdbc.Driver", sqlConfigDateTime)
-    val genStr = getSqlGenerator("com.mysql.cj.jdbc.Driver", sqlConfigString)
-    val genNum = getSqlGenerator("com.mysql.cj.jdbc.Driver", sqlConfigNumber)
-    val genEscaped = getSqlGenerator("com.mysql.cj.jdbc.Driver", sqlConfigEscape)
-
-    "generate count queries without date ranges" in {
-      assert(genDate.getCountQuery("A") == "SELECT COUNT(*) FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT `A`, `D`, `Column with spaces` FROM `A`")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
-        assert(genDate.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE DATE(D) = '2020-08-17'")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE DATE(D) >= '2020-08-17' AND DATE(D) <= '2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` = '2020-08-17'")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` >= '2020-08-17' AND `Info date` <= '2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE DATE(D) = '2020-08-17'")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE DATE(D) >= '2020-08-17' AND DATE(D) <= '2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = '2020-08-17' LIMIT 100")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30' LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(genDate.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) t")
-      }
-    }
-
-    "quote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.quote("System User.`Table Name`")
-
-        assert(actual == "`System User`.`Table Name`")
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.unquote("System User.`Table Name`")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
-  }
-
-  "DB2 SQL generator" should {
-    val genDate = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigDate)
-    val genDateTime = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigDateTime)
-    val genStr = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigString)
-    val genNum = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigNumber)
-    val genEscaped = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigEscape)
-
-    "generate count queries without date ranges" in {
-      assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = DATE '2020-08-17'")
-        assert(genDate.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= DATE '2020-08-17' AND D <= DATE '2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) = DATE '2020-08-17'")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) >= DATE '2020-08-17' AND CAST(D AS DATE) <= DATE '2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
-          "SELECT COUNT(*) AS \"CNT\" FROM \"Input Table\" WHERE \"Info date\" = DATE '2020-08-17'")
-        assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
-          "SELECT COUNT(*) AS \"CNT\" FROM \"Input Table\" WHERE \"Info date\" >= DATE '2020-08-17' AND \"Info date\" <= DATE '2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = DATE '2020-08-17'")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= DATE '2020-08-17' AND D <= DATE '2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) = DATE '2020-08-17'")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) >= DATE '2020-08-17' AND CAST(D AS DATE) <= DATE '2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = DATE '2020-08-17' LIMIT 100")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= DATE '2020-08-17' AND D <= DATE '2020-08-30' LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(genDate.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) AS T")
-      }
-    }
-
-    "quote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.quote("System User.\"Table Name\"")
-
-        assert(actual == "\"System User\".\"Table Name\"")
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.unquote("System User.\"Table Name\"")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
-  }
-
-  "HSQL generator" should {
-    val genDate = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigDate)
-    val genDateTime = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigDateTime)
-    val genStr = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigString)
-    val genNum = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigNumber)
-    val genEscaped = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigEscape)
-
-    "generate count queries without date ranges" in {
-      assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
-        assert(genDate.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND D <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND CAST(D AS DATE) <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("SELECT", date1, date1) ==
-          "SELECT COUNT(*) AS \"CNT\" FROM \"SELECT\" WHERE \"Info date\" = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
-        assert(genEscaped.getCountQuery("SELECT", date1, date2) ==
-          "SELECT COUNT(*) AS \"CNT\" FROM \"SELECT\" WHERE \"Info date\" >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND \"Info date\" <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND D <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) = TO_DATE('2020-08-17', 'YYYY-MM-DD')")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND CAST(D AS DATE) <= TO_DATE('2020-08-30', 'YYYY-MM-DD')")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = TO_DATE('2020-08-17', 'YYYY-MM-DD') LIMIT 100")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= TO_DATE('2020-08-17', 'YYYY-MM-DD') AND D <= TO_DATE('2020-08-30', 'YYYY-MM-DD') LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B)")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(genDate.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) t")
-      }
-    }
-
-    "quote" should {
-      "escape each subfields separately" in {
-        val actual = genDate.quote("System User.\"Table Name\"")
-
-        assert(actual == "\"System User\".\"Table Name\"")
-      }
-
-      "throw an exception if a column contains a single quote" in {
-        assertThrows[IllegalArgumentException] {
-          genDate.quote("ABC ' DEF")
-        }
-      }
-
-      "throw an exception if a column contains a semicolon" in {
-        assertThrows[IllegalArgumentException] {
-          genDate.quote("ABC ; DEF")
-        }
-      }
-    }
-  }
-
   "Generic SQL generator" should {
-    val genDate = getSqlGenerator("generic", sqlConfigDate)
-    val genDateTime = getSqlGenerator("generic", sqlConfigDateTime)
-    val genStr = getSqlGenerator("generic", sqlConfigString)
-    val genNum = getSqlGenerator("generic", sqlConfigNumber)
-    val genEscaped = getSqlGenerator("generic", sqlConfigEscape)
-
-    "generate count queries without date ranges" in {
-      assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A")
-    }
-
-    "generate data queries without date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
-    }
-
-    "generate data queries when list of columns is specified" in {
-      assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
-    }
-
-    "generate data queries with limit clause date ranges" in {
-      assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
-    }
-
-    "generate ranged count queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = date'2020-08-17'")
-        assert(genDate.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
-        assert(genDateTime.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getCountQuery("A", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D = 20200817")
-        assert(genNum.getCountQuery("A", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "the table name and column name need to be escaped" in {
-        assert(genEscaped.getCountQuery("SELECT", date1, date1) ==
-          "SELECT COUNT(*) AS CNT FROM \"SELECT\" WHERE \"Info date\" = date'2020-08-17'")
-        assert(genEscaped.getCountQuery("SELECT", date1, date2) ==
-          "SELECT COUNT(*) AS CNT FROM \"SELECT\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
-      }
-    }
-
-    "generate ranged data queries" when {
-      "date is in DATE format" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17'")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
-      }
-
-      "date is in DATETIME format" in {
-        assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
-        assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
-      }
-
-      "date is in STRING format" in {
-        assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = '2020-08-17'")
-        assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
-      }
-
-      "date is in NUMBER format" in {
-        assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
-          "SELECT * FROM A WHERE D = 20200817")
-        assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
-          "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
-      }
-
-      "with limit records" in {
-        assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D = date'2020-08-17' LIMIT 100")
-        assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
-          "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' LIMIT 100")
-      }
-    }
-
-    "getCountQueryForSql" should {
-      "generate count queries for an SQL subquery" in {
-        assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
-      }
-    }
-
-    "getDtable" should {
-      "return the original table when a table is provided" in {
-        assert(genDate.getDtable("A") == "A")
-      }
-
-      "wrapped query without alias for SQL queries " in {
-        assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) AS t")
-      }
-    }
-
-    "quote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.quote("System User.\"Table Name\"")
-
-        assert(actual == "\"System User\".\"Table Name\"")
-      }
-
-      "escape should not escape if turned off by config" in {
-        val sql = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D", identifierQuotingPolicy = QuotingPolicy.Never)
-        val genDate = getSqlGenerator("generic", sql)
-
-        val actual = genDate.asInstanceOf[SqlGeneratorBase].escape("System User.\"Table Name\"")
-
-        assert(actual == "System User.\"Table Name\"")
-      }
-
-      "throw an exception if a column contains a single quote" in {
-        assertThrows[IllegalArgumentException] {
-          genDate.quote("ABC ' DEF")
-        }
-      }
-
-      "throw an exception if a column contains a semicolon" in {
-        val ex = intercept[IllegalArgumentException] {
-          genDate.quote("ABC ; DEF")
-        }
-
-        assert(ex.getMessage == "The character ';' (0x3B) cannot be used as part of column name in 'ABC ; DEF'.")
-      }
-
-      "throw an exception if a column contains a back slash" in {
-        assertThrows[IllegalArgumentException] {
-          genDate.quote("ABC \\n DEF")
-        }
-      }
-
-      "throw an exception if a column contains a new line" in {
-        assertThrows[IllegalArgumentException] {
-          genDate.quote("ABC \n DEF")
-        }
-      }
-    }
-
-    "unquote" should {
-      "quote each subfields separately" in {
-        val actual = genDate.unquote("System User.\"Table Name\"")
-
-        assert(actual == "System User.Table Name")
-      }
-    }
   }
 
 }
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorMicrosoftSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorMicrosoftSuite.scala
new file mode 100644
index 000000000..77b50f43f
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorMicrosoftSuite.scala
@@ -0,0 +1,473 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.{OffsetInfo, OffsetType, OffsetValue}
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+import za.co.absa.pramen.core.sql.SqlGeneratorMicrosoft
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorMicrosoftSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D", offsetInfo = Some(OffsetInfo("offset", OffsetType.IntegralType)))
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D", offsetInfo = Some(OffsetInfo("offset", OffsetType.DateTimeType)))
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "net.sourceforge.jtds.jdbc.Driver"
+
+  val genDate: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genStr2: SqlGenerator = getSqlGenerator(driver, DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, dateFormatApp = "yyyyMMdd",  infoDateColumn = "D"))
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+  val genEscaped2: SqlGenerator = getSqlGenerator(driver,  DummySqlConfigFactory.getDummyConfig(infoDateColumn = "[Info date]", identifierQuotingPolicy = QuotingPolicy.Auto))
+
+  "generate count queries without date ranges" in {
+    assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK)")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A WITH (NOLOCK)")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT [A], [D], [Column with spaces] FROM [A] WITH (NOLOCK)")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT TOP 100 * FROM A WITH (NOLOCK)")
+  }
+
+  "generate data queries with limit clause date ranges when table is quoted" in {
+    assert(genDate.getDataQuery("\"A\"", Nil, Some(100)) == "SELECT TOP 100 * FROM \"A\" WITH (NOLOCK)")
+  }
+
+  "generate data queries with limit clause date ranges when table is escaped in brackets" in {
+    assert(genDate.getDataQuery("[A]", Nil, Some(100)) == "SELECT TOP 100 * FROM [A] WITH (NOLOCK)")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genDate.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) AND CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "date is in STRING ISO format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) " +
+          "AND TRY_CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "date is in STRING non ISO format" in {
+      assert(genStr2.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = '20200817'")
+      assert(genStr2.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= '20200817' AND D <= '20200830'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK) WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) AS [CNT] FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) AS [CNT] FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17', 23) AND [Info date] <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "the table name and column name already escaped" in {
+      assert(genEscaped2.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genEscaped2.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) AS CNT FROM [Input Table] WITH (NOLOCK) WHERE [Info date] >= CONVERT(DATE, '2020-08-17', 23) AND [Info date] <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) AND CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "date is in STRING ISO format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE TRY_CONVERT(DATE, D, 23) >= CONVERT(DATE, '2020-08-17', 23) " +
+          "AND TRY_CONVERT(DATE, D, 23) <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+
+    "date is in STRING non-ISO format" in {
+      assert(genStr2.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE D = '20200817'")
+      assert(genStr2.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE D >= '20200817' AND D <= '20200830'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WITH (NOLOCK) WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT TOP 100 * FROM A WITH (NOLOCK) WHERE D >= CONVERT(DATE, '2020-08-17', 23) AND D <= CONVERT(DATE, '2020-08-30', 23)")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(genDate.getDtable("A") == "A")
+    }
+
+    "wrapped query with alias for SQL queries " in {
+      assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) AS tbl")
+    }
+  }
+
+  "quote" should {
+    "escape each subfields separately" in {
+      val actual = genDate.quote("System User.[Table Name]")
+
+      assert(actual == "[System User].[Table Name]")
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately using quotes" in {
+      val actual = genDate.unquote("System User.\"Table Name\"")
+
+      assert(actual == "System User.Table Name")
+    }
+
+    "quote each subfields separately using brackets" in {
+      val actual = genDate.unquote("[System User].[Table Name]")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+
+  "splitComplexIdentifier" should {
+    "throw on an empty identifier" in {
+      assertThrows[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier(" ")
+      }
+    }
+
+    "keep original column name as is" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User")
+
+      assert(actual == Seq("System User"))
+    }
+
+    "split a complex column with brackets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.[Table Name]")
+
+      assert(actual == Seq("System User", "[Table Name]"))
+    }
+
+    "split a complex column with quotes 1" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System User\".Table Name")
+
+      assert(actual == Seq("\"System User\"", "Table Name"))
+    }
+
+    "split a complex column with quotes 2" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("[System User].\"Table Name\"")
+
+      assert(actual == Seq("[System User]", "\"Table Name\""))
+    }
+
+    "handle escaped dots with brackets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("[System User.Table Name]")
+
+      assert(actual == Seq("[System User.Table Name]"))
+    }
+
+    "handle escaped dots with quotes" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System User.Table Name\"")
+
+      assert(actual == Seq("\"System User.Table Name\""))
+    }
+
+    "throw an exception if brackets are found inside the identifier" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System Use[r.T]able Name")
+      }
+
+      assert(ex.getMessage.contains("Invalid character '[' in the identifier 'System Use[r.T]able Name', position 10."))
+    }
+
+    "throw an exception if quotes are found inside the identifier" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System Use\"r.T\"able Name")
+      }
+
+      assert(ex.getMessage.contains("Invalid character '\"' in the identifier 'System Use\"r.T\"able Name', position 10."))
+    }
+
+    "throw on unmatched open bracket" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.[Table Name")
+      }
+
+      assert(ex.getMessage.contains("Found not matching '[' in the identifier 'System User.[Table Name'"))
+    }
+
+    "throw on unmatched open double quote" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.\"Table Name")
+      }
+
+      assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.\"Table Name'"))
+    }
+
+    "throw on unmatched closing bracket" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.Table Name]")
+      }
+
+      assert(ex.getMessage.contains("Found not matching ']' in the identifier 'System User.Table Name]'"))
+    }
+
+    "throw on unmatched double quote at the and" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("System User.Table Name\"")
+      }
+
+      assert(ex.getMessage.contains("Found not matching '\"' in the identifier 'System User.Table Name\"'"))
+    }
+
+    "throw on invalid open double-quote" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System User.\"Table Name")
+      }
+
+      assert(ex.getMessage.contains("Invalid character '\"' in the identifier '\"System User.\"Table Name'"))
+    }
+
+    "throw on invalid close double-quote" in {
+      val ex = intercept[IllegalArgumentException] {
+        genDate.asInstanceOf[SqlGeneratorMicrosoft].splitComplexIdentifier("\"System Use\"r.Table Name")
+      }
+
+      assert(ex.getMessage.contains("Invalid character '\"' in the identifier '\"System Use\"r.Table Name'"))
+    }
+  }
+
+  "getDataQueryIncremental" should {
+    "integral offset" when {
+      "info date is absent" when {
+        "work without offsets" in {
+          val sql = genDate.getDataQueryIncremental("table1", None, None, None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK)")
+        }
+
+        "work with only from offset" in {
+          val sql = genDate.getDataQueryIncremental("table1", None, Some(OffsetValue.IntegralValue(1)), None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE offset > 1")
+        }
+
+        "work with only to offset" in {
+          val sql = genDate.getDataQueryIncremental("table1", None, None, Some(OffsetValue.IntegralValue(1)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE offset <= 1")
+        }
+
+        "work with from and to offsets" in {
+          val sql = genDate.getDataQueryIncremental("table1", None, Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE offset >= 1 AND offset <= 2")
+        }
+
+        "work with from and to offsets, column projection" in {
+          val sql = genDate.getDataQueryIncremental("table1", None, Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), columns)
+
+          assert(sql == "SELECT A, D, [Column with spaces] FROM table1 WITH (NOLOCK) WHERE offset >= 1 AND offset <= 2")
+        }
+      }
+      "info date is present" when {
+        "work without offsets" in {
+          val sql = genDate.getDataQueryIncremental("table1", Some(date1), None, None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23)")
+        }
+
+        "work with only from offset" in {
+          val sql = genDate.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.IntegralValue(1)), None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23) AND offset > 1")
+        }
+
+        "work with only to offset" in {
+          val sql = genDate.getDataQueryIncremental("table1", Some(date1), None, Some(OffsetValue.IntegralValue(1)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23) AND offset <= 1")
+        }
+
+        "work with from and to offsets" in {
+          val sql = genDate.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23) AND offset >= 1 AND offset <= 2")
+        }
+
+        "work with from and to offsets, column projection" in {
+          val sql = genDate.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.IntegralValue(1)), Some(OffsetValue.IntegralValue(2)), columns)
+
+          assert(sql == "SELECT A, D, [Column with spaces] FROM table1 WITH (NOLOCK) WHERE D = CONVERT(DATE, '2020-08-17', 23) AND offset >= 1 AND offset <= 2")
+        }
+      }
+    }
+    "datetime offset" when {
+      val offset1 = Instant.ofEpochMilli(1727761000)
+      val offset2 = Instant.ofEpochMilli(1727762000)
+      "info date is absent" when {
+        "work without offsets" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", None, None, None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK)")
+        }
+
+        "work with only from offset" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", None, Some(OffsetValue.DateTimeValue(offset1)), None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE offset > '1970-01-21 01:56:01.000'")
+        }
+
+        "work with only to offset" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", None, None, Some(OffsetValue.DateTimeValue(offset1)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE offset <= '1970-01-21 01:56:01.000'")
+        }
+
+        "work with from and to offsets" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", None, Some(OffsetValue.DateTimeValue(offset1)), Some(OffsetValue.DateTimeValue(offset2)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE offset >= '1970-01-21 01:56:01.000' AND offset <= '1970-01-21 01:56:02.000'")
+        }
+
+        "work with from and to offsets, column projection" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", None, Some(OffsetValue.DateTimeValue(offset1)), Some(OffsetValue.DateTimeValue(offset2)), columns)
+
+          assert(sql == "SELECT A, D, [Column with spaces] FROM table1 WITH (NOLOCK) WHERE offset >= '1970-01-21 01:56:01.000' AND offset <= '1970-01-21 01:56:02.000'")
+        }
+      }
+      "info date is present" when {
+        "work without offsets" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", Some(date1), None, None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23)")
+        }
+
+        "work with only from offset" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.DateTimeValue(offset1)), None, Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23) AND offset > '1970-01-21 01:56:01.000'")
+        }
+
+        "work with only to offset" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", Some(date1), None, Some(OffsetValue.DateTimeValue(offset1)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23) AND offset <= '1970-01-21 01:56:01.000'")
+        }
+
+        "work with from and to offsets" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.DateTimeValue(offset1)), Some(OffsetValue.DateTimeValue(offset2)), Seq.empty)
+
+          assert(sql == "SELECT * FROM table1 WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23) AND offset >= '1970-01-21 01:56:01.000' AND offset <= '1970-01-21 01:56:02.000'")
+        }
+
+        "work with from and to offsets, column projection" in {
+          val sql = genDateTime.getDataQueryIncremental("table1", Some(date1), Some(OffsetValue.DateTimeValue(offset1)), Some(OffsetValue.DateTimeValue(offset2)), columns)
+
+          assert(sql == "SELECT A, D, [Column with spaces] FROM table1 WITH (NOLOCK) WHERE CONVERT(DATE, D, 23) = CONVERT(DATE, '2020-08-17', 23) AND offset >= '1970-01-21 01:56:01.000' AND offset <= '1970-01-21 01:56:02.000'")
+        }
+      }
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorMicrosoft]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorMySqlSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorMySqlSuite.scala
new file mode 100644
index 000000000..bb7560bcf
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorMySqlSuite.scala
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorMySqlSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "com.mysql.cj.jdbc.Driver"
+
+  val genDate: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+
+  "generate count queries without date ranges" in {
+    assert(genDate.getCountQuery("A") == "SELECT COUNT(*) FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT `A`, `D`, `Column with spaces` FROM `A`")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
+      assert(genDate.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17 00:00:00' AND D < '2020-08-18 00:00:00'")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17 00:00:00' AND D < '2020-08-31 00:00:00'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` = '2020-08-17'")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) FROM `Input Table` WHERE `Info date` >= '2020-08-17' AND `Info date` <= '2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17 00:00:00' AND D < '2020-08-18 00:00:00'")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17 00:00:00' AND D < '2020-08-31 00:00:00'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = '2020-08-17' LIMIT 100")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30' LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(genDate.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) t")
+    }
+  }
+
+  "quote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.quote("System User.`Table Name`")
+
+      assert(actual == "`System User`.`Table Name`")
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.unquote("System User.`Table Name`")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorOracleSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorOracleSuite.scala
new file mode 100644
index 000000000..2479aa8eb
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorOracleSuite.scala
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorOracleSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "oracle.jdbc.OracleDriver"
+
+  val gen: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+
+  "generate count queries without date ranges" in {
+    assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(gen.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(gen.getDataQuery("A", columns, None) == "SELECT A, D, \"Column with spaces\" FROM A")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(gen.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A WHERE ROWNUM <= 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(gen.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
+      assert(gen.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17'")
+      assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE TO_DATE(D, 'YYYY-MM-DD') = date'2020-08-17'")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE TO_DATE(D, 'YYYY-MM-DD') >= date'2020-08-17' AND TO_DATE(D, 'YYYY-MM-DD') <= date'2020-08-30'")
+    }
+
+    "with limit records" in {
+      assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17' AND ROWNUM <= 100")
+      assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' AND ROWNUM <= 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(gen.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B)")
+    }
+  }
+
+  "quote" should {
+    "quote each subfields separately" in {
+      val actual = gen.quote("System User.\"Table Name\"")
+
+      assert(actual == "\"System User\".\"Table Name\"")
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately" in {
+      val actual = gen.unquote("System User.\"Table Name\"")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > TIMESTAMP '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorPostgreSqlSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorPostgreSqlSuite.scala
new file mode 100644
index 000000000..4693c3660
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorPostgreSqlSuite.scala
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorPostgreSqlSuite extends AnyWordSpec {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driver = "org.postgresql.Driver"
+
+  val genDate: SqlGenerator = getSqlGenerator(driver, sqlConfigDate)
+  val genDateTime: SqlGenerator = getSqlGenerator(driver, sqlConfigDateTime)
+  val genStr: SqlGenerator = getSqlGenerator(driver, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driver, sqlConfigNumber)
+  val genEscaped: SqlGenerator = getSqlGenerator(driver, sqlConfigEscape)
+
+  "generate count queries without date ranges" in {
+    assert(genDate.getCountQuery("A") == "SELECT COUNT(*) FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, None) == "SELECT * FROM A")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("A", columns, None) == "SELECT \"A\", \"D\", \"Column with spaces\" FROM \"A\"")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(genDate.getDataQuery("A", Nil, Some(100)) == "SELECT * FROM A LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
+      assert(genDate.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D < '2020-08-18'")
+      assert(genDateTime.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D < '2020-08-31'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17'")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in DATETIME format" in {
+      assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D < '2020-08-18'")
+      assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D < '2020-08-31'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
+        "SELECT * FROM A WHERE D = 20200817")
+      assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
+        "SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(genDate.getDataQuery("A", date1, date1, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D = date'2020-08-17' LIMIT 100")
+      assert(genDate.getDataQuery("A", date1, date2, Nil, Some(100)) ==
+        "SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(genDate.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(genDate.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(genDate.getDtable("SELECT A FROM B") == "(SELECT A FROM B) t")
+    }
+  }
+
+  "quote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.quote("System User.\"Table Name\"")
+
+      assert(actual == "\"System User\".\"Table Name\"")
+    }
+  }
+
+  "unquote" should {
+    "quote each subfields separately" in {
+      val actual = genDate.unquote("System User.\"Table Name\"")
+
+      assert(actual == "System User.Table Name")
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > '1970-01-21 01:56:01.000'")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = genDate.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSasSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSasSuite.scala
new file mode 100644
index 000000000..8a3a547a5
--- /dev/null
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSasSuite.scala
@@ -0,0 +1,214 @@
+/*
+ * Copyright 2022 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.pramen.core.tests.sql
+
+import org.scalatest.wordspec.AnyWordSpec
+import za.co.absa.pramen.api.offset.OffsetValue
+import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGenerator, SqlGeneratorBase}
+import za.co.absa.pramen.core.fixtures.RelationalDbFixture
+import za.co.absa.pramen.core.mocks.DummySqlConfigFactory
+import za.co.absa.pramen.core.samples.RdbExampleTable
+
+import java.sql.Connection
+import java.time.{Instant, LocalDate}
+
+class SqlGeneratorSasSuite extends AnyWordSpec with RelationalDbFixture {
+
+  import za.co.absa.pramen.core.sql.SqlGeneratorLoader._
+
+  private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D")
+  private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always)
+  private val sqlConfigDateTime = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATETIME, infoDateColumn = "D")
+  private val sqlConfigString = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, infoDateColumn = "D")
+  private val sqlConfigNumber = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.NUMBER, infoDateColumn = "D", dateFormatApp = "yyyyMMdd")
+  private val columns = Seq("A", "D", "Column with spaces")
+
+  private val date1 = LocalDate.of(2020, 8, 17)
+  private val date2 = LocalDate.of(2020, 8, 30)
+
+  val driverSas = "com.sas.rio.MVADriver"
+
+  val connection: Connection = getConnection
+
+  val gen: SqlGenerator = getSqlGenerator(driverSas, sqlConfigDate)
+  val genStr: SqlGenerator = getSqlGenerator(driverSas, sqlConfigString)
+  val genNum: SqlGenerator = getSqlGenerator(driverSas, sqlConfigNumber)
+  val genDateTime: SqlGenerator = getSqlGenerator(driverSas, sqlConfigDateTime)
+  val genEscaped: SqlGenerator = getSqlGenerator(driverSas, sqlConfigEscape)
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    RdbExampleTable.Company.initTable(getConnection)
+    gen.setConnection(connection)
+    genStr.setConnection(connection)
+    genNum.setConnection(connection)
+  }
+
+  override protected def afterAll(): Unit = {
+    RdbExampleTable.Company.dropTable(getConnection)
+    super.afterAll()
+  }
+
+  "generate count queries without date ranges" in {
+    assert(gen.getCountQuery("A") == "SELECT COUNT(*) AS cnt 'cnt' FROM A")
+  }
+
+  "generate data queries without date ranges" in {
+    assert(gen.getDataQuery("company", Nil, None) == "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company")
+  }
+
+  "generate data queries when list of columns is specified" in {
+    assert(genEscaped.getDataQuery("company", columns, None) == "SELECT 'A'n, 'D'n, 'Column with spaces'n FROM 'company'n")
+  }
+
+  "generate data queries with limit clause date ranges" in {
+    assert(gen.getDataQuery("company", Nil, Some(100)) == "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company LIMIT 100")
+  }
+
+  "generate ranged count queries" when {
+    "date is in DATE format" in {
+      assert(gen.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D = date'2020-08-17'")
+      assert(gen.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D = '2020-08-17'")
+      assert(genStr.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getCountQuery("A", date1, date1) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D = 20200817")
+      assert(genNum.getCountQuery("A", date1, date2) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM A WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "date is in DATETIME format" in {
+      assertThrows[UnsupportedOperationException] {
+        genDateTime.getDataQuery("A", date1, date1, Nil, None)
+      }
+    }
+
+    "the table name and column name need to be escaped" in {
+      assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM 'Input Table'n WHERE 'Info date'n = date'2020-08-17'")
+      assert(genEscaped.getCountQuery("Input Table", date1, date2) ==
+        "SELECT COUNT(*) AS cnt 'cnt' FROM 'Input Table'n WHERE 'Info date'n >= date'2020-08-17' AND 'Info date'n <= date'2020-08-30'")
+    }
+  }
+
+  "generate ranged data queries" when {
+    "date is in DATE format" in {
+      assert(gen.getDataQuery("company", date1, date1, Nil, None) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = date'2020-08-17'")
+      assert(gen.getDataQuery("company", date1, date2, Nil, None) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
+    }
+
+    "date is in STRING format" in {
+      assert(genStr.getDataQuery("company", date1, date1, Nil, None) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = '2020-08-17'")
+      assert(genStr.getDataQuery("company", date1, date2, Nil, None) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
+    }
+
+    "date is in NUMBER format" in {
+      assert(genNum.getDataQuery("company", date1, date1, Nil, None) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = 20200817")
+      assert(genNum.getDataQuery("company", date1, date2, Nil, None) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= 20200817 AND D <= 20200830")
+    }
+
+    "with limit records" in {
+      assert(gen.getDataQuery("company", date1, date1, Nil, Some(100)) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D = date'2020-08-17' LIMIT 100")
+      assert(gen.getDataQuery("company", date1, date2, Nil, Some(100)) ==
+        "SELECT ID 'ID', NAME 'NAME', DESCRIPTION 'DESCRIPTION', EMAIL 'EMAIL', FOUNDED 'FOUNDED', LAST_UPDATED 'LAST_UPDATED', INFO_DATE 'INFO_DATE' FROM company WHERE D >= date'2020-08-17' AND D <= date'2020-08-30' LIMIT 100")
+    }
+  }
+
+  "getCountQueryForSql" should {
+    "generate count queries for an SQL subquery" in {
+      assert(gen.getCountQueryForSql("SELECT A FROM B") == "SELECT COUNT(*) FROM (SELECT A FROM B) AS query")
+    }
+  }
+
+  "getDtable" should {
+    "return the original table when a table is provided" in {
+      assert(gen.getDtable("A") == "A")
+    }
+
+    "wrapped query without alias for SQL queries " in {
+      assert(gen.getDtable("SELECT A FROM B") == "(SELECT A FROM B)")
+    }
+  }
+
+  "quote" should {
+    "quote an identifier" in {
+      assert(gen.quote("database.table") == "'database'n.'table'n")
+    }
+
+    "quote a partially quoted identifier" in {
+      assert(gen.quote("'database'n.table") == "'database'n.'table'n")
+    }
+
+    "not quote already quoted identifier" in {
+      assert(gen.quote("'database'n.'table'n") == "'database'n.'table'n")
+    }
+  }
+
+  "unquote" should {
+    "unquote a quoted identifier" in {
+      assert(gen.unquote("'database'n.'table'n") == "database.table")
+    }
+
+    "unquote a partially quoted identifier" in {
+      assert(gen.unquote("'database'n.table") == "database.table")
+    }
+
+    "not unquote a not quoted identifier" in {
+      assert(gen.unquote("database.table") == "database.table")
+    }
+  }
+
+  "getOffsetWhereCondition" should {
+    "return the correct condition for integral offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", "<", OffsetValue.IntegralValue(1))
+
+      assert(actual == "offset < 1")
+    }
+
+    "return the correct condition for datetime offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
+
+      assert(actual == "offset > '21Jan1970:01:56:01.000'dt")
+    }
+
+    "return the correct condition for string offsets" in {
+      val actual = gen.asInstanceOf[SqlGeneratorBase]
+        .getOffsetWhereCondition("offset", ">=", OffsetValue.StringValue("AAA"))
+
+      assert(actual == "offset >= 'AAA'")
+    }
+  }
+}
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/utils/SparkUtilsSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/utils/SparkUtilsSuite.scala
index f572dce6d..4e40b0506 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/utils/SparkUtilsSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/utils/SparkUtilsSuite.scala
@@ -19,7 +19,7 @@ package za.co.absa.pramen.core.tests.utils
 import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.functions.lit
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.{DataFrame, Row, types}
+import org.apache.spark.sql.{DataFrame, Row}
 import org.scalatest.wordspec.AnyWordSpec
 import za.co.absa.pramen.api.FieldChange._
 import za.co.absa.pramen.core.NestedDataFrameFactory
@@ -263,9 +263,6 @@ class SparkUtilsSuite extends AnyWordSpec with SparkTestBase with TempDirFixture
       val newField2 = schema1Orig.fields.head.copy(metadata = metadata2)
       val schema2 = schema1Orig.copy(fields = newField2 +: schema1Orig.fields.tail)
 
-      println(schema1.prettyJson)
-      println(schema2.prettyJson)
-
       val diff = compareSchemas(schema1, schema2)
 
       assert(diff.length == 1)
@@ -444,6 +441,52 @@ class SparkUtilsSuite extends AnyWordSpec with SparkTestBase with TempDirFixture
     }
   }
 
+  "getLengthFromMetadata" should {
+    "return length for long type" in {
+      val metadata = new MetadataBuilder
+      metadata.putLong(MAX_LENGTH_METADATA_KEY, 10L)
+
+      val len = SparkUtils.getLengthFromMetadata(metadata.build())
+
+      assert(len.contains(10))
+    }
+
+    "return length for string type" in {
+      val metadata = new MetadataBuilder
+      metadata.putString(MAX_LENGTH_METADATA_KEY, "10")
+
+      val len = SparkUtils.getLengthFromMetadata(metadata.build())
+
+      assert(len.contains(10))
+    }
+
+    "return None for wrong type" in {
+      val metadata = new MetadataBuilder
+      metadata.putString(MAX_LENGTH_METADATA_KEY, "abc")
+
+      val len = SparkUtils.getLengthFromMetadata(metadata.build())
+
+      assert(len.isEmpty)
+    }
+
+    "return None for wrong value" in {
+      val metadata = new MetadataBuilder
+      metadata.putDouble(MAX_LENGTH_METADATA_KEY, 12.25)
+
+      val len = SparkUtils.getLengthFromMetadata(metadata.build())
+
+      assert(len.isEmpty)
+    }
+
+    "return None if not specified" in {
+      val metadata = new MetadataBuilder
+
+      val len = SparkUtils.getLengthFromMetadata(metadata.build())
+
+      assert(len.isEmpty)
+    }
+  }
+
   "removeNestedMetadata" should {
     "remove metadata, but only from nested fields" in {
       val metadata1 = new MetadataBuilder().putLong("maxLength", 5).putString("comment", "Employee name").build()
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/transformers/IdentityTransformerSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/transformers/IdentityTransformerSuite.scala
index 26783c2c3..7e3facee9 100644
--- a/pramen/core/src/test/scala/za/co/absa/pramen/core/transformers/IdentityTransformerSuite.scala
+++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/transformers/IdentityTransformerSuite.scala
@@ -36,7 +36,7 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
 
   "validate()" should {
     "pass when the mandatory option is present" in {
-      val (transformer, metastore) = getUseCase
+      val (transformer, metastore) = getUseCase()
 
       val outcome = transformer.validate(metastore, infoDateWithData, Map("input.table" -> "table1"))
 
@@ -44,7 +44,7 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
     }
 
     "pass when the legacy mandatory option is present" in {
-      val (transformer, metastore) = getUseCase
+      val (transformer, metastore) = getUseCase()
 
       val outcome = transformer.validate(metastore, infoDateWithData, Map("table" -> "table1"))
 
@@ -52,7 +52,7 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
     }
 
     "pass when empty is allowed" in {
-      val (transformer, metastore) = getUseCase
+      val (transformer, metastore) = getUseCase(true)
 
       val outcome = transformer.validate(metastore, infoDateWithEmptyDf, Map("table" -> "table1", "empty.allowed" -> "true"))
 
@@ -60,7 +60,7 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
     }
 
     "return SkipOnce when empty is not allowed" in {
-      val (transformer, metastore) = getUseCase
+      val (transformer, metastore) = getUseCase(true)
 
       val outcome = transformer.validate(metastore, infoDateWithEmptyDf, Map("table" -> "table1", "empty.allowed" -> "false"))
 
@@ -68,7 +68,7 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
     }
 
     "fail when the mandatory option is absent" in {
-      val (transformer, metastore) = getUseCase
+      val (transformer, metastore) = getUseCase()
 
       val ex = intercept[IllegalArgumentException] {
         transformer.validate(metastore, infoDateWithData, Map.empty)
@@ -89,7 +89,7 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
           ||C  |3  |
           |+---+---+
           |""".stripMargin
-      val (transformer, metastore) = getUseCase
+      val (transformer, metastore) = getUseCase()
 
       val outcome = transformer.run(metastore, infoDateWithData, Map("table" -> "table1"))
         .orderBy("a")
@@ -100,12 +100,16 @@ class IdentityTransformerSuite extends AnyWordSpec with SparkTestBase with TextC
     }
   }
 
-  def getUseCase: (IdentityTransformer, MetastoreReader) = {
+  def getUseCase(isEmptyToday: Boolean = false): (IdentityTransformer, MetastoreReader) = {
     val metastoreReadeMock = mock(classOf[MetastoreReader])
 
     whenMock(metastoreReadeMock.getTable("table1")).thenReturn(exampleDf)
     whenMock(metastoreReadeMock.getTable("table1", Some(infoDateWithData), Some(infoDateWithData))).thenReturn(exampleDf)
     whenMock(metastoreReadeMock.getTable("table1", Some(infoDateWithEmptyDf), Some(infoDateWithEmptyDf))).thenReturn(emptyDf)
+    if (isEmptyToday)
+      whenMock(metastoreReadeMock.getCurrentBatch("table1")).thenReturn(emptyDf)
+    else
+      whenMock(metastoreReadeMock.getCurrentBatch("table1")).thenReturn(exampleDf)
 
     (new IdentityTransformer(), metastoreReadeMock)
   }
diff --git a/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/ReaderSpy.scala b/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/ReaderSpy.scala
index d8da4b5c3..33b50cb4c 100644
--- a/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/ReaderSpy.scala
+++ b/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/ReaderSpy.scala
@@ -16,10 +16,11 @@
 
 package za.co.absa.pramen.extras.mocks
 
-import java.time.LocalDate
 import org.apache.spark.sql.{DataFrame, SparkSession}
+import za.co.absa.pramen.api.offset.OffsetValue
 import za.co.absa.pramen.api.{Query, TableReader}
 
+import java.time.LocalDate
 import scala.collection.mutable.ListBuffer
 
 class ReaderSpy(numRecords: Long = 0L)(implicit spark: SparkSession)
@@ -42,4 +43,6 @@ class ReaderSpy(numRecords: Long = 0L)(implicit spark: SparkSession)
     val df = generatedList.toDF("v")
     df
   }
+
+  override def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): DataFrame = ???
 }
diff --git a/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/TestPrototypes.scala b/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/TestPrototypes.scala
index faf594ccb..b9b65fffe 100644
--- a/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/TestPrototypes.scala
+++ b/pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/TestPrototypes.scala
@@ -33,13 +33,14 @@ object TestPrototypes {
     DataFormat.Null(),
     "pramen_info_date",
     "yyyy-MM-dd",
+    "pramen_batchid",
     None,
     None,
     LocalDate.MIN,
     Map.empty,
     Map.empty)
 
-  val taskStatus: RunStatus = RunStatus.Succeeded(None, 100, None, TaskRunReason.New, Seq.empty, Seq.empty, Seq.empty, Seq.empty)
+  val taskStatus: RunStatus = RunStatus.Succeeded(None, Some(100), None, None, TaskRunReason.New, Seq.empty, Seq.empty, Seq.empty, Seq.empty)
 
   val taskNotification: TaskResult = status.TaskResult(
     "Dummy Job",