Repository: spetlr-org/spetlr (language: python)

Issue:
feat: update tenant/subscription
## What type of PR is this? (check all applicable)

- Refactor
- Bug Fix

## Description

**NB:** I have manually created the deploying spn (using the `setup_deploying_spn.ps1`). Also updated the github secrets.

### Overview
This PR changes the tenant and subscription for the deployment of the azure environment

### What is the current behavior?
In  the moment the resources are deployed to one subscription.

### What is the new behavior?
Now the moment the resources are deployed to another subscription B.
Also, I might need to change the location of some resources due to Azure limitations.

### Does this PR introduce a breaking change?
All resources and SPN will be redployed. It will not change the spetlr python code - only deployment for tests.
We can experience issues do to this update. But the IaC should be able to handle it all.
The location of the resources needed to be changed due to Azure limitations of the resources. This should be investigated further.

feat: eh2delta incremental when append
When appending data to silver - it makes sense to use incremental extraction.

Now, when using "append" - all data is appended each time the orchestrator is runned.
Feature/validfrom transformer
A transformer for introducing SCD2 to tables.


Discussion / hints:

Fix patch:
diff --git a/.github/deploy-bicep/resources-integration.bicep b/.github/deploy-bicep/resources-integration.bicep
index 2508ab63..8d8aab18 100644
--- a/.github/deploy-bicep/resources-integration.bicep
+++ b/.github/deploy-bicep/resources-integration.bicep
@@ -35,7 +35,7 @@ resource rsdatabricks 'Microsoft.Databricks/workspaces@2022-04-01-preview' = {
 
 resource staccount 'Microsoft.Storage/storageAccounts@2021-09-01' = {
   name: dataLakeName
-  location: location
+  location: 'northeurope'
   tags: resourceTags
   sku: {
     name: 'Standard_LRS'
@@ -115,7 +115,7 @@ resource eventhubs 'Microsoft.EventHub/namespaces@2021-11-01' = {
 
 resource sqlserver 'Microsoft.Sql/servers@2022-02-01-preview' = {
   name: databaseServerName
-  location: location
+  location: 'eastus'
   tags: resourceTags
   identity: {
     type: 'SystemAssigned'
@@ -157,7 +157,7 @@ resource sqlserver 'Microsoft.Sql/servers@2022-02-01-preview' = {
 resource sqlDb 'Microsoft.Sql/servers/databases@2022-02-01-preview' = {
   name: deliveryDatabase
   parent: sqlserver
-  location: location
+  location: 'eastus'
   tags: resourceTags
   sku: {
     name: 'GP_S_Gen5'
diff --git a/.github/deploy-bicep/resources-permanent.bicep b/.github/deploy-bicep/resources-permanent.bicep
index b634af65..cbe98f72 100644
--- a/.github/deploy-bicep/resources-permanent.bicep
+++ b/.github/deploy-bicep/resources-permanent.bicep
@@ -18,6 +18,7 @@ resource kw 'Microsoft.KeyVault/vaults@2022-07-01' = {
   location: location
   tags: tags
   properties: {
+
     sku: {
       family: 'A'
       name: 'standard'
@@ -36,6 +37,7 @@ resource kw 'Microsoft.KeyVault/vaults@2022-07-01' = {
         tenantId: tenant().tenantId
       }
     ]
+
   }
 }
 
@@ -53,8 +55,11 @@ resource csdb 'Microsoft.DocumentDB/databaseAccounts@2022-05-15' = {
     locations: [ {
         failoverPriority: 0
         isZoneRedundant: false
-        locationName: location
+        locationName: 'northeurope'
       } ]
+    //capabilities: [ {
+    //    name: 'EnableServerless'
+     // } ]
   }
 }
 
diff --git a/.github/deploy/setup_deploying_SPN.ps1 b/.github/deploy/setup_deploying_SPN.ps1
index 8d0e9adc..a5be6cc3 100644
--- a/.github/deploy/setup_deploying_SPN.ps1
+++ b/.github/deploy/setup_deploying_SPN.ps1
@@ -7,14 +7,24 @@
 #az account list
 ## find the corect one
 #az account set --subscription $mySpetlrAzureSubscription
-Write-Host "Check correct subsctiption is selected."
-$account = az account show | ConvertFrom-Json
-Write-Host "Current subscription is $($account.name)"
-if ($account.name -notmatch "ATC"){
-  Write-Host "Expected subscription to match ATC" -ForegroundColor Red
+
+# Get the current subscription name
+$subscriptionName = az account show --query name -o tsv
+
+# Use the subscription name to get the subscription ID
+$subscriptionId = az account list --query "[?name=='$subscriptionName'].id" -o tsv
+
+# Output the subscription ID
+Write-Output "Current subscription ID: $subscriptionId"
+
+
+Write-Host "Checkinjg.. correct subsription is selected."
+if ($subscriptionId -ne "f861842b-e686-40fb-8b34-87e8735e8749"){
+  Write-Host "Expected subscription to match f861842b-e686-40fb-8b34-87e8735e8749" -ForegroundColor Red
   Write-Host "Please change subscription" -ForegroundColor Red
   Exit 1
 }
+Write-Host "Subscription Id is correct!" -ForegroundColor Green
 
 ####################################################################################
 ## Step 2. Create app registration
diff --git a/.github/deploy/steps/00-Config.ps1 b/.github/deploy/steps/00-Config.ps1
index fcf33948..f91455ce 100644
--- a/.github/deploy/steps/00-Config.ps1
+++ b/.github/deploy/steps/00-Config.ps1
@@ -11,9 +11,9 @@ if (-not $secrets) {
 $repoRoot = (git rev-parse --show-toplevel)
 $sqlSourceDir = Resolve-Path $PSScriptRoot/sql
 
-$permanentResourceName = "githubatc"
-$permanentResourceGroup = "atc-permanent"
-$keyVaultName = "atcGithubCiCd"
+$permanentResourceName = "spetlrpermanent"
+$permanentResourceGroup = "spetlr-permanent"
+$keyVaultName = "spetlrGithubCiCdv2"
 
 # at some point, the following will be made variable between deployments
 $resourceName = "githubspetlr$uniqueRunId"
@@ -86,13 +86,13 @@ $sqlAdminSpnName = $cicdSpnName
 
 $logAnalyticsWsName = $resourceGroupName
 
-$metastoreStorageAccountName = "githubspetlrmetastore"
-$metastoreContainerName = "metastore"
-$metastoreAccessConnectorName = "ac-metastore"
-$metastoreDatabricksName = "dbws-metastore"
-$metastoreName = "spetlr-metastore"
-$metastoreCatalogName = "spetlr_catalog"
-$subscriptionId = "c3a3d88f-e2a3-40c1-8ce7-e88453a01548"
+$metastoreStorageAccountName = "githubspetlrmetastorev2"
+$metastoreContainerName = "metastorev2"
+$metastoreAccessConnectorName = "ac-metastorev2"
+$metastoreDatabricksName = "dbws-metastorev2"
+$metastoreName = "spetlr-metastorev2"
+$metastoreCatalogName = "spetlr_catalogv2"
+$subscriptionId = "f861842b-e686-40fb-8b34-87e8735e8749"
 
 Write-Host "**********************************************************************" -ForegroundColor White
 Write-Host "* Base Configuration       *******************************************" -ForegroundColor White
diff --git a/.github/deploy/steps/10-Verify-Arguments.ps1 b/.github/deploy/steps/10-Verify-Arguments.ps1
index d0df58b1..108b0eb0 100644
--- a/.github/deploy/steps/10-Verify-Arguments.ps1
+++ b/.github/deploy/steps/10-Verify-Arguments.ps1
@@ -5,10 +5,6 @@ if (!$tenantId) {
   $tenantId = (az account show --query tenantId --out tsv)
 }
 
-if (-not (Check-AzureAccountTag "spetlr")) {
-  throw "Refusing to deploy to incorrectly tagged subscription."
-}
-
 # Allow dynamic install of extensions to be able to use az databricks commands
 az config set extension.use_dynamic_install=yes_without_prompt
 
diff --git a/.github/deploy/steps/25-Provision-Service-Principal.ps1 b/.github/deploy/steps/25-Provision-Service-Principal.ps1
index c018e78d..f56c9363 100644
--- a/.github/deploy/steps/25-Provision-Service-Principal.ps1
+++ b/.github/deploy/steps/25-Provision-Service-Principal.ps1
@@ -34,4 +34,4 @@ $secrets.addSecret("Databricks--ClientSecret", $mountSpn.secretText)
 
 # there is a chicken-and-egg problem where we want to save the new SPN secret in the
 # keyvault, but the keyvault may not exist yet. This doesn't matter since the keyvault
-# is never destroyed, and by the second run, it will exist.
+# is never destroyed, and by the second run, it will exist.
\ No newline at end of file
diff --git a/.github/deploy/steps/30-deploy-bicep.ps1 b/.github/deploy/steps/30-deploy-bicep.ps1
index 987e0fe1..f76c45fa 100644
--- a/.github/deploy/steps/30-deploy-bicep.ps1
+++ b/.github/deploy/steps/30-deploy-bicep.ps1
@@ -11,6 +11,7 @@ $sqlAdminSpnName=$dbSpn.name
 $sqlAdminObjectId=$dbSpn.objectId
 
 $output = az deployment sub create `
+  --name $resourceName `
   --location $location `
   --template-file $repoRoot\.github\deploy-bicep\main.bicep `
   --parameters `
diff --git a/.github/workflows/azure-cleanup.yml b/.github/workflows/azure-cleanup.yml
index 7c33a43e..531a6e82 100644
--- a/.github/workflows/azure-cleanup.yml
+++ b/.github/workflows/azure-cleanup.yml
@@ -24,7 +24,8 @@ jobs:
           az login --service-principal `
             -u ${{ secrets.SPN_CLIENT_ID }} `
             -p ${{ secrets.SPN_CLIENT_SECRET }} `
-            --tenant ${{ secrets.SPN_TENANT_ID }}
+            --tenant ${{ secrets.SPN_TENANT_ID }} `
+            --output none
       - name: Delete Deployment
         shell: pwsh
         run: |
diff --git a/.github/workflows/pre-integration.yml b/.github/workflows/pre-integration.yml
index 5dfd108a..43a76ed2 100644
--- a/.github/workflows/pre-integration.yml
+++ b/.github/workflows/pre-integration.yml
@@ -64,7 +64,9 @@ jobs:
           az login --service-principal `
             -u ${{ secrets.SPN_CLIENT_ID }} `
             -p ${{ secrets.SPN_CLIENT_SECRET }} `
-            --tenant ${{ secrets.SPN_TENANT_ID }}
+            --tenant ${{ secrets.SPN_TENANT_ID }} `
+            --output none
+            
       - name: Create Deployment
         shell: pwsh
         run: |
@@ -111,6 +113,8 @@ jobs:
           az login --service-principal `
             -u ${{ secrets.SPN_CLIENT_ID }} `
             -p ${{ secrets.SPN_CLIENT_SECRET }} `
-            --tenant ${{ secrets.SPN_TENANT_ID }}
+            --tenant ${{ secrets.SPN_TENANT_ID }} `
+            --output none
+
           .github/deploy/destroy.ps1 `
             -uniqueRunId "${{ github.run_id }}${{ github.run_attempt }}"
diff --git a/.github/workflows/spn-cleanup.yml b/.github/workflows/spn-cleanup.yml
index e264c38f..99e07a13 100644
--- a/.github/workflows/spn-cleanup.yml
+++ b/.github/workflows/spn-cleanup.yml
@@ -21,7 +21,9 @@ jobs:
           az login --service-principal `
             -u ${{ secrets.SPN_CLIENT_ID }} `
             -p ${{ secrets.SPN_CLIENT_SECRET }} `
-            --tenant ${{ secrets.SPN_TENANT_ID }}
+            --tenant ${{ secrets.SPN_TENANT_ID }} `
+            --output none
+            
       - name: Delete Deployment
         shell: pwsh
         run: |
diff --git a/docs/power_bi/README.md b/docs/power_bi/README.md
new file mode 100644
index 00000000..f92f2c59
--- /dev/null
+++ b/docs/power_bi/README.md
@@ -0,0 +1,226 @@
+
+# PowerBi and PowerBiClient classes
+
+The `PowerBi` and `PowerBiClient` classes contain logic for refreshing
+PowerBI datasets and for checking if the last dataset refresh completed
+successfully. 
+
+For easier PowerBI credential handling (service principal or AD user),
+the first parameter to the `PowerBi` constructor must be a `PowerBiClient`
+class object. 
+
+## PowerBI Permissions
+
+To enable PowerBI API access in your PowerBI, you need to enable
+the "Service principals can use Fabric APIs" setting (see the screen-shot).
+Additionally, you need to specify the user group that should have access 
+to the API.
+
+![Power BI admin settings](./admin_settings.png)
+
+Apart from this, each PowerBI dataset should have a user or service principal
+attached, that is part of this user group.
+
+
+## Links
+
+[Register an App and give the needed permissions. A very well how-to-guide can be found here.](https://www.sqlshack.com/how-to-access-power-bi-rest-apis-programmatically/)
+
+[How to Refresh a Power BI Dataset with Python.](https://pbi-guy.com/2022/01/07/refresh-a-power-bi-dataset-with-python/)
+
+### API documentation:
+
+[Datasets - Refresh Dataset In Group](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/refresh-dataset-in-group)
+
+[Datasets - Get Refresh History In Group](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/get-refresh-history-in-group)
+
+
+# Usage of PowerBi and PowerBiClient classes
+
+## Step 1: Create PowerBI credentials
+
+The client ID, client secret, and tenant ID values should be stored in a key vault,
+and loaded from the key vault or Databricks secrets scope.
+
+```python
+# example PowerBiClient credentials object
+from spetlr.power_bi.PowerBiClient import PowerBiClient 
+from my_proj.env import secrets
+
+class MyPowerBiClient(PowerBiClient):
+    def __init__(self):
+        super().__init__(
+            client_id=secrets.get_power_bi_client(),
+            client_secret=secrets.get_power_bi_secret(),
+            tenant_id=secrets.get_power_bi_tenant(),
+        )
+```
+
+## Step 2: List available workspaces
+
+If no workspace parameter is specified, a list of available workspaces
+is shown using Pandas. This logic can be used in a notebook.
+
+```python
+# example listing of available workspaces
+from spetlr.power_bi.PowerBi import PowerBi 
+
+client = MyPowerBiClient()
+PowerBi(client).check()
+```
+
+```
+Available workspaces:   
++----+--------------------------------------+----------------+
+|    |            workspace_id              | workspace_name |
++----+--------------------------------------+----------------+
+|   1| 614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0 | Finance        |
+|   2| 5da990e9-089e-472c-a7fa-4fc3dd096d01 | CRM            |
++----+--------------------------------------+----------------+
+```
+
+## Step 3: List available datasets
+
+If no dataset parameter is specified, a list of available datasets
+in the given workspace is shown using Pandas.
+This logic can be used in a notebook.
+
+The workspace can be specified either as name or id, but not both.
+
+```python
+# example listing of available datasets
+from spetlr.power_bi.PowerBi import PowerBi 
+
+client = MyPowerBiClient()
+PowerBi(client, workspace_name="Finance").check()
+
+# alternatively:
+PowerBi(client, workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0").check()
+```
+
+```
+Available datasets:
++----+--------------------------------------+----------------+
+|    |             dataset_id               |  dataset_name  |
++----+--------------------------------------+----------------+
+|   1| b1f0a07e-e348-402c-a2b2-11f3e31181ce | Invoicing      |
+|   2| 2e848e9a-47a3-4b0e-a22a-af35507ec8c4 | Reimbursement  |
+|   3| 4de28a6f-f7d4-4186-a529-bf6c65e67b31 | Fees           |
++----+--------------------------------------+----------------+
+```
+
+## Step 4: Check the status and time of the last refresh of a given dataset
+
+The check() method can be used to check the status and time of the last
+refresh of a dataset. An exception will be cast if the last refresh failed,
+or if the last refresh finished more the given number of minutes ago.
+The number of minutes can be specified in the optional
+"max_minutes_after_last_refresh" parameter (default is 12 hours).
+
+You can also specify the optional "local_timezone_name" parameter to show
+the last refresh time of the PowerBI dataset in a local timezone.
+Default timezone is UTC.
+
+```python
+# example last refresh time checking
+from spetlr.power_bi.PowerBi import PowerBi 
+
+client = MyPowerBiClient()
+PowerBi(client,
+        workspace_name="Finance",
+        dataset_name="Invoicing",
+        max_minutes_after_last_refresh=2*60,
+        local_timezone_name="Europe/Copenhagen").check()
+
+# alternatively:
+PowerBi(client,
+        workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0",
+        dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce",
+        max_minutes_after_last_refresh=2*60,
+        local_timezone_name="Europe/Copenhagen").check()
+```
+
+```
+Refresh completed successfully at 2024-02-01 10:15 (local time).
+True   
+```
+
+```
+Exception: Last refresh finished more than 15 minutes ago
+at 2024-02-01 10:15 (local time) !
+```
+
+## Step 5: Start a new refresh of a given dataset without waiting
+
+The start_refresh() method starts a new refresh of the given PowerBI
+dataset asynchronously. You need to call the check() method after waiting
+for some sufficient time (e.g. from a separate monitoring job) to verify
+if the refresh succeeded.
+
+```python
+# example starting of a dataset refresh
+from spetlr.power_bi.PowerBi import PowerBi 
+
+client = MyPowerBiClient()
+PowerBi(client,
+        workspace_name="Finance",
+        dataset_name="Invoicing").start_refresh()
+
+# alternatively:
+PowerBi(client,
+        workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0",
+        dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce").start_refresh()
+```
+
+```
+A new refresh has been successfully triggered.
+True
+```
+
+## Step 6: Start a new refresh of a given dataset and wait for the result
+
+The refresh() method starts a new refresh of the given PowerBI dataset
+synchronously. It waits until the refresh is finished or until a time-out
+occurs. The time-out can be specified using the optional "timeout_in_seconds"
+parameter (default is 15 minutes). 
+If the refresh fails or a time-out occurs, the method casts an exception.
+
+The wait time between calls to the PowerBI API is synchronized with the
+execution time of the previous dataset refresh, making sure as few requests
+to the PowerBI API would be made as possible, while ensuring the method
+would finish as soon as possible.
+
+```python
+# example starting of a dataset refresh
+from spetlr.power_bi.PowerBi import PowerBi 
+
+client = MyPowerBiClient()
+PowerBi(client,
+        workspace_name="Finance",
+        dataset_name="Invoicing",
+        timeout_in_seconds=10*60).refresh()
+
+# alternatively:
+PowerBi(client,
+        workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0",
+        dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce",
+        timeout_in_seconds=10*60).refresh()
+```
+
+```
+A new refresh has been successfully triggered.
+Waiting 60 seconds...
+Waiting 15 seconds...
+Refresh completed successfully at 2024-02-02 09:02 (local time).
+True
+```
+
+# Testing
+
+Due to license restrictions, testing requires a valid PowerBI license.
+Because of this, testing must be executed manually in each project
+that uses spetlr to refresh datasets.
+
+Recommended integration tests should include all above examples, i.e. 
+listing of workspaces and datasets, checking a refresh, and possibly
+triggering a new refresh.
diff --git a/docs/power_bi/admin_settings.png b/docs/power_bi/admin_settings.png
new file mode 100644
index 00000000..81e80732
Binary files /dev/null and b/docs/power_bi/admin_settings.png differ
diff --git a/setup.cfg b/setup.cfg
index b2cab253..079cfed4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -42,6 +42,8 @@ install_requires =
     more_itertools
     pycountry
     databricks-sdk
+    msal
+    pandas
     # Numpy does not support 3.8 if above 1.24
     numpy<=1.24;python_version<="3.8"
     numpy;python_version>"3.8"
diff --git a/src/spetlr/power_bi/PowerBi.py b/src/spetlr/power_bi/PowerBi.py
new file mode 100644
index 00000000..897aac1e
--- /dev/null
+++ b/src/spetlr/power_bi/PowerBi.py
@@ -0,0 +1,459 @@
+import time
+from datetime import datetime, timedelta
+
+import msal
+import pandas as pd
+import requests
+from dateutil import parser
+from pytz import timezone, utc
+
+from spetlr.exceptions import SpetlrException
+
+from .PowerBiClient import PowerBiClient
+
+
+class PowerBi:
+    def __init__(
+        self,
+        client: PowerBiClient,
+        *,
+        workspace_id: str = None,
+        workspace_name: str = None,
+        dataset_id: str = None,
+        dataset_name: str = None,
+        max_minutes_after_last_refresh: int = 60 * 12,
+        timeout_in_seconds: int = 60 * 15,
+        local_timezone_name: str = "UTC",
+        ignore_errors: bool = False,
+    ):
+        """
+        Allows refreshing PowerBI datasets and checking if the last refresh
+            completed successfully.
+        If no workspace is specified, a list of available workspaces will be displayed.
+        If no dataset is specified, a list of available datasets in the given workspace
+            will be displayed.
+
+        :param PowerBiClient client: PowerBI client credentials.
+
+        :param str workspace_id: The GUID of the workspace.
+        :param str workspace_name: The name of the workspace
+            (specified instead of the workspace_id).
+        :param str dataset_id: The GUID of the dataset.
+        :param str dataset_name: The name of the dataset
+            (specified instead of the dataset_id).
+        :param int max_minutes_after_last_refresh: The number of minutes
+            for which the last succeeded refresh is considered valid,
+            or 0 to disable time checking. Default is 12 hours.
+        :param bool timeout_in_seconds: The number of seconds after which
+            the refresh() method times out. Default is 15 minutes.
+        :param str local_timezone_name: The timezone to use when showing
+            refresh timestamps. Only used for printing timestamps.
+            Default timezone is UTC.
+        :param bool ignore_errors: True to print errors in the output
+            or False (default) to cast a SpetlrException.
+        """
+
+        if workspace_id is not None and workspace_name is not None:
+            raise ValueError("Specify either 'workspace_id' or 'workspace_name'!")
+        if dataset_id is not None and dataset_name is not None:
+            raise ValueError("Specify either 'dataset_id' or 'dataset_name'!")
+
+        self.workspace_id = workspace_id
+        self.workspace_name = workspace_name
+        self.dataset_id = dataset_id
+        self.dataset_name = dataset_name
+
+        # Set access parameters
+        self.client = client
+
+        self.max_minutes_after_last_refresh = max_minutes_after_last_refresh
+        self.timeout_in_seconds = timeout_in_seconds
+        self.local_timezone_name = local_timezone_name
+        self.ignore_errors = ignore_errors
+        self.api_header = None
+        self.expire_time = 0
+
+        self.last_status = None
+        self.last_exception = None
+        self.last_refresh_utc = None
+        self.last_duration = 0
+
+    def _raise_error(self, message: str):
+        if self.ignore_errors:
+            print(message)
+        else:
+            raise SpetlrException(message)
+
+    def _raise_api_error(self, message: str, api_call: requests.Response):
+        print(api_call.text)
+        self._raise_error(
+            message + f" Response: {api_call.status_code} {api_call.reason}"
+        )
+
+    def _get_access_token(self) -> bool:
+        """
+        Acquires an access token to connect to PowerBI.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if self.expire_time != 0:
+            if time.time() < self.expire_time:
+                return True
+            print("Renewing access token...")
+
+        # Prepare URLs (note: we fetch only the latest refresh record, i.e. top=1)
+        authority_url = f"https://login.microsoftonline.com/{self.client.tenant_id}/"
+        scope = ["https://analysis.windows.net/powerbi/api/.default"]
+        self.powerbi_url = "https://api.powerbi.com/v1.0/myorg/"
+
+        # Use MSAL to get an access token
+        app = msal.ConfidentialClientApplication(
+            self.client.client_id,
+            authority=authority_url,
+            client_credential=self.client.client_secret,
+        )
+        result = app.acquire_token_for_client(scopes=scope)
+        if "access_token" not in result:
+            print(result)
+            self._raise_error("Failed to acquire token!")
+            return False
+
+        default_expires_in = 5 * 60
+        extra_seconds = 3
+
+        access_token = result["access_token"]
+        if "expires_in" in result:
+            self.expire_time = time.time() + result["expires_in"] - extra_seconds
+        else:
+            self.expire_time = time.time() + default_expires_in - extra_seconds
+
+        # Get latest Power BI Dataset refresh record
+        self.api_header = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {access_token}",
+        }
+        return True
+
+    @staticmethod
+    def _show_workspaces(df: pd.DataFrame):
+        print("Available workspaces:")
+        df.rename(
+            columns={"id": "workspace_id", "name": "workspace_name"},
+            inplace=True,
+        )
+        df.display()
+
+    @staticmethod
+    def _show_datasets(df: pd.DataFrame):
+        print("Available datasets:")
+        df.rename(
+            columns={"id": "dataset_id", "name": "dataset_name"},
+            inplace=True,
+        )
+        df.display()
+
+    def _get_workspace(self) -> bool:
+        """
+        Gets workspace ID based on the workspace name, or shows all workspaces
+            if no parameter was specified.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if self.workspace_id is None:
+            api_call = requests.get(
+                url=f"{self.powerbi_url}groups", headers=self.api_header
+            )
+            if api_call.status_code != 200:
+                self._raise_api_error("Failed to fetch workspaces!", api_call)
+                return False
+            df = pd.DataFrame(api_call.json()["value"], columns=["id", "name"])
+
+            if self.workspace_name is None:
+                self._show_workspaces(df)
+                return False
+
+            rows = df.loc[df["name"] == self.workspace_name, "id"]
+            if rows.empty:
+                self._raise_error(
+                    f"Workspace name '{self.workspace_name}' cannot be found!"
+                )
+                return False
+            self.workspace_id = rows.values[0]
+
+        return True
+
+    def _get_dataset(self) -> bool:
+        """
+        Gets dataset ID based on the dataset name, or shows all datasets
+            if no parameter was specified.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if self.dataset_id is None:
+            api_call = requests.get(
+                url=f"{self.powerbi_url}groups/{self.workspace_id}/datasets",
+                headers=self.api_header,
+            )
+            if api_call.status_code != 200:
+                self._raise_api_error("Failed to fetch datasets!", api_call)
+                return False
+            df = pd.DataFrame(api_call.json()["value"], columns=["id", "name"])
+
+            if self.dataset_name is None:
+                self._show_datasets(df)
+                return False
+
+            rows = df.loc[df["name"] == self.dataset_name, "id"]
+            if rows.empty:
+                self._raise_error(
+                    f"Dataset name '{self.dataset_name}' cannot be found, "
+                    "or the dataset doesn't have a user with the required permissions!"
+                )
+                return False
+            self.dataset_id = rows.values[0]
+
+        return True
+
+    def _connect(self) -> bool:
+        """
+        Connects or reconnects to PowerBI to fetch refresh history or trigger a refresh.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if not self._get_access_token():
+            return False
+        if not self._get_workspace():
+            return False
+        if not self._get_dataset():
+            return False
+
+        return True
+
+    def _get_last_refresh(self) -> bool:
+        """
+        Gets the latest record in the PowerBI dataset refresh history.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if not self._connect():
+            return False
+
+        api_url = (
+            f"{self.powerbi_url}groups/{self.workspace_id}"
+            f"/datasets/{self.dataset_id}/refreshes?$top=1"
+        )
+        api_call = requests.get(url=api_url, headers=self.api_header)
+        if api_call.status_code == 200:
+            json = api_call.json()
+            df = pd.DataFrame(
+                json["value"],
+                columns=[
+                    "requestId",
+                    "id",
+                    "refreshType",
+                    "startTime",
+                    "endTime",
+                    "status",
+                    "serviceExceptionJson",
+                ],
+            )
+            df.set_index("id")
+            if len(df.index) > 0:
+                self.last_status = df.status[0]
+                self.last_exception = df.serviceExceptionJson[0]
+                if (
+                    self.last_status == "Completed"
+                    and df.endTime[0] is not None
+                    and len(df.endTime[0]) > 0
+                ):
+                    self.last_refresh_utc = parser.parse(df.endTime[0]).replace(
+                        tzinfo=utc
+                    )
+                    if df.startTime[0] is not None and len(df.startTime[0]) > 0:
+                        self.last_duration = int(
+                            (
+                                parser.parse(df.endTime[0])
+                                - parser.parse(df.startTime[0])
+                            ).total_seconds()
+                        )
+            return True
+        elif api_call.status_code == 404:
+            self._raise_error(
+                "The specified dataset or workspace cannot be found, "
+                "or the dataset doesn't have a user with the required permissions!"
+            )
+        else:
+            self._raise_api_error("Failed to fetch refresh history!", api_call)
+        return False
+
+    def _verify_last_refresh(self) -> bool:
+        """
+        Checks if the last refresh of the PowerBI dataset completed successfully,
+            and verifies if it happened recently enough.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if self.last_status is None:
+            self._raise_error("Refresh is still in progress or never triggered!")
+        elif self.last_status == "Completed":
+            if self.last_refresh_utc is None:
+                self._raise_error("Completed at unknown refresh time!")
+            else:
+                last_refresh_str = self.last_refresh_utc.astimezone(
+                    timezone(self.local_timezone_name)
+                ).strftime("%Y-%m-%d %H:%M") + (
+                    " (UTC)"
+                    if self.local_timezone_name.lower() == "utc"
+                    else " (local time)"
+                )
+                min_refresh_time_utc = datetime.now(utc) - timedelta(
+                    minutes=self.max_minutes_after_last_refresh
+                )
+                if (
+                    self.max_minutes_after_last_refresh > 0
+                    and self.last_refresh_utc < min_refresh_time_utc
+                ):
+                    self._raise_error(
+                        "Last refresh finished more than "
+                        f"{self.max_minutes_after_last_refresh} "
+                        f"minutes ago at {last_refresh_str} !"
+                    )
+                else:
+                    print(f"Refresh completed successfully at {last_refresh_str}.")
+                    return True
+        elif self.last_status == "Unknown":
+            self._raise_error("Refresh is still in progress!")
+        elif self.last_status == "Disabled":
+            self._raise_error("Refresh is disabled!")
+        elif self.last_status == "Failed":
+            self._raise_error(f"Last refresh failed! {self.last_exception}")
+        else:
+            self._raise_error(
+                f"Unknown refresh status: {self.last_status}! {self.last_exception}"
+            )
+        return False
+
+    def _trigger_new_refresh(self) -> bool:
+        """
+        Starts a refresh of the PowerBI dataset.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        if self.last_status is None or self.last_status in ["Completed", "Failed"]:
+            if self.last_status == "Failed":
+                print(f"Warning: Last refresh failed! {self.last_exception}")
+                print()
+
+            api_url = (
+                f"{self.powerbi_url}groups/{self.workspace_id}"
+                f"/datasets/{self.dataset_id}/refreshes"
+            )
+            api_call = requests.post(url=api_url, headers=self.api_header)
+            if api_call.status_code == 202:
+                print("A new refresh has been successfully triggered.")
+                return True
+            else:
+                self._raise_api_error("Failed to trigger a refresh!", api_call)
+        elif self.last_status == "Unknown":
+            print("Refresh is already in progress!")
+            return True
+        elif self.last_status == "Disabled":
+            self._raise_error("Refresh is disabled!")
+        else:
+            self._raise_error(
+                f"Unknown refresh status: {self.last_status}! {self.last_exception}"
+            )
+        return False
+
+    def _get_seconds_to_wait(self, elapsed: int) -> int:
+        """
+        Returns the number of seconds to wait before rechecking
+            if the refresh has completed. The method makes sure as few requests
+            to the PowerBI API as possible would be made.
+
+        :return: number of seconds to wait
+        :rtype: int
+        """
+
+        # Microsoft set a limit of how many API requests in an hour can be made
+        # to the PowerBI API. This logic is a fine compromise between how often
+        # (small delay) we check through polling if the refresh has finished and
+        # how soon (large delay) will the refresh() method complete.
+
+        if self.last_duration > 0:
+            elapsed = abs(self.last_duration - elapsed)
+
+        return 15 if elapsed < 40 else 60 if elapsed < (3 * 60) else (5 * 60)
+
+    def check(self) -> bool:
+        """
+        Checks if the last refresh of the PowerBI dataset completed successfully,
+            and verifies if it happened recently enough.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        return self._get_last_refresh() and self._verify_last_refresh()
+
+    def start_refresh(self) -> bool:
+        """
+        Starts a refresh of the PowerBI dataset without waiting.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        return self._get_last_refresh() and self._trigger_new_refresh()
+
+    def refresh(self) -> bool:
+        """
+        Starts a refresh of the PowerBI dataset and waits until completed.
+
+        :return: True if succeeded or False if failed (when ignore_errors==True)
+        :rtype: bool
+        :raises SpetlrException: if failed and ignore_errors==False
+        """
+
+        start_time = time.time()
+        if not self.start_refresh():
+            return False
+
+        while True:
+            elapsed = int(time.time() - start_time)
+            if elapsed > self.timeout_in_seconds:
+                print("Timeout!")
+                break
+            wait_seconds = self._get_seconds_to_wait(elapsed)
+            print(f"Waiting {wait_seconds} seconds...")
+            time.sleep(wait_seconds)
+
+            if not self._get_last_refresh():
+                return False
+            if self.last_status != "Unknown":
+                break
+
+        return self._verify_last_refresh()
diff --git a/src/spetlr/power_bi/PowerBiClient.py b/src/spetlr/power_bi/PowerBiClient.py
new file mode 100644
index 00000000..092d6449
--- /dev/null
+++ b/src/spetlr/power_bi/PowerBiClient.py
@@ -0,0 +1,19 @@
+class PowerBiClient:
+    def __init__(
+        self,
+        *,
+        client_id: str = None,
+        client_secret: str = None,
+        tenant_id: str = None,
+    ):
+        """
+        Creates PowerBI client credentials.
+
+        :param str client_id: PowerBI client ID.
+        :param str client_secret: PowerBI client secret.
+        :param str tenant_id: PowerBI tenant ID.
+        """
+
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.tenant_id = tenant_id
diff --git a/src/spetlr/power_bi/__init__.py b/src/spetlr/power_bi/__init__.py
new file mode 100644
index 00000000..e69de29b


Test patch:
diff --git a/tests/local/power_bi/__init__.py b/tests/local/power_bi/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/local/power_bi/test_power_bi.py b/tests/local/power_bi/test_power_bi.py
new file mode 100644
index 00000000..ff3fcb88
--- /dev/null
+++ b/tests/local/power_bi/test_power_bi.py
@@ -0,0 +1,259 @@
+import unittest
+from datetime import datetime, timedelta
+from unittest.mock import Mock, patch
+
+from pytz import utc
+
+from spetlr.exceptions import SpetlrException
+from spetlr.power_bi.PowerBi import PowerBi
+from spetlr.power_bi.PowerBiClient import PowerBiClient
+
+
+class TestPowerBi(unittest.TestCase):
+    @patch("requests.get")
+    def test_get_workspace_success(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "value": [{"id": "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0", "name": "Finance"}]
+        }
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(PowerBiClient(), workspace_name="Finance")
+        sut.powerbi_url = "test"
+
+        # Act
+        result = sut._get_workspace()
+
+        # Assert
+        self.assertTrue(result)
+        self.assertEqual(sut.workspace_id, "614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0")
+
+    @patch("requests.get")
+    def test_get_workspace_failure(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_response.text = "error"
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(PowerBiClient(), workspace_name="Finance")
+        sut.powerbi_url = "test"
+
+        # Act
+        with self.assertRaises(SpetlrException) as context:
+            sut._get_workspace()
+
+        # Assert
+        self.assertIn("Failed to fetch workspaces", str(context.exception))
+
+    @patch("requests.get")
+    def test_get_dataset_success(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "value": [
+                {"id": "b1f0a07e-e348-402c-a2b2-11f3e31181ce", "name": "Invoicing"}
+            ]
+        }
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(
+            PowerBiClient(),
+            workspace_name="Finance",
+            dataset_name="Invoicing",
+        )
+        sut.powerbi_url = "test"
+
+        # Act
+        result = sut._get_dataset()
+
+        # Assert
+        self.assertTrue(result)
+        self.assertEqual(sut.dataset_id, "b1f0a07e-e348-402c-a2b2-11f3e31181ce")
+
+    @patch("requests.get")
+    def test_get_dataset_failure(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_response.text = "error"
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(
+            PowerBiClient(),
+            workspace_name="Finance",
+            dataset_name="Invoicing",
+        )
+        sut.powerbi_url = "test"
+
+        # Act
+        with self.assertRaises(SpetlrException) as context:
+            sut._get_dataset()
+
+        # Assert
+        self.assertIn("Failed to fetch datasets", str(context.exception))
+
+    @patch("requests.get")
+    def test_get_last_refresh_success(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "value": [
+                {
+                    "requestId": "1",
+                    "id": "1",
+                    "refreshType": "Manual",
+                    "startTime": "2024-02-26T10:00:00Z",
+                    "endTime": "2024-02-26T10:05:00Z",
+                    "status": "Completed",
+                    "serviceExceptionJson": None,
+                }
+            ]
+        }
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test")
+        sut.powerbi_url = "test"
+        sut._connect = lambda: True
+
+        # Act
+        result = sut._get_last_refresh()
+
+        # Assert
+        self.assertTrue(result)
+        self.assertEqual(sut.last_status, "Completed")
+        self.assertIsNone(sut.last_exception)
+        self.assertEqual(str(sut.last_refresh_utc), "2024-02-26 10:05:00+00:00")
+        self.assertEqual(sut.last_duration, 300)
+
+    @patch("requests.get")
+    def test_get_last_refresh_failure(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 404  # dataset or workspace not found
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(PowerBiClient(), workspace_id="test", dataset_id="test")
+        sut.powerbi_url = "test"
+        sut._connect = lambda: True
+
+        # Act
+        with self.assertRaises(SpetlrException) as context:
+            sut._get_last_refresh()
+
+        # Assert
+        self.assertIn(
+            "The specified dataset or workspace cannot be found",
+            str(context.exception),
+        )
+
+    def test_verify_last_refresh_success(self):
+        # Arrange
+        sut = PowerBi(
+            PowerBiClient(),
+            workspace_id="test",
+            dataset_id="test",
+            max_minutes_after_last_refresh=5,
+            local_timezone_name="Europe/Copenhagen",
+        )
+        sut.last_status = "Completed"
+        sut.last_refresh_utc = datetime.now(utc) - timedelta(minutes=1)
+
+        # Act
+        result = sut._verify_last_refresh()
+
+        # Assert
+        self.assertTrue(result)
+
+    def test_verify_last_refresh_failure(self):
+        # Arrange
+        sut = PowerBi(
+            PowerBiClient(),
+            workspace_id="test",
+            dataset_id="test",
+            max_minutes_after_last_refresh=5,
+            local_timezone_name="Europe/Copenhagen",
+        )
+        sut.last_status = "Completed"
+        sut.last_refresh_utc = datetime.now(utc) - timedelta(minutes=10)
+
+        # Act
+        with self.assertRaises(SpetlrException) as context:
+            sut._verify_last_refresh()
+
+        # Assert
+        self.assertIn("Last refresh finished more than", str(context.exception))
+
+    @patch("requests.post")
+    def test_trigger_new_refresh_success(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 202
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(
+            PowerBiClient(),
+            workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0",
+            dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce",
+        )
+        sut.last_status = "Completed"
+        sut.powerbi_url = "test"
+
+        # Act
+        result = sut._trigger_new_refresh()
+
+        # Assert
+        self.assertTrue(result)
+
+    @patch("requests.post")
+    def test_trigger_new_refresh_failure(self, mock_get):
+        # Arrange
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_response.text = "error"
+        mock_get.return_value = mock_response
+
+        sut = PowerBi(
+            PowerBiClient(),
+            workspace_id="614850c2-3a5c-4d2d-bcaa-d3f20f32a2e0",
+            dataset_id="b1f0a07e-e348-402c-a2b2-11f3e31181ce",
+        )
+        sut.last_status = "Completed"
+        sut.powerbi_url = "test"
+
+        # Act
+        with self.assertRaises(SpetlrException) as context:
+            sut._trigger_new_refresh()
+
+        # Assert
+        self.assertIn("Failed to trigger a refresh", str(context.exception))
+
+    def test_get_seconds_to_wait_first(self):
+        # Arrange
+        sut = PowerBi(PowerBiClient())
+        sut.last_duration = 0
+        elapsed = 5
+        expected_result = 15
+
+        # Act
+        result = sut._get_seconds_to_wait(elapsed)
+
+        # Assert
+        self.assertEqual(result, expected_result)
+
+    def test_get_seconds_to_wait_next(self):
+        # Arrange
+        sut = PowerBi(PowerBiClient())
+        sut.last_duration = 60 * 15
+        elapsed = 5
+        expected_result = 60 * 5
+
+        # Act
+        result = sut._get_seconds_to_wait(elapsed)
+
+        # Assert
+        self.assertEqual(result, expected_result)