Repository: softwareone-platform/ffc-finops-operations
Pull request #184: feat: add --dry-run flag to charge files generation command (MPT-9808)
Author: login: arturbalabanov
Created: 2025-05-12T15:27:17Z | Merged: 2025-05-13T11:58:41Z



Diff:
diff --git a/app/commands/generate_monthly_charges.py b/app/commands/generate_monthly_charges.py
index 2b542c77..c4ae9342 100644
--- a/app/commands/generate_monthly_charges.py
+++ b/app/commands/generate_monthly_charges.py
@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import pathlib
+import tempfile
 import zipfile
 from collections.abc import AsyncGenerator, Sequence
 from copy import copy
@@ -17,7 +18,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.blob_storage import upload_charges_file
-from app.conf import Settings, get_settings
+from app.conf import get_settings
 from app.currency import CurrencyConverter
 from app.db.base import session_factory
 from app.db.handlers import (
@@ -461,32 +462,49 @@ async def genenerate_monthly_charges(
     account: Account,
     currency_converter: CurrencyConverter,
     exports_dir: pathlib.Path,
+    dry_run: bool,
 ) -> None:
     today = datetime.now(UTC).date()
     loop = asyncio.get_event_loop()
 
     async with session.begin():
-        generated_charges_file = await fetch_existing_generated_charges_file(
-            session, account, currency
-        )
+        if dry_run:
+            logger.info("Dry run enabled, skipping fetching existing charges file")
+        else:
+            generated_charges_file = await fetch_existing_generated_charges_file(
+                session, account, currency
+            )
 
-        if generated_charges_file is not None:
-            return
+            if generated_charges_file is not None:
+                return
 
         generator = ChargesFileGenerator(account, currency, currency_converter, exports_dir)
         async for ds_exp in fetch_datasource_expenses(session, currency):
             await loop.run_in_executor(None, generator.add_datasource_expense, ds_exp)
 
-        if not generator.has_entries:
-            return
+        if dry_run:
+            logger.info("Dry run enabled, skipping creating charges file database record")
+
+            # When in dry_run mode, we don't have access to the the charges_file ID,
+            # so we should use a different filename
+            filename = f"{account.id}_{currency}_{today.strftime('%Y_%m')}.zip"
+        else:
+            if not generator.has_entries:
+                # Since we're not saving the results to an excel file in this case, we need to
+                # explicitly close the temporary file openpyxl created to avoid leaks
+                generator.worksheet.close()
+                return
 
-        charges_file_db_record = await get_or_create_draft_charges_file(
-            session, account, currency, today
-        )
+            charges_file_db_record = await get_or_create_draft_charges_file(
+                session, account, currency, today
+            )
+            filename = f"{charges_file_db_record.id}.zip"
 
-    zip_file_path = await loop.run_in_executor(
-        None, generator.save, f"{charges_file_db_record.id}.zip"
-    )
+    zip_file_path = await loop.run_in_executor(None, generator.make_archive, filename)
+
+    if dry_run:
+        logger.info("Dry run enabled, skipping upload to Azure Blob Storage")
+        return
 
     successful_upload = await upload_charges_file_to_azure(charges_file_db_record, zip_file_path)
 
@@ -503,9 +521,9 @@ async def genenerate_monthly_charges(
 
 async def main(
     exports_dir: pathlib.Path,
-    settings: Settings,
     currency: str | None = None,
     account_id: str | None = None,
+    dry_run: bool = False,
 ) -> None:
     async with session_factory() as session:
         async with session.begin():
@@ -536,15 +554,22 @@ async def main(
         for currency in unique_billing_currencies:
             for account in accounts:
                 await genenerate_monthly_charges(
-                    session, currency, account, currency_converter, exports_dir
+                    session, currency, account, currency_converter, exports_dir, dry_run
                 )
 
 
 def command(
     ctx: typer.Context,
     exports_dir: Annotated[
-        pathlib.Path, typer.Option("--exports-dir", help="Directory to export the charge files to")
-    ],
+        pathlib.Path | None,
+        typer.Option(
+            "--exports-dir",
+            help=(
+                "Directory to export the charge files to. "
+                "If not set the command will create a temporary directory"
+            ),
+        ),
+    ] = None,
     currency: Annotated[
         str | None,
         typer.Option(
@@ -561,6 +586,14 @@ def command(
             show_default=False,
         ),
     ] = None,
+    dry_run: Annotated[
+        bool,
+        typer.Option(
+            "--dry-run",
+            help="If set, only generate charge files but do not upload them to Azure Blob Storage",
+            show_default=False,
+        ),
+    ] = False,
 ) -> None:
     """
     Generate monthly charge files for the previous month and upload them to Azure Blob Storage.
@@ -570,15 +603,29 @@ def command(
     """
     logger.info("Starting command function")
 
-    if exports_dir.is_file():  # pragma: no cover
-        raise ValueError("The exports directory must be a directory, not a file.")
+    if exports_dir is None:
+        exports_dir = pathlib.Path(tempfile.TemporaryDirectory().name)
+        logger.info(
+            "No exports directory specified, using a temporary directory: %s",
+            str(exports_dir.resolve()),
+        )
+    else:
+        if exports_dir.is_file():  # pragma: no cover
+            raise ValueError("The exports directory must be a directory, not a file.")
 
-    if not exports_dir.exists():  # pragma: no cover
-        logger.info("Exports directory %s does not exist, creating it", str(exports_dir.resolve()))
-        exports_dir.mkdir(parents=True)
+        if not exports_dir.exists():  # pragma: no cover
+            logger.info(
+                "Exports directory %s does not exist, creating it", str(exports_dir.resolve())
+            )
+            exports_dir.mkdir(parents=True)
 
     asyncio.run(
-        main(exports_dir=exports_dir, currency=currency, account_id=account_id, settings=ctx.obj)
+        main(
+            exports_dir=exports_dir,
+            currency=currency,
+            account_id=account_id,
+            dry_run=dry_run,
+        )
     )
 
     logger.info("Completed command function")
diff --git a/tests/commands/snapshots/test_generate_monthtly_charges/test_generate_monthly_charges_dry_run/charges.csv b/tests/commands/snapshots/test_generate_monthtly_charges/test_generate_monthly_charges_dry_run/charges.csv
new file mode 100644
index 00000000..4cb8c8de
--- /dev/null
+++ b/tests/commands/snapshots/test_generate_monthtly_charges/test_generate_monthly_charges_dry_run/charges.csv
@@ -0,0 +1,5 @@
+Entry ID,Subscription Search Criteria,Subscription Search Value,Item Search Criteria,Item Search Value,Usage Start Time,Usage End Time,Quantity,Purchase Price,Total Purchase Price,External Reference,Vendor Description 1,Vendor Description 2,Vendor Reference
+1,subscription.externalIds.vendor,FORG-7330-1645-9608,item.externalIds.vendor,FIN-0001-P1M,1-Mar-2025,31-Mar-2025,1,0.56,0.56,06cb0fb3-9a1d-4644-815e-f6d13b8faa18,AWS Account,11111111111,
+2,subscription.externalIds.vendor,FORG-7330-1645-9608,item.externalIds.vendor,FIN-0001-P1M,1-Mar-2025,31-Mar-2025,1,-0.56,-0.56,06cb0fb3-9a1d-4644-815e-f6d13b8faa18,AWS Account,11111111111,
+3,subscription.externalIds.vendor,FORG-7330-1645-9608,item.externalIds.vendor,FIN-0001-P1M,1-Mar-2025,31-Mar-2025,1,0.65,0.65,06cb0fb3-9a1d-4644-815e-f6d13b8faa18,GCP Account,22222222222,
+4,subscription.externalIds.vendor,FORG-7330-1645-9608,item.externalIds.vendor,FIN-0001-P1M,1-Mar-2025,31-Mar-2025,1,-0.65,-0.65,06cb0fb3-9a1d-4644-815e-f6d13b8faa18,GCP Account,22222222222,
diff --git a/tests/commands/test_generate_monthtly_charges.py b/tests/commands/test_generate_monthtly_charges.py
index 7011277a..2b9f533d 100644
--- a/tests/commands/test_generate_monthtly_charges.py
+++ b/tests/commands/test_generate_monthtly_charges.py
@@ -1,6 +1,7 @@
 import io
 import logging
 import pathlib
+import tempfile
 import zipfile
 from contextlib import nullcontext
 from datetime import UTC, date, datetime, timedelta
@@ -463,10 +464,16 @@ def test_charges_file_generator_append_row(
     currency_converter: CurrencyConverter,
     operations_account: Account,
     test_settings: Settings,
+    request: pytest.FixtureRequest,
     tmp_path: pathlib.Path,
 ):
     generator = ChargesFileGenerator(operations_account, "USD", currency_converter, tmp_path)
 
+    # Since we're not saving to a file in this test, we need to close the temporary file
+    # openpyxl creates explicitly when we're done with writing to it to avoid a warning
+    # in the pytest output.
+    request.addfinalizer(generator.worksheet.close)
+
     assert not generator.has_entries
     assert generator.total_rows == 0
     assert generator.running_total == Decimal("0.00")
@@ -532,7 +539,6 @@ async def test_charges_file_generator_add_datasource_expense(
     usd_org_billed_in_usd_expenses: Organization,
     usd_org_billed_in_eur_expenses: Organization,
     eur_org_billed_in_gbp_expenses: Organization,
-    test_settings: Settings,
     db_session: AsyncSession,
     request: pytest.FixtureRequest,
     tmp_path: pathlib.Path,
@@ -1039,7 +1045,7 @@ async def test_full_run(
     # (both new records and updating existing ones), so that we can filter the affected records
     # bellow
     with caplog.at_level(logging.INFO), time_machine.travel("2025-04-10T11:00:00Z", tick=False):
-        await generate_monthly_charges_main(exports_dir=tmp_path, settings=test_settings)
+        await generate_monthly_charges_main(exports_dir=tmp_path)
 
     await db_session.refresh(generated_charges_file)
     await db_session.refresh(processed_charges_file)
@@ -1205,7 +1211,6 @@ async def test_command_filters(
     affiliate_account: Account,
     another_affiliate_account: Account,
     operations_account: Account,
-    test_settings: Settings,
     db_session: AsyncSession,
     caplog: pytest.LogCaptureFixture,
     tmp_path: pathlib.Path,
@@ -1234,7 +1239,6 @@ async def test_command_filters(
     with pytest.raises(exception.__class__, match=str(exception)) if exception else nullcontext():
         await generate_monthly_charges_main(
             exports_dir=tmp_path,
-            settings=test_settings,
             account_id=account_id,
             currency=currency,
         )
@@ -1250,6 +1254,52 @@ async def test_command_filters(
     assert sorted(actual_calls_args) == sorted(expected_calls_args)
 
 
+@pytest.mark.fixed_random_seed
+@time_machine.travel("2025-04-10T10:00:00Z", tick=False)
+async def test_generate_monthly_charges_dry_run(
+    exchange_rates_factory: ModelFactory[ExchangeRates],
+    usd_org_billed_in_eur_expenses: Organization,
+    operations_account: Account,
+    db_session: AsyncSession,
+    tmp_path: pathlib.Path,
+    mocker: MockerFixture,
+    snapshot: Snapshot,
+    caplog: pytest.LogCaptureFixture,
+):
+    mock_upload_to_azure = mocker.patch(
+        "app.commands.generate_monthly_charges.upload_charges_file_to_azure"
+    )
+    charges_file_handler = ChargesFileHandler(db_session)
+    assert await charges_file_handler.count() == 0
+
+    await exchange_rates_factory(base_currency="EUR")
+    await exchange_rates_factory(base_currency="USD")
+    await exchange_rates_factory(base_currency="GBP")
+
+    with caplog.at_level(logging.INFO):
+        await generate_monthly_charges_main(
+            exports_dir=tmp_path,
+            account_id=operations_account.id,
+            currency="EUR",
+            dry_run=True,
+        )
+
+    assert "Dry run enabled, skipping upload to Azure Blob Storage" in caplog.text
+    assert "Dry run enabled, skipping creating charges file database record" in caplog.text
+    assert "Dry run enabled, skipping fetching existing charges file" in caplog.text
+
+    assert not mock_upload_to_azure.called
+    assert await charges_file_handler.count() == 0
+    expected_generated_file_path = tmp_path / f"{operations_account.id}_EUR_2025_04.zip"
+    assert expected_generated_file_path.exists()
+
+    with zipfile.ZipFile(expected_generated_file_path, "r") as archive:
+        assert sorted(archive.namelist()) == ["charges.xlsx", "exchange_rates_USD.json"]
+
+        excel_file_path = pathlib.Path(archive.extract("charges.xlsx", tmp_path))
+        assert_generated_file_matches_snapshot(excel_file_path, snapshot)
+
+
 def test_cli_command(mocker: MockerFixture, test_settings: Settings, tmp_path: pathlib.Path):
     mocker.patch("app.cli.get_settings", return_value=test_settings)
     mock_command_coro = mocker.MagicMock()
@@ -1266,5 +1316,24 @@ def test_cli_command(mocker: MockerFixture, test_settings: Settings, tmp_path: p
         exports_dir=tmp_path,
         currency=None,
         account_id=None,
-        settings=test_settings,
+        dry_run=False,
     )
+
+
+def test_cli_command_default_exports_dir(mocker: MockerFixture, test_settings: Settings):
+    mocker.patch("app.cli.get_settings", return_value=test_settings)
+    mock_command_coro = mocker.MagicMock()
+    mock_command = mocker.MagicMock(return_value=mock_command_coro)
+
+    mocker.patch("app.commands.generate_monthly_charges.main", mock_command)
+    mock_run = mocker.patch("app.commands.generate_monthly_charges.asyncio.run")
+    runner = CliRunner()
+
+    result = runner.invoke(app, ["generate-monthly-charges"])
+    assert result.exit_code == 0
+    mock_run.assert_called_once_with(mock_command_coro)
+
+    calls = mock_command.call_args_list
+    assert len(calls) == 1
+    exports_dir = calls[0].kwargs.get("exports_dir")
+    assert exports_dir.is_relative_to(tempfile.gettempdir())


Reviews:
user: login: ffaraone
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-12T15:58:51Z
user: login: arturbalabanov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T11:23:11Z
user: login: arturbalabanov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T11:23:33Z

Review comments:
diff_hunk: @@ -506,6 +523,7 @@ async def main(
     settings: Settings,
path: app/commands/generate_monthly_charges.py
user: login: ffaraone
body: Looks like settings here are not used at all, could we remove the function argument ?
created_at: 2025-05-12T15:55:39Z
author_association: COLLABORATOR
diff_hunk: @@ -578,7 +604,13 @@ def command(
         exports_dir.mkdir(parents=True)
 
     asyncio.run(
-        main(exports_dir=exports_dir, currency=currency, account_id=account_id, settings=ctx.obj)
+        main(
+            exports_dir=exports_dir,
path: app/commands/generate_monthly_charges.py
user: login: ffaraone
body: Could we default the exports dir to some /tmp/whatever so we can have the options not required ?
created_at: 2025-05-12T15:58:00Z
author_association: COLLABORATOR
diff_hunk: @@ -506,6 +523,7 @@ async def main(
     settings: Settings,
path: app/commands/generate_monthly_charges.py
user: login: arturbalabanov
body: Removed
created_at: 2025-05-13T11:23:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -578,7 +604,13 @@ def command(
         exports_dir.mkdir(parents=True)
 
     asyncio.run(
-        main(exports_dir=exports_dir, currency=currency, account_id=account_id, settings=ctx.obj)
+        main(
+            exports_dir=exports_dir,
path: app/commands/generate_monthly_charges.py
user: login: arturbalabanov
body: Good idea, implemented this :)
created_at: 2025-05-13T11:23:33Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-05-13T11:25:55Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=softwareone-platform_ffc-finops-operations&pullRequest=184) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [1 New issue](https://sonarcloud.io/project/issues?id=softwareone-platform_ffc-finops-operations&pullRequest=184&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=softwareone-platform_ffc-finops-operations&pullRequest=184&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=softwareone-platform_ffc-finops-operations&pullRequest=184&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [100.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=softwareone-platform_ffc-finops-operations&pullRequest=184&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=softwareone-platform_ffc-finops-operations&pullRequest=184&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=softwareone-platform_ffc-finops-operations&pullRequest=184)


author_association: NONE