Repository: rotki/rotki

Issue:
Average cost basis method - all transactions unrealistic loss
## Problem Definition

This was also reported by @a7e8 in discord.

Running a PnL report for a random address (say `0x72c3d8b1F8B34B075C18498B66CaEabC11045F79`) from etherscan seem to report an unrealistically big loss for each transaction.

#### FiFo

![2023-02-05_01-09_1](https://user-images.githubusercontent.com/1658405/216794954-2359b416-5fc1-43e2-9aee-0f1c48b0f39c.png)

#### Average cost basis

![2023-02-05_01-10](https://user-images.githubusercontent.com/1658405/216794962-ed91da82-d73a-464c-aa94-bfd71be3168f.png)


## Task

- [ ] Figure out what exactly is wrong and fix it.
- [ ] Also test ACB more and add more test cases.
- [x] Also make sure to remind frontend to properly rename it to Average cost basis. (done at https://github.com/rotki/rotki/pull/5564)

### System Description


Operating system: Any
Rotki version: 1.27.0



Discussion:

Resolving patch:
diff --git a/rotkehlchen/accounting/cost_basis/base.py b/rotkehlchen/accounting/cost_basis/base.py
--- a/rotkehlchen/accounting/cost_basis/base.py
+++ b/rotkehlchen/accounting/cost_basis/base.py
@@ -205,7 +205,8 @@ def calculate_spend_cost_basis(
                 at_taxfree_period = acquisition_event.timestamp + settings.taxfree_after_period < timestamp  # noqa: E501
 
             if remaining_sold_amount < acquisition_event.remaining_amount:
-                acquisition_cost = acquisition_event.rate * remaining_sold_amount if average_cost_basis is None else average_cost_basis  # noqa: E501
+                acquisition_rate = acquisition_event.rate if average_cost_basis is None else average_cost_basis  # noqa: E501
+                acquisition_cost = acquisition_rate * remaining_sold_amount
 
                 taxable = True
                 if at_taxfree_period:
@@ -237,7 +238,8 @@ def calculate_spend_cost_basis(
                 break
 
             remaining_sold_amount -= acquisition_event.remaining_amount
-            acquisition_cost = acquisition_event.rate * acquisition_event.remaining_amount if average_cost_basis is None else average_cost_basis  # noqa: E501
+            acquisition_rate = acquisition_event.rate if average_cost_basis is None else average_cost_basis  # noqa: E501
+            acquisition_cost = acquisition_rate * acquisition_event.remaining_amount
             taxable = True
             if at_taxfree_period:
                 taxfree_amount += acquisition_event.remaining_amount
@@ -340,15 +342,24 @@ def add_acquisition(self, acquisition: AssetAcquisitionEvent) -> None:
 class AverageCostBasisMethod(BaseCostBasisMethod):
     """
     Accounting in Average Cost Basis(ACB) method.
-    https://www.investopedia.com/terms/a/averagecostbasismethod.asp
-    """
+
+    This accounting method is used for several jurisdictions. The examples are:
+        1. For Canada: https://www.canada.ca/content/dam/cra-arc/formspubs/pub/t4037/t4037-22e.pdf, page 23
+        2. For UK: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/877369/HS284_Example_3_2020.pdf
+
+    Also an overall explanation of the method can be found here:
+        https://www.investopedia.com/terms/a/averagecostbasismethod.asp
+
+    For more details and explanations go here:
+        https://github.com/rotki/rotki/issues/5561#issuecomment-1423338938
+    """  # noqa: E501
     def __init__(self) -> None:
         super().__init__()
         self._count = ZERO
         # keeps track of the amount of the asset remaining after every acquisition or spend
-        self.remaining_amount = ZERO
-        # the average cost basis of last event(buy/sell) that occurred
-        self.current_average_cost_basis = ZERO
+        self.current_amount = ZERO
+        # the current total cost basis of the asset
+        self.current_total_acb = ZERO
 
     def add_acquisition(self, acquisition: AssetAcquisitionEvent) -> None:
         """
@@ -364,13 +375,18 @@ def add_acquisition(self, acquisition: AssetAcquisitionEvent) -> None:
             self._acquisitions_heap,
             AssetAcquisitionHeapElement(self._count, acquisition),
         )
-        self.current_average_cost_basis += (acquisition.rate * acquisition.amount)
-        self.remaining_amount += acquisition.amount
+        self.current_total_acb += acquisition.amount * acquisition.rate
+        self.current_amount += acquisition.amount
         self._count += 1
 
     def consume_result(self, used_amount: FVal) -> None:
-        """Same as its parent function but also deducts `used_amount` from `remaining_amount`."""
-        self.remaining_amount -= used_amount
+        """
+        Same as its parent function but also deducts `used_amount` from `current_amount`.
+        `current_amount` is guaranteed to be greater than zero since `consume_result` is
+        supposed to be called under `processing_iterator`.
+        """
+        self.current_total_acb *= (self.current_amount - used_amount) / self.current_amount  # noqa: E501
+        self.current_amount -= used_amount
         super().consume_result(used_amount)
 
     def calculate_spend_cost_basis(
@@ -384,31 +400,25 @@ def calculate_spend_cost_basis(
             timestamp_to_date: Callable[[Timestamp], str],
             average_cost_basis: Optional[FVal] = None,  # pylint: disable=unused-argument
     ) -> 'CostBasisInfo':
-        """
-        Calculates the cost basis of the spend using the average cost basis method.
-
-        Formula is taken from:
-        https://www.adjustedcostbase.ca/blog/how-to-calculate-adjusted-cost-base-acb-and-capital-gains/
-        """
-        if self.remaining_amount == ZERO:
+        """Calculates the cost basis of the spend using the average cost basis method."""
+        if self.current_amount == ZERO:
             missing_acquisitions.append(
                 MissingAcquisition(
                     asset=spending_asset,
                     time=timestamp,
-                    found_amount=self.remaining_amount,
+                    found_amount=self.current_amount,
                     missing_amount=spending_amount,
                 ),
             )
 
             return CostBasisInfo(
-                taxable_amount=ZERO,
+                taxable_amount=spending_amount,
                 taxable_bought_cost=ZERO,
                 taxfree_bought_cost=ZERO,
                 matched_acquisitions=[],
                 is_complete=False,
             )
 
-        self.current_average_cost_basis *= ((self.remaining_amount - spending_amount) / self.remaining_amount)  # noqa: E501
         return super().calculate_spend_cost_basis(
             spending_amount=spending_amount,
             spending_asset=spending_asset,
@@ -417,7 +427,11 @@ def calculate_spend_cost_basis(
             used_acquisitions=used_acquisitions,
             settings=settings,
             timestamp_to_date=timestamp_to_date,
-            average_cost_basis=self.current_average_cost_basis,
+            # Important note: calculation of the average cost basis of the current event has to
+            # happen before calling `consume_result`. For correct results `current_total_acb` and
+            # `current_amount` have to be used before applying the effect of the event that is
+            # being processed.
+            average_cost_basis=self.current_total_acb / self.current_amount,
         )
 
 


Test patch:
diff --git a/rotkehlchen/tests/unit/test_cost_basis.py b/rotkehlchen/tests/unit/test_cost_basis.py
--- a/rotkehlchen/tests/unit/test_cost_basis.py
+++ b/rotkehlchen/tests/unit/test_cost_basis.py
@@ -17,6 +17,43 @@
 from rotkehlchen.types import CostBasisMethod, Location, Timestamp, make_evm_tx_hash
 
 
+def add_acquisition(pot, amount, asset=A_ETH, price=ONE, taxable=False):
+    """
+    Util function to add an acquisition to an accounting pot.
+    Timestamp doesn't matter here since we provide `given_price`, but has to be big enough so that
+    `handle_prefork_acquisitions` is not called.
+    """
+    pot.add_acquisition(
+        event_type=AccountingEventType.TRANSACTION_EVENT,
+        notes='Test',
+        location=Location.BLOCKCHAIN,
+        timestamp=Timestamp(1675483017),
+        asset=asset,
+        amount=amount,
+        taxable=taxable,
+        given_price=price,
+    )
+
+
+def add_spend(pot, amount, asset=A_ETH, price=ONE, taxable=True):
+    """
+    Util function to add a spend event to an accounting pot.
+    Timestamp doesn't matter here since we provide `given_price`, but has to be big enough so that
+    `handle_prefork_acquisitions` is not called.
+    """
+    pot.add_spend(
+        event_type=AccountingEventType.TRANSACTION_EVENT,
+        notes='Test',
+        location=Location.BLOCKCHAIN,
+        timestamp=Timestamp(1675483017),
+        asset=asset,
+        amount=amount,
+        taxable=taxable,
+        given_price=price,
+        count_entire_amount_spend=False,
+    )
+
+
 @pytest.mark.parametrize('accounting_initialize_parameters', [True])
 def test_calculate_spend_cost_basis_after_year(accountant):
     asset = A_BTC
@@ -786,136 +823,104 @@ def test_missing_acquisitions(accountant):
 
 
 def test_accounting_average_cost_basis(accountant):
-    """
-    Test data is gotten from:
-    https://www.adjustedcostbase.ca/blog/how-to-calculate-adjusted-cost-base-acb-and-capital-gains/
-    """
-    asset = A_ETH
-    cost_basis = accountant.pots[0].cost_basis
-    cost_basis.reset(DBSettings(cost_basis_method=CostBasisMethod.ACB))
-    asset_events = cost_basis.get_events(asset)
-
-    # check that the average cost basis is calculated properly
-    # whenever a spend happens after an acquisition
-    assert asset_events.acquisitions_manager.remaining_amount == ZERO
-    event1 = AssetAcquisitionEvent(
-        amount=FVal(100),
-        timestamp=1,
-        rate=FVal(50),
-        index=1,
-    )
-    asset_events.acquisitions_manager.add_acquisition(event1)
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(100)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(5000)
-    cost_basis_result = asset_events.acquisitions_manager.calculate_spend_cost_basis(
-        spending_amount=FVal(50),
-        spending_asset=asset,
-        timestamp=0,
-        missing_acquisitions=cost_basis.missing_acquisitions,
-        used_acquisitions=asset_events.used_acquisitions,
-        settings=cost_basis.settings,
-        timestamp_to_date=cost_basis.timestamp_to_date,
-        average_cost_basis=asset_events.acquisitions_manager.current_average_cost_basis,
-    )
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(50)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(2500)
-    assert cost_basis_result.taxable_bought_cost == FVal(2500)
-    assert cost_basis_result.is_complete is True
-
-    # repeat the above process again to see that it works as expected
-    # and the average cost basis calculated is correct.
-    asset_events.acquisitions_manager.add_acquisition(AssetAcquisitionEvent(
-        amount=FVal(50),
-        timestamp=3,
-        rate=FVal(130),
-        index=3,
-    ))
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(100)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(9000)
-    cost_basis_result = asset_events.acquisitions_manager.calculate_spend_cost_basis(
-        spending_amount=FVal(40),
-        spending_asset=asset,
-        timestamp=0,
-        missing_acquisitions=cost_basis.missing_acquisitions,
-        used_acquisitions=asset_events.used_acquisitions,
-        settings=cost_basis.settings,
-        timestamp_to_date=cost_basis.timestamp_to_date,
-        average_cost_basis=asset_events.acquisitions_manager.current_average_cost_basis,
-    )
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(60)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(5400)
-    assert cost_basis_result.taxable_bought_cost == FVal(5400)
-    assert cost_basis_result.is_complete is True
-
-    # reset the cost basis and
-    # now see that having two consecutive acquisitions followed by a spend
-    # and see that the average cost basis calculated is correct.
+    """Test various scenarios in average cost basis calculation"""
+    pot = accountant.pots[0]
+    events = pot.processed_events
+    cost_basis = pot.cost_basis
     cost_basis.reset(DBSettings(cost_basis_method=CostBasisMethod.ACB))
-    asset_events = cost_basis.get_events(asset)
-    assert asset_events.acquisitions_manager.remaining_amount == ZERO
-    event3 = AssetAcquisitionEvent(
-        amount=FVal(1),
-        timestamp=3,
-        rate=FVal(100),
-        index=3,
-    )
-    event4 = AssetAcquisitionEvent(
-        amount=FVal(1),
-        timestamp=4,
-        rate=FVal(200),
-        index=4,
-    )
-    asset_events.acquisitions_manager.add_acquisition(event3)
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(1)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(100)
-    asset_events.acquisitions_manager.add_acquisition(event4)
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(2)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(300)
-    cost_basis_result = asset_events.acquisitions_manager.calculate_spend_cost_basis(
-        spending_amount=FVal(0.5),
-        spending_asset=asset,
-        timestamp=0,
-        missing_acquisitions=cost_basis.missing_acquisitions,
-        used_acquisitions=asset_events.used_acquisitions,
-        settings=cost_basis.settings,
-        timestamp_to_date=cost_basis.timestamp_to_date,
-        average_cost_basis=asset_events.acquisitions_manager.current_average_cost_basis,
-    )
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(1.5)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(225)
-    assert cost_basis_result.is_complete is True
-    assert cost_basis_result.taxable_bought_cost == FVal(225)
-    asset_events.acquisitions_manager.add_acquisition(AssetAcquisitionEvent(
-        amount=FVal(0.5),
-        timestamp=5,
-        rate=FVal(500),
-        index=5,
-    ))
-    assert asset_events.acquisitions_manager.remaining_amount == FVal(2)
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(475)
-
-    # see that using more than the available acquisitions adds a MissingAcquisition
-    assert asset_events.acquisitions_manager.calculate_spend_cost_basis(
-        spending_amount=FVal(3.5),
-        spending_asset=asset,
-        timestamp=0,
-        missing_acquisitions=cost_basis.missing_acquisitions,
-        used_acquisitions=asset_events.used_acquisitions,
-        settings=cost_basis.settings,
-        timestamp_to_date=cost_basis.timestamp_to_date,
-        average_cost_basis=asset_events.acquisitions_manager.current_average_cost_basis,
-    ).is_complete is False
-    assert asset_events.acquisitions_manager.remaining_amount == ZERO
-    # it is negative due to the missing acquisition.
-    assert asset_events.acquisitions_manager.current_average_cost_basis == FVal(-356.25)
-    assert cost_basis.missing_acquisitions == [
-        MissingAcquisition(
-            asset=asset,
-            time=0,
-            found_amount=FVal(2),
-            missing_amount=FVal(1.5),
-        ),
-    ]
+    manager = cost_basis.get_events(A_ETH).acquisitions_manager
+
+    # Step 1. Add an acquisition
+    add_acquisition(pot, amount=FVal(2), price=FVal(10))  # Buy 2 ETH for $10  total acb: $20
+    assert events[0].pnl.taxable == ZERO  # No profit for acquisitions
+    current_total_acb = events[0].price * events[0].free_amount
+    current_amount = events[0].free_amount  # 2
+    assert manager.current_total_acb == current_total_acb == FVal(20)
+    assert manager.current_amount == current_amount == FVal(2)
+
+    # Step 2. Add a spend with positive pnl
+    add_spend(pot, amount=FVal(1), price=FVal(15))  # Sell 1 ETH for $15  pnl: 1 * (15 - 10) = $5
+    assert events[1].pnl.taxable == events[1].taxable_amount * (events[1].price - current_total_acb / current_amount) == FVal(5)  # noqa: E501
+    current_total_acb *= (current_amount - events[1].taxable_amount) / current_amount  # total acb: 20 * (2 - 1) / 2 = $10  # noqa: E501
+    current_amount -= events[1].taxable_amount  # 1
+    assert manager.current_total_acb == current_total_acb == FVal(10)
+    assert manager.current_amount == current_amount == FVal(1)
+
+    # Step 3. Add another acquisition
+    add_acquisition(pot, amount=FVal(1), price=FVal(30))  # Buy 1 ETH for $30  total acb: 10 + 1 * 30 = $40  # noqa: E501
+    assert events[2].pnl.taxable == ZERO  # No profit for acquisitions
+    current_total_acb += events[2].price * events[2].free_amount
+    current_amount += events[2].free_amount  # 2
+    assert manager.current_total_acb == current_total_acb == FVal(40)
+    assert manager.current_amount == current_amount == FVal(2)
+
+    # Step 4. Add a spend with negative pnl
+    add_spend(pot, amount=FVal(1.5), price=FVal(10))  # Sell 1.5 ETH for $40  pnl: 1.5 * (10 - 20) = -$15  # noqa: E501
+    assert events[3].pnl.taxable == events[3].taxable_amount * (events[3].price - current_total_acb / current_amount) == FVal(-15)  # noqa: E501
+    current_total_acb *= (current_amount - events[3].taxable_amount) / current_amount  # total acb: 40 * (2 - 1.5) / 2 = $10  # noqa: E501
+    current_amount -= events[3].taxable_amount  # 0.5
+    assert manager.current_total_acb == current_total_acb == FVal(10)
+    assert manager.current_amount == current_amount == FVal(0.5)
+
+    # Step 5. Add another acquisition
+    add_acquisition(pot, amount=FVal(3), price=FVal(20))  # Buy 3 ETH for $20  total acb: 10 + 3 * 20 = $70  # noqa: E501
+    assert events[4].pnl.taxable == ZERO  # No profit for acquisitions
+    current_total_acb += events[4].price * events[4].free_amount
+    current_amount += events[4].free_amount  # 3.5
+    assert manager.current_total_acb == current_total_acb == FVal(70)
+    assert manager.current_amount == current_amount == FVal(3.5)
+
+    # Step 6. Add a spend after a sequence of acquisitions and spends
+    add_spend(pot, amount=FVal(2), price=FVal(30))  # Sell 2 ETH for $30  pnl: 2 * (30 - 20) = $20
+    assert events[5].pnl.taxable == events[5].taxable_amount * (events[5].price - current_total_acb / current_amount) == FVal(20)  # noqa: E501
+    current_total_acb *= (current_amount - events[5].taxable_amount) / current_amount  # total acb: 70 * (3.5 - 2) / 3.5 = $30 # noqa: E501
+    current_amount -= events[5].taxable_amount  # 1.5
+    assert manager.current_total_acb == current_total_acb == FVal(30)
+    assert manager.current_amount == current_amount == FVal(1.5)
+
+    # Step 7. Check that spending up the entire remaining amount works.
+    add_spend(pot, amount=FVal(1.5), price=FVal(10))  # Sell 1.5 ETH for $10 pnl: 1.5*10 - (30/1.5)*1.5 = -$15 # noqa: E501
+    assert events[6].pnl.taxable == events[6].taxable_amount * (events[6].price - current_total_acb / current_amount) == FVal(-15)  # noqa: E501
+    current_total_acb *= (current_amount - events[6].taxable_amount) / current_amount  # total acb: 30 * (1.5 - 1.5) / 1.5 = $0  # noqa: E501
+    current_amount -= events[6].taxable_amount  # 0
+    assert manager.current_total_acb == current_total_acb == ZERO
+    assert manager.current_amount == current_amount == ZERO
+
+    # Step 8. Add one more acquisition
+    add_acquisition(pot, amount=FVal(1), price=FVal(10))  # Buy 1 ETH for $10  total acb: 0 + 1 * 10 = $10  # noqa: E501
+    assert events[7].pnl.taxable == ZERO  # No profit for acquisitions
+    current_total_acb += events[7].price * events[7].free_amount
+    current_amount += events[7].free_amount  # 1
+    assert manager.current_total_acb == current_total_acb == FVal(10)
+    assert manager.current_amount == current_amount == FVal(1)
+
+    # Step 9. Check that negative pnl is correctly handled
+    add_spend(pot, amount=FVal(0.5), price=FVal(5))  # Sell 0.5 ETH for $5  pnl: 0.5 * (5 - 10) = -$2.5  # noqa: E501
+    assert events[8].pnl.taxable == events[8].taxable_amount * (events[8].price - current_total_acb / current_amount) == FVal(-2.5)  # noqa: E501
+    current_total_acb *= (current_amount - events[8].taxable_amount) / current_amount  # total acb: 10 * (1 - 0.5) / 1 = $5  # noqa: E501
+    current_amount -= events[8].taxable_amount  # 0.5
+    assert manager.current_total_acb == current_total_acb == FVal(5)
+    assert manager.current_amount == current_amount == FVal(0.5)
+
+    # Step 10. Try to spend more than the remaining amount
+    add_spend(pot, amount=FVal(0.6), price=FVal(5))  # Sell 0.6 ETH for $5  pnl: 0.5 * (5 - 10) + (0.1 * 5 <-- amount that has no matching acquisition) = -$2  # noqa: E501
+    assert len(cost_basis.missing_acquisitions) == 1
+    missing_acquisition = cost_basis.missing_acquisitions[0]
+    assert missing_acquisition.found_amount == current_amount == FVal(0.5)
+    assert missing_acquisition.missing_amount == events[9].taxable_amount - current_amount == FVal(0.1)  # 0.6 - 0.5  # noqa: E501
+    assert events[9].pnl.taxable == missing_acquisition.found_amount * (events[9].price - current_total_acb / current_amount) + missing_acquisition.missing_amount * events[9].price == FVal(-2)  # noqa: E501
+    assert manager.current_total_acb == ZERO
+    assert manager.current_amount == ZERO
+
+    # Step 11. Try to spend when remaining amount is zero
+    add_spend(pot, amount=FVal(0.5), price=FVal(5))  # Sell 0.5 ETH for $5  pnl: 0.5 * 5 = $2.5  # noqa: E501
+    assert len(cost_basis.missing_acquisitions) == 2
+    missing_acquisition = cost_basis.missing_acquisitions[1]
+    assert missing_acquisition.found_amount == ZERO
+    assert missing_acquisition.missing_amount == events[10].taxable_amount == FVal(0.5)
+    assert events[10].pnl.taxable == missing_acquisition.missing_amount * events[10].price == FVal(2.5)  # noqa: E501
+    assert manager.current_total_acb == ZERO
+    assert manager.current_amount == ZERO
 
 
 @pytest.mark.parametrize('mocked_price_queries', [{