Repository: rotki/rotki

Issue:
Include Maker Vaults in Net Worth balance
Maker Vaults are currently not included in your Net Worth balance. Suggest to treat vaults as another asset owned by your ethereum accounts so your net worth is updated accordingly 

The USD value = Collateral value - debt outstanding



Discussion:
Thank you for the issue @JakeBarclay and for suggesting this in the discord chat!

## Frontend

So @isidorosp and @kelsos this should be easy on the front-end I suppose.

How about we include it in the assets as @JakeBarclay said in the "balances per asset" or in a new Defi tab with USD value being `= Collateral value - debt outstanding`

The asset would be a vault with type the type of vault collateral. e.g. `ETH-A`, `USDC-A`, `USDC-B` etc.

## Backend

The balances should also be saved in the backend though. We should include this in the snapshot saving with a proper location. This may need some thinking for proper syncing with the frontend.

Resolving patch:
diff --git a/rotkehlchen/chain/ethereum/makerdao/vaults.py b/rotkehlchen/chain/ethereum/makerdao/vaults.py
--- a/rotkehlchen/chain/ethereum/makerdao/vaults.py
+++ b/rotkehlchen/chain/ethereum/makerdao/vaults.py
@@ -6,6 +6,7 @@
 from eth_utils.address import to_checksum_address
 from gevent.lock import Semaphore
 
+from rotkehlchen.accounting.structures import Balance
 from rotkehlchen.assets.asset import Asset
 from rotkehlchen.chain.ethereum.makerdao.common import (
     MAKERDAO_REQUERY_PERIOD,
@@ -38,6 +39,7 @@
 from rotkehlchen.db.dbhandler import DBHandler
 from rotkehlchen.fval import FVal
 from rotkehlchen.history.price import query_usd_price_or_use_default
+from rotkehlchen.inquirer import Inquirer
 from rotkehlchen.premium.premium import Premium
 from rotkehlchen.typing import ChecksumEthAddress, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
@@ -163,6 +165,19 @@ class MakerDAOVaultDetails(NamedTuple):
     events: List[VaultEvent]
 
 
+def get_vault_normalized_balance(vault: MakerDAOVault) -> Balance:
+    """Get the balance in the vault's collateral asset after deducting the generated debt"""
+    collateral_usd_price = Inquirer().find_usd_price(vault.collateral_asset)
+    dai_usd_price = Inquirer().find_usd_price(A_DAI)
+    debt_usd_value = dai_usd_price * vault.debt_value
+    normalized_usd_value = vault.collateral_usd_value - debt_usd_value
+
+    return Balance(
+        amount=normalized_usd_value / collateral_usd_price,
+        usd_value=normalized_usd_value,
+    )
+
+
 class MakerDAOVaults(MakerDAOCommon):
 
     def __init__(
@@ -603,13 +618,26 @@ def _get_vaults_of_address(
     def get_vaults(self) -> List[MakerDAOVault]:
         """Detects vaults the user has and returns basic info about each one
 
+        If the vaults have been queried in the past REQUERY_PERIOD
+        seconds then the old result is used.
+
         May raise:
         - RemoteError if etherscan is used and there is a problem with
         reaching it or with the returned result.
         - BlockchainQueryError if an ethereum node is used and the contract call
         queries fail for some reason
         """
+        now = ts_now()
+        if now - self.last_vault_mapping_query_ts < MAKERDAO_REQUERY_PERIOD:
+            prequeried_vaults = []
+            for _, vaults in self.vault_mappings.items():
+                prequeried_vaults.extend(vaults)
+
+            prequeried_vaults.sort(key=lambda vault: vault.identifier)
+            return prequeried_vaults
+
         with self.lock:
+            self.vault_mappings = defaultdict(list)
             proxy_mappings = self._get_accounts_having_maker_proxy()
             vaults = []
             for user_address, proxy in proxy_mappings.items():
@@ -622,27 +650,6 @@ def get_vaults(self) -> List[MakerDAOVault]:
             vaults.sort(key=lambda vault: vault.identifier)
         return vaults
 
-    def _get_vault_mappings(self) -> Dict[ChecksumEthAddress, List[MakerDAOVault]]:
-        """Returns a mapping of accounts to vault lists
-
-        If the mappings have been queried in the past REQUERY_PERIOD
-        seconds then the old result is used.
-
-        May raise:
-        - RemoteError if etherscan is used and there is a problem with
-        reaching it or with the returned result.
-        - BlockchainQueryError if an ethereum node is used and the contract call
-        queries fail for some reason
-        """
-        now = ts_now()
-        if now - self.last_vault_mapping_query_ts < MAKERDAO_REQUERY_PERIOD:
-            return self.vault_mappings
-
-        # reset vault mappings, and set them with get_vaults
-        self.vault_mappings = defaultdict(list)
-        self.get_vaults()
-        return self.vault_mappings
-
     def get_vault_details(self) -> List[MakerDAOVaultDetails]:
         """Queries vault details for the auto detected vaults of the user
 
@@ -664,19 +671,32 @@ def get_vault_details(self) -> List[MakerDAOVaultDetails]:
         self.vault_details = []
         proxy_mappings = self._get_accounts_having_maker_proxy()
         # Make sure that before querying vault details there has been a recent vaults call
-        vault_mappings = self._get_vault_mappings()
-        for address, vaults in vault_mappings.items():
-            proxy = proxy_mappings[address]
-            for vault in vaults:
-                vault_detail = self._query_vault_details(vault, proxy, vault.urn)
-                if vault_detail:
-                    self.vault_details.append(vault_detail)
+        vaults = self.get_vaults()
+        for vault in vaults:
+            proxy = proxy_mappings[vault.owner]
+            vault_detail = self._query_vault_details(vault, proxy, vault.urn)
+            if vault_detail:
+                self.vault_details.append(vault_detail)
 
         # Returns vault details sorted. Oldest identifier first
         self.vault_details.sort(key=lambda details: details.identifier)
         self.last_vault_details_query_ts = ts_now()
         return self.vault_details
 
+    def get_normalized_balances(self) -> Dict[Asset, Balance]:
+        """Return a mapping of asset to balance indicating all normalized balances in vaults
+
+        Normalized balance is defined as the balance of the collateral asset minus
+        that of the generated debt.
+        """
+        vaults = self.get_vaults()
+        balances: Dict[Asset, Balance] = defaultdict(Balance)
+        for vault in vaults:
+            normalized_balance = get_vault_normalized_balance(vault)
+            balances[vault.collateral_asset] += normalized_balance
+
+        return balances
+
     # -- Methods following the EthereumModule interface -- #
     def on_startup(self) -> None:
         super().on_startup()
diff --git a/rotkehlchen/chain/manager.py b/rotkehlchen/chain/manager.py
--- a/rotkehlchen/chain/manager.py
+++ b/rotkehlchen/chain/manager.py
@@ -996,3 +996,8 @@ def query_ethereum_balances(self) -> None:
 
         self.query_defi_balances()
         self.query_ethereum_tokens(self.owned_eth_tokens)
+        vaults_module = self.makerdao_vaults
+        if vaults_module is not None:
+            normalized_balances = vaults_module.get_normalized_balances()
+            for asset, normalized_vault_balance in normalized_balances.items():
+                self.totals[asset] += normalized_vault_balance


Test patch:
diff --git a/rotkehlchen/tests/api/test_aave.py b/rotkehlchen/tests/api/test_aave.py
--- a/rotkehlchen/tests/api/test_aave.py
+++ b/rotkehlchen/tests/api/test_aave.py
@@ -48,6 +48,7 @@ def test_query_aave_balances(rotkehlchen_api_server, ethereum_accounts):  # pyli
 @pytest.mark.parametrize('start_with_valid_premium', [True])
 @pytest.mark.parametrize('mocked_price_queries', [aave_mocked_historical_prices])
 @pytest.mark.parametrize('mocked_current_prices', [aave_mocked_current_prices])
+@pytest.mark.parametrize('default_mock_price_value', [FVal(1)])
 def test_query_aave_history(rotkehlchen_api_server, ethereum_accounts):  # pylint: disable=unused-argument  # noqa: E501
     """Check querying the aave histoy endpoint works. Uses real data."""
     response = requests.get(api_url_for(
@@ -64,7 +65,8 @@ def test_query_aave_history(rotkehlchen_api_server, ethereum_accounts):  # pylin
     assert FVal(total_earned['aDAI']['amount']) >= FVal('24.207179802347627414')
     assert FVal(total_earned['aDAI']['usd_value']) >= FVal('24.580592532348742989192')
     expected_events = process_result_list(expected_aave_test_events)
-    assert events == expected_events
+    assert len(events) >= 12
+    assert events[:12] == expected_events
 
 
 @pytest.mark.parametrize('ethereum_modules', [['aave']])
diff --git a/rotkehlchen/tests/unit/test_aave.py b/rotkehlchen/tests/unit/test_aave.py
--- a/rotkehlchen/tests/unit/test_aave.py
+++ b/rotkehlchen/tests/unit/test_aave.py
@@ -41,8 +41,9 @@ def test_get_lending_profit_for_address(aave, price_historian):  # pylint: disab
         given_to_block=10386830,  # test was written at this block
     )
 
-    assert history.events == expected_aave_test_events
+    assert history.events == expected_aave_test_events[:10]
     assert len(history.total_earned) == 1
     # comparison is greater than since interest is always accruing since writing the test
-    assert history.total_earned['aDAI'].amount >= FVal('24.207179802347627414')
-    assert history.total_earned['aDAI'].usd_value >= FVal('24.580592532348742989192')
+    # and 7 "should be" the principal balance at the given block
+    assert history.total_earned['aDAI'].amount >= FVal('7')
+    assert history.total_earned['aDAI'].usd_value >= FVal('7')
diff --git a/rotkehlchen/tests/unit/test_makerdao.py b/rotkehlchen/tests/unit/test_makerdao.py
--- a/rotkehlchen/tests/unit/test_makerdao.py
+++ b/rotkehlchen/tests/unit/test_makerdao.py
@@ -1,7 +1,12 @@
 import pytest
 from web3 import Web3
 
-from rotkehlchen.chain.ethereum.makerdao.vaults import MakerDAOVault, MakerDAOVaults
+from rotkehlchen.accounting.structures import Balance
+from rotkehlchen.chain.ethereum.makerdao.vaults import (
+    MakerDAOVault,
+    MakerDAOVaults,
+    get_vault_normalized_balance,
+)
 from rotkehlchen.constants.assets import A_ETH
 from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.fval import FVal
@@ -108,3 +113,26 @@ def test_get_vaults(makerdao_vaults, makerdao_test_data):
 
     for idx, vault in enumerate(vaults):
         assert_vaults_equal(vault, makerdao_test_data.vaults[idx])
+
+
+@pytest.mark.parametrize('mocked_current_prices', [{
+    'ETH': FVal('200'),
+    'DAI': FVal('1.01'),
+}])
+def test_get_vault_normalized_balance(inquirer):  # pylint: disable=unused-argument
+    vault = MakerDAOVault(
+        identifier=1,
+        collateral_type='ETH-A',
+        collateral_asset=A_ETH,
+        owner=make_ethereum_address(),
+        collateral_amount=FVal('100'),
+        collateral_usd_value=FVal('20000'),
+        debt_value=FVal('2000'),
+        collateralization_ratio='990%',
+        liquidation_ratio=FVal(1.5),
+        liquidation_price=FVal('50'),  # not calculated to be correct
+        urn=make_ethereum_address(),
+        stability_fee=ZERO,
+    )
+    expected_result = Balance(amount=FVal('89.9'), usd_value=FVal('17980'))
+    assert get_vault_normalized_balance(vault) == expected_result
diff --git a/rotkehlchen/tests/utils/aave.py b/rotkehlchen/tests/utils/aave.py
--- a/rotkehlchen/tests/utils/aave.py
+++ b/rotkehlchen/tests/utils/aave.py
@@ -14,6 +14,7 @@
             1588463542: FVal('1.009'),
             1588430911: FVal('1.009'),
             1592175763: FVal('1.013'),
+            1594502373: FVal('1.019'),
         },
     },
     'DAI': {
@@ -24,6 +25,7 @@
             1588463542: FVal('1.009'),
             1588430911: FVal('1.009'),
             1592175763: FVal('1.013'),
+            1594502373: FVal('1.019'),
         },
     },
 }
@@ -130,5 +132,25 @@
         block_number=10266740,
         timestamp=Timestamp(1592175763),
         tx_hash='0x90b818ba8d3b55f332b64f3df58bf37f33addcbfc1f27bd1ec6102ae4bf2d871',
+    ), AaveEvent(
+        event_type='deposit',
+        asset=A_DAI,
+        value=Balance(
+            amount=FVal('1939.840878392183347402'),
+            usd_value=FVal('1976.697855081634831002638'),
+        ),
+        block_number=10440633,
+        timestamp=Timestamp(1594502373),
+        tx_hash='0xc3a8978418afa1a4f139e9314ac787cacfbed79b1daa28e146bb0bf6fdf79a41',
+    ), AaveEvent(
+        event_type='interest',
+        asset=A_ADAI,
+        value=Balance(
+            amount=FVal('27.824509817913242961'),
+            usd_value=FVal('28.353175504453594577259'),
+        ),
+        block_number=10440633,
+        timestamp=Timestamp(1594502373),
+        tx_hash='0xc3a8978418afa1a4f139e9314ac787cacfbed79b1daa28e146bb0bf6fdf79a41',
     ),
 ]