Repository: rotki/rotki

Issue:
Introduce per protocol setting for which accounts to query for historical data
## Abstract

Introduce a setting per protocol that will specify which accounts to query for that protocol's historical data.

## Motivation

Querying protocols for historical data is expensive and when a user wants to query a protocol but has lots of accounts the query can take a long time.

Along with https://github.com/rotki/rotki/issues/1084 this can really improve the speed of historical queries for protocols.

## Specification

- [ ] Add a new setting per protocol that will be a list of accounts to query. By default all accounts are queried for each protocol. Allow customization of this setting from the frontend.
- [ ] For each protocol in the backend query only the selected addresses.


Discussion:
Reopening this issue as it still needs the frontend work to be done. @kelsos we discussed how this would look like.

(1) a setting entry somewhere
(2) first time you enter the DeFi page a wizard like thingy asking you which addresses you use for which protocol

Resolving patch:
diff --git a/rotkehlchen/api/v1/encoding.py b/rotkehlchen/api/v1/encoding.py
--- a/rotkehlchen/api/v1/encoding.py
+++ b/rotkehlchen/api/v1/encoding.py
@@ -700,6 +700,7 @@ class ModifiableSettingsSchema(Schema):
     date_display_format = fields.String(missing=None)
     kraken_account_type = KrakenAccountTypeField(missing=None)
     active_modules = fields.List(fields.String(), missing=None)
+    frontend_settings = fields.String(missing=None)
 
     @validates_schema  # type: ignore
     def validate_settings_schema(  # pylint: disable=no-self-use
@@ -736,6 +737,7 @@ def transform_data(  # pylint: disable=no-self-use
             submit_usage_analytics=data['submit_usage_analytics'],
             kraken_account_type=data['kraken_account_type'],
             active_modules=data['active_modules'],
+            frontend_settings=data['frontend_settings'],
         )
 
 
diff --git a/rotkehlchen/db/settings.py b/rotkehlchen/db/settings.py
--- a/rotkehlchen/db/settings.py
+++ b/rotkehlchen/db/settings.py
@@ -46,6 +46,7 @@ class DBSettings(NamedTuple):
     submit_usage_analytics: bool = DEFAULT_SUBMIT_USAGE_ANALYTICS
     kraken_account_type: KrakenAccountType = DEFAULT_KRAKEN_ACCOUNT_TYPE
     active_modules: List[str] = DEFAULT_ACTIVE_MODULES
+    frontend_settings: str = ''
 
 
 class ModifiableDBSettings(NamedTuple):
@@ -63,6 +64,7 @@ class ModifiableDBSettings(NamedTuple):
     submit_usage_analytics: Optional[bool] = None
     kraken_account_type: Optional[KrakenAccountType] = None
     active_modules: Optional[List[str]] = None
+    frontend_settings: Optional[str] = None
 
     def serialize(self) -> Dict[str, Any]:
         settings_dict = {}
@@ -159,6 +161,8 @@ def db_settings_from_dict(
             specified_args[key] = KrakenAccountType.deserialize(value)
         elif key == 'active_modules':
             specified_args[key] = json.loads(value)
+        elif key == 'frontend_settings':
+            specified_args[key] = str(value)
         else:
             msg_aggregator.add_warning(
                 f'Unknown DB setting {key} given. Ignoring it. Should not '


Test patch:
diff --git a/frontend/app/tests/common/utils.ts b/frontend/app/tests/common/utils.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/common/utils.ts
@@ -0,0 +1,3 @@
+export function stub<T>(partial?: Partial<T>): T {
+  return partial != null ? (partial as T) : ({} as T);
+}
diff --git a/frontend/app/tests/e2e/pages/defi-page.ts b/frontend/app/tests/e2e/pages/defi-page.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/e2e/pages/defi-page.ts
@@ -0,0 +1,31 @@
+export class DefiPage {
+  visit() {
+    cy.get('.v-app-bar__nav-icon').click();
+    cy.get('.navigation__defi').click();
+  }
+
+  goToSelectModules() {
+    cy.get('.defi-wizard__select-modules').click();
+  }
+
+  selectModules() {
+    cy.get('#defi-module-makerdao_dsr').find('button').click();
+    cy.get('#defi-module-makerdao_vaults').find('button').click();
+    cy.get('#defi-module-aave').should('be.visible');
+    cy.get('#defi-module-makerdao_dsr').should('not.be.visible');
+    cy.get('#defi-module-makerdao_vaults').should('not.be.visible');
+    cy.get('.defi-wizard__select-accounts').click();
+  }
+
+  selectAccounts() {
+    cy.get('.defi-address-selector')
+      .find('.v-stepper__header')
+      .children()
+      .should('have.length', 1);
+    cy.get('.defi-wizard__done').click();
+  }
+
+  defiOverviewIsVisible() {
+    cy.contains('Defi Overview').should('be.visible');
+  }
+}
diff --git a/frontend/app/tests/e2e/specs/defi.spec.ts b/frontend/app/tests/e2e/specs/defi.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/e2e/specs/defi.spec.ts
@@ -0,0 +1,29 @@
+import { Guid } from '../../common/guid';
+import { DefiPage } from '../pages/defi-page';
+import { RotkiApp } from '../pages/rotki-app';
+
+describe('defi', () => {
+  let username: string;
+  let app: RotkiApp;
+  let page: DefiPage;
+
+  before(() => {
+    username = Guid.newGuid().toString();
+    app = new RotkiApp();
+    page = new DefiPage();
+    app.visit();
+    app.createAccount(username);
+  });
+
+  it('goes through the wizard', () => {
+    page.visit();
+    page.goToSelectModules();
+    page.selectModules();
+    page.selectAccounts();
+    page.defiOverviewIsVisible();
+  });
+
+  after(() => {
+    app.logout();
+  });
+});
diff --git a/frontend/app/tests/unit/components/defi/wizard/defi-module-selector.spec.ts b/frontend/app/tests/unit/components/defi/wizard/defi-module-selector.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/components/defi/wizard/defi-module-selector.spec.ts
@@ -0,0 +1,48 @@
+import { mount, Wrapper } from '@vue/test-utils';
+import Vue from 'vue';
+import Vuetify from 'vuetify';
+import DefiModuleSelector from '@/components/defi/wizard/DefiModuleSelector.vue';
+import { api } from '@/services/rotkehlchen-api';
+import store from '@/store/store';
+import { GeneralSettings } from '@/typing/types';
+
+jest.mock('@/services/rotkehlchen-api');
+
+Vue.use(Vuetify);
+
+describe('DefiModuleSelector.vue', () => {
+  let wrapper: Wrapper<DefiModuleSelector>;
+
+  function createWrapper() {
+    const vuetify = new Vuetify();
+    return mount(DefiModuleSelector, {
+      store,
+      vuetify,
+      stubs: ['v-tooltip']
+    });
+  }
+
+  beforeEach(() => {
+    document.body.setAttribute('data-app', 'true');
+    const settings: GeneralSettings = {
+      ...store.state.session!.generalSettings,
+      activeModules: ['aave']
+    };
+    store.commit('session/generalSettings', settings);
+
+    wrapper = createWrapper();
+  });
+
+  test('displays active modules', async () => {
+    expect(wrapper.find('#defi-module-aave').exists()).toBe(true);
+  });
+
+  test('removes active modules on click', async () => {
+    expect.assertions(2);
+    api.setSettings = jest.fn().mockResolvedValue({ active_modules: [] });
+    wrapper.find('#defi-module-aave').find('button').trigger('click');
+    await wrapper.vm.$nextTick();
+    expect(wrapper.find('#defi-module-aave').exists()).toBe(false);
+    expect(store.state.session!.generalSettings.activeModules).toEqual([]);
+  });
+});
diff --git a/frontend/app/tests/unit/components/defi/wizard/defi-wizard.spec.ts b/frontend/app/tests/unit/components/defi/wizard/defi-wizard.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/components/defi/wizard/defi-wizard.spec.ts
@@ -0,0 +1,44 @@
+import { mount, Wrapper } from '@vue/test-utils';
+import Vue from 'vue';
+import Vuetify from 'vuetify';
+import DefiWizard from '@/components/defi/wizard/DefiWizard.vue';
+import { DEFI_SETUP_DONE } from '@/store/settings/consts';
+import store from '@/store/store';
+
+Vue.use(Vuetify);
+
+describe('DefiWizard.vue', () => {
+  let wrapper: Wrapper<DefiWizard>;
+  function createWrapper() {
+    const vuetify = new Vuetify();
+    return mount(DefiWizard, {
+      store,
+      vuetify,
+      stubs: ['v-tooltip', 'defi-module-selector', 'defi-address-selector']
+    });
+  }
+
+  beforeEach(() => {
+    wrapper = createWrapper();
+  });
+
+  test('wizard completes when use default is pressed', async () => {
+    expect.assertions(1);
+    wrapper.find('.defi-wizard__use-default').trigger('click');
+    await wrapper.vm.$nextTick();
+    // @ts-ignore
+    expect(store.state.settings[DEFI_SETUP_DONE]).toBeTruthy();
+  });
+
+  test('wizard completes when complete is pressed', async () => {
+    expect.assertions(1);
+    wrapper.find('.defi-wizard__select-modules').trigger('click');
+    await wrapper.vm.$nextTick();
+    wrapper.find('.defi-wizard__select-accounts').trigger('click');
+    await wrapper.vm.$nextTick();
+    wrapper.find('.defi-wizard__done').trigger('click');
+    await wrapper.vm.$nextTick();
+    // @ts-ignore
+    expect(store.state.settings[DEFI_SETUP_DONE]).toBeTruthy();
+  });
+});
diff --git a/frontend/app/tests/unit/components/display/account-display.spec.ts b/frontend/app/tests/unit/components/display/account-display.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/components/display/account-display.spec.ts
@@ -0,0 +1,46 @@
+import { mount, Wrapper } from '@vue/test-utils';
+import Vue from 'vue';
+import Vuetify from 'vuetify';
+import AccountDisplay from '@/components/display/AccountDisplay.vue';
+import store from '@/store/store';
+import { GeneralAccount } from '@/typing/types';
+
+Vue.use(Vuetify);
+
+describe('AccountDisplay.vue', () => {
+  let wrapper: Wrapper<AccountDisplay>;
+
+  const account: GeneralAccount = {
+    chain: 'ETH',
+    address: '0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5',
+    label: 'Test Account',
+    tags: []
+  };
+
+  function createWrapper() {
+    const vuetify = new Vuetify();
+    return mount(AccountDisplay, {
+      store,
+      vuetify,
+      stubs: ['v-tooltip', 'crypto-icon'],
+      propsData: {
+        account
+      }
+    });
+  }
+
+  beforeEach(() => {
+    store.dispatch('session/logout');
+    wrapper = createWrapper();
+  });
+
+  test('does not blur anything by default', async () => {
+    expect(wrapper.find('.blur-content').exists()).toBe(false);
+  });
+
+  test('blurs address on privacy mode', async () => {
+    store.commit('session/privacyMode', true);
+    await wrapper.vm.$nextTick();
+    expect(wrapper.find('.blur-content').exists()).toBe(true);
+  });
+});
diff --git a/frontend/app/tests/unit/components/status/node-status-indicator.spec.ts b/frontend/app/tests/unit/components/status/node-status-indicator.spec.ts
--- a/frontend/app/tests/unit/components/status/node-status-indicator.spec.ts
+++ b/frontend/app/tests/unit/components/status/node-status-indicator.spec.ts
@@ -11,6 +11,7 @@ describe('NodeStatusIndicator.vue', () => {
   let wrapper: Wrapper<NodeStatusIndicator>;
 
   beforeEach(() => {
+    document.body.setAttribute('data-app', 'true');
     vuetify = new Vuetify();
     wrapper = mount(NodeStatusIndicator, {
       store,
diff --git a/frontend/app/tests/unit/store/session/actions.spec.ts b/frontend/app/tests/unit/store/session/actions.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/store/session/actions.spec.ts
@@ -0,0 +1,104 @@
+import { api } from '@/services/rotkehlchen-api';
+import {
+  QueriedAddresses,
+  QueriedAddressPayload
+} from '@/services/session/types';
+import store from '@/store/store';
+
+jest.mock('@/services/rotkehlchen-api');
+
+describe('session:actions', () => {
+  beforeEach(() => {
+    store.dispatch('session/logout');
+  });
+
+  test('fetchQueriedAddresses', async () => {
+    expect.assertions(2);
+    const response: QueriedAddresses = {
+      makerdao_dsr: ['0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5']
+    };
+    api.session.queriedAddresses = jest.fn().mockResolvedValue(response);
+    await store.dispatch('session/fetchQueriedAddresses');
+    expect(api.session.queriedAddresses).toHaveBeenCalledTimes(1);
+    expect(store.state.session!.queriedAddresses).toMatchObject(response);
+  });
+
+  test('fetchQueriedAddresses fails', async () => {
+    expect.assertions(3);
+    api.session.queriedAddresses = jest
+      .fn()
+      .mockRejectedValue(new Error('failed'));
+    await store.dispatch('session/fetchQueriedAddresses');
+    expect(api.session.queriedAddresses).toHaveBeenCalledTimes(1);
+    expect(store.state.session!.queriedAddresses).toMatchObject({});
+    expect(store.state.message.description).toBeTruthy();
+  });
+
+  test('addQueriedAddress', async () => {
+    expect.assertions(2);
+    const payload: QueriedAddressPayload = {
+      module: 'makerdao_dsr',
+      address: '0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5'
+    };
+    const response: QueriedAddresses = {
+      makerdao_dsr: ['0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5']
+    };
+    api.session.addQueriedAddress = jest.fn().mockResolvedValue(response);
+    await store.dispatch('session/addQueriedAddress', payload);
+    expect(api.session.addQueriedAddress).toHaveBeenCalledWith(payload);
+    expect(store.state.session!.queriedAddresses).toMatchObject(response);
+  });
+
+  test('addQueriedAddress fails', async () => {
+    expect.assertions(3);
+    const payload: QueriedAddressPayload = {
+      module: 'makerdao_dsr',
+      address: '0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5'
+    };
+
+    api.session.addQueriedAddress = jest
+      .fn()
+      .mockRejectedValue(new Error('failed'));
+    await store.dispatch('session/addQueriedAddress', payload);
+    expect(api.session.addQueriedAddress).toHaveBeenCalledWith(payload);
+    expect(store.state.session!.queriedAddresses).toMatchObject({});
+    expect(store.state.message.description).toBeTruthy();
+  });
+
+  test('deletedQueriedAddress', async () => {
+    expect.assertions(2);
+    const originalState: QueriedAddresses = {
+      makerdao_dsr: ['0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5']
+    };
+    store.commit('session/queriedAddresses', originalState);
+    const payload: QueriedAddressPayload = {
+      module: 'makerdao_dsr',
+      address: '0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5'
+    };
+
+    api.session.deleteQueriedAddress = jest.fn().mockResolvedValue({});
+    await store.dispatch('session/deleteQueriedAddress', payload);
+    expect(api.session.deleteQueriedAddress).toHaveBeenCalledWith(payload);
+    expect(store.state.session!.queriedAddresses).toMatchObject({});
+  });
+
+  test('deletedQueriedAddress failed', async () => {
+    expect.assertions(3);
+    const originalState: QueriedAddresses = {
+      makerdao_dsr: ['0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5']
+    };
+    store.commit('session/queriedAddresses', originalState);
+    const payload: QueriedAddressPayload = {
+      module: 'makerdao_dsr',
+      address: '0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5'
+    };
+
+    api.session.deleteQueriedAddress = jest
+      .fn()
+      .mockRejectedValue(new Error('failed'));
+    await store.dispatch('session/deleteQueriedAddress', payload);
+    expect(api.session.deleteQueriedAddress).toHaveBeenCalledWith(payload);
+    expect(store.state.session!.queriedAddresses).toMatchObject(originalState);
+    expect(store.state.message.description).toBeTruthy();
+  });
+});
diff --git a/frontend/app/tests/unit/store/settings/actions.spec.ts b/frontend/app/tests/unit/store/settings/actions.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/store/settings/actions.spec.ts
@@ -0,0 +1,34 @@
+import { api } from '@/services/rotkehlchen-api';
+import { DEFI_SETUP_DONE } from '@/store/settings/consts';
+import { FrontendSettingsPayload } from '@/store/settings/types';
+import store from '@/store/store';
+
+jest.mock('@/services/rotkehlchen-api');
+
+describe('settings:actions', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+  test('updates settings on valid payload', async () => {
+    expect.assertions(1);
+    await store.dispatch('settings/updateSetting', {
+      [DEFI_SETUP_DONE]: true
+    } as FrontendSettingsPayload);
+
+    expect(api.setSettings).toHaveBeenCalledWith(
+      expect.objectContaining({
+        frontend_settings: JSON.stringify({ [DEFI_SETUP_DONE]: true })
+      })
+    );
+  });
+
+  test('does not update settings on missing payload', async () => {
+    expect.assertions(1);
+    await store.dispatch(
+      'settings/updateSetting',
+      {} as FrontendSettingsPayload
+    );
+
+    expect(api.setSettings).toHaveBeenCalledTimes(0);
+  });
+});
diff --git a/frontend/app/tests/unit/store/settings/mutations.spec.ts b/frontend/app/tests/unit/store/settings/mutations.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/store/settings/mutations.spec.ts
@@ -0,0 +1,13 @@
+import { DEFI_SETUP_DONE } from '@/store/settings/consts';
+import { SettingsState } from '@/store/settings/types';
+import store from '@/store/store';
+
+describe('settings:mutations', () => {
+  test('restore', async () => {
+    const state: SettingsState = {
+      [DEFI_SETUP_DONE]: true
+    };
+    store.commit('settings/restore', state);
+    expect(store.state.settings!.defiSetupDone).toBe(true);
+  });
+});
diff --git a/frontend/app/tests/unit/store/settings/utils.spec.ts b/frontend/app/tests/unit/store/settings/utils.spec.ts
new file mode 100644
--- /dev/null
+++ b/frontend/app/tests/unit/store/settings/utils.spec.ts
@@ -0,0 +1,40 @@
+import { DEFI_SETUP_DONE } from '@/store/settings/consts';
+import { loadFrontendSettings } from '@/store/settings/utils';
+
+describe('settings:utils', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  test('restore nothing if no valid properties are found', async () => {
+    expect.assertions(1);
+    const commit = jest.fn();
+    loadFrontendSettings(commit, JSON.stringify({ random: 1 }));
+    expect(commit).toHaveBeenCalledTimes(0);
+  });
+
+  test('restore nothing is the loaded value is not a valid json', async () => {
+    expect.assertions(1);
+    const commit = jest.fn();
+    loadFrontendSettings(commit, 'dasda');
+    expect(commit).toHaveBeenCalledTimes(0);
+  });
+
+  test('restore nothing is the loaded value has an unexpected type', async () => {
+    expect.assertions(1);
+    const commit = jest.fn();
+    loadFrontendSettings(commit, JSON.stringify({ [DEFI_SETUP_DONE]: 1 }));
+    expect(commit).toHaveBeenCalledTimes(0);
+  });
+
+  test('restore valid properties', async () => {
+    expect.assertions(1);
+    const commit = jest.fn();
+    loadFrontendSettings(commit, JSON.stringify({ [DEFI_SETUP_DONE]: true }));
+    expect(commit).toHaveBeenCalledWith(
+      'settings/restore',
+      { [DEFI_SETUP_DONE]: true },
+      { root: true }
+    );
+  });
+});
diff --git a/rotkehlchen/tests/api/test_settings.py b/rotkehlchen/tests/api/test_settings.py
--- a/rotkehlchen/tests/api/test_settings.py
+++ b/rotkehlchen/tests/api/test_settings.py
@@ -87,6 +87,8 @@ def test_set_settings(rotkehlchen_api_server):
             value = str(KrakenAccountType.PRO)
         elif setting == 'active_modules':
             value = ['makerdao_vaults']
+        elif setting == 'frontend_settings':
+            value = ''
         else:
             raise AssertionError(f'Unexpected settting {setting} encountered')
 
diff --git a/rotkehlchen/tests/db/test_db.py b/rotkehlchen/tests/db/test_db.py
--- a/rotkehlchen/tests/db/test_db.py
+++ b/rotkehlchen/tests/db/test_db.py
@@ -175,7 +175,7 @@ def test_export_import_db(data_dir, username):
     assert int(fiat_balances[A_EUR]) == 10
 
 
-def test_writting_fetching_data(data_dir, username):
+def test_writing_fetching_data(data_dir, username):
     msg_aggregator = MessagesAggregator()
     data = DataHandler(data_dir, msg_aggregator)
     data.unlock(username, '123', create_new=True)
@@ -262,6 +262,7 @@ def test_writting_fetching_data(data_dir, username):
         'last_write_ts': 0,
         'kraken_account_type': DEFAULT_KRAKEN_ACCOUNT_TYPE,
         'active_modules': DEFAULT_ACTIVE_MODULES,
+        'frontend_settings': '',
     }
     assert len(expected_dict) == len(DBSettings()), 'One or more settings are missing'
 
@@ -320,6 +321,8 @@ def test_settings_entry_types(database):
     assert res.submit_usage_analytics is False
     assert isinstance(res.active_modules, list)
     assert res.active_modules == DEFAULT_ACTIVE_MODULES
+    assert isinstance(res.frontend_settings, str)
+    assert res.frontend_settings == ''
 
 
 def test_balance_save_frequency_check(data_dir, username):