Repository: ethyca/fidesops (language: python)

Issue:
Rollbar (Access)



Discussion / hints:

Fix patch:
diff --git a/data/saas/config/rollbar_config.yml b/data/saas/config/rollbar_config.yml
new file mode 100644
index 000000000..e03f84aa3
--- /dev/null
+++ b/data/saas/config/rollbar_config.yml
@@ -0,0 +1,91 @@
+saas_config:
+  fides_key: <instance_fides_key>
+  name: Rollbar SaaS Config
+  type: rollbar
+  description: A sample schema representing the Rollbar connector for Fidesops
+  version: 0.0.1
+
+  connector_params:
+    - name: domain
+    - name: read_access_token
+    - name: write_access_token
+    - name: page_limit
+      default_value: 100
+      description: The maximum number of pages to return from Rollbar's "instances" endpoint
+
+  client_config:
+    protocol: https
+    host: <domain>
+
+  test_request:
+    method: GET
+    path: /api/1/status/ping
+
+  endpoints:
+    - name: projects
+      requests:
+        read:
+          method: GET
+          path: /api/1/projects
+          headers:
+            - name: X-Rollbar-Access-Token
+              value: <read_access_token>
+          param_values:
+            - name: read_access_token
+              connector_param: read_access_token
+            - name: placeholder
+              identity: email
+          data_path: result
+    - name: project_access_tokens
+      requests:
+        read:
+          method: GET
+          path: /api/1/project/<project_id>/access_tokens
+          headers:
+            - name: X-Rollbar-Access-Token
+              value: <read_access_token>
+          param_values:
+            - name: read_access_token
+              connector_param: read_access_token
+            - name: project_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: projects.id
+                  direction: from
+          postprocessors:
+            - strategy: filter
+              configuration:
+                field: name
+                value: read
+          data_path: result
+    - name: instances
+      requests:
+        read:
+          method: GET
+          path: /api/1/instances
+          headers:
+            - name: X-Rollbar-Access-Token
+              value: <project_read_token>
+          query_params:
+            - name: page
+              value: 1
+          param_values:
+            - name: project_read_token
+              references:
+                - dataset: <instance_fides_key>
+                  field: project_access_tokens.access_token
+                  direction: from
+          pagination:
+            strategy: offset
+            configuration:
+              incremental_param: page
+              increment_by: 1
+              limit:
+                connector_param: page_limit
+          postprocessors:
+            - strategy: filter
+              configuration:
+                field: data.person.email
+                value:
+                  identity: email
+          data_path: result.instances
diff --git a/data/saas/dataset/rollbar_dataset.yml b/data/saas/dataset/rollbar_dataset.yml
new file mode 100644
index 000000000..3d30d54a1
--- /dev/null
+++ b/data/saas/dataset/rollbar_dataset.yml
@@ -0,0 +1,175 @@
+dataset:
+  - fides_key: <instance_fides_key>
+    name: Rollbar Dataset
+    description: A sample dataset representing the Rollbar connector for Fidesops
+    collections:
+      - name: projects
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: account_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: date_created
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: date_modified
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: name
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+      - name: project_access_tokens
+        fields:
+          - name: project_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: name
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: status
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: date_created
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: date_modified
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: scopes
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string[]
+      - name: instances
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: project_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: timestamp
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: version
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: data
+            fidesops_meta:
+              data_type: object
+            fields:
+              - name: environment
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: string
+              - name: body
+                fidesops_meta:
+                  data_type: object
+                fields:
+                  - name: message
+                    fidesops_meta:
+                      data_type: object
+                    fields:
+                      - name: body
+                        data_categories: [user]
+                        fidesops_meta:
+                          data_type: string
+                      - name: route
+                        data_categories: [system.operations]
+                        fidesops_meta:
+                          data_type: string
+                      - name: time_elapsed
+                        data_categories: [system.operations]
+                        fidesops_meta:
+                          data_type: float
+                  - name: level
+                    data_categories: [system.operations]
+                    fidesops_meta:
+                      data_type: string
+              - name: person
+                fidesops_meta:
+                  data_type: object
+                fields:
+                  - name: id
+                    data_categories: [system.operations]
+                    fidesops_meta:
+                      data_type: string
+                  - name: username
+                    data_categories: [user]
+                    fidesops_meta:
+                      data_type: string
+                  - name: email
+                    data_categories: [user.contact.email]
+                    fidesops_meta:
+                      data_type: string
+              - name: metadata
+                fidesops_meta:
+                  data_type: object
+                fields:
+                  - name: timestamp_ms
+                    data_categories: [system.operations]
+                    fidesops_meta:
+                      data_type: integer
+                  - name: api_server_hostname
+                    data_categories: [system.operations]
+                    fidesops_meta:
+                      data_type: string
+                  - name: debug
+                    fidesops_meta:
+                      data_type: object
+                    fields:
+                      - name: routes
+                        fidesops_meta:
+                          data_type: object
+                        fields:
+                          - name: start_time
+                            data_categories: [system.operations]
+                            fidesops_meta:
+                              data_type: integer
+                          - name: counters
+                            fidesops_meta:
+                              data_type: object
+                            fields:
+                              - name: post_item
+                                data_categories: [system.operations]
+                                fidesops_meta:
+                                  data_type: integer
+              - name: framework
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: string
+              - name: uuid
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: string
+              - name: timestamp
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: integer
+          - name: billable
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
+          - name: item_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: integer
diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml
index b5187baf3..24e728203 100644
--- a/data/saas/saas_connector_registry.toml
+++ b/data/saas/saas_connector_registry.toml
@@ -82,3 +82,9 @@ config = "data/saas/config/zendesk_config.yml"
 dataset = "data/saas/dataset/zendesk_dataset.yml"
 icon = "data/saas/icon/zendesk.svg"
 human_readable = "Zendesk"
+
+[rollbar]
+config = "data/saas/config/rollbar_config.yml"
+dataset = "data/saas/dataset/rollbar_dataset.yml"
+icon = "data/saas/icon/default.svg"
+human_readable = "Rollbar"
\ No newline at end of file
diff --git a/src/fidesops/ops/schemas/saas/saas_config.py b/src/fidesops/ops/schemas/saas/saas_config.py
index 3e00e384a..c21199fc5 100644
--- a/src/fidesops/ops/schemas/saas/saas_config.py
+++ b/src/fidesops/ops/schemas/saas/saas_config.py
@@ -263,6 +263,7 @@ class SaaSType(Enum):
     custom = "custom"
     sendgrid = "sendgrid"
     datadog = "datadog"
+    rollbar = "rollbar"
 
 
 class SaaSConfigBase(BaseModel):


Test patch:
diff --git a/pytest.ini b/pytest.ini
index 716b4e22f..d62d6050b 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,6 +21,7 @@ markers =
     integration_stripe
     integration_hubspot
     integration_datadog
+    integration_rollbar
     integration_segment
     integration_sendgrid
     integration_auth0
diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py
index 0aed88046..cc40d329a 100644
--- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py
@@ -48,7 +48,7 @@ def test_get_connection_types(
         resp = api_client.get(url, headers=auth_header)
         data = resp.json()["items"]
         assert resp.status_code == 200
-        assert len(data) == 23
+        assert len(data) == 24
 
         assert {
             "identifier": ConnectionType.postgres.value,
@@ -152,7 +152,7 @@ def test_search_system_type(self, api_client, generate_auth_header, url):
         resp = api_client.get(url + "?system_type=saas", headers=auth_header)
         assert resp.status_code == 200
         data = resp.json()["items"]
-        assert len(data) == 14
+        assert len(data) == 15
 
         resp = api_client.get(url + "?system_type=database", headers=auth_header)
         assert resp.status_code == 200
diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py
index 461f3fa26..94840bec6 100644
--- a/tests/ops/conftest.py
+++ b/tests/ops/conftest.py
@@ -49,6 +49,7 @@
 from .fixtures.saas.mailchimp_fixtures import *
 from .fixtures.saas.outreach_fixtures import *
 from .fixtures.saas.request_override.mailchimp_override_fixtures import *
+from .fixtures.saas.rollbar_fixtures import *
 from .fixtures.saas.salesforce_fixtures import *
 from .fixtures.saas.segment_fixtures import *
 from .fixtures.saas.sendgrid_fixtures import *
diff --git a/tests/ops/fixtures/saas/rollbar_fixtures.py b/tests/ops/fixtures/saas/rollbar_fixtures.py
new file mode 100644
index 000000000..0b440cd25
--- /dev/null
+++ b/tests/ops/fixtures/saas/rollbar_fixtures.py
@@ -0,0 +1,101 @@
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+from fideslib.db import session
+from sqlalchemy.orm import Session
+
+from fidesops.ops.models.connectionconfig import (
+    AccessLevel,
+    ConnectionConfig,
+    ConnectionType,
+)
+from fidesops.ops.models.datasetconfig import DatasetConfig
+from fidesops.ops.util.saas_util import (
+    load_config_with_replacement,
+    load_dataset_with_replacement,
+)
+from tests.ops.test_helpers.vault_client import get_secrets
+
+secrets = get_secrets("rollbar")
+
+
+@pytest.fixture(scope="session")
+def rollbar_secrets(saas_config):
+    return {
+        "domain": pydash.get(saas_config, "rollbar.domain") or secrets["domain"],
+        "read_access_token": pydash.get(saas_config, "rollbar.read_access_token")
+        or secrets["read_access_token"],
+        "write_access_token": pydash.get(saas_config, "rollbar.write_access_token")
+        or secrets["write_access_token"],
+        "page_limit": pydash.get(saas_config, "rollbar.page_limit")
+        or secrets["page_limit"],
+    }
+
+
+@pytest.fixture(scope="session")
+def rollbar_identity_email(saas_config):
+    return (
+        pydash.get(saas_config, "rollbar.identity_email") or secrets["identity_email"]
+    )
+
+
+@pytest.fixture
+def rollbar_config() -> Dict[str, Any]:
+    return load_config_with_replacement(
+        "data/saas/config/rollbar_config.yml",
+        "<instance_fides_key>",
+        "rollbar_instance",
+    )
+
+
+@pytest.fixture
+def rollbar_dataset() -> Dict[str, Any]:
+    return load_dataset_with_replacement(
+        "data/saas/dataset/rollbar_dataset.yml",
+        "<instance_fides_key>",
+        "rollbar_instance",
+    )[0]
+
+
+@pytest.fixture(scope="function")
+def rollbar_connection_config(
+    db: session, rollbar_config, rollbar_secrets
+) -> Generator:
+    fides_key = rollbar_config["fides_key"]
+    connection_config = ConnectionConfig.create(
+        db=db,
+        data={
+            "key": fides_key,
+            "name": fides_key,
+            "connection_type": ConnectionType.saas,
+            "access": AccessLevel.write,
+            "secrets": rollbar_secrets,
+            "saas_config": rollbar_config,
+        },
+    )
+    yield connection_config
+    connection_config.delete(db)
+
+
+@pytest.fixture
+def rollbar_dataset_config(
+    db: Session,
+    rollbar_connection_config: ConnectionConfig,
+    rollbar_dataset,
+    rollbar_config,
+) -> Generator:
+    fides_key = rollbar_config["fides_key"]
+    rollbar_connection_config.name = fides_key
+    rollbar_connection_config.key = fides_key
+    rollbar_connection_config.save(db=db)
+    dataset = DatasetConfig.create(
+        db=db,
+        data={
+            "connection_config_id": rollbar_connection_config.id,
+            "fides_key": fides_key,
+            "dataset": rollbar_dataset,
+        },
+    )
+    yield dataset
+    dataset.delete(db=db)
diff --git a/tests/ops/integration_tests/saas/test_rollbar_task.py b/tests/ops/integration_tests/saas/test_rollbar_task.py
new file mode 100644
index 000000000..cb292201b
--- /dev/null
+++ b/tests/ops/integration_tests/saas/test_rollbar_task.py
@@ -0,0 +1,88 @@
+import logging
+import random
+
+import pytest
+
+from fidesops.ops.graph.graph import DatasetGraph
+from fidesops.ops.models.privacy_request import PrivacyRequest
+from fidesops.ops.schemas.redis_cache import Identity
+from fidesops.ops.service.connectors import get_connector
+from fidesops.ops.task import graph_task
+from tests.ops.graph.graph_test_util import assert_rows_match
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.integration_saas
+@pytest.mark.integration_rollbar
+def test_rollbar_connection_test(rollbar_connection_config) -> None:
+    get_connector(rollbar_connection_config).test_connection()
+
+
+@pytest.mark.integration_saas
+@pytest.mark.integration_rollbar
+@pytest.mark.asyncio
+async def test_rollbar_access_request_task(
+    db,
+    policy,
+    rollbar_connection_config,
+    rollbar_dataset_config,
+    rollbar_identity_email,
+) -> None:
+    """Full access request based on the Rollbar SaaS config"""
+
+    privacy_request = PrivacyRequest(
+        id=f"test_rollbar_access_request_task_{random.randint(0, 1000)}"
+    )
+    identity = Identity(**{"email": rollbar_identity_email})
+    privacy_request.cache_identity(identity)
+
+    dataset_name = rollbar_connection_config.get_saas_config().fides_key
+    merged_graph = rollbar_dataset_config.get_graph()
+    graph = DatasetGraph(merged_graph)
+
+    v = await graph_task.run_access_request(
+        privacy_request,
+        policy,
+        graph,
+        [rollbar_connection_config],
+        {"email": rollbar_identity_email},
+        db,
+    )
+
+    assert_rows_match(
+        v[f"{dataset_name}:projects"],
+        min_size=1,
+        keys=["id", "account_id", "status", "date_created", "date_modified", "name"],
+    )
+
+    assert_rows_match(
+        v[f"{dataset_name}:project_access_tokens"],
+        min_size=1,
+        keys=[
+            "project_id",
+            "name",
+            "status",
+            "date_created",
+            "date_modified",
+            "scopes",
+        ],
+    )
+
+    assert_rows_match(
+        v[f"{dataset_name}:instances"],
+        min_size=1,
+        keys=[
+            "id",
+            "project_id",
+            "timestamp",
+            "version",
+            "data",
+            "billable",
+            "item_id",
+        ],
+    )
+
+    # verify we only returned data for our identity email
+    for instance in v[f"{dataset_name}:instances"]:
+        assert instance["data"]["person"]["email"] == rollbar_identity_email