Repository: ethyca/fideslog (language: python)

Issue:
Create a Python analytics SDK
The SDK should implement a "managed API" interface for Python applications that wish to send analytics events to the API server. It should enable applications to:
- Create an `AnalyticsClient` instance, which enables caller-specific configuration, such as API keys, UUID, etc.
- Create an `AnalyticsEvent` instance, which validates attributes for presence and content, as necessary
- Abstract away the details of sending a `POST` request to the API server's `/events` endpoint
- Identify and handle errors

The SDK should not create any log entries. Include usage and reference documentation in a `README.md` file.

Usage may be as simple as:

```python
from logging import getLogger
from os import getenv
from fideslog.sdk.python import (
	AnalyticsClient,
	AnalyticsError,
	AnalyticsEvent,
	FIDESCTL_CLI,
	get_or_create_uuid,
)
	
logger = getLogger(__name__)
client = AnalyticsClient(getenv("FIDESLOG_API_KEY"), get_or_create_uuid(), FIDESCTL_CLI)
event = AnalyticsEvent("event type", "command name", ...)

try:
	client.send(event)
except AnalyticsError as e:
	logger.warn("failed to send analytics event: %s", e["message"])
except:
	logger.warn("something else has gone wrong")
else:
	logger.info("analytics event sent")
```


Discussion / hints:

Fix patch:
diff --git a/Makefile b/Makefile
index edc9008..cbf1def 100644
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,8 @@ xenon:
 	--max-absolute B \
 	--max-modules B \
 	--max-average A \
-	--ignore "tests"
+	--ignore "tests" \
+	--exclude "fideslog/sdk/python/event.py"
 
 ####################
 # Utils
diff --git a/dev-requirements.txt b/dev-requirements.txt
index e8e201f..33ecde5 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -3,5 +3,6 @@ mypy==0.910
 pylint==2.6.0
 pytest==6.2.2
 pytest-cov==2.11.1
+types-requests
 types-toml
 xenon==0.7.3
diff --git a/fideslog/sdk/python/__init__.py b/fideslog/sdk/python/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/fideslog/sdk/python/client.py b/fideslog/sdk/python/client.py
new file mode 100644
index 0000000..5856680
--- /dev/null
+++ b/fideslog/sdk/python/client.py
@@ -0,0 +1,122 @@
+# pylint: disable=too-many-arguments
+
+from typing import Dict, Optional
+from urllib.error import HTTPError
+
+from requests import PreparedRequest, post
+from requests.auth import AuthBase
+from requests.exceptions import RequestException
+
+from fideslog.sdk.python.event import AnalyticsEvent
+from fideslog.sdk.python.exceptions import AnalyticsException
+
+
+class AnalyticsAuth(AuthBase):
+    """
+    Attaches fideslog authentication to a given Request object.
+    """
+
+    def __init__(self, api_key: str):
+        self.token = api_key
+
+    def __call__(self, request: PreparedRequest) -> PreparedRequest:
+        request.headers["Authorization"] = f"Token {self.token}"
+        return request
+
+
+class AnalyticsClient:
+    """
+    An instance of a fides tool that wishes to send
+    analytics events to the fideslog server.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        client_id: str,
+        os: str,
+        product_name: str,
+        production_version: str,
+        extra_data: Optional[Dict] = None,
+    ) -> None:
+        # pylint: disable=line-too-long
+        """
+        Define a new client from which to send analytics events to the fideslog server.
+
+        :param api_key: The authorization token, administered by Ethyca, that enables query access to the fideslog server.
+        :param client_id: The fully anonymized, globally unique identifier for this client.
+        :param os: The operating system on which this client is running, as returned by [platform.system()](https://docs.python.org/3/library/platform.html#platform.system).
+        :param product_name: The name of the fides tool in which this client is integrated.
+        :param production_version: The semantic version number of the fides tool in which this client is integrated.
+        :param extra_data: Any additional information that should be included in all analytics events sent by this client. Any key/value pairs included here will be merged with key/value pairs included directly on specific `AnalyticsEvent`s, with the `AnalyticsEvent`'s `extra_data` taking priority.
+        """
+        # pylint: enable=line-too-long
+
+        for arg in [api_key, client_id, os, product_name, production_version]:
+            assert arg is not None and arg != "", (
+                f"{arg=}".split("=")[0] + " must be provided"
+            )
+
+        self.api_key = api_key
+        self.client_id = client_id
+        self.os = os
+        self.product_name = product_name
+        self.production_version = production_version
+        self.extra_data = extra_data
+
+    async def send(self, event: AnalyticsEvent) -> None:
+        """
+        Record a new event.
+        """
+
+        payload = {
+            "client_id": self.client_id,
+            "docker": event.docker,
+            "event": event.event,
+            "event_created_at": event.event_created_at,
+            "local_host": event.local_host,
+            "os": self.os,
+            "product_name": self.product_name,
+            "production_version": self.production_version,
+        }
+
+        payload_extras = [
+            "command",
+            "endpoint",
+            "error",
+            # "extra_data",
+            "flags",
+            "resource_counts",
+            "status_code",
+        ]
+
+        event_dict = event.dict()
+        for extra in payload_extras:
+            if event_dict[extra]:
+                payload[extra] = event_dict[extra]
+
+        extra_data: Dict = {}
+        if self.extra_data is not None:
+            extra_data = self.extra_data
+
+        if event.extra_data is not None:
+            for key, val in event.extra_data.items():
+                extra_data[key] = val
+
+        payload["extra_data"] = extra_data
+
+        try:
+            response = post(
+                "http://localhost:8080",
+                auth=AnalyticsAuth(self.api_key),
+                json=payload,
+                timeout=(3.05, 120),
+            )
+
+            try:
+                response.raise_for_status()
+            except HTTPError as e:
+                raise AnalyticsException(e.reason, e.args, status_code=e.code) from e
+
+        except RequestException as e:
+            raise AnalyticsException(e.strerror, e.args) from e
diff --git a/fideslog/sdk/python/event.py b/fideslog/sdk/python/event.py
new file mode 100644
index 0000000..c5ab20a
--- /dev/null
+++ b/fideslog/sdk/python/event.py
@@ -0,0 +1,92 @@
+# pylint: disable= too-many-arguments, too-many-instance-attributes
+
+from datetime import datetime, timezone
+from json import dumps
+from typing import Dict, Optional
+from urllib.parse import urlparse
+
+
+class AnalyticsEvent:
+    """
+    A discrete event, representing a user action within a fides tool.
+    """
+
+    def __init__(
+        self,
+        event: str,
+        event_created_at: datetime,
+        command: Optional[str] = None,
+        docker: bool = False,
+        endpoint: Optional[str] = None,
+        error: Optional[str] = None,
+        extra_data: Optional[Dict] = None,
+        flags: Optional[list[str]] = None,
+        local_host: bool = False,
+        resource_counts: Optional[dict[str, int]] = None,
+        status_code: Optional[int] = None,
+    ) -> None:
+        # pylint: disable=line-too-long
+        """
+        Define a new analytics event to send to the fideslog server.
+
+        :param event: The name/type of this event.
+        :param event_created_at: The UTC timestamp when the event occurred, in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format. Must include the UTC timezone, and represent a datetime in the past.
+        :param command: For events submitted as a result of running CLI commands, the name of the command that was submitted. May include the subcommand name(s).
+        :param docker: `True` if the command was submitted within a Docker container. Default: `False`.
+        :param endpoint: For events submitted as a result of making API server requests, the API endpoint path that was requested. If a fully-qualified URL is provided, only the URL path will be persisted.
+        :param error: For events submitted as a result of running CLI commands that exit with a non-0 status code, or events submitted as a result of API server requests that respond with a non-2xx status code, the error type, without specific error details.
+        :param extra_data: Any additional key/value pairs that should be associated with this event.
+        :param flags: For events submitted as a result of running CLI commands, the flags in use when the command was submitted. Omits flag values (when they exist) by persisting only the portion of each string in this list that come before `=` or `space` characters.
+        :param local_host: For events submitted as a result of making API server requests, `True` if the API server is running on the user's local host. Default: `False`.
+        :param resource_counts: Should contain the counts of dataset, policy, and system manifests in use when this event was submitted. Include all three keys, even if one or more of their values are `0`. Ex: `{ "datasets": 7, "policies": 26, "systems": 9 }`.
+        :param status_code: For events submitted as a result of making API server requests, the HTTP status code included in the response.
+        """
+        # pylint: enable=line-too-long
+
+        assert event is not None, "An event name/type is required"
+        self.event = event
+
+        assert (
+            event_created_at is not None
+        ), "event_created_at is required, in UTC and ISO 8601 format"
+        assert (
+            event_created_at.tzinfo is not None
+            and event_created_at.tzinfo == timezone.utc
+        ), "event_created_at must include the UTC timezone"
+        assert event_created_at < datetime.now(
+            timezone.utc
+        ), "event_created_at must be in the past"
+        self.event_created_at = event_created_at
+
+        self.resource_counts = None
+        if resource_counts is not None:
+            for key in ["datasets", "policies", "systems"]:
+                val = resource_counts.get(key)
+                assert val is not None, f'resource_counts must include a "{key}" key'
+                assert isinstance(
+                    val, int
+                ), f'The value of resource_counts["{key}"] must be an integer'
+
+            self.resource_counts = dumps(resource_counts)
+
+        self.endpoint = None
+        if endpoint is not None:
+            assert urlparse(endpoint).path != "", "endpoint must include a URL path"
+            self.endpoint = endpoint
+
+        self.command = command
+        self.docker = docker
+        self.error = error
+        self.extra_data = extra_data
+        self.flags = flags
+        self.local_host = local_host
+        self.status_code = status_code
+
+        if self.command is not None or self.endpoint is not None:
+            assert self.status_code is not None, "status_code must be provided"
+
+        if self.error is not None:
+            assert self.status_code is not None, "status_code must be provided"
+            assert self.status_code > 0 and (
+                self.status_code < 200 or self.status_code > 299
+            ), "An error was provided, but the provided status_code indicates success"
diff --git a/fideslog/sdk/python/exceptions.py b/fideslog/sdk/python/exceptions.py
new file mode 100644
index 0000000..11b998d
--- /dev/null
+++ b/fideslog/sdk/python/exceptions.py
@@ -0,0 +1,23 @@
+from typing import Optional
+
+
+class AnalyticsException(Exception):
+    """
+    To be raised wherever an exception is required in this package.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *args: object,
+        status_code: Optional[int] = None,
+    ) -> None:
+        self.message = message
+        self.status_code = status_code
+        super().__init__(self.message, *args)
+
+    def __str__(self) -> str:
+        if self.status_code is not None:
+            return f"{self.status_code} response from fideslog: {self.message}"
+
+        return f"Failed to send analytics event: {self.message}"
diff --git a/fideslog/sdk/python/utils.py b/fideslog/sdk/python/utils.py
new file mode 100644
index 0000000..15195fa
--- /dev/null
+++ b/fideslog/sdk/python/utils.py
@@ -0,0 +1,23 @@
+from hashlib import md5
+from uuid import uuid1
+
+from bcrypt import gensalt
+
+FIDESCTL_API = b"fidesctl-api"
+FIDESCTL_CLI = b"fidesctl-cli"
+FIDESOPS = b"fidesops"
+
+
+def generate_client_id(application: bytes) -> str:
+    # pylint: disable=line-too-long
+    """
+    Generates a cryptographically secure, fully anonymized string
+    that is globally unique to the machine on which it is generated.
+    The hash digest's component values are never stored in memory.
+
+    :param application: A bytestring corresponding to the fides tool currently in use. Consider `import`ing and using the constants available in this function's package.
+    :returns: The hash digest, as a hexadecimal string, to be used as the analytics identifier for the machine on which this function is executed.
+    """
+    # pylint: enable=line-too-long
+
+    return md5(uuid1().bytes + gensalt() + application).hexdigest()
diff --git a/requirements.txt b/requirements.txt
index 2430642..ccc3eb8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
+bcrypt~=3.2.0
 fastapi==0.74.0
 pydantic==1.9.0
+requests==2.27.1
 snowflake-sqlalchemy==1.3.3
 sqlalchemy==1.4.31
 uvicorn==0.17.5


Test patch:
diff --git a/tests/sdk/test_sdk.py b/tests/sdk/test_sdk.py
new file mode 100644
index 0000000..17a031c
--- /dev/null
+++ b/tests/sdk/test_sdk.py
@@ -0,0 +1,54 @@
+import pytest
+from datetime import datetime, timezone
+
+from fideslog.sdk.python import client, event
+
+
+API_KEY = "Token 12345"
+
+
+@pytest.fixture()
+def test_create_client():
+    yield client.AnalyticsClient(
+        api_key=API_KEY,
+        client_id="fake_client_id",
+        os="Darwin",
+        product_name="fideslog",
+        production_version="1.2.3",
+    )
+
+
+@pytest.fixture()
+def test_basic_additional_payload():
+    yield event.AnalyticsEvent(
+        event="test_event",
+        event_created_at=datetime.now(timezone.utc),
+    )
+
+
+@pytest.fixture()
+def test_rich_additional_payload():
+    yield event.AnalyticsEvent(
+        event="test_event",
+        event_created_at=datetime.now(timezone.utc),
+        command="test_command",
+        docker=True,
+        status_code=200,
+        extra_data={"extra": "data"},
+    )
+
+
+def test_create_client_attribute(test_create_client):
+    assert test_create_client.client_id == "fake_client_id"
+    assert test_create_client.extra_data == None
+
+
+def test_basic_event_payload(test_basic_additional_payload):
+    assert test_basic_additional_payload.event == "test_event"
+
+
+def test_rich_event_payload(test_rich_additional_payload):
+    assert test_rich_additional_payload.command == "test_command"
+    assert test_rich_additional_payload.docker
+    assert test_rich_additional_payload.status_code == 200
+    assert isinstance(test_rich_additional_payload.extra_data, dict)