Upload 195 files

steamship/__init__.py

@@ -0,0 +1,57 @@
+from importlib.metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError
+
+from .base import (
+    Configuration,
+    MimeTypes,
+    RuntimeEnvironments,
+    SteamshipError,
+    Task,
+    TaskState,
+    check_environment,
+)
+from .data import (
+    Block,
+    DocTag,
+    EmbeddingIndex,
+    File,
+    Package,
+    PackageInstance,
+    PackageVersion,
+    PluginInstance,
+    PluginVersion,
+    Tag,
+    Workspace,
+)
+
+from .client import Steamship  # isort:skip
+
+__all__ = [
+    "Steamship",
+    "Configuration",
+    "SteamshipError",
+    "MimeTypes",
+    "Package",
+    "PackageInstance",
+    "PackageVersion",
+    "File",
+    "Task",
+    "TaskState",
+    "Block",
+    "Tag",
+    "Workspace",
+    "PluginInstance",
+    "PluginVersion",
+    "DocTag",
+    "EmbeddingIndex",
+    "check_environment",
+    "RuntimeEnvironments",
+]

steamship/base/__init__.py

@@ -0,0 +1,15 @@
+from .configuration import Configuration
+from .environments import RuntimeEnvironments, check_environment
+from .error import SteamshipError
+from .mime_types import MimeTypes
+from .tasks import Task, TaskState
+
+__all__ = [
+    "Configuration",
+    "SteamshipError",
+    "Task",
+    "TaskState",
+    "MimeTypes",
+    "RuntimeEnvironments",
+    "check_environment",
+]

steamship/base/client.py

@@ -0,0 +1,635 @@
+from __future__ import annotations
+
+import logging
+import typing
+from abc import ABC
+from inspect import isclass
+from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union
+
+import inflection
+from pydantic import BaseModel, PrivateAttr
+from requests import Session
+
+from steamship.base.configuration import Configuration
+from steamship.base.error import SteamshipError
+from steamship.base.mime_types import MimeTypes
+from steamship.base.model import CamelModel, to_camel
+from steamship.base.request import Request
+from steamship.base.tasks import Task, TaskState
+from steamship.utils.url import Verb, is_local
+
+_logger = logging.getLogger(__name__)
+
+T = TypeVar("T")  # TODO (enias): Do we need this?
+
+
+def _multipart_name(path: str, val: Any) -> List[Tuple[Optional[str], str, Optional[str]]]:
+    """Decode any object into a series of HTTP Multi-part segments that Vapor will consume.
+
+    https://github.com/vapor/multipart-kit
+    When sending a JSON object in a MultiPart request, Vapor wishes to see multi part segments as follows:
+
+    single_key
+    array_key[idx]
+    obj_key[prop]
+
+    So a File with a list of one tag with kind=Foo would be transmitted as setting the part:
+    [tags][0][kind]
+    """
+    ret = []
+    if isinstance(val, dict):
+        for key, subval in val.items():
+            ret.extend(_multipart_name(f"{path}[{key}]", subval))
+    elif isinstance(val, list):
+        for idx, subval in enumerate(val):
+            ret.extend(_multipart_name(f"{path}[{idx}]", subval))
+    elif val is not None:
+        ret.append((path, val, None))
+    return ret
+
+
+class Client(CamelModel, ABC):
+    """Client model.py class.
+
+    Separated primarily as a hack to prevent circular imports.
+    """
+
+    config: Configuration
+    _session: Session = PrivateAttr()
+
+    def __init__(
+        self,
+        api_key: str = None,
+        api_base: str = None,
+        app_base: str = None,
+        web_base: str = None,
+        workspace: str = None,
+        fail_if_workspace_exists: bool = False,
+        profile: str = None,
+        config_file: str = None,
+        config: Configuration = None,
+        trust_workspace_config: bool = False,  # For use by lambda_handler; don't fetch the workspace
+        **kwargs,
+    ):
+        """Create a new client.
+
+        If `workspace` is provided, it will anchor the client in a workspace by that name, creating it if necessary.
+        Otherwise the `default` workspace will be used.
+        """
+        if config is not None and not isinstance(config, Configuration):
+            config = Configuration.parse_obj(config)
+
+        self._session = Session()
+        config = config or Configuration(
+            api_key=api_key,
+            api_base=api_base,
+            app_base=app_base,
+            web_base=web_base,
+            workspace_handle=workspace,
+            profile=profile,
+            config_file=config_file,
+        )
+
+        super().__init__(config=config)
+        # The lambda_handler will pass in the workspace via the workspace_id, so we need to plumb this through to make sure
+        # that the workspace switch performed doesn't mistake `workspace=None` as a request for the default workspace
+        self.switch_workspace(
+            workspace_handle=workspace or config.workspace_handle,
+            workspace_id=config.workspace_id,
+            fail_if_workspace_exists=fail_if_workspace_exists,
+            trust_workspace_config=trust_workspace_config,
+        )
+
+    def switch_workspace(  # noqa: C901
+        self,
+        workspace_handle: str = None,
+        workspace_id: str = None,
+        fail_if_workspace_exists: bool = False,
+        trust_workspace_config: bool = False,
+        # For use by lambda_handler; don't fetch the workspacetrust_workspace_config: bool = False, # For use by lambda_handler; don't fetch the workspace
+    ):
+        """Switches this client to the requested workspace, possibly creating it. If all arguments are None, the client
+        actively switches into the default workspace.
+
+        - API calls are performed manually to not result in circular imports.
+        - Note that the default workspace is technically not necessary for API usage; it will be assumed by the Engine
+          in the absense of a Workspace ID or Handle being manually specified in request headers.
+        """
+        workspace = None
+
+        if workspace_handle is None and workspace_id is None:
+            # Switch to the default workspace since no named or ID'ed workspace was provided
+            workspace_handle = "default"
+
+        if fail_if_workspace_exists:
+            logging.info(
+                f"[Client] Creating workspace with handle/id: {workspace_handle}/{workspace_id}."
+            )
+        else:
+            logging.info(
+                f"[Client] Creating/Fetching workspace with handle/id: {workspace_handle}/{workspace_id}."
+            )
+
+        # Zero out the workspace_handle on the config block in case we're being invoked from
+        # `init` (otherwise we'll attempt to create the space IN that non-existant workspace)
+        old_workspace_handle = self.config.workspace_handle
+        self.config.workspace_handle = None
+
+        if trust_workspace_config:
+            if workspace_handle is None or workspace_id is None:
+                raise SteamshipError(
+                    message="Attempted a trusted workspace switch without providing both workspace handle and workspace id."
+                )
+            return_id = workspace_id
+            return_handle = workspace_handle
+        else:
+            try:
+                if workspace_handle is not None and workspace_id is not None:
+                    get_params = {
+                        "handle": workspace_handle,
+                        "id": workspace_id,
+                        "fetchIfExists": False,
+                    }
+                    workspace = self.post("workspace/get", get_params)
+                elif workspace_handle is not None:
+                    get_params = {
+                        "handle": workspace_handle,
+                        "fetchIfExists": not fail_if_workspace_exists,
+                    }
+                    workspace = self.post("workspace/create", get_params)
+                elif workspace_id is not None:
+                    get_params = {"id": workspace_id}
+                    workspace = self.post("workspace/get", get_params)
+
+            except SteamshipError as e:
+                self.config.workspace_handle = old_workspace_handle
+                raise e
+
+            if workspace is None:
+                raise SteamshipError(
+                    message="Was unable to switch to new workspace: server returned empty Workspace."
+                )
+
+            return_id = workspace.get("workspace", {}).get("id")
+            return_handle = workspace.get("workspace", {}).get("handle")
+
+        if return_id is None or return_handle is None:
+            raise SteamshipError(
+                message="Was unable to switch to new workspace: server returned empty ID and Handle."
+            )
+
+        # Finally, set the new workspace.
+        self.config.workspace_id = return_id
+        self.config.workspace_handle = return_handle
+        logging.info(f"[Client] Switched to workspace {return_handle}/{return_id}")
+
+    def dict(self, **kwargs) -> dict:
+        # Because of the trick we do to hack these in as both static and member methods (with different
+        # implementations), Pydantic will try to include them by default. So we have to suppress that otherwise
+        # downstream serialization into JSON will fail.
+        if "exclude" not in kwargs:
+            kwargs["exclude"] = {
+                "use": True,
+                "use_plugin": True,
+                "_instance_use": True,
+                "_instance_use_plugin": True,
+                "config": {"api_key"},
+            }
+        elif isinstance(kwargs["exclude"], set):
+            kwargs["exclude"].add("use")
+            kwargs["exclude"].add("use_plugin")
+            kwargs["exclude"].add("_instance_use")
+            kwargs["exclude"].add("_instance_use_plugin")
+            kwargs["exclude"].add(
+                "config"
+            )  # the set version cannot exclude subcomponents; we must remove all config
+        elif isinstance(kwargs["exclude"], dict):
+            kwargs["exclude"]["use"] = True
+            kwargs["exclude"]["use_plugin"] = True
+            kwargs["exclude"]["_instance_use"] = True
+            kwargs["exclude"]["_instance_use_plugin"] = True
+            kwargs["exclude"]["config"] = {"api_key"}
+
+        return super().dict(**kwargs)
+
+    def _url(
+        self,
+        is_package_call: bool = False,
+        package_owner: str = None,
+        operation: str = None,
+    ):
+        if not is_package_call:
+            # Regular API call
+            base = self.config.api_base
+        else:
+            # Do the invocable version
+            if package_owner is None:
+                return SteamshipError(
+                    code="UserMissing",
+                    message="Cannot invoke a package endpoint without the package owner's user handle.",
+                    suggestion="Provide the package_owner option, or initialize your package with a lookup.",
+                )
+
+            base = self.config.app_base
+            if not is_local(base):
+                # We want to prepend the user handle
+                parts = base.split("//")
+                base = f"{parts[0]}//{package_owner}.{'//'.join(parts[1:])}"
+
+        # Clean leading and trailing "/"
+        if base[len(base) - 1] == "/":
+            base = base[:-1]
+        if operation[0] == "/":
+            operation = operation[1:]
+
+        return f"{base}/{operation}"
+
+    def _headers(  # noqa: C901
+        self,
+        is_package_call: bool = False,
+        package_owner: str = None,
+        package_id: str = None,
+        package_instance_id: str = None,
+        as_background_task: bool = False,
+        wait_on_tasks: List[Union[str, Task]] = None,
+    ):
+        headers = {"Authorization": f"Bearer {self.config.api_key.get_secret_value()}"}
+
+        if self.config.workspace_id:
+            headers["X-Workspace-Id"] = self.config.workspace_id
+        elif self.config.workspace_handle:
+            headers["X-Workspace-Handle"] = self.config.workspace_handle
+
+        if is_package_call:
+            if package_owner:
+                headers["X-Package-Owner-Handle"] = package_owner
+            if package_id:
+                headers["X-Package-Id"] = package_id
+            if package_instance_id:
+                headers["X-Package-Instance-Id"] = package_instance_id
+
+        if wait_on_tasks:
+            # Will result in the engine persisting the inbound HTTP request as a Task for deferred
+            # execution. Additionally, the task will be scheduled to first wait on the other tasks
+            # provided in the list of IDs. Accepts a list of EITHER Task objects OR task_id strings.
+            as_background_task = True
+            task_ids = []
+            for task_or_id in wait_on_tasks:
+                if isinstance(task_or_id, str):
+                    task_ids.append(task_or_id)
+                elif isinstance(task_or_id, Task):
+                    task_ids.append(task_or_id.task_id)
+                else:
+                    raise SteamshipError(
+                        message=f"`wait_on_tasks` should only contain Task or str objects. Got a {type(task_or_id)}."
+                    )
+
+            headers["X-Task-Dependency"] = ",".join(task_ids)
+
+        if as_background_task:
+            # Will result in the engine persisting the inbound HTTP request as a Task for deferred
+            # execution. The client will receive task information back instead of the synchronous API response.
+            # That task can be polled for eventual completion.
+            headers["X-Task-Background"] = "true"
+
+        return headers
+
+    @staticmethod
+    def _prepare_data(payload: Union[Request, dict]):
+        if payload is None:
+            data = {}
+        elif isinstance(payload, dict):
+            data = payload
+        elif isinstance(payload, BaseModel):
+            data = payload.dict(by_alias=True)
+        else:
+            raise RuntimeError(f"Unable to parse payload of type {type(payload)}")
+
+        return data
+
+    @staticmethod
+    def _response_data(resp, raw_response: bool = False):
+        if resp is None:
+            return None
+
+        if raw_response:
+            return resp.content
+
+        if resp.headers:
+            ct = None
+            if "Content-Type" in resp.headers:
+                ct = resp.headers["Content-Type"]
+            if "content-type" in resp.headers:
+                ct = resp.headers["content-type"]
+            if ct is not None:
+                ct = ct.split(";")[0]  # application/json; charset=utf-8
+                if ct in [MimeTypes.TXT, MimeTypes.MKD, MimeTypes.HTML]:
+                    return resp.text
+                elif ct == MimeTypes.JSON:
+                    return resp.json()
+                else:
+                    return resp.content
+
+    @staticmethod
+    def _prepare_multipart_data(data, file):
+        # Note: requests seems to have a bug passing boolean (and maybe numeric?)
+        # values in the midst of multipart form data. You need to manually convert
+        # it to a string; otherwise it will pass as False or True (with the capital),
+        # which is not standard notation outside of Python.
+        for key in data:
+            if data[key] is False:
+                data[key] = "false"
+            elif data[key] is True:
+                data[key] = "true"
+
+        result = {}
+        for key, val in data.items():
+            for t in _multipart_name(key, val):
+                result[t[0]] = t
+        result["file"] = file
+        return result
+
+    def _add_client_to_response(self, expect: Type, response_data: Any):
+        if isinstance(response_data, dict):
+            self._add_client_to_object(expect, response_data)
+        elif isinstance(response_data, list):
+            for el in response_data:
+                typing_parameters = typing.get_args(expect)
+                self._add_client_to_response(
+                    typing_parameters[0] if typing_parameters else None, el
+                )
+
+        return response_data
+
+    def _add_client_to_object(self, expect, response_data):
+        if expect and isclass(expect):
+            if len(response_data.keys()) == 1 and list(response_data.keys())[0] in (
+                to_camel(expect.__name__),
+                to_camel(expect.__name__).replace("package", "invocable"),
+                # Hack since engine uses "App" instead of "Package"
+                "index",
+                "pluginInstance",  # Inlined here since `expect` may be a subclass of pluginInstance
+            ):
+                # TODO (enias): Hack since the engine responds with incosistent formats e.g. {"plugin" : {plugin_fields}}
+                for _, v in response_data.items():
+                    self._add_client_to_response(expect, v)
+            elif issubclass(expect, BaseModel):
+                response_data["client"] = self
+                try:
+                    key_to_type = typing.get_type_hints(expect)
+                    for k, v in response_data.items():
+                        self._add_client_to_response(key_to_type.get(inflection.underscore(k)), v)
+                except NameError:
+                    # typing.get_type_hints fails for Workspace
+                    pass
+
+    def call(  # noqa: C901
+        self,
+        verb: Verb,
+        operation: str,
+        payload: Union[Request, dict] = None,
+        file: Any = None,
+        expect: Type[T] = None,
+        debug: bool = False,
+        raw_response: bool = False,
+        is_package_call: bool = False,
+        package_owner: str = None,
+        package_id: str = None,
+        package_instance_id: str = None,
+        as_background_task: bool = False,
+        wait_on_tasks: List[Union[str, Task]] = None,
+        timeout_s: Optional[float] = None,
+    ) -> Union[
+        Any, Task
+    ]:  # TODO (enias): I would like to list all possible return types using interfaces instead of Any
+        """Post to the Steamship API.
+
+        All responses have the format::
+
+        .. code-block:: json
+
+           {
+               "data": "<actual response>",
+               "error": {"reason": "<message>"}
+           } # noqa: RST203
+
+        For the Python client we return the contents of the `data` field if present, and we raise an exception
+        if the `error` field is filled in.
+        """
+        # TODO (enias): Review this codebase
+        url = self._url(
+            is_package_call=is_package_call,
+            package_owner=package_owner,
+            operation=operation,
+        )
+
+        headers = self._headers(
+            is_package_call=is_package_call,
+            package_owner=package_owner,
+            package_id=package_id,
+            package_instance_id=package_instance_id,
+            as_background_task=as_background_task,
+            wait_on_tasks=wait_on_tasks,
+        )
+
+        data = self._prepare_data(payload=payload)
+
+        logging.debug(
+            f"Making {verb} to {url} in workspace {self.config.workspace_handle}/{self.config.workspace_id}"
+        )
+        if verb == Verb.POST:
+            if file is not None:
+                files = self._prepare_multipart_data(data, file)
+                resp = self._session.post(url, files=files, headers=headers, timeout=timeout_s)
+            else:
+                resp = self._session.post(url, json=data, headers=headers, timeout=timeout_s)
+        elif verb == Verb.GET:
+            resp = self._session.get(url, params=data, headers=headers, timeout=timeout_s)
+        else:
+            raise Exception(f"Unsupported verb: {verb}")
+
+        logging.debug(f"From {verb} to {url} got HTTP {resp.status_code}")
+
+        if debug is True:
+            logging.debug(f"Got response {resp}")
+
+        response_data = self._response_data(resp, raw_response=raw_response)
+
+        logging.debug(f"Response JSON {response_data}")
+
+        task = None
+        error = None
+
+        if isinstance(response_data, dict):
+            if "status" in response_data:
+                try:
+                    task = Task.parse_obj(
+                        {**response_data["status"], "client": self, "expect": expect}
+                    )
+                    if "state" in response_data["status"]:
+                        if response_data["status"]["state"] == "failed":
+                            error = SteamshipError.from_dict(response_data["status"])
+                            logging.warning(f"Client received error from server: {error}")
+                except TypeError as e:
+                    # There's an edge case here -- if a Steamship package returns the JSON dictionary
+                    #
+                    # { "status": "status string" }
+                    #
+                    # Then the above handler will attempt to parse it and throw... But we don't actually want to throw
+                    # since we don't take a strong opinion on what the response type of a package endpoint ought to be.
+                    # It *may* choose to conform to the SteamshipResponse<T> type, but it doesn't have to.
+                    if not is_package_call:
+                        raise e
+
+                if task is not None and task.state == TaskState.failed:
+                    error = task.as_error()
+
+            if "data" in response_data:
+                if expect is not None:
+                    if issubclass(expect, SteamshipError):
+                        data = expect.from_dict({**response_data["data"], "client": self})
+                    elif issubclass(expect, BaseModel):
+                        data = expect.parse_obj(
+                            self._add_client_to_response(expect, response_data["data"])
+                        )
+                    else:
+                        raise RuntimeError(f"obj of type {expect} does not have a from_dict method")
+                else:
+                    data = response_data["data"]
+
+                if task:
+                    task.output = data
+            else:
+                data = response_data
+
+        else:
+            data = response_data
+
+        if error is not None:
+            logging.warning(f"Client received error from server: {error}", exc_info=error)
+            raise error
+
+        if not resp.ok:
+            raise SteamshipError(
+                f"API call did not complete successfully.  Server returned: {response_data}"
+            )
+
+        elif task is not None:
+            return task
+        elif data is not None and expect is not None:
+            # if we have data AND we expect it to be of a certain type,
+            # we should probably make sure that expectation is met.
+            if not isinstance(data, expect):
+                raise SteamshipError(
+                    message=f"Inconsistent response from server (data does not match expected type: {expect}.)",
+                    suggestion="Please contact support via hello@steamship.com and report what caused this error.",
+                )
+            return data
+        elif data is not None:
+            return data
+        else:
+            raise SteamshipError("Inconsistent response from server. Please contact support.")
+
+    def post(
+        self,
+        operation: str,
+        payload: Union[Request, dict, BaseModel] = None,
+        file: Any = None,
+        expect: Any = None,
+        debug: bool = False,
+        raw_response: bool = False,
+        is_package_call: bool = False,
+        package_owner: str = None,
+        package_id: str = None,
+        package_instance_id: str = None,
+        as_background_task: bool = False,
+        wait_on_tasks: List[Union[str, Task]] = None,
+        timeout_s: Optional[float] = None,
+    ) -> Union[
+        Any, Task
+    ]:  # TODO (enias): I would like to list all possible return types using interfaces instead of Any
+        return self.call(
+            verb=Verb.POST,
+            operation=operation,
+            payload=payload,
+            file=file,
+            expect=expect,
+            debug=debug,
+            raw_response=raw_response,
+            is_package_call=is_package_call,
+            package_owner=package_owner,
+            package_id=package_id,
+            package_instance_id=package_instance_id,
+            as_background_task=as_background_task,
+            wait_on_tasks=wait_on_tasks,
+            timeout_s=timeout_s,
+        )
+
+    def get(
+        self,
+        operation: str,
+        payload: Union[Request, dict] = None,
+        file: Any = None,
+        expect: Any = None,
+        debug: bool = False,
+        raw_response: bool = False,
+        is_package_call: bool = False,
+        package_owner: str = None,
+        package_id: str = None,
+        package_instance_id: str = None,
+        as_background_task: bool = False,
+        wait_on_tasks: List[Union[str, Task]] = None,
+        timeout_s: Optional[float] = None,
+    ) -> Union[
+        Any, Task
+    ]:  # TODO (enias): I would like to list all possible return types using interfaces instead of Any
+        return self.call(
+            verb=Verb.GET,
+            operation=operation,
+            payload=payload,
+            file=file,
+            expect=expect,
+            debug=debug,
+            raw_response=raw_response,
+            is_package_call=is_package_call,
+            package_owner=package_owner,
+            package_id=package_id,
+            package_instance_id=package_instance_id,
+            as_background_task=as_background_task,
+            wait_on_tasks=wait_on_tasks,
+            timeout_s=timeout_s,
+        )
+
+    def logs(
+        self,
+        offset: int = 0,
+        number: int = 50,
+        invocable_handle: Optional[str] = None,
+        instance_handle: Optional[str] = None,
+        invocable_version_handle: Optional[str] = None,
+        path: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Return generated logs for a client.
+
+        The logs will be workspace-scoped. You will only receive logs
+        for packages and plugins that you own.
+
+        :param offset: The index of the first log entry to return. This can be used with `number` to page through logs.
+        :param number: The number of log entries to return. This can be used with `offset` to page through logs.
+        :param invocable_handle: Enables optional filtering based on the handle of package or plugin. Example: `my-package`
+        :param instance_handle: Enables optional filtering based on the handle of package instance or plugin instance. Example: `my-instance`
+        :param invocable_version_handle: Enables optional filtering based on the version handle of package or plugin. Example: `0.0.2`
+        :param path: Enables optional filtering based on request path. Example: `/generate`.
+        :return: Returns a dictionary containing the offset and number of log entries as well as a list of `entries` that match the specificed filters.
+        """
+        args = {"from": offset, "size": number}
+        if invocable_handle:
+            args["invocableHandle"] = invocable_handle
+        if instance_handle:
+            args["invocableInstanceHandle"] = instance_handle
+        if invocable_version_handle:
+            args["invocableVersionHandle"] = invocable_version_handle
+        if path:
+            args["invocablePath"] = path
+
+        return self.post("logs/list", args)

steamship/base/configuration.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+import inflection
+from pydantic import HttpUrl, SecretStr
+
+from steamship.base.model import CamelModel
+from steamship.cli.login import login
+from steamship.utils.utils import format_uri
+
+DEFAULT_WEB_BASE = "https://steamship.com/"
+DEFAULT_APP_BASE = "https://steamship.run/"
+DEFAULT_API_BASE = "https://api.steamship.com/api/v1/"
+
+ENVIRONMENT_VARIABLES_TO_PROPERTY = {
+    "STEAMSHIP_API_KEY": "api_key",
+    "STEAMSHIP_API_BASE": "api_base",
+    "STEAMSHIP_APP_BASE": "app_base",
+    "STEAMSHIP_WEB_BASE": "web_base",
+    "STEAMSHIP_WORKSPACE_ID": "workspace_id",
+    "STEAMSHIP_WORKSPACE_HANDLE": "workspace_handle",
+}
+DEFAULT_CONFIG_FILE = Path.home() / ".steamship.json"
+
+# This stops us from including the `client` object in the dict() output, which is fine in a dict()
+# but explodes if that dict() is turned into JSON. Sadly the `exclude` option in Pydantic doesn't
+# cascade down nested objects, so we have to use this structure to catch all the possible combinations
+EXCLUDE_FROM_DICT = {
+    "client": True,
+    "blocks": {"__all__": {"client": True, "tags": {"__all__": {"client": True}}}},
+    "tags": {"__all__": {"client": True}},
+}
+
+
+class Configuration(CamelModel):
+    api_key: SecretStr
+    api_base: HttpUrl = DEFAULT_API_BASE
+    app_base: HttpUrl = DEFAULT_APP_BASE
+    web_base: HttpUrl = DEFAULT_WEB_BASE
+    workspace_id: str = None
+    workspace_handle: str = None
+    profile: Optional[str] = None
+
+    def __init__(
+        self,
+        config_file: Optional[Path] = None,
+        **kwargs,
+    ):
+        # First set the profile
+        kwargs["profile"] = profile = kwargs.get("profile") or os.getenv("STEAMSHIP_PROFILE")
+
+        # Then load configuration from a file if provided
+        config_dict = self._load_from_file(
+            config_file or DEFAULT_CONFIG_FILE,
+            profile,
+            raise_on_exception=config_file is not None,
+        )
+        config_dict.update(self._get_config_dict_from_environment())
+        kwargs.update({k: v for k, v in config_dict.items() if kwargs.get(k) is None})
+
+        kwargs["api_base"] = format_uri(kwargs.get("api_base"))
+        kwargs["app_base"] = format_uri(kwargs.get("app_base"))
+        kwargs["web_base"] = format_uri(kwargs.get("web_base"))
+
+        if not kwargs.get("api_key") and not kwargs.get("apiKey"):
+            api_key = login(
+                kwargs.get("api_base") or DEFAULT_API_BASE,
+                kwargs.get("web_base") or DEFAULT_WEB_BASE,
+            )
+            Configuration._save_api_key_to_file(
+                api_key, profile, config_file or DEFAULT_CONFIG_FILE
+            )
+            kwargs["api_key"] = api_key
+
+        super().__init__(**kwargs)
+
+    @staticmethod
+    def _load_from_file(
+        file: Path, profile: str = None, raise_on_exception: bool = False
+    ) -> Optional[dict]:
+        try:
+            with file.open() as f:
+                config_file = json.load(f)
+                if profile:
+                    if "profiles" not in config_file or profile not in config_file["profiles"]:
+                        raise RuntimeError(f"Profile {profile} requested but not found in {file}")
+                    config = config_file["profiles"][profile]
+                else:
+                    config = config_file
+                return {inflection.underscore(k): v for k, v in config.items()}
+        except FileNotFoundError:
+            if raise_on_exception:
+                raise Exception(f"Tried to load configuration file at {file} but it did not exist.")
+        except Exception as err:
+            if raise_on_exception:
+                raise err
+        return {}
+
+    @staticmethod
+    def _get_config_dict_from_environment():
+        """Overrides configuration with environment variables."""
+        return {
+            property_name: os.getenv(environment_variable_name, None)
+            for environment_variable_name, property_name in ENVIRONMENT_VARIABLES_TO_PROPERTY.items()
+            if environment_variable_name in os.environ
+        }
+
+    @staticmethod
+    def _save_api_key_to_file(new_api_key: Optional[str], profile: Optional[str], file_path: Path):
+        # Minimally rewrite config file, adding api key
+        try:
+            with file_path.open() as f:
+                config_file = json.load(f)
+            if profile:
+                if "profiles" not in config_file or profile not in config_file["profiles"]:
+                    raise RuntimeError(f"Could not update API key for {profile} in {file_path}")
+                config = config_file["profiles"][profile]
+            else:
+                config = config_file
+        except FileNotFoundError:
+            config_file = {}
+            config = config_file
+
+        config["apiKey"] = new_api_key
+
+        with file_path.open("w") as f:
+            json.dump(config_file, f, indent="\t")
+
+    @staticmethod
+    def default_config_file_has_api_key() -> bool:
+        return Configuration._load_from_file(DEFAULT_CONFIG_FILE).get("api_key") is not None
+
+    @staticmethod
+    def remove_api_key_from_default_config():
+        Configuration._save_api_key_to_file(None, None, DEFAULT_CONFIG_FILE)

steamship/base/environments.py

@@ -0,0 +1,99 @@
+import os
+from enum import Enum
+
+from steamship.base.configuration import Configuration
+from steamship.base.error import SteamshipError
+
+
+class RuntimeEnvironments(str, Enum):
+    REPLIT = "replit"
+    LOCALHOST = "localhost"
+
+
+def _interactively_get_key(env: RuntimeEnvironments):
+    print(
+        """Get your free API key here: https://steamship.com/account/api
+
+You'll get immediate access to our SDK for AI models, including OpenAI, GPT, Cohere, and more.
+"""
+    )
+
+    api_key = input("Paste your API key to run: ")
+
+    while len(api_key.strip()) == 0:
+        api_key = input("API Key: ")
+
+    os.environ["STEAMSHIP_API_KEY"] = api_key
+
+    if env == RuntimeEnvironments.REPLIT:
+        print(
+            """
+This key is set temporarily. In the future, you can:
+- Set the STEAMSHIP_API_KEY Replit Secret
+- Close and re-open any Replit shells to make sure secrets are refreshed.
+
+"""
+        )
+    elif env == RuntimeEnvironments.LOCALHOST:
+        print(
+            """
+This key is set temporarily. In the future, you can:
+- Set the STEAMSHIP_API_KEY environment variable
+- Run `ship login` to create a ~/.steamship.json credential file
+
+"""
+        )
+
+
+def _report_error_and_exit(env: RuntimeEnvironments):
+    if env == RuntimeEnvironments.REPLIT:
+        print(
+            """To run this Replit, you will need a Steamship API Key.
+
+1) If you're viewing someone else's Replit, clone it
+
+2) Visit https://steamship.com/account/api to get a key
+
+3) Add your key as a Replit secret named STEAMSHIP_API_KEY
+
+4) Close and re-open any shells to make sure your new secret is available
+
+Then try running again!"""
+        )
+    elif env == RuntimeEnvironments.LOCALHOST:
+        print(
+            """To run this script, you will need a Steamship API Key.
+
+1) Visit https://steamship.com/account/api to get a key
+
+2) Set your key as the environment variable STEAMSHIP_API_KEY
+
+Then try running again!
+
+If you have pip-installed `steamship`, you can also try setting your key by simply running `ship login`.
+"""
+        )
+    exit(-1)
+
+
+def check_environment(env: RuntimeEnvironments, interactively_set_key: bool = True):
+    # This will try loading from STEAMSHIP_API_KEY and also ~/.steamship.json
+    try:
+        config = Configuration()
+
+        # If an API key is set, we're good to go!
+        if config.api_key:
+            return
+    except SteamshipError:
+        # The Configuration object will throw an error if there is no API Key found.
+        # Since that error is expected from the context of this function, we pass on it to handle it in a more
+        # user-interactive way.
+        pass
+
+    # If we're hot-loading config, do it here!
+    if interactively_set_key:
+        _interactively_get_key(env)
+        return
+
+    # If we're still here, we're not interactively setting the key. Display an error message and exit.
+    _report_error_and_exit(env)

steamship/base/error.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Union
+
+DEFAULT_ERROR_MESSAGE = "Undefined remote error"
+
+
+class SteamshipError(Exception):
+    message: str = None
+    internal_message: str = None
+    suggestion: str = None
+    code: str = None
+    error: str = None
+
+    def __init__(
+        self,
+        message: str = DEFAULT_ERROR_MESSAGE,
+        internal_message: str = None,
+        suggestion: str = None,
+        code: str = None,
+        error: Union[Exception, str] = None,
+    ):
+        super().__init__()
+        self.message = message
+        self.suggestion = suggestion
+        self.internal_message = internal_message
+        self.code = code
+        if error is not None:
+            self.error = str(error)
+
+        parts = []
+        if code is not None:
+            parts.append(f"[{code}]")
+        if message is not None:
+            parts.append(message)
+        if internal_message is not None:
+            parts.append(f"Internal Message: {internal_message}")
+        if suggestion is not None:
+            parts.append(f"Suggestion: {suggestion}")
+
+        super().__init__("\n".join(parts))
+
+    def log(self):
+        logging.error(
+            f"[{self.code}] {self.message}. [Internal: {self.internal_message}] [Suggestion: {self.suggestion}]",
+            exc_info=self,
+        )
+        if self.error:
+            logging.error(self.error)
+
+    def to_dict(self) -> dict:
+        # Required since Exception cannot be combined with pydantic.BaseModel
+        return {
+            "message": self.message,
+            "internalMessage": self.internal_message,
+            "suggestion": self.suggestion,
+            "code": self.code,
+            "error": self.error,
+        }
+
+    @staticmethod
+    def from_dict(d: Any) -> SteamshipError:
+        """Last resort if subclass doesn't override: pass through."""
+        # Required since Exception cannot be combined with pydantic.BaseModel
+        return SteamshipError(
+            message=d.get("statusMessage", d.get("message")),
+            internal_message=d.get("internalMessage"),
+            suggestion=d.get("statusSuggestion", d.get("suggestion")),
+            code=d.get("statusCode", d.get("code")),
+            error=d.get("error", d.get("error")),
+        )

steamship/base/mime_types.py

@@ -0,0 +1,42 @@
+from enum import Enum
+
+
+class MimeTypes(str, Enum):
+    UNKNOWN = "unknown"
+    TXT = "text/plain"
+    JSON = "application/json"
+    MKD = "text/markdown"
+    EPUB = "application/epub+zip"
+    PDF = "application/pdf"
+    JPG = "image/jpeg"
+    PNG = "image/png"
+    TIFF = "image/tiff"
+    GIF = "image/gif"
+    HTML = "text/html"
+    DOC = "application/msword"
+    DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    PPT = "applicatino/ms-powerpoint"
+    PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+    RTF = "application/rtf"
+    BINARY = "application/octet-stream"
+    STEAMSHIP_BLOCK_JSON = "application/vnd.steamship-block.json.v1"
+    WAV = "audio/wav"
+    MP3 = "audio/mp3"
+    MP4_VIDEO = "video/mp4"
+    MP4_AUDIO = "audio/mp4"
+    WEBM_VIDEO = "video/webm"
+    WEBM_AUDIO = "audio/webm"
+    FILE_JSON = "fileJson"
+
+
+class ContentEncodings:
+    BASE64 = "base64"
+
+
+TEXT_MIME_TYPES = [
+    MimeTypes.TXT,
+    MimeTypes.MKD,
+    MimeTypes.HTML,
+    MimeTypes.DOCX,
+    MimeTypes.PPTX,
+]

steamship/base/model.py

@@ -0,0 +1,29 @@
+import re
+from typing import TypeVar
+
+import inflection
+from pydantic import BaseModel
+from pydantic.generics import GenericModel
+
+T = TypeVar("T")  # Declare type variable
+
+
+def to_camel(s: str) -> str:
+    s = re.sub("_(url)$", lambda m: f"_{m.group(1).upper()}", s)
+    return inflection.camelize(s, uppercase_first_letter=False)
+
+
+class CamelModel(BaseModel):
+    def __init__(self, **kwargs):
+        kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        super().__init__(**kwargs)
+
+    class Config:
+        alias_generator = to_camel
+        allow_population_by_field_name = True
+        # Populate enum values with their value, rather than the raw enum. Important to serialise model.dict()
+        use_enum_values = True
+
+
+class GenericCamelModel(CamelModel, GenericModel):
+    pass

steamship/base/package_spec.py

@@ -0,0 +1,150 @@
+"""Objects for recording and reporting upon the introspected interface of a Steamship Package."""
+import inspect
+from enum import Enum
+from typing import Dict, List, Optional, Union, get_args, get_origin
+
+from steamship import SteamshipError
+from steamship.base.configuration import CamelModel
+from steamship.utils.url import Verb
+
+
+class ArgSpec(CamelModel):
+    """An argument passed to a method."""
+
+    # The name of the argument.
+    name: str
+    # The kind of the argument, reported by str(annotation) via the `inspect` library. E.g. <class 'int'>
+    kind: str
+    # Possible values, if the kind is an enum type
+    values: Optional[List[str]]
+
+    def __init__(self, name: str, parameter: inspect.Parameter):
+        if name == "self":
+            raise SteamshipError(
+                message="Attempt to interpret the `self` object as a method parameter."
+            )
+        values = None
+        if isinstance(parameter.annotation, type):
+            if issubclass(parameter.annotation, Enum):
+                values = [choice.value for choice in parameter.annotation]
+        elif get_origin(parameter.annotation) is Union:
+            args = get_args(parameter.annotation)
+            # For now, only deal with the case where the Union is an Optional[Enum]
+            if len(args) == 2 and type(None) in args:
+                optional_arg = [t for t in args if t != type(None)][0]  # noqa: E721
+                if issubclass(optional_arg, Enum):
+                    values = [choice.value for choice in optional_arg]
+
+        super().__init__(name=name, kind=str(parameter.annotation), values=values)
+
+    def pprint(self, name_width: Optional[int] = None, prefix: str = "") -> str:
+        """Returns a pretty printable representation of this argument."""
+        width = name_width or len(self.name)
+        ret = f"{prefix}{self.name.ljust(width)} - {self.kind}"
+        return ret
+
+
+class MethodSpec(CamelModel):
+    """A method, callable remotely, on an object."""
+
+    # The HTTP Path at which the method is callable.
+    path: str
+
+    # The HTTP Verb at which the method is callable. Defaults to POST
+    verb: str
+
+    # The return type. Reported by str(annotation) via the `inspect` library. E.g. <class 'int'>
+    returns: str
+
+    # The docstring of the method.
+    doc: Optional[str] = None
+
+    # The named arguments of the method. Positional arguments are not permitted.
+    args: Optional[List[ArgSpec]] = None
+
+    # Additional configuration around this endpoint.
+    # Note: The actual type of this is Optional[Dict[str, Union[str, bool, int, float]]]
+    #       But if Pydantic sees that, it attempts to force all values to be str, which is wrong.
+    config: Optional[Dict] = None
+
+    @staticmethod
+    def clean_path(path: str = "") -> str:
+        """Ensure that the path always starts with /, and at minimum must be at least /."""
+        if not path:
+            path = "/"
+        elif path[0] != "/":
+            path = f"/{path}"
+        return path
+
+    def __init__(
+        self,
+        cls: object,
+        name: str,
+        path: str = None,
+        verb: Verb = Verb.POST,
+        config: Dict[str, Union[str, bool, int, float]] = None,
+    ):
+        # Set the path
+        if path is None and name is not None:
+            path = f"/{name}"
+        path = MethodSpec.clean_path(path)
+
+        # Get the function on the class so that we can inspect it
+        func = getattr(cls, name)
+        sig = inspect.signature(func)
+
+        # Set the return type
+        returns = str(sig.return_annotation)
+
+        # Set the docstring
+        doc = func.__doc__
+
+        # Set the arguments
+        args = []
+        for p in sig.parameters:
+            if p == "self":
+                continue
+            args.append(ArgSpec(p, sig.parameters[p]))
+
+        super().__init__(path=path, verb=verb, returns=returns, doc=doc, args=args, config=config)
+
+    def pprint(self, name_width: Optional[int] = None, prefix: str = "  ") -> str:
+        """Returns a pretty printable representation of this method."""
+
+        width = name_width or len(self.path)
+        ret = f"{self.verb.ljust(4)} {self.path.lstrip('/').ljust(width)} -> {self.returns}"
+        if self.args:
+            name_width = max([(len(arg.name) if arg.name else 0) for arg in self.args])
+            for arg in self.args:
+                arg_doc_string = arg.print(name_width, prefix)
+                ret += f"\n{arg_doc_string}"
+        return ret
+
+
+class PackageSpec(CamelModel):
+    """A package, representing a remotely instantiable service."""
+
+    # The name of the package
+    name: str
+
+    # The docstring of the package
+    doc: Optional[str] = None
+
+    # The list of methods the package exposes remotely
+    methods: Optional[List[MethodSpec]] = None
+
+    def pprint(self, prefix: str = "  ") -> str:
+        """Returns a pretty printable representation of this package."""
+        underline = "=" * len(self.name)
+        ret = f"{self.name}\n{underline}\n"
+        if self.doc:
+            ret += f"{self.doc}\n\n"
+        else:
+            ret += "\n"
+
+        if self.methods:
+            name_width = max([len(method.path) or 0 for method in self.methods])
+            for method in self.methods:
+                method_doc_string = method.pprint(name_width, prefix)
+                ret += f"\n{method_doc_string}"
+        return ret

steamship/base/request.py

@@ -0,0 +1,33 @@
+from steamship.base.model import CamelModel
+
+
+class Request(CamelModel):
+    pass
+
+
+class GetRequest(Request):
+    id: str = None
+    handle: str = None
+
+
+class CreateRequest(Request):
+    id: str = None
+    handle: str = None
+
+
+class UpdateRequest(Request):
+    id: str = None
+    handle: str = None
+
+
+class IdentifierRequest(Request):
+    id: str = None
+    handle: str = None
+
+
+class ListRequest(Request):
+    pass
+
+
+class DeleteRequest(Request):
+    id: str

steamship/base/response.py

@@ -0,0 +1,5 @@
+from steamship.base.model import CamelModel
+
+
+class Response(CamelModel):
+    pass

steamship/base/tasks.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Callable, Dict, Generic, List, Optional, Set, Type, TypeVar
+
+from pydantic import BaseModel, Field
+
+from steamship.base.error import SteamshipError
+from steamship.base.model import CamelModel, GenericCamelModel
+from steamship.base.request import DeleteRequest, IdentifierRequest, Request
+from steamship.utils.metadata import metadata_to_str, str_to_metadata
+
+T = TypeVar("T")
+
+
+class CreateTaskCommentRequest(Request):
+    task_id: str
+    external_id: str = None
+    external_type: str = None
+    external_group: str = None
+    metadata: str = None
+
+
+class ListTaskCommentRequest(Request):
+    task_id: str = None
+    external_id: str = None
+    external_type: str = None
+    external_group: str = None
+
+
+class TaskComment(CamelModel):
+    client: Client = Field(None, exclude=True)
+    id: str = None
+    user_id: str = None
+    task_id: str = None
+    external_id: str = None
+    external_type: str = None
+    external_group: str = None
+    metadata: Any = None
+    created_at: str = None
+
+    def __init__(self, **kwargs):
+        kwargs["metadata"] = str_to_metadata(kwargs.get("metadata"))
+        super().__init__(**kwargs)
+
+    @classmethod
+    def parse_obj(cls: Type[BaseModel], obj: Any) -> BaseModel:
+        # TODO (enias): This needs to be solved at the engine side
+        obj = obj["taskComment"] if "taskComment" in obj else obj
+        return super().parse_obj(obj)
+
+    @staticmethod
+    def create(
+        client: Client,
+        task_id: str = None,
+        external_id: str = None,
+        external_type: str = None,
+        external_group: str = None,
+        metadata: Any = None,
+    ) -> TaskComment:
+        req = CreateTaskCommentRequest(
+            taskId=task_id,
+            external_id=external_id,
+            external_type=external_type,
+            externalGroup=external_group,
+            metadata=metadata_to_str(metadata),
+        )
+        return client.post(
+            "task/comment/create",
+            req,
+            expect=TaskComment,
+        )
+
+    @staticmethod
+    def list(
+        client: Client,
+        task_id: str = None,
+        external_id: str = None,
+        external_type: str = None,
+        external_group: str = None,
+    ) -> TaskCommentList:
+        req = ListTaskCommentRequest(
+            taskId=task_id,
+            external_id=external_id,
+            external_type=external_type,
+            externalGroup=external_group,
+        )
+        return client.post(
+            "task/comment/list",
+            req,
+            expect=TaskCommentList,
+        )
+
+    def delete(self) -> TaskComment:
+        req = DeleteRequest(id=self.id)
+        return self.client.post(
+            "task/comment/delete",
+            req,
+            expect=TaskComment,
+        )
+
+
+class TaskCommentList(CamelModel):
+    comments: List[TaskComment]
+
+
+class TaskState:
+    waiting = "waiting"
+    running = "running"
+    succeeded = "succeeded"
+    failed = "failed"
+
+
+class TaskType:
+    internal_api = "internalApi"
+    train = "train"
+    infer = "infer"
+
+
+class TaskRunRequest(Request):
+    task_id: str
+
+
+class TaskStatusRequest(Request):
+    task_id: str
+
+
+class Task(GenericCamelModel, Generic[T]):
+    """Encapsulates a unit of asynchronously performed work."""
+
+    # Note: The Field object prevents this from being serialized into JSON (and causing a crash)
+    client: Client = Field(None, exclude=True)  # Steamship client
+
+    task_id: str = None  # The id of this task
+    user_id: str = None  # The user who requested this task
+    workspace_id: str = None  # The workspace in which this task is executing
+
+    # Note: The Field object prevents this from being serialized into JSON (and causing a crash)
+    expect: Type = Field(
+        None, exclude=True
+    )  # Type of the expected output once the output is complete.
+
+    input: str = None  # The input provided to the task
+    output: T = None  # The output of the task
+    state: str = None  # A value in class TaskState
+
+    status_message: str = None  # User-facing message concerning task status
+    status_suggestion: str = None  # User-facing suggestion concerning error remediation
+    status_code: str = None  # User-facing error code for support assistance
+    status_created_on: str = None  # When the status fields were last set
+
+    task_type: str = None  # A value in class TaskType; for internal routing
+    task_executor: str = None  #
+    task_created_on: str = None  # When the task object was created
+    task_last_modified_on: str = None  # When the task object was last modified
+
+    # Long Running Plugin Support
+    # The `remote_status_*` fields govern how Steamship Plugins can communicate long-running work back to the engine.
+    # If instead of sending data, the plugin sends a status with these fields set, the engine will begin polling for
+    # updates, echoing the contents of these fields back to the plugin to communicate, e.g., the jobId of the work
+    # being checked. When the work is complete, simply respond with the Response `data` field set as per usual.
+    remote_status_input: Optional[
+        Dict
+    ] = None  # For re-hydrating state in order to check remote status.
+    remote_status_output: Optional[
+        Dict
+    ] = None  # For reporting structured JSON state for error diagnostics.
+    remote_status_message: str = None  # User facing message string to report on remote status.
+
+    assigned_worker: str = None  # The worker assigned to complete this task
+    started_at: str = None  # When the work on this task began
+
+    max_retries: int = None  # The maximum number of retries allowed for this task
+    retries: int = None  # The number of retries already used.
+
+    def as_error(self) -> SteamshipError:
+        return SteamshipError(
+            message=self.status_message, suggestion=self.status_suggestion, code=self.status_code
+        )
+
+    @classmethod
+    def parse_obj(cls: Type[BaseModel], obj: Any) -> Task:
+        obj = obj["task"] if "task" in obj else obj
+        return super().parse_obj(obj)
+
+    @staticmethod
+    def get(
+        client,
+        _id: str = None,
+        handle: str = None,
+    ) -> Task:
+        return client.post(
+            "task/get",
+            IdentifierRequest(id=_id, handle=handle),
+            expect=Task,
+        )
+
+    def update(self, other: Optional[Task] = None):
+        """Incorporates a `Task` into this object."""
+        other = other or Task()
+        for k, v in other.__dict__.items():
+            self.__dict__[k] = v
+
+    def add_comment(
+        self,
+        external_id: str = None,
+        external_type: str = None,
+        external_group: str = None,
+        metadata: Any = None,
+    ) -> TaskComment:
+        return TaskComment.create(
+            client=self.client,
+            task_id=self.task_id,
+            external_id=external_id,
+            external_type=external_type,
+            external_group=external_group,
+            metadata=metadata,
+        )
+
+    def post_update(self, fields: Set[str] = None) -> Task:
+        """Updates this task in the Steamship Engine."""
+        if not isinstance(fields, set):
+            raise RuntimeError(f'Unexpected type of "fields": {type(fields)}. Expected type set.')
+        body = self.dict(by_alias=True, include={*fields, "task_id"})
+        return self.client.post("task/update", body, expect=Task)
+
+    def wait(
+        self,
+        max_timeout_s: float = 180,
+        retry_delay_s: float = 1,
+        on_each_refresh: "Optional[Callable[[int, float, Task], None]]" = None,
+    ):
+        """Polls and blocks until the task has succeeded or failed (or timeout reached).
+
+        Parameters
+        ----------
+        max_timeout_s : int
+            Max timeout in seconds. Default: 180s. After this timeout, an exception will be thrown.
+        retry_delay_s : float
+            Delay between status checks. Default: 1s.
+        on_each_refresh : Optional[Callable[[int, float, Task], None]]
+            Optional call back you can get after each refresh is made, including success state refreshes.
+            The signature represents: (refresh #, total elapsed time, task)
+
+            WARNING: Do not pass a long-running function to this variable. It will block the update polling.
+        """
+        t0 = time.perf_counter()
+        refresh_count = 0
+        while time.perf_counter() - t0 < max_timeout_s and self.state not in (
+            TaskState.succeeded,
+            TaskState.failed,
+        ):
+            time.sleep(retry_delay_s)
+            self.refresh()
+            refresh_count += 1
+
+            # Possibly make a callback so the caller knows we've tried again
+            if on_each_refresh:
+                on_each_refresh(refresh_count, time.perf_counter() - t0, self)
+
+        # If the task did not complete within the timeout, throw an error
+        if self.state not in (TaskState.succeeded, TaskState.failed):
+            raise SteamshipError(
+                message=f"Task {self.task_id} did not complete within requested timeout of {max_timeout_s}s. The task is still running on the server. You can retrieve its status via Task.get() or try waiting again with wait()."
+            )
+
+    def refresh(self):
+        if self.task_id is None:
+            raise SteamshipError(message="Unable to refresh task because `task_id` is None")
+
+        req = TaskStatusRequest(taskId=self.task_id)
+        # TODO (enias): A status call can return both data and task
+        # In this case both task and data will include the output (one is string serialized, the other is parsed)
+        # Ideally task status only returns the status, not the full output object
+        resp = self.client.post("task/status", payload=req, expect=self.expect)
+        self.update(resp)
+
+
+from .client import Client  # noqa: E402
+
+Task.update_forward_refs()
+TaskComment.update_forward_refs()

steamship/cli/cli.py

@@ -0,0 +1,195 @@
+import json
+import logging
+import sys
+import time
+from os import path
+from typing import Optional
+
+import click
+
+import steamship
+from steamship import Steamship, SteamshipError
+from steamship.base.configuration import Configuration
+from steamship.cli.deploy import (
+    PackageDeployer,
+    PluginDeployer,
+    bundle_deployable,
+    update_config_template,
+)
+from steamship.cli.manifest_init_wizard import manifest_init_wizard
+from steamship.cli.requirements_init_wizard import requirements_init_wizard
+from steamship.cli.ship_spinner import ship_spinner
+from steamship.data.manifest import DeployableType, Manifest
+from steamship.data.user import User
+
+
+@click.group()
+def cli():
+    pass
+
+
+def initialize(suppress_message: bool = False):
+    logging.root.setLevel(logging.FATAL)
+    if not suppress_message:
+        click.echo(f"Steamship PYTHON cli version {steamship.__version__}")
+
+
+@click.command()
+def login():
+    """Log in to Steamship, creating ~/.steamship.json"""
+    initialize()
+    click.echo("Logging into Steamship.")
+    if sys.argv[1] == "login":
+        if Configuration.default_config_file_has_api_key():
+            overwrite = click.confirm(
+                text="You already have an API key in your .steamship.json file.  Do you want to remove it and login?",
+                default=False,
+            )
+            if not overwrite:
+                sys.exit(0)
+            Configuration.remove_api_key_from_default_config()
+
+        # Carry on with login
+        client = Steamship()
+        user = User.current(client)
+        click.secho(f"🚢🚢🚢 Hooray! You're logged in with user handle: {user.handle} 🚢🚢🚢", fg="green")
+
+
+@click.command()
+def ships():
+    """Ship some ships"""
+    initialize()
+    click.secho("Here are some ships:", fg="cyan")
+    with ship_spinner():
+        time.sleep(5)
+    click.secho()
+
+
+@click.command()
+def deploy():
+    """Deploy the package or plugin in this directory"""
+    initialize()
+    client = None
+    try:
+        client = Steamship()
+    except SteamshipError as e:
+        click.secho(e.message, fg="red")
+        click.get_current_context().abort()
+
+    user = User.current(client)
+    if path.exists("steamship.json"):
+        manifest = Manifest.load_manifest()
+    else:
+        manifest = manifest_init_wizard(client)
+        manifest.save()
+
+    if not path.exists("requirements.txt"):
+        requirements_init_wizard()
+
+    deployable_type = manifest.type
+
+    update_config_template(manifest)
+
+    deployer = None
+    if deployable_type == DeployableType.PACKAGE:
+        deployer = PackageDeployer()
+    elif deployable_type == DeployableType.PLUGIN:
+        deployer = PluginDeployer()
+    else:
+        click.secho("Deployable must be of type package or plugin.", fg="red")
+        click.get_current_context().abort()
+
+    deployable = deployer.create_or_fetch_deployable(client, user, manifest)
+
+    click.echo("Bundling content... ", nl=False)
+    bundle_deployable(manifest)
+    click.echo("Done. 📦")
+
+    _ = deployer.create_version(client, manifest, deployable.id)
+
+    thing_url = f"{client.config.web_base}{deployable_type}s/{manifest.handle}"
+    click.echo(
+        f"Deployment was successful. View and share your new {deployable_type} here:\n\n{thing_url}\n"
+    )
+
+    # Common error conditions:
+    # - Package/plugin handle already taken. [handled; asks user for new]
+    # - Version handle already deployed. [handled; asks user for new]
+    # - Bad parameter configuration. [mitigated by deriving template from Config object]
+    # - Package content fails health check (ex. bad import) [Error caught while checking config object]
+
+
+@click.command()
+@click.option(
+    "--workspace",
+    "-w",
+    required=True,
+    type=str,
+    help="Workspace handle used for scoping logs request. All requests MUST be scoped by workspace.",
+)
+@click.option(
+    "--offset",
+    "-o",
+    default=0,
+    type=int,
+    help="Starting index from sorted logs to return a chunk. Used for paging. Defaults to 0.",
+)
+@click.option(
+    "--number",
+    "-n",
+    default=50,
+    type=int,
+    help="Number of logs to return in a single batch. Defaults to 50.",
+)
+@click.option(
+    "--package",
+    "-p",
+    type=str,
+    help="Package handle. Used to filter logs returend to a specific package (across all instances).",
+)
+@click.option(
+    "--instance",
+    "-i",
+    type=str,
+    help="Instance handle. Used to filter logs returned to a specific instance of a package.",
+)
+@click.option(
+    "--version",
+    "-v",
+    type=str,
+    help="Version handle. Used to filter logs returned to a specific version of a package.",
+)
+@click.option(
+    "--path",
+    "request_path",
+    type=str,
+    help="Path invoked by a client operation. Used to filter logs returned to a specific invocation path.",
+)
+def logs(
+    workspace: str,
+    offset: int,
+    number: int,
+    package: Optional[str] = None,
+    instance: Optional[str] = None,
+    version: Optional[str] = None,
+    request_path: Optional[str] = None,
+):
+    initialize(suppress_message=True)
+    client = None
+    try:
+        client = Steamship(workspace=workspace)
+    except SteamshipError as e:
+        raise click.UsageError(message=e.message)
+
+    click.echo(json.dumps(client.logs(offset, number, package, instance, version, request_path)))
+
+
+cli.add_command(login)
+cli.add_command(deploy)
+cli.add_command(deploy, name="it")
+cli.add_command(ships)
+cli.add_command(logs)
+
+
+if __name__ == "__main__":
+    deploy([])

steamship/cli/deploy.py

@@ -0,0 +1,285 @@
+import importlib.machinery as machinery
+import os
+import sys
+import traceback
+import zipfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+import click
+from semver import VersionInfo
+
+from steamship import Package, PackageVersion, PluginVersion, Steamship, SteamshipError
+from steamship.cli.manifest_init_wizard import validate_handle, validate_version_handle
+from steamship.cli.ship_spinner import ship_spinner
+from steamship.data import Plugin
+from steamship.data.manifest import Manifest
+from steamship.data.user import User
+from steamship.invocable.lambda_handler import get_class_from_module
+
+DEFAULT_BUILD_IGNORE = [
+    "build",
+    ".git",
+    ".venv",
+    ".ipynb_checkpoints",
+    ".DS_Store",
+    "venv",
+    "tests",
+    "examples",
+    ".idea",
+    "__pycache__",
+]
+
+
+def update_config_template(manifest: Manifest):
+
+    path = Path("src/api.py")
+    if not path.exists():
+        path = Path("api.py")
+        if not path.exists():
+            raise SteamshipError("Could not find api.py either in root directory or in src.")
+
+    api_module = None
+    try:
+        sys.path.append(str(path.parent.absolute()))
+
+        # load the API module to allow config inspection / generation
+        api_module = machinery.SourceFileLoader("api", str(path)).load_module()
+    except Exception:
+        click.secho(
+            "An error occurred while loading your api.py to check configuration parameters. Full stack trace below.",
+            fg="red",
+        )
+        traceback.print_exc()
+        click.get_current_context().abort()
+
+    invocable_type = get_class_from_module(api_module)
+
+    config_parameters = invocable_type.config_cls().get_config_parameters()
+
+    if manifest.configTemplate != config_parameters:
+        if len(config_parameters) > 0:
+            click.secho("Config parameters changed; updating steamship.json.", fg="cyan")
+            for param_name, param in config_parameters.items():
+                click.echo(f"{param_name}:")
+                click.echo(f"\tType: {param.type}")
+                click.echo(f"\tDefault: {param.default}")
+                click.echo(f"\tDescription: {param.description}")
+        else:
+            click.secho("Config parameters removed; updating steamship.json.", fg="cyan")
+
+    manifest.configTemplate = config_parameters
+    manifest.save()
+
+
+def get_archive_path(manifest: Manifest) -> Path:
+    return Path(".") / "build" / "archives" / f"{manifest.handle}_v{manifest.version}.zip"
+
+
+def bundle_deployable(manifest: Manifest):
+    archive_path = get_archive_path(manifest)
+    archive_path.parent.mkdir(parents=True, exist_ok=True)
+    excludes = DEFAULT_BUILD_IGNORE + manifest.build_config.get("ignore", [])
+
+    archive_path.unlink(missing_ok=True)
+
+    # This zipfile packaging is modeled after the typescript CLI.
+    # Items in non-excluded root folders are moved to the top-level.
+
+    with zipfile.ZipFile(
+        file=archive_path, mode="a", compression=zipfile.ZIP_DEFLATED, allowZip64=False
+    ) as zip_file:
+
+        root = Path(".")
+        for file_path in root.iterdir():
+            if file_path.name not in excludes:
+                if file_path.is_dir():
+                    for directory, _, files in os.walk(file_path):
+                        subdirectory_path = Path(directory)
+                        if Path(directory).name not in excludes:
+                            for file in files:
+                                pypi_file = subdirectory_path / file
+                                relative_to = pypi_file.relative_to(file_path)
+                                zip_file.write(pypi_file, relative_to)
+
+                else:
+                    zip_file.write(file_path)
+
+
+class DeployableDeployer(ABC):
+    @abstractmethod
+    def _create_version(self, client: Steamship, manifest: Manifest, thing_id: str):
+        pass
+
+    @abstractmethod
+    def create_object(self, client: Steamship, manifest: Manifest):
+        pass
+
+    @abstractmethod
+    def update_object(self, deployable, client: Steamship, manifest: Manifest):
+        pass
+
+    @abstractmethod
+    def deployable_type(self):
+        pass
+
+    def create_or_fetch_deployable(self, client: Steamship, user: User, manifest: Manifest):
+        if not manifest.handle or len(manifest.handle) == 0:
+            self.ask_for_new_handle(manifest, was_missing=True)
+
+        deployable = None
+        while deployable is None:
+            click.echo(
+                f"Creating / fetching {self.deployable_type()} with handle [{manifest.handle}]... ",
+                nl=False,
+            )
+            try:
+                deployable = self.create_object(client, manifest)
+                if deployable.user_id != user.id:
+                    self.ask_for_new_handle(manifest)
+                    deployable = None
+            except SteamshipError as e:
+                if e.message == "Something went wrong.":
+                    self.ask_for_new_handle(manifest)
+                else:
+                    click.secho(
+                        f"Unable to create / fetch {self.deployable_type()}. Server returned message: {e.message}"
+                    )
+                    click.get_current_context().abort()
+
+        self.update_object(deployable, client, manifest)
+
+        click.echo("Done.")
+        return deployable
+
+    def ask_for_new_handle(self, manifest: Manifest, was_missing: bool = False):
+        if not was_missing:
+            try_again = click.confirm(
+                click.style(
+                    f"\nIt looks like that handle [{manifest.handle}] is already in use. Would you like to change the handle and try again?",
+                    fg="yellow",
+                ),
+                default=True,
+            )
+            if not try_again:
+                click.get_current_context().abort()
+
+        new_handle = click.prompt(
+            f"What handle would you like to use for your {self.deployable_type()}? Valid characters are a-z and -",
+            value_proc=validate_handle,
+        )
+        manifest.handle = new_handle
+        manifest.save()
+
+    def create_version(self, client: Steamship, manifest: Manifest, thing_id: str):
+        version = None
+
+        if not manifest.version or len(manifest.version) == 0:
+            self.ask_for_new_version_handle(manifest, was_missing=True)
+
+        while version is None:
+            click.echo(f"Deploying version {manifest.version} of [{manifest.handle}]... ", nl=False)
+            try:
+                with ship_spinner():
+                    version = self._create_version(client, manifest, thing_id)
+            except SteamshipError as e:
+                if "The object you are trying to create already exists." in e.message:
+                    self.ask_for_new_version_handle(manifest)
+                else:
+                    click.secho(f"\nUnable to deploy {self.deployable_type()} version.", fg="red")
+                    click.secho(f"Server returned message: {e.message}")
+                    if "ModuleNotFoundError" in e.message:
+                        click.secho(
+                            "It looks like you may be missing a dependency in your requirements.txt.",
+                            fg="yellow",
+                        )
+                    click.get_current_context().abort()
+        click.echo("\nDone. 🚢")
+
+    def ask_for_new_version_handle(self, manifest: Manifest, was_missing: bool = False):
+        if not was_missing:
+            try_again = click.confirm(
+                click.style(
+                    f"\nIt looks like that version [{manifest.version}] has already been deployed. Would you like to change the version handle and try again?",
+                    fg="yellow",
+                ),
+                default=True,
+            )
+            if not try_again:
+                click.get_current_context().abort()
+
+        default_new = "1.0.0"
+        try:
+            default_new = str(VersionInfo.parse(manifest.version).bump_prerelease())
+        except ValueError:
+            pass
+        old_archive_path = get_archive_path(manifest)
+        new_version_handle = click.prompt(
+            "What should the new version be? Valid characters are a-z, 0-9, . and -",
+            value_proc=validate_version_handle,
+            default=default_new,
+        )
+        manifest.version = new_version_handle
+        manifest.save()
+        new_archive_path = get_archive_path(manifest)
+        old_archive_path.rename(new_archive_path)
+
+
+class PackageDeployer(DeployableDeployer):
+    def _create_version(self, client: Steamship, manifest: Manifest, thing_id: str):
+        return PackageVersion.create(
+            client=client,
+            config_template=manifest.config_template_as_dict(),
+            handle=manifest.version,
+            filename=f"build/archives/{manifest.handle}_v{manifest.version}.zip",
+            package_id=thing_id,
+        )
+
+    def create_object(self, client: Steamship, manifest: Manifest):
+        return Package.create(
+            client,
+            handle=manifest.handle,
+            profile=manifest,
+            is_public=manifest.public,
+            fetch_if_exists=True,
+        )
+
+    def update_object(self, deployable, client: Steamship, manifest: Manifest):
+        deployable.profile = manifest
+
+        package = deployable.update(client)
+        return package
+
+    def deployable_type(self):
+        return "package"
+
+
+class PluginDeployer(DeployableDeployer):
+    def _create_version(self, client: Steamship, manifest: Manifest, thing_id: str):
+        return PluginVersion.create(
+            client=client,
+            config_template=manifest.config_template_as_dict(),
+            handle=manifest.version,
+            filename=f"build/archives/{manifest.handle}_v{manifest.version}.zip",
+            plugin_id=thing_id,
+        )
+
+    def create_object(self, client: Steamship, manifest: Manifest):
+        return Plugin.create(
+            client,
+            description=manifest.description,
+            is_public=manifest.public,
+            transport=manifest.plugin.transport,
+            type_=manifest.plugin.type,
+            handle=manifest.handle,
+            fetch_if_exists=True,
+        )
+
+    def update_object(self, deployable, client: Steamship, manifest: Manifest):
+        deployable.profile = manifest
+
+        plugin = deployable.update(client)
+        return plugin
+
+    def deployable_type(self):
+        return "plugin"

steamship/cli/login.py

@@ -0,0 +1,62 @@
+import time
+import webbrowser
+
+import requests
+
+from steamship.base.error import SteamshipError
+
+
+def login(api_base: str, web_base: str) -> str:  # noqa: C901
+
+    # create login token
+    try:
+        token_result = requests.post(api_base + "account/create_login_attempt")
+        token_data = token_result.json().get("data")
+    except Exception as e:
+        raise SteamshipError("Could not create login token when attempting login.", error=e)
+
+    if token_data is None:
+        raise SteamshipError("Could not create login token when attempting login.")
+    token = token_data.get("token")
+    if token is None:
+        raise SteamshipError("Could not create login token when attempting login.")
+
+    # Launch login attempt in browser
+    login_browser_url = (
+        f"{web_base}account/client-login?attemptToken={token}&client=pycli&version=0.0.1"
+    )
+    try:
+        opened_browser = webbrowser.open(login_browser_url)
+    except Exception as e:
+        raise SteamshipError("Exception attempting to launch browser for login.", error=e)
+
+    if not opened_browser:
+        raise SteamshipError(
+            """Could not launch browser to log in to Steamship.
+
+If you are in Replit:
+
+1) Get an API key at https://steamship.com/account/api
+2) Set the STEAMSHIP_API_KEY Replit Secret
+3) Close and reopen this shell so that secrets refresh
+
+If you are in a different headless environment, visit https://docs.steamship.com/configuration/authentication.html"""
+        )
+
+    # Wait on result
+    total_poll_time_s = 0
+    time_between_polls_s = 1
+    api_key = None
+    while total_poll_time_s < 300:  # Five minutes
+        params = {"token": token}
+        login_response = requests.post(f"{api_base}account/poll_login_attempt", json=params).json()
+        if login_response.get("data", {}).get("status") == "done":
+            api_key = login_response.get("data", {}).get("apiKey")
+            break
+        time.sleep(time_between_polls_s)
+        total_poll_time_s += time_between_polls_s
+
+    if api_key is None:
+        raise SteamshipError("Could not fetch api key after login attempt in allotted time.")
+
+    return api_key

steamship/cli/manifest_init_wizard.py

@@ -0,0 +1,96 @@
+import re
+
+import click
+from click import BadParameter
+
+from steamship import Steamship
+from steamship.data.manifest import Manifest, PluginConfig, SteamshipRegistry
+from steamship.data.user import User
+
+
+def validate_handle(handle: str) -> str:
+    if re.fullmatch(r"[a-z\-]+", handle) is not None:
+        return handle
+    else:
+        raise BadParameter("Handle must only include lowercase letters and -")
+
+
+def validate_version_handle(handle: str) -> str:
+    if re.fullmatch(r"[a-z0-9\-.]+", handle) is not None:
+        return handle
+    else:
+        raise BadParameter("Handle must only include lowercase letters, numbers, . and -")
+
+
+def manifest_init_wizard(client: Steamship):
+    click.secho(
+        "It looks like you don't yet have a steamship.json to deploy. Let's create one.",
+        fg="cyan",
+    )
+
+    deployable_type = click.prompt(
+        "Is this a package or a plugin?",
+        default="package",
+        type=click.Choice(["package", "plugin"]),
+        show_choices=False,
+    )
+
+    handle = click.prompt(
+        f"What handle would you like to use for your {deployable_type}? Valid characters are a-z and -",
+        value_proc=validate_handle,
+    )
+
+    # TODO: claim the handle right here!
+
+    version_handle = "0.0.1"
+
+    plugin_detail = None
+    if deployable_type == "plugin":
+        plugin_type = click.prompt(
+            "What type of plugin is this?",
+            default="tagger",
+            type=click.Choice(
+                ["tagger", "blockifier", "exporter", "fileImporter", "corpusImporter", "generator"]
+            ),
+            show_choices=True,
+        )
+        if plugin_type == "tagger":
+            trainable = click.confirm("Is the plugin trainable?", default=False)
+        else:
+            trainable = False
+        plugin_detail = PluginConfig(isTrainable=trainable, type=plugin_type)
+
+    public = click.confirm(f"Do you want this {deployable_type} to be public?", default=True)
+
+    user = User.current(client)
+
+    author = click.prompt("How should we list your author name?", default=user.handle)
+
+    tagline = None
+    author_github = None
+    if public:
+        tagline = click.prompt(f"Want to give the {deployable_type} a tagline?", default="")
+        author_github = click.prompt(
+            "If you'd like this associated with your github account, please your github username",
+            default="",
+        )
+
+    tag_string = click.prompt(
+        f"Want to give the {deployable_type} some tags? (comma separated)", default="Prompt API"
+    )
+    tags = [tag.strip() for tag in tag_string.split(",")]
+
+    return Manifest(
+        type=deployable_type,
+        handle=handle,
+        version=version_handle,
+        description="",
+        author=author,
+        public=public,
+        plugin=plugin_detail,
+        build_config={"ignore": ["tests", "examples"]},
+        configTemplate={},
+        steamshipRegistry=SteamshipRegistry(
+            tagline=tagline, authorGithub=author_github, authorName=author, tags=tags
+        ),
+    )

steamship/cli/requirements_init_wizard.py

@@ -0,0 +1,20 @@
+import click
+
+import steamship
+
+
+def requirements_init_wizard():
+    click.secho(
+        "Steamship uses requirements.txt to specify dependencies. You do not currently have a requirements.txt in this directory.",
+        fg="yellow",
+    )
+    if not click.confirm("Would you like to create one automatically?", default=True):
+        click.secho("Please manually create a requirements.txt and try again.")
+        click.get_current_context().abort()
+
+    with open("requirements.txt", "w") as requirements_file:
+        requirements_file.write(f"steamship=={steamship.__version__}\n")
+
+    click.secho(
+        "Created a requirements.txt with the steamship dependency. If you need others, they must be added manually."
+    )

steamship/cli/ship_spinner.py

@@ -0,0 +1,48 @@
+import itertools
+import threading
+
+import click
+
+
+class Spinner(object):
+    # ["   🚢", "  🚢 ", " 🚢  ", "🚢   "]
+    # Unfortunately, backspacing doesn't seem to work correctly for emoji in iTerm, so leaving the "spinner"
+    # as adding ships for now
+    spinner_cycle = itertools.cycle(["🚢"])
+
+    def __init__(self):
+        self.stop_running = None
+        self.spin_thread = None
+
+    def start(self):
+        self.stop_running = threading.Event()
+        self.spin_thread = threading.Thread(target=self.init_spin)
+        self.spin_thread.start()
+
+    def stop(self):
+        if self.spin_thread:
+            self.stop_running.set()
+            self.spin_thread.join()
+
+    def init_spin(self):
+        while not self.stop_running.is_set():
+            click.echo(next(self.spinner_cycle), nl=False)
+            self.stop_running.wait(1)
+            # click.echo("\b", nl=False)
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+        return False
+
+
+def ship_spinner():
+    """This function creates a context manager that is used to display a
+    spinner on stdout as long as the context has not exited.
+    The spinner is created only if stdout is not redirected, or if the spinner
+    is forced using the `force` parameter.
+    """
+    return Spinner()

steamship/client/__init__.py

@@ -0,0 +1,3 @@
+from .steamship import Steamship
+
+__all__ = ["Steamship"]

steamship/client/skill_to_provider.py

@@ -0,0 +1,51 @@
+from typing import Any, Dict
+
+from pydantic import BaseModel
+
+from steamship.client.skills import Skill
+from steamship.client.vendors import Vendor
+
+
+class SkillVendorConfig(BaseModel):
+    plugin_handle: str
+    config: Dict[str, Any]
+
+
+SKILL_TO_PROVIDER: Dict[Skill, Dict[Vendor, SkillVendorConfig]] = {
+    Skill.ENTITIES: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger",
+            config={"skills": ["names", "numbers", "business-entities"]},
+        )
+    },
+    Skill.SUMMARY: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger", config={"skills": ["summarize"]}
+        )
+    },
+    Skill.SENTIMENTS: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger", config={"skills": ["sentiments"]}
+        )
+    },
+    Skill.EMOTIONS: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger", config={"skills": ["emotions"]}
+        )
+    },
+    Skill.TOPICS: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger", config={"skills": ["article-topics"]}
+        ),
+    },
+    Skill.HIGHLIGHTS: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger", config={"skills": ["highlights"]}
+        )
+    },
+    Skill.KEYWORDS: {
+        Vendor.OneAI: SkillVendorConfig(
+            plugin_handle="oneai-tagger", config={"skills": ["keywords"]}
+        )
+    },
+}

steamship/client/skills.py

@@ -0,0 +1,11 @@
+from enum import Enum
+
+
+class Skill(str, Enum):
+    ENTITIES = "entities"
+    SUMMARY = "summary"
+    SENTIMENTS = "sentiments"
+    EMOTIONS = "emotions"
+    TOPICS = "topics"
+    HIGHLIGHTS = "highlights"
+    KEYWORDS = "keywords"

steamship/client/steamship.py

@@ -0,0 +1,327 @@
+from __future__ import annotations
+
+import logging
+import uuid
+from contextlib import contextmanager
+from typing import Any, Dict, Generator, List, Optional
+
+from pydantic import BaseModel
+
+from steamship.base.client import Client
+from steamship.base.configuration import Configuration
+from steamship.base.error import SteamshipError
+from steamship.client.skill_to_provider import SKILL_TO_PROVIDER
+from steamship.client.skills import Skill
+from steamship.client.vendors import Vendor
+from steamship.data.embeddings import EmbedAndSearchRequest, QueryResults
+from steamship.data.package.package_instance import PackageInstance
+from steamship.data.plugin.index_plugin_instance import EmbeddingIndexPluginInstance
+from steamship.data.plugin.plugin_instance import PluginInstance
+from steamship.data.plugin.prompt_generation_plugin_instance import PromptGenerationPluginInstance
+from steamship.data.workspace import Workspace
+from steamship.utils.metadata import hash_dict
+
+_logger = logging.getLogger(__name__)
+
+
+class Steamship(Client):
+    """Steamship Python Client."""
+
+    # Some plugin instances use special subclasses which provide helper methods and/or more complex
+    # behavior than typical PluginInstance subclass permits. Examples are:
+    #
+    # - Embedding indices (which much coordinate both embedding taggers & vector indices)
+    # - Prompt generators (which benefit from supporting, prompt-specific, methods)
+    _PLUGIN_INSTANCE_SUBCLASS_OVERRIDES = {
+        "prompt-generation-default": PromptGenerationPluginInstance,
+        "prompt-generation-trainable-default": PromptGenerationPluginInstance,
+        "gpt3": PromptGenerationPluginInstance,
+        "gpt-3": PromptGenerationPluginInstance,
+        "cerebrium": PromptGenerationPluginInstance,
+        "embedding-index": EmbeddingIndexPluginInstance,
+    }
+
+    def __init__(
+        self,
+        api_key: str = None,
+        api_base: str = None,
+        app_base: str = None,
+        web_base: str = None,
+        workspace: str = None,
+        fail_if_workspace_exists: bool = False,
+        profile: str = None,
+        config_file: str = None,
+        config: Configuration = None,
+        trust_workspace_config: bool = False,  # For use by lambda_handler; don't fetch the workspace
+        **kwargs,
+    ):
+        super().__init__(
+            api_key=api_key,
+            api_base=api_base,
+            app_base=app_base,
+            web_base=web_base,
+            workspace=workspace,
+            fail_if_workspace_exists=fail_if_workspace_exists,
+            profile=profile,
+            config_file=config_file,
+            config=config,
+            trust_workspace_config=trust_workspace_config,
+            **kwargs,
+        )
+        # We use object.__setattr__ here in order to bypass Pydantic's overloading of it (which would block this
+        # set unless we were to add this as a field)
+        object.__setattr__(self, "use", self._instance_use)
+        object.__setattr__(self, "use_plugin", self._instance_use_plugin)
+
+    def __repr_args__(self: BaseModel) -> Any:
+        """Because of the trick we've done with `use` and `use_plugin`, we need to exclude these from __repr__
+        otherwise we'll get an infinite recursion."""
+        return [
+            (key, value)
+            for key, value in self.__dict__.items()
+            if key != "use" and key != "use_plugin"
+        ]
+
+    def embed_and_search(
+        self,
+        query: str,
+        docs: List[str],
+        plugin_instance: str,
+        k: int = 1,
+    ) -> QueryResults:
+        req = EmbedAndSearchRequest(query=query, docs=docs, plugin_instance=plugin_instance, k=k)
+        return self.post(
+            "plugin/instance/embeddingSearch",
+            req,
+            expect=QueryResults,
+        )
+
+    @staticmethod
+    @contextmanager
+    def temporary_workspace(**kwargs) -> Generator["Steamship", None, None]:
+        """Create a client rooted in a temporary workspace that will be deleted after use."""
+        # Create a new client and switch to a temporary workspace
+        client = Steamship(**kwargs)
+        temporary_handle = "temp-" + str(uuid.uuid4())
+        client.switch_workspace(temporary_handle)
+
+        # Safety check that we are now working form the new workspace.
+        if client.config.workspace_handle != temporary_handle:
+            raise SteamshipError(
+                message=f"Attempted to switch to temporary workspace {temporary_handle} but the client claimed to be working from {client.config.workspace_handle}"
+            )
+
+        yield client
+
+        # Safely delete the temporary workspace. Here we re-fetch the workspace using the temporary_handle
+        # in case the user switched workspaces yet again upon the client.
+        workspace = Workspace.get(client, handle=temporary_handle)
+        if workspace.handle != temporary_handle:
+            raise SteamshipError(
+                message=f"Was about to delete temporary workspace {temporary_handle} but its handle is different: {workspace.handle}"
+            )
+        else:
+            workspace.delete()
+
+    @staticmethod
+    def use(
+        package_handle: str,
+        instance_handle: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+        version: Optional[str] = None,
+        fetch_if_exists: bool = True,
+        workspace_handle: Optional[str] = None,
+        **kwargs,
+    ) -> PackageInstance:
+        """Creates/loads an instance of package `package_handle`.
+
+        The instance is named `instance_handle` and located in the Workspace named `instance_handle`. If no
+        `instance_handle` is provided, the default is `package_handle`.
+
+        For example, one may write the following to always get back the same package instance, no matter how many
+        times you run it, scoped into its own workspace:
+
+        ```python
+        instance = Steamship.use('package-handle', 'instance-handle')
+        ```
+
+        One may also write:
+
+        ```python
+        instance = Steamship.use('package-handle') # Instance will also be named `package-handle`
+        ```
+
+        If you wish to override the usage of a workspace named `instance_handle`, you can provide the `workspace_handle`
+        parameter.
+        """
+        if instance_handle is None:
+            instance_handle = package_handle
+        kwargs["workspace"] = workspace_handle or instance_handle
+        client = Steamship(**kwargs)
+        return client._instance_use(
+            package_handle=package_handle,
+            instance_handle=instance_handle,
+            config=config,
+            version=version,
+            fetch_if_exists=fetch_if_exists,
+        )
+
+    def _instance_use(
+        self,
+        package_handle: str,
+        instance_handle: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+        version: Optional[str] = None,
+        fetch_if_exists: bool = True,
+    ) -> PackageInstance:
+        """Creates/loads an instance of package `package_handle`.
+
+        The instance is named `instance_handle` and located in the workspace this client is anchored to.
+        If no `instance_handle` is provided, the default is `package_handle`.
+        """
+
+        if instance_handle is None:
+            if config is None:
+                instance_handle = package_handle
+            else:
+                instance_handle = f"{package_handle}-{hash_dict(config)}"
+
+        return PackageInstance.create(
+            self,
+            package_handle=package_handle,
+            package_version_handle=version,
+            handle=instance_handle,
+            config=config,
+            fetch_if_exists=fetch_if_exists,
+        )
+
+    @staticmethod
+    def use_plugin(
+        plugin_handle: str,
+        instance_handle: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+        version: Optional[str] = None,
+        fetch_if_exists: bool = True,
+        workspace_handle: Optional[str] = None,
+        **kwargs,
+    ) -> PluginInstance:
+        """Creates/loads an instance of plugin `plugin_handle`.
+
+        The instance is named `instance_handle` and located in the Workspace named `instance_handle`.
+        If no `instance_handle` is provided, the default is `plugin_handle`.
+
+        For example, one may write the following to always get back the same plugin instance, no matter how many
+        times you run it, scoped into its own workspace:
+
+        ```python
+        instance = Steamship.use_plugin('plugin-handle', 'instance-handle')
+        ```
+
+        One may also write:
+
+        ```python
+        instance = Steamship.use('plugin-handle') # Instance will also be named `plugin-handle`
+        ```
+        """
+        if instance_handle is None:
+            instance_handle = plugin_handle
+        kwargs["workspace"] = workspace_handle or instance_handle
+        client = Steamship(**kwargs)
+        return client._instance_use_plugin(
+            plugin_handle=plugin_handle,
+            instance_handle=instance_handle,
+            config=config,
+            version=version,
+            fetch_if_exists=fetch_if_exists,
+        )
+
+    def use_skill(
+        self,
+        skill: Skill,
+        provider: Optional[Vendor] = None,
+        instance_handle: Optional[str] = None,
+        fetch_if_exists: Optional[bool] = True,
+    ) -> PluginInstance:
+
+        if skill not in SKILL_TO_PROVIDER:
+            raise SteamshipError(
+                f"Unsupported skill provided. "
+                f"Use one of our supported skills: {','.join(SKILL_TO_PROVIDER)}"
+            )
+
+        if provider and provider not in SKILL_TO_PROVIDER[skill]:
+            raise SteamshipError(
+                f"The provider {provider} has no support for the skill {skill}."
+                f"Use one of the providers that support your skill: "
+                f"{','.join(SKILL_TO_PROVIDER[skill])}"
+            )
+
+        plugin_setup = (
+            SKILL_TO_PROVIDER[skill][provider]
+            if provider
+            else list(SKILL_TO_PROVIDER[skill].values())[0]
+        )
+        return self._instance_use_plugin(
+            plugin_handle=plugin_setup["plugin_handle"],
+            instance_handle=instance_handle,
+            config=plugin_setup["config"],
+            fetch_if_exists=fetch_if_exists,
+        )
+
+    def _instance_use_plugin(
+        self,
+        plugin_handle: str,
+        instance_handle: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+        version: Optional[str] = None,
+        fetch_if_exists: Optional[bool] = True,
+    ) -> PluginInstance:
+        """Creates/loads an instance of plugin `plugin_handle`.
+
+        The instance is named `instance_handle` and located in the workspace this client is anchored to.
+        If no `instance_handle` is provided, the default is `plugin_handle`.
+        """
+
+        if instance_handle is None:
+            if config is None:
+                instance_handle = plugin_handle
+            else:
+                instance_handle = f"{plugin_handle}-{hash_dict(config)}"
+
+        if plugin_handle in Steamship._PLUGIN_INSTANCE_SUBCLASS_OVERRIDES:
+            return Steamship._PLUGIN_INSTANCE_SUBCLASS_OVERRIDES[plugin_handle].create(
+                self,
+                plugin_handle=plugin_handle,
+                plugin_version_handle=version,
+                handle=instance_handle,
+                config=config,
+                fetch_if_exists=fetch_if_exists,
+            )
+
+        return PluginInstance.create(
+            self,
+            plugin_handle=plugin_handle,
+            plugin_version_handle=version,
+            handle=instance_handle,
+            config=config,
+            fetch_if_exists=fetch_if_exists,
+        )
+
+    def get_workspace(self) -> Workspace:
+        # We should probably add a hard-coded way to get this. The client in a Steamship Plugin/App comes
+        # pre-configured with an API key and the Workspace in which this client should be operating.
+        # This is a way to load the model object for that workspace.
+        logging.info(
+            f"get_workspace() called on client with config workspace {self.config.workspace_handle}/{self.config.workspace_id}"
+        )
+        workspace = Workspace.get(
+            self, id_=self.config.workspace_id, handle=self.config.workspace_handle
+        )
+        if not workspace:
+            logging.error("Unable to get workspace.")
+            raise SteamshipError(
+                message="Error while retrieving the Workspace associated with this client config.",
+                internal_message=f"workspace_id={self.config.workspace_id} workspace_handle={self.config.workspace_handle}",
+            )
+        logging.info(f"Got workspace: {workspace.handle}/{workspace.id}")
+        return workspace