Repository: great-expectations/great_expectations

Issue:
list_datasources prints passwords in clear text even if their values are stored in a secret store/environment variable
**Describe the bug**
Great Expectations supports loading credentials and other secrets from environment variables and other secret stores using ${VAR} syntax in order to not put them in the config file. The `list_datasources` method prints passwords in clear text even if their values are stored in a secret store/environment variable.

**To Reproduce**
Steps to reproduce the behavior:
1. Create a Data Context (e.g., by running great_expectations init)
2. Configure a datasource (e.g., to connect to a Postgres database) with password specified by a variable ${MY_DB_PASSWORD} that is substituted from an env variable in runtime (follow these instuctions: https://docs.greatexpectations.io/en/latest/guides/how_to_guides/configuring_data_contexts/how_to_use_a_yaml_file_or_environment_variables_to_populate_credentials.html)
3. Run the following Python snippet (in Jupyter notebook or IDE of your choice):
```
import great_expectations as ge

context = ge.data_context.DataContext()
context.list_datasources()

```
4. See the clear text password in the output:
![image](https://user-images.githubusercontent.com/80363/102553342-903af180-4077-11eb-8e65-29ef462698ee.png)


**Expected behavior**
The value of the password is masked.

**Environment (please complete the following information):**
 - OS: any
 - GE Version: 0.13.3

**Additional context**
None


Discussion:

Resolving patch:
diff --git a/great_expectations/data_context/data_context.py b/great_expectations/data_context/data_context.py
--- a/great_expectations/data_context/data_context.py
+++ b/great_expectations/data_context/data_context.py
@@ -57,6 +57,7 @@
     ValidationResultIdentifier,
 )
 from great_expectations.data_context.util import (
+    PasswordMasker,
     file_relative_path,
     instantiate_class_from_config,
     load_class,
@@ -1760,7 +1761,7 @@ def list_expectation_suites(self):
         return keys
 
     def list_datasources(self):
-        """List currently-configured datasources on this context.
+        """List currently-configured datasources on this context. Masks passwords.
 
         Returns:
             List(dict): each dictionary includes "name", "class_name", and "module_name" keys
@@ -1771,6 +1772,17 @@ def list_datasources(self):
             value,
         ) in self._project_config_with_variables_substituted.datasources.items():
             value["name"] = key
+
+            if "credentials" in value:
+                if "password" in value["credentials"]:
+                    value["credentials"][
+                        "password"
+                    ] = PasswordMasker.MASKED_PASSWORD_STRING
+                if "url" in value["credentials"]:
+                    value["credentials"]["url"] = PasswordMasker.mask_db_url(
+                        value["credentials"]["url"]
+                    )
+
             datasources.append(value)
         return datasources
 
diff --git a/great_expectations/data_context/util.py b/great_expectations/data_context/util.py
--- a/great_expectations/data_context/util.py
+++ b/great_expectations/data_context/util.py
@@ -3,7 +3,9 @@
 import logging
 import os
 import re
+import warnings
 from collections import OrderedDict
+from urllib.parse import urlparse
 
 from great_expectations.data_context.types.base import (
     DataContextConfig,
@@ -12,6 +14,11 @@
 from great_expectations.exceptions import MissingConfigVariableError
 from great_expectations.util import load_class, verify_dynamic_loading_support
 
+try:
+    import sqlalchemy as sa
+except ImportError:
+    sa = None
+
 logger = logging.getLogger(__name__)
 
 
@@ -205,3 +212,65 @@ def file_relative_path(dunderfile, relative_path):
     H/T https://github.com/dagster-io/dagster/blob/8a250e9619a49e8bff8e9aa7435df89c2d2ea039/python_modules/dagster/dagster/utils/__init__.py#L34
     """
     return os.path.join(os.path.dirname(dunderfile), relative_path)
+
+
+class PasswordMasker:
+    """
+    Used to mask passwords in Datasources. Does not mask sqlite urls.
+
+    Example usage
+    masked_db_url = PasswordMasker.mask_db_url(url)
+    where url = "postgresql+psycopg2://username:password@host:65432/database"
+    and masked_url = "postgresql+psycopg2://username:***@host:65432/database"
+
+    """
+
+    MASKED_PASSWORD_STRING = "***"
+
+    @staticmethod
+    def mask_db_url(url: str, use_urlparse: bool = False, **kwargs) -> str:
+        """
+        Mask password in database url.
+        Uses sqlalchemy engine parsing if sqlalchemy is installed, otherwise defaults to using urlparse from the stdlib which does not handle kwargs.
+        Args:
+            url: Database url e.g. "postgresql+psycopg2://username:password@host:65432/database"
+            use_urlparse: Skip trying to parse url with sqlalchemy and use urlparse
+            **kwargs: passed to create_engine()
+
+        Returns:
+            url with password masked e.g. "postgresql+psycopg2://username:***@host:65432/database"
+        """
+        if sa is not None and use_urlparse is False:
+            engine = sa.create_engine(url, **kwargs)
+            return engine.url.__repr__()
+        else:
+            warnings.warn(
+                "SQLAlchemy is not installed, using urlparse to mask database url password which ignores **kwargs."
+            )
+
+            # oracle+cx_oracle does not parse well using urlparse, parse as oracle then swap back
+            replace_prefix = None
+            if url.startswith("oracle+cx_oracle"):
+                replace_prefix = {"original": "oracle+cx_oracle", "temporary": "oracle"}
+                url = url.replace(
+                    replace_prefix["original"], replace_prefix["temporary"]
+                )
+
+            parsed_url = urlparse(url)
+
+            # Do not parse sqlite
+            if parsed_url.scheme == "sqlite":
+                return url
+
+            colon = ":" if parsed_url.port is not None else ""
+            masked_url = (
+                f"{parsed_url.scheme}://{parsed_url.username}:{PasswordMasker.MASKED_PASSWORD_STRING}"
+                f"@{parsed_url.hostname}{colon}{parsed_url.port or ''}{parsed_url.path or ''}"
+            )
+
+            if replace_prefix is not None:
+                masked_url = masked_url.replace(
+                    replace_prefix["temporary"], replace_prefix["original"]
+                )
+
+            return masked_url


Test patch:
diff --git a/tests/data_context/test_data_context.py b/tests/data_context/test_data_context.py
--- a/tests/data_context/test_data_context.py
+++ b/tests/data_context/test_data_context.py
@@ -283,6 +283,88 @@ def test_list_datasources(data_context_parameterized_expectation_suite):
         },
     ]
 
+    # Make sure passwords are masked in password or url fields
+    data_context_parameterized_expectation_suite.add_datasource(
+        "postgres_source_with_password",
+        initialize=False,
+        module_name="great_expectations.datasource",
+        class_name="SqlAlchemyDatasource",
+        credentials={
+            "drivername": "postgresql",
+            "host": "localhost",
+            "port": "65432",
+            "username": "username_str",
+            "password": "password_str",
+            "database": "database_str",
+        },
+    )
+
+    data_context_parameterized_expectation_suite.add_datasource(
+        "postgres_source_with_password_in_url",
+        initialize=False,
+        module_name="great_expectations.datasource",
+        class_name="SqlAlchemyDatasource",
+        credentials={
+            "url": "postgresql+psycopg2://username:password@host:65432/database",
+        },
+    )
+
+    datasources = data_context_parameterized_expectation_suite.list_datasources()
+
+    assert datasources == [
+        {
+            "name": "mydatasource",
+            "class_name": "PandasDatasource",
+            "module_name": "great_expectations.datasource",
+            "data_asset_type": {"class_name": "PandasDataset"},
+            "batch_kwargs_generators": {
+                "mygenerator": {
+                    "base_directory": "../data",
+                    "class_name": "SubdirReaderBatchKwargsGenerator",
+                    "reader_options": {"engine": "python", "sep": None},
+                }
+            },
+        },
+        {
+            "name": "second_pandas_source",
+            "class_name": "PandasDatasource",
+            "module_name": "great_expectations.datasource",
+            "data_asset_type": {
+                "class_name": "PandasDataset",
+                "module_name": "great_expectations.dataset",
+            },
+        },
+        {
+            "name": "postgres_source_with_password",
+            "class_name": "SqlAlchemyDatasource",
+            "module_name": "great_expectations.datasource",
+            "data_asset_type": {
+                "class_name": "SqlAlchemyDataset",
+                "module_name": "great_expectations.dataset",
+            },
+            "credentials": {
+                "drivername": "postgresql",
+                "host": "localhost",
+                "port": "65432",
+                "username": "username_str",
+                "password": "***",
+                "database": "database_str",
+            },
+        },
+        {
+            "name": "postgres_source_with_password_in_url",
+            "class_name": "SqlAlchemyDatasource",
+            "module_name": "great_expectations.datasource",
+            "data_asset_type": {
+                "class_name": "SqlAlchemyDataset",
+                "module_name": "great_expectations.dataset",
+            },
+            "credentials": {
+                "url": "postgresql+psycopg2://username:***@host:65432/database",
+            },
+        },
+    ]
+
 
 @freeze_time("09/26/2019 13:42:41")
 def test_data_context_get_validation_result(titanic_data_context):
diff --git a/tests/data_context/test_data_context_utils.py b/tests/data_context/test_data_context_utils.py
--- a/tests/data_context/test_data_context_utils.py
+++ b/tests/data_context/test_data_context_utils.py
@@ -1,6 +1,7 @@
 import pytest
 
 import great_expectations.exceptions as gee
+from great_expectations.data_context.util import PasswordMasker
 from great_expectations.util import load_class
 
 
@@ -12,3 +13,217 @@ def test_load_class_raises_error_when_module_not_found():
 def test_load_class_raises_error_when_class_not_found():
     with pytest.raises(gee.PluginClassNotFoundError):
         load_class("TotallyNotARealClass", "great_expectations.datasource")
+
+
+def test_password_masker_mask_db_url():
+    """
+    What does this test and why?
+    The PasswordMasker.mask_db_url() should mask passwords consistently in database urls. The output of mask_db_url should be the same whether user_urlparse is set to True or False.
+    This test uses database url examples from
+    https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls
+    """
+    # PostgreSQL
+    # default
+    assert (
+        PasswordMasker.mask_db_url(
+            "postgresql://scott:tiger@localhost:65432/mydatabase"
+        )
+        == "postgresql://scott:***@localhost:65432/mydatabase"
+    )
+    assert (
+        PasswordMasker.mask_db_url(
+            "postgresql://scott:tiger@localhost:65432/mydatabase", use_urlparse=True
+        )
+        == "postgresql://scott:***@localhost:65432/mydatabase"
+    )
+    # missing port number, using urlparse
+    assert (
+        PasswordMasker.mask_db_url(
+            "postgresql://scott:tiger@localhost/mydatabase", use_urlparse=True
+        )
+        == "postgresql://scott:***@localhost/mydatabase"
+    )
+
+    # psycopg2
+    assert (
+        PasswordMasker.mask_db_url(
+            "postgresql+psycopg2://scott:tiger@localhost:65432/mydatabase"
+        )
+        == "postgresql+psycopg2://scott:***@localhost:65432/mydatabase"
+    )
+    assert (
+        PasswordMasker.mask_db_url(
+            "postgresql+psycopg2://scott:tiger@localhost:65432/mydatabase",
+            use_urlparse=True,
+        )
+        == "postgresql+psycopg2://scott:***@localhost:65432/mydatabase"
+    )
+
+    # pg8000 (if installed in test environment)
+    try:
+        assert (
+            PasswordMasker.mask_db_url(
+                "postgresql+pg8000://scott:tiger@localhost:65432/mydatabase"
+            )
+            == "postgresql+pg8000://scott:***@localhost:65432/mydatabase"
+        )
+    except ModuleNotFoundError:
+        pass
+    assert (
+        PasswordMasker.mask_db_url(
+            "postgresql+pg8000://scott:tiger@localhost:65432/mydatabase",
+            use_urlparse=True,
+        )
+        == "postgresql+pg8000://scott:***@localhost:65432/mydatabase"
+    )
+
+    # MySQL
+    # default (if installed in test environment)
+    try:
+        assert (
+            PasswordMasker.mask_db_url("mysql://scott:tiger@localhost:65432/foo")
+            == "mysql://scott:***@localhost:65432/foo"
+        )
+    except ModuleNotFoundError:
+        pass
+
+    assert (
+        PasswordMasker.mask_db_url(
+            "mysql://scott:tiger@localhost:65432/foo", use_urlparse=True
+        )
+        == "mysql://scott:***@localhost:65432/foo"
+    )
+
+    # mysqlclient (a maintained fork of MySQL-Python) (if installed in test environment)
+    try:
+        assert (
+            PasswordMasker.mask_db_url(
+                "mysql+mysqldb://scott:tiger@localhost:65432/foo"
+            )
+            == "mysql+mysqldb://scott:***@localhost:65432/foo"
+        )
+    except ModuleNotFoundError:
+        pass
+    assert (
+        PasswordMasker.mask_db_url(
+            "mysql+mysqldb://scott:tiger@localhost:65432/foo", use_urlparse=True
+        )
+        == "mysql+mysqldb://scott:***@localhost:65432/foo"
+    )
+
+    # PyMySQL
+    assert (
+        PasswordMasker.mask_db_url("mysql+pymysql://scott:tiger@localhost:65432/foo")
+        == "mysql+pymysql://scott:***@localhost:65432/foo"
+    )
+    assert (
+        PasswordMasker.mask_db_url(
+            "mysql+pymysql://scott:tiger@localhost:65432/foo", use_urlparse=True
+        )
+        == "mysql+pymysql://scott:***@localhost:65432/foo"
+    )
+
+    # Oracle (if installed in test environment)
+    try:
+        assert (
+            PasswordMasker.mask_db_url("oracle://scott:tiger@127.0.0.1:1521/sidname")
+            == "oracle://scott:***@127.0.0.1:1521/sidname"
+        )
+    except ModuleNotFoundError:
+        pass
+
+    assert (
+        PasswordMasker.mask_db_url(
+            "oracle://scott:tiger@127.0.0.1:1521/sidname", use_urlparse=True
+        )
+        == "oracle://scott:***@127.0.0.1:1521/sidname"
+    )
+
+    try:
+        assert (
+            PasswordMasker.mask_db_url("oracle+cx_oracle://scott:tiger@tnsname")
+            == "oracle+cx_oracle://scott:***@tnsname"
+        )
+    except ModuleNotFoundError:
+        pass
+    assert (
+        PasswordMasker.mask_db_url(
+            "oracle+cx_oracle://scott:tiger@tnsname", use_urlparse=True
+        )
+        == "oracle+cx_oracle://scott:***@tnsname"
+    )
+
+    # Microsoft SQL Server
+    # pyodbc
+    assert (
+        PasswordMasker.mask_db_url("mssql+pyodbc://scott:tiger@mydsn")
+        == "mssql+pyodbc://scott:***@mydsn"
+    )
+    assert (
+        PasswordMasker.mask_db_url(
+            "mssql+pyodbc://scott:tiger@mydsn", use_urlparse=True
+        )
+        == "mssql+pyodbc://scott:***@mydsn"
+    )
+
+    # pymssql (if installed in test environment)
+    try:
+        assert (
+            PasswordMasker.mask_db_url(
+                "mssql+pymssql://scott:tiger@hostname:12345/dbname"
+            )
+            == "mssql+pymssql://scott:***@hostname:12345/dbname"
+        )
+    except ModuleNotFoundError:
+        pass
+    assert (
+        PasswordMasker.mask_db_url(
+            "mssql+pymssql://scott:tiger@hostname:12345/dbname", use_urlparse=True
+        )
+        == "mssql+pymssql://scott:***@hostname:12345/dbname"
+    )
+
+    # SQLite
+    # relative path
+    assert PasswordMasker.mask_db_url("sqlite:///foo.db") == "sqlite:///foo.db"
+    assert (
+        PasswordMasker.mask_db_url("sqlite:///foo.db", use_urlparse=True)
+        == "sqlite:///foo.db"
+    )
+
+    # absolute path
+    # Unix/Mac - 4 initial slashes in total
+    assert (
+        PasswordMasker.mask_db_url("sqlite:////absolute/path/to/foo.db")
+        == "sqlite:////absolute/path/to/foo.db"
+    )
+    assert (
+        PasswordMasker.mask_db_url(
+            "sqlite:////absolute/path/to/foo.db", use_urlparse=True
+        )
+        == "sqlite:////absolute/path/to/foo.db"
+    )
+
+    # Windows
+    assert (
+        PasswordMasker.mask_db_url("sqlite:///C:\\path\\to\\foo.db")
+        == "sqlite:///C:\\path\\to\\foo.db"
+    )
+    assert (
+        PasswordMasker.mask_db_url("sqlite:///C:\\path\\to\\foo.db", use_urlparse=True)
+        == "sqlite:///C:\\path\\to\\foo.db"
+    )
+
+    # Windows alternative using raw string
+    assert (
+        PasswordMasker.mask_db_url(r"sqlite:///C:\path\to\foo.db")
+        == r"sqlite:///C:\path\to\foo.db"
+    )
+    assert (
+        PasswordMasker.mask_db_url(r"sqlite:///C:\path\to\foo.db", use_urlparse=True)
+        == r"sqlite:///C:\path\to\foo.db"
+    )
+
+    # in-memory
+    assert PasswordMasker.mask_db_url("sqlite://") == "sqlite://"
+    assert PasswordMasker.mask_db_url("sqlite://", use_urlparse=True) == "sqlite://"