Repository: amundsen-io/amundsen

Issue:
Bug Report - Postegres / Redshift Extractors have inconsistent behavior
<!--- Provide a general summary of the issue in the Title above -->
<!--- Look through existing open and closed issues to see if someone has reported the issue before -->

## Expected Behavior

```python
# For RedshiftMetadataExtractor
redshift_metadata_extractor_config = ConfigFactory.from_dict({
    'extractor.redshift_metadata.where_clause_suffix':  'schema <> "foo"'
})
```

## Current Behavior

```python
# For RedshiftMetadataExtractor
redshift_metadata_extractor_config = ConfigFactory.from_dict({
    'extractor.redshift_metadata.where_clause_suffix':  'where schema <> "foo"'
})
```

## Possible Solution

Update redshift extractor to properly build suffix

## Screenshots (if appropriate)

## Context

Both the `PostgresMetadataExtractor` and the `RedshiftMetadataExtractor` take the config `extractor.redshift_metadata.where_clause_suffix` as they are both based off of `BasePostgresMetadataExtractor`.

However they require slightly different different sql partials, which is unexpected: 

The `where_clause_suffix` for `RedshiftMetadataExtractor` takes a partial sql statement **without** a `WHERE`.
The `where_clause_suffix` for `PostgresMetadataExtractor` take a partial sql statement **with** a `WHERE`.

The name `where_clause_suffix` implies that its a statement appended to a where clause and should  _not_ (IMHO) require a `WHERE` statement.


<!--- How has this issue affected you? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Amunsen version used: amundsen-databuilder 6.3.1
* Data warehouse stores: Neo4j
* Deployment (k8s or native): terraform
* Link to your fork or repository:


Discussion:
While changing this behavior might break code for some users, I think that this difference in usage is confusing enough to newer users that its worth fixing.

Resolving patch:
diff --git a/databuilder/databuilder/extractor/redshift_metadata_extractor.py b/databuilder/databuilder/extractor/redshift_metadata_extractor.py
--- a/databuilder/databuilder/extractor/redshift_metadata_extractor.py
+++ b/databuilder/databuilder/extractor/redshift_metadata_extractor.py
@@ -1,6 +1,7 @@
 # Copyright Contributors to the Amundsen project.
 # SPDX-License-Identifier: Apache-2.0
 
+import logging
 from typing import (  # noqa: F401
     Any, Dict, Iterator, Union,
 )
@@ -9,11 +10,14 @@
 
 from databuilder.extractor.base_postgres_metadata_extractor import BasePostgresMetadataExtractor
 
+LOGGER = logging.getLogger(__name__)
+
 
 class RedshiftMetadataExtractor(BasePostgresMetadataExtractor):
     """
     Extracts Redshift table and column metadata from underlying meta store database using SQLAlchemyExtractor
 
+
     This differs from the PostgresMetadataExtractor because in order to support Redshift's late binding views,
     we need to join the INFORMATION_SCHEMA data against the function PG_GET_LATE_BINDING_VIEW_COLS().
     """
@@ -24,6 +28,15 @@ def get_sql_statement(self, use_catalog_as_cluster_name: bool, where_clause_suff
         else:
             cluster_source = f"'{self._cluster}'"
 
+        if where_clause_suffix:
+            if where_clause_suffix.lower().startswith("where"):
+                LOGGER.warning("you no longer need to begin with 'where' in your suffix")
+                where_clause = where_clause_suffix
+            else:
+                where_clause = f"where {where_clause_suffix}"
+        else:
+            where_clause = ""
+
         return """
         SELECT
             *
@@ -74,11 +87,11 @@ def get_sql_statement(self, use_catalog_as_cluster_name: bool, where_clause_suff
             FROM svv_external_columns
         )
 
-        {where_clause_suffix}
+        {where_clause}
         ORDER by cluster, schema, name, col_sort_order ;
         """.format(
             cluster_source=cluster_source,
-            where_clause_suffix=where_clause_suffix,
+            where_clause=where_clause,
         )
 
     def get_scope(self) -> str:


Test patch:
diff --git a/databuilder/tests/unit/extractor/test_redshift_metadata_extractor.py b/databuilder/tests/unit/extractor/test_redshift_metadata_extractor.py
--- a/databuilder/tests/unit/extractor/test_redshift_metadata_extractor.py
+++ b/databuilder/tests/unit/extractor/test_redshift_metadata_extractor.py
@@ -104,6 +104,32 @@ def _union(self,
 
 
 class TestRedshiftMetadataExtractorWithWhereClause(unittest.TestCase):
+    def setUp(self) -> None:
+        logging.basicConfig(level=logging.INFO)
+        self.where_clause_suffix = """
+        table_schema in ('public') and table_name = 'movies'
+        """
+
+        config_dict = {
+            RedshiftMetadataExtractor.WHERE_CLAUSE_SUFFIX_KEY: self.where_clause_suffix,
+            f'extractor.sqlalchemy.{SQLAlchemyExtractor.CONN_STRING}':
+                'TEST_CONNECTION'
+        }
+        self.conf = ConfigFactory.from_dict(config_dict)
+
+    def test_sql_statement(self) -> None:
+        """
+        Test extraction sql properly includes where suffix
+        """
+        with patch.object(SQLAlchemyExtractor, '_get_connection'):
+            extractor = RedshiftMetadataExtractor()
+            extractor.init(self.conf)
+            expected_where_clause = f'where {self.where_clause_suffix}'
+
+            self.assertTrue(expected_where_clause in extractor.sql_stmt)
+
+
+class TestRedshiftMetadataExtractorWithLegacyWhereClause(unittest.TestCase):
     def setUp(self) -> None:
         logging.basicConfig(level=logging.INFO)
         self.where_clause_suffix = """
@@ -119,11 +145,12 @@ def setUp(self) -> None:
 
     def test_sql_statement(self) -> None:
         """
-        Test Extraction with empty result from query
+        Test extraction sql properly includes where suffix using legacy specification
         """
         with patch.object(SQLAlchemyExtractor, '_get_connection'):
             extractor = RedshiftMetadataExtractor()
             extractor.init(self.conf)
+
             self.assertTrue(self.where_clause_suffix in extractor.sql_stmt)