Repository: bluelabsio/records-mover (language: python)

Issue:
Circleci editor/14/rm 2 fix table to table integration tests
tbl2tbl test was failing because of a problem with pandas build:
https://app.circleci.com/pipelines/github/bluelabsio/records-mover/2119/workflows/5f195957-0cbd-4b93-b81c-5bab34e24ca7/jobs/18862

Turns out that pandas version < 1 doesn't build on python 3.9; just specified pandas version number here and the tests passed now that BQ integration is a go:
https://app.circleci.com/pipelines/github/bluelabsio/records-mover/2281/workflows/c443de7c-aafd-4437-9cd3-50024f716a45/jobs/18928


Discussion / hints:

Fix patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index a2cffcc47..d7a55819f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -625,6 +625,19 @@ workflows:
           filters:
             tags:
               only: /v\d+\.\d+\.\d+(-[\w]+)?/
+      - integration_test_with_dbs:
+          name: tbl2tbl-itest
+          extras: '[literally_every_single_database_binary,itest]'
+          python_version: "3.9"
+          pandas_version: "==1.1.5"
+          command: |
+            . venv/bin/activate
+            export PATH=${PATH}:${PWD}/tests/integration/bin:/opt/vertica/bin
+            export DB_FACTS_PATH=${PWD}/tests/integration/circleci-dbfacts.yml
+            export RECORDS_MOVER_SESSION_TYPE=env
+            mkdir -p test-reports/itest
+            cd tests/integration
+            python3 -m records.multi_db.test_records_table2table
       - integration_test:
           name: bigquery-no-gcs-itest
           extras: '[bigquery,itest]'
@@ -646,23 +659,6 @@ workflows:
           filters:
             tags:
               only: /v\d+\.\d+\.\d+(-[\w]+)?/
-      # - integration_test_with_dbs:
-      #     name: tbl2tbl-itest
-      #     extras: '[literally_every_single_database_binary,itest]'
-      #     python_version: "3.9"
-      #     command: |
-      #       . venv/bin/activate
-      #       export PATH=${PATH}:${PWD}/tests/integration/bin:/opt/vertica/bin
-      #       export DB_FACTS_PATH=${PWD}/tests/integration/circleci-dbfacts.yml
-      #       export RECORDS_MOVER_SESSION_TYPE=env
-      #       mkdir -p test-reports/itest
-      #       cd tests/integration
-      #       python3 -m records.multi_db.test_records_table2table
-      #     requires:
-      #       - redshift-s3-itest
-      #     filters:
-      #       tags:
-      #         only: /v\d+\.\d+\.\d+(-[\w]+)?/
       - cli-extra-test:
           name: cli-extra-test
           requires:
@@ -674,7 +670,7 @@ workflows:
             - test-3.8
             - test-3.9
             - cli-extra-test
-            # - tbl2tbl-itest
+            - tbl2tbl-itest
             # - bigquery-no-gcs-itest
             - bigquery-gcs-itest
             - redshift-no-s3-itest
@@ -693,3 +689,5 @@ workflows:
               only: /v\d+\.\d+\.\d+(-[\w]+)?/
             branches:
               ignore: /.*/
+
+# VS Code Extension Version: 1.4.0


Test patch:
diff --git a/tests/integration/records/expected_column_types.py b/tests/integration/records/expected_column_types.py
index c3a2070db..982007bea 100644
--- a/tests/integration/records/expected_column_types.py
+++ b/tests/integration/records/expected_column_types.py
@@ -117,7 +117,7 @@
     ('postgresql', 'postgresql'): [
         'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
         'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'TIME WITHOUT TIME ZONE',
-        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
+        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
     ],
     ('postgresql', 'vertica'): [
         'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
@@ -138,6 +138,11 @@
         'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'VARCHAR(8)',
         'TIMESTAMP', 'TIMESTAMP'
     ],
+    ('redshift', 'postgresql'): [
+        'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
+        'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'VARCHAR(8)',
+        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
+    ],
     ('bigquery', 'redshift'): [
         'BIGINT', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
         'VARCHAR(256)', 'VARCHAR(256)', 'INTEGER', 'INTEGER',
@@ -150,7 +155,7 @@
     ('bigquery', 'postgresql'): [
         'BIGINT', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
         'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'TIME WITHOUT TIME ZONE',
-        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
+        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
     ],
     ('bigquery', 'vertica'): [
         'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
@@ -191,7 +196,7 @@
     ('vertica', 'postgresql'): [
         'BIGINT', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
         'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'TIME WITHOUT TIME ZONE',
-        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
+        'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
     ],
     ('vertica', 'redshift'): [
         'BIGINT', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
diff --git a/tests/integration/records/table_validator.py b/tests/integration/records/table_validator.py
index 6d6b6fe83..d6ebd6077 100644
--- a/tests/integration/records/table_validator.py
+++ b/tests/integration/records/table_validator.py
@@ -79,20 +79,40 @@ def format_type(column: Dict[str, Any]) -> str:
             return str(column['type']) + suffix
 
         actual_column_types = [format_type(column) for column in columns]
+
+        def format_actual_expected_column_types(*expected):
+            expected_str = '\nOR: '.join([str(x) for x in expected])
+            return (f'\nACTUAL: {actual_column_types}\n'
+                    f'EXPECTED: {expected_str}')
+
+        expected_target_df_types = expected_df_loaded_database_column_types.get(
+            self.target_db_engine.name)
+        expected_target_single_types = \
+            expected_single_database_column_types.get(
+                self.target_db_engine.name)
+
         if self.source_db_engine is None:
             if self.file_variant is None:
-                assert actual_column_types in\
-                    (expected_df_loaded_database_column_types.get(self.target_db_engine.name),
-                     expected_single_database_column_types[self.target_db_engine.name]),\
+                actual_expected_str = format_actual_expected_column_types(
+                    expected_target_df_types,
+                    expected_target_single_types)
+
+                assert \
+                    actual_column_types in (
+                        expected_target_df_types,
+                        expected_target_single_types), \
                     f'Could not find column types filed under ' \
                     f"{('df', self.target_db_engine.name)} or : " \
-                    f"{self.target_db_engine.name}: " \
-                    f'{actual_column_types}'
+                    f'{self.target_db_engine.name}: ' \
+                    f'{actual_expected_str}'
             else:
-                assert actual_column_types ==\
-                    expected_single_database_column_types[self.target_db_engine.name],\
-                    f'Could not find column types filed under {self.target_db_engine.name}: ' +\
-                    f'{actual_column_types}'
+                actual_expected_str = format_actual_expected_column_types(
+                    expected_target_single_types)
+
+                assert actual_column_types == expected_target_single_types, \
+                    f'Could not find column types filed under ' \
+                    f'{self.target_db_engine.name}: ' \
+                    f'{actual_expected_str}'
         else:
             assert (actual_column_types in
                     (expected_table2table_column_types.get((self.source_db_engine.name,