Repository: gestaogovbr/FastETL (language: python)

Issue:
Merge conntype
* Split fast_etl.py into db_connection.py, get_table_cols_name.py, table_comments.py, copy_db_extensions.py
* change providers to airflow connection conn_type
* create auxiliary functions
* update tests


Discussion / hints:

Fix patch:
diff --git a/Makefile b/Makefile
index d793eae..7460306 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,7 @@
 .PHONY: setup
 setup:
 	docker-compose -f tests/docker-compose.yml up -d --force-recreate --remove-orphans
+	sleep 12 # wait for container to be ready
 	docker exec airflow sh -c "airflow db reset -y && airflow scheduler -D"
 
 .PHONY: down


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index cbb693e..8c2c5e4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -19,20 +19,20 @@ def pytest_configure(config):
     # define some models to get the tests to pass.
     db.merge_conn(
         models.Connection(
-            conn_id='pg-source-conn', conn_type='postgres',
-            host='pg-source', schema='db',
+            conn_id='postgres-source-conn', conn_type='postgres',
+            host='postgres-source', schema='db',
             login='root', password='root')
     )
     db.merge_conn(
         models.Connection(
-            conn_id='pg-destination-conn', conn_type='postgres',
-            host='pg-destination', schema='db',
+            conn_id='postgres-destination-conn', conn_type='postgres',
+            host='postgres-destination', schema='db',
             login='root', password='root')
     )
     db.merge_conn(
         models.Connection(
-            conn_id='pg-destination-fake-conn', conn_type='postgres',
-            host='pg-destination', schema='db',
+            conn_id='postgres-destination-fake-conn', conn_type='postgres',
+            host='postgres-destination', schema='db',
             login='fake', password='fake')
     )
 
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index 37b630f..5810c3e 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -44,6 +44,7 @@
           AIRFLOW__CORE__EXECUTOR: SequentialExecutor
           AIRFLOW__CORE__ENABLE_XCOM_PICKLING: 'true'
           AIRFLOW__CORE__FERNET_KEY: 'Uqv6pwYWJn7xifR3QAkifMTkupos8mkxaQL2Q7Ydsqw='
+          AIRFLOW__CORE__DEFAULT_TIMEZONE: 'America/Sao_Paulo'
           AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
           AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
         volumes:
@@ -53,7 +54,7 @@
           - ../tests:/opt/airflow/tests
           - ../tests/test_dag.py:/opt/airflow/dags/test_dag.py
         container_name: airflow
-        command: webserver
+        command: bash -c "airflow db init && airflow webserver"
         ports:
           - 8080:8080
         healthcheck:
@@ -63,10 +64,10 @@
           retries: 5
         restart: always
 
-      pg-source:
+      postgres-source:
         image: postgres:11.4-alpine
         restart: always
-        container_name: pg-source
+        container_name: postgres-source
         ports:
           - 54321:5432
         environment:
@@ -74,10 +75,10 @@
           - POSTGRES_PASSWORD=root
           - POSTGRES_DB=db
 
-      pg-destination:
+      postgres-destination:
         image: postgres:11.4-alpine
         restart: always
-        container_name: pg-destination
+        container_name: postgres-destination
         ports:
           - 54322:5432
         environment:
diff --git a/tests/sql/init/create_destination_table_pg.sql b/tests/sql/init/create_destination_table_postgres.sql
similarity index 100%
rename from tests/sql/init/create_destination_table_pg.sql
rename to tests/sql/init/create_destination_table_postgres.sql
diff --git a/tests/sql/init/create_source_table_pg.sql b/tests/sql/init/create_source_table_postgres.sql
similarity index 100%
rename from tests/sql/init/create_source_table_pg.sql
rename to tests/sql/init/create_source_table_postgres.sql
diff --git a/tests/test_dag.py b/tests/test_dag.py
index 6c1d944..dcbae72 100644
--- a/tests/test_dag.py
+++ b/tests/test_dag.py
@@ -15,7 +15,7 @@
     ) as dag:
     db_confs = [
         ('mssql', 'dbo'),
-        ('pg', 'public')
+        ('postgres', 'public')
     ]
     for source_conf in db_confs:
         for dest_conf in db_confs:
diff --git a/tests/test_db_to_db_operator.py b/tests/test_db_to_db_operator.py
index 46e444f..c84579d 100644
--- a/tests/test_db_to_db_operator.py
+++ b/tests/test_db_to_db_operator.py
@@ -84,14 +84,14 @@ def _insert_initial_source_table_n_data(table_name: str, hook: DbApiHook,
 @pytest.mark.parametrize(
     'source_conn_id, source_hook_cls, source_provider, dest_conn_id, dest_hook_cls, destination_provider, has_dest_table',
     [
-        ('pg-source-conn', PostgresHook, 'postgres', 'pg-destination-conn', PostgresHook, 'postgres', True),
+        ('postgres-source-conn', PostgresHook, 'postgres', 'postgres-destination-conn', PostgresHook, 'postgres', True),
         ('mssql-source-conn', OdbcHook, 'mssql', 'mssql-destination-conn', OdbcHook, 'mssql', True),
-        ('pg-source-conn', PostgresHook, 'postgres', 'mssql-destination-conn', OdbcHook, 'mssql', True),
-        ('mssql-source-conn', OdbcHook, 'mssql', 'pg-destination-conn', PostgresHook, 'postgres', True),
-        ('pg-source-conn', PostgresHook, 'postgres', 'pg-destination-conn', PostgresHook, 'postgres', False),
+        ('postgres-source-conn', PostgresHook, 'postgres', 'mssql-destination-conn', OdbcHook, 'mssql', True),
+        ('mssql-source-conn', OdbcHook, 'mssql', 'postgres-destination-conn', PostgresHook, 'postgres', True),
+        ('postgres-source-conn', PostgresHook, 'postgres', 'postgres-destination-conn', PostgresHook, 'postgres', False),
         ('mssql-source-conn', OdbcHook, 'mssql', 'mssql-destination-conn', OdbcHook, 'mssql', False),
-        ('pg-source-conn', PostgresHook, 'postgres', 'mssql-destination-conn', OdbcHook, 'mssql', False),
-        ('mssql-source-conn', OdbcHook, 'mssql', 'pg-destination-conn', PostgresHook, 'postgres', False),
+        ('postgres-source-conn', PostgresHook, 'postgres', 'mssql-destination-conn', OdbcHook, 'mssql', False),
+        ('mssql-source-conn', OdbcHook, 'mssql', 'postgres-destination-conn', PostgresHook, 'postgres', False),
     ])
 def test_full_table_replication_various_db_types(
         source_conn_id: str,