Repository: astronomer/astro-sdk

Issue:
Fix concurency issue on `main` branch
**Describe the bug**
Since we introduced some parallelism in running our tests, we have seen the CI getting red more often than desired, many times breaking tests with parts of the code that were not changed.

This is happening because some tests do not use unique table names and do not implement tear down effectively.

**Version**
* Astro: 0.7.0

**To Reproduce**
Create a pull request / or merge it and - intermittently - see the tests failing:
![Screenshot from 2022-03-14 09-30-23](https://user-images.githubusercontent.com/272048/158144250-bbb7ab22-3dfb-41b6-88d2-c95b712238f3.png)

**Expected behavior**
The tests should not break unless there is a bug in the code.

**Acceptance criteria**
* [ ] Make the tests more reliable, by avoiding table name clashes.



Discussion:

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,6 @@
 import os
+import random
+import string
 
 import pytest
 import yaml
@@ -15,6 +17,7 @@
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
 
 OUTPUT_TABLE_NAME = test_utils.get_table_name("integration_test_table")
+UNIQUE_HASH_SIZE = 16
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -33,9 +36,20 @@ def create_database_connections():
             session.add(conn)
 
 
+def _create_unique_dag_id():
+    # To use 32 chars was too long for the DAG & SQL name table, for this reason we are not using:
+    # unique_id = str(uuid.uuid4()).replace("-", "")
+    unique_id = "".join(
+        random.choice(string.ascii_lowercase + string.digits)
+        for _ in range(UNIQUE_HASH_SIZE)
+    )
+    return f"test_dag_{unique_id}"
+
+
 @pytest.fixture
 def sample_dag():
-    yield DAG("test_dag", default_args={"owner": "airflow", "start_date": DEFAULT_DATE})
+    dag_id = _create_unique_dag_id()
+    yield DAG(dag_id, default_args={"owner": "airflow", "start_date": DEFAULT_DATE})
     with create_session() as session:
         session.query(DagRun).delete()
         session.query(TI).delete()
@@ -46,17 +60,28 @@ def tmp_table(sql_server):
     sql_name, hook = sql_server
 
     if isinstance(hook, SnowflakeHook):
-        return TempTable(
+        temporary_table = TempTable(
             conn_id=hook.snowflake_conn_id,
             database=hook.database,
             warehouse=hook.warehouse,
         )
     elif isinstance(hook, PostgresHook):
-        return TempTable(conn_id=hook.postgres_conn_id, database=hook.schema)
+        temporary_table = TempTable(conn_id=hook.postgres_conn_id, database=hook.schema)
     elif isinstance(hook, SqliteHook):
-        return TempTable(conn_id=hook.sqlite_conn_id, database="sqlite")
+        temporary_table = TempTable(conn_id=hook.sqlite_conn_id, database="sqlite")
     elif isinstance(hook, BigQueryHook):
-        return TempTable(conn_id=hook.gcp_conn_id)
+        temporary_table = TempTable(conn_id=hook.gcp_conn_id)
+    yield temporary_table
+
+    if isinstance(hook, SqliteHook):
+        hook.run(f"DROP TABLE IF EXISTS {temporary_table.table_name}")
+        hook.run(f"DROP INDEX IF EXISTS unique_index")
+    elif isinstance(hook, SnowflakeHook):
+        hook.run(f"DROP TABLE IF EXISTS {temporary_table.table_name}")
+    elif not isinstance(hook, BigQueryHook):
+        # There are some tests (e.g. test_agnostic_merge.py) which create stuff which are not being deleted
+        # Example: tables which are not fixtures and constraints. This is an agressive approach towards tearing down:
+        hook.run(f"DROP SCHEMA IF EXISTS {temporary_table.schema} CASCADE;")
 
 
 @pytest.fixture
diff --git a/src/astro/sql/table.py b/src/astro/sql/table.py
--- a/src/astro/sql/table.py
+++ b/src/astro/sql/table.py
@@ -16,6 +16,8 @@
 from airflow.hooks.base import BaseHook
 from airflow.models import DagRun, TaskInstance
 
+MAX_TABLE_NAME_SIZE = 63
+
 
 class Table:
     template_fields = (
@@ -68,11 +70,17 @@ def __str__(self):
 
 class TempTable(Table):
     def __init__(self, conn_id=None, database=None, warehouse=""):
+        self.table_name = ""
         super().__init__(
-            table_name="", conn_id=conn_id, database=database, warehouse=warehouse
+            table_name=self.table_name,
+            conn_id=conn_id,
+            database=database,
+            warehouse=warehouse,
         )
 
     def to_table(self, table_name: str, schema: str) -> Table:
+        self.table_name = table_name
+        self.schema = schema
         return Table(
             table_name=table_name,
             conn_id=self.conn_id,
@@ -87,7 +95,7 @@ def create_table_name(context) -> str:
     dag_run: DagRun = ti.get_dagrun()
     table_name = f"{dag_run.dag_id}_{ti.task_id}_{dag_run.id}".replace(
         "-", "_"
-    ).replace(".", "__")
+    ).replace(".", "__")[:MAX_TABLE_NAME_SIZE]
     if not table_name.isidentifier():
         table_name = f'"{table_name}"'
     return table_name


Test patch:
diff --git a/tests/operators/test_agnostic_load_file.py b/tests/operators/test_agnostic_load_file.py
--- a/tests/operators/test_agnostic_load_file.py
+++ b/tests/operators/test_agnostic_load_file.py
@@ -669,4 +669,5 @@ def test_load_file_chunks(sample_dag, sql_server):
         task_params = create_task_parameters(database_name, file_type)
         test_utils.create_and_run_task(sample_dag, load_file, (), task_params)
 
-    assert mock_chunk_function.call_args_list[0].kwargs[chunk_size_argument] == 1000000
+    _, kwargs = mock_chunk_function.call_args
+    assert kwargs[chunk_size_argument] == 1000000