Repository: reata/sqllineage (2019-07-26 ~ 2019-08-11)

problem_statement: subquery mistake alias as table name
```
$ python -m sqllineage.core -e "SELECT col1 FROM (SELECT col2 from tab1) dt"
Statements(#): 1
Source Tables:
    dt
Target Tables:

```

expect source table as tab1
date: 2019-07-26
patch: diff --git a/sqllineage/core.py b/sqllineage/core.py
index 301c818..2f5e0b6 100644
--- a/sqllineage/core.py
+++ b/sqllineage/core.py
@@ -3,7 +3,7 @@ import sys
 from typing import List, Set
 
 import sqlparse
-from sqlparse.sql import Function, Identifier, IdentifierList, Statement, TokenList
+from sqlparse.sql import Function, Identifier, Parenthesis, Statement, TokenList
 from sqlparse.tokens import Keyword, Token, Whitespace
 
 SOURCE_TABLE_TOKENS = ('FROM', 'JOIN', 'INNER JOIN', 'LEFT JOIN', 'RIGHT JOIN', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN',
@@ -51,11 +51,9 @@ Target Tables:
         return self._target_tables
 
     def _extract_from_token(self, token: Token):
-        if not isinstance(token, TokenList):
-            return
         source_table_token_flag = target_table_token_flag = temp_table_token_flag = False
         for sub_token in token.tokens:
-            if isinstance(token, TokenList) and not isinstance(sub_token, (Identifier, IdentifierList)):
+            if isinstance(sub_token, TokenList):
                 self._extract_from_token(sub_token)
             if sub_token.ttype in Keyword:
                 if sub_token.normalized in SOURCE_TABLE_TOKENS:
@@ -75,7 +73,13 @@ Target Tables:
                     continue
                 else:
                     assert isinstance(sub_token, Identifier)
-                    self._source_tables.add(sub_token.get_real_name())
+                    if isinstance(sub_token.token_first(), Parenthesis):
+                        # SELECT col1 FROM (SELECT col2 FROM tab1) dt, the subquery will be parsed as Identifier
+                        # and this Identifier's get_real_name method would return alias name dt
+                        # referring https://github.com/andialbrecht/sqlparse/issues/218 for further information
+                        pass
+                    else:
+                        self._source_tables.add(sub_token.get_real_name())
                     source_table_token_flag = False
             elif target_table_token_flag:
                 if sub_token.ttype == Whitespace:

test_patch: diff --git a/tests/test_select.py b/tests/test_select.py
index cea96ef..49fe323 100644
--- a/tests/test_select.py
+++ b/tests/test_select.py
@@ -40,6 +40,10 @@ def test_select_count():
     helper("SELECT COUNT(*) FROM tab1", {"tab1"})
 
 
+def test_select_subquery():
+    helper("SELECT col1 FROM (SELECT col2 FROM tab1) dt", {"tab1"})
+
+
 def test_select_inner_join():
     helper("SELECT * FROM tab1 INNER JOIN tab2", {"tab1", "tab2"})
 


----

problem_statement: multi-line sql causes AssertionError
```sql
SELECT * FROM
tab1
```
causes following exception

Traceback (most recent call last):
  File "/home/admin/.pyenv/versions/3.6.8/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/admin/.pyenv/versions/3.6.8/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/admin/repos/sqllineage/sqllineage/core.py", line 131, in <module>
    main()
  File "/home/admin/repos/sqllineage/sqllineage/core.py", line 119, in main
    print(LineageParser(sql))
  File "/home/admin/repos/sqllineage/sqllineage/core.py", line 23, in __init__
    self._extract_from_token(stmt)
  File "/home/admin/repos/sqllineage/sqllineage/core.py", line 75, in _extract_from_token
    assert isinstance(sub_token, Identifier)
AssertionError
date: 2019-08-11
patch: diff --git a/sqllineage/core.py b/sqllineage/core.py
index 2f5e0b6..fbf17f2 100644
--- a/sqllineage/core.py
+++ b/sqllineage/core.py
@@ -4,7 +4,7 @@ from typing import List, Set
 
 import sqlparse
 from sqlparse.sql import Function, Identifier, Parenthesis, Statement, TokenList
-from sqlparse.tokens import Keyword, Token, Whitespace
+from sqlparse.tokens import Keyword, Token
 
 SOURCE_TABLE_TOKENS = ('FROM', 'JOIN', 'INNER JOIN', 'LEFT JOIN', 'RIGHT JOIN', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN',
                        'FULL OUTER JOIN', 'CROSS JOIN')
@@ -69,7 +69,7 @@ Target Tables:
                 self._target_tables.add(sub_token.get_alias())
                 continue
             if source_table_token_flag:
-                if sub_token.ttype == Whitespace:
+                if sub_token.is_whitespace:
                     continue
                 else:
                     assert isinstance(sub_token, Identifier)
@@ -82,7 +82,7 @@ Target Tables:
                         self._source_tables.add(sub_token.get_real_name())
                     source_table_token_flag = False
             elif target_table_token_flag:
-                if sub_token.ttype == Whitespace:
+                if sub_token.is_whitespace:
                     continue
                 elif isinstance(sub_token, Function):
                     # insert into tab (col1, col2), tab (col1, col2) will be parsed as Function
@@ -95,7 +95,7 @@ Target Tables:
                     self._target_tables.add(sub_token.get_real_name())
                     target_table_token_flag = False
             elif temp_table_token_flag:
-                if sub_token.ttype == Whitespace:
+                if sub_token.is_whitespace:
                     continue
                 else:
                     assert isinstance(sub_token, Identifier)

test_patch: diff --git a/tests/test_select.py b/tests/test_select.py
index 49fe323..8837091 100644
--- a/tests/test_select.py
+++ b/tests/test_select.py
@@ -5,6 +5,11 @@ def test_select():
     helper("SELECT col1 FROM tab1", {"tab1"})
 
 
+def test_select_multi_line():
+    helper("""SELECT col1 FROM
+tab1""", {"tab1"})
+
+
 def test_select_asterisk():
     helper("SELECT * FROM tab1", {"tab1"})
 


----

problem_statement: drop table parsed as target table
```
$ sqllineage -e "DROP TABLE IF EXISTS tab1"
Statements(#): 1
Source Tables:
    
Target Tables:
    tab1
```

expect:
When a table is in target_tables, dropping table result in a removal from target_tables. Otherwise  the DROP DML doesn't affect the result.
date: 2019-08-11
patch: diff --git a/sqllineage/core.py b/sqllineage/core.py
index fbf17f2..74c178d 100644
--- a/sqllineage/core.py
+++ b/sqllineage/core.py
@@ -4,7 +4,7 @@ from typing import List, Set
 
 import sqlparse
 from sqlparse.sql import Function, Identifier, Parenthesis, Statement, TokenList
-from sqlparse.tokens import Keyword, Token
+from sqlparse.tokens import DDL, Keyword, Token
 
 SOURCE_TABLE_TOKENS = ('FROM', 'JOIN', 'INNER JOIN', 'LEFT JOIN', 'RIGHT JOIN', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN',
                        'FULL OUTER JOIN', 'CROSS JOIN')
@@ -20,7 +20,10 @@ class LineageParser(object):
         self._target_tables = set()
         self._stmt = sqlparse.parse(sql, self._encoding)
         for stmt in self._stmt:
-            self._extract_from_token(stmt)
+            if stmt.token_first().ttype == DDL and stmt.token_first().normalized == "DROP":
+                self._target_tables -= {t.get_real_name() for t in stmt.tokens if isinstance(t, Identifier)}
+            else:
+                self._extract_from_token(stmt)
         self._tmp_tables = self._source_tables.intersection(self._target_tables)
         self._source_tables -= self._tmp_tables
         self._target_tables -= self._tmp_tables

test_patch: diff --git a/tests/test_others.py b/tests/test_others.py
index a6f8472..6fc57c9 100644
--- a/tests/test_others.py
+++ b/tests/test_others.py
@@ -6,6 +6,26 @@ def test_use():
     helper("USE db1")
 
 
+def test_create():
+    helper("CREATE TABLE tab1 (col1 STRING)", None, {"tab1"})
+
+
+def test_create_if_not_exist():
+    helper("CREATE TABLE IF NOT EXISTS tab1 (col1 STRING)", None, {"tab1"})
+
+
+def test_create_after_drop():
+    helper("DROP TABLE IF EXISTS tab1; CREATE TABLE IF NOT EXISTS tab1 (col1 STRING)", None, {"tab1"})
+
+
+def test_drop():
+    helper("DROP TABLE IF EXISTS tab1", None, None)
+
+
+def test_drop_after_create():
+    helper("CREATE TABLE IF NOT EXISTS tab1 (col1 STRING);DROP TABLE IF EXISTS tab1", None, None)
+
+
 def test_create_select():
     helper("CREATE TABLE tab1 SELECT * FROM tab2", {"tab2"}, {"tab1"})