Repository: tobymao/sqlglot

Issue:
Parsing of insert overwrite directory fails in hive dialect
**Before you file an issue**
[x] Make sure you specify the "read" dialect eg. `parse_one(sql, read="spark")`
[x] Make sure you specify the "write" dialect eg. `ast.sql(dialect="duckdb")`
[x] Check if the issue still exists on main

**Fully reproducible code snippet**
```
q = """INSERT OVERWRITE DIRECTORY
's3a://path' ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' LINES TERMINATED BY '
'
STORED AS TEXTFILE
SELECT * FROM `a`.`b`"""
asts = sqlglot.parse(q, dialect='hive')
```

Exception:
```
File /opt/miniconda3/envs/databricks-repos/lib/python3.10/site-packages/sqlglot/parser.py:1170, in Parser.parse(self, raw_tokens, sql)
   1156 def parse(
   1157     self, raw_tokens: t.List[Token], sql: t.Optional[str] = None
   1158 ) -> t.List[t.Optional[exp.Expression]]:
   1159     """
   1160     Parses a list of tokens and returns a list of syntax trees, one tree
   1161     per parsed SQL statement.
   (...)
   1168         The list of the produced syntax trees.
   1169     """
-> 1170     return self._parse(
   1171         parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql
   1172     )

File /opt/miniconda3/envs/databricks-repos/lib/python3.10/site-packages/sqlglot/parser.py:1239, in Parser._parse(self, parse_method, raw_tokens, sql)
   1236     expressions.append(parse_method(self))
   1238     if self._index < len(self._tokens):
-> 1239         self.raise_error("Invalid expression / Unexpected token")
   1241     self.check_errors()
   1243 return expressions

File /opt/miniconda3/envs/databricks-repos/lib/python3.10/site-packages/sqlglot/parser.py:1280, in Parser.raise_error(self, message, token)
   1268 error = ParseError.new(
   1269     f"{message}. Line {token.line}, Col: {token.col}.\n"
   1270     f"  {start_context}\033[4m{highlight}\033[0m{end_context}",
   (...)
   1276     end_context=end_context,
   1277 )
   1279 if self.error_level == ErrorLevel.IMMEDIATE:
-> 1280     raise error
   1282 self.errors.append(error)

ParseError: Invalid expression / Unexpected token. Line 4, Col: 6.
  INATED BY '' COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' LINES TERMINATED BY '
'
STORED AS TEXTFILE
SELECT * FROM `a`.`b`
```

**Official Documentation**
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DML


Comments:

Patch:
diff --git a/sqlglot/expressions.py b/sqlglot/expressions.py
index 5adbb1e5..d97807a3 100644
--- a/sqlglot/expressions.py
+++ b/sqlglot/expressions.py
@@ -2063,6 +2063,7 @@ class Insert(DDL, DML):
         "where": False,
         "ignore": False,
         "by_name": False,
+        "stored": False,
     }
 
     def with_(
diff --git a/sqlglot/generator.py b/sqlglot/generator.py
index b7da18b3..23b8d9c6 100644
--- a/sqlglot/generator.py
+++ b/sqlglot/generator.py
@@ -1520,6 +1520,8 @@ class Generator(metaclass=_Generator):
         else:
             this = self.INSERT_OVERWRITE if overwrite else " INTO"
 
+        stored = self.sql(expression, "stored")
+        stored = f" {stored}" if stored else ""
         alternative = expression.args.get("alternative")
         alternative = f" OR {alternative}" if alternative else ""
         ignore = " IGNORE" if expression.args.get("ignore") else ""
@@ -1545,7 +1547,7 @@ class Generator(metaclass=_Generator):
         else:
             expression_sql = f"{returning}{expression_sql}{on_conflict}"
 
-        sql = f"INSERT{hint}{alternative}{ignore}{this}{by_name}{exists}{partition_sql}{where}{expression_sql}"
+        sql = f"INSERT{hint}{alternative}{ignore}{this}{stored}{by_name}{exists}{partition_sql}{where}{expression_sql}"
         return self.prepend_ctes(expression, sql)
 
     def intersect_sql(self, expression: exp.Intersect) -> str:
diff --git a/sqlglot/parser.py b/sqlglot/parser.py
index 2aaba600..9c075dc7 100644
--- a/sqlglot/parser.py
+++ b/sqlglot/parser.py
@@ -2248,6 +2248,7 @@ class Parser(metaclass=_Parser):
             hint=hint,
             is_function=is_function,
             this=this,
+            stored=self._match_text_seq("STORED") and self._parse_stored(),
             by_name=self._match_text_seq("BY", "NAME"),
             exists=self._parse_exists(),
             partition=self._parse_partition(),


Test patch:
diff --git a/tests/dialects/test_hive.py b/tests/dialects/test_hive.py
index 33294ee0..d52510d2 100644
--- a/tests/dialects/test_hive.py
+++ b/tests/dialects/test_hive.py
@@ -428,6 +428,9 @@ class TestHive(Validator):
         self.validate_identity(
             "INSERT OVERWRITE TABLE zipcodes PARTITION(state = 0) VALUES (896, 'US', 'TAMPA', 33607)"
         )
+        self.validate_identity(
+            "INSERT OVERWRITE DIRECTORY 'x' ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' LINES TERMINATED BY '' STORED AS TEXTFILE SELECT * FROM `a`.`b`"
+        )
         self.validate_identity(
             "SELECT a, b, SUM(c) FROM tabl AS t GROUP BY a, b, GROUPING SETS ((a, b), a)"
         )