Repository: fal-ai/dbt-fal

Issue:
write_to_source throws on certain characters
We tried to fix this for https://github.com/fal-ai/fal/issues/121 in https://github.com/fal-ai/fal/pull/150 by simply escaping the known jinja syntax: `{` and `}`; but it comes back with other characters making it fail too:

![image](https://user-images.githubusercontent.com/2745502/155373592-47dc1c2b-14fd-4e55-ae33-2737e58eeb14.png)
> image from @cesar-loadsmart in [Discord](https://discord.com/channels/908693336280432750/940966301680156683/946065009857921105)

***** 

I think the real solution for this would be to try and disable Jinja processing altogether in `write_to_source` (and other functions, maybe?)

Internally, we call dbt (since it holds the credentials from the profiles) in this lib method
https://github.com/fal-ai/fal/blob/d76cf2cfdcb1d6bf6d44553e8665fa97ec792856/src/faldbt/lib.py#L103-L132

We could maybe control how `adapter.execute` is called to avoid processing Jinja in it.




Discussion:

Resolving patch:
diff --git a/src/faldbt/cp/parser/sql.py b/src/faldbt/cp/parser/sql.py
deleted file mode 100644
--- a/src/faldbt/cp/parser/sql.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# NOTE: COPIED FROM https://github.com/dbt-labs/dbt-core/blob/89907f09c8a94d95d2882f909f05143e098746a0/core/dbt/parser/sql.py
-import os
-from dataclasses import dataclass
-from typing import Iterable
-
-from dbt.contracts.graph.manifest import SourceFile
-from dbt.contracts.graph.unparsed import UnparsedMacro
-from dbt.exceptions import InternalException
-from dbt.parser.base import SimpleSQLParser
-from dbt.parser.macros import MacroParser
-from dbt.parser.search import FileBlock
-
-from faldbt.cp.contracts.graph.parsed import ParsedSqlNode, ParsedMacro
-from faldbt.cp.node_types import NodeType
-
-
-@dataclass
-class SqlBlock(FileBlock):
-    block_name: str
-
-    @property
-    def name(self):
-        return self.block_name
-
-
-class SqlBlockParser(SimpleSQLParser[ParsedSqlNode]):
-    def parse_from_dict(self, dct, validate=True) -> ParsedSqlNode:
-        if validate:
-            ParsedSqlNode.validate(dct)
-        return ParsedSqlNode.from_dict(dct)
-
-    @property
-    def resource_type(self) -> NodeType:
-        return NodeType.SqlOperation
-
-    @staticmethod
-    def get_compiled_path(block: FileBlock):
-        # we do it this way to make mypy happy
-        if not isinstance(block, SqlBlock):
-            raise InternalException(
-                'While parsing SQL operation, got an actual file block instead of '
-                'an SQL block: {}'.format(block)
-            )
-
-        return os.path.join('sql', block.name)
-
-    def parse_remote(self, sql: str, name: str) -> ParsedSqlNode:
-        source_file = SourceFile.remote(sql, self.project.project_name)
-        contents = SqlBlock(block_name=name, file=source_file)
-        return self.parse_node(contents)
-
-
-class SqlMacroParser(MacroParser):
-    def parse_remote(self, contents) -> Iterable[ParsedMacro]:
-        base = UnparsedMacro(
-            path='from remote system',
-            original_file_path='from remote system',
-            package_name=self.project.project_name,
-            raw_sql=contents,
-            root_path=self.project.project_root,
-            resource_type=NodeType.Macro,
-        )
-        for node in self.parse_unparsed_macros(base):
-            yield node
diff --git a/src/faldbt/lib.py b/src/faldbt/lib.py
--- a/src/faldbt/lib.py
+++ b/src/faldbt/lib.py
@@ -12,7 +12,6 @@
 from dbt.config.runtime import RuntimeConfig
 from dbt.contracts.connection import AdapterResponse
 from dbt.contracts.graph.manifest import Manifest
-from dbt.parser.manifest import process_node
 from dbt.logger import GLOBAL_LOGGER as logger
 
 from . import parse
@@ -23,18 +22,14 @@
 import sqlalchemy
 from sqlalchemy.sql.ddl import CreateTable
 from sqlalchemy.sql import Insert
-from sqlalchemy.sql.schema import MetaData
-
 
 DBT_V1 = dbt.semver.VersionSpecifier.from_version_string("1.0.0")
 DBT_VCURRENT = dbt.version.get_installed_version()
 
 if DBT_VCURRENT.compare(DBT_V1) >= 0:
-    from dbt.parser.sql import SqlBlockParser
     from dbt.contracts.graph.parsed import ParsedModelNode, ParsedSourceDefinition
     from dbt.contracts.sql import ResultTable, RemoteRunResult
 else:
-    from faldbt.cp.parser.sql import SqlBlockParser
     from faldbt.cp.contracts.graph.parsed import ParsedModelNode, ParsedSourceDefinition
     from faldbt.cp.contracts.sql import ResultTable, RemoteRunResult
 
@@ -75,23 +70,6 @@ def register_adapters(config: RuntimeConfig):
     adapters_factory.register_adapter(config)
 
 
-def _get_operation_node(manifest: Manifest, project_path, profiles_dir, sql):
-
-    config = parse.get_dbt_config(project_path, profiles_dir)
-    block_parser = SqlBlockParser(
-        project=config,
-        manifest=manifest,
-        root_project=config,
-    )
-
-    # NOTE: nodes get registered to the manifest automatically,
-    # HACK: we need to include uniqueness (UUID4) to avoid clashes
-    name = "SQL:" + str(hash(sql)) + ":" + str(uuid4())
-    sql_node = block_parser.parse_remote(sql, name)
-    process_node(config, manifest, sql_node)
-    return sql_node
-
-
 # NOTE: Once we get an adapter, we must call `connection_for` or `connection_named` to use it
 def _get_adapter(project_path: str, profiles_dir: str):
     config = parse.get_dbt_config(project_path, profiles_dir)
@@ -103,13 +81,14 @@ def _get_adapter(project_path: str, profiles_dir: str):
 def _execute_sql(
     manifest: Manifest, project_path: str, profiles_dir: str, sql: str
 ) -> Tuple[AdapterResponse, RemoteRunResult]:
-    node = _get_operation_node(manifest, project_path, profiles_dir, sql)
     adapter = _get_adapter(project_path, profiles_dir)
 
     logger.debug("Running query\n{}", sql)
 
+    # HACK: we need to include uniqueness (UUID4) to avoid clashes
+    name = "SQL:" + str(hash(sql)) + ":" + str(uuid4())
     result = None
-    with adapter.connection_for(node):
+    with adapter.connection_named(name, node=None):
         adapter.connections.begin()
         response, execute_result = adapter.execute(sql, fetch=True)
 
@@ -121,7 +100,7 @@ def _execute_sql(
         result = RemoteRunResult(
             raw_sql=sql,
             compiled_sql=sql,
-            node=node,
+            node=None,
             table=table,
             timing=[],
             logs=[],
@@ -177,7 +156,7 @@ def write_target(
     project_path: str,
     profiles_dir: str,
     target: Union[ParsedModelNode, ParsedSourceDefinition],
-    dtype=None
+    dtype=None,
 ) -> RemoteRunResult:
     adapter = _get_adapter(project_path, profiles_dir)
 
@@ -190,10 +169,6 @@ def write_target(
 
     column_names: List[str] = list(data.columns)
 
-    # Escape { and } in row data
-    data = data.replace('{', r'\{', regex=True)
-    data = data.replace('}', r'\}', regex=True)
-
     rows = data.to_records(index=False)
     row_dicts = list(map(lambda row: dict(zip(column_names, row)), rows))
 


Test patch:
diff --git a/tests/mock/models/schema.yml b/tests/mock/models/schema.yml
--- a/tests/mock/models/schema.yml
+++ b/tests/mock/models/schema.yml
@@ -6,6 +6,7 @@ sources:
     schema: public
     tables:
       - name: single_col
+      - name: sql_col
 
 models:
   - name: model_with_scripts
diff --git a/tests/test_import.py b/tests/test_import.py
--- a/tests/test_import.py
+++ b/tests/test_import.py
@@ -6,6 +6,7 @@
 profiles_dir = os.path.join(Path.cwd(), "tests/mock/mockProfile")
 project_dir = os.path.join(Path.cwd(), "tests/mock")
 
+
 def test_initialize():
     faldbt = FalDbt(
         profiles_dir=profiles_dir,
diff --git a/tests/test_regressions.py b/tests/test_regressions.py
new file mode 100644
--- /dev/null
+++ b/tests/test_regressions.py
@@ -0,0 +1,23 @@
+from fal import FalDbt
+from pathlib import Path
+import os
+import pandas as pd
+
+profiles_dir = os.path.join(Path.cwd(), "tests/mock/mockProfile")
+project_dir = os.path.join(Path.cwd(), "tests/mock")
+
+
+# https://github.com/fal-ai/fal/issues/154
+def test_write_to_source_not_processing_jinja():
+    faldbt = FalDbt(
+        profiles_dir=profiles_dir,
+        project_dir=project_dir,
+    )
+
+    df = pd.DataFrame({"sql": [r"SELECT 1 FROM {{ wrong jinja }}"]})
+
+    faldbt.write_to_source(df, "test_sources", "sql_col")
+
+    # TODO: look at df data
+    df = faldbt.source("test_sources", "sql_col")
+    assert df.sql.get(0) == r"SELECT 1 FROM {{ wrong jinja }}"