Repository: prql/prql
Pull request #405 (2022-04-23 14:35:04)

Adding to_json in prql-compiler/src/lib , and exposing to python
@max-sixty There are some python tests in prql-python/python/tests that would be nice to have run before the Release phase as a check before uploading, I could not get them to run correctly.  Pinging @rbpatt2019 also he might know how to integrate running the python tests before Release in .github/workflows/python.yaml 

max-sixty: PR looks good!

Why wouldn't those tests run correctly? What happens if we try running pytest on them?
rbpatt2019: Can definitely add a pytest step to this. Does that need to come before or after the builds? My recommendation would be before, if possible - save ourselves the build time if the tests don't pass.
rbpatt2019: Once I build locally, the tests fail complaining that there is no prql_python.compile method. Think we need to export this directly to pyo3 if you want to call it in python.

charlie-sanders: @rbpatt2019 after the local build are you calling 'maturin develop' ?  Thats what builds the python module, its happens with this step 
```
      - uses: messense/maturin-action@v1
        with:
          manylinux: auto
          command: build
          args: --release -o dist -m prql-python/Cargo.toml
``` , so we probably need a pytest after that for each platform.

@max-sixty the workflow would completely stop running when I was trying to develop the workflow , let me try again might have been a fluke.


rbpatt2019: Yep. I build the wheel, install the wheel, `import prql_python as prql`, and then get `module prql_python has no attribute compile`.

Also, can't push commits, but I can definitely use a composite action to refactor the action into a single matrix covering all OS and python versions.

rbpatt2019: Locally, I've got everything resolved (action for multiple os build, multiple python test) excepting the error. Just need to expose compile in some fashion...put I'll need push permission for the PR to share all this.

charlie-sanders: Oh in the python bindings they were renamed to `to_sql`, when I added the `to_json` method @rbpatt2019 , should we add a compile as well ?  Can you dir(prql_python) see what methods are defined on it ? 
charlie-sanders: Closing this so not to dirty the Actions with my fumbling about
max-sixty: > Closing this so not to dirty the Actions with my fumbling about

There's no dirtying! It's totally fine to do whatever is helpful on any non-main branch!
rbpatt2019: Sorry I missed the bug here! What was causing the issues with the actions?

rbpatt2019: I've fixed the python tests - specified the wrong diet directory for installing `prql_python`. They are pushed [here](https://github.com/charlie-sanders/prql).

Patch:
diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml
new file mode 100644
index 00000000..d9124159
--- /dev/null
+++ b/.github/actions/build/action.yaml
@@ -0,0 +1,35 @@
+name: build
+description: "Use maturin to build python dists of prql-python. Conditional if checks ensure only the correct OX build is called."
+
+inputs:
+  OS:
+    description: "The operating system of the build"
+    required: True
+    default: 'ubuntu-latest'
+
+runs:
+  using: composite
+  steps:
+      - uses: actions/checkout@v2
+      - uses: Swatinem/rust-cache@v1
+      - uses: messense/maturin-action@v1
+        if: inputs.OS == 'ubuntu-latest'
+        with:
+          manylinux: auto
+          command: build
+          args: --release -o dist -m prql-python/Cargo.toml
+      - uses: messense/maturin-action@v1
+        if: inputs.OS == 'windows-latest'
+        with:
+          command: build
+          args: --release --no-sdist -o dist -m prql-python/Cargo.toml
+      - uses: messense/maturin-action@v1
+        if: inputs.OS == 'macos-latest'
+        with:
+          command: build
+          args: --release --no-sdist -o dist --universal2 -m prql-python/Cargo.toml
+      - name: Upload wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: dist
diff --git a/.github/actions/setup_nox/action.yml b/.github/actions/setup_nox/action.yml
new file mode 100644
index 00000000..58c88875
--- /dev/null
+++ b/.github/actions/setup_nox/action.yml
@@ -0,0 +1,33 @@
+name: Setup Nox
+description: 'Prepare and cache nox, install poetry.'
+
+inputs:
+  cache_number:
+    description: "A manually specified cache number. Useful for triggering new caches."
+    required: True
+    default: 0
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v3
+      with:
+        python-version: "3.7"
+    - uses: actions/setup-python@v3
+      with:
+        python-version: "3.8"
+    - uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: "Install nox"
+      run: pipx install nox
+      shell: bash
+    - name: "Cache Nox"
+      uses: actions/cache@v2
+      with:
+        path: .nox
+        key:
+          nox-${{inputs.cache_number }}-${{ hashFiles('prql-python/noxfile.py') }}-${{ hashFiles('prql-python/requirements.txt') }}
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index e0171d22..0b90308f 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -23,67 +23,29 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # TODO: would there be a way of integrating these into a single matrix job?
-  # Note that the commands are slightly different atm.
-  linux:
-    runs-on: ubuntu-latest
-    steps:
-      - name: 📂 Checkout code
-        uses: actions/checkout@v2
-      - name: 💰 Cache
-        uses: Swatinem/rust-cache@v1
-      - uses: messense/maturin-action@v1
-        with:
-          manylinux: auto
-          command: build
-          args: --release -o dist -m prql-python/Cargo.toml
-      - name: Upload wheels
-        uses: actions/upload-artifact@v2
-        with:
-          name: wheels
-          path: dist
-
-  windows:
-    runs-on: windows-latest
+  build_python:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
     steps:
-      - name: 📂 Checkout code
+      - name: Checkout
         uses: actions/checkout@v2
-      - name: 💰 Cache
-        uses: Swatinem/rust-cache@v1
-      - uses: messense/maturin-action@v1
+      - name: Build Wheels
+        uses: ./.github/actions/build
         with:
-          command: build
-          args: --release --no-sdist -o dist -m prql-python/Cargo.toml
-      - name: Upload wheels
-        uses: actions/upload-artifact@v2
+          OS: ${{ matrix.os }}
+      - name: Setup Nox
+        uses: ./.github/actions/setup_nox
         with:
-          name: wheels
-          path: dist
-
-  macos:
-    runs-on: macos-latest
-    steps:
-      - name: 📂 Checkout code
-        uses: actions/checkout@v2
-      - name: 💰 Cache
-        uses: Swatinem/rust-cache@v1
-      - uses: messense/maturin-action@v1
-        with:
-          command: build
-          args: --release --no-sdist -o dist --universal2 -m prql-python/Cargo.toml
-      - name: Upload wheels
-        uses: actions/upload-artifact@v2
-        with:
-          name: wheels
-          path: dist
-
+          cache_number: 0
+      - name: Run tests
+        run: nox -s tests -f prql-python/noxfile.py
   release:
     name: Release
     runs-on: ubuntu-latest
-    # TODO: Upload PYPI_API_TOKEN, confirm how we release all crates together.
-    # if: "startsWith(github.ref, 'refs/tags/')"
-    if: false
-    needs: [macos, windows, linux]
+    if: "startsWith(github.ref, 'refs/tags/')"
+    needs: [ build_python ]
     steps:
       - uses: actions/download-artifact@v2
         with:
@@ -91,6 +53,7 @@ jobs:
       - name: Publish to PyPI
         uses: messense/maturin-action@v1
         env:
+          # TODO: Upload PYPI_API_TOKEN, confirm how we release all crates together.
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
         with:
           command: upload
diff --git a/Cargo.lock b/Cargo.lock
index f062e13f..637fbe6c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1134,6 +1134,7 @@ dependencies = [
  "pulldown-cmark",
  "pulldown-cmark-to-cmark",
  "serde",
+ "serde_json",
  "serde_yaml",
  "similar",
  "sqlformat",
@@ -1156,7 +1157,7 @@ dependencies = [
 
 [[package]]
 name = "prql-python"
-version = "0.0.2"
+version = "0.0.3"
 dependencies = [
  "prql-compiler",
  "pyo3",
diff --git a/README.md b/README.md
index b0071939..804eb061 100644
--- a/README.md
+++ b/README.md
@@ -265,7 +265,7 @@ See above for fuller examples of PRQL.
 ### Python implementation
 
 There is a python implementation at
-[qorrect/PyPrql](https://github.com/qorrect/PyPrql), which can be installed with
+[prql/PyPrql](https://github.com/prql/PyPrql), which can be installed with
 `pip install pyprql`. It has some great features, including a native interactive
 console with auto-complete for column names.
 
@@ -434,7 +434,7 @@ decisions on the direction of the language, and project administration:
 
 - [**@aljazerzen**](https://github.com/aljazerzen) — Aljaž Mur Eržen
 - [**@max-sixty**](https://github.com/max-sixty) — Maximilian Roos
-- [**@qorrect**](https://github.com/qorrect) — Charlie Sando
+- [**@charlie-sanders**](https://github.com/charlie-sanders) — Charlie Sanders
 
 We welcome others to join who have a track record of contributions.
 
diff --git a/prql-compiler/Cargo.toml b/prql-compiler/Cargo.toml
index 60103241..2ed38642 100644
--- a/prql-compiler/Cargo.toml
+++ b/prql-compiler/Cargo.toml
@@ -26,6 +26,7 @@ itertools = "0.10.3"
 pest = "2.1.3"
 pest_derive = "2.1.0"
 serde_yaml = "0.8.23"
+serde_json = "1.0.79"
 sqlformat = "0.1.8"
 strum = { version = "0.24.0", features = ["std", "derive"] }# for converting enum variants to string
 strum_macros = "0.24.0"
diff --git a/prql-compiler/src/lib.rs b/prql-compiler/src/lib.rs
index 90e078e3..cc9ed016 100644
--- a/prql-compiler/src/lib.rs
+++ b/prql-compiler/src/lib.rs
@@ -35,6 +35,11 @@ pub fn format(prql: &str) -> Result<String> {
     parse(prql).map(display)
 }
 
+/// Compile a PRQL string into a JSON version of the Query.
+pub fn to_json(prql: &str) -> Result<String> {
+    Ok(serde_json::to_string(&parse(prql)?)?)
+}
+
 /// Exposes some library internals.
 ///
 /// They are primarily exposed for documentation. There may be issues with using
@@ -45,3 +50,19 @@ pub mod internals {
     pub use crate::ast::Node;
     pub use crate::utils::{IntoOnly, Only};
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::to_json;
+
+    #[test]
+    fn test_to_json() -> Result<()> {
+        let json = to_json("from employees | take 10")?;
+        // Since the AST is so in flux right now just test that the brackets are present
+        assert_eq!(json.chars().next().unwrap(), '{');
+        assert_eq!(json.chars().nth(json.len() - 1).unwrap(), '}');
+
+        Ok(())
+    }
+}
diff --git a/prql-python/Cargo.toml b/prql-python/Cargo.toml
index 339c2848..49bf027e 100644
--- a/prql-python/Cargo.toml
+++ b/prql-python/Cargo.toml
@@ -2,7 +2,7 @@
 build = "build.rs"
 edition = "2021"
 name = "prql-python"
-version = "0.0.2"
+version = "0.0.3"
 
 [lib]
 crate-type = ["cdylib"]
diff --git a/prql-python/noxfile.py b/prql-python/noxfile.py
new file mode 100644
index 00000000..cf239406
--- /dev/null
+++ b/prql-python/noxfile.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""Nox session configuration."""
+import os
+from typing import List
+from pathlib import Path
+import os
+import nox
+from nox.sessions import Session
+
+VERSIONS: List[str] = [
+    "3.7",
+    "3.8",
+    "3.9",
+    "3.10",
+]
+
+nox.options.stop_on_first_error = False
+nox.options.reuse_existing_virtualenvs = False
+
+
+@nox.session(python=VERSIONS)
+def tests(session: Session) -> None:
+    """Run the test suite with pytest."""
+    print('CWD', os.getcwd())
+    session.install("-v", "--no-index", f"--find-links={Path('..', 'dist')}", "prql_python")
+    session.install("-v", "-r", "requirements.txt")
+    session.run("pytest", str(Path("python", "tests")))
diff --git a/prql-python/pyproject.toml b/prql-python/pyproject.toml
index 2632e273..87ae6671 100644
--- a/prql-python/pyproject.toml
+++ b/prql-python/pyproject.toml
@@ -4,9 +4,15 @@ build-backend = "maturin"
 
 [project]
 name = "prql-python"
-requires-python = ">=3.6"
+requires-python = ">=3.7"
 classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
+
+[tool.pytest.ini_options]
+addopts = """
+-vv
+-n auto
+"""
diff --git a/prql-python/requirements.txt b/prql-python/requirements.txt
new file mode 100644
index 00000000..6c3ef184
--- /dev/null
+++ b/prql-python/requirements.txt
@@ -0,0 +1,3 @@
+pytest==7.0.0
+pytest-xdist==2.5.0
+pytest-clarity==1.0.1
diff --git a/prql-python/src/lib.rs b/prql-python/src/lib.rs
index 877fbc0f..f887bf93 100644
--- a/prql-python/src/lib.rs
+++ b/prql-python/src/lib.rs
@@ -15,9 +15,21 @@ pub fn to_sql(query: &str) -> PyResult<String> {
     }
 }
 
+#[pyfunction]
+pub fn to_json(query: &str) -> PyResult<String> {
+    match prql_compiler::to_json(query) {
+        Ok(sql) => Ok(sql),
+        Err(err) => Err(PyErr::new::<exceptions::PySyntaxError, _>(format!(
+            "{}",
+            err
+        ))),
+    }
+}
 #[pymodule]
 fn prql_python(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(to_sql, m)?)?;
+    m.add_function(wrap_pyfunction!(to_json, m)?)?;
+
     Ok(())
 }
 // This test below is causing a linking error here https://github.com/qorrect/prql/runs/6099160429?check_suite_focus=true


Test patch:
diff --git a/prql-python/python/tests/test_all.py b/prql-python/python/tests/test_all.py
new file mode 100644
index 00000000..5949fa2b
--- /dev/null
+++ b/prql-python/python/tests/test_all.py
@@ -0,0 +1,21 @@
+import prql_python as prql
+import json
+
+
+def test_all():
+    """
+    Test the basic python functions
+    """
+
+    prql_query = "from employee"
+
+    # Since the AST is so in flux lets just take these dont throw exceptions
+    res = json.loads(prql.to_json(prql_query))
+    assert res is not None
+
+    res = prql.to_sql(prql_query)
+    assert res is not None
+
+
+if __name__ == '__main__':
+    test_all()