Repository: getredash/redash

Issue:
Queries using other queries as a data source
I couldnt find an existing ticket for this so made this one. 

Redash seems to advertise that it supports adding queries as first class data sources here: https://redash.io/help/queries/using-datasets-as-data-sources.html

It would be really cool to have this functionality land in redash. @arikfr any plans on open sourcing the preview, or would you accept any PRs that add this functionality directly into redash?



Discussion:
It will be part of the v3 release.

Resolving patch:
diff --git a/redash/query_runner/query_results.py b/redash/query_runner/query_results.py
new file mode 100644
--- /dev/null
+++ b/redash/query_runner/query_results.py
@@ -0,0 +1,167 @@
+import json
+import logging
+import numbers
+import re
+import sqlite3
+
+from dateutil import parser
+
+from redash import models
+from redash.permissions import has_access, not_view_only
+from redash.query_runner import (TYPE_BOOLEAN, TYPE_DATETIME, TYPE_FLOAT,
+                                 TYPE_INTEGER, TYPE_STRING, BaseQueryRunner,
+                                 register)
+from redash.utils import JSONEncoder
+
+logger = logging.getLogger(__name__)
+
+
+class PermissionError(Exception):
+    pass
+
+
+def _guess_type(value):
+    if value == '' or value is None:
+        return TYPE_STRING
+
+    if isinstance(value, numbers.Integral):
+        return TYPE_INTEGER
+
+    if isinstance(value, float):
+        return TYPE_FLOAT
+
+    if unicode(value).lower() in ('true', 'false'):
+        return TYPE_BOOLEAN
+
+    try:
+        parser.parse(value)
+        return TYPE_DATETIME
+    except (ValueError, OverflowError):
+        pass
+
+    return TYPE_STRING
+
+
+def extract_query_ids(query):
+    queries = re.findall(r'(?:join|from) query_(\d+)', query, re.IGNORECASE)
+    return [int(q) for q in queries]
+
+
+def _load_query(user, query_id):
+    query = models.Query.get_by_id(query_id)
+
+    if user.org_id != query.org_id:
+        raise PermissionError("Query id {} not found.".format(query.id))
+
+    if not has_access(query.data_source.groups, user, not_view_only):
+        raise PermissionError(u"You are not allowed to execute queries on {} data source (used for query id {}).".format(
+            query.data_source.name, query.id))
+
+    return query
+
+
+def create_tables_from_query_ids(user, connection, query_ids):
+    for query_id in set(query_ids):
+        query = _load_query(user, query_id)
+
+        results, error = query.data_source.query_runner.run_query(
+            query.query_text, user)
+
+        if error:
+            raise Exception(
+                "Failed loading results for query id {}.".format(query.id))
+
+        results = json.loads(results)
+        table_name = 'query_{query_id}'.format(query_id=query_id)
+        create_table(connection, table_name, results)
+
+
+def fix_column_name(name):
+    return name.replace(':', '_').replace('.', '_').replace(' ', '_')
+
+
+def create_table(connection, table_name, query_results):
+    columns = [column['name']
+               for column in query_results['columns']]
+    safe_columns = [fix_column_name(column) for column in columns]
+
+    column_list = ", ".join(safe_columns)
+    create_table = "CREATE TABLE {table_name} ({column_list})".format(
+        table_name=table_name, column_list=column_list)
+    logger.debug("CREATE TABLE query: %s", create_table)
+    connection.execute(create_table)
+
+    insert_template = u"insert into {table_name} ({column_list}) values ({place_holders})".format(
+        table_name=table_name,
+        column_list=column_list,
+        place_holders=','.join(['?'] * len(columns)))
+
+    for row in query_results['rows']:
+        values = [row.get(column) for column in columns]
+        connection.execute(insert_template, values)
+
+
+class Results(BaseQueryRunner):
+    noop_query = 'SELECT 1'
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+            }
+        }
+
+    @classmethod
+    def annotate_query(cls):
+        return False
+
+    @classmethod
+    def name(cls):
+        return "Query Results (Beta)"
+
+    def run_query(self, query, user):
+        connection = sqlite3.connect(':memory:')
+
+        query_ids = extract_query_ids(query)
+        create_tables_from_query_ids(user, connection, query_ids)
+
+        cursor = connection.cursor()
+
+        try:
+            cursor.execute(query)
+
+            if cursor.description is not None:
+                columns = self.fetch_columns(
+                    [(i[0], None) for i in cursor.description])
+
+                rows = []
+                column_names = [c['name'] for c in columns]
+
+                for i, row in enumerate(cursor):
+                    for j, col in enumerate(row):
+                        guess = _guess_type(col)
+
+                        if columns[j]['type'] is None:
+                            columns[j]['type'] = guess
+                        elif columns[j]['type'] != guess:
+                            columns[j]['type'] = TYPE_STRING
+
+                    rows.append(dict(zip(column_names, row)))
+
+                data = {'columns': columns, 'rows': rows}
+                error = None
+                json_data = json.dumps(data, cls=JSONEncoder)
+            else:
+                error = 'Query completed but it returned no data.'
+                json_data = None
+        except KeyboardInterrupt:
+            connection.cancel()
+            error = "Query cancelled by user."
+            json_data = None
+        finally:
+            connection.close()
+        return json_data, error
+
+
+register(Results)
diff --git a/redash/settings.py b/redash/settings.py
--- a/redash/settings.py
+++ b/redash/settings.py
@@ -222,7 +222,8 @@ def all_settings():
     'redash.query_runner.jql',
     'redash.query_runner.google_analytics',
     'redash.query_runner.axibase_tsd',
-    'redash.query_runner.salesforce'
+    'redash.query_runner.salesforce',
+    'redash.query_runner.query_results'
 ]
 
 enabled_query_runners = array_from_string(os.environ.get("REDASH_ENABLED_QUERY_RUNNERS", ",".join(default_query_runners)))


Test patch:
diff --git a/tests/query_runner/test_query_results.py b/tests/query_runner/test_query_results.py
new file mode 100644
--- /dev/null
+++ b/tests/query_runner/test_query_results.py
@@ -0,0 +1,87 @@
+import sqlite3
+from unittest import TestCase
+
+from redash.query_runner.query_results import (PermissionError, _load_query, create_table,
+                                               extract_query_ids)
+from tests import BaseTestCase
+
+
+class TestExtractQueryIds(TestCase):
+    def test_works_with_simple_query(self):
+        query = "SELECT 1"
+        self.assertEquals([], extract_query_ids(query))
+
+    def test_finds_queries_to_load(self):
+        query = "SELECT * FROM query_123"
+        self.assertEquals([123], extract_query_ids(query))
+
+    def test_finds_queries_in_joins(self):
+        query = "SELECT * FROM query_123 JOIN query_4566"
+        self.assertEquals([123, 4566], extract_query_ids(query))
+
+
+class TestCreateTable(TestCase):
+    def test_creates_table_with_colons_in_column_name(self):
+        connection = sqlite3.connect(':memory:')
+        results = {'columns': [{'name': 'ga:newUsers'}, {
+            'name': 'test2'}], 'rows': [{'ga:newUsers': 123, 'test2': 2}]}
+        table_name = 'query_123'
+        create_table(connection, table_name, results)
+        connection.execute('SELECT 1 FROM query_123')
+
+    def test_creates_table(self):
+        connection = sqlite3.connect(':memory:')
+        results = {'columns': [{'name': 'test1'},
+                               {'name': 'test2'}], 'rows': []}
+        table_name = 'query_123'
+        create_table(connection, table_name, results)
+        connection.execute('SELECT 1 FROM query_123')
+
+    def test_creates_table_with_missing_columns(self):
+        connection = sqlite3.connect(':memory:')
+        results = {'columns': [{'name': 'test1'}, {'name': 'test2'}], 'rows': [
+            {'test1': 1, 'test2': 2}, {'test1': 3}]}
+        table_name = 'query_123'
+        create_table(connection, table_name, results)
+        connection.execute('SELECT 1 FROM query_123')
+
+    def test_creates_table_with_spaces_in_column_name(self):
+        connection = sqlite3.connect(':memory:')
+        results = {'columns': [{'name': 'two words'}, {'name': 'test2'}], 'rows': [
+            {'two words': 1, 'test2': 2}, {'test1': 3}]}
+        table_name = 'query_123'
+        create_table(connection, table_name, results)
+        connection.execute('SELECT 1 FROM query_123')
+
+    def test_loads_results(self):
+        connection = sqlite3.connect(':memory:')
+        rows = [{'test1': 1, 'test2': 'test'}, {'test1': 2, 'test2': 'test2'}]
+        results = {'columns': [{'name': 'test1'},
+                               {'name': 'test2'}], 'rows': rows}
+        table_name = 'query_123'
+        create_table(connection, table_name, results)
+        self.assertEquals(
+            len(list(connection.execute('SELECT * FROM query_123'))), 2)
+
+
+class TestGetQuery(BaseTestCase):
+    # test query from different account
+    def test_raises_exception_for_query_from_different_account(self):
+        query = self.factory.create_query()
+        user = self.factory.create_user(org=self.factory.create_org())
+
+        self.assertRaises(PermissionError, lambda: _load_query(user, query.id))
+
+    def test_raises_exception_for_query_with_different_groups(self):
+        ds = self.factory.create_data_source(group=self.factory.create_group())
+        query = self.factory.create_query(data_source=ds)
+        user = self.factory.create_user()
+
+        self.assertRaises(PermissionError, lambda: _load_query(user, query.id))
+
+    def test_returns_query(self):
+        query = self.factory.create_query()
+        user = self.factory.create_user()
+
+        loaded = _load_query(user, query.id)
+        self.assertEquals(query, loaded)