Repository: getredash/redash

Issue:
Inner objects of MongoDB results are not handled correctly
### Issue Summary

When querying a mongo object that has inner objects and returning only a given field of an inner object, results are not displayed in the UI. 
### Steps to Reproduce
1. Have a MongoDB data_source that contains an object looking like that

``` json
{
    "foo": {
        "bar":0,
        "foo":1
    }
}
```
1. Query this data source like that: 

``` json
{
    "collection": "foo",
    "query": {
    },
    "fields": {
        "foo.bar":1
     }
}
```
1. Notice that the results table is empty. 
2. Look at the response using the network tab of your browser's console.
3. Notice that the data is correctly returned but incorrectly handled in the UI. 

I expect the data to be handled correctly. Since it is not in the table, it is impossible to make any visualization. Inner objects are common in MongoDB. I believer it can be fixed either on the UI or the server since when it is an inner field, it isn't in the same format as the other fields. 
### Technical details:
- Redash Version: 0.10
- Browser/OS: Firefox/Linux
- How did you install Redash: Docker



Discussion:
Can you share a GitHub Gist with the query result of the above (the result you get from the API)? This will help with reproducing and fixing.

I unfortunately have no public instance of redash, so the Gist wouldn't actually work. However, here's the results I get from the API (a real life example): 

``` json
{
    "query_result": {
        "retrieved_at": "2016-07-21T13:50:37.155615+00:00",
        "query_hash": "ac18633652fccb3940d0e752fecb10f2",
        "query": "{\n  \"collection\": \"jenkins_builds\",\n  \"query\": {\n    \"slave_info.slaveName\": \"optimusprime\"\n  },\n  \"fields\":{\n    \"slave_info.slaveName\":1,\n    \"build_url\":1\n  }\n}",
        "runtime": 0.074466,
        "data": {
            "rows": [{
                "slave_info": {
                    "slaveName": "optimusprime"
                },
                "build_url": "aUrl",
                "_id": "5784e1161f7a4b0006edebeb"
            }, {
                "slave_info": {
                    "slaveName": "optimusprime"
                },
                "build_url": "anotherUrl",
                "_id": "5784f6ef1f7a4b0006edede3"
            }],
            "columns": [null, {
                "friendly_name": "build_url",
                "type": "string",
                "name": "build_url"
            }]
        },
        "id": 45,
        "data_source_id": 1
    }
}
```

Here's how a document looks like in the collection: 

``` json
{
    "_id": ObjectId("578f6caac3775823842c9b1a"),
    "queue_time": 29,
    "result": "ABORTED",
    "scm_checkout_end_time": ISODate("2016-07-20T08:20:59.237Z"),
    "end_time": ISODate("2016-07-20T08:21:07.912Z"),
    "ci_url": "http://localhost:8080/jenkins/",
    "context_id": 102490963,
    "full_job_name": "jenkins_test",
    "parameters": {

    },
    "build_failure_causes": [{
        "name": "abort",
        "categories": [
            "aaa"
        ],
        "id": "ccff5d15-1003-4570-b38f-f844255c6be1",
        "description": "aborted"
    }],
    "build_url": "aUrl",
    "build_cause": "Démarré par l'utilisateur anonymous",
    "start_time": ISODate("2016-07-20T08:20:58.406Z"),
    "number": 252,
    "scm_checkout_start_time": ISODate("2016-07-20T08:20:58.450Z"),
    "started_username": "anonymous",
    "scm_checkout_duration": 787,
    "build_steps": [{
        "start_time": ISODate("2016-07-20T08:20:59.242Z"),
        "end_time": ISODate("2016-07-20T08:21:00.127Z"),
        "duration": 885,
        "build_step_type": "hudson.tasks.Shell"
    }, {
        "start_time": ISODate("2016-07-20T08:21:00.128Z"),
        "end_time": ISODate("1969-12-31T19:00:00Z"),
        "build_step_type": "hudson.tasks.Maven"
    }, {
        "start_time": ISODate("2016-07-20T08:21:07.710Z"),
        "end_time": ISODate("2016-07-20T08:21:07.711Z"),
        "duration": 1,
        "build_step_type": "com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty"
    }, {
        "start_time": ISODate("2016-07-20T08:21:07.712Z"),
        "end_time": ISODate("2016-07-20T08:21:07.714Z"),
        "duration": 2,
        "build_step_type": "jenkins.model.BuildDiscarderProperty"
    }],
    "job_name": "jenkins_test",
    "slave_info": {
        "slaveName": "optimusprime",
        "executor": "1",
        "label": "master, windows,"
    },
    "duration": 9452,
    "scm_info": {
        "url": "aGithubUrl",
        "commit": "aCommitHash",
        "branch": "origin/master"
    },
    "started_user_id": "anonymous"
}
```

And here's the JS errors we get in the UI: 

```
TypeError: Cannot read property 'name' of null
    at http://localhost:5000/scripts/scripts.f1054183.js:1:4880
    at Array.forEach (native)
    at Function.k.each.k.forEach (http://localhost:5000/scripts/plugins.ec2c390a.js:15:7459)
    at n.o [as update] (http://localhost:5000/scripts/scripts.f1054183.js:1:4829)
    at http://localhost:5000/scripts/scripts.f1054183.js:1:8914
    at http:/localhost:5000/scripts/plugins.ec2c390a.js:15:1819
    at c (http://localhost:5000/scripts/plugins.ec2c390a.js:5:16812)
    at http://localhost:5000/scripts/plugins.ec2c390a.js:5:17489
    at c.$eval (http://localhost:5000/scripts/plugins.ec2c390a.js:5:22696)
    at c.$digest (http://localhost:5000/scripts/plugins.ec2c390a.js:5:21221)
```

and 

```
Error: [$interpolate:interr] Can't interpolate: {{queryResult.getData().length}}
TypeError: Cannot read property 'name' of null
http://errors.angularjs.org/1.2.18/$interpolate/interr?p0=%7B%7BqueryResult…ngth%7D%7D&p1=TypeError%3A%20Cannot%20read%20property%20'name'%20of%20null
    at plugins.ec2c390a.js:3
    at Object.f (plugins.ec2c390a.js:5)
    at c.$digest (plugins.ec2c390a.js:5)
    at c.$apply (plugins.ec2c390a.js:5)
    at l (plugins.ec2c390a.js:5)
    at v (plugins.ec2c390a.js:5)
    at XMLHttpRequest.w.onreadystatechange (plugins.ec2c390a.js:5)
```

If you need more information or if anything is not clear, let me know. 

I have the same issue.  I'm using `aggregate` instead of `query` to get around the issue.


Resolving patch:
diff --git a/redash/query_runner/mongodb.py b/redash/query_runner/mongodb.py
--- a/redash/query_runner/mongodb.py
+++ b/redash/query_runner/mongodb.py
@@ -74,6 +74,49 @@ def parse_query_json(query):
     return query_data
 
 
+def _get_column_by_name(columns, column_name):
+    for c in columns:
+        if "name" in c and c["name"] == column_name:
+            return c
+
+    return None
+
+
+def parse_results(results):
+    rows = []
+    columns = []
+
+    for row in results:
+        parsed_row = {}
+
+        for key in row:
+            if isinstance(row[key], dict):
+                for inner_key in row[key]:
+                    column_name = '{}.{}'.format(key, inner_key)
+                    if _get_column_by_name(columns, column_name) is None:
+                        columns.append({
+                            "name": column_name,
+                            "friendly_name": column_name,
+                            "type": TYPES_MAP.get(type(row[key][inner_key]), TYPE_STRING)
+                        })
+                    
+                    parsed_row[column_name] = row[key][inner_key]
+
+            else:
+                if _get_column_by_name(columns, key) is None:
+                    columns.append({
+                        "name": key,
+                        "friendly_name": key,
+                        "type": TYPES_MAP.get(type(row[key]), TYPE_STRING)
+                    })
+                
+                parsed_row[key] = row[key]
+
+        rows.append(parsed_row)
+
+    return rows, columns
+
+
 class MongoDB(BaseQueryRunner):
     @classmethod
     def configuration_schema(cls):
@@ -113,13 +156,6 @@ def __init__(self, configuration):
 
         self.is_replica_set = True if "replicaSetName" in self.configuration and self.configuration["replicaSetName"] else False
 
-    def _get_column_by_name(self, columns, column_name):
-        for c in columns:
-            if "name" in c and c["name"] == column_name:
-                return c
-
-        return None
-
     def _get_db(self):
         if self.is_replica_set:
             db_connection = pymongo.MongoReplicaSetClient(self.configuration["connectionString"], replicaSet=self.configuration["replicaSetName"])
@@ -259,21 +295,14 @@ def run_query(self, query, user):
 
             rows.append({ "count" : cursor })
         else:
-            for r in cursor:
-                for k in r:
-                    if self._get_column_by_name(columns, k) is None:
-                        columns.append({
-                            "name": k,
-                            "friendly_name": k,
-                            "type": TYPES_MAP.get(type(r[k]), TYPE_STRING)
-                        })
-
-                rows.append(r)
+            rows, columns = parse_results(cursor)
 
         if f:
             ordered_columns = []
             for k in sorted(f, key=f.get):
-                ordered_columns.append(self._get_column_by_name(columns, k))
+                column = _get_column_by_name(columns, k)
+                if column:
+                    ordered_columns.append(column)
 
             columns = ordered_columns
 


Test patch:
diff --git a/tests/query_runner/test_mongodb.py b/tests/query_runner/test_mongodb.py
--- a/tests/query_runner/test_mongodb.py
+++ b/tests/query_runner/test_mongodb.py
@@ -2,7 +2,7 @@
 import json
 from unittest import TestCase
 from pytz import utc
-from redash.query_runner.mongodb import parse_query_json
+from redash.query_runner.mongodb import parse_query_json, parse_results, _get_column_by_name
 
 from redash.utils import parse_human_time
 
@@ -105,4 +105,42 @@ def test_supports_relative_timestamps(self):
         self.assertEqual(query_data['ts'], one_hour_ago)
 
 
-
+class TestMongoResults(TestCase):
+    def test_parses_regular_results(self):
+        raw_results = [
+            {'column': 1, 'column2': 'test'},
+            {'column': 2, 'column2': 'test', 'column3': 'hello'}
+        ]
+        rows, columns = parse_results(raw_results)
+
+        for i, row in enumerate(rows):
+            self.assertDictEqual(row, raw_results[i])
+
+        self.assertIsNotNone(_get_column_by_name(columns, 'column'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'column2'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'column3'))
+    
+    def test_parses_nested_results(self):
+        raw_results = [
+            {'column': 1, 'column2': 'test', 'nested': {
+                'a': 1,
+                'b': 'str'
+            }},
+            {'column': 2, 'column2': 'test', 'column3': 'hello', 'nested': {
+                'a': 2,
+                'b': 'str2',
+                'c': 'c'
+            }}
+        ]
+
+        rows, columns = parse_results(raw_results)
+
+        self.assertDictEqual(rows[0], { 'column': 1, 'column2': 'test', 'nested.a': 1, 'nested.b': 'str' })
+        self.assertDictEqual(rows[1], { 'column': 2, 'column2': 'test', 'column3': 'hello', 'nested.a': 2, 'nested.b': 'str2', 'nested.c': 'c' })
+
+        self.assertIsNotNone(_get_column_by_name(columns, 'column'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'column2'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'column3'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'nested.a'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'nested.b'))
+        self.assertIsNotNone(_get_column_by_name(columns, 'nested.c'))
\ No newline at end of file