Repository: getredash/redash

Issue:
Visualizations workflow & object
Visualizations (widget?) should have an object of their own containing the following data:
-  query
-  type (chart, cohort, grid, ...)
-  options

Tasks:
- [x] Visualization object
- [x] UI to create new visualizations instead of the hardcoded option we have today
- [x] Change the dashboard widgets to use visualizations rather than queries
- [ ] Friendlier selector when adding new widgets to dashboard



Discussion:
There was a case (EvMe's query 607), where the data was unsorted and had too many series, which resulted in HighCharts dying. When we revisit visualizations we should take into account limits for different visualizers, and prevent the user from killing his browser.

cc: @shayel.

http://app.raw.densitydesign.org/#/

@amirnissim this requires some changes on the backend too, but let's try to move forward without them and on Sunday we will discuss the needed changes.

Below is a "brain dump" about this feature, ask any questions you feel necessary:

Basically the idea is to have different types of visualizations, and the ability to create a new visualization from any dataset (query). Eventually, this will replace the "Chart" and "Cohort" static tabs.

Each visualization will define:
1. Name
2. Description
3. Properties (mandatory & optional + default values)
4. Rendering logic

When creating new visualization it will have:
1. query_id - reference to queries table
2. visualization_type - string
3. options - JSON
4. ?

Until we add the visualization object, let's start by creating the "infrastructure" for this in the frontend code.

In terms of UI, I think we will represent each visualization as a tab where we currently have the "Chart" and "Cohort" tabs and also have a "+" tab, where you define a new visualization.


Resolving patch:
diff --git a/rd_service/data/models.py b/rd_service/data/models.py
--- a/rd_service/data/models.py
+++ b/rd_service/data/models.py
@@ -51,7 +51,8 @@ class Meta:
         app_label = 'redash'
         db_table = 'queries'
 
-    def to_dict(self, with_result=True, with_stats=False):
+    def to_dict(self, with_result=True, with_stats=False,
+                with_visualizations=False):
         d = {
             'id': self.id,
             'latest_query_data_id': self.latest_query_data_id,
@@ -75,6 +76,10 @@ def to_dict(self, with_result=True, with_stats=False):
         if with_result and self.latest_query_data_id:
             d['latest_query_data'] = self.latest_query_data.to_dict()
 
+        if with_visualizations:
+            d['visualizations'] = [vis.to_dict(with_query=False)
+                                   for vis in self.visualizations.all()]
+
         return d
 
     @classmethod
@@ -148,10 +153,41 @@ def __unicode__(self):
         return u"%s=%s" % (self.id, self.name)
 
 
+class Visualization(models.Model):
+    id = models.AutoField(primary_key=True)
+    type = models.CharField(max_length=100)
+    query = models.ForeignKey(Query, related_name='visualizations')
+    name = models.CharField(max_length=255)
+    description = models.CharField(max_length=4096)
+    options = models.TextField()
+
+    class Meta:
+        app_label = 'redash'
+        db_table = 'visualizations'
+
+    def to_dict(self, with_query=True):
+        d = {
+            'id': self.id,
+            'type': self.type,
+            'name': self.name,
+            'description': self.description,
+            'options': json.loads(self.options),
+        }
+
+        if with_query:
+            d['query'] = self.query.to_dict()
+
+        return d
+
+    def __unicode__(self):
+        return u"%s=>%s" % (self.id, self.query_id)
+
+
 class Widget(models.Model):
     id = models.AutoField(primary_key=True)
-    query = models.ForeignKey(Query)
     type = models.CharField(max_length=100)
+    query = models.ForeignKey(Query, related_name='widgets')
+    visualization = models.ForeignKey(Visualization, related_name='widgets')
     width = models.IntegerField()
     options = models.TextField()
     dashboard = models.ForeignKey(Dashboard, related_name='widgets')
@@ -163,10 +199,10 @@ class Meta:
     def to_dict(self):
         return {
             'id': self.id,
-            'query': self.query.to_dict(),
             'type': self.type,
             'width': self.width,
             'options': json.loads(self.options),
+            'visualization': self.visualization.to_dict(),
             'dashboard_id': self.dashboard_id
         }
 
diff --git a/rd_service/migrate.py b/rd_service/migrate.py
new file mode 100644
--- /dev/null
+++ b/rd_service/migrate.py
@@ -0,0 +1,50 @@
+import json
+import settings
+from data.models import *
+
+# first run:
+
+# CREATE TABLE "visualizations" (
+#     "id" serial NOT NULL PRIMARY KEY,
+#     "type" varchar(100) NOT NULL,
+#     "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED,
+#     "name" varchar(255) NOT NULL,
+#     "description" varchar(4096),
+#     "options" text NOT NULL
+# )
+# ;
+
+# ALTER TABLE widgets ADD COLUMN "visualization_id" integer REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED;
+
+if __name__ == '__main__':
+    print 'migrating Widgets -> Visualizations ...'
+
+    for query in Query.objects.filter(name__icontains="cohort"):
+        vis = Visualization(query=query, name=query.name,
+                            description=query.description,
+                            type="COHORT", options="{}")
+        vis.save()
+
+
+    for widget in Widget.objects.all():
+        print 'processing widget %d' % widget.id
+        query = widget.query
+        vis_type = widget.type.upper()
+
+        vis = query.visualizations.filter(type=vis_type)
+        if vis:
+            print 'found'
+            widget.visualization = vis[0]
+            widget.save()
+
+        else:
+            options = json.loads(widget.options)
+            vis_options = {"series": options} if options else {}
+            vis_options = json.dumps(vis_options)
+
+            vis = Visualization(query=query, name=query.name,
+                                            description=query.description,
+                                            type=vis_type, options=vis_options)
+            vis.save()
+            widget.visualization = vis
+            widget.save()
\ No newline at end of file
diff --git a/rd_service/server.py b/rd_service/server.py
--- a/rd_service/server.py
+++ b/rd_service/server.py
@@ -167,7 +167,7 @@ def delete(self, widget_id):
 class DashboardHandler(BaseAuthenticatedHandler):
     def get(self, dashboard_slug=None):
         if dashboard_slug:
-            dashboard = data.models.Dashboard.objects.prefetch_related('widgets__query__latest_query_data').get(slug=dashboard_slug)
+            dashboard = data.models.Dashboard.objects.prefetch_related('widgets__visualization__query__latest_query_data').get(slug=dashboard_slug)
             self.write_json(dashboard.to_dict(with_widgets=True))
         else:
             dashboards = [d.to_dict() for d in
@@ -204,6 +204,7 @@ def post(self, id=None):
             query_def['created_at'] = dateutil.parser.parse(query_def['created_at'])
 
         query_def.pop('latest_query_data', None)
+        query_def.pop('visualizations', None)
 
         if id:
             query = data.models.Query(**query_def)
@@ -221,7 +222,7 @@ def get(self, id=None):
         if id:
             q = data.models.Query.objects.get(pk=id)
             if q:
-                self.write_json(q.to_dict())
+                self.write_json(q.to_dict(with_visualizations=True))
             else:
                 self.send_error(404)
         else:
@@ -251,6 +252,30 @@ def post(self, _):
             self.write({'job': job.to_dict()})
 
 
+class VisualizationHandler(BaseAuthenticatedHandler):
+    def get(self, id):
+        pass
+
+    def post(self, id=None):
+        kwargs = json.loads(self.request.body)
+        kwargs['options'] = json.dumps(kwargs['options'])
+
+        if id:
+            vis = data.models.Visualization(**kwargs)
+            fields = kwargs.keys()
+            fields.remove('id')
+            vis.save(update_fields=fields)
+        else:
+            vis = data.models.Visualization(**kwargs)
+            vis.save()
+
+        self.write_json(vis.to_dict(with_query=False))
+
+    def delete(self, id):
+        vis = data.models.Visualization.objects.get(pk=id)
+        vis.delete()
+
+
 class CsvQueryResultsHandler(BaseAuthenticatedHandler):
     def get_current_user(self):
         user = super(CsvQueryResultsHandler, self).get_current_user()
@@ -312,6 +337,7 @@ def get_application(static_path, is_debug, redis_connection, data_manager):
                                     (r"/api/queries(?:/([0-9]*))?", QueriesHandler),
                                     (r"/api/query_results(?:/([0-9]*))?", QueryResultsHandler),
                                     (r"/api/jobs/(.*)", JobsHandler),
+                                    (r"/api/visualizations(?:/([0-9]*))?", VisualizationHandler),
                                     (r"/api/widgets(?:/([0-9]*))?", WidgetsHandler),
                                     (r"/api/dashboards(?:/(.*))?", DashboardHandler),
                                     (r"/admin/(.*)", MainHandler),


Test patch: