Repository: bokeh/bokeh

Issue:
Add the ComputedTransforms to the DataSpec System
Currently, to enable a Transform to be performed, the user must actively click on a control to force the calculation.  Integrating this into the DataSpec system will allow for the transformation to be performed on the fly without the need for a user to force the calculation.



Discussion:
Hi @justacec we can talk more soon, but a quick info dump.

All of the `*Spec` properties ultimately resolve into one of two JSON representations: 
- `{ 'value': 10 }`
- `{ 'field': 'foo' }`

There can also be a `'units'` field, but let's ignore that for now. The differences between all the different "spec" types on the python is really just to apply validation, and set `'units` correctly automatically. 

The entire point of a `DataSpec` really to distinguish (in the Bokeh JSON)  between cases like:

```
p.circle(x=10, ...) # x gets always has one fixed value
```

and 

```
p.circle(x='foo', source=source, ... ) # x takes values from source.data['foo']
```

Note that we have made it easy to write things like the above, but it's always possible to actually just provide the "spec" dicts as values:

```
p.text(text={ 'value', 'hello' }, ...)   # to a bokeh,plotting glyph method

Line(x={ 'field': 'foo' }, ...)  # or a bokeh.models Glyph object
```

Then on the BokehJS side there is lots of machinery to compute columns of screen coordinates for the various renderers, and it switches on the `value`/`field` distinction to do this work. The main point of doing this at all is to reduce the size of the data we have to encode and send to BokehJS (we could turn `x=10` into an array of `[10, 10, 10, 10, ...]` and send that via JSON, and then BokehJS could be simpler, only having to ever deal with arrays, but that gets very wasteful in terms of the JSON payload size)

---

So at the first, lowest level, this task is about allow the "spec" dictionaries to have a new optional key, something possibly like:

```
{ 'field': 'foo', 'transform': Jitter(width=0.5) }
```

Then on the BokehJS side where it computes all the columns for the actual renderers there would be an extra step to look for and apply any transform before giving the result to the renderer. 

After that, we can look at how to make this easy to spell, e.g. maybe it would be nice to do:

```
p.circle(x=jitter('foo'), ...)
```

and have that automatically translate to the equivalent of

```
p.circle(x={'field': 'foo', 'transform': Jitter()), ...)
```

---

In terms of the difficulty of the various parts, my rough though are that the last part (convenient spelling) is probably pretty simple and straightforward. Then making BokehJS use a transform at the right place if one is present is probably also not so bad. The most challenging part is getting the serialization/deserialization machinery working correctly and also having it well-tested. 

@bryevdv Got it.  Thanks for the pointers.  This will get me started on thinking...  We can talk more about it later.

@bryevdv I have started to play with this and have made progress.  I implemented your suggested workflow above with no major problems.  I am having difficulty find out how:

```
plt.Circle([0, 1, 2], [5, 4, 3])
```

Gets converted into a colum data source in the json.  

Anyway,  just wanted you to know I am making progress.


Resolving patch:
diff --git a/bokeh/core/properties.py b/bokeh/core/properties.py
--- a/bokeh/core/properties.py
+++ b/bokeh/core/properties.py
@@ -1630,9 +1630,11 @@ def set_from_json(self, obj, json, models=None):
 
 class DataSpec(Either):
     def __init__(self, typ, default, help=None):
-        super(DataSpec, self).__init__(String, Dict(String, Either(String, typ)), typ, default=default, help=help)
+        super(DataSpec, self).__init__(String, Dict(String, Either(String, Instance('bokeh.models.transforms.Transform'), typ)), typ, default=default, help=help)
         self._type = self._validate_type_param(typ)
 
+    # TODO (bev) add stricter validation on keys
+
     def make_properties(self, base_name):
         return [ DataSpecProperty(descriptor=self, name=base_name) ]
 
diff --git a/examples/plotting/file/jitter.py b/examples/plotting/file/jitter.py
new file mode 100644
--- /dev/null
+++ b/examples/plotting/file/jitter.py
@@ -0,0 +1,19 @@
+import numpy as np
+
+from bokeh.models import ColumnDataSource, Jitter
+from bokeh.plotting import figure, show, output_file
+
+p = figure(plot_width=500, plot_height=400, x_range=(0,3), y_range=(0,10))
+
+y1 = np.random.random(2500) * 10
+y2 = np.random.normal(size=2500)*2 + 5
+
+p.circle(x={'value': 1, 'transform': Jitter(width=0.4)}, y=y1,
+         color="navy", alpha=0.3)
+
+p.circle(x={'value': 2, 'transform': Jitter(width=0.4)}, y=y2,
+         color="firebrick", alpha=0.3)
+
+output_file("jitter.html")
+
+show(p)


Test patch:
diff --git a/bokehjs/test/core/properties.coffee b/bokehjs/test/core/properties.coffee
--- a/bokehjs/test/core/properties.coffee
+++ b/bokehjs/test/core/properties.coffee
@@ -7,6 +7,15 @@ HasProps = utils.require "core/has_props"
 enums = utils.require "core/enums"
 ColumnDataSource = utils.require("models/sources/column_data_source").Model
 svg_colors = utils.require "core/util/svg_colors"
+Transform = utils.require "models/transforms/transform"
+
+class TestTransform extends Transform.Model
+  compute: (x) -> x+1
+  v_compute: (xs) ->
+    ret = []
+    for i in [0...xs.length]
+      ret.push(xs[i]+i)
+    ret
 
 class SomeHasProps extends HasProps
   type: 'SomeHasProps'
@@ -34,11 +43,13 @@ describe "properties module", ->
     validation_error prop, undefined
     validation_error prop, new SomeHasProps()
 
-  fixed           = {a: 1}
-  spec_field      = {a: {field: 'foo'}, b: 30}
-  spec_field_only = {a: {field: 'foo'}}
-  spec_value      = {a: {value: 2}}
-  spec_value_null = {a: {value: null}}
+  fixed            = {a: 1}
+  spec_field       = {a: {field: 'foo'}, b: 30}
+  spec_field_only  = {a: {field: 'foo'}}
+  spec_field_trans = {a: {field: 'foo', transform: new TestTransform()}}
+  spec_value       = {a: {value: 2}}
+  spec_value_trans = {a: {value: 2, transform: new TestTransform()}}
+  spec_value_null  = {a: {value: null}}
 
   describe "Property", ->
 
@@ -112,6 +123,10 @@ describe "properties module", ->
         prop = new properties.Property({obj: new SomeHasProps(spec_value), attr: 'a'})
         expect(prop.value()).to.be.equal 2
 
+      it "should return a transformed value if there is a value spec with transform", ->
+        prop = new properties.Property({obj: new SomeHasProps(spec_value_trans), attr: 'a'})
+        expect(prop.value()).to.be.equal 3
+
       it "should allow a fixed null value", ->
         prop = new properties.Property({obj: new SomeHasProps(spec_value_null), attr: 'a'})
         expect(prop.value()).to.be.equal null
@@ -177,6 +192,32 @@ describe "properties module", ->
           arr = prop.array(source)
         expect(fn).to.throw Error, /attempted to retrieve property array for nonexistent field 'foo'/
 
+      it "should apply a spec transform to a field", ->
+        source = new ColumnDataSource({data: {foo: [0,1,2,3,10]}})
+        prop = new properties.Property({obj: new SomeHasProps(spec_field_trans), attr: 'a'})
+        prop.dataspec = true
+        arr = prop.array(source)
+        expect(arr).to.be.instanceof Array
+        expect(arr.length).to.be.equal 5
+        expect(arr[0]).to.be.equal 0
+        expect(arr[1]).to.be.equal 2
+        expect(arr[2]).to.be.equal 4
+        expect(arr[3]).to.be.equal 6
+        expect(arr[4]).to.be.equal 14
+
+      it "should apply a spec transform to a value array", ->
+        source = new ColumnDataSource({data: {foo: [0,1,2,3,10]}})
+        prop = new properties.Property({obj: new SomeHasProps(spec_value_trans), attr: 'a'})
+        prop.dataspec = true
+        arr = prop.array(source)
+        expect(arr).to.be.instanceof Array
+        expect(arr.length).to.be.equal 5
+        expect(arr[0]).to.be.equal 2
+        expect(arr[1]).to.be.equal 3
+        expect(arr[2]).to.be.equal 4
+        expect(arr[3]).to.be.equal 5
+        expect(arr[4]).to.be.equal 6
+
     describe "init", ->
       it "should return nothing by default", ->
         p = new properties.Property({obj: new SomeHasProps(a: {value: "foo"}), attr: 'a'})