Repository: blaze/blaze

Issue:
Confusing truncate behavior
```
In [75]: import pandas.util.testing as tm

In [76]: df = tm.makeTimeDataFrame().reset_index().rename(columns={'index': 'date'})

In [77]: d = Data(df)

In [78]: d
Out[78]:
         date         A         B         C         D
0  2000-01-03  2.537552 -0.361743  1.846507  0.849406
1  2000-01-04  0.919578 -0.943919  0.845371  0.536620
2  2000-01-05 -1.094128  0.349832 -2.037541  1.573696
3  2000-01-06  0.600101 -0.407543 -0.565710 -0.035369
4  2000-01-07  2.452163  0.429239  0.087082 -1.497300
5  2000-01-10 -0.268406  1.920480 -1.114133  0.457976
6  2000-01-11 -0.907569 -1.578275  0.804834  0.102532
7  2000-01-12  0.725926  0.650127 -0.421118 -0.422981
8  2000-01-13  0.057786 -0.319255  0.464328 -0.745487
9  2000-01-14 -0.638116 -0.540182 -1.512453  0.332363
...

In [79]: result = into(pd.Series, d.date.truncate(days=5))

In [80]: result.head()
Out[80]:
0   1999-12-31
1   1999-12-31
2   2000-01-05
3   2000-01-05
4   2000-01-05
Name: date, dtype: datetime64[ns]

In [81]: expected = pd.Series(((into(pd.Series, d.date).values.astype('M8[D]').astype('int64') // 5) * 5).astype('M8[D]'))

In [82]: expected.head()
Out[82]:
0   1999-12-30
1   2000-01-04
2   2000-01-04
3   2000-01-04
4   2000-01-04
dtype: datetime64[ns]
```



Discussion:
Why is there a +4 in this code? 

``` python
@dispatch(DateTimeTruncate, (np.ndarray, np.datetime64))
def compute_up(expr, data, **kwargs):
    np_dtype = precision_map[expr.unit]
    if expr.unit in ['day', 'week']:
        if expr.unit == 'day':
            measure = expr.measure
        else:
            measure = expr.measure * 7
        return (((data.astype('M8[D]')
                      .astype('i8')
                      + 4)
                      // measure
                      *  measure
                      - 4)
                     .astype('M8[D]'))
    else:
        return ((data.astype(np_dtype)
                     .astype('i8')
                     // expr.measure
                     * expr.measure)
                     .astype(np_dtype))
```

if there's a reason for it, i think a comment on why it's there would be nice, for future readers

in the meantime, i'll `git blame` and see if there are some breadcrumbs so that i can write a comment and submit a PR

I agree that a comment would be good.  Grepping through code to find similar comments now and not finding as many as I thought there would be.  Short story is that we need to make Python and NumPy agree on where day zero is.

These numbers might be relevant.

``` Python
In [1]: import datetime

In [2]: import numpy as np

In [3]: dt = datetime.datetime(2000, 1, 1); dt
Out[3]: datetime.datetime(2000, 1, 1, 0, 0)

In [4]: x = np.datetime64('2000-01-01', dtype='M8[D]'); x
Out[4]: numpy.datetime64('2000-01-01')

In [5]: dt.toordinal()
Out[5]: 730120

In [6]: x.astype('i8')
Out[6]: 10957
```

``` Python
In [11]: np.int64(1).astype('M8[D]')
Out[11]: numpy.datetime64('1970-01-02')

In [12]: datetime.datetime.fromordinal(1)
Out[12]: datetime.datetime(1, 1, 1, 0, 0)
```

that looks like numpy is just wrong, unless the definition of day one is different

Numpy defines day 0 as epoch.  Python defines day 1 as 1-1-1

ok, but where does 4 come from?

We want truncation computations on each to resolve in the same result.

``` Python
In [22]: dt = datetime.date(2000, 1, 1); dt
Out[22]: datetime.date(2000, 1, 1)

In [23]: x = np.datetime64('2000-01-01', dtype='M8[D]'); x
Out[23]: numpy.datetime64('2000-01-01')

In [24]: s = symbol('s', 'datetime')

In [28]: compute(s.truncate(days=7), x)
Out[28]: numpy.datetime64('1999-12-26')

In [29]: compute(s.truncate(days=7), dt)
Out[29]: datetime.date(1999, 12, 26)
```

To do this we need to make sure that when we perform integer division they each have the same zero point.

However, as I say this, I recognize that they don't have the same zero point.  They only have the same zero point mod 7 which was what I was likely testing against.  Indeed, this fails for truncations that are not weeks

``` Python
In [30]: compute(s.truncate(days=3), dt)
Out[30]: datetime.date(1999, 12, 31)

In [31]: compute(s.truncate(days=3), x)
Out[31]: numpy.datetime64('1999-12-30')
```

Probably we don't want to add 4 but instead want to add something like `730120 - 10957`

`q` follows _yet another_ convention, the zeroth day is `2000.01.01`:

```
q)`long $ 2000.01.01
0
```


Resolving patch:
diff --git a/blaze/api/into.py b/blaze/api/into.py
--- a/blaze/api/into.py
+++ b/blaze/api/into.py
@@ -487,6 +487,11 @@ def into(a, df, **kwargs):
     return x
 
 
+@dispatch(np.ndarray, pd.Series)
+def into(_, s, **kwargs):
+    return s.values
+
+
 @dispatch(nd.array)
 def discover(arr):
     return dshape(nd.dshape_of(arr))
diff --git a/blaze/compute/numpy.py b/blaze/compute/numpy.py
--- a/blaze/compute/numpy.py
+++ b/blaze/compute/numpy.py
@@ -1,5 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
+import datetime
+
 import numpy as np
 from pandas import DataFrame, Series
 from datashape import to_numpy
@@ -202,9 +204,16 @@ def compute_up(expr, data, **kwargs):
     return result
 
 
+
+# Note the use of 'week': 'M8[D]' here.
+
+# We truncate week offsets "manually" in the compute_up implementation by first
+# converting to days then multiplying our measure by 7 this simplifies our code
+# by only requiring us to calculate the week offset relative to the day of week.
+
 precision_map = {'year': 'M8[Y]',
                  'month': 'M8[M]',
-                 'week': 'M8[W]',
+                 'week': 'M8[D]',
                  'day': 'M8[D]',
                  'hour': 'M8[h]',
                  'minute': 'M8[m]',
@@ -213,27 +222,29 @@ def compute_up(expr, data, **kwargs):
                  'microsecond': 'M8[us]',
                  'nanosecond': 'M8[ns]'}
 
+
+# these offsets are integers in units of their representation
+
+epoch = datetime.datetime(1970, 1, 1)
+offsets = {
+    'week': epoch.isoweekday(),
+    'day': epoch.toordinal() # number of days since *Python's* epoch (01/01/01)
+}
+
+
 @dispatch(DateTimeTruncate, (np.ndarray, np.datetime64))
 def compute_up(expr, data, **kwargs):
     np_dtype = precision_map[expr.unit]
-    if expr.unit in ['day', 'week']:
-        if expr.unit == 'day':
-            measure = expr.measure
-        else:
-            measure = expr.measure * 7
-        return (((data.astype('M8[D]')
-                      .astype('i8')
-                      + 4)
-                      // measure
-                      *  measure
-                      - 4)
-                     .astype('M8[D]'))
-    else:
-        return ((data.astype(np_dtype)
-                     .astype('i8')
-                     // expr.measure
-                     * expr.measure)
-                     .astype(np_dtype))
+    offset = offsets.get(expr.unit, 0)
+    measure = expr.measure * 7 if expr.unit == 'week' else expr.measure
+    result = (((data.astype(np_dtype)
+                    .view('int64')
+                    + offset)
+                    // measure
+                    * measure
+                    - offset)
+                    .astype(np_dtype))
+    return result
 
 
 @dispatch(np.ndarray)


Test patch:
diff --git a/blaze/compute/tests/test_pandas_compute.py b/blaze/compute/tests/test_pandas_compute.py
--- a/blaze/compute/tests/test_pandas_compute.py
+++ b/blaze/compute/tests/test_pandas_compute.py
@@ -593,7 +593,7 @@ def test_nelements():
     assert compute(t.nrows, df) == len(df)
 
 
-def test_datetime_truncation():
+def test_datetime_truncation_minutes():
     data = Series(['2000-01-01T12:10:00Z', '2000-06-25T12:35:12Z'],
                   dtype='M8[ns]')
     s = symbol('s', 'var * datetime')
@@ -601,11 +601,44 @@ def test_datetime_truncation():
             list(Series(['2000-01-01T12:00:00Z', '2000-06-25T12:20:00Z'],
                         dtype='M8[ns]'))
 
+
+def test_datetime_truncation_nanoseconds():
+    data = Series(['2000-01-01T12:10:00.000000005',
+                   '2000-01-01T12:10:00.000000025'],
+                  dtype='M8[ns]')
+    s = symbol('s', 'var * datetime')
+    result = Series(['2000-01-01T12:10:00.000000000',
+                     '2000-01-01T12:10:00.000000020'],
+                    dtype='M8[ns]')
+    assert list(compute(s.truncate(nanoseconds=20), data)) == list(result)
+
+
+def test_datetime_truncation_weeks():
+    data = Series(['2000-01-01T12:10:00Z', '2000-06-25T12:35:12Z'],
+                  dtype='M8[ns]')
+    s = symbol('s', 'var * datetime')
     assert list(compute(s.truncate(2, 'weeks'), data)) == \
-            list(Series(['1999-12-19T00:00:00Z', '2000-06-18T00:00:00Z'],
+            list(Series(['1999-12-19', '2000-06-18'],
                         dtype='M8[ns]'))
 
 
+def test_datetime_truncation_days():
+    data = Series(['2000-01-01T12:10:00Z', '2000-06-25T12:35:12Z'],
+                  dtype='M8[ns]')
+    s = symbol('s', 'var * datetime')
+    assert list(compute(s.truncate(days=3), data)) == \
+            list(Series(['1999-12-31', '2000-06-25'],
+                        dtype='M8[ns]'))
+
+
+def test_datetime_truncation_same_as_python():
+    data = Series(['2000-01-01T12:10:00Z', '2000-06-25T12:35:12Z'],
+                  dtype='M8[ns]')
+    s = symbol('s', 'var * datetime')
+    assert (compute(s.truncate(weeks=2), data[0].to_pydatetime()) ==
+            datetime(1999, 12, 26).date())
+
+
 def test_complex_group_by():
     expr = by(merge(tbig.amount // 10, tbig.id % 2),
               count=tbig.name.count())