Repository: blaze/blaze

Issue:
Bug in sample(frac=) expression
When randomly sampling a fraction of the results for an SQL backend, it always returns 1 row - tested against postgres. Will update test_postgres_compute() to include a failing test right after opening this issues so it can be referenced in the xfail.

The following expression always returns 1 row regardless of the number of rows contained in the `nyc` table (test in postgres backend): `result2 = compute(t.sample(frac=0.5), nyc)` 



Discussion:

Resolving patch:
diff --git a/blaze/compute/pandas.py b/blaze/compute/pandas.py
--- a/blaze/compute/pandas.py
+++ b/blaze/compute/pandas.py
@@ -546,9 +546,31 @@ def compute_up(t, s, **kwargs):
         return s.order(ascending=t.ascending)
 
 
-@dispatch(Sample, (Series, DataFrame, DaskDataFrame, DaskSeries))
+@dispatch(Sample, (Series, DataFrame))
 def compute_up(t, df, **kwargs):
-    frac = t.frac if t.frac is not None else float(t.n) / len(df)
+    from math import modf
+    if t.frac is not None:
+        # Work around annoying edge case: Python's round() builtin (which
+        # Pandas' sample() uses) rounds 0.5, 2.5, 4.5, ... down to 0, 2, 4, ...,
+        # while it rounds 1.5, 3.5, 5.5, ... up.  This is inconsistent with any
+        # sane implementation of floating point rounding, including SQL's, so
+        # we work around it here.
+        fractional, integral = modf(t.frac * df.shape[0])
+        n = int(integral + (0 if fractional < 0.5 else 1))
+    else:
+        n = min(t.n, df.shape[0])
+    return df.sample(n=n)
+
+
+@dispatch(Sample, (DaskDataFrame, DaskSeries))
+def compute_up(t, df, **kwargs):
+    # Dask doesn't support sample(n=...), only sample(frac=...), so we have a
+    # separate dispatch for dask objects.
+    if t.frac is not None:
+        frac = t.frac
+    else:
+        nrows = len(df)
+        frac = float(min(t.n, nrows)) / nrows
     return df.sample(frac=frac)
 
 
diff --git a/blaze/compute/sql.py b/blaze/compute/sql.py
--- a/blaze/compute/sql.py
+++ b/blaze/compute/sql.py
@@ -951,13 +951,16 @@ def compute_up(t, s, **kwargs):
     cols = [direction(lower_column(s.c[c])) for c in listpack(t.key)]
     return s.order_by(*cols)
 
+
 def _samp_compute_up(t, s, **kwargs):
     if t.n is not None:
         limit = t.n
     else:
-        limit = select([safuncs.count() * t.frac]).as_scalar()
+        limit = sa.select([safuncs.count() * t.frac],
+                          from_obj=s.alias()).as_scalar()
     return s.order_by(safuncs.random()).limit(limit)
 
+
 @dispatch(Sample, sa.Table)
 def compute_up(t, s, **kwargs):
     return _samp_compute_up(t, select(s), **kwargs)


Test patch:
diff --git a/blaze/compute/tests/test_pandas_compute.py b/blaze/compute/tests/test_pandas_compute.py
--- a/blaze/compute/tests/test_pandas_compute.py
+++ b/blaze/compute/tests/test_pandas_compute.py
@@ -386,8 +386,17 @@ def test_tail():
 def test_sample():
     samp = compute(t.sample(n=2), df)
     assert len(samp) == 2
-    samp = compute(t.sample(frac=0.5), df)
-    assert len(samp) == int(np.ceil(len(df) * 0.5))
+
+
+def test_sample_frac_rounding_edge_case():
+    samp_big = compute(tbig.sample(frac=0.1), dfbig)
+    assert len(samp_big) == int(np.ceil(len(dfbig) * 0.1))
+
+
+def test_sample_clip():
+    samp_series = compute(t.name.sample(n=2*len(df)), df)
+    samp_df = compute(t.sample(n=2*len(df)), df)
+    assert len(samp_series) == len(samp_df) == len(df)
 
 
 def test_label():
diff --git a/blaze/compute/tests/test_postgresql_compute.py b/blaze/compute/tests/test_postgresql_compute.py
--- a/blaze/compute/tests/test_postgresql_compute.py
+++ b/blaze/compute/tests/test_postgresql_compute.py
@@ -1,6 +1,7 @@
 from datetime import timedelta
 from operator import methodcaller
 import itertools
+import math
 
 import pytest
 
@@ -68,13 +69,16 @@ def sql(url):
             drop(t)
 
 
+
+
 @pytest.yield_fixture(scope='module')
 def nyc(pg_ip):
+    # odoing csv -> pandas -> postgres is more robust, as it doesn't require
+    # the postgres server to be on the same filesystem as the csv file.
+    nyc_pd = odo(example('nyc.csv'), pd.DataFrame)
     try:
-        t = odo(
-            example('nyc.csv'),
-            'postgresql://postgres@{}/test::nyc'.format(pg_ip),
-        )
+        t = odo(nyc_pd,
+                'postgresql://postgres@{}/test::nyc'.format(pg_ip))
     except sa.exc.OperationalError as e:
         pytest.skip(str(e))
     else:
@@ -84,6 +88,20 @@ def nyc(pg_ip):
             drop(t)
 
 
+@pytest.yield_fixture
+def big_sql(url):
+    try:
+        t = data(url % next(names), dshape='var * {A: string, B: int64}')
+    except sa.exc.OperationalError as e:
+        pytest.skip(str(e))
+    else:
+        t = odo(zip(list('a'*100), list(range(100))), t)
+        try:
+            yield t
+        finally:
+            drop(t)
+
+
 @pytest.yield_fixture
 def sqla(url):
     try:
@@ -269,6 +287,27 @@ def sql_with_float(url):
             drop(t)
 
 
+@pytest.yield_fixture(scope='module')
+def nyc_csv(pg_ip):
+    try:
+        t = odo(
+            example('nyc.csv'),
+            'postgresql://postgres@{}/test::nyc'.format(pg_ip),
+        )
+    except sa.exc.OperationalError as e:
+        pytest.skip(str(e))
+    else:
+        try:
+            yield t
+        finally:
+            drop(t)
+
+
+def test_nyc_csv(nyc_csv):
+    t = symbol('t', discover(nyc_csv))
+    assert compute(t.nrows, nyc_csv, return_type='core') > 0
+
+
 def test_postgres_create(sql):
     assert odo(sql, list) == [('a', 1), ('b', 2)]
 
@@ -594,6 +633,34 @@ def test_interactive_len(sql):
     assert len(t) == int(t.count())
 
 
+def test_sample_n(nyc):
+    t = symbol('t', discover(nyc))
+    result = compute(t.sample(n=14), nyc)
+    s = odo(result, pd.DataFrame)
+    assert len(s) == 14
+
+
+def test_sample_frac(nyc):
+    t = symbol('t', discover(nyc))
+    result = compute(t.sample(frac=0.5), nyc, return_type=pd.DataFrame)
+    num_rows = compute(t.nrows, nyc, return_type=int)
+    # *Sigh* have to do proper rounding manually; Python's round() builtin is
+    # borked.
+    fractional, integral = math.modf(num_rows * 0.5)
+    assert int(integral + (0 if fractional < 0.5 else 1)) == len(result)
+
+
+def test_sample(big_sql):
+    nn = symbol('nn', discover(big_sql))
+    nrows = odo(compute(nn.nrows, big_sql), int)
+    result = compute(nn.sample(n=nrows // 2), big_sql)
+    s = odo(result, pd.DataFrame)
+    assert len(s) == nrows // 2
+    result2 = compute(nn.sample(frac=0.5), big_sql)
+    s2 = odo(result2, pd.DataFrame)
+    assert len(s) == len(s2)
+
+
 def test_core_compute(nyc):
     t = symbol('t', discover(nyc))
     assert isinstance(compute(t, nyc, return_type='core'), pd.DataFrame)
@@ -602,15 +669,6 @@ def test_core_compute(nyc):
     assert isinstance(compute(t, nyc, return_type=list), list)
 
 
-def test_sample(sql):
-    t = symbol('t', discover(sql))
-    result = compute(t.sample(n=1), sql)
-    s = odo(result, pd.DataFrame)
-    assert len(s) == 1
-    result2 = compute(t.sample(frac=0.5), sql)
-    s2 = odo(result2, pd.DataFrame)
-    assert len(s) == len(s2)
-
 @pytest.fixture
 def gl_data(sql_two_tables):
     u_data, t_data = sql_two_tables
diff --git a/blaze/compute/tests/test_sql_compute.py b/blaze/compute/tests/test_sql_compute.py
--- a/blaze/compute/tests/test_sql_compute.py
+++ b/blaze/compute/tests/test_sql_compute.py
@@ -537,8 +537,6 @@ def test_head():
 def test_sample():
     order_by = select(s).order_by(sa.func.random())
     assert str(compute(t.sample(n=5), s)) == str(order_by.limit(5))
-    limit_frac = select([sa.func.count() * 0.5]).as_scalar()
-    assert str(compute(t.sample(frac=0.5), s)) == str(order_by.limit(limit_frac))
 
 
 def test_label():