Repository: rootpy/rootpy

Issue:
Documentation issue for merging overflow bins
Hi, 

This time not something where I didn't read the docs carefully enough :)

The documentation states that there's an easy trick to use merge_bins to merge the overflow and the last real bin: 

> newhist = hist.merge_bins([(0, 1), (-2, -1)])

However, it doesn't seem to work as one would expect. First, let's start with a non-uniform binning:

    >>> h = Hist([0, 10, 20, 50], name="dummy")
    >>> h.Fill(60)
    4
    >>> newhist = h.merge_bins([(0, 1), (-2, -1)])
    >>> newhist.GetBinContent(0)
    0.0
    >>> newhist.GetBinContent(1)
    1.0
    >>> newhist.GetBinContent(2)
    0.0
    >>> newhist.GetBinContent(3)
    0.0
    >>> newhist.GetBinContent(4)
    0.0

Using this kind of functionality:
    h[h.nbins()] = h[h.nbins()].value + h.overflow()
    h[h.nbins()+1] = 0

Would put it in the correct bin number. 

It is not an issue due to the binning. If I'd do the same with a 10-bin histogram:

    >>> h = Hist(10, 0, 10, name="dummy")
    >>> h.Fill(60)
    11
    >>> newhist = h.merge_bins([(0, 1), (-2, -1)])
    >>> newhist.Draw()
    >>> newhist.GetBinContent(1)
    0.0
    >>> newhist.GetBinContent(2)
    0.0
    >>> newhist.GetBinContent(3)
    0.0
    >>> newhist.GetBinContent(4)
    0.0
    >>> newhist.GetBinContent(5)
    0.0
    >>> newhist.GetBinContent(6)
    0.0
    >>> newhist.GetBinContent(7)
    0.0
    >>> newhist.GetBinContent(8)
    1.0
    >>> newhist.GetBinContent(9)
    0.0
    >>> newhist.GetBinContent(10)
    0.0

This is a bit surprising. Either the merge_bins() function does something different, or the indexing is wrong. However, that last functionality seems to behave OK:

    >>> h.Fill(60)
    11
    >>> h[-2].value
    0.0
    >>> h[-1].value
    1.0
    >>> h[h.nbins()].value
    0.0
    >>> h[h.nbins()+1].value
    1.0

Is the piece in the docs perhaps a holdover from when the overflow bins were still indexed differently? 

In general, for 1D histograms, a member function like this might be a quite handy addition (now still written as non-member; it's my own utility function):

    def mergeOverflow(h):
        h[h.nbins()] = h[h.nbins()].value + h.overflow()
        h[h.nbins()+1] = 0

Documentation issue for merging overflow bins
Hi, 

This time not something where I didn't read the docs carefully enough :)

The documentation states that there's an easy trick to use merge_bins to merge the overflow and the last real bin: 

> newhist = hist.merge_bins([(0, 1), (-2, -1)])

However, it doesn't seem to work as one would expect. First, let's start with a non-uniform binning:

    >>> h = Hist([0, 10, 20, 50], name="dummy")
    >>> h.Fill(60)
    4
    >>> newhist = h.merge_bins([(0, 1), (-2, -1)])
    >>> newhist.GetBinContent(0)
    0.0
    >>> newhist.GetBinContent(1)
    1.0
    >>> newhist.GetBinContent(2)
    0.0
    >>> newhist.GetBinContent(3)
    0.0
    >>> newhist.GetBinContent(4)
    0.0

Using this kind of functionality:
    h[h.nbins()] = h[h.nbins()].value + h.overflow()
    h[h.nbins()+1] = 0

Would put it in the correct bin number. 

It is not an issue due to the binning. If I'd do the same with a 10-bin histogram:

    >>> h = Hist(10, 0, 10, name="dummy")
    >>> h.Fill(60)
    11
    >>> newhist = h.merge_bins([(0, 1), (-2, -1)])
    >>> newhist.Draw()
    >>> newhist.GetBinContent(1)
    0.0
    >>> newhist.GetBinContent(2)
    0.0
    >>> newhist.GetBinContent(3)
    0.0
    >>> newhist.GetBinContent(4)
    0.0
    >>> newhist.GetBinContent(5)
    0.0
    >>> newhist.GetBinContent(6)
    0.0
    >>> newhist.GetBinContent(7)
    0.0
    >>> newhist.GetBinContent(8)
    1.0
    >>> newhist.GetBinContent(9)
    0.0
    >>> newhist.GetBinContent(10)
    0.0

This is a bit surprising. Either the merge_bins() function does something different, or the indexing is wrong. However, that last functionality seems to behave OK:

    >>> h.Fill(60)
    11
    >>> h[-2].value
    0.0
    >>> h[-1].value
    1.0
    >>> h[h.nbins()].value
    0.0
    >>> h[h.nbins()+1].value
    1.0

Is the piece in the docs perhaps a holdover from when the overflow bins were still indexed differently? 

In general, for 1D histograms, a member function like this might be a quite handy addition (now still written as non-member; it's my own utility function):

    def mergeOverflow(h):
        h[h.nbins()] = h[h.nbins()].value + h.overflow()
        h[h.nbins()+1] = 0



Discussion:

Resolving patch:
diff --git a/rootpy/plotting/hist.py b/rootpy/plotting/hist.py
--- a/rootpy/plotting/hist.py
+++ b/rootpy/plotting/hist.py
@@ -1314,14 +1314,16 @@ def merge_bins(self, bin_ranges, axis=0):
         for window in bin_ranges:
             if len(window) != 2:
                 raise ValueError(
-                    "bin range tuples must contain two elements")
+                    "bin range tuples must contain "
+                    "two elements: {0!r}".format(window))
             l, r = window
             if l == r:
                 raise ValueError(
-                    "bin indices must not be equal in a merging window")
-            if l < 0 and r >= 0:
+                    "bin indices must not be equal "
+                    "in a merging window: {0!r}".format(window))
+            if (l < 0 and r >= 0) or (l > 0 and r > 0 and l > r):
                 raise ValueError(
-                    "invalid bin range")
+                    "invalid bin range: {0!r}".format(window))
             if r == -1:
                 r = axis_bins
             else:
@@ -1336,11 +1338,10 @@ def merge_bins(self, bin_ranges, axis=0):
 
         # check that windows do not overlap
         if len(windows) > 1:
-            full_list = windows[0]
-            for window in windows[1:]:
-                full_list += window
-            if len(full_list) != len(set(full_list)):
-                raise ValueError("bin index windows overlap")
+            flattened = [idx for window in windows for idx in window]
+            if len(flattened) != len(set(flattened)):
+                raise ValueError(
+                    "bin index windows overlap: {0!r}".format(bin_ranges))
 
         # construct a mapping from old to new bin index along this axis
         windows.sort()
diff --git a/rootpy/roosh.py b/rootpy/roosh.py
--- a/rootpy/roosh.py
+++ b/rootpy/roosh.py
@@ -363,8 +363,7 @@ def do_ls(self, args=None):
                         if i > 0:
                             print()
                         print("{0}:".format(_dir.GetName()))
-                    keys = _dir.keys(latest=True)
-                    keys.sort(key=lambda key: key.GetName())
+                    keys = sorted(_dir.keys(latest=True), key=lambda key: key.GetName())
                     things = [color_key(key) for key in keys]
                     if things:
                         self.columnize(things)


Test patch:
diff --git a/rootpy/plotting/tests/test_hist.py b/rootpy/plotting/tests/test_hist.py
--- a/rootpy/plotting/tests/test_hist.py
+++ b/rootpy/plotting/tests/test_hist.py
@@ -226,6 +226,28 @@ def test_merge_bins():
     assert_equal(h3d.GetSumOfWeights(), h3d_merged.GetSumOfWeights())
     assert_equal(h3d_merged.nbins(1), 6)
 
+    h = Hist(4, 0, 1)
+    h.Fill(-1)
+    h.Fill(2)
+    h.Fill(0.8, 2)
+    h.Fill(0.5, 1)
+
+    # invalid ranges
+    assert_raises(ValueError, h.merge_bins, [(0, 0)])
+    assert_raises(ValueError, h.merge_bins, [(0, 1, 2)])
+    assert_raises(ValueError, h.merge_bins, [(2, 1)])
+    assert_raises(ValueError, h.merge_bins, [(-2, 1)])
+    # overlapping windows
+    assert_raises(ValueError, h.merge_bins, [(0, 1), (1, 2)])
+
+    assert_equal(list(h.y(overflow=True)), [1., 0., 0., 1., 2., 1.])
+    assert_equal(list(h.merge_bins([(1, 2)]).y(overflow=True)), [1., 0., 1., 2., 1.])
+    assert_equal(list(h.merge_bins([(-3, -2)]).y(overflow=True)), [1., 0., 0., 3., 1.] )
+    assert_equal(list(h.merge_bins([(-2, -1)]).y(overflow=True)), [1., 0., 0., 1., 3., 0.])
+    assert_equal(list(h.merge_bins([(0, 1), (-2, -1)]).y(overflow=True)), [0., 1., 0., 1., 3., 0.])
+    assert_equal(list(h.merge_bins([(0, 1)]).merge_bins([(-2, -1)]).y(overflow=True)), [0., 1., 0., 1., 3., 0.])
+    assert_equal(list(h.merge_bins([(1, 2), (3, 4)]).y(overflow=True)), [1., 0., 3., 1.])
+
 
 def test_rebinning():
     h1d = Hist(100, 0, 1)
diff --git a/rootpy/tests/test_root2hdf5.py b/rootpy/tests/test_root2hdf5.py
--- a/rootpy/tests/test_root2hdf5.py
+++ b/rootpy/tests/test_root2hdf5.py
@@ -29,13 +29,13 @@ def test_root2hdf5():
     except ImportError:
         raise SkipTest
 
-    from rootpy.root2hdf5 import root2hdf5
+    from rootpy.root2hdf5 import root2hdf5, tables_open
 
     rfile = get_file('test_tree.root')
     hfilename = os.path.join(TEMPDIR, 'out.h5')
     root2hdf5(rfile, hfilename)
 
-    hfile = tables.openFile(hfilename)
+    hfile = tables_open(hfilename)
     assert_equal(len(hfile.root.test), 1000)
     hfile.close()
 
@@ -47,13 +47,13 @@ def test_root2hdf5_chunked():
     except ImportError:
         raise SkipTest
 
-    from rootpy.root2hdf5 import root2hdf5
+    from rootpy.root2hdf5 import root2hdf5, tables_open
 
     rfile = get_file('test_tree.root')
     hfilename = os.path.join(TEMPDIR, 'out.h5')
     root2hdf5(rfile, hfilename, entries=10)
 
-    hfile = tables.openFile(hfilename)
+    hfile = tables_open(hfilename)
     assert_equal(len(hfile.root.test), 1000)
     hfile.close()
 
@@ -65,13 +65,13 @@ def test_root2hdf5_chunked_selected():
     except ImportError:
         raise SkipTest
 
-    from rootpy.root2hdf5 import root2hdf5
+    from rootpy.root2hdf5 import root2hdf5, tables_open
 
     rfile = get_file('test_tree.root')
     hfilename = os.path.join(TEMPDIR, 'out.h5')
     root2hdf5(rfile, hfilename, entries=90, selection='i % 2 == 0')
 
-    hfile = tables.openFile(hfilename)
+    hfile = tables_open(hfilename)
     assert_equal(len(hfile.root.test), 500)
     hfile.close()