Repository: lisa-lab/pylearn2

Issue:
Monitor stores num_examples as a float, causing failures with large numbers of examples
```
  File "/home/ubuntu/pylearn2/pylearn2/train.py", line 196, in main_loop
    self.run_callbacks_and_monitoring()
  File "/home/ubuntu/pylearn2/pylearn2/train.py", line 242, in run_callbacks_and_monitoring
    self.model.monitor()
  File "/home/ubuntu/pylearn2/pylearn2/monitor.py", line 264, in __call__
    (ne, actual_ne))
RuntimeError: At compile time, your iterator said it had 60002992 examples total, but at runtime it gave us 60002994.
```

This seems to be caused by monitor.py line 441-442:

```
        self.num_examples = [np.cast[config.floatX](float(i.num_examples))
                             for i in it]
```

a float32 can't represent the number 60002994:

```
>>> import numpy as np
>>> np.cast["float32"](float(60002994.0))
array(60002992.0, dtype=float32)
```



Discussion:
Thanks for the report.
TODO:
- store Monitor.num_examples as an int64
- make sure monitors serialized previously can still be reloaded
- write tests.


Resolving patch:
diff --git a/pylearn2/monitor.py b/pylearn2/monitor.py
--- a/pylearn2/monitor.py
+++ b/pylearn2/monitor.py
@@ -438,8 +438,7 @@ def redo_theano(self):
             it.append(d.iterator(mode=i, num_batches=n, batch_size=b,
                                  data_specs=self._flat_data_specs,
                                  return_tuple=True))
-        self.num_examples = [np.cast[config.floatX](float(i.num_examples))
-                             for i in it]
+        self.num_examples = [i.num_examples for i in it]
         givens = [OrderedDict() for d in self._datasets]
         updates = [OrderedDict() for d in self._datasets]
         for i, channel in enumerate(self.channels.values()):
@@ -474,8 +473,8 @@ def redo_theano(self):
                 if n == 0:
                     raise ValueError("Iterating over 0 examples results in " +
                                      "divide by 0")
-                val = (channel.val * T.cast(batch_size, config.floatX)
-                       / cur_num_examples)
+                val = T.cast(channel.val * T.cast(batch_size, 'float64')
+                             / cur_num_examples, config.floatX)
             u[channel.val_shared] = channel.val_shared + val
 
         with log_timing(log, "Compiling accum"):


Test patch:
diff --git a/pylearn2/tests/test_monitor.py b/pylearn2/tests/test_monitor.py
--- a/pylearn2/tests/test_monitor.py
+++ b/pylearn2/tests/test_monitor.py
@@ -134,6 +134,31 @@ def test_counting():
     assert isinstance(model.monitor.get_examples_seen(), py_integer_types)
     assert isinstance(model.monitor.get_batches_seen(), py_integer_types)
 
+def test_large_examples():
+    BATCH_SIZE = 10000
+    num_examples = 60002994
+    NUM_FEATURES = 3
+
+    model = DummyModel(NUM_FEATURES)
+    monitor = Monitor.get_monitor(model)
+
+    monitoring_dataset = DummyDataset(num_examples = num_examples,
+            num_features = NUM_FEATURES)
+
+    monitor.add_dataset(monitoring_dataset, 'sequential', batch_size=BATCH_SIZE)
+
+    name = 'z'
+
+    monitor.add_channel(name = name,
+            ipt = model.input_space.make_theano_batch(),
+            val = 0.,
+            data_specs=(model.get_input_space(), model.get_input_source()))
+
+    try:
+        monitor()
+    except RuntimeError:
+        assert False
+
 def test_reject_empty():
 
     # Test that Monitor raises an error if asked to iterate over 0 batches