Repository: pytorch/ignite

Issue:
[Feature Request] Parameter scheduling
Providing an abstraction to adjust optimizer parameters during training seems like it might be useful - techniques like [SGDR](https://arxiv.org/pdf/1608.03983.pdf) seem applicable to many types of models.

The [`torch.optim.lr_scheduler`](https://pytorch.org/docs/master/optim.html#how-to-adjust-learning-rate) module in PyTorch core implements some useful schedulers, but (a) can only adjust the LR, and (b) only adjusts it per-epoch.

On the other hand, the `Engine` event API seems like a really natural way to adjust parameter values, since handlers that manipulate them could be added for either `ITERATION_*` or `EPOCH_*` events, and modifying multiple parameters at once (e.g. LR and momentum) would be straightforward too.

I wrote a short [IPython notebook](https://gist.github.com/sean-adler/467321a23805736f8dc06a19157b0567) as a prototype of one way it could look to do this with the event API in a general way (plots are at the very bottom). I left most of the actual scheduler code in separate files for now to try and see if the idea is even worth it first. Would this be useful?

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 460e31d2..b50ef8a2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,16 @@
 language: python
+
 python:
   - "2.7"
   - "3.5"
 
 stages:
-  - lint_check
-  - test
-  - docs
+  - Lint check
+  - Test
+  - Docs
+  - Deploy
 
-install:
+before_install: &before_install
   - sudo apt-get update
   - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
   - bash miniconda.sh -b -p $HOME/miniconda
@@ -18,13 +20,15 @@ install:
   - conda update -q conda
   # Useful for debugging any issues with conda
   - conda info -a
-  - conda create -q -n test-environment -c pytorch python=$TRAVIS_PYTHON_VERSION numpy mock pytorch-cpu
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install enum34; fi
+  - conda create -q -n test-environment -c pytorch python=$TRAVIS_PYTHON_VERSION pytorch-cpu
   - source activate test-environment
-  - python setup.py install
-  - pip install --upgrade pytest codecov pytest-cov
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install enum34; fi
   # Test contrib dependencies
   - pip install tqdm scikit-learn
+
+install:
+  - python setup.py install
+  - pip install numpy mock pytest codecov pytest-cov
   # Examples dependencies
   - pip install visdom torchvision tensorboardX
   - pip install gym
@@ -52,7 +56,7 @@ script:
   # RL
   # 1) Actor-Critic
   - python examples/reinforcement_learning/actor_critic.py --max-episodes=2
-  # 1) Reinforce
+  # 2) Reinforce
   - python examples/reinforcement_learning/reinforce.py --max-episodes=2
 
   #fast-neural-style
@@ -60,37 +64,37 @@ script:
   - python examples/fast_neural_style/neural_style.py train --epochs 1 --cuda 0 --dataset test --dataroot . --image_size 32 --style_image examples/fast_neural_style/images/style_images/mosaic.jpg --style_size 32
 
 after_success:
-  # Ignore codecov failures as the codecov server is not
-  # very reliable but we don't want travis to report a failure
-  # in the github UI just because the coverage report failed to
-  # be published.
-  - codecov || echo "codecov upload failed"
+  - codecov
+
 
 jobs:
   include:
-    - stage: lint_check
-      python: "2.7"
+    - stage: Lint check
+      python: "3.5"
+      before_install: # Nothing to do
       install: pip install flake8
       script: flake8
       after_success: # Nothing to do
 
-
     # GitHub Pages Deployment: https://docs.travis-ci.com/user/deployment/pages/
-    - stage: docs
+    - stage: Docs
       python: "3.5"
+
+      # Use previously defined before_install
+      before_install: *before_install
+
       install:
-        # Minimal install : ignite and dependencies just to build the docs
         - pip install -r docs/requirements.txt
-        - pip install http://download.pytorch.org/whl/cpu/torch-0.4.1-cp35-cp35m-linux_x86_64.whl
-        # Add contrib dependencies (otherwise doc is not built)
-        - pip install scikit-learn scipy tqdm
         # `pip install .` vs `python setup.py install` : 1st works better to produce _module/ignite with source links
         - pip install .
+
       script:
         - cd docs && make html
         # Create .nojekyll file to serve correctly _static and friends
         - touch build/html/.nojekyll
       after_success: # Nothing to do
+
+      # Deploy built docs when PR is merged to master
       deploy:
         provider: pages
         skip-cleanup: true
@@ -99,3 +103,55 @@ jobs:
         local_dir: docs/build/html
         on:
           branch: master
+
+    - stage: Deploy
+      python: "3.5"
+      if: tag IS present
+
+      # Use previously defined before_install
+      before_install: *before_install
+
+      install:
+        - python setup.py install
+
+      script: true
+
+      after_success: # Nothing to do
+
+      before_deploy:
+        # Conda deploy if on tag
+        # ANACONDA_TOKEN should be provided by Travis
+        # How to generate ANACONDA_TOKEN: https://docs.anaconda.com/anaconda-cloud/user-guide/tasks/work-with-accounts#creating-access-tokens
+
+        # https://conda.io/docs/user-guide/tasks/build-packages/install-conda-build.html
+        - conda install -y conda-build conda-verify anaconda-client
+        - conda config --set anaconda_upload no
+        - conda build --quiet --no-test --output-folder conda_build conda.recipe
+        # Convert to other platforms: OSX, WIN
+        - conda convert --platform win-64 conda_build/linux-64/*.tar.bz2 -o conda_build/
+        - conda convert --platform osx-64 conda_build/linux-64/*.tar.bz2 -o conda_build/
+        # Upload to Anaconda
+        # We could use --all but too much platforms to uploaded
+        - ls conda_build/*/*.tar.bz2 | xargs -I {} anaconda -v -t $ANACONDA_TOKEN upload -u pytorch {}
+
+
+      # PyPI Deployment: https://docs.travis-ci.com/user/deployment/pypi/
+      deploy:
+
+        provider: pypi
+
+        user: $PYPI_USER
+        # If password contains non alphanumeric characters
+        # https://github.com/travis-ci/dpl/issues/377
+        # pass it as secured variable
+        password: $PYPI_TOKEN
+        # otherwise, follow "How to encrypt the password": https://docs.travis-ci.com/user/encryption-keys/
+        # `travis encrypt deploy.password="password"`
+        #  secure: "secured_password"
+
+        skip_cleanup: true
+
+        distributions: "sdist bdist_wheel"
+
+        on:
+          tags: true
diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml
new file mode 100644
index 00000000..ac94cebf
--- /dev/null
+++ b/conda.recipe/conda_build_config.yaml
@@ -0,0 +1,5 @@
+python:
+  - 2.7
+  - 3.5
+  - 3.6
+  - 3.7
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
new file mode 100644
index 00000000..05db78d4
--- /dev/null
+++ b/conda.recipe/meta.yaml
@@ -0,0 +1,38 @@
+{% set data = load_setup_py_data() %}
+
+package:
+  name: ignite
+  version: {{ data['version'] }}
+
+source:
+  path: ..
+
+build:
+  number: 0
+  script: python setup.py install --single-version-externally-managed --record=record.txt
+
+# https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#export-runtime-requirements
+requirements:
+  build:
+    - python
+    - setuptools
+    - pytorch
+    - enum34 # [py < 34]
+
+  run:
+    - python
+    - pytorch
+    - enum34 # [py < 34]
+
+test:
+  imports:
+    - ignite
+    - ignite.engine
+    - ignite.handlers
+    - ignite.metrics
+    - ignite.contrib
+
+about:
+  home: {{ data['url'] }}
+  license: {{ data['license'] }}
+  summary: {{ data['description'] }}
diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst
index 6180e50b..04cfa862 100644
--- a/docs/source/concepts.rst
+++ b/docs/source/concepts.rst
@@ -28,7 +28,7 @@ For example, model trainer for a supervised task:
         loss = loss_fn(y_pred, y)
         loss.backward()
         optimizer.step()
-        return loss.data[0]
+        return loss.item()
 
     trainer = Engine(update_model)
     trainer.run(data, max_epochs=100)
diff --git a/docs/source/contrib/metrics.rst b/docs/source/contrib/metrics.rst
index f66573ef..9a1b1078 100644
--- a/docs/source/contrib/metrics.rst
+++ b/docs/source/contrib/metrics.rst
@@ -8,3 +8,5 @@ Contribution module of metrics
 .. autoclass:: ROC_AUC
 
 .. autoclass:: AveragePrecision
+
+.. autoclass:: MaximumAbsoluteError
diff --git a/examples/gan/dcgan.py b/examples/gan/dcgan.py
index cbeef985..03669add 100644
--- a/examples/gan/dcgan.py
+++ b/examples/gan/dcgan.py
@@ -308,8 +308,11 @@ def main(dataset, dataroot,
 
     # attach running average metrics
     monitoring_metrics = ['errD', 'errG', 'D_x', 'D_G_z1', 'D_G_z2']
-    for metric in monitoring_metrics:
-        RunningAverage(alpha=alpha, output_transform=lambda x: x[metric]).attach(trainer, metric)
+    RunningAverage(alpha=alpha, output_transform=lambda x: x['errD']).attach(trainer, 'errD')
+    RunningAverage(alpha=alpha, output_transform=lambda x: x['errG']).attach(trainer, 'errG')
+    RunningAverage(alpha=alpha, output_transform=lambda x: x['D_x']).attach(trainer, 'D_x')
+    RunningAverage(alpha=alpha, output_transform=lambda x: x['D_G_z1']).attach(trainer, 'D_G_z1')
+    RunningAverage(alpha=alpha, output_transform=lambda x: x['D_G_z2']).attach(trainer, 'D_G_z2')
 
     # attach progress bar
     pbar = ProgressBar()
diff --git a/ignite/__init__.py b/ignite/__init__.py
index ecd84fe4..65cd025d 100644
--- a/ignite/__init__.py
+++ b/ignite/__init__.py
@@ -1,4 +1,7 @@
 import ignite.engine
 import ignite.handlers
+import ignite.metrics
+import ignite.exceptions
+import ignite.contrib
 
 __version__ = '0.1.0'
diff --git a/ignite/contrib/engines/tbptt.py b/ignite/contrib/engines/tbptt.py
index 33a85690..ba71a557 100644
--- a/ignite/contrib/engines/tbptt.py
+++ b/ignite/contrib/engines/tbptt.py
@@ -4,8 +4,8 @@ from enum import Enum
 
 import torch
 
-from ignite._utils import convert_tensor, apply_to_tensor
-from ignite.engine import Engine
+from ignite._utils import apply_to_tensor
+from ignite.engine import Engine, _prepare_batch
 
 
 class Tbptt_Events(Enum):
@@ -19,18 +19,6 @@ class Tbptt_Events(Enum):
     TIME_ITERATION_COMPLETED = "time_iteration_completed"
 
 
-def _prepare_tbptt_batch(batch, tbptt_step, dim=0, device=None):
-    """Prepare batch for tbptt trainer.
-
-    Batch come from the dataloader. It is split in chunks along the time
-    dimension and fed to the truncated backpropagation throught time trainer.
-    """
-    x, y = batch
-    x = convert_tensor(x, device=device)
-    y = convert_tensor(y, device=device)
-    return zip(x.split(tbptt_step, dim=dim), y.split(tbptt_step, dim=dim))
-
-
 def _detach_hidden(hidden):
     """Cut backpropagation graph.
 
@@ -46,7 +34,9 @@ def create_supervised_tbptt_trainer(
     loss_fn,
     tbtt_step,
     dim=0,
-    device=None
+    device=None,
+    non_blocking=False,
+    prepare_batch=_prepare_batch
 ):
     """Create a trainer for truncated backprop through time supervised models.
 
@@ -70,6 +60,11 @@ def create_supervised_tbptt_trainer(
         dim (int): axis representing the time dimension
         device (str, optional): device type specification (default: None).
             Applies to both model and batches.
+        non_blocking (bool, optional): if True and this copy is between CPU and GPU,
+            the copy may occur asynchronously with respect to the host. For other cases,
+            this argument has no effect.
+        prepare_batch (Callable, optional): function that receives `batch`, `device`,
+            `non_blocking` and outputs tuple of tensors `(batch_x, batch_y)`.
 
     Returns:
         Engine: a trainer engine with supervised update function
@@ -82,11 +77,9 @@ def create_supervised_tbptt_trainer(
         loss_list = []
         hidden = None
 
-        # Batches split in time chunks
-        batch_splits = _prepare_tbptt_batch(
-            batch, tbtt_step, dim=dim, device=device
-        )
-        for x_t, y_t in batch_splits:
+        x, y = batch
+        for batch_t in zip(x.split(tbtt_step, dim=dim), y.split(tbtt_step, dim=dim)):
+            x_t, y_t = prepare_batch(batch_t, device=device, non_blocking=non_blocking)
             # Fire event for start of iteration
             engine.fire_event(Tbptt_Events.TIME_ITERATION_STARTED)
             # Forward, backward and
diff --git a/ignite/contrib/handlers/param_scheduler.py b/ignite/contrib/handlers/param_scheduler.py
index 6a663385..0ec27bbc 100644
--- a/ignite/contrib/handlers/param_scheduler.py
+++ b/ignite/contrib/handlers/param_scheduler.py
@@ -4,7 +4,14 @@ import numpy as np
 
 
 class ParamScheduler(object):
-    """Updates an optimizer's parameter value during training.
+    """An abstract class for updating an optimizer's parameter value during
+    training.
+
+    Args:
+        optimizer (`torch.optim.Optimizer`): the optimizer to use
+        param_name (str): name of optimizer's parameter to update
+        save_history (bool, optional): whether to log the parameter values
+            (default=False)
     """
     def __init__(self, optimizer, param_name, save_history=False):
         self.optimizer = optimizer
@@ -28,16 +35,28 @@ class ParamScheduler(object):
 
     def get_param(self):
         """Method to get current optimizer's parameter value
-
         """
         raise NotImplementedError()
 
 
 class CyclicalScheduler(ParamScheduler):
-    """Updates an optimizer's parameter value over a cycle of some size.
+    """An abstract class for updating an optimizer's parameter value over a
+    cycle of some size.
+
+    Args:
+        optimizer (`torch.optim.Optimizer`): the optimizer to use
+        param_name (str): name of optimizer's parameter to update
+        start_value (float): value at start of cycle
+        end_value (float) : value at the middle of the cycle
+        cycle_size (int) : length of cycle.
+        cycle_mult (float, optional) : ratio by which to change the cycle_size
+            at the end of each cycle (default=1),
+        save_history (bool, optional): whether to log the parameter values
+            (default: False)
 
-    NOTE: If the scheduler is bound to an 'ITERATION_*' event, 'cycle_size' should usually be
-    the number of batches in an epoch.
+    Note:
+        If the scheduler is bound to an 'ITERATION_*' event, 'cycle_size' should
+        usually be the number of batches in an epoch.
     """
     def __init__(self,
                  optimizer,
@@ -47,7 +66,11 @@ class CyclicalScheduler(ParamScheduler):
                  cycle_size,
                  cycle_mult=1,
                  save_history=False):
-        super(CyclicalScheduler, self).__init__(optimizer, param_name, save_history=save_history)
+        super(CyclicalScheduler, self).__init__(
+            optimizer,
+            param_name,
+            save_history=save_history
+        )
         self.start_value = start_value
         self.end_value = end_value
         self.cycle_size = cycle_size
@@ -64,25 +87,172 @@ class CyclicalScheduler(ParamScheduler):
 
 
 class LinearCyclicalScheduler(CyclicalScheduler):
-    """
-    Linearly adjusts param value to 'end_value' for a half-cycle, then linearly
+    """Linearly adjusts param value to 'end_value' for a half-cycle, then linearly
     adjusts it back to 'start_value' for a half-cycle.
-    """
-    def __init__(self, *args, **kwargs):
-        super(LinearCyclicalScheduler, self).__init__(*args, **kwargs)
 
+    Args:
+        optimizer (`torch.optim.Optimizer`): the optimizer to use
+        param_name (str): name of optimizer's parameter to update
+        start_value (float): value at start of cycle
+        end_value (float) : value at the middle of the cycle
+        cycle_size (int) : length of cycle.
+        cycle_mult (float, optional) : ratio by which to change the cycle_size
+            at the end of each cycle (default=1),
+        save_history (bool, optional): whether to log the parameter values
+            (default: False)
+
+    Note:
+        If the scheduler is bound to an 'ITERATION_*' event, 'cycle_size' should
+        usually be the number of batches in an epoch.
+
+    Examples:
+
+    .. code-block:: python
+
+        from ignite.contrib.handlers.param_scheduler import LinearCyclicalScheduler
+
+        scheduler = LinearCyclicalScheduler(optimizer, 'lr', 1e-3, 1e-1, len(train_loader))
+        trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler)
+        #
+        # Linearly increases the learning rate from 1e-3 to 1e-1 and back to 1e-3
+        # over the course of 1 epoch
+        #
+    """
     def get_param(self):
         cycle_progress = self.event_index / self.cycle_size
         return self.end_value + (self.start_value - self.end_value) * abs(cycle_progress - 0.5) * 2
 
 
 class CosineAnnealingScheduler(CyclicalScheduler):
-    """
-    Anneals 'start_value' to 'end_value' over each cycle.
-    """
-    def __init__(self, *args, **kwargs):
-        super(CosineAnnealingScheduler, self).__init__(*args, **kwargs)
+    """Anneals 'start_value' to 'end_value' over each cycle.
+
+    The annealing takes the form of the first half of a cosine
+    wave (as suggested in [Smith17]_).
+
+    Args:
+        optimizer (`torch.optim.Optimizer`): the optimizer to use
+        param_name (str): name of optimizer's parameter to update
+        start_value (float): value at start of cycle
+        end_value (float) : value at the end of the cycle
+        cycle_size (int) : length of cycle.
+        cycle_mult (float, optional) : ratio by which to change the cycle_size
+            at the end of each cycle (default=1),
+        save_history (bool, optional): whether to log the parameter values
+            (default: False)
+
+    Note:
+        If the scheduler is bound to an 'ITERATION_*' event, 'cycle_size' should
+        usually be the number of batches in an epoch.
 
+    Examples:
+
+    .. code-block:: python
+
+        from ignite.contrib.handlers.param_scheduler import CosineAnnealingScheduler
+
+        scheduler = CosineAnnealingScheduler(optimizer, 'lr', 1e-1, 1e-3, len(train_loader))
+        trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler)
+        #
+        # Anneals the learning rate from 1e-1 to 1e-3 over the course of 1 epoch.
+        #
+
+    .. [Smith17] Smith, Leslie N. "Cyclical learning rates for training neural networks."
+                 Applications of Computer Vision (WACV), 2017 IEEE Winter Conference on. IEEE, 2017
+    """
     def get_param(self):
+        """Method to get current optimizer's parameter value
+        """
         cycle_progress = self.event_index / self.cycle_size
-        return self.start_value + ((self.end_value - self.start_value) / 2) * (1 + np.cos(np.pi * cycle_progress))
+        return self.start_value + ((self.end_value - self.start_value) / 2) * (1 - np.cos(np.pi * cycle_progress))
+
+
+class ConcatScheduler(ParamScheduler):
+    """Concat a list of Schedulers.
+
+    The `ConcatScheduler` cycles through a list of schedulers (given by
+    `schedulers_list`). Each element in the list is a tuple whose first
+    element is the scheduler class, the second is the parameters used for
+    instantiating the scheduler, and the third is the duration of the
+    scheduler. If duration is `None` the `ConcatScheduler` will not
+    switch to the next scheduler.
+
+    Args:
+        optimizer (`torch.optim.Optimizer`): the optimizer to use
+        param_name (str): name of optimizer's parameter to update
+        schedulers_list (list): List of three tuple of the order (scheduler_cls,
+            scheduler_kwds, duration).
+        save_history (bool, optional): whether to log the parameter values
+            (default: False)
+
+    Examples:
+
+    .. code-block:: python
+
+        from ignite.contrib.handlers.param_scheduler import ConcatScheduler
+        from ignite.contrib.handlers.param_scheduler import LinearCyclicalScheduler
+        from ignite.contrib.handlers.param_scheduler import CosineAnnealingScheduler
+
+        scheduler = ConcatScheduler(
+            optimizer,
+            "lr",
+            [
+                (
+                    LinearCyclicalScheduler,
+                    dict(
+                        start_value=0.1,
+                        end_value=0.5,
+                        cycle_size=60
+                    ),
+                    30
+                ),
+                (
+                    CosineAnnealingScheduler,
+                    dict(
+                        start_value=0.5,
+                        end_value=0.01,
+                        cycle_size=60
+                    ),
+                    None
+                ),
+            ],
+        )
+        trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler)
+        #
+        # Sets the Learning rate linearly from 0.1 to 0.5 over 30 iterations. Then
+        # starts an annealing schedule from 0.5 to 0.01 over 60 iterations.
+        # The annealing cycles are repeated indefinitely.
+        #
+    """
+    def __init__(self,
+                 optimizer,
+                 param_name,
+                 schedulers_list,
+                 save_history=False):
+        super(ConcatScheduler, self).__init__(optimizer, param_name, save_history=save_history)
+        self._schedulers_list = schedulers_list
+        self._schedulers_index = 0
+        self._next_scheduler_switch = 0
+
+    def _next_scheduler(self):
+        scheduler_cls, scheduler_kwds, self._next_scheduler_switch = \
+            self._schedulers_list[self._schedulers_index]
+
+        kwds = scheduler_kwds.copy()
+        kwds.update(
+            dict(
+                optimizer=self.optimizer,
+                param_name=self.param_name,
+                save_history=self.save_history
+            )
+        )
+
+        self._scheduler = scheduler_cls(**kwds)
+        self._schedulers_index = (self._schedulers_index + 1) % len(self._schedulers_list)
+
+    def __call__(self, engine):
+        if self._next_scheduler_switch is not None:
+            self._next_scheduler_switch -= 1
+            if self._next_scheduler_switch <= 0:
+                self._next_scheduler()
+
+        return self._scheduler(engine)
diff --git a/ignite/contrib/handlers/tqdm_logger.py b/ignite/contrib/handlers/tqdm_logger.py
index 196f57f7..7cc0723a 100644
--- a/ignite/contrib/handlers/tqdm_logger.py
+++ b/ignite/contrib/handlers/tqdm_logger.py
@@ -10,46 +10,83 @@ class ProgressBar:
     """
     TQDM progress bar handler to log training progress and computed metrics.
 
+    Args:
+        persist (bool, optional): set to ``True`` to persist the progress bar after completion (default = ``False``)
+
     Examples:
 
-        Create a progress bar that shows you some metrics as they are computed,
-        by simply attaching the progress bar object to your engine.
+        Simple progress bar
+
+        .. code-block:: python
+
+            trainer = create_supervised_trainer(model, optimizer, loss)
+
+            pbar = ProgressBar()
+            pbar.attach(trainer)
+
+        Attach metrics that already have been computed at `ITERATION_COMPLETED` (such as `RunningAverage`)
 
         .. code-block:: python
 
+            trainer = create_supervised_trainer(model, optimizer, loss)
+
+            RunningAverage(output_transform=lambda x: x).attach(trainer, 'loss')
+
             pbar = ProgressBar()
             pbar.attach(trainer, ['loss'])
 
+        Directly attach the engine's output
+
+        .. code-block:: python
+
+            trainer = create_supervised_trainer(model, optimizer, loss)
+
+            pbar = ProgressBar()
+            pbar.attach(trainer, output_transform=lambda x: {'loss': x})
+
     Note:
         When adding attaching the progress bar to an engine, it is recommend that you replace
         every print operation in the engine's handlers triggered every iteration with
         ``pbar.log_message`` to guarantee the correct format of the stdout.
     """
 
-    def __init__(self):
+    def __init__(self, persist=False):
         self.pbar = None
+        self.persist = persist
 
     def _reset(self, engine):
         self.pbar = tqdm(
             total=len(engine.state.dataloader),
-            leave=False,
+            leave=self.persist,
             bar_format='{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]')
 
     def _close(self, engine):
         self.pbar.close()
         self.pbar = None
 
-    def _update(self, engine, metric_names=None):
+    def _update(self, engine, metric_names=None, output_transform=None):
         if self.pbar is None:
             self._reset(engine)
 
-        self.pbar.set_description('Epoch {}'.format(engine.state.epoch))
+        self.pbar.set_description('Epoch [{}/{}]'.format(engine.state.epoch, engine.state.max_epochs))
 
+        metrics = {}
         if metric_names is not None:
             if not all(metric in engine.state.metrics for metric in metric_names):
+                self._close(engine)
                 raise KeyError("metrics not found in engine.state.metrics")
 
-            metrics = {name: '{:.2e}'.format(engine.state.metrics[name]) for name in metric_names}
+            metrics.update({name: '{:.2e}'.format(engine.state.metrics[name]) for name in metric_names})
+
+        if output_transform is not None:
+            output_dict = output_transform(engine.state.output)
+
+            if not isinstance(output_dict, dict):
+                output_dict = {"output": output_dict}
+
+            metrics.update({name: '{:.2e}'.format(value) for name, value in output_dict.items()})
+
+        if metrics:
             self.pbar.set_postfix(**metrics)
 
         self.pbar.update()
@@ -64,16 +101,23 @@ class ProgressBar:
         """
         tqdm.write(message)
 
-    def attach(self, engine, metric_names=None):
+    def attach(self, engine, metric_names=None, output_transform=None):
         """
         Attaches the progress bar to an engine object
 
         Args:
             engine (Engine): engine object
-            metric_names (list): (Optional) list of the metrics names to log as the bar progresses
+            metric_names (list, optional): list of the metrics names to log as the bar progresses
+            output_transform (Callable, optional): a function to select what you want to print from the engine's
+                output. This function may return either a dictionary with entries in the format of ``{name: value}``,
+                or a single scalar, which will be displayed with the default name `output`.
         """
-        if not isinstance(metric_names, list):
+        if metric_names is not None and not isinstance(metric_names, list):
             raise TypeError("metric_names should be a list, got {} instead".format(type(metric_names)))
 
+        if output_transform is not None and not callable(output_transform):
+            raise TypeError("output_transform should be a function, got {} instead"
+                            .format(type(output_transform)))
+
         engine.add_event_handler(Events.EPOCH_COMPLETED, self._close)
-        engine.add_event_handler(Events.ITERATION_COMPLETED, self._update, metric_names)
+        engine.add_event_handler(Events.ITERATION_COMPLETED, self._update, metric_names, output_transform)
diff --git a/ignite/contrib/metrics/__init__.py b/ignite/contrib/metrics/__init__.py
index 3e52ca91..9f0d40b1 100644
--- a/ignite/contrib/metrics/__init__.py
+++ b/ignite/contrib/metrics/__init__.py
@@ -1,3 +1,4 @@
 
 from ignite.contrib.metrics.average_precision import AveragePrecision
 from ignite.contrib.metrics.roc_auc import ROC_AUC
+from ignite.contrib.metrics.mae import MaximumAbsoluteError
diff --git a/ignite/contrib/metrics/mae.py b/ignite/contrib/metrics/mae.py
new file mode 100644
index 00000000..45f3fbae
--- /dev/null
+++ b/ignite/contrib/metrics/mae.py
@@ -0,0 +1,24 @@
+from ignite.metrics import Metric
+from ignite.exceptions import NotComputableError
+import torch
+
+
+class MaximumAbsoluteError(Metric):
+    """
+    Calculates the maximum absolute error.
+
+    - `update` must receive output of the form `(y_pred, y)`.
+    """
+    def reset(self):
+        self._max_of_absolute_errors = -1
+
+    def update(self, output):
+        y_pred, y = output
+        mae = torch.abs(y_pred - y.view_as(y_pred)).max().item()
+        if self._max_of_absolute_errors < mae:
+            self._max_of_absolute_errors = mae
+
+    def compute(self):
+        if self._max_of_absolute_errors < 0:
+            raise NotComputableError('MaximumAbsoluteError must have at least one example before it can be computed')
+        return self._max_of_absolute_errors
diff --git a/ignite/engine/__init__.py b/ignite/engine/__init__.py
index ac7128d3..ccb4de54 100644
--- a/ignite/engine/__init__.py
+++ b/ignite/engine/__init__.py
@@ -5,6 +5,9 @@ from ignite._utils import convert_tensor
 
 
 def _prepare_batch(batch, device=None, non_blocking=False):
+    """Prepare batch for training: pass to a device with options
+
+    """
     x, y = batch
     return (convert_tensor(x, device=device, non_blocking=non_blocking),
             convert_tensor(y, device=device, non_blocking=non_blocking))
diff --git a/ignite/metrics/loss.py b/ignite/metrics/loss.py
index ca809855..41fe6deb 100644
--- a/ignite/metrics/loss.py
+++ b/ignite/metrics/loss.py
@@ -20,12 +20,16 @@ class Loss(Metric):
             The output is is expected to be a tuple (prediction, target) or
             (prediction, target, kwargs) where kwargs is a dictionary of extra
             keywords arguments.
+        batch_size (callable): a callable taking a target tensor that returns the
+            first dimension size (usually the batch size).
 
     """
 
-    def __init__(self, loss_fn, output_transform=lambda x: x):
+    def __init__(self, loss_fn, output_transform=lambda x: x,
+                 batch_size=lambda x: x.shape[0]):
         super(Loss, self).__init__(output_transform)
         self._loss_fn = loss_fn
+        self._batch_size = batch_size
 
     def reset(self):
         self._sum = 0
@@ -42,8 +46,9 @@ class Loss(Metric):
         if len(average_loss.shape) != 0:
             raise ValueError('loss_fn did not return the average loss')
 
-        self._sum += average_loss.item() * y.shape[0]
-        self._num_examples += y.shape[0]
+        N = self._batch_size(y)
+        self._sum += average_loss.item() * N
+        self._num_examples += N
 
     def compute(self):
         if self._num_examples == 0:


Test patch:
diff --git a/tests/ignite/contrib/handlers/test_param_scheduler.py b/tests/ignite/contrib/handlers/test_param_scheduler.py
index 927d70da..d6c3bf0b 100644
--- a/tests/ignite/contrib/handlers/test_param_scheduler.py
+++ b/tests/ignite/contrib/handlers/test_param_scheduler.py
@@ -4,6 +4,7 @@ import torch
 
 from ignite.engine import Engine, Events
 from ignite.contrib.handlers.param_scheduler import LinearCyclicalScheduler, CosineAnnealingScheduler
+from ignite.contrib.handlers.param_scheduler import ConcatScheduler
 
 
 def test_linear_scheduler():
@@ -56,7 +57,7 @@ def test_cosine_annealing_scheduler():
     tensor = torch.zeros([1], requires_grad=True)
     optimizer = torch.optim.SGD([tensor], lr=0)
 
-    scheduler = CosineAnnealingScheduler(optimizer, 'lr', 1, 0, 10)
+    scheduler = CosineAnnealingScheduler(optimizer, 'lr', 0, 1, 10)
     lrs = []
 
     def save_lr(engine):
@@ -75,6 +76,56 @@ def test_cosine_annealing_scheduler():
     ]))
 
 
+def test_concat_scheduler():
+    tensor = torch.zeros([1], requires_grad=True)
+    optimizer = torch.optim.SGD([tensor], lr=0)
+
+    concat_scheduler = ConcatScheduler(
+        optimizer=optimizer,
+        param_name='lr',
+        schedulers_list=[
+            (
+                LinearCyclicalScheduler,
+                dict(
+                    start_value=1,
+                    end_value=0,
+                    cycle_size=10
+                ),
+                10
+            ),
+            (
+                CosineAnnealingScheduler,
+                dict(
+                    start_value=0,
+                    end_value=1,
+                    cycle_size=10
+                ),
+                None
+            )
+        ],
+        save_history=True
+    )
+
+    lrs = []
+
+    def save_lr(engine):
+        lrs.append(optimizer.param_groups[0]['lr'])
+
+    trainer = Engine(lambda engine, batch: None)
+    trainer.add_event_handler(Events.ITERATION_COMPLETED, concat_scheduler)
+    trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
+    trainer.run([0] * 10, max_epochs=2)
+
+    assert lrs == list(map(pytest.approx, [
+        # Cycle 1 of the LinearCyclicalScheduler
+        1.0, 0.8, 0.6, 0.4, 0.2,
+        0.0, 0.2, 0.4, 0.6, 0.8,
+        # Cycle 1 of the CosineAnnealingScheduler
+        0.0, 0.02447174185242318, 0.09549150281252627, 0.20610737385376332, 0.3454915028125263,
+        0.5, 0.6545084971874737, 0.7938926261462365, 0.9045084971874737, 0.9755282581475768,
+    ]))
+
+
 def test_save_param_history():
     tensor = torch.zeros([1], requires_grad=True)
     optimizer = torch.optim.SGD([tensor], lr=0)
diff --git a/tests/ignite/contrib/handlers/test_pbar.py b/tests/ignite/contrib/handlers/test_pbar.py
index aeecbb51..68d1d1f1 100644
--- a/tests/ignite/contrib/handlers/test_pbar.py
+++ b/tests/ignite/contrib/handlers/test_pbar.py
@@ -9,8 +9,9 @@ from ignite.metrics import CategoricalAccuracy
 
 
 def update_fn(engine, batch):
-    engine.state.metrics['a'] = 1
-    return None
+    a = 1
+    engine.state.metrics['a'] = a
+    return a
 
 
 def test_pbar(capsys):
@@ -28,7 +29,7 @@ def test_pbar(capsys):
     err = captured.err.split('\r')
     err = list(map(lambda x: x.strip(), err))
     err = list(filter(None, err))
-    expected = u'Epoch 2: [1/2]  50%|█████     , a=1.00e+00 [00:00<00:00]'
+    expected = u'Epoch [2/2]: [1/2]  50%|█████     , a=1.00e+00 [00:00<00:00]'
     assert err[-1] == expected
 
 
@@ -66,3 +67,67 @@ def test_pbar_with_metric():
 
     with pytest.raises(KeyError):
         trainer.run(data=data, max_epochs=1)
+
+
+def test_pbar_no_metric_names(capsys):
+
+    n_epochs = 2
+    loader = [1, 2]
+    engine = Engine(update_fn)
+
+    pbar = ProgressBar()
+    pbar.attach(engine)
+
+    engine.run(loader, max_epochs=n_epochs)
+
+    captured = capsys.readouterr()
+    err = captured.err.split('\r')
+    err = list(map(lambda x: x.strip(), err))
+    err = list(filter(None, err))
+    actual = err[-1]
+    expected = u'Epoch [2/2]: [1/2]  50%|█████      [00:00<00:00]'
+    assert actual == expected
+
+
+def test_pbar_with_output(capsys):
+    n_epochs = 2
+    loader = [1, 2]
+    engine = Engine(update_fn)
+
+    pbar = ProgressBar()
+    pbar.attach(engine, output_transform=lambda x: {'a': x})
+
+    engine.run(loader, max_epochs=n_epochs)
+
+    captured = capsys.readouterr()
+    err = captured.err.split('\r')
+    err = list(map(lambda x: x.strip(), err))
+    err = list(filter(None, err))
+    expected = u'Epoch [2/2]: [1/2]  50%|█████     , a=1.00e+00 [00:00<00:00]'
+    assert err[-1] == expected
+
+
+def test_pbar_fail_with_non_callable_transform():
+    engine = Engine(update_fn)
+    pbar = ProgressBar()
+
+    with pytest.raises(TypeError):
+        pbar.attach(engine, output_transform=1)
+
+
+def test_pbar_with_scalar_output(capsys):
+    n_epochs = 2
+    loader = [1, 2]
+    engine = Engine(update_fn)
+
+    pbar = ProgressBar()
+    pbar.attach(engine, output_transform=lambda x: x)
+
+    engine.run(loader, max_epochs=n_epochs)
+
+    captured = capsys.readouterr()
+    err = captured.err.split('\r')
+    err = list(map(lambda x: x.strip(), err))
+    err = list(filter(None, err))
+    expected = u'Epoch [2/2]: [1/2]  50%|█████     , output=1.00e+00 [00:00<00:00]'
+    assert err[-1] == expected
diff --git a/tests/ignite/contrib/metrics/test_mae.py b/tests/ignite/contrib/metrics/test_mae.py
new file mode 100644
index 00000000..46f36cf4
--- /dev/null
+++ b/tests/ignite/contrib/metrics/test_mae.py
@@ -0,0 +1,26 @@
+from ignite.contrib.metrics import MaximumAbsoluteError
+import torch
+from pytest import approx
+
+
+def test_maximum_absolute_error():
+    a = torch.tensor([0.0, 0.2, -0.8])
+    b = torch.tensor([1.0, -2.0, 3.0])
+    c = torch.tensor([0.0, 2.0, 0.0])
+    d = torch.tensor([0.0, 0.0, -5.0])
+    ground_truth = torch.tensor([0.0, 0.0, 0.0])
+
+    m = MaximumAbsoluteError()
+    m.reset()
+
+    m.update((a, ground_truth))
+    assert m.compute() == approx(0.8)
+
+    m.update((b, ground_truth))
+    assert m.compute() == approx(3.0)
+
+    m.update((c, ground_truth))
+    assert m.compute() == approx(3.0)
+
+    m.update((d, ground_truth))
+    assert m.compute() == approx(5.0)