def save(self, path_yaml):
        """ Write options dictionary to a yaml file
        """
        Options.save_yaml_opts(self.options, path_yaml)

def save ( self , path_yaml ) : Options . save_yaml_opts ( self . options , path_yaml )


def load_yaml_opts(source):
        """ Load options dictionary from a yaml file
        """
        result = {}
        if isinstance(source, str):
            with open(source, 'r') as yaml_file:
                options_dict = yaml.safe_load(yaml_file)
        elif isinstance(source, dict):
            options_dict = source
        else:
            raise TypeError('Unsupported source type: {}'.format(type(source)))
        includes = options_dict.get('__include__', False)
        if includes:
            if type(includes) != list:
                includes = [includes]
            for include in includes:
                filename = '{}/{}'.format(os.path.dirname(source), include)
                if os.path.isfile(filename):
                    parent = Options.load_yaml_opts(filename)
                else:
                    parent = Options.load_yaml_opts(include)
                merge_dictionaries(result, parent)
        merge_dictionaries(result, options_dict)  # to be sure the main options overwrite the parent options
        result.pop('__include__', None)
        result = OptionsDict(result)
        return result


Collects information about the environment for reproducibility

def env_info():
    """ Collects information about the environment, for reproducibility """
    info = {}
    info['python_version'] = sys.version
    info['command'] = subprocess.list2cmdline(sys.argv)
    with open(os.devnull, 'w') as devnull:
        info['pip_modules'] = subprocess.check_output(['pip', 'freeze'], stderr=devnull)
        try:
            git_branch_cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD']
            info['git_branch'] = subprocess.check_output(git_branch_cmd, stderr=devnull).strip().decode('UTF-8')
            git_local_commit_cmd = ['git', 'rev-parse', 'HEAD']
            git_status_cmd = ['git', 'status']
            git_origin_commit_cmd = ['git', 'rev-parse', 'origin/{}'.format(info['git_branch'])]
            git_diff_origin_commit_cmd = ['git', 'diff', 'origin/{}'.format(info['git_branch'])]
            info['git_origin_commit'] = subprocess.check_output(git_origin_commit_cmd, stderr=devnull)
            git_log_since_origin_cmd = ['git', 'log', '--pretty=oneline', '{}..HEAD'.format(info['git_origin_commit'])]
            info['git_local_commit'] = subprocess.check_output(git_local_commit_cmd, stderr=devnull)
            info['git_status'] = subprocess.check_output(git_status_cmd, stderr=devnull)
            info['git_diff_origin_commit'] = subprocess.check_output(git_diff_origin_commit_cmd, stderr=devnull)
            info['git_log_since_origin'] = subprocess.check_output(git_log_since_origin_cmd, stderr=devnull)
        except subprocess.CalledProcessError:
            pass
    info['creation_time'] = time.strftime('%y-%m-%d-%H-%M-%S')
    info['sysname'] = os.uname()[0]
    info['nodename'] = os.uname()[1]
    info['release'] = os.uname()[2]
    info['version'] = os.uname()[3]
    info['architecture'] = os.uname()[4]
    info['user'] = os.environ['USER']
    info['path'] = os.environ['PWD']
    info['conda_environment'] = os.environ.get('CONDA_DEFAULT_ENV', '')
    info['environment_vars'] = dict(os.environ)
    info = {k: info[k].decode("UTF-8") if type(info[k]) == bytes else info[k] for k in info}
    return info

def env_info ( ) : info = { } info [ 'python_version' ] = sys . version info [ 'command' ] = subprocess . list2cmdline ( sys . argv ) with open ( os . devnull , 'w' ) as devnull : info [ 'pip_modules' ] = subprocess . check_output ( [ 'pip' , 'freeze' ] , stderr = devnull ) try : git_branch_cmd = [ 'git' , 'rev-parse' , '--abbrev-ref' , 'HEAD' ] info [ 'git_branch' ] = subprocess . check_output ( git_branch_cmd , stderr = devnull ) . strip ( ) . decode ( 'UTF-8' ) git_local_commit_cmd = [ 'git' , 'rev-parse' , 'HEAD' ] git_status_cmd = [ 'git' , 'status' ] git_origin_commit_cmd = [ 'git' , 'rev-parse' , 'origin/{}' . format ( info [ 'git_branch' ] ) ] git_diff_origin_commit_cmd = [ 'git' , 'diff' , 'origin/{}' . format ( info [ 'git_branch' ] ) ] info [ 'git_origin_commit' ] = subprocess . check_output ( git_origin_commit_cmd , stderr = devnull ) git_log_since_origin_cmd = [ 'git' , 'log' , '--pretty=oneline' , '{}..HEAD' . format ( info [ 'git_origin_commit' ] ) ] info [ 'git_local_commit' ] = subprocess . check_output ( git_local_commit_cmd , stderr = devnull ) info [ 'git_status' ] = subprocess . check_output ( git_status_cmd , stderr = devnull ) info [ 'git_diff_origin_commit' ] = subprocess . check_output ( git_diff_origin_commit_cmd , stderr = devnull ) info [ 'git_log_since_origin' ] = subprocess . check_output ( git_log_since_origin_cmd , stderr = devnull ) except subprocess . CalledProcessError : pass info [ 'creation_time' ] = time . strftime ( '%y-%m-%d-%H-%M-%S' ) info [ 'sysname' ] = os . uname ( ) [ 0 ] info [ 'nodename' ] = os . uname ( ) [ 1 ] info [ 'release' ] = os . uname ( ) [ 2 ] info [ 'version' ] = os . uname ( ) [ 3 ] info [ 'architecture' ] = os . uname ( ) [ 4 ] info [ 'user' ] = os . environ [ 'USER' ] info [ 'path' ] = os . environ [ 'PWD' ] info [ 'conda_environment' ] = os . environ . get ( 'CONDA_DEFAULT_ENV' , '' ) info [ 'environment_vars' ] = dict ( os . environ ) info = { k : info [ k ] . decode ( "UTF-8" ) if type ( info [ k ] ) == bytes else info [ k ] for k in info } return info


def eval(self):
        """ Activate evaluation mode
        """
        super(Model, self).train(mode=False)
        self.mode = 'eval'

def eval ( self ) : super ( Model , self ) . train ( mode = False ) self . mode = 'eval'


def train(self):
        """ Activate training mode
        """
        super(Model, self).train(mode=True)
        self.mode = 'train'

def train ( self ) : super ( Model , self ) . train ( mode = True ) self . mode = 'train'


Moves all model parameters and buffers to the GPU .

def cuda(self, device_id=None):
        """ Moves all model parameters and buffers to the GPU.

            Args:
                device_id (int, optional): if specified, all parameters will be
                    copied to that device
        """
        self.is_cuda = True
        return self._apply(lambda t: t.cuda(device_id))

def cuda ( self , device_id = None ) : self . is_cuda = True return self . _apply ( lambda t : t . cuda ( device_id ) )


Moves all model parameters and buffers to the CPU .

def cpu(self):
        """ Moves all model parameters and buffers to the CPU.
        """
        self.is_cuda = False
        return self._apply(lambda t: t.cpu())

def cpu ( self ) : self . is_cuda = False return self . _apply ( lambda t : t . cpu ( ) )


Prepare a batch with two functions : cuda_tf and detach_tf ( only in eval mode )

def prepare_batch(self, batch):
        """ Prepare a batch with two functions: cuda_tf and detach_tf (only in eval mode)
        """
        if self.is_cuda:
            batch = self.cuda_tf()(batch)
        if self.mode == 'eval':
            batch = self.detach_tf()(batch)
        return batch

def prepare_batch ( self , batch ) : if self . is_cuda : batch = self . cuda_tf ( ) ( batch ) if self . mode == 'eval' : batch = self . detach_tf ( ) ( batch ) return batch


Prepare the batch and feed it to the network criterion and metric .

def forward(self, batch):
        """ Prepare the batch and feed it to the network, criterion and metric.

            Returns:
                out (dict): a dictionary of outputs
        """
        batch = self.prepare_batch(batch)
        net_out = self.network(batch)

        cri_out = {}
        if self.mode in self.criterions:
            cri_tmp = self.criterions[self.mode](net_out, batch)
            if cri_tmp is not None:
                cri_out = cri_tmp

        met_out = {}
        if self.mode in self.metrics:
            met_tmp = self.metrics[self.mode](cri_out, net_out, batch)
            if met_tmp is not None:
                met_out = met_tmp

        out = {}
        if type(net_out) is dict:
            for key, value in net_out.items():
                out[key] = value
        if type(cri_out) is dict:
            for key, value in cri_out.items():
                out[key] = value
        if type(met_out) is dict:
            for key, value in met_out.items():
                out[key] = value
        return out

def forward ( self , batch ) : batch = self . prepare_batch ( batch ) net_out = self . network ( batch ) cri_out = { } if self . mode in self . criterions : cri_tmp = self . criterions [ self . mode ] ( net_out , batch ) if cri_tmp is not None : cri_out = cri_tmp met_out = { } if self . mode in self . metrics : met_tmp = self . metrics [ self . mode ] ( cri_out , net_out , batch ) if met_tmp is not None : met_out = met_tmp out = { } if type ( net_out ) is dict : for key , value in net_out . items ( ) : out [ key ] = value if type ( cri_out ) is dict : for key , value in cri_out . items ( ) : out [ key ] = value if type ( met_out ) is dict : for key , value in met_out . items ( ) : out [ key ] = value return out


def _init_network(self, engine=None):
        """ Create the network using the bootstrap network factory
        """
        return net_factory(engine)

def _init_network ( self , engine = None ) : return net_factory ( engine )


def _init_criterions(self, engine=None):
        """ Create the two criterions using the bootstrap criterion factory
        """
        # by default all modes have criterions
        if engine:
            modes = list(engine.dataset.keys())  # [train, val] for mnist
        else:
            modes = ['train', 'eval']

        criterions = {}
        for mode in modes:
            tmp_cri = cri_factory(engine, mode)
            if tmp_cri is not None:
                criterions[mode] = tmp_cri
        return criterions

def
_init_criterions
(
self
,
engine
=
)
:
# by default all modes have criterions
if
engine
:
modes
=
list
(
engine
.
dataset
.
keys
(
)
)
# [train, val] for mnist
else
:
modes
=
[
'train'
,
'eval'
]
criterions
=
{
}
for
mode
in
modes
:
tmp_cri
=
cri_factory
(
engine
,
mode
)
if
tmp_cri
is
not
:
criterions
[
mode
]
=
tmp_cri
return
criterions


def _init_metrics(self, engine=None):
        """ Create the two metrics using the bootstrap metric factory
        """
        # by default all modes have metrics
        if engine:
            modes = list(engine.dataset.keys())
        else:
            modes = ['train', 'eval']

        metrics = {}
        for mode in modes:
            tmp_met = met_factory(engine, mode)
            if tmp_met is not None:
                metrics[mode] = tmp_met
        return metrics

def
_init_metrics
(
self
,
engine
=
)
:
# by default all modes have metrics
if
engine
:
modes
=
list
(
engine
.
dataset
.
keys
(
)
)
else
:
modes
=
[
'train'
,
'eval'
]
metrics
=
{
}
for
mode
in
modes
:
tmp_met
=
met_factory
(
engine
,
mode
)
if
tmp_met
is
not
:
metrics
[
mode
]
=
tmp_met
return
metrics


The forward call to the network uses batch [ data ] instead of batch

def forward(self, batch):
        """ The forward call to the network uses batch['data'] instead of batch
        """
        batch = self.prepare_batch(batch)
        net_out = self.network(batch['data'])

        cri_out = {}
        if self.mode in self.criterions:
            cri_tmp = self.criterions[self.mode](net_out, batch)
            if cri_tmp is not None:
                cri_out = cri_tmp

        met_out = {}
        if self.mode in self.metrics:
            met_tmp = self.metrics[self.mode](cri_out, net_out, batch)
            if met_tmp is not None:
                met_out = met_tmp

        out = {}
        if type(net_out) is dict:
            for key, value in net_out.items():
                out[key] = value
        if type(cri_out) is dict:
            for key, value in cri_out.items():
                out[key] = value
        if type(met_out) is dict:
            for key, value in met_out.items():
                out[key] = value
        return out

def forward ( self , batch ) : batch = self . prepare_batch ( batch ) net_out = self . network ( batch [ 'data' ] ) cri_out = { } if self . mode in self . criterions : cri_tmp = self . criterions [ self . mode ] ( net_out , batch ) if cri_tmp is not None : cri_out = cri_tmp met_out = { } if self . mode in self . metrics : met_tmp = self . metrics [ self . mode ] ( cri_out , net_out , batch ) if met_tmp is not None : met_out = met_tmp out = { } if type ( net_out ) is dict : for key , value in net_out . items ( ) : out [ key ] = value if type ( cri_out ) is dict : for key , value in cri_out . items ( ) : out [ key ] = value if type ( met_out ) is dict : for key , value in met_out . items ( ) : out [ key ] = value return out


def accuracy(output, target, topk=None, ignore_index=None):
    """Computes the precision@k for the specified values of k"""
    topk = topk or [1, 5]

    if ignore_index is not None:
        target_mask = (target != ignore_index)
        target = target[target_mask]
        output_mask = target_mask.unsqueeze(1)
        output_mask = output_mask.expand_as(output)
        output = output[output_mask]
        output = output.view(-1, output_mask.size(1))

    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size)[0])
    return res

def accuracy ( output , target , topk = None , ignore_index = None ) : topk = topk or [ 1 , 5 ] if ignore_index is not None : target_mask = ( target != ignore_index ) target = target [ target_mask ] output_mask = target_mask . unsqueeze ( 1 ) output_mask = output_mask . expand_as ( output ) output = output [ output_mask ] output = output . view ( - 1 , output_mask . size ( 1 ) ) maxk = max ( topk ) batch_size = target . size ( 0 ) _ , pred = output . topk ( maxk , 1 , True , True ) pred = pred . t ( ) correct = pred . eq ( target . view ( 1 , - 1 ) . expand_as ( pred ) ) res = [ ] for k in topk : correct_k = correct [ : k ] . view ( - 1 ) . float ( ) . sum ( 0 , keepdim = True ) res . append ( correct_k . mul_ ( 100.0 / batch_size ) [ 0 ] ) return res


r Gathers tensors from different GPUs on a specified device ( - 1 means the CPU ) .

def gather(outputs, target_device, dim=0):
    r"""
    Gathers tensors from different GPUs on a specified device
      (-1 means the CPU).
    """
    def gather_map(outputs):
        out = outputs[0]
        if torch.is_tensor(out):
            return Gather.apply(target_device, dim, *outputs)
        if out is None:
            return None
        if isinstance(out, dict):
            if not all((len(out) == len(d) for d in outputs)):
                raise ValueError('All dicts must have the same number of keys')
            return type(out)(((k, gather_map([d[k] for d in outputs]))
                              for k in out))
        return type(out)(map(gather_map, zip(*outputs)))

    # Recursive function calls like this create reference cycles.
    # Setting the function to None clears the refcycle.
    try:
        return gather_map(outputs)
    finally:
        gather_map = None


Generate a view . html via an asynchronous call to self . view . generate ()

def generate_view(self):
        """ Generate a view.html via an asynchronous call to `self.view.generate()`
        """
        if self.view is not None:
            if hasattr(self.view, 'current_thread') and self.view.current_thread.is_alive():
                Logger()('Skipping view generation: another view is already being generated', log_level=Logger.WARNING)
                Logger()('Consider removing batch entries from views.items in order to speed it up', log_level=Logger.WARNING)
            else:
                # TODO: Redesign this threading system so it wont slow down training
                # Python threads don't really exist, because of the interpreter lock
                # we might need multi-proessing or some other way.
                self.view.current_thread = threading.Thread(target=self.view.generate)
                self.view.current_thread.start()


Run all the callback functions that have been registered for a hook .

def hook(self, name):
        """ Run all the callback functions that have been registered
            for a hook.

            Args:
                name: the name of the hook
        """
        if name in self.hooks:
            for func in self.hooks[name]:
                func()

def hook ( self , name ) : if name in self . hooks : for func in self . hooks [ name ] : func ( )


Register a callback function to be triggered when the hook is called .

def register_hook(self, name, func):
        """ Register a callback function to be triggered when the hook
            is called.

            Args:
                name: the name of the hook
                func: the callback function (no argument)

            Example usage:

            .. code-block:: python

                def func():
                    print('hooked!')

                engine.register_hook('train_on_start_batch', func)
        """
        if name not in self.hooks:
            self.hooks[name] = []
        self.hooks[name].append(func)

def register_hook ( self , name , func ) : if name not in self . hooks : self . hooks [ name ] = [ ] self . hooks [ name ] . append ( func )


Resume a checkpoint using the bootstrap . lib . options . Options

def resume(self, map_location=None):
        """ Resume a checkpoint using the `bootstrap.lib.options.Options`
        """
        Logger()('Loading {} checkpoint'.format(Options()['exp']['resume']))
        self.load(Options()['exp']['dir'],
                  Options()['exp']['resume'],
                  self.model, self.optimizer,
                  map_location=map_location)
        self.epoch += 1

def
resume
(
self
,
map_location
=
)
:
Logger
(
)
(
'Loading {} checkpoint'
.
format
(
Options
(
)
[
'exp'
]
[
'resume'
]
)
)
self
.
load
(
Options
(
)
[
'exp'
]
[
'dir'
]
,
Options
(
)
[
'exp'
]
[
'resume'
]
,
self
.
model
,
self
.
optimizer
,
map_location
=
map_location
)
self
.
epoch
+=
1


def eval(self):
        """ Launch evaluation procedures
        """
        Logger()('Launching evaluation procedures')

        if Options()['dataset']['eval_split']:
            # self.epoch-1 to be equal to the same resumed epoch
            # or to be equal to -1 when not resumed
            self.eval_epoch(self.model, self.dataset['eval'], self.epoch - 1, logs_json=True)

        Logger()('Ending evaluation procedures')


def train(self):
        """ Launch training procedures

            List of the hooks:

            - train_on_start: before the full training procedure

        """
        Logger()('Launching training procedures')

        self.hook('train_on_start')
        while self.epoch < Options()['engine']['nb_epochs']:
            self.train_epoch(self.model, self.dataset['train'], self.optimizer, self.epoch)

            if Options()['dataset']['eval_split']:
                out = self.eval_epoch(self.model, self.dataset['eval'], self.epoch)

                if 'saving_criteria' in Options()['engine'] and Options()['engine']['saving_criteria'] is not None:
                    for saving_criteria in Options()['engine']['saving_criteria']:
                        if self.is_best(out, saving_criteria):
                            name = saving_criteria.split(':')[0]
                            Logger()('Saving best checkpoint for strategy {}'.format(name))
                            self.save(Options()['exp']['dir'], 'best_{}'.format(name), self.model, self.optimizer)

            Logger()('Saving last checkpoint')
            self.save(Options()['exp']['dir'], 'last', self.model, self.optimizer)
            self.epoch += 1

        Logger()('Ending training procedures')


def train_epoch(self, model, dataset, optimizer, epoch, mode='train'):
        """ Launch training procedures for one epoch

            List of the hooks:

            - train_on_start_epoch: before the training procedure for an epoch
            - train_on_start_batch: before the training precedure for a batch
            - train_on_forward: after the forward of the model
            - train_on_backward: after the backward of the loss
            - train_on_update: after the optimization step
            - train_on_print: after the print to the terminal
            - train_on_end_batch: end of the training procedure for a batch
            - train_on_end_epoch: before saving the logs in logs.json
            - train_on_flush: end of the training procedure for an epoch
        """
        utils.set_random_seed(Options()['misc']['seed'] + epoch)  # to be able to reproduce exps on reload
        Logger()('Training model on {}set for epoch {}'.format(dataset.split, epoch))
        model.train()

        timer = {
            'begin': time.time(),
            'elapsed': time.time(),
            'process': None,
            'load': None,
            'run_avg': 0
        }
        out_epoch = {}
        batch_loader = dataset.make_batch_loader()

        self.hook(f'{mode}_on_start_epoch')
        for i, batch in enumerate(batch_loader):
            timer['load'] = time.time() - timer['elapsed']
            self.hook(f'{mode}_on_start_batch')

            optimizer.zero_grad()
            out = model(batch)
            self.hook(f'{mode}_on_forward')

            if not torch.isnan(out['loss']):
                out['loss'].backward()
            else:
                Logger()('NaN detected')
            # torch.cuda.synchronize()
            self.hook(f'{mode}_on_backward')

            optimizer.step()
            # torch.cuda.synchronize()
            self.hook(f'{mode}_on_update')

            timer['process'] = time.time() - timer['elapsed']
            if i == 0:
                timer['run_avg'] = timer['process']
            else:
                timer['run_avg'] = timer['run_avg'] * 0.8 + timer['process'] * 0.2

            Logger().log_value(f'{mode}_batch.epoch', epoch, should_print=False)
            Logger().log_value(f'{mode}_batch.batch', i, should_print=False)
            Logger().log_value(f'{mode}_batch.timer.process', timer['process'], should_print=False)
            Logger().log_value(f'{mode}_batch.timer.load', timer['load'], should_print=False)

            for key, value in out.items():
                if torch.is_tensor(value):
                    if value.numel() <= 1:
                        value = value.item()  # get number from a torch scalar
                    else:
                        continue
                if isinstance(value, (list, dict, tuple)):
                    continue
                if key not in out_epoch:
                    out_epoch[key] = []
                out_epoch[key].append(value)
                Logger().log_value(f'{mode}_batch.' + key, value, should_print=False)

            if i % Options()['engine']['print_freq'] == 0 or i == len(batch_loader) - 1:
                Logger()("{}: epoch {} | batch {}/{}".format(mode, epoch, i, len(batch_loader) - 1))
                Logger()("{} elapsed: {} | left: {}".format(
                    ' ' * len(mode),
                    datetime.timedelta(seconds=math.floor(time.time() - timer['begin'])),
                    datetime.timedelta(seconds=math.floor(timer['run_avg'] * (len(batch_loader) - 1 - i)))))
                Logger()("{} process: {:.5f} | load: {:.5f}".format(' ' * len(mode), timer['process'], timer['load']))
                Logger()("{} loss: {:.5f}".format(' ' * len(mode), out['loss'].data.item()))
                self.hook(f'{mode}_on_print')

            timer['elapsed'] = time.time()
            self.hook(f'{mode}_on_end_batch')

        Logger().log_value(f'{mode}_epoch.epoch', epoch, should_print=True)
        for key, value in out_epoch.items():
            Logger().log_value(f'{mode}_epoch.' + key, np.asarray(value).mean(), should_print=True)

        self.hook(f'{mode}_on_end_epoch')
        Logger().flush()
        self.hook(f'{mode}_on_flush')


def eval_epoch(self, model, dataset, epoch, mode='eval', logs_json=True):
        """ Launch evaluation procedures for one epoch

            List of the hooks (``mode='eval'`` by default):

            - mode_on_start_epoch: before the evaluation procedure for an epoch
            - mode_on_start_batch: before the evaluation precedure for a batch
            - mode_on_forward: after the forward of the model
            - mode_on_print: after the print to the terminal
            - mode_on_end_batch: end of the evaluation procedure for a batch
            - mode_on_end_epoch: before saving the logs in logs.json
            - mode_on_flush: end of the evaluation procedure for an epoch

            Returns:
                out(dict): mean of all the scalar outputs of the model, indexed by output name, for this epoch
        """
        utils.set_random_seed(Options()['misc']['seed'] + epoch)  # to be able to reproduce exps on reload
        Logger()('Evaluating model on {}set for epoch {}'.format(dataset.split, epoch))
        model.eval()

        timer = {
            'begin': time.time(),
            'elapsed': time.time(),
            'process': None,
            'load': None,
            'run_avg': 0
        }
        out_epoch = {}
        batch_loader = dataset.make_batch_loader()

        self.hook('{}_on_start_epoch'.format(mode))
        for i, batch in enumerate(batch_loader):
            timer['load'] = time.time() - timer['elapsed']
            self.hook('{}_on_start_batch'.format(mode))

            with torch.no_grad():
                out = model(batch)
            # torch.cuda.synchronize()
            self.hook('{}_on_forward'.format(mode))

            timer['process'] = time.time() - timer['elapsed']
            if i == 0:
                timer['run_avg'] = timer['process']
            else:
                timer['run_avg'] = timer['run_avg'] * 0.8 + timer['process'] * 0.2

            Logger().log_value('{}_batch.batch'.format(mode), i, should_print=False)
            Logger().log_value('{}_batch.epoch'.format(mode), epoch, should_print=False)
            Logger().log_value('{}_batch.timer.process'.format(mode), timer['process'], should_print=False)
            Logger().log_value('{}_batch.timer.load'.format(mode), timer['load'], should_print=False)

            for key, value in out.items():
                if torch.is_tensor(value):
                    if value.dim() <= 1:
                        value = value.item()  # get number from a torch scalar
                    else:
                        continue
                if isinstance(value, (list, dict, tuple)):
                    continue
                if key not in out_epoch:
                    out_epoch[key] = []
                out_epoch[key].append(value)
                Logger().log_value('{}_batch.{}'.format(mode, key), value, should_print=False)

            if i % Options()['engine']['print_freq'] == 0:
                Logger()("{}: epoch {} | batch {}/{}".format(mode, epoch, i, len(batch_loader) - 1))
                Logger()("{}  elapsed: {} | left: {}".format(
                    ' ' * len(mode),
                    datetime.timedelta(seconds=math.floor(time.time() - timer['begin'])),
                    datetime.timedelta(seconds=math.floor(timer['run_avg'] * (len(batch_loader) - 1 - i)))))
                Logger()("{}  process: {:.5f} | load: {:.5f}".format(' ' * len(mode), timer['process'], timer['load']))
                self.hook('{}_on_print'.format(mode))

            timer['elapsed'] = time.time()
            self.hook('{}_on_end_batch'.format(mode))

        out = {}
        for key, value in out_epoch.items():
            out[key] = sum(value) / len(value)

        Logger().log_value('{}_epoch.epoch'.format(mode), epoch, should_print=True)
        for key, value in out.items():
            Logger().log_value('{}_epoch.{}'.format(mode, key), value, should_print=True)

        self.hook('{}_on_end_epoch'.format(mode))
        if logs_json:
            Logger().flush()

        self.hook('{}_on_flush'.format(mode))
        return out


def is_best(self, out, saving_criteria):
        """ Verify if the last model is the best for a specific saving criteria

            Args:
                out(dict): mean of all the scalar outputs of model indexed by output name
                saving_criteria(str):

            Returns:
                is_best(bool)

            Example usage:

            .. code-block:: python

                out = {
                    'loss': 0.2,
                    'acctop1': 87.02
                }

                engine.is_best(out, 'loss:min')
        """
        if ':min' in saving_criteria:
            name = saving_criteria.replace(':min', '')
            order = '<'
        elif ':max' in saving_criteria:
            name = saving_criteria.replace(':max', '')
            order = '>'
        else:
            error_msg = """'--engine.saving_criteria' named '{}' does not specify order,
            you need to chose between '{}' or '{}' to specify if the criteria needs to be minimize or maximize""".format(
                saving_criteria, saving_criteria + ':min', saving_criteria + ':max')
            raise ValueError(error_msg)

        if name not in out:
            raise KeyError("'--engine.saving_criteria' named '{}' not in outputs '{}'".format(name, list(out.keys())))

        if name not in self.best_out:
            self.best_out[name] = out[name]
            return True
        else:
            if eval('{} {} {}'.format(out[name], order, self.best_out[name])):
                self.best_out[name] = out[name]
                return True

        return False


def load(self, dir_logs, name, model, optimizer, map_location=None):
        """ Load a checkpoint

            Args:
                dir_logs: directory of the checkpoint
                name: name of the checkpoint
                model: model associated to the checkpoint
                optimizer: optimizer associated to the checkpoint
        """
        path_template = os.path.join(dir_logs, 'ckpt_{}_{}.pth.tar')

        Logger()('Loading model...')
        model_state = torch.load(path_template.format(name, 'model'), map_location=map_location)
        model.load_state_dict(model_state)

        if Options()['dataset']['train_split'] is not None:
            if os.path.isfile(path_template.format(name, 'optimizer')):
                Logger()('Loading optimizer...')
                optimizer_state = torch.load(path_template.format(name, 'optimizer'), map_location=map_location)
                optimizer.load_state_dict(optimizer_state)
            else:
                Logger()('No optimizer checkpoint', log_level=Logger.WARNING)

        if os.path.isfile(path_template.format(name, 'engine')):
            Logger()('Loading engine...')
            engine_state = torch.load(path_template.format(name, 'engine'), map_location=map_location)
            self.load_state_dict(engine_state)
        else:
            Logger()('No engine checkpoint', log_level=Logger.WARNING)

def
load
(
self
,
dir_logs
,
name
,
model
,
optimizer
,
map_location
=
)
:
path_template
=
os
.
path
.
join
(
dir_logs
,
'ckpt_{}_{}.pth.tar'
)
Logger
(
)
(
'Loading model...'
)
model_state
=
torch
.
load
(
path_template
.
format
(
name
,
'model'
)
,
map_location
=
map_location
)
model
.
load_state_dict
(
model_state
)
if
Options
(
)
[
'dataset'
]
[
'train_split'
]
is
not
:
if
os
.
path
.
isfile
(
path_template
.
format
(
name
,
'optimizer'
)
)
:
Logger
(
)
(
'Loading optimizer...'
)
optimizer_state
=
torch
.
load
(
path_template
.
format
(
name
,
'optimizer'
)
,
map_location
=
map_location
)
optimizer
.
load_state_dict
(
optimizer_state
)
else
:
Logger
(
)
(
'No optimizer checkpoint'
,
log_level
=
Logger
.
WARNING
)
if
os
.
path
.
isfile
(
path_template
.
format
(
name
,
'engine'
)
)
:
Logger
(
)
(
'Loading engine...'
)
engine_state
=
torch
.
load
(
path_template
.
format
(
name
,
'engine'
)
,
map_location
=
map_location
)
self
.
load_state_dict
(
engine_state
)
else
:
Logger
(
)
(
'No engine checkpoint'
,
log_level
=
Logger
.
WARNING
)


def save(self, dir_logs, name, model, optimizer):
        """ Save a checkpoint

            Args:
                dir_logs: directory of the checkpoint
                name: name of the checkpoint
                model: model associated to the checkpoint
                optimizer: optimizer associated to the checkpoint
        """
        path_template = os.path.join(dir_logs, 'ckpt_{}_{}.pth.tar')

        Logger()('Saving model...')
        model_state = model.state_dict()
        torch.save(model_state, path_template.format(name, 'model'))

        Logger()('Saving optimizer...')
        optimizer_state = optimizer.state_dict()
        torch.save(optimizer_state, path_template.format(name, 'optimizer'))

        Logger()('Saving engine...')
        engine_state = self.state_dict()
        torch.save(engine_state, path_template.format(name, 'engine'))

def
save
(
self
,
dir_logs
,
name
,
model
,
optimizer
)
:
path_template
=
os
.
path
.
join
(
dir_logs
,
'ckpt_{}_{}.pth.tar'
)
Logger
(
)
(
'Saving model...'
)
model_state
=
model
.
state_dict
(
)
torch
.
save
(
model_state
,
path_template
.
format
(
name
,
'model'
)
)
Logger
(
)
(
'Saving optimizer...'
)
optimizer_state
=
optimizer
.
state_dict
(
)
torch
.
save
(
optimizer_state
,
path_template
.
format
(
name
,
'optimizer'
)
)
Logger
(
)
(
'Saving engine...'
)
engine_state
=
self
.
state_dict
(
)
torch
.
save
(
engine_state
,
path_template
.
format
(
name
,
'engine'
)
)