Repository: activeloopai/Hub
File: deeplake/integrations/mmdet/mmdet_runners.py (Python)

Code before:
        while self.iter < self._max_iters:
            for i, flow in enumerate(workflow):
                self._inner_iter = 0
                mode, iters = flow
                if not isinstance(mode, str) or not hasattr(self, mode):
                    raise ValueError(
                        'runner has no method named "{}" to run a workflow'.format(mode)
                    )
                iter_runner = getattr(self, mode)
                for _ in range(iters):
                    if mode == "train" and self.iter >= self._max_iters:
                        break

                    iter_time = time.time()

                    if iter_time - start_time > TIME_INTERVAL_FOR_CUDA_MEMORY_CLEANING:
                        torch.cuda.empty_cache()
                        start_time = iter_time
                    iter_runner(iter_loaders[i], **kwargs)

        time.sleep(1)  # wait for some hooks like loggers to finish
        self.call_hook("after_epoch")
        self.call_hook("after_run")


@runner.RUNNERS.register_module()
class DeeplakeEpochBasedRunner(runner.EpochBasedRunner):
    def train(self, data_loader, **kwargs):
        start_time = time.time()
        self.model.train()
        self.mode = "train"
        self.data_loader = data_loader
        self._max_iters = self._max_epochs * len(self.data_loader)
        self.call_hook("before_train_epoch")
        time.sleep(2)  # Prevent possible deadlock during epoch transition
        for i, data_batch in enumerate(self.data_loader):
            self.data_batch = data_batch
            self._inner_iter = i
            self.call_hook("before_train_iter")
            self.run_iter(data_batch, train_mode=True, **kwargs)
            self.call_hook("after_train_iter")
            del self.data_batch
            self._iter += 1
            iter_time = time.time()
            if iter_time - start_time > TIME_INTERVAL_FOR_CUDA_MEMORY_CLEANING:
                torch.cuda.empty_cache()
                start_time = iter_time

        self.call_hook("after_train_epoch")
        self._epoch += 1


Review comment (suggestion):
all of the code is taken from mmcv.runner.epoch_based_runner.py, the only difference is that I added time.time(). Essentially, the idea is that we want to clear gpu every 10 minutes, otherwise there would be a greater gpu memory fluctuations.

Code after:
                mode, iters = flow
                if not isinstance(mode, str) or not hasattr(self, mode):
                    raise ValueError(
                        'runner has no method named "{}" to run a workflow'.format(mode)
                    )
                iter_runner = getattr(self, mode)
                for _ in range(iters):
                    if mode == "train" and self.iter >= self._max_iters:
                        break

                    iter_time = time.time()

                    if iter_time - start_time > TIME_INTERVAL_FOR_CUDA_MEMORY_CLEANING:
                        empty_cuda()
                        start_time = iter_time
                    iter_runner(iter_loaders[i], **kwargs)

        time.sleep(1)  # wait for some hooks like loggers to finish
        self.call_hook("after_epoch")
        self.call_hook("after_run")


@runner.RUNNERS.register_module()
class DeeplakeEpochBasedRunner(runner.EpochBasedRunner):
    def train(self, data_loader, **kwargs):
        start_time = time.time()
        self.model.train()
        self.mode = "train"
        self.data_loader = data_loader
        self._max_iters = self._max_epochs * len(self.data_loader)
        self.call_hook("before_train_epoch")
        time.sleep(2)  # Prevent possible deadlock during epoch transition
        for i, data_batch in enumerate(self.data_loader):
            self.data_batch = data_batch
            self._inner_iter = i
            self.call_hook("before_train_iter")
            self.run_iter(data_batch, train_mode=True, **kwargs)
            self.call_hook("after_train_iter")
            del self.data_batch
            self._iter += 1
            iter_time = time.time()
            if iter_time - start_time > TIME_INTERVAL_FOR_CUDA_MEMORY_CLEANING:
                empty_cuda()
                start_time = iter_time

        self.call_hook("after_train_epoch")
        self._epoch += 1

    @torch.no_grad()
    def val(self, data_loader, **kwargs):
        start_time = time.time()