Repository: activeloopai/Hub
File: deeplake/core/transform/transform.py (Python)

Code before:
        if not check_lengths:
            skip_ok = True

        if checkpoint_interval > 0:
            if num_workers > 0 and checkpoint_interval % num_workers != 0:
                raise ValueError(
                    "checkpoint_interval should be a multiple of num_workers if num_workers > 0"
                )
            if overwrite:
                raise ValueError(
                    "checkpoint_interval > 0 and ds_out is None. Cannot checkpoint during inplace transform."
                )
            datas_in = [
                data_in[i : i + checkpoint_interval]
                for i in range(0, len(data_in), checkpoint_interval)
            ]

        else:
            datas_in = [data_in]

        samples_processed = 0
        desc = get_pbar_description(self.functions)
        pbar = get_progressbar(len(data_in), desc)
        pqueue = compute_provider.create_queue()
        for i, data_in in enumerate(datas_in):
            progress = int((i + 1) / len(datas_in) * 100)
            end = progress == 100
            progress_args = {"compute_id": compute_id, "progress": progress, "end": end}

            try:
                self.run(
                    data_in,
                    target_ds,
                    compute_provider,
                    num_workers,
                    scheduler,
                    progressbar,
                    overwrite,
                    skip_ok,
                    read_only_ok and overwrite,
                    pbar,
                    pqueue,
                    **kwargs,
                )
                target_ds._send_compute_progress(**progress_args, status="success")
                target_ds.commit(
                    f"Auto-commit during {desc} compute after {progress}% progress"
                )
                samples_processed += len(data_in)
            except Exception as e:
                if checkpoint_interval > 0:

Review comment (bug):
@AbhinavTuli FYI this is mathematically slightly incorrect because each element in the `datas_in`  list does not have the same number of samples. Not sure if it's work fixing, but if possible+simple, we should print the sample_level progress. 

Code after:
        total_samples = len(data_in)
        if checkpointing_enabled:
            check_checkpoint_interval(
                data_in, checkpoint_interval, num_workers, overwrite
            )
            datas_in = [
                data_in[i : i + checkpoint_interval]
                for i in range(0, len(data_in), checkpoint_interval)
            ]

        else:
            datas_in = [data_in]

        samples_processed = 0
        desc = get_pbar_description(self.functions)
        if progressbar:
            pbar = get_progress_bar(len(data_in), desc)
            pqueue = compute_provider.create_queue()
        else:
            pbar, pqueue = None, None
        desc = desc.split()[1]
        completed = False
        progress = 0.0
        for data_in in datas_in:
            if checkpointing_enabled and progress > 0:
                target_ds.commit(
                    f"Auto-commit during deeplake.compute of {desc} after {progress}% progress"
                )
            progress = round(
                (samples_processed + len(data_in)) / total_samples * 100, 2
            )
            end = progress == 100
            progress_args = {"compute_id": compute_id, "progress": progress, "end": end}

            try:
                self.run(
                    data_in,
                    target_ds,
                    compute_provider,
                    num_workers,
                    scheduler,
                    progressbar,
                    overwrite,
                    skip_ok,
                    read_only_ok and overwrite,
                    pbar,
                    pqueue,
                    ignore_errors,
                    **kwargs,
                )
                target_ds._send_compute_progress(**progress_args, status="success")