utils/commons/os_utils.py

import os
import subprocess
import glob
from utils.commons.multiprocess_utils import multiprocess_run_tqdm


def link_file(from_file, to_file):
    subprocess.check_call(
        f'ln -s "`realpath --relative-to="{os.path.dirname(to_file)}" "{from_file}"`" "{to_file}"', shell=True)


def move_file(from_file, to_file):
    subprocess.check_call(f'mv "{from_file}" "{to_file}"', shell=True)


def copy_file(from_file, to_file):
    subprocess.check_call(f'cp -r "{from_file}" "{to_file}"', shell=True)


def remove_file(*fns):
    for f in fns:
        subprocess.check_call(f'rm -rf "{f}"', shell=True)

def glob_job(d, f):
    pattern = os.path.join(d, f)
    return glob.glob(pattern)

def multiprocess_glob(pattern, num_workers=None):
    split_pattern = pattern.split("/")
    recursive_depth = 0 # number of recursive depth
    for split in split_pattern:
        if '*' in split:
            recursive_depth += 1
    if recursive_depth == 1:
        return glob.glob(pattern)
    else:
        dirs = glob.glob('/'.join(split_pattern[:-1]))
        ret = []
        args = [(d, split_pattern[-1]) for d in dirs]
        for (i,res) in multiprocess_run_tqdm(glob_job, args=args, desc=f"globing {pattern}", num_workers=num_workers):
            ret += res
        return ret