File size: 1,411 Bytes
17dfad1 5c7ac99 17dfad1 5c7ac99 17dfad1 5c7ac99 17dfad1 de6938d 17dfad1 5c7ac99 17dfad1 5c7ac99 de6938d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
from pathlib import Path
from datasets import disable_caching, enable_caching, is_caching_enabled
from datasets.utils.py_utils import get_imports
from .file_utils import get_all_files_in_dir
HF_CACHING_ENABLED = False
class HFCachingContextManager:
def __init__(self, should_cache):
self.should_cache = should_cache
self.was_caching_enabled = None
def __enter__(self):
self.was_caching_enabled = is_caching_enabled()
if self.should_cache:
if not self.was_caching_enabled:
enable_caching()
else:
if self.was_caching_enabled:
disable_caching()
def __exit__(self, exc_type, exc_val, exc_tb):
if self.was_caching_enabled != is_caching_enabled():
if self.was_caching_enabled:
enable_caching()
else:
disable_caching()
def get_missing_imports(file, exclude=None):
if exclude is None:
exclude = []
src_dir = Path(__file__).parent
python_files = get_all_files_in_dir(src_dir, file_extension=".py")
# get only the file without the path and extension
required_modules = [Path(p).stem for p in python_files]
imports = get_imports(file)
imported_modules = [i[1] for i in imports if i[0] == "internal"]
return [
i for i in required_modules if i not in imported_modules and i not in exclude
]
|