Elron commited on
Commit
d5c8a0f
1 Parent(s): 1e56308

Upload register.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. register.py +31 -17
register.py CHANGED
@@ -1,23 +1,15 @@
1
  import importlib
2
  import inspect
3
  import os
 
4
 
5
  from .artifact import Artifact, Artifactories
6
- from .catalog import PATHS_SEP, EnvironmentLocalCatalog, GithubCatalog, LocalCatalog
 
7
  from .utils import Singleton
8
 
9
- UNITXT_ARTIFACTORIES_ENV_VAR = "UNITXT_ARTIFACTORIES"
10
-
11
- # Usage
12
- non_registered_files = [
13
- "__init__.py",
14
- "artifact.py",
15
- "utils.py",
16
- "register.py",
17
- "metric.py",
18
- "dataset.py",
19
- "blocks.py",
20
- ]
21
 
22
 
23
  def _register_catalog(catalog: LocalCatalog):
@@ -28,12 +20,32 @@ def _unregister_catalog(catalog: LocalCatalog):
28
  Artifactories().unregister(catalog)
29
 
30
 
 
 
 
 
 
 
 
 
 
 
31
  def register_local_catalog(catalog_path: str):
32
  assert os.path.exists(catalog_path), f"Catalog path {catalog_path} does not exist."
33
  assert os.path.isdir(
34
  catalog_path
35
  ), f"Catalog path {catalog_path} is not a directory."
36
- _register_catalog(LocalCatalog(location=catalog_path))
 
 
 
 
 
 
 
 
 
 
37
 
38
 
39
  def _catalogs_list():
@@ -50,8 +62,10 @@ def _reset_env_local_catalogs():
50
  for catalog in _catalogs_list():
51
  if isinstance(catalog, EnvironmentLocalCatalog):
52
  _unregister_catalog(catalog)
53
- if UNITXT_ARTIFACTORIES_ENV_VAR in os.environ:
54
- for path in os.environ[UNITXT_ARTIFACTORIES_ENV_VAR].split(PATHS_SEP):
 
 
55
  _register_catalog(EnvironmentLocalCatalog(location=path))
56
 
57
 
@@ -62,7 +76,7 @@ def _register_all_artifacts():
62
  for file in os.listdir(dir):
63
  if (
64
  file.endswith(".py")
65
- and file not in non_registered_files
66
  and file != file_name
67
  ):
68
  module_name = file.replace(".py", "")
 
1
  import importlib
2
  import inspect
3
  import os
4
+ from pathlib import Path
5
 
6
  from .artifact import Artifact, Artifactories
7
+ from .catalog import EnvironmentLocalCatalog, GithubCatalog, LocalCatalog
8
+ from .settings_utils import get_constants, get_settings
9
  from .utils import Singleton
10
 
11
+ constants = get_constants()
12
+ settings = get_settings()
 
 
 
 
 
 
 
 
 
 
13
 
14
 
15
  def _register_catalog(catalog: LocalCatalog):
 
20
  Artifactories().unregister(catalog)
21
 
22
 
23
+ def is_local_catalog_registered(catalog_path: str):
24
+ if os.path.isdir(catalog_path):
25
+ for catalog in _catalogs_list():
26
+ if isinstance(catalog, LocalCatalog):
27
+ if os.path.isdir(catalog.location):
28
+ if Path(catalog.location).resolve() == Path(catalog_path).resolve():
29
+ return True
30
+ return False
31
+
32
+
33
  def register_local_catalog(catalog_path: str):
34
  assert os.path.exists(catalog_path), f"Catalog path {catalog_path} does not exist."
35
  assert os.path.isdir(
36
  catalog_path
37
  ), f"Catalog path {catalog_path} is not a directory."
38
+ if not is_local_catalog_registered(catalog_path=catalog_path):
39
+ _register_catalog(LocalCatalog(location=catalog_path))
40
+
41
+
42
+ def unregister_local_catalog(catalog_path: str):
43
+ if is_local_catalog_registered(catalog_path=catalog_path):
44
+ for catalog in _catalogs_list():
45
+ if isinstance(catalog, LocalCatalog):
46
+ if os.path.isdir(catalog.location):
47
+ if Path(catalog.location).resolve() == Path(catalog_path).resolve():
48
+ _unregister_catalog(catalog)
49
 
50
 
51
  def _catalogs_list():
 
62
  for catalog in _catalogs_list():
63
  if isinstance(catalog, EnvironmentLocalCatalog):
64
  _unregister_catalog(catalog)
65
+ if settings.artifactories:
66
+ for path in settings.artifactories.split(
67
+ constants.env_local_catalogs_paths_sep
68
+ ):
69
  _register_catalog(EnvironmentLocalCatalog(location=path))
70
 
71
 
 
76
  for file in os.listdir(dir):
77
  if (
78
  file.endswith(".py")
79
+ and file not in constants.non_registered_files
80
  and file != file_name
81
  ):
82
  module_name = file.replace(".py", "")