Elron commited on
Commit
c3f3d16
1 Parent(s): 6452fbf

Upload artifact.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. artifact.py +14 -116
artifact.py CHANGED
@@ -32,116 +32,6 @@ class Artifactories(object):
32
  self.artifactories = [artifactory] + self.artifactories
33
 
34
 
35
- # class BaseArtifact(ABC):
36
- # _class_register = {}
37
-
38
- # @classmethod
39
- # def is_artifact_dict(cls, d):
40
- # return isinstance(d, dict) and "type" in d and d["type"] in cls._class_register
41
-
42
- # @classmethod
43
- # def register_class(cls, artifact_class):
44
- # assert issubclass(artifact_class, BaseArtifact), "Artifact class must be a subclass of BaseArtifact"
45
- # assert is_camel_case(
46
- # artifact_class.__name__
47
- # ), f"Artifact class name must be legal camel case, got {artifact_class.__name__}"
48
-
49
- # snake_case_key = camel_to_snake_case(artifact_class.__name__)
50
-
51
- # if snake_case_key in cls._class_register:
52
- # assert (
53
- # cls._class_register[snake_case_key] == artifact_class
54
- # ), f"Artifact class name must be unique, {snake_case_key} already exists for {cls._class_register[snake_case_key]}"
55
-
56
- # cls._class_register[snake_case_key] = artifact_class
57
-
58
- # return snake_case_key
59
-
60
- # @classmethod
61
- # def is_artifact_file(cls, path):
62
- # if not os.path.exists(path) or not os.path.isfile(path):
63
- # return False
64
- # with open(path, "r") as f:
65
- # d = json.load(f)
66
- # return cls.is_artifact_dict(d)
67
-
68
- # @final
69
- # def __init__(self, *args, **kwargs):
70
- # super().__init__(*args, **kwargs)
71
-
72
- # @final
73
- # def __init_subclass__(cls, **kwargs):
74
- # super().__init_subclass__(**kwargs)
75
- # cls = dataclass(cls)
76
-
77
- # def prepare(self):
78
- # pass
79
-
80
- # def verify(self):
81
- # pass
82
-
83
- # @final
84
- # def __post_init__(self):
85
- # self.type = self.register_class(self.__class__)
86
-
87
- # self._args_dict = asdict(self)
88
-
89
- # for field in fields(self):
90
- # if getattr(self, field.name) == "cards.wnli":
91
- # print("cards.wnli")
92
- # if issubtype(field.type, Union[BaseArtifact, List[BaseArtifact], Dict[str, BaseArtifact]]):
93
- # value = getattr(self, field.name)
94
- # value = map_values_in_place(value, maybe_recover_artifact)
95
- # setattr(self, field.name, value)
96
-
97
- # self.prepare()
98
- # self.verify()
99
-
100
- # def to_dict(self):
101
- # return self._args_dict
102
-
103
- # def save(self, path):
104
- # with open(path, "w") as f:
105
- # json.dump(self.to_dict(), f, indent=4)
106
-
107
- # # def __getstate__(self):
108
- # # print('getstate', self.__dict__)
109
- # # return self.to_dict()
110
-
111
- # @classmethod
112
- # def _recursive_load(cls, d):
113
- # if isinstance(d, dict):
114
- # new_d = {}
115
- # for key, value in d.items():
116
- # new_d[key] = cls._recursive_load(value)
117
- # d = new_d
118
- # elif isinstance(d, list):
119
- # d = [cls._recursive_load(value) for value in d]
120
- # else:
121
- # pass
122
- # if cls.is_artifact_dict(d):
123
- # instance = cls._class_register[d.pop("type")](**d)
124
- # return instance
125
- # else:
126
- # return d
127
-
128
- # @classmethod
129
- # def from_dict(cls, d):
130
- # assert cls.is_artifact_dict(d), "Input must be a dict with type field"
131
- # return cls._recursive_load(d)
132
-
133
- # @classmethod
134
- # def load(cls, path):
135
- # with open(path, "r") as f:
136
- # d = json.load(f)
137
-
138
- # assert "type" in d, "Saved artifact must have a type field"
139
- # return cls._recursive_load(d)
140
- # # assert d['type'] in cls._class_register, f'Artifact type "{d["type"]}" is not registered'
141
- # # cls = cls._class_register[d.pop('type')]
142
- # # return cls(**d)
143
-
144
-
145
  def map_values_in_place(object, mapper):
146
  if isinstance(object, dict):
147
  for key, value in object.items():
@@ -163,6 +53,10 @@ class Artifact(Dataclass):
163
  def is_artifact_dict(cls, d):
164
  return isinstance(d, dict) and "type" in d and d["type"] in cls._class_register
165
 
 
 
 
 
166
  @classmethod
167
  def register_class(cls, artifact_class):
168
  assert issubclass(
@@ -179,6 +73,8 @@ class Artifact(Dataclass):
179
  cls._class_register[snake_case_key] == artifact_class
180
  ), f"Artifact class name must be unique, {snake_case_key} already exists for {cls._class_register[snake_case_key]}"
181
 
 
 
182
  cls._class_register[snake_case_key] = artifact_class
183
 
184
  return snake_case_key
@@ -221,7 +117,7 @@ class Artifact(Dataclass):
221
  assert "type" in d, "Saved artifact must have a type field"
222
  return cls._recursive_load(d)
223
  except Exception as e:
224
- raise IOError(f"Failed loading {cls.type} from {path}")
225
 
226
  def prepare(self):
227
  pass
@@ -229,12 +125,14 @@ class Artifact(Dataclass):
229
  def verify(self):
230
  pass
231
 
 
 
 
 
232
  @final
233
  def __post_init__(self):
234
  self.type = self.register_class(self.__class__)
235
 
236
- self._init_dict = asdict(self)
237
-
238
  for field in fields(self):
239
  if issubtype(field.type, Union[Artifact, List[Artifact], Dict[str, Artifact]]):
240
  value = getattr(self, field.name)
@@ -244,12 +142,12 @@ class Artifact(Dataclass):
244
  self.prepare()
245
  self.verify()
246
 
247
- def get_init_dict(self):
248
- return self._init_dict
249
 
250
  def save(self, path):
251
  with open(path, "w") as f:
252
- init_dict = self.get_init_dict()
253
  json.dump(init_dict, f, indent=4)
254
 
255
 
 
32
  self.artifactories = [artifactory] + self.artifactories
33
 
34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  def map_values_in_place(object, mapper):
36
  if isinstance(object, dict):
37
  for key, value in object.items():
 
53
  def is_artifact_dict(cls, d):
54
  return isinstance(d, dict) and "type" in d and d["type"] in cls._class_register
55
 
56
+ @classmethod
57
+ def get_artifact_type(cls):
58
+ return camel_to_snake_case(cls.__name__)
59
+
60
  @classmethod
61
  def register_class(cls, artifact_class):
62
  assert issubclass(
 
73
  cls._class_register[snake_case_key] == artifact_class
74
  ), f"Artifact class name must be unique, {snake_case_key} already exists for {cls._class_register[snake_case_key]}"
75
 
76
+ return snake_case_key
77
+
78
  cls._class_register[snake_case_key] = artifact_class
79
 
80
  return snake_case_key
 
117
  assert "type" in d, "Saved artifact must have a type field"
118
  return cls._recursive_load(d)
119
  except Exception as e:
120
+ raise Exception(f"{e}\n\nFailed to load artifact from {path} see above for more details.")
121
 
122
  def prepare(self):
123
  pass
 
125
  def verify(self):
126
  pass
127
 
128
+ @final
129
+ def __pre_init__(self, **kwargs):
130
+ self._init_dict = kwargs
131
+
132
  @final
133
  def __post_init__(self):
134
  self.type = self.register_class(self.__class__)
135
 
 
 
136
  for field in fields(self):
137
  if issubtype(field.type, Union[Artifact, List[Artifact], Dict[str, Artifact]]):
138
  value = getattr(self, field.name)
 
142
  self.prepare()
143
  self.verify()
144
 
145
+ def _to_raw_dict(self):
146
+ return {"type": self.type, **self._init_dict}
147
 
148
  def save(self, path):
149
  with open(path, "w") as f:
150
+ init_dict = self.to_dict()
151
  json.dump(init_dict, f, indent=4)
152
 
153