Elron commited on
Commit
f4655a2
1 Parent(s): 3ffaf6a

Upload schema.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. schema.py +6 -1
schema.py CHANGED
@@ -34,9 +34,14 @@ class ToUnitxtGroup(StreamInstanceOperatorValidator):
34
 
35
  def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
36
  if self.remove_unnecessary_fields:
 
 
37
  for key in instance.keys():
38
  if key not in UNITXT_DATASET_SCHEMA:
39
- del instance[key]
 
 
 
40
 
41
  instance["group"] = self.group
42
  if self.metrics is not None:
 
34
 
35
  def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
36
  if self.remove_unnecessary_fields:
37
+ keys_to_delete = []
38
+
39
  for key in instance.keys():
40
  if key not in UNITXT_DATASET_SCHEMA:
41
+ keys_to_delete.append(key)
42
+
43
+ for key in keys_to_delete:
44
+ del instance[key]
45
 
46
  instance["group"] = self.group
47
  if self.metrics is not None: