Upload folder using huggingface_hub

artifact.py

@@ -4,6 +4,7 @@ import json
 import os
 import pkgutil
 import re
+import warnings
 from abc import abstractmethod
 from typing import Any, Dict, List, Optional, Tuple, Union, final
 
@@ -138,6 +139,12 @@ class Artifact(Dataclass):
     )
     __id__: str = InternalField(default=None, required=False, also_positional=False)
 
+    # if not None, the artifact is deprecated, and once instantiated, that msg
+    # is logged as a warning
+    __deprecated_msg__: str = NonPositionalField(
+        default=None, required=False, also_positional=False
+    )
+
     data_classification_policy: List[str] = NonPositionalField(
         default=None, required=False, also_positional=False
     )
@@ -237,6 +244,11 @@ class Artifact(Dataclass):
     @classmethod
     def load(cls, path, artifact_identifier=None, overwrite_args=None):
         d = artifacts_json_cache(path)
+        if "artifact_linked_to" in d and d["artifact_linked_to"] is not None:
+            # d stands for an ArtifactLink
+            artifact_link = ArtifactLink.from_dict(d)
+            return artifact_link.load(overwrite_args)
+
         new_artifact = cls.from_dict(d, overwrite_args=overwrite_args)
         new_artifact.__id__ = artifact_identifier
         return new_artifact
@@ -247,7 +259,8 @@ class Artifact(Dataclass):
         return self.__class__.__name__
 
     def prepare(self):
-        pass
+        if self.__deprecated_msg__:
+            warnings.warn(self.__deprecated_msg__, DeprecationWarning, stacklevel=2)
 
     def verify(self):
         pass
@@ -396,6 +409,57 @@ class Artifact(Dataclass):
         return instance
 
 
+class ArtifactLink(Artifact):
+    # the artifact linked to, expressed by its catalog id
+    artifact_linked_to: str = Field(default=None, required=True)
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        assert isinstance(d, dict), f"argument must be a dictionary, got: d = {d}."
+        assert (
+            "artifact_linked_to" in d and d["artifact_linked_to"] is not None
+        ), f"A non-none field named 'artifact_linked_to' is expected in input argument d, but got: {d}."
+        artifact_linked_to = d["artifact_linked_to"]
+        # artifact_linked_to is a name of catalog entry
+        assert isinstance(
+            artifact_linked_to, str
+        ), f"'artifact_linked_to' should be a string expressing a name of a catalog entry. Got{artifact_linked_to}."
+        msg = d["__deprecated_msg__"] if "__deprecated_msg__" in d else None
+        return ArtifactLink(
+            artifact_linked_to=artifact_linked_to, __deprecated_msg__=msg
+        )
+
+    def load(self, overwrite_args: dict) -> Artifact:
+        # identify the catalog for the artifact_linked_to
+        assert (
+            self.artifact_linked_to is not None
+        ), "'artifact_linked_to' must be non-None in order to load it from the catalog. Currently, it is None."
+        assert isinstance(
+            self.artifact_linked_to, str
+        ), f"'artifact_linked_to' should be a string (expressing a name of a catalog entry). Currently, its type is: {type(self.artifact_linked_to)}."
+        needed_catalog = None
+        catalogs = list(Catalogs())
+        for catalog in catalogs:
+            if self.artifact_linked_to in catalog:
+                needed_catalog = catalog
+
+        if needed_catalog is None:
+            raise UnitxtArtifactNotFoundError(self.artifact_linked_to, catalogs)
+
+        path = needed_catalog.path(self.artifact_linked_to)
+        d = artifacts_json_cache(path)
+        # if needed, follow, in a recursive manner, over multiple links,
+        # passing through instantiating of the ArtifactLink-s on the way, triggering
+        # deprecatioin warning as needed.
+        if "artifact_linked_to" in d and d["artifact_linked_to"] is not None:
+            # d stands for an ArtifactLink
+            artifact_link = ArtifactLink.from_dict(d)
+            return artifact_link.load(overwrite_args)
+        new_artifact = Artifact.from_dict(d, overwrite_args=overwrite_args)
+        new_artifact.__id__ = self.artifact_linked_to
+        return new_artifact
+
+
 def get_raw(obj):
     if isinstance(obj, Artifact):
         return obj._to_raw_dict()
@@ -456,20 +520,27 @@ def fetch_artifact(artifact_rep) -> Tuple[Artifact, Union[AbstractCatalog, None]
     (5) Otherwise, check that the artifact representation is a dictionary and build an Artifact object from it.
     """
     if isinstance(artifact_rep, Artifact):
+        if isinstance(artifact_rep, ArtifactLink):
+            return fetch_artifact(artifact_rep.artifact_linked_to)
         return artifact_rep, None
 
     # If local file
     if isinstance(artifact_rep, str) and Artifact.is_artifact_file(artifact_rep):
-        return Artifact.load(artifact_rep), None
+        artifact_to_return = Artifact.load(artifact_rep)
+        if isinstance(artifact_rep, ArtifactLink):
+            artifact_to_return = fetch_artifact(artifact_to_return.artifact_linked_to)
 
-    # If artifact name in catalog
+        return artifact_to_return, None
+
+    # if artifact is a name of a catalog entry
     if isinstance(artifact_rep, str):
         name, _ = separate_inside_and_outside_square_brackets(artifact_rep)
         if is_name_legal_for_catalog(name):
             catalog, artifact_rep, args = get_catalog_name_and_args(name=artifact_rep)
-            return catalog.get_with_overwrite(
+            artifact_to_return = catalog.get_with_overwrite(
                 artifact_rep, overwrite_args=args
-            ), catalog
+            )
+            return artifact_to_return, catalog
 
     # If Json string, first load into dictionary
     if isinstance(artifact_rep, str):

augmentors.py

@@ -80,21 +80,26 @@ class AugmentWhitespace(TextAugmentor):
 
 
 class AugmentPrefixSuffix(TextAugmentor):
-    r"""Augments the input by prepending and appending randomly selected (typically, whitespace) patterns.
+    r"""Augments the input by prepending and appending randomly selected patterns (typically, whitespace).
 
     Args:
-     prefixes, suffixes (list or dict) : the potential patterns (typically, whitespace) to select from. The dictionary version allows the specification relative weights for the different patterns.
-     prefix_len, suffix_len (positive int) : The added prefix or suffix will be of a certain length.
-     remove_existing_whitespaces : Clean any existing leading and trailing whitespaces. The strings made of repetitions of the selected pattern(s) are then prepended and/or appended to the potentially trimmed input.
-     If only either just prefixes or just suffixes are needed, set the other to None.
+       prefixes (list or dict or None): the potential patterns (typically, whitespace) to select prefix from. The dictionary version allows the specification of relative weights for the different patterns. Set to None if not needed (i.e., only suffixes are needed).
+
+       suffixes (list or dict or None): the potential patterns (typically, whitespace) to select suffix from. The dictionary version allows the specification of relative weights for the different patterns. Set to None if not needed (i.e., only prefixes are needed).
+
+       prefix_len (positive int): the length of the prefix to be added.
+
+       suffix_len (positive int): The length of the suffix to be added.
+
+       remove_existing_whitespaces (bool): Clean any existing leading and trailing whitespaces. The selected pattern(s) are then prepended and/or appended to the potentially trimmed input.
 
     Examples:
-        To prepend the input with a prefix made of 4 '\n'-s or '\t'-s, employ
-        AugmentPrefixSuffix(augment_model_input=True, prefixes=['\n','\t'], prefix_len=4, suffixes = None)
-        To append the input with a suffix made of 3 '\n'-s or '\t'-s, with triple '\n' suffixes
-        being preferred over triple '\t', at 2:1 ratio, employ
-        AugmentPrefixSuffix(augment_model_input=True, suffixes={'\n':2,'\t':1}, suffix_len=3, prefixes = None)
-        which will append '\n'-s twice as often as '\t'-s.
+        To prepend the input with a prefix made of 4 ``\n``-s or ``\t``-s, employ
+        ``AugmentPrefixSuffix(augment_model_input=True, prefixes=['\n','\t'], prefix_len=4, suffixes = None)``.
+
+        To append the input with a suffix made of 3 ``\n``-s or ``\t``-s, with ``\n`` being preferred over ``\t``, at 2:1 ratio, employ
+        ``AugmentPrefixSuffix(augment_model_input=True, suffixes={'\n':2,'\t':1}, suffix_len=3, prefixes = None)``
+        which will append ``\n``-s twice as often as ``\t``-s.
 
     """
 

catalog.py

@@ -10,6 +10,7 @@ import requests
 from .artifact import (
     AbstractCatalog,
     Artifact,
+    ArtifactLink,
     Catalogs,
     get_catalog_name_and_args,
     reset_artifacts_json_cache,
@@ -51,7 +52,9 @@ class LocalCatalog(Catalog):
         ), f"Artifact with name {artifact_identifier} does not exist"
         path = self.path(artifact_identifier)
         return Artifact.load(
-            path, artifact_identifier=artifact_identifier, overwrite_args=overwrite_args
+            path,
+            artifact_identifier=artifact_identifier,
+            overwrite_args=overwrite_args,
         )
 
     def __getitem__(self, name) -> Artifact:
@@ -132,10 +135,36 @@ def add_to_catalog(
             catalog_path = constants.default_catalog_path
         catalog = LocalCatalog(location=catalog_path)
     verify_legal_catalog_name(name)
-    catalog.save_artifact(
-        artifact, name, overwrite=overwrite, verbose=verbose
-    )  # remove collection (its actually the dir).
-    # verify name
+    catalog.save_artifact(artifact, name, overwrite=overwrite, verbose=verbose)
+
+
+def add_link_to_catalog(
+    artifact_linked_to: str,
+    name: str,
+    deprecate: bool = False,
+    catalog: Catalog = None,
+    overwrite: bool = False,
+    catalog_path: Optional[str] = None,
+    verbose=True,
+):
+    if deprecate:
+        deprecated_msg = f"Artifact '{name}' is deprecated. Artifact '{artifact_linked_to}' will be instantiated instead. "
+        deprecated_msg += f"In future uses, please reference artifact '{artifact_linked_to}' directly."
+    else:
+        deprecated_msg = None
+
+    artifact_link = ArtifactLink(
+        artifact_linked_to=artifact_linked_to, __deprecated_msg__=deprecated_msg
+    )
+
+    add_to_catalog(
+        artifact=artifact_link,
+        name=name,
+        catalog=catalog,
+        overwrite=overwrite,
+        catalog_path=catalog_path,
+        verbose=verbose,
+    )
 
 
 @lru_cache(maxsize=None)

llm_as_judge.py

@@ -297,9 +297,15 @@ class LLMAsJudge(LLMAsJudgeBase):
 
     def prepare_instances(self, references, predictions, task_data):
         input_instances = self._get_input_instances(task_data)
-        return self._get_instance_for_judge_model(
+        instances = self._get_instance_for_judge_model(
             input_instances, predictions, references
         )
+        # Copy the data classification policy from the original instance
+        for instance, single_task_data in zip(instances, task_data):
+            instance["data_classification_policy"] = single_task_data.get(
+                "metadata", {}
+            ).get("data_classification_policy")
+        return instances
 
 
 class TaskBasedLLMasJudge(LLMAsJudgeBase):

operator.py

@@ -427,7 +427,7 @@ class InstanceOperator(StreamOperator):
                 raise e
             else:
                 raise ValueError(
-                    f"Error processing instance '{_index}' from stream '{stream_name}' in {self.__class__.__name__} due to: {e}"
+                    f"Error processing instance '{_index}' from stream '{stream_name}' in {self.__class__.__name__} due to the exception above."
                 ) from e
 
     def _process_instance(

operators.py

@@ -190,7 +190,7 @@ class MapInstanceValues(InstanceOperator):
             if value is not None:
                 if (self.process_every_value is True) and (not isinstance(value, list)):
                     raise ValueError(
-                        f"'process_every_field' == True is allowed only when all fields which have mappers, i.e., {list(self.mappers.keys())} are lists. Instance = {instance}"
+                        f"'process_every_field' == True is allowed only for fields whose values are lists, but value of field '{key}' is '{value}'"
                     )
                 if isinstance(value, list) and self.process_every_value:
                     for i, val in enumerate(value):
@@ -211,7 +211,7 @@ class MapInstanceValues(InstanceOperator):
             return recursive_copy(mapper[val_as_str])
         if self.strict:
             raise KeyError(
-                f"value '{val}' in instance '{instance}' is not found in mapper '{mapper}', associated with field '{key}'."
+                f"value '{val_as_str}', the string representation of the value in field '{key}', is not found in mapper '{mapper}'"
             )
         return val
 
@@ -454,7 +454,7 @@ class InstanceFieldOperator(InstanceOperator):
                     old_value = self.get_default
             except Exception as e:
                 raise ValueError(
-                    f"Failed to get '{from_field}' from {instance} due to : {e}"
+                    f"Failed to get '{from_field}' from instance due to the exception above."
                 ) from e
             try:
                 if self.process_every_value:
@@ -466,7 +466,7 @@ class InstanceFieldOperator(InstanceOperator):
                     new_value = self.process_instance_value(old_value, instance)
             except Exception as e:
                 raise ValueError(
-                    f"Failed to process '{from_field}' from {instance} due to : {e}"
+                    f"Failed to process field '{from_field}' from instance due to the exception above."
                 ) from e
             dict_set(
                 instance,
@@ -977,7 +977,7 @@ class CastFields(InstanceOperator):
             if self.process_every_value:
                 assert isinstance(
                     value, list
-                ), f"'process_every_value' can be set to True only for fields that contain lists, whereas in instance {instance}, the contents of field '{field_name}' is of type '{type(value)}'"
+                ), f"'process_every_field' == True is allowed only for fields whose values are lists, but value of field '{field_name}' is '{value}'"
                 casted_value = self._cast_multiple(value, type, field_name)
             else:
                 casted_value = self._cast_single(value, type, field_name)
@@ -1154,7 +1154,7 @@ class FilterByCondition(StreamOperator):
                 instance_key = dict_get(instance, key)
             except ValueError as ve:
                 raise ValueError(
-                    f"Required filter field ('{key}') in FilterByCondition is not found in {instance}"
+                    f"Required filter field ('{key}') in FilterByCondition is not found in instance."
                 ) from ve
             if self.condition == "in":
                 if instance_key not in value:
@@ -1194,13 +1194,13 @@ class FilterByConditionBasedOnFields(FilterByCondition):
                 instance_key = dict_get(instance, key)
             except ValueError as ve:
                 raise ValueError(
-                    f"Required filter field ('{key}') in FilterByCondition is not found in {instance}"
+                    f"Required filter field ('{key}') in FilterByCondition is not found in instance"
                 ) from ve
             try:
                 instance_value = dict_get(instance, value)
             except ValueError as ve:
                 raise ValueError(
-                    f"Required filter field ('{value}') in FilterByCondition is not found in {instance}"
+                    f"Required filter field ('{value}') in FilterByCondition is not found in instance"
                 ) from ve
             if self.condition == "in":
                 if instance_key not in instance_value:
@@ -1551,7 +1551,7 @@ class SplitByNestedGroup(MultiStreamOperator):
             for instance in stream:
                 if self.field_name_of_group not in instance:
                     raise ValueError(
-                        f"Field {self.field_name_of_group} is missing from instance {instance}"
+                        f"Field {self.field_name_of_group} is missing from instance. Available fields: {instance.keys()}"
                     )
                 signature = (
                     stream_name

version.py

@@ -1 +1 @@
-version = "1.15.8"
+version = "1.15.9"