Upload folder using huggingface_hub

README.md

@@ -40,11 +40,11 @@ https://github.com/IBM/unitxt/assets/23455264/baef9131-39d4-4164-90b2-05da52919f
 
 ### 🦄 Currently on Unitxt Catalog
 
-![NLP Tasks](https://img.shields.io/badge/NLP_tasks-40-blue)
-![Dataset Cards](https://img.shields.io/badge/Dataset_Cards-457-blue)
-![Templates](https://img.shields.io/badge/Templates-229-blue)
-![Formats](https://img.shields.io/badge/Formats-18-blue)
-![Metrics](https://img.shields.io/badge/Metrics-98-blue)
+![NLP Tasks](https://img.shields.io/badge/NLP_tasks-48-blue)
+![Dataset Cards](https://img.shields.io/badge/Dataset_Cards-537-blue)
+![Templates](https://img.shields.io/badge/Templates-265-blue)
+![Formats](https://img.shields.io/badge/Formats-23-blue)
+![Metrics](https://img.shields.io/badge/Metrics-136-blue)
 
 ### 🦄 Run Unitxt Exploration Dashboard
 

augmentors.py

@@ -0,0 +1,195 @@
+from random import Random
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
+
+from .operators import FieldOperator
+from .random_utils import new_random_generator
+from .type_utils import isoftype
+
+
+class Augmentor(FieldOperator):
+    """A stream operator that augments the values of either the task input fields before rendering with the template,  or the input passed to the model after rendering of the template."""
+
+    operator: FieldOperator
+
+    def process_value(self, value: Any) -> Any:
+        return self.operator.process_value(value)
+
+
+class TaskInputsAugmentor(Augmentor):
+    def set_fields(self, fields: List[str]):
+        fields = ["input_fields/" + field for field in fields]
+        self.field_to_field = {field: field for field in fields}
+
+
+class FinalStateInputsAugmentor(Augmentor):
+    pass
+
+
+class ModelInputAugmentor(FinalStateInputsAugmentor):
+    field = "source"
+
+
+class ImagesAugmentor(FinalStateInputsAugmentor):
+    field = "media/images"
+    process_every_value = True
+
+
+class Identity(FieldOperator):
+    def process_value(self, value: Any) -> Any:
+        return value
+
+
+class NullAugmentor(Augmentor):
+    """Does not change the input string."""
+
+    operator = Identity()
+
+
+class AugmentWhitespace(FieldOperator):
+    """Augments the inputs by replacing existing whitespaces with other whitespaces.
+
+    Currently, each whitespace is replaced by a random choice of 1-3 whitespace characters (space, tab, newline).
+    """
+
+    def process_value(self, value: str) -> str:
+        import re
+
+        words = re.split(r"(\s+)", value)
+        new_value = ""
+
+        random_generator = new_random_generator(sub_seed=value)
+        for word in words:
+            if word.isspace():
+                new_value += random_generator.choice(
+                    ["\n", "\t", " "]
+                ) * random_generator.randint(1, 3)
+            else:
+                new_value += word
+        return new_value
+
+
+class AugmentPrefixSuffix(FieldOperator):
+    r"""Augments the input by prepending and appending randomly selected (typically, whitespace) patterns.
+
+    Args:
+     prefixes, suffixes (list or dict) : the potential (typically, whitespace) patterns to select from.
+        The dictionary version allows the specification relative weights for the different patterns.
+     prefix_len, suffix_len (positive int) : The added prefix or suffix will be of a certain length.
+     remove_existing_whitespaces : Clean any existing leading and trailing whitespaces.
+        The strings made of repetitions of the selected pattern(s) are then prepended and/or appended to the potentially
+        trimmed input.
+     If only either just prefixes or just suffixes are needed, set the other to None.
+
+    Examples:
+        To prepend the input with a prefix made of 4 '\n'-s or '\t'-s, employ
+        AugmentPrefixSuffix(augment_model_input=True, prefixes=['\n','\t'], prefix_len=4, suffixes = None)
+        To append the input with a suffix made of 3 '\n'-s or '\t'-s, with triple '\n' suffixes
+        being preferred over triple '\t', at 2:1 ratio, employ
+        AugmentPrefixSuffix(augment_model_input=True, suffixes={'\n':2,'\t':1}, suffix_len=3, prefixes = None)
+        which will append '\n'-s twice as often as '\t'-s.
+
+    """
+
+    prefixes: Optional[Union[List[str], Dict[str, int]]] = {
+        " ": 20,
+        "\\t": 10,
+        "\\n": 40,
+        "": 30,
+    }
+    prefix_len: Optional[int] = 3
+    suffixes: Optional[Union[List[str], Dict[str, int]]] = {
+        " ": 20,
+        "\\t": 10,
+        "\\n": 40,
+        "": 30,
+    }
+    suffix_len: Optional[int] = 3
+    remove_existing_whitespaces: Optional[bool] = False
+
+    def verify(self):
+        assert (
+            self.prefixes or self.suffixes
+        ), "At least one of prefixes/suffixes should be not None."
+        for arg, arg_name in zip(
+            [self.prefixes, self.suffixes], ["prefixes", "suffixes"]
+        ):
+            assert (
+                arg is None or isoftype(arg, List[str]) or isoftype(arg, Dict[str, int])
+            ), f"Argument {arg_name} should be either None or a list of strings or a dictionary str->int. {arg} is none of the above."
+        assert (
+            self.prefix_len > 0
+        ), f"prefix_len must be positive, got {self.prefix_len}"
+        assert (
+            self.suffix_len > 0
+        ), f"suffix_len must be positive, got {self.suffix_len}"
+        super().verify()
+
+    def _calculate_distributions(self, prefs_or_suffs):
+        if prefs_or_suffs is None:
+            return None, None
+        patterns = (
+            prefs_or_suffs
+            if isinstance(prefs_or_suffs, list)
+            else [k for k, v in prefs_or_suffs.items()]
+        )
+        total_weight = (
+            len(patterns)
+            if isinstance(prefs_or_suffs, list)
+            else sum([v for k, v in prefs_or_suffs.items()])
+        )
+        weights = (
+            [1.0 / total_weight] * len(patterns)
+            if isinstance(prefs_or_suffs, list)
+            else [float(prefs_or_suffs[p]) / total_weight for p in patterns]
+        )
+        return patterns, weights
+
+    def prepare(self):
+        # Being an artifact, prepare is invoked before verify. Here we need verify before the actions
+        self.verify()
+        self._prefix_pattern_distribution = {"length": self.prefix_len}
+        self._suffix_pattern_distribution = {"length": self.suffix_len}
+
+        (
+            self._prefix_pattern_distribution["patterns"],
+            self._prefix_pattern_distribution["weights"],
+        ) = self._calculate_distributions(self.prefixes)
+        (
+            self._suffix_pattern_distribution["patterns"],
+            self._suffix_pattern_distribution["weights"],
+        ) = self._calculate_distributions(self.suffixes)
+        super().prepare()
+
+    def _get_random_pattern(
+        self, pattern_distribution, random_generator: Random
+    ) -> str:
+        string_to_add = ""
+        if pattern_distribution["patterns"]:
+            string_to_add = "".join(
+                random_generator.choices(
+                    pattern_distribution["patterns"],
+                    pattern_distribution["weights"],
+                    k=pattern_distribution["length"],
+                )
+            )
+        return string_to_add
+
+    def process_value(self, value: Any) -> Any:
+        assert value is not None, "input value should not be None"
+        new_value = str(value)
+        if self.remove_existing_whitespaces:
+            new_value = new_value.strip()
+        random_generator = new_random_generator(sub_seed=value)
+        prefix = self._get_random_pattern(
+            self._prefix_pattern_distribution, random_generator
+        )
+        suffix = self._get_random_pattern(
+            self._suffix_pattern_distribution, random_generator
+        )
+        return prefix + new_value + suffix

dataset.py

@@ -4,6 +4,7 @@ import datasets
 
 from .api import __file__ as _
 from .artifact import __file__ as _
+from .augmentors import __file__ as _
 from .benchmark import __file__ as _
 from .blocks import __file__ as _
 from .card import __file__ as _
@@ -43,6 +44,7 @@ from .random_utils import __file__ as _
 from .recipe import __file__ as _
 from .register import __file__ as _
 from .schema import __file__ as _
+from .serializers import __file__ as _
 from .settings_utils import __file__ as _
 from .settings_utils import get_constants
 from .span_lableing_operators import __file__ as _
@@ -58,6 +60,7 @@ from .task import __file__ as _
 from .templates import __file__ as _
 from .text_utils import __file__ as _
 from .type_utils import __file__ as _
+from .types import __file__ as _
 from .utils import __file__ as _
 from .utils import is_package_installed
 from .validate import __file__ as _

deprecation_utils.py

@@ -1,6 +1,7 @@
 import functools
 import warnings
 
+from .error_utils import UnitxtWarning
 from .settings_utils import get_constants, get_settings
 
 constants = get_constants()
@@ -98,3 +99,39 @@ def deprecation(version, alternative=None, msg=None):
         return depraction_wrapper(func, version, alt_text)
 
     return decorator
+
+
+def init_warning(msg=""):
+    # Decorator that raises warning when class is initialized
+    def decorator(initiated_class):
+        UnitxtWarning(msg)
+        return initiated_class
+
+    return decorator
+
+
+def warn_on_call(warning_type=UserWarning, msg=""):
+    def decorator(obj):
+        if isinstance(obj, type):
+            original_init = obj.__init__
+
+            @functools.wraps(original_init)
+            def new_init(self, *args, **kwargs):
+                warnings.warn(msg, warning_type, stacklevel=2)
+                original_init(self, *args, **kwargs)
+
+            obj.__init__ = new_init
+            return obj
+
+        if callable(obj):
+
+            @functools.wraps(obj)
+            def wrapper(*args, **kwargs):
+                warnings.warn(msg, warning_type, stacklevel=2)
+                return obj(*args, **kwargs)
+
+            return wrapper
+
+        raise TypeError("This decorator can only be applied to classes or functions.")
+
+    return decorator

dialog_operators.py

@@ -11,7 +11,6 @@ dialog = [
     {"user": "kkk", "system": ""},
 ]
 """
-
 from typing import Any, Dict, List, Optional
 
 from .formats import SystemFormat
@@ -34,10 +33,11 @@ class SerializeDialog(InstanceFieldOperator):
         context_field (Optional[str]): Field that contains additional context to be prepended to the dialog.
     """
 
-    format: Optional[SystemFormat] = None
+    format: SystemFormat = None
     last_response_to_field: Optional[str] = None
     context_field: Optional[str] = None
     context_separator: str = " "
+    slice_first_and_last_turns_format: bool = True
 
     def standardize_format(self, demo_format):
         turn_format = demo_format.replace("{source}", "{user}")
@@ -54,10 +54,11 @@ class SerializeDialog(InstanceFieldOperator):
         return turn_format[: turn_format.index("{user}") + len("{user}")]
 
     def get_turn_format(self, turn_format, step, length):
-        if step == 0:
+        if step == 0 and self.slice_first_and_last_turns_format:
             turn_format = self.slice_first_turn(turn_format)
         if step == length - 1:
-            turn_format = self.slice_last_turn(turn_format)
+            if self.slice_first_and_last_turns_format:
+                turn_format = self.slice_last_turn(turn_format)
             if self.last_response_to_field is not None:
                 turn_format = self.slice_last_response(turn_format)
         return turn_format
@@ -87,3 +88,149 @@ class SerializeDialog(InstanceFieldOperator):
         if self.last_response_to_field is not None:
             instance[self.last_response_to_field] = turn["system"]
         return dialog
+
+
+class SerializeOpenAiFormatDialog(SerializeDialog):
+    """Serializes dialog data for feeding into a model.
+
+    This class takes structured dialog data in the OpenAi format, and converts it into a text format
+    according to a specified template. It allows for the inclusion or exclusion
+    of system responses and can operate on a per-turn basis or aggregate the entire
+    dialog.
+
+    Attributes:
+        field (str): The field in the input data that contains the dialog.
+        to_field (Optional[str]): The field in the output data where the serialized dialog will be stored.
+        last_user_turn_to_field (Optional[str]): Field to store the last user turn.
+        last_system_turn_to_field (Optional[str]): Field to store the last system turn.
+        context_field (Optional[str]): Field that contains additional context to be prepended to the dialog.
+    """
+
+    is_last_turn_user_only: bool = True
+
+    @staticmethod
+    def validate_openai_dialog_format(dialog: List[Dict[str, str]]) -> None:
+        """Validates that the given dialog follows the correct OpenAI format.
+
+        The function checks that:
+        1. The dialog is a list of dictionaries.
+        2. Each dictionary contains the keys 'role' and 'content'.
+        3. The 'role' value is either 'user' or 'assistant'.
+        4. Both 'role' and 'content' values are strings.
+        5. The first 'role' is 'user'
+
+        If the dialog does not conform to the expected format, a descriptive
+        ValueError is raised indicating the issue.
+
+        Args:
+            dialog (List[Dict[str, str]]): The dialog to validate.
+
+        Raises:
+            ValueError: If the dialog does not meet the format requirements.
+        """
+        if not isinstance(dialog, list):
+            raise ValueError("Dialog must be a list of dictionaries.")
+
+        for i, entry in enumerate(dialog):
+            if not isinstance(entry, dict):
+                raise ValueError(
+                    f"Entry {i} is not a dictionary: {entry}. Each entry in the dialog must be a dictionary."
+                )
+
+            if "role" not in entry:
+                raise ValueError(
+                    f"Entry {i} is missing the 'role' key: {entry}. Each dictionary must have a 'role' key."
+                )
+
+            if "content" not in entry:
+                raise ValueError(
+                    f"Entry {i} is missing the 'content' key: {entry}. Each dictionary must have a 'content' key."
+                )
+
+            if not isinstance(entry["role"], str):
+                raise ValueError(
+                    f"Entry {i} has a non-string 'role': {entry['role']}. The 'role' value must be a string."
+                )
+
+            if not isinstance(entry["content"], str):
+                raise ValueError(
+                    f"Entry {i} has a non-string 'content': {entry['content']}. The 'content' value must be a string."
+                )
+
+            if entry["role"] not in {"user", "assistant"}:
+                raise ValueError(
+                    f"Entry {i} has an invalid role: {entry['role']}. Allowed roles are 'user' and 'assistant'."
+                )
+
+        first_entry = dialog[0]
+        if first_entry["role"] != "user":
+            raise ValueError(
+                f"First entry role is expected to be 'user' It is  {first_entry['role']}."
+            )
+
+    @staticmethod
+    def merge_dialog_entries(dialog: List[Dict[str, str]]) -> List[Dict[str, str]]:
+        """Merges consecutive dialog entries with the same role.
+
+        Args:
+            dialog (List[Dict[str, str]]): The input dialog list where each dictionary has a 'role' and 'content'.
+
+        Returns:
+            List[Dict[str, str]]: A new list where consecutive entries with the same role are merged.
+        """
+        if len(dialog) == 0:
+            return []
+
+        merged_dialog = [dialog[0]]
+
+        for entry in dialog[1:]:
+            if entry["role"] == merged_dialog[-1]["role"]:
+                merged_dialog[-1]["content"] += " " + entry["content"]
+            else:
+                merged_dialog.append(entry)
+
+        return merged_dialog
+
+    def transform_dialog_to_standard_format(
+        self, dialog: List[Dict[str, str]]
+    ) -> List[Dict[str, str]]:
+        """Transforms a dialog from OpenAI format to a simplified format.
+
+        Each dictionary
+        contains 'user' and 'system' keys with their respective contents. Consecutive entries
+        with the same role are merged. Entries with invalid roles raise an error.
+
+        Args:
+            dialog (List[Dict[str, str]]): The input dialog in OpenAI format.
+
+        Returns:
+            List[Dict[str, str]]: The transformed dialog.
+
+        Raises:
+            ValueError: If an invalid role is detected.
+        """
+        SerializeOpenAiFormatDialog.validate_openai_dialog_format(dialog)
+        merged_dialog = SerializeOpenAiFormatDialog.merge_dialog_entries(dialog)
+        # self.validate_dialog_have_complete_pairs(merged_dialog)
+
+        result = []
+        for i in range(0, len(merged_dialog) - 1, 2):
+            user_entry = merged_dialog[i]
+            system_entry = merged_dialog[i + 1]
+
+            result.append(
+                {"user": user_entry["content"], "system": system_entry["content"]}
+            )
+        if len(merged_dialog) % 2 != 0:
+            user_entry = merged_dialog[-1]
+            result.append({"user": user_entry["content"], "system": ""})
+
+        return result
+
+    def process_instance_value(
+        self, structured_dialog: List[Dict[str, str]], instance: Dict[str, Any]
+    ):
+        standard_format_dialog = self.transform_dialog_to_standard_format(
+            structured_dialog
+        )
+        return super().process_instance_value(standard_format_dialog, instance)

formats.py

@@ -164,7 +164,7 @@ class SystemFormat(BaseFormat):
                 demos is not None and isoftype(demos, List[Dict[str, Any]])
             ), f"A list of dict-s is expected in field '{self.demos_field}'. Received instance: {instance}"
             demo_instances = demos
-            instance.pop(self.demos_field)
+            # instance.pop(self.demos_field)
 
         demos_string = ""
         for demo_instance in demo_instances:
@@ -226,14 +226,16 @@ class HFSystemFormat(BaseFormat):
     """
 
     model_name: str
+    _requirements_list = ["transformers"]
 
-    def process(
-        self, instance: Dict[str, Any], stream_name: Optional[str] = None
-    ) -> Dict[str, Any]:
+    def prepare(self):
         from transformers import AutoTokenizer
 
-        tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
 
+    def process(
+        self, instance: Dict[str, Any], stream_name: Optional[str] = None
+    ) -> Dict[str, Any]:
         assert (
             "source" in instance
         ), f"field 'source' is expected to be in the input instance. Received instance: {instance}"
@@ -267,7 +269,7 @@ class HFSystemFormat(BaseFormat):
                 demos is not None and isoftype(demos, List[Dict[str, Any]])
             ), f"A list of dict-s is expected in field '{self.demos_field}'. Received instance: {instance}"
             demo_instances = demos
-            instance.pop(self.demos_field)
+            # instance.pop(self.demos_field)
 
         for demo_instance in demo_instances:
             messages.extend(
@@ -280,7 +282,7 @@ class HFSystemFormat(BaseFormat):
                 ]
             )
         messages.extend([{"role": "user", "content": source}])
-        tokenized_chat = tokenizer.apply_chat_template(
+        tokenized_chat = self.tokenizer.apply_chat_template(
             messages, tokenize=False, add_generation_prompt=True
         )
 

image_operators.py

@@ -1,8 +1,25 @@
+import base64
+import io
 import re
+from abc import abstractmethod
 from typing import Any, Dict
 
+import numpy as np
+
 from .dict_utils import dict_get
-from .operators import InstanceFieldOperator
+from .operators import FieldOperator, InstanceFieldOperator, PackageRequirementsMixin
+
+
+class PillowMixin(PackageRequirementsMixin):
+    _requirements_list = {"PIL": "pip install pillow"}
+
+    def prepare(self):
+        super().prepare()
+        import PIL
+        from PIL import Image
+
+        self.pil = PIL
+        self.image = Image
 
 
 def extract_images(text, instance):
@@ -15,12 +32,44 @@ def extract_images(text, instance):
     return images
 
 
-class ImageToText(InstanceFieldOperator):
+class DecodeImage(FieldOperator, PillowMixin):
+    def decode_base64_to_image(self, base64_string):
+        image_data = base64.b64decode(base64_string)
+        return self.image.open(io.BytesIO(image_data))
+
+    def process_value(self, value: Any) -> Any:
+        return {"image": self.decode_base64_to_image(value)}
+
+
+class ToImage(InstanceFieldOperator):
     def process_instance_value(self, value: Any, instance: Dict[str, Any]):
-        if "media" not in instance:
-            instance["media"] = {}
-        if "images" not in instance["media"]:
-            instance["media"]["images"] = []
-        idx = len(instance["media"]["images"])
-        instance["media"]["images"].append(value)
-        return f'<img src="media/images/{idx}">'
+        return {"image": value}
+
+
+class ImageFieldOperator(FieldOperator, PillowMixin):
+    @abstractmethod
+    def process_image(self, image):
+        pass
+
+    def process_value(self, value: Any) -> Any:
+        if not isinstance(value, self.image.Image):
+            raise ValueError(f"ImageFieldOperator requires image, got {type(value)}.")
+        return self.process_image(value)
+
+
+class GrayScale(ImageFieldOperator):
+    def process_image(self, image):
+        # Convert the image to grayscale
+        grayscale_image = image.convert("L")
+
+        # Convert the grayscale image to a NumPy array
+        grayscale_array = np.array(grayscale_image)
+
+        # Add a dummy channel dimension to make it (height, width, 1)
+        grayscale_array = np.expand_dims(grayscale_array, axis=-1)
+
+        # Repeat the channel to have (height, width, 3) if needed for compatibility
+        grayscale_array = np.repeat(grayscale_array, 3, axis=-1)
+
+        # Convert back to a PIL image with 3 channels
+        return self.image.fromarray(grayscale_array)

inference.py

@@ -5,12 +5,15 @@ from typing import Any, Dict, List, Literal, Optional, Union
 
 from tqdm import tqdm
 
-from .artifact import Artifact
+from .artifact import Artifact, fetch_artifact
 from .dataclass import InternalField, NonPositionalField
 from .deprecation_utils import deprecation
 from .image_operators import extract_images
 from .logging_utils import get_logger
 from .operator import PackageRequirementsMixin
+from .settings_utils import get_settings
+
+settings = get_settings()
 
 
 class InferenceEngine(abc.ABC, Artifact):
@@ -21,9 +24,20 @@ class InferenceEngine(abc.ABC, Artifact):
         """Perform inference on the input dataset."""
         pass
 
+    @abc.abstractmethod
+    def prepare_engine(self):
+        """Perform inference on the input dataset."""
+        pass
+
+    def prepare(self):
+        if not settings.mock_inference_mode:
+            self.prepare_engine()
+
     def infer(self, dataset) -> str:
         """Verifies instances of a dataset and performs inference."""
         [self.verify_instance(instance) for instance in dataset]
+        if settings.mock_inference_mode:
+            return [instance["source"] for instance in dataset]
         return self._infer(dataset)
 
     @deprecation(version="2.0.0")
@@ -122,7 +136,7 @@ class HFPipelineBasedInferenceEngine(
             model=self.model_name, trust_remote_code=True, **model_args
         )
 
-    def prepare(self):
+    def prepare_engine(self):
         if not self.lazy_load:
             self._prepare_pipeline()
 
@@ -144,13 +158,17 @@ class HFPipelineBasedInferenceEngine(
 class MockInferenceEngine(InferenceEngine):
     model_name: str
 
-    def prepare(self):
+    def prepare_engine(self):
         return
 
     def _infer(self, dataset):
         return ["[[10]]" for instance in dataset]
 
 
+class MockModeMixin(Artifact):
+    mock_mode: bool = False
+
+
 class IbmGenAiInferenceEngineParamsMixin(Artifact):
     beam_width: Optional[int] = None
     decoding_method: Optional[Literal["greedy", "sample"]] = None
@@ -190,6 +208,57 @@ class IbmGenAiInferenceEngineParams(Artifact):
     typical_p: Optional[float] = None
 
 
+class GenericInferenceEngine(InferenceEngine):
+    default: Optional[str] = None
+
+    def prepare_engine(self):
+        if "UNITXT_INFERENCE_ENGINE" in os.environ:
+            engine_reference = os.environ["UNITXT_INFERENCE_ENGINE"]
+        else:
+            assert self.default is not None, (
+                "GenericInferenceEngine could not be initialized"
+                '\nThis is since both the "UNITXT_INFERENCE_ENGINE" environmental variable is not set and no default engine was not inputted.'
+                "\nFor example, you can fix it by setting"
+                "\nexport UNITXT_INFERENCE_ENGINE=engines.ibm_gen_ai.llama_3_70b_instruct"
+                "\nto your ~/.bashrc"
+                "\nor passing a similar required engine in the default argument"
+            )
+            engine_reference = self.default
+        self.engine, _ = fetch_artifact(engine_reference)
+
+    def _infer(self, dataset):
+        return self.engine._infer(dataset)
+
+
+class OllamaInferenceEngine(InferenceEngine, PackageRequirementsMixin):
+    label: str = "ollama"
+    model_name: str
+    _requirements_list = {
+        "ollama": "Install ollama package using 'pip install --upgrade ollama"
+    }
+    data_classification_policy = ["public", "proprietary"]
+
+    def prepare_engine(self):
+        pass
+
+    def _infer(self, dataset):
+        import ollama
+
+        result = [
+            ollama.chat(
+                model="llama2",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": instance["source"],
+                    },
+                ],
+            )
+            for instance in dataset
+        ]
+        return [element["message"]["content"] for element in result]
+
+
 class IbmGenAiInferenceEngine(
     InferenceEngine, IbmGenAiInferenceEngineParamsMixin, PackageRequirementsMixin
 ):
@@ -201,11 +270,12 @@ class IbmGenAiInferenceEngine(
     data_classification_policy = ["public", "proprietary"]
     parameters: Optional[IbmGenAiInferenceEngineParams] = None
 
-    def prepare(self):
+    def prepare_engine(self):
         from genai import Client, Credentials
 
         api_key_env_var_name = "GENAI_KEY"
         api_key = os.environ.get(api_key_env_var_name)
+
         assert api_key is not None, (
             f"Error while trying to run IbmGenAiInferenceEngine."
             f" Please set the environment param '{api_key_env_var_name}'."
@@ -242,9 +312,9 @@ class OpenAiInferenceEngineParamsMixin(Artifact):
     top_p: Optional[float] = None
     top_logprobs: Optional[int] = 20
     logit_bias: Optional[Dict[str, int]] = None
-    logprobs: Optional[bool] = None
+    logprobs: Optional[bool] = True
     n: Optional[int] = None
-    parallel_tool_calls: bool = None
+    parallel_tool_calls: Optional[bool] = None
     service_tier: Optional[Literal["auto", "default"]] = None
 
 
@@ -259,9 +329,9 @@ class OpenAiInferenceEngineParams(Artifact):
     top_p: Optional[float] = None
     top_logprobs: Optional[int] = 20
     logit_bias: Optional[Dict[str, int]] = None
-    logprobs: Optional[bool] = None
+    logprobs: Optional[bool] = True
     n: Optional[int] = None
-    parallel_tool_calls: bool = None
+    parallel_tool_calls: Optional[bool] = None
     service_tier: Optional[Literal["auto", "default"]] = None
 
 
@@ -279,7 +349,7 @@ class OpenAiInferenceEngine(
     data_classification_policy = ["public"]
     parameters: Optional[OpenAiInferenceEngineParams] = None
 
-    def prepare(self):
+    def prepare_engine(self):
         from openai import OpenAI
 
         api_key_env_var_name = "OPENAI_API_KEY"
@@ -293,6 +363,13 @@ class OpenAiInferenceEngine(
 
         self._set_inference_parameters()
 
+    def _get_completion_kwargs(self):
+        return {
+            k: v
+            for k, v in self.to_dict([OpenAiInferenceEngineParamsMixin]).items()
+            if v is not None
+        }
+
     def _infer(self, dataset):
         outputs = []
         for instance in tqdm(dataset, desc="Inferring with openAI API"):
@@ -308,7 +385,7 @@ class OpenAiInferenceEngine(
                     }
                 ],
                 model=self.model_name,
-                **self.to_dict([OpenAiInferenceEngineParamsMixin]),
+                **self._get_completion_kwargs(),
             )
             output = response.choices[0].message.content
 
@@ -331,7 +408,7 @@ class OpenAiInferenceEngine(
                     }
                 ],
                 model=self.model_name,
-                **self.to_dict([OpenAiInferenceEngineParamsMixin]),
+                **self._get_completion_kwargs(),
             )
             top_logprobs_response = response.choices[0].logprobs.content
             output = [
@@ -347,6 +424,96 @@ class OpenAiInferenceEngine(
         return outputs
 
 
+class TogetherAiInferenceEngineParamsMixin(Artifact):
+    max_tokens: Optional[int] = None
+    stop: Optional[List[str]] = None
+    temperature: Optional[float] = None
+    top_p: Optional[float] = None
+    top_k: Optional[int] = None
+    repetition_penalty: Optional[float] = None
+    logprobs: Optional[int] = None
+    echo: Optional[bool] = None
+    n: Optional[int] = None
+    min_p: Optional[float] = None
+    presence_penalty: Optional[float] = None
+    frequency_penalty: Optional[float] = None
+
+
+class TogetherAiInferenceEngine(
+    InferenceEngine, TogetherAiInferenceEngineParamsMixin, PackageRequirementsMixin
+):
+    label: str = "together"
+    model_name: str
+    _requirements_list = {
+        "together": "Install together package using 'pip install --upgrade together"
+    }
+    data_classification_policy = ["public"]
+    parameters: Optional[TogetherAiInferenceEngineParamsMixin] = None
+
+    def prepare_engine(self):
+        from together import Together
+        from together.types.models import ModelType
+
+        api_key_env_var_name = "TOGETHER_API_KEY"
+        api_key = os.environ.get(api_key_env_var_name)
+        assert api_key is not None, (
+            f"Error while trying to run TogetherAiInferenceEngine."
+            f" Please set the environment param '{api_key_env_var_name}'."
+        )
+        self.client = Together(api_key=api_key)
+        self._set_inference_parameters()
+
+        # Get model type from Together List Models API
+        together_models = self.client.models.list()
+        together_model_id_to_type = {
+            together_model.id: together_model.type for together_model in together_models
+        }
+        model_type = together_model_id_to_type.get(self.model_name)
+        assert model_type is not None, (
+            f"Could not find model {self.model_name} " "in Together AI model list"
+        )
+        assert model_type in [ModelType.CHAT, ModelType.LANGUAGE, ModelType.CODE], (
+            f"Together AI model type {model_type} is not supported; "
+            "supported types are 'chat', 'language' and 'code'."
+        )
+        self.model_type = model_type
+
+    def _get_infer_kwargs(self):
+        return {
+            k: v
+            for k, v in self.to_dict([TogetherAiInferenceEngineParamsMixin]).items()
+            if v is not None
+        }
+
+    def _infer_chat(self, prompt: str) -> str:
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=[{"role": "user", "content": prompt}],
+            **self._get_infer_kwargs(),
+        )
+        return response.choices[0].message.content
+
+    def _infer_text(self, prompt: str) -> str:
+        response = self.client.completions.create(
+            model=self.model_name,
+            prompt=prompt,
+            **self._get_infer_kwargs(),
+        )
+        return response.choices[0].text
+
+    def _infer(self, dataset):
+        from together.types.models import ModelType
+
+        outputs = []
+        if self.model_type == ModelType.CHAT:
+            for instance in tqdm(dataset, desc="Inferring with Together AI Chat API"):
+                outputs.append(self._infer_chat(instance["source"]))
+        else:
+            for instance in tqdm(dataset, desc="Inferring with Together AI Text API"):
+                outputs.append(self._infer_text(instance["source"]))
+        return outputs
+
+
 class WMLInferenceEngineParamsMixin(Artifact):
     decoding_method: Optional[Literal["greedy", "sample"]] = None
     length_penalty: Optional[Dict[str, Union[int, float]]] = None
@@ -400,6 +567,7 @@ class WMLInferenceEngine(
         parameters (WMLInferenceEngineParams, optional): Instance of WMLInferenceEngineParams
             which defines inference parameters and their values. Deprecated attribute, please
             pass respective parameters directly to the WMLInferenceEngine class instead.
+        concurrency_limit (int): number of requests that will be sent in parallel, max is 10.
 
     Examples:
         from .api import load_dataset
@@ -433,7 +601,7 @@ class WMLInferenceEngine(
     }
     data_classification_policy = ["public", "proprietary"]
     parameters: Optional[WMLInferenceEngineParams] = None
-
+    concurrency_limit: int = 10
     _client: Any = InternalField(default=None, name="WML client")
 
     def verify(self):
@@ -490,7 +658,7 @@ class WMLInferenceEngine(
         client.set.default_project(self.credentials["project_id"])
         return client
 
-    def prepare(self):
+    def prepare_engine(self):
         self._client = self._initialize_wml_client()
 
         self._set_inference_parameters()
@@ -504,10 +672,19 @@ class WMLInferenceEngine(
             api_client=self._client,
         )
 
-        return model.generate_text(
-            prompt=dataset["source"],
-            params=self.to_dict([WMLInferenceEngineParamsMixin], keep_empty=False),
-        )
+        # the class was previously used with a dataset that is a single instance
+        dataset = dataset if isinstance(dataset, list) else [dataset]
+
+        result = [
+            model.generate_text(
+                prompt=instance["source"],
+                params=self.to_dict([WMLInferenceEngineParamsMixin], keep_empty=False),
+            )
+            for instance in dataset
+        ]
+
+        # the class was previously used with a dataset that is a single instance
+        return result[0] if not isinstance(dataset, list) else result
 
 
 class HFLlavaInferenceEngine(InferenceEngine, LazyLoadMixin):
@@ -541,7 +718,7 @@ class HFLlavaInferenceEngine(InferenceEngine, LazyLoadMixin):
 
         self.processor = AutoProcessor.from_pretrained(self.model_name)
 
-    def prepare(self):
+    def prepare_engine(self):
         if not self.lazy_load:
             self._prepare_engine()
 

llm_as_judge.py

@@ -144,13 +144,13 @@ class LLMAsJudge(BulkInstanceMetric):
             )
 
         if isinstance(self.inference_model, OpenAiInferenceEngine):
-            if self.format:
+            if self.format and type(self.format) is not SystemFormat:
                 raise ValueError(
                     "Error in 'LLMAsJudge' metric. Inference model 'OpenAiInferenceEngine' does "
                     "not support formatting. Please remove the format definition from the recipe"
                     " (OpenAi Chat API take care of the formatting automatically)."
                 )
-            if self.system_prompt:
+            if self.system_prompt and type(self.system_prompt) is not EmptySystemPrompt:
                 raise ValueError(
                     "Error in 'LLMAsJudge' metric. Inference model 'OpenAiInferenceEngine' does "
                     "not support system prompt. Please remove the system_prompt definition from the recipe"
@@ -181,9 +181,17 @@ class LLMAsJudge(BulkInstanceMetric):
         results = []
         for instance in outputs:
             if self.task == "pairwise_comparative_rating.single_turn":
-                is_model_b_the_baseline = (
-                    instance["task_data"]["model_b"] == "baseline_model"
+                import json
+
+                # seems like the task data sometimes comes as a string, not a dict
+                # this fixes it
+                task_data = (
+                    json.loads(instance["task_data"])
+                    if isinstance(instance["task_data"], str)
+                    else instance["task_data"]
                 )
+
+                is_model_b_the_baseline = task_data["model_b"] == "baseline_model"
                 if is_model_b_the_baseline:
                     model_a_preference_score = instance["prediction"]
                 else:

loaders.py

@@ -151,6 +151,7 @@ class LoadHF(Loader):
         data_dir: Optional directory to store downloaded data.
         split: Optional specification of which split to load.
         data_files: Optional specification of particular data files to load.
+        revision: Optional. The revision of the dataset. Often the commit id. Use in case you want to set the dataset version.
         streaming: Bool indicating if streaming should be used.
         filtering_lambda: A lambda function for filtering the data after loading.
         num_proc: Optional integer to specify the number of processes to use for parallel dataset loading.
@@ -170,6 +171,7 @@ class LoadHF(Loader):
     data_files: Optional[
         Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
     ] = None
+    revision: Optional[str] = None
     streaming: bool = True
     filtering_lambda: Optional[str] = None
     num_proc: Optional[int] = None
@@ -199,6 +201,7 @@ class LoadHF(Loader):
                         name=self.name,
                         data_dir=self.data_dir,
                         data_files=self.data_files,
+                        revision=self.revision,
                         streaming=self.streaming,
                         cache_dir=None if self.streaming else dir_to_be_deleted,
                         split=self.split,
@@ -488,6 +491,7 @@ class LoadFromIBMCloud(Loader):
         bucket_name: Name of the S3 bucket from which to load data.
         data_dir: Optional directory path within the bucket.
         data_files: Union type allowing either a list of file names or a mapping of splits to file names.
+        data_field: The dataset key for nested JSON file, i.e. when multiple datasets are nested in the same file
         caching: Bool indicating if caching is enabled to avoid re-downloading data.
 
     Example:
@@ -511,6 +515,7 @@ class LoadFromIBMCloud(Loader):
     data_dir: str = None
 
     data_files: Union[Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
+    data_field: str = None
     caching: bool = True
     data_classification_policy = ["proprietary"]
 
@@ -636,10 +641,13 @@ class LoadFromIBMCloud(Loader):
                     )
 
         if isinstance(self.data_files, list):
-            dataset = hf_load_dataset(local_dir, streaming=False)
+            dataset = hf_load_dataset(local_dir, streaming=False, field=self.data_field)
         else:
             dataset = hf_load_dataset(
-                local_dir, streaming=False, data_files=self.data_files
+                local_dir,
+                streaming=False,
+                data_files=self.data_files,
+                field=self.data_field,
             )
 
         return MultiStream.from_iterables(dataset)

metric.py

@@ -4,6 +4,7 @@ import evaluate
 
 from .api import __file__ as _
 from .artifact import __file__ as _
+from .augmentors import __file__ as _
 from .benchmark import __file__ as _
 from .blocks import __file__ as _
 from .card import __file__ as _
@@ -42,6 +43,7 @@ from .random_utils import __file__ as _
 from .recipe import __file__ as _
 from .register import __file__ as _
 from .schema import __file__ as _
+from .serializers import __file__ as _
 from .settings_utils import __file__ as _
 from .settings_utils import get_constants
 from .span_lableing_operators import __file__ as _
@@ -57,6 +59,7 @@ from .task import __file__ as _
 from .templates import __file__ as _
 from .text_utils import __file__ as _
 from .type_utils import __file__ as _
+from .types import __file__ as _
 from .utils import __file__ as _
 from .utils import is_package_installed
 from .validate import __file__ as _

metrics.py

@@ -421,7 +421,7 @@ class MetricWithConfidenceInterval(Metric):
             full_score_name = ci_score_prefix + score_name
             result[f"{full_score_name}_ci_low"] = ci.low
             result[f"{full_score_name}_ci_high"] = ci.high
-            if score_name == self.main_score:
+            if score_name == self.score_prefix + self.main_score:
                 result["score_ci_low"] = ci.low
                 result["score_ci_high"] = ci.high
         return result
@@ -1183,7 +1183,11 @@ class InstanceMetric(StreamOperator, MetricWithConfidenceInterval):
         return instances
 
     def get_group_scores(
-        self, instances: List[dict], score_names: List[str], group_aggregation_func
+        self,
+        instances: List[dict],
+        score_names: List[str],
+        group_aggregation_func,
+        prepend_score_prefix: bool = True,
     ):
         """Group scores by the group_id and subgroup_type fields of each instance, and compute group_aggregation_func by group.
 
@@ -1193,6 +1197,8 @@ class InstanceMetric(StreamOperator, MetricWithConfidenceInterval):
             group_aggregation_func: Callable aggregation function accepting a list of numeric scores;
                 or, if self.subgroup_column is not None, a dict of subgroup types scores by subgroup_column value.
                 callable function returns a single score for the group
+            prepend_score_prefix: if True - prepend the score_prefix to the score names in the returned dicts. Set to False
+                if down the stream such a prepending is expected.
 
         Returns:
             List of dicts, each corresponding to a group of instances (defined by 'group_id'),
@@ -1222,7 +1228,9 @@ class InstanceMetric(StreamOperator, MetricWithConfidenceInterval):
             )
             for score_name in score_names:
                 group_to_instance_scores[group_key][score_name][subgroup_type].append(
-                    instance["score"]["instance"][score_name]
+                    instance["score"]["instance"][
+                        (self.score_prefix if prepend_score_prefix else "") + score_name
+                    ]
                 )
 
         # if group_aggregation_func expects a subgroup-types score dict, pass it; otherwise pass the default type list of scores
@@ -1230,7 +1238,8 @@ class InstanceMetric(StreamOperator, MetricWithConfidenceInterval):
             {
                 "score": {
                     "instance": {
-                        score_name: group_aggregation_func(
+                        (self.score_prefix if prepend_score_prefix else "")
+                        + score_name: group_aggregation_func(
                             score_dict
                             if uses_subgroups
                             else score_dict[default_subgroup_name]
@@ -1268,7 +1277,7 @@ class InstanceMetric(StreamOperator, MetricWithConfidenceInterval):
                 group_aggregation_func=group_aggregation_func,
             ):
                 group_scores = self.get_group_scores(
-                    instances, [field_name], group_aggregation_func
+                    instances, [field_name], group_aggregation_func, False
                 )
                 return nan_mean(
                     [group["score"]["instance"][field_name] for group in group_scores]
@@ -4565,8 +4574,7 @@ class NormalizedSacrebleu(HuggingfaceMetric):
     scaled_fields = ["sacrebleu", "precisions"]
     hf_additional_input_fields_pass_one_value = ["tokenize"]
     _requirements_list = {
-        "mecab_ko": KO_ERROR_MESSAGE,
-        "mecab_ko_dic": KO_ERROR_MESSAGE,
+        "sacrebleu": "Additional dependencies required. To install them, run: `pip install sacrebleu`."
     }
 
 

operators.py

@@ -531,230 +531,6 @@ class AddConstant(FieldOperator):
         return self.add + value
 
 
-class Augmentor(InstanceOperator):
-    """A stream operator that augments the values of either the task input fields before rendering with the template,  or the input passed to the model after rendering of the template.
-
-    Args:
-        augment_model_input: Whether to augment the input to the model.
-        augment_task_input:  Whether to augment the task input fields.  The specific fields are defined in the Task operator.
-
-    """
-
-    augment_task_input: bool = False
-    augment_model_input: bool = False
-
-    def verify(self):
-        assert not (
-            self.augment_task_input and self.augment_model_input
-        ), "Augmentor must set either 'augment_task_input' and 'augment_model_input' but not both"
-        assert (
-            self.augment_task_input or self.augment_model_input
-        ), "Augmentor must set either 'augment_task_input' or 'augment_model_input'"
-
-        super().verify()
-
-    @abstractmethod
-    def process_value(self, value: Any) -> Any:
-        pass
-
-    def prepare(self):
-        pass
-
-    def set_task_input_fields(self, task_input_fields: List[str]):
-        self._task_input_fields = [
-            "input_fields/" + task_input_field for task_input_field in task_input_fields
-        ]
-
-    def process(
-        self, instance: Dict[str, Any], stream_name: Optional[str] = None
-    ) -> Dict[str, Any]:
-        if self.augment_task_input:
-            assert (
-                len(self._task_input_fields) > 0
-            ), "No augmentable input fields were defined in Task, and augmentation was requested. Specify the fields to augment in 'argumentable_inputs' attribute of the Task."
-            fields = self._task_input_fields
-            assert not self.augment_model_input
-
-        if self.augment_model_input:
-            fields = ["source"]
-            assert not self.augment_task_input
-
-        for field_name in fields:
-            try:
-                old_value = dict_get(
-                    instance,
-                    field_name,
-                    default="",
-                    not_exist_ok=False,
-                )
-            except ValueError as e:
-                raise TypeError(f"Failed to get {field_name} from {instance}") from e
-
-            try:
-                new_value = self.process_value(old_value)
-            except Exception as e:
-                raise RuntimeError(
-                    f"Error augmenting value '{old_value}' from '{field_name}' in instance: {instance}"
-                ) from e
-            dict_set(instance, field_name, new_value, not_exist_ok=True)
-        return instance
-
-
-class NullAugmentor(Augmentor):
-    """Does not change the input string."""
-
-    def verify(self):
-        pass
-
-    def process_value(self, value: Any) -> Any:
-        return value
-
-
-class AugmentWhitespace(Augmentor):
-    """Augments the inputs by replacing existing whitespaces with other whitespaces.
-
-    Currently, each whitespace is replaced by a random choice of 1-3 whitespace characters (space, tab, newline).
-    """
-
-    def process_value(self, value: Any) -> Any:
-        import re
-
-        words = re.split(r"(\s+)", value)
-        new_value = ""
-
-        random_generator = new_random_generator(sub_seed=value)
-        for word in words:
-            if word.isspace():
-                new_value += random_generator.choice(
-                    ["\n", "\t", " "]
-                ) * random_generator.randint(1, 3)
-            else:
-                new_value += word
-        return new_value
-
-
-class AugmentPrefixSuffix(Augmentor):
-    r"""Augments the input by prepending and appending to it a randomly selected (typically, whitespace) patterns.
-
-    Args:
-     prefixes, suffixes (list or dict) : the potential (typically, whitespace) patterns to select from.
-        The dictionary version allows to specify relative weights of the different patterns.
-     prefix_len, suffix_len (positive int) : The added prefix or suffix will be of length
-        prefix_len of suffix_len, respectively, repetitions of the randomly selected patterns.
-     remove_existing_whitespaces : allows to first clean any existing leading and trailing whitespaces.
-        The strings made of repetitions of the selected pattern(s) are then prepended and/or appended to the potentially
-        trimmed input.
-     If only one of prefixes/suffixes is needed, set the other to None.
-
-    Examples:
-        To prepend the input with a prefix made of 4 '\n'-s or '\t'-s, employ
-        AugmentPrefixSuffix(augment_model_input=True, prefixes=['\n','\t'], prefix_len=4, suffixes = None)
-        To append the input with a suffix made of 3 '\n'-s or '\t'-s, with triple '\n' suffixes
-        being preferred over triple '\t', at 2:1 ratio, employ
-        AugmentPrefixSuffix(augment_model_input=True, suffixes={'\n':2,'\t':1}, suffix_len=3, prefixes = None)
-        which will append '\n'-s twice as often as '\t'-s.
-
-    """
-
-    prefixes: Optional[Union[List[str], Dict[str, int]]] = {
-        " ": 20,
-        "\\t": 10,
-        "\\n": 40,
-        "": 30,
-    }
-    prefix_len: Optional[int] = 3
-    suffixes: Optional[Union[List[str], Dict[str, int]]] = {
-        " ": 20,
-        "\\t": 10,
-        "\\n": 40,
-        "": 30,
-    }
-    suffix_len: Optional[int] = 3
-    remove_existing_whitespaces: Optional[bool] = False
-
-    def verify(self):
-        assert (
-            self.prefixes or self.suffixes
-        ), "At least one of prefixes/suffixes should be not None."
-        for arg, arg_name in zip(
-            [self.prefixes, self.suffixes], ["prefixes", "suffixes"]
-        ):
-            assert (
-                arg is None or isoftype(arg, List[str]) or isoftype(arg, Dict[str, int])
-            ), f"Argument {arg_name} should be either None or a list of strings or a dictionary str->int. {arg} is none of the above."
-        assert (
-            self.prefix_len > 0
-        ), f"prefix_len must be positive, got {self.prefix_len}"
-        assert (
-            self.suffix_len > 0
-        ), f"suffix_len must be positive, got {self.suffix_len}"
-        super().verify()
-
-    def _calculate_distributions(self, prefs_or_suffs):
-        if prefs_or_suffs is None:
-            return None, None
-        patterns = (
-            prefs_or_suffs
-            if isinstance(prefs_or_suffs, list)
-            else [k for k, v in prefs_or_suffs.items()]
-        )
-        total_weight = (
-            len(patterns)
-            if isinstance(prefs_or_suffs, list)
-            else sum([v for k, v in prefs_or_suffs.items()])
-        )
-        weights = (
-            [1.0 / total_weight] * len(patterns)
-            if isinstance(prefs_or_suffs, list)
-            else [float(prefs_or_suffs[p]) / total_weight for p in patterns]
-        )
-        return patterns, weights
-
-    def prepare(self):
-        # Being an artifact, prepare is invoked before verify. Here we need verify before the actions
-        self.verify()
-        self._prefix_pattern_distribution = {"length": self.prefix_len}
-        self._suffix_pattern_distribution = {"length": self.suffix_len}
-
-        (
-            self._prefix_pattern_distribution["patterns"],
-            self._prefix_pattern_distribution["weights"],
-        ) = self._calculate_distributions(self.prefixes)
-        (
-            self._suffix_pattern_distribution["patterns"],
-            self._suffix_pattern_distribution["weights"],
-        ) = self._calculate_distributions(self.suffixes)
-        super().prepare()
-
-    def _get_random_pattern(
-        self, pattern_distribution, random_generator: Random
-    ) -> str:
-        string_to_add = ""
-        if pattern_distribution["patterns"]:
-            string_to_add = "".join(
-                random_generator.choices(
-                    pattern_distribution["patterns"],
-                    pattern_distribution["weights"],
-                    k=pattern_distribution["length"],
-                )
-            )
-        return string_to_add
-
-    def process_value(self, value: Any) -> Any:
-        assert value is not None, "input value should not be None"
-        new_value = str(value)
-        if self.remove_existing_whitespaces:
-            new_value = new_value.strip()
-        random_generator = new_random_generator(sub_seed=value)
-        prefix = self._get_random_pattern(
-            self._prefix_pattern_distribution, random_generator
-        )
-        suffix = self._get_random_pattern(
-            self._suffix_pattern_distribution, random_generator
-        )
-        return prefix + new_value + suffix
-
-
 class ShuffleFieldValues(FieldOperator):
     """Shuffles a list of values found in a field."""
 
@@ -1445,7 +1221,7 @@ class ComputeExpressionMixin(Artifact):
 
     def compute_expression(self, instance: dict) -> Any:
         if settings.allow_unverified_code:
-            return eval(self.expression, self.globals, instance)
+            return eval(self.expression, {**self.globals, **instance})
 
         raise ValueError(
             f"Cannot evaluate expression in {self} when unitxt.settings.allow_unverified_code=False - either set it to True or set {settings.allow_unverified_code_key} environment variable."

schema.py

@@ -69,23 +69,36 @@ class Finalize(InstanceOperatorValidator):
 
         return instance
 
-    def process(
-        self, instance: Dict[str, Any], stream_name: Optional[str] = None
+    def _get_instance_task_data(
+        self, instance: Dict[str, Any], use_reference_fields=True
     ) -> Dict[str, Any]:
-        metadata = {
-            "data_classification_policy": instance["data_classification_policy"],
-            "template": self.artifact_to_jsonable(
-                instance["recipe_metadata"]["template"]
-            ),
-            "num_demos": instance["recipe_metadata"]["num_demos"],
-        }
         task_data = {
             **instance["input_fields"],
-            "metadata": metadata,
+            "metadata": {
+                "data_classification_policy": instance["data_classification_policy"],
+            },
         }
-
-        if stream_name != constants.inference_stream:
+        if use_reference_fields:
             task_data = {**task_data, **instance["reference_fields"]}
+        return task_data
+
+    def process(
+        self, instance: Dict[str, Any], stream_name: Optional[str] = None
+    ) -> Dict[str, Any]:
+        task_data = self._get_instance_task_data(
+            instance,
+            use_reference_fields=stream_name != constants.inference_stream,
+        )
+
+        task_data["metadata"]["num_demos"] = instance["recipe_metadata"]["num_demos"]
+        task_data["metadata"]["template"] = self.artifact_to_jsonable(
+            instance["recipe_metadata"]["template"]
+        )
+        if "demos" in instance:
+            task_data["demos"] = [
+                self._get_instance_task_data(instance)
+                for instance in instance.pop("demos")
+            ]
 
         instance["task_data"] = json.dumps(task_data)
 
@@ -99,7 +112,7 @@ class Finalize(InstanceOperatorValidator):
             for key in keys_to_delete:
                 del instance[key]
 
-        data = {**task_data, **metadata}
+        data = {**task_data, **task_data["metadata"]}
         groups = []
         for group_attributes in self.group_by:
             group = {}

serializers.py

@@ -0,0 +1,142 @@
+import csv
+import io
+from abc import abstractmethod
+from typing import Any, Dict, List, Union
+
+from .dataclass import AbstractField, Field
+from .operators import InstanceFieldOperator
+from .type_utils import isoftype, to_type_string
+from .types import Dialog, Image, Number, Table
+
+
+class Serializer(InstanceFieldOperator):
+    def process_instance_value(self, value: Any, instance: Dict[str, Any]) -> str:
+        return self.serialize(value, instance)
+
+    @abstractmethod
+    def serialize(self, value: Any, instance: Dict[str, Any]) -> str:
+        pass
+
+
+class DefaultSerializer(Serializer):
+    def serialize(self, value: Any, instance: Dict[str, Any]) -> str:
+        return str(value)
+
+
+class SingleTypeSerializer(InstanceFieldOperator):
+    serialized_type: object = AbstractField()
+
+    def process_instance_value(self, value: Any, instance: Dict[str, Any]) -> str:
+        if not isoftype(value, self.serialized_type):
+            raise ValueError(
+                f"SingleTypeSerializer for type {self.serialized_type} should get this type. got {to_type_string(value)}"
+            )
+        return self.serialize(value, instance)
+
+
+class DefaultListSerializer(Serializer):
+    def serialize(self, value: Any, instance: Dict[str, Any]) -> str:
+        if isinstance(value, list):
+            return ", ".join(str(item) for item in value)
+        return str(value)
+
+
+class ListSerializer(SingleTypeSerializer):
+    serialized_type = list
+
+    def serialize(self, value: Any, instance: Dict[str, Any]) -> str:
+        return ", ".join(str(item) for item in value)
+
+
+class DialogSerializer(SingleTypeSerializer):
+    serialized_type = Dialog
+
+    def serialize(self, value: Dialog, instance: Dict[str, Any]) -> str:
+        # Convert the Dialog into a string representation, typically combining roles and content
+        return "\n".join(f"{turn['role']}: {turn['content']}" for turn in value)
+
+
+class NumberSerializer(SingleTypeSerializer):
+    serialized_type = Number
+
+    def serialize(self, value: Number, instance: Dict[str, Any]) -> str:
+        # Check if the value is an integer or a float
+        if isinstance(value, int):
+            return str(value)
+        # For floats, format to one decimal place
+        if isinstance(value, float):
+            return f"{value:.1f}"
+        raise ValueError("Unsupported type for NumberSerializer")
+
+
+class NumberQuantizingSerializer(NumberSerializer):
+    serialized_type = Number
+    quantum: Union[float, int] = 0.1
+
+    def serialize(self, value: Number, instance: Dict[str, Any]) -> str:
+        if isoftype(value, Number):
+            quantized_value = round(value / self.quantum) / (1 / self.quantum)
+            if isinstance(self.quantum, int):
+                quantized_value = int(quantized_value)
+            return str(quantized_value)
+        raise ValueError("Unsupported type for NumberSerializer")
+
+
+class TableSerializer(SingleTypeSerializer):
+    serialized_type = Table
+
+    def serialize(self, value: Table, instance: Dict[str, Any]) -> str:
+        output = io.StringIO()
+        writer = csv.writer(output, lineterminator="\n")
+
+        # Write the header and rows to the CSV writer
+        writer.writerow(value["header"])
+        writer.writerows(value["rows"])
+
+        # Retrieve the CSV string
+        return output.getvalue().strip()
+
+
+class ImageSerializer(SingleTypeSerializer):
+    serialized_type = Image
+
+    def serialize(self, value: Image, instance: Dict[str, Any]) -> str:
+        if "media" not in instance:
+            instance["media"] = {}
+        if "images" not in instance["media"]:
+            instance["media"]["images"] = []
+        idx = len(instance["media"]["images"])
+        instance["media"]["images"].append(value["image"])
+        value["image"] = f'<img src="media/images/{idx}">'
+        return value["image"]
+
+
+class MultiTypeSerializer(Serializer):
+    serializers: List[SingleTypeSerializer] = Field(
+        default_factory=lambda: [
+            ImageSerializer(),
+            TableSerializer(),
+            DialogSerializer(),
+        ]
+    )
+
+    def verify(self):
+        super().verify()
+        self._verify_serializers(self.serializers)
+
+    def _verify_serializers(self, serializers):
+        if not isoftype(serializers, List[SingleTypeSerializer]):
+            raise ValueError(
+                "MultiTypeSerializer requires the list of serializers to be List[SingleTypeSerializer]."
+            )
+
+    def add_serializers(self, serializers: List[SingleTypeSerializer]):
+        self._verify_serializers(serializers)
+        self.serializers = serializers + self.serializers
+
+    def serialize(self, value: Any, instance: Dict[str, Any]) -> Any:
+        for serializer in self.serializers:
+            if isoftype(value, serializer.serialized_type):
+                return serializer.serialize(value, instance)
+
+        return str(value)

settings_utils.py

@@ -146,6 +146,7 @@ if Settings.is_uninitilized():
     settings.seed = (int, 42)
     settings.skip_artifacts_prepare_and_verify = (bool, False)
     settings.data_classification_policy = None
+    settings.mock_inference_mode = (bool, False)
 
 if Constants.is_uninitilized():
     constants = Constants()

standard.py

@@ -1,20 +1,27 @@
 from typing import List, Optional, Union
 
+from .augmentors import (
+    Augmentor,
+    FinalStateInputsAugmentor,
+    NullAugmentor,
+    TaskInputsAugmentor,
+)
 from .card import TaskCard
 from .collections_operators import GetLength
 from .dataclass import Field, InternalField, NonPositionalField, OptionalField
 from .formats import Format, SystemFormat
 from .logging_utils import get_logger
 from .operator import SequentialOperator, SourceSequentialOperator, StreamingOperator
-from .operators import Augmentor, NullAugmentor, Set, StreamRefiner
+from .operators import Set, StreamRefiner
 from .recipe import Recipe
 from .schema import Finalize
+from .serializers import SingleTypeSerializer
 from .settings_utils import get_constants
 from .splitters import ConstantSizeSample, RandomSizeSample, Sampler, SeparateSplit
 from .stream import MultiStream
 from .system_prompts import EmptySystemPrompt, SystemPrompt
 from .task import Task
-from .templates import ApplyRandomTemplate, ApplySingleTemplate, Template
+from .templates import ApplyRandomTemplate, ApplySingleTemplate, Template, TemplatesList
 
 constants = get_constants()
 logger = get_logger()
@@ -29,9 +36,10 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
     # Base parameters
     card: TaskCard = None
     task: Task = None
-    template: Union[Template, List[Template]] = None
+    template: Union[Template, List[Template], TemplatesList] = None
     system_prompt: SystemPrompt = Field(default_factory=EmptySystemPrompt)
     format: Format = Field(default_factory=SystemFormat)
+    serializer: Union[SingleTypeSerializer, List[SingleTypeSerializer]] = None
 
     # Additional parameters
     template_card_index: int = NonPositionalField(default=None)
@@ -140,6 +148,11 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
         else:
             self.verify_template(self.template)
 
+        if self.serializer is not None:
+            if not isinstance(self.serializer, list):
+                self.serializer = [self.serializer]
+            self.template.serializer.add_serializers(self.serializer)
+
     def prepare_refiners(self):
         self.train_refiner.max_instances = self.max_train_instances
         self.train_refiner.apply_to_streams = ["train"]
@@ -281,8 +294,8 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
 
         self.processing.steps.append(self.task)
 
-        if self.augmentor.augment_task_input:
-            self.augmentor.set_task_input_fields(self.card.task.augmentable_inputs)
+        if isinstance(self.augmentor, TaskInputsAugmentor):
+            self.augmentor.set_fields(self.card.task.augmentable_inputs)
             self.processing.steps.append(self.augmentor)
 
         if self.has_custom_demos_pool:
@@ -362,7 +375,7 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
 
         self.verbalization.steps.append(self.system_prompt)
         self.verbalization.steps.append(self.format)
-        if self.augmentor.augment_model_input:
+        if isinstance(self.augmentor, FinalStateInputsAugmentor):
             self.verbalization.steps.append(self.augmentor)
 
         if self.postprocessors is not None:
@@ -376,6 +389,8 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
         self.finalize.steps.append(Finalize(group_by=self.group_by))
 
     def prepare(self):
+        if isinstance(self.template, TemplatesList):
+            self.template = self.template.items
         self.reset_pipeline()
 
 

struct_data_operators.py

@@ -29,15 +29,62 @@ import pandas as pd
 
 from .dict_utils import dict_get
 from .operators import FieldOperator, InstanceOperator
+from .random_utils import new_random_generator
+from .serializers import TableSerializer
+from .types import Table
 from .utils import deepcopy
 
 
-class SerializeTable(ABC, FieldOperator):
+def shuffle_columns(table: Table, seed=0) -> Table:
+    # extract header & rows from the dictionary
+    header = table.get("header", [])
+    rows = table.get("rows", [])
+    # shuffle the indices first
+    indices = list(range(len(header)))
+    random_generator = new_random_generator({"table": table, "seed": seed})
+    random_generator.shuffle(indices)
+
+    # shuffle the header & rows based on that indices
+    shuffled_header = [header[i] for i in indices]
+    shuffled_rows = [[row[i] for i in indices] for row in rows]
+
+    table["header"] = shuffled_header
+    table["rows"] = shuffled_rows
+
+    return table
+
+
+def shuffle_rows(table: Table, seed=0) -> Table:
+    # extract header & rows from the dictionary
+    rows = table.get("rows", [])
+    # shuffle rows
+    random_generator = new_random_generator({"table": table, "seed": seed})
+    random_generator.shuffle(rows)
+    table["rows"] = rows
+
+    return table
+
+
+class SerializeTable(ABC, TableSerializer):
     """TableSerializer converts a given table into a flat sequence with special symbols.
 
     Output format varies depending on the chosen serializer. This abstract class defines structure of a typical table serializer that any concrete implementation should follow.
     """
 
+    seed: int = 0
+    shuffle_rows: bool = False
+    shuffle_columns: bool = False
+
+    def serialize(self, value: Table, instance: Dict[str, Any]) -> str:
+        value = deepcopy(value)
+        if self.shuffle_columns:
+            value = shuffle_columns(table=value, seed=self.seed)
+
+        if self.shuffle_rows:
+            value = shuffle_rows(table=value, seed=self.seed)
+
+        return self.serialize_table(value)
+
     # main method to serialize a table
     @abstractmethod
     def serialize_table(self, table_content: Dict) -> str:
@@ -60,10 +107,6 @@ class SerializeTableAsIndexedRowMajor(SerializeTable):
     Format:  col : col1 | col2 | col 3 row 1 : val1 | val2 | val3 | val4 row 2 : val1 | ...
     """
 
-    def process_value(self, table: Any) -> Any:
-        table_input = deepcopy(table)
-        return self.serialize_table(table_content=table_input)
-
     # main method that processes a table
     # table_content must be in the presribed input format
     def serialize_table(self, table_content: Dict) -> str:
@@ -111,10 +154,6 @@ class SerializeTableAsMarkdown(SerializeTable):
     ...
     """
 
-    def process_value(self, table: Any) -> Any:
-        table_input = deepcopy(table)
-        return self.serialize_table(table_content=table_input)
-
     # main method that serializes a table.
     # table_content must be in the presribed input format.
     def serialize_table(self, table_content: Dict) -> str:
@@ -159,10 +198,6 @@ class SerializeTableAsDFLoader(SerializeTable):
     index=[0,1,2])
     """
 
-    def process_value(self, table: Any) -> Any:
-        table_input = deepcopy(table)
-        return self.serialize_table(table_content=table_input)
-
     # main method that serializes a table.
     # table_content must be in the presribed input format.
     def serialize_table(self, table_content: Dict) -> str:
@@ -199,10 +234,6 @@ class SerializeTableAsJson(SerializeTable):
     }
     """
 
-    def process_value(self, table: Any) -> Any:
-        table_input = deepcopy(table)
-        return self.serialize_table(table_content=table_input)
-
     # main method that serializes a table.
     # table_content must be in the presribed input format.
     def serialize_table(self, table_content: Dict) -> str:
@@ -493,20 +524,7 @@ class ShuffleTableRows(FieldOperator):
 
     def process_value(self, table: Any) -> Any:
         table_input = deepcopy(table)
-        return self.shuffle_rows(table_content=table_input)
-
-    # shuffles table rows randomly
-    def shuffle_rows(self, table_content: Dict) -> str:
-        # extract header & rows from the dictionary
-        header = table_content.get("header", [])
-        rows = table_content.get("rows", [])
-        assert header and rows, "Incorrect input table format"
-
-        # shuffle rows
-        random.shuffle(rows)
-        table_content["rows"] = rows
-
-        return table_content
+        return shuffle_rows(table_input)
 
 
 class ShuffleTableColumns(FieldOperator):
@@ -527,27 +545,7 @@ class ShuffleTableColumns(FieldOperator):
 
     def process_value(self, table: Any) -> Any:
         table_input = deepcopy(table)
-        return self.shuffle_columns(table_content=table_input)
-
-    # shuffles table columns randomly
-    def shuffle_columns(self, table_content: Dict) -> str:
-        # extract header & rows from the dictionary
-        header = table_content.get("header", [])
-        rows = table_content.get("rows", [])
-        assert header and rows, "Incorrect input table format"
-
-        # shuffle the indices first
-        indices = list(range(len(header)))
-        random.shuffle(indices)  #
-
-        # shuffle the header & rows based on that indices
-        shuffled_header = [header[i] for i in indices]
-        shuffled_rows = [[row[i] for i in indices] for row in rows]
-
-        table_content["header"] = shuffled_header
-        table_content["rows"] = shuffled_rows
-
-        return table_content
+        return shuffle_columns(table_input)
 
 
 class LoadJson(FieldOperator):

templates.py

@@ -10,6 +10,15 @@ from .dict_utils import dict_set
 from .error_utils import Documentation, UnitxtError
 from .operator import InstanceOperator
 from .random_utils import new_random_generator
+from .serializers import (
+    DialogSerializer,
+    ImageSerializer,
+    ListSerializer,
+    MultiTypeSerializer,
+    NumberQuantizingSerializer,
+    Serializer,
+    TableSerializer,
+)
 from .settings_utils import get_constants
 from .type_utils import isoftype
 
@@ -46,17 +55,26 @@ class Template(InstanceOperator):
     instruction: str = NonPositionalField(default="")
     target_prefix: str = NonPositionalField(default="")
     title_fields: List[str] = NonPositionalField(default_factory=list)
+    serializer: Serializer = NonPositionalField(
+        default_factory=lambda: MultiTypeSerializer(
+            serializers=[
+                ImageSerializer(),
+                TableSerializer(),
+                DialogSerializer(),
+                ListSerializer(),
+            ]
+        )
+    )
 
     def input_fields_to_instruction_and_target_prefix(self, input_fields):
         instruction = self.apply_formatting(
-            input_fields, "input field", self.instruction, "instruction", serialize=True
+            input_fields, "input field", self.instruction, "instruction"
         )
         target_prefix = self.apply_formatting(
             input_fields,
             "input field",
             self.target_prefix,
             "target_prefix",
-            serialize=True,
         )
         return instruction, target_prefix
 
@@ -65,6 +83,12 @@ class Template(InstanceOperator):
     ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         return input_fields, reference_fields
 
+    def preprocess_input_fields(self, input_fields: Dict[str, Any]):
+        return input_fields
+
+    def preprocess_reference_fields(self, reference_fields: Dict[str, Any]):
+        return reference_fields
+
     def process(
         self, instance: Dict[str, Any], stream_name: Optional[str] = None
     ) -> Dict[str, Any]:
@@ -78,14 +102,21 @@ class Template(InstanceOperator):
 
         input_fields = instance.get("input_fields")
         reference_fields = instance.get("reference_fields")
-        input_fields, reference_fields = self.preprocess_input_and_reference_fields(
-            input_fields, reference_fields
-        )
+
+        if stream_name != constants.inference_stream:
+            input_fields, reference_fields = self.preprocess_input_and_reference_fields(
+                input_fields, reference_fields
+            )
+
+        input_fields = self.preprocess_input_fields(input_fields)
 
         self.set_titles(input_fields)
-        source = self.input_fields_to_source(input_fields)
+
+        serialized_inputs = self.serialize(input_fields, instance)
+
+        source = self.input_fields_to_source(serialized_inputs)
         instruction, target_prefix = self.input_fields_to_instruction_and_target_prefix(
-            input_fields
+            serialized_inputs
         )
 
         result = {
@@ -97,19 +128,33 @@ class Template(InstanceOperator):
         }
 
         if stream_name == constants.inference_stream:
-            return result
+            return self.post_process_instance(result)
 
         if reference_fields is None:
             raise ValueError("Should have reference_fields")
 
+        reference_fields = self.preprocess_reference_fields(reference_fields)
+
+        serialized_references = self.serialize(
+            reference_fields, instance
+        )  # Dict[str, str]
+
         target, references = self.reference_fields_to_target_and_references(
-            reference_fields
+            serialized_references
         )
 
         result["target"] = target
         result["references"] = references
 
-        return result
+        return self.post_process_instance(result)
+
+    def post_process_instance(self, instance):
+        return instance
+
+    def serialize(
+        self, data: Dict[str, Any], instance: Dict[str, Any]
+    ) -> Dict[str, str]:
+        return {k: self.serializer.serialize(v, instance) for k, v in data.items()}
 
     @abstractmethod
     def input_fields_to_source(self, input_fields: Dict[str, object]) -> str:
@@ -125,21 +170,13 @@ class Template(InstanceOperator):
     ) -> Tuple[str, List[str]]:
         pass
 
-    def serialize_data(self, data):
-        return {
-            k: ", ".join(str(t) for t in v) if isinstance(v, list) else v
-            for k, v in data.items()
-        }
-
     def apply_formatting(
-        self, data, data_type, format_str, format_name, serialize=False
+        self, data: Dict[str, Any], data_type: str, format_str: str, format_name: str
     ) -> str:
-        if serialize:
-            data = self.serialize_data(data)
         try:
             if format_str is None:
                 raise UnitxtError(
-                    f"Required field 'output_format' of class {self.__class__.__name__} not set in {self.__class__.__name__}",
+                    f"Required field '{format_name}' of class {self.__class__.__name__} not set in {self.__class__.__name__}",
                     Documentation.ADDING_TEMPLATE,
                 )
             return format_str.format(**data)
@@ -197,26 +234,21 @@ class ApplyRandomTemplate(ApplyTemplate):
         return random_generator.choice(self.templates)
 
 
-class InputOutputTemplate(Template):
-    """Generate field 'source' from fields designated as input, and fields 'target' and 'references' from fields designated as output, of the processed instance.
-
-    Args specify the formatting strings with which to glue together the input and reference fields of the processed instance into one string ('source' and 'target'), and into a list of strings ('references').
-    """
-
+class InputFormatTemplate(Template):
     input_format: str
-    output_format: str = None
 
-    def input_fields_to_source(
-        self, input_fields: Dict[str, object]
-    ) -> Tuple[str, str]:
+    def input_fields_to_source(self, input_fields: Dict[str, object]) -> str:
         return self.apply_formatting(
             input_fields,
             "input field",
             self.input_format,
             "input_format",
-            serialize=True,
         )
 
+
+class OutputFormatTemplate(Template):
+    output_format: str = None
+
     def reference_fields_to_target_and_references(
         self, reference_fields: Dict[str, object]
     ) -> str:
@@ -225,12 +257,20 @@ class InputOutputTemplate(Template):
             "reference field",
             self.output_format,
             "output_format",
-            serialize=True,
         )
         references = [target]
         return target, references
 
 
+class InputOutputTemplate(InputFormatTemplate, OutputFormatTemplate):
+    """Generate field 'source' from fields designated as input, and fields 'target' and 'references' from fields designated as output, of the processed instance.
+
+    Args specify the formatting strings with which to glue together the input and reference fields of the processed instance into one string ('source' and 'target'), and into a list of strings ('references').
+    """
+
+    pass
+
+
 class InputOutputTemplateWithCustomTarget(InputOutputTemplate):
     reference: str
 
@@ -242,14 +282,12 @@ class InputOutputTemplateWithCustomTarget(InputOutputTemplate):
             "reference field",
             self.output_format,
             "output_format",
-            serialize=True,
         )
         reference = self.apply_formatting(
             reference_fields,
             "reference field",
             self.reference,
             "reference",
-            serialize=True,
         )
         return target, [reference]
 
@@ -374,22 +412,12 @@ class DialogTemplate(InputOutputTemplate):
             input_fields[dialog_fields.dialog_field] = dialog_str
         return input_fields
 
-    def preprocess_input_and_reference_fields(
-        self, input_fields: Dict[str, Any], reference_fields: Dict[str, Any]
-    ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
-        return self.process_dialog(input_fields), reference_fields
+    def preprocess_input_fields(self, input_fields: Dict[str, Any]):
+        return self.process_dialog(input_fields)
 
 
 class DialogPairwiseChoiceTemplate(DialogTemplate, PairwiseChoiceTemplate):
-    def preprocess_input_and_reference_fields(
-        self, input_fields: Dict[str, Any], reference_fields: Dict[str, Any]
-    ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
-        inputs, reference_fields = DialogTemplate.preprocess_input_and_reference_fields(
-            self, input_fields, reference_fields
-        )
-        return PairwiseChoiceTemplate.preprocess_input_and_reference_fields(
-            self, input_fields, reference_fields
-        )
+    pass
 
 
 class PairwiseComparativeRatingTemplate(InputOutputTemplate):
@@ -448,10 +476,9 @@ class PairwiseComparativeRatingTemplate(InputOutputTemplate):
         return input_fields, reference_fields
 
 
-class MultipleChoiceTemplate(Template):
+class MultipleChoiceTemplate(InputFormatTemplate):
     """Formats the input (that specifies the question), the multiple choices to select the answer from, and specifies the field with the correct answer."""
 
-    input_format: str
     target_prefix: str = ""
     choices_field: str = "choices"
     target_field: str = "label"
@@ -493,7 +520,7 @@ class MultipleChoiceTemplate(Template):
                 "XX",
             ]
 
-    def inputs_to_choices(self, data: Dict[str, object], choice_format: str) -> str:
+    def inputs_to_choices(self, data: Dict[str, Any], choice_format: str) -> str:
         choices = data[self.choices_field]
         enumrated_choices = []
         for i, choice in enumerate(choices):
@@ -505,12 +532,12 @@ class MultipleChoiceTemplate(Template):
             )
         return enumrated_choices
 
-    def inputs_to_numerals(self, input_fields: Dict[str, object]) -> Tuple[str, str]:
+    def inputs_to_numerals(self, input_fields: Dict[str, Any]) -> Tuple[str, str]:
         return self.inputs_to_choices(input_fields, "{choice_numeral}")
 
     def prepare_multiple_choice_inputs(
-        self, input_fields: Dict[str, object]
-    ) -> Dict[str, object]:
+        self, input_fields: Dict[str, Any]
+    ) -> Dict[str, Any]:
         choices = self.inputs_to_choices(input_fields, self.source_choice_format)
         return {
             "numerals": self.inputs_to_numerals(input_fields),
@@ -518,23 +545,10 @@ class MultipleChoiceTemplate(Template):
             self.choices_field: self.choices_separator.join(choices),
         }
 
-    def input_fields_to_source(
-        self, input_fields: Dict[str, object]
-    ) -> Tuple[str, str]:
-        input_fields = self.prepare_multiple_choice_inputs(input_fields)
-        return self.apply_formatting(
-            input_fields,
-            "input field",
-            self.input_format,
-            "input_format",
-            serialize=True,
-        )
-
-    def input_fields_to_instruction_and_target_prefix(self, input_fields):
-        input_fields = self.prepare_multiple_choice_inputs(input_fields)
-        return super().input_fields_to_instruction_and_target_prefix(input_fields)
+    def preprocess_input_fields(self, input_fields: Dict[str, Any]) -> Dict[str, Any]:
+        return self.prepare_multiple_choice_inputs(input_fields)
 
-    def outputs_to_target_index(self, reference_fields: Dict[str, object]) -> str:
+    def outputs_to_target_index(self, reference_fields: Dict[str, object]) -> int:
         target = reference_fields[self.target_field]
 
         if not isinstance(target, int):
@@ -547,9 +561,7 @@ class MultipleChoiceTemplate(Template):
                 ) from e
         return target
 
-    def reference_fields_to_target_and_references(
-        self, reference_fields: Dict[str, object]
-    ) -> str:
+    def preprocess_reference_fields(self, reference_fields: Dict[str, Any]):
         target = reference_fields[self.target_field]
 
         if not isinstance(target, int):
@@ -571,51 +583,40 @@ class MultipleChoiceTemplate(Template):
                 Documentation.ADDING_TEMPLATE,
             ) from e
 
+        return {self.target_field: target}
+
+    def reference_fields_to_target_and_references(
+        self, reference_fields: Dict[str, object]
+    ) -> str:
+        target = reference_fields[self.target_field]
         return target, [target]
 
-    def _shuffle_choices(self, instance, stream_name):
-        if stream_name != constants.inference_stream:
-            target_index = self.outputs_to_target_index(instance["reference_fields"])
-            original_label_choice = instance["reference_fields"][self.choices_field][
-                target_index
-            ]
-        choices = instance["input_fields"][self.choices_field]
+    def preprocess_input_and_reference_fields(
+        self, input_fields: Dict[str, Any], reference_fields: Dict[str, Any]
+    ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+        if self.shuffle_choices:
+            target_index = self.outputs_to_target_index(reference_fields)
+            original_label_choice = reference_fields[self.choices_field][target_index]
+            choices = input_fields[self.choices_field]
+            random_seed = {**input_fields}
 
-        random_seed = {**instance["input_fields"]}
+            random_generator = new_random_generator(random_seed)
+            random_generator.shuffle(choices)
+            input_fields[self.choices_field] = choices
 
-        random_generator = new_random_generator(random_seed)
-        random_generator.shuffle(choices)
-        instance["input_fields"][self.choices_field] = choices
+            reference_fields[self.choices_field] = choices
+            reference_fields[self.target_field] = choices.index(original_label_choice)
 
-        if stream_name == constants.inference_stream:
-            return instance
+        return input_fields, reference_fields
 
-        instance["reference_fields"][self.choices_field] = choices
-        instance["reference_fields"][self.target_field] = choices.index(
-            original_label_choice
+    def post_process_instance(self, instance):
+        instance["input_fields"]["options"] = self.inputs_to_choices(
+            instance["input_fields"], self.target_choice_format
         )
-
         return instance
 
-    def process(
-        self, instance: Dict[str, Any], stream_name: Optional[str] = None
-    ) -> Dict[str, Any]:
-        if self.shuffle_choices:
-            instance = self._shuffle_choices(instance, stream_name)
-        result = super().process(instance, stream_name)
-        if stream_name == constants.inference_stream:
-            result["input_fields"]["options"] = self.inputs_to_choices(
-                instance["input_fields"], self.target_choice_format
-            )
-        else:
-            if "options" not in result["reference_fields"]:
-                result["reference_fields"]["options"] = self.inputs_to_choices(
-                    instance["reference_fields"], self.target_choice_format
-                )
-        return result
-
 
-class YesNoTemplate(Template):
+class YesNoTemplate(InputFormatTemplate):
     """A template for generating binary Yes/No questions asking whether an input text is of a specific class.
 
     input_format:
@@ -641,17 +642,6 @@ class YesNoTemplate(Template):
     yes_answer: str = "Yes"
     no_answer: str = "No"
 
-    def input_fields_to_source(
-        self, input_fields: Dict[str, object]
-    ) -> Tuple[str, str]:
-        return self.apply_formatting(
-            input_fields,
-            "input field",
-            self.input_format,
-            "input_format",
-            serialize=True,
-        )
-
     def reference_fields_to_target_and_references(
         self, reference_fields: Dict[str, object]
     ) -> str:
@@ -695,16 +685,13 @@ class KeyValTemplate(Template):
     def process_dict(
         self, data: Dict[str, object], key_val_sep, pairs_sep, use_keys
     ) -> str:
-        data = self.serialize_data(data)
         pairs = []
         for key, val in data.items():
             key_val = [key, str(val)] if use_keys else [str(val)]
             pairs.append(key_val_sep.join(key_val))
         return pairs_sep.join(pairs)
 
-    def input_fields_to_source(
-        self, input_fields: Dict[str, object]
-    ) -> Tuple[str, str]:
+    def input_fields_to_source(self, input_fields: Dict[str, object]) -> str:
         return self.process_dict(
             input_fields,
             key_val_sep=self.key_val_separator,
@@ -725,25 +712,16 @@ class KeyValTemplate(Template):
 
 
 class OutputQuantizingTemplate(InputOutputTemplate):
-    quantum: Union[float, int] = 0.1  # Now supports both int and float
+    serializer: MultiTypeSerializer = NonPositionalField(
+        default_factory=MultiTypeSerializer
+    )
+    quantum: Union[float, int] = 0.1
 
-    def reference_fields_to_target_and_references(
-        self, reference_fields: Dict[str, object]
-    ) -> str:
-        if isinstance(self.quantum, int):
-            # When quantum is an int, format quantized values as ints
-            quantized_outputs = {
-                key: f"{int(round(value / self.quantum) * self.quantum)}"
-                for key, value in reference_fields.items()
-            }
-        else:
-            # When quantum is a float, format quantized values with precision based on quantum
-            quantum_str = f"{self.quantum:.10f}".rstrip("0").rstrip(".")
-            quantized_outputs = {
-                key: f"{round(value / self.quantum) * self.quantum:{quantum_str}}"
-                for key, value in reference_fields.items()
-            }
-        return super().reference_fields_to_target_and_references(quantized_outputs)
+    def prepare(self):
+        super().prepare()
+        self.serializer.add_serializers(
+            [NumberQuantizingSerializer(quantum=self.quantum)]
+        )
 
 
 class MultiLabelTemplate(InputOutputTemplate):
@@ -753,9 +731,9 @@ class MultiLabelTemplate(InputOutputTemplate):
     output_format: str = "{labels}"
     empty_label: str = "None"
 
-    def reference_fields_to_target_and_references(
-        self, reference_fields: Dict[str, object]
-    ) -> str:
+    def preprocess_reference_fields(
+        self, reference_fields: Dict[str, Any]
+    ) -> Dict[str, Any]:
         labels = reference_fields[self.labels_field]
         if not isinstance(labels, list):
             raise UnitxtError(
@@ -765,18 +743,29 @@ class MultiLabelTemplate(InputOutputTemplate):
         if len(labels) == 0:
             labels = [self.empty_label]
         labels_str = self.labels_separator.join(labels)
-        return super().reference_fields_to_target_and_references(
-            {self.labels_field: labels_str}
-        )
+        return {self.labels_field: labels_str}
 
 
 class MultiReferenceTemplate(InputOutputTemplate):
     references_field: str = "references"
     random_reference: bool = False
+    serializer: Serializer = NonPositionalField(default_factory=MultiTypeSerializer)
+
+    def serialize(
+        self, data: Dict[str, Any], instance: Dict[str, Any]
+    ) -> Dict[str, str]:
+        result = {}
+        for k, v in data.items():
+            if k == self.references_field:
+                v = [self.serializer.serialize(item, instance) for item in v]
+            else:
+                v = self.serializer.serialize(v, instance)
+            result[k] = v
+        return result
 
     def reference_fields_to_target_and_references(
         self, reference_fields: Dict[str, object]
-    ) -> List[str]:
+    ) -> Tuple[str, List[str]]:
         references = reference_fields[self.references_field]
         if not isoftype(references, List[str]):
             raise UnitxtError(
@@ -825,12 +814,12 @@ class SpanLabelingBaseTemplate(MultiLabelTemplate):
             if self.labels_support is None or span[3] in self.labels_support:
                 yield span[2], span[3]
 
-    def reference_fields_to_target_and_references(
-        self, reference_fields: Dict[str, object]
-    ) -> Dict[str, object]:
+    def preprocess_reference_fields(
+        self, reference_fields: Dict[str, Any]
+    ) -> Dict[str, Any]:
         span_labels_pairs = self.extract_span_label_pairs(reference_fields)
         targets = self.span_label_pairs_to_targets(span_labels_pairs)
-        return super().reference_fields_to_target_and_references({"labels": targets})
+        return super().preprocess_reference_fields({"labels": targets})
 
     @abstractmethod
     def span_label_pairs_to_targets(self, pairs):

type_utils.py

@@ -4,48 +4,75 @@ import io
 import itertools
 import re
 import typing
+from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
 
 from .utils import safe_eval
 
-_supported_types_strings = [
-    "Any",
-    "List[...]",
-    "Dict[...]",
-    "Tuple[...]",
-    "Union[...]",
-    "Optional[...]",
-    "int",
-    "float",
-    "dict",
-    "double",
-    "str",
-]
+_registered_types = {
+    "Any": typing.Any,
+    "List": typing.List,
+    "Dict": typing.Dict,
+    "Tuple": typing.Tuple,
+    "Union": typing.Union,
+    "Optional": typing.Optional,
+    "Literal": typing.Literal,
+    "int": int,
+    "str": str,
+    "float": float,
+    "bool": bool,
+}
+
+
+def register_type(new_type):
+    assert is_new_type(new_type) or is_typed_dict(
+        new_type
+    ), "Can register only typing.NewType or typing.TypedDict"
+    _registered_types[new_type.__name__] = new_type
+
 
 Type = typing.Any
 
 
 class UnsupportedTypeError(ValueError):
     def __init__(self, type_object):
-        supported_types = ", ".join(_supported_types_strings)
+        supported_types = ", ".join(_registered_types.keys())
         super().__init__(
             f"Type: '{type_object!s}' is not supported type. Use one of {supported_types}"
         )
 
 
+class GenericTypedDict(TypedDict):
+    pass
+
+
 _generics = [
-    typing.List[typing.Any],
-    typing.Dict[typing.Any, typing.Any],
-    typing.Tuple[typing.Any],
-    typing.Union[typing.Any, typing.Any],
-    typing.Optional[typing.Any],
-    typing.Any,
+    List[Any],
+    Dict[Any, Any],
+    Tuple[Any],
+    Union[Any, Any],
+    Optional[Any],
+    Any,
+    Literal,
 ]
 
 _generics_types = [type(t) for t in _generics]
 
 
+def is_new_type(object):
+    return callable(object) and hasattr(object, "__supertype__")
+
+
+def is_typed_dict(object):
+    return isinstance(object, type(GenericTypedDict))
+
+
 def is_type(object):
-    return isinstance(object, (type, *_generics_types))
+    """Checks if the provided object is a type, including generics, Literal, TypedDict, and NewType."""
+    return (
+        isinstance(object, (type, *_generics_types))
+        or is_new_type(object)
+        or is_typed_dict(object)
+    )
 
 
 def is_type_dict(object):
@@ -215,34 +242,31 @@ def parse_type_string(type_string: str) -> typing.Any:
     and basic Python data types. It also defines a list of safe tokens that are allowed
     in the type string.
     """
-    safe_context = {
-        "Any": typing.Any,
-        "List": typing.List,
-        "Dict": typing.Dict,
-        "Tuple": typing.Tuple,
-        "Union": typing.Union,
-        "int": int,
-        "str": str,
-        "float": float,
-        "bool": bool,
-        "Optional": typing.Optional,
-    }
-
     type_string = format_type_string(type_string)
 
-    safe_tokens = ["[", "]", ",", " "]
-    return safe_eval(type_string, safe_context, safe_tokens)
+    return safe_eval(
+        type_string, context=_registered_types, allowed_tokens=["[", "]", ",", " "]
+    )
+
+
+def replace_class_names(full_string: str) -> str:
+    # Regular expression to match any fully qualified class name and extract the class name
+    pattern = r"(?:\w+\.)*<locals>\.(\w+)|(?:\w+\.)*(\w+)"
+
+    # Function to replace the matched pattern with just the class name
+    def replacement(match):
+        # If the match has a group for <locals>
+        if match.group(1):
+            return match.group(1)
+        # Otherwise, return the last group (class name)
+        return match.group(2)
+
+    # Use re.sub to replace all occurrences in the string
+    return re.sub(pattern, replacement, full_string)
 
 
 def to_type_string(typing_type):
-    if not is_type(typing_type):
-        raise UnsupportedTypeError(typing_type)
-    type_string = (
-        str(typing_type)
-        .replace("typing.", "")
-        .replace("<class '", "")
-        .replace("'>", "")
-    )
+    type_string = strtype(typing_type)
     assert parse_type_string(type_string), "Is not parsed well"
     return type_string
 
@@ -447,9 +471,9 @@ def infer_type_string(obj: typing.Any) -> str:
 def isoftype(object, typing_type):
     """Checks if an object is of a certain typing type, including nested types.
 
-    This function supports simple types (like `int`, `str`), typing types
-    (like `List[int]`, `Tuple[str, int]`, `Dict[str, int]`), and nested typing
-    types (like `List[List[int]]`, `Tuple[List[str], int]`, `Dict[str, List[int]]`).
+    This function supports simple types, typing types (List[int], Tuple[str, int]),
+    nested typing types (List[List[int]], Tuple[List[str], int]), Literal, TypedDict,
+    and NewType.
 
     Args:
         object: The object to check.
@@ -457,19 +481,21 @@ def isoftype(object, typing_type):
 
     Returns:
         bool: True if the object is of the specified type, False otherwise.
-
-    Examples:
-    .. highlight:: python
-    .. code-block:: python
-
-        isoftype(1, int) # True
-        isoftype([1, 2, 3], typing.List[int]) # True
-        isoftype([1, 2, 3], typing.List[str]) # False
-        isoftype([[1, 2], [3, 4]], typing.List[typing.List[int]]) # True
     """
     if not is_type(typing_type):
         raise UnsupportedTypeError(typing_type)
 
+    if is_new_type(typing_type):
+        typing_type = typing_type.__supertype__
+
+    if is_typed_dict(typing_type):
+        if not isinstance(object, dict):
+            return False
+        for key, expected_type in typing_type.__annotations__.items():
+            if key not in object or not isoftype(object[key], expected_type):
+                return False
+        return True
+
     if typing_type == typing.Any:
         return True
 
@@ -477,15 +503,16 @@ def isoftype(object, typing_type):
         origin = typing_type.__origin__
         type_args = typing.get_args(typing_type)
 
+        if origin is Literal:
+            return object in type_args
+
         if origin is typing.Union:
             return any(isoftype(object, sub_type) for sub_type in type_args)
 
         if not isinstance(object, origin):
             return False
-
         if origin is list or origin is set:
             return all(isoftype(element, type_args[0]) for element in object)
-
         if origin is dict:
             return all(
                 isoftype(key, type_args[0]) and isoftype(value, type_args[1])
@@ -496,11 +523,77 @@ def isoftype(object, typing_type):
                 isoftype(element, type_arg)
                 for element, type_arg in zip(object, type_args)
             )
-        return None
 
     return isinstance(object, typing_type)
 
 
+def strtype(typing_type) -> str:
+    """Converts a typing type to its string representation.
+
+    Args:
+        typing_type (Any): The typing type to be converted. This can include standard types,
+            custom types, or types from the `typing` module, such as `Literal`, `Union`,
+            `List`, `Dict`, `Tuple`, `TypedDict`, and `NewType`.
+
+    Returns:
+        str: The string representation of the provided typing type.
+
+    Raises:
+        UnsupportedTypeError: If the provided `typing_type` is not a recognized type.
+
+    Notes:
+        - If `typing_type` is `Literal`, `NewType`, or `TypedDict`, the function returns
+          the name of the type.
+        - If `typing_type` is `Any`, it returns the string `"Any"`.
+        - For other typing constructs like `Union`, `List`, `Dict`, and `Tuple`, the function
+          recursively converts each part of the type to its string representation.
+        - The function checks the `__origin__` attribute to determine the base type and formats
+          the type arguments accordingly.
+    """
+    if not is_type(typing_type):
+        raise UnsupportedTypeError(typing_type)
+
+    if is_new_type(typing_type) or is_typed_dict(typing_type):
+        return typing_type.__name__
+
+    if typing_type == typing.Any:
+        return "Any"
+
+    if hasattr(typing_type, "__origin__"):
+        origin = typing_type.__origin__
+        type_args = typing.get_args(typing_type)
+
+        if type_args[-1] is type(None):
+            return (
+                "Optional["
+                + ", ".join([strtype(sub_type) for sub_type in type_args[:-1]])
+                + "]"
+            )
+
+        if origin is Literal:
+            return str(typing_type).replace("typing.", "")
+        if origin is typing.Union:
+            return (
+                "Union["
+                + ", ".join([strtype(sub_type) for sub_type in type_args])
+                + "]"
+            )
+        if origin is list or origin is set:
+            return "List[" + strtype(type_args[0]) + "]"
+        if origin is set:
+            return "Set[" + strtype(type_args[0]) + "]"
+        if origin is dict:
+            return "Dict[" + strtype(type_args[0]) + ", " + strtype(type_args[1]) + "]"
+        if origin is tuple:
+            return (
+                "Tuple["
+                + ", ".join([strtype(sub_type) for sub_type in type_args])
+                + "]"
+            )
+
+    return typing_type.__name__
+
+
 # copied from: https://github.com/bojiang/typing_utils/blob/main/typing_utils/__init__.py
 # liscened under Apache License 2.0
 

types.py

@@ -0,0 +1,36 @@
+from typing import Any, List, Literal, NewType, TypedDict, Union
+
+from .type_utils import register_type
+
+Text = NewType("Text", str)
+Number = NewType("Number", Union[float, int])
+
+
+class Turn(TypedDict):
+    role: Literal["system", "user", "agent"]
+    content: Text
+
+
+Dialog = NewType("Dialog", List[Turn])
+
+
+class Image(TypedDict):
+    image: Any
+
+
+class Audio(TypedDict):
+    audio: Any
+
+
+class Table(TypedDict):
+    header: List[str]
+    rows: List[List[Any]]
+
+
+register_type(Text)
+register_type(Number)
+register_type(Turn)
+register_type(Dialog)
+register_type(Table)
+register_type(Audio)
+register_type(Image)

utils.py

@@ -2,6 +2,7 @@ import copy
 import importlib.util
 import json
 import os
+import re
 from functools import lru_cache
 from typing import Any, Dict
 
@@ -87,6 +88,23 @@ def is_module_available(module_name):
         return False
 
 
+def remove_numerics_and_quoted_texts(input_str):
+    # Remove floats first to avoid leaving stray periods
+    input_str = re.sub(r"\d+\.\d+", "", input_str)
+
+    # Remove integers
+    input_str = re.sub(r"\d+", "", input_str)
+
+    # Remove strings in single quotes
+    input_str = re.sub(r"'.*?'", "", input_str)
+
+    # Remove strings in double quotes
+    input_str = re.sub(r'".*?"', "", input_str)
+
+    # Remove strings in triple quotes
+    return re.sub(r'""".*?"""', "", input_str, flags=re.DOTALL)
+
+
 def safe_eval(expression: str, context: dict, allowed_tokens: list) -> any:
     """Evaluates a given expression in a restricted environment, allowing only specified tokens and context variables.
 
@@ -109,7 +127,9 @@ def safe_eval(expression: str, context: dict, allowed_tokens: list) -> any:
         by restricting the available tokens and not exposing built-in functions.
     """
     allowed_sub_strings = list(context.keys()) + allowed_tokens
-    if is_made_of_sub_strings(expression, allowed_sub_strings):
+    if is_made_of_sub_strings(
+        remove_numerics_and_quoted_texts(expression), allowed_sub_strings
+    ):
         return eval(expression, {"__builtins__": {}}, context)
     raise ValueError(
         f"The expression '{expression}' can not be evaluated because it contains tokens outside the allowed list of {allowed_sub_strings}."

version.py

@@ -1 +1 @@
-version = "1.12.4"
+version = "1.13.0"