Repository: iai-group/DialogueKit (language: python)

Issue:
Create annotation file converters
Create an abstract class for annotation converters, and also implement a converter from dialoguekit/moviebot data to rasa specific format. In addition create files that may be useful as *entity type* to *entity examples*, etc.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 68959aa9..e67d9174 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,6 @@
 .pyc
 __pycache__/
 .vscode/
+.rasa/*
+moviebot/*
+testing/*
\ No newline at end of file
diff --git a/dialoguekit/utils/annotation_converter.py b/dialoguekit/utils/annotation_converter.py
new file mode 100644
index 00000000..e7bc4fce
--- /dev/null
+++ b/dialoguekit/utils/annotation_converter.py
@@ -0,0 +1,40 @@
+"""Annotation converter interface.
+
+As the different modules used for NLU use different formats for training file
+converters are needed."""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+
+class AnnotationConverter(ABC):
+    """Annotation converter interface.
+
+    Attributes:
+        filepath: String path to the original annotation file.
+        save_to_path: Optional String path to the destination directory.
+    """
+
+    def __init__(
+        self, filepath: str, save_to_path: Optional[str] = None
+    ) -> None:
+        self._filepath = filepath
+        self._save_to_path = save_to_path
+        self._converted = {}
+
+    @abstractmethod
+    def run(self) -> Dict[str, str]:
+        """Runs the converter.
+
+        The conversion will run and save to 'save_to_path' directory. If the
+        converter supports different file outputs, this method will run all the
+        supported conversions.
+
+        Returns:
+            Dictionary with filename and filepath pairs.
+
+        Raises:
+            NotImplementedError
+        """
+
+        raise NotImplementedError
diff --git a/dialoguekit/utils/annotation_converter_dialoguekit_to_rasa.py b/dialoguekit/utils/annotation_converter_dialoguekit_to_rasa.py
new file mode 100644
index 00000000..2ebdf9d2
--- /dev/null
+++ b/dialoguekit/utils/annotation_converter_dialoguekit_to_rasa.py
@@ -0,0 +1,174 @@
+import json
+import yaml
+from collections import defaultdict
+from yaml.representer import SafeRepresenter
+from yaml.nodes import ScalarNode
+from typing import Optional, Dict, List
+
+from dialoguekit.utils.annotation_converter import AnnotationConverter
+
+
+class AnnotationConverterRasa(AnnotationConverter):
+    def __init__(
+        self, filepath: str, save_to_path: Optional[str] = None
+    ) -> None:
+        super().__init__(filepath, save_to_path)
+        self._intent_examples = {"USER": {}, "AGENT": {}}
+        self._slot_value_pairs = defaultdict(set)
+        self._data = {}
+
+    def read_original(self) -> None:
+        f = open(self._filepath)
+        data = json.load(f)
+        self._data["original"] = data
+
+        for conversation in data:
+            for turn in conversation["conversation"]:
+                intent = turn.get("intent", None)
+                slot_values = turn.get("slot_values", [])
+                utterance = turn.get("utterance", "").strip()
+                if len(slot_values) > 0:
+                    # Utterance with annotation
+                    turn["utterance_annotated"] = utterance
+                    for pair in slot_values:
+                        turn["utterance_annotated"] = turn[
+                            "utterance_annotated"
+                        ].replace(pair[1], "[{}]({})".format(pair[1], pair[0]))
+
+                        # Annotation types with examples
+                        self._slot_value_pairs[pair[0]].add(pair[1])
+
+                # Intent with examples
+                if intent not in self._intent_examples[turn["participant"]]:
+                    self._intent_examples[turn["participant"]][intent] = set()
+
+                self._intent_examples[turn["participant"]][intent].add(
+                    turn.get("utterance_annotated", utterance)
+                )
+
+        self._slot_value_pairs = {
+            k: list(v) for k, v in self._slot_value_pairs.items()
+        }
+        self._intent_examples = {
+            k: {i: list(l) for i, l in v.items()}
+            for k, v in self._intent_examples.items()
+        }
+
+    def run(self) -> Dict[str, str]:
+        """Generates 4 coversions of DialogueKit to Rasa compatible files.
+
+        The generated files are saved in the self._save_to_path.
+
+        Genereted files:
+            1. <originalname>_reformat.yml
+                The original file saved as a yml
+
+            2. <originalname>_types_w_examples.yml
+                The entity types with the corresponding entities identified in
+                the text. Used to debug and give an overview
+
+            3. <originalname>_rasa_user.yml
+                Conversion of the annotation file to a rasa nlu doc.
+                This file represents the USER utterances with intents
+
+            4. <originalname>_rasa_agent.yml
+                Conversion of the annotation file to a rasa nlu doc.
+                This file represents the AGENT utterances with intents
+
+        Returns:
+            Filename: path to file pairs
+        """
+
+        if len(self._slot_value_pairs.values()) <= 0:
+            raise TypeError(
+                "Your need to use the read_original() "
+                "function before running run()"
+            )
+
+        return_dictionary = {}
+        save_path_name = (
+            self._save_to_path + self._filepath.split("/")[-1].split(".")[-2]
+        )
+        save_name_base = self._filepath.split("/")[-1].split(".")[-2]
+
+        # Save original as yml
+        extention = "_reformat.yml"
+        filename = save_name_base + extention
+        return_dictionary[filename] = save_path_name + extention
+        with open(return_dictionary[filename], "w") as outfile:
+            yaml.dump(self._data["original"], outfile, default_flow_style=False)
+
+        # Save the intent types with examples
+        extention = "_types_w_examples.yml"
+        filename = save_name_base + extention
+        return_dictionary[filename] = save_path_name + extention
+        with open(return_dictionary[filename], "w") as outfile:
+            yaml.dump(self._slot_value_pairs, outfile, default_flow_style=False)
+
+        # Used for yml formating
+        class LiteralString(str):
+            pass
+
+        def change_style(style: str, representer: ScalarNode):
+            """Used to change the python yml data representation
+
+            Args:
+                style (str): Style used to represent type
+                representer (ScalarNode): representer type ->
+                            (SafeRepresenter.represent_str)
+            """
+
+            def new_representer(dumper, data):
+                scalar = representer(dumper, data)
+                scalar.style = style
+                return scalar
+
+            return new_representer
+
+        represent_literal_list = change_style(
+            "|", SafeRepresenter.represent_str
+        )
+        yaml.add_representer(LiteralString, represent_literal_list)
+
+        def rasa_string(v: List[str]) -> str:
+            formated_string = LiteralString(
+                "".join(["- " + s.strip() + "\n" for i, s in enumerate(v)])
+            )
+            return formated_string
+
+        # Create dict with rasa compatible format
+        rasa_dict_user = {"version": "3.0", "nlu": []}
+        rasa_dict_agent = {"version": "3.0", "nlu": []}
+        for k, v in self._intent_examples["USER"].items():
+            formated_dict = {"intent": k, "examples": rasa_string(v)}
+            rasa_dict_user["nlu"].append(formated_dict)
+
+        for k, v in self._intent_examples["AGENT"].items():
+            formated_dict = {"intent": k, "examples": rasa_string(v)}
+            rasa_dict_agent["nlu"].append(formated_dict)
+
+        # Save rasa compatible format
+        extention = "_rasa_user.yml"
+        filename = save_name_base + extention
+        return_dictionary[filename] = save_path_name + extention
+        with open(return_dictionary[filename], "w") as outfile:
+            yaml.dump(rasa_dict_user, outfile, default_flow_style=False)
+
+        extention = "_rasa_agent.yml.yml"
+        filename = save_name_base + extention
+        return_dictionary[filename] = save_path_name + extention
+        with open(return_dictionary[filename], "w") as outfile:
+            yaml.dump(rasa_dict_agent, outfile, default_flow_style=False)
+
+        return return_dictionary
+
+
+if __name__ == "__main__":
+    converter = AnnotationConverterRasa(
+        filepath="/Users/aleksanderdrzewiecki/Documents/GitHub/"
+        + "dialoguekit/moviebot/annotated_dialogues.json",
+        save_to_path="/Users/aleksanderdrzewiecki/Documents/GitHub/"
+        "dialoguekit/testing/",
+    )
+    converter.read_original()
+    converter.run()


Test patch:
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/utils/test_annotation_converter_dialoguekit_to_rasa.py b/tests/utils/test_annotation_converter_dialoguekit_to_rasa.py
new file mode 100644
index 00000000..7db1d7aa
--- /dev/null
+++ b/tests/utils/test_annotation_converter_dialoguekit_to_rasa.py
@@ -0,0 +1,49 @@
+from dialoguekit.utils.annotation_converter_dialoguekit_to_rasa import (
+    AnnotationConverterRasa,
+)
+
+
+def test_read_original(tmp_path):
+    save_to_dir = tmp_path
+    # save_to_dir.mkdir()
+    full_path = save_to_dir.absolute()
+    my_path = full_path.as_posix()
+
+    converter = AnnotationConverterRasa(
+        filepath="tests/data/annotated_dialogues.json",
+        save_to_path=my_path + "/",
+    )
+    converter.read_original()
+
+    # Check slot value pairs
+    assert len(converter._slot_value_pairs.keys()) == 5
+    assert len(list(converter._slot_value_pairs.values())[0]) == 16
+
+    # Check intent examples
+    assert len(converter._intent_examples.keys()) == 2
+    assert "USER" in converter._intent_examples.keys()
+    assert "AGENT" in converter._intent_examples.keys()
+    assert len(converter._intent_examples["USER"].keys()) == 10
+    assert len(converter._intent_examples["AGENT"].keys()) == 9
+
+
+def test_run(tmp_path):
+    save_to_dir = tmp_path
+    full_path = save_to_dir.absolute()
+    my_path = full_path.as_posix()
+
+    converter = AnnotationConverterRasa(
+        filepath="tests/data/annotated_dialogues.json",
+        save_to_path=my_path + "/",
+    )
+
+    converter.read_original()
+    converter.run()
+
+    p = save_to_dir.glob("**/*")
+    files = [x for x in p if x.is_file()]
+
+    # Check amount of docs
+    assert len(files) == 4
+    # TODO validate generated yml files.
+    # https://github.com/iai-group/dialoguekit/issues/57