Upload 34 files

app.py

@@ -0,0 +1,42 @@
+import gradio as gr
+import os
+from common.config import Config
+from common.model_manager import ModelManager
+
+config = Config.load_from_yaml("config/app.yaml")
+model_manager = ModelManager(config)
+model_manager.load()
+
+
+def text_analysis(text):
+    print(text)
+    data = model_manager.predict(text)
+    html = """<link href="https://cdn.staticfile.org/twitter-bootstrap/5.1.1/css/bootstrap.min.css" rel="stylesheet">
+                <script src="https://cdn.staticfile.org/twitter-bootstrap/5.1.1/js/bootstrap.bundle.min.js"></script>"""
+    html += """<div style="background: white; padding: 16px;"><b>Intent:</b>"""
+
+    for intent in data["intent"]:
+        html += """<button type="button" class="btn btn-white">
+                        <span class="badge text-dark btn-light">""" + intent + """</span> </button>"""
+    html += """<br /> <b>Slot:</b>"""
+    for t, slot in zip(data["text"], data["slot"]):
+        html += """<button type="button" class="btn btn-white">"""+t+"""<span class="badge text-dark" style="background-color: rgb(255, 255, 255);
+                            color: rgb(62 62 62);
+                            box-shadow: 2px 2px 7px 1px rgba(210, 210, 210, 0.42);">"""+slot+\
+                            """</span>
+                    </button>"""
+    html+="</div>"
+    return html
+
+
+demo = gr.Interface(
+    text_analysis,
+    gr.Textbox(placeholder="Enter sentence here..."),
+    ["html"],
+    examples=[
+        ["What a beautiful morning for a walk!"],
+        ["It was the best of times, it was the worst of times."],
+    ],
+)
+
+demo.launch()

common/__init__.py

@@ -0,0 +1 @@
+

common/config.py

@@ -0,0 +1,191 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-26 10:55:43
+Description: Configuration class to manage all process in OpenSLU like model construction, learning processing and so on.
+
+'''
+import re
+
+from ruamel import yaml
+import datetime
+
+class Config(dict):
+    def __init__(self, *args, **kwargs):
+        """ init with dict as args
+        """
+        dict.__init__(self, *args, **kwargs)
+        self.__dict__ = self
+        self.start_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')
+        self.__autowired()
+
+    @staticmethod
+    def load_from_yaml(file_path:str)->"Config":
+        """load config files with path
+
+        Args:
+            file_path (str): yaml configuration file path.
+
+        Returns:
+            Config: config object.
+        """
+        with open(file_path) as stream:
+            try:
+                return Config(yaml.safe_load(stream))
+            except yaml.YAMLError as exc:
+                print(exc)
+
+    @staticmethod
+    def load_from_args(args)->"Config":
+        """ load args to replace item value in config files assigned with '--config_path' or '--model'
+
+        Args:
+            args (Any): args with command line.
+
+        Returns:
+            Config: _description_
+        """
+        if args.model is not None:
+            args.config_path = "config/" + args.model + ".yaml"
+        config = Config.load_from_yaml(args.config_path)
+        if args.dataset is not None:
+            config.__update_dataset(args.dataset)
+        if args.device is not None:
+            config["base"]["device"] = args.device
+        if args.learning_rate is not None:
+            config["optimizer"]["lr"] = args.learning_rate
+        if args.epoch_num is not None:
+            config["base"]["epoch_num"] = args.epoch_num
+        return config
+
+    def autoload_template(self):
+        """ search '{*}' template to excute as python code, support replace variable as any configure item
+        """
+        self.__autoload_template(self.__dict__)
+
+    def __get_autoload_value(self, matched):
+        keys = matched.group()[1:-1].split(".")
+        temp = self.__dict__
+        for k in keys:
+            temp = temp[k]
+        return str(temp)
+
+    def __autoload_template(self, config:dict):
+        for k in config:
+            if isinstance(config, dict):
+                sub_config = config[k]
+            elif isinstance(config, list):
+                sub_config = k
+            else:
+                continue
+            if isinstance(sub_config, dict) or isinstance(sub_config, list):
+                self.__autoload_template(sub_config)
+            if isinstance(sub_config, str) and "{" in sub_config and "}" in sub_config:
+                res = re.sub(r'{.*?}', self.__get_autoload_value, config[k])
+                res_dict= {"res": None}
+                exec("res=" + res, res_dict)
+                config[k] = res_dict["res"]
+
+    def __update_dataset(self, dataset_name):
+        if dataset_name is not None and isinstance(dataset_name, str):
+            self.__dict__["dataset"]["dataset_name"] = dataset_name
+
+    def get_model_config(self):
+        return self.__dict__["model"]
+
+    def __autowired(self):
+        # Set encoder
+        encoder_config = self.__dict__["model"]["encoder"]
+        encoder_type = encoder_config["_model_target_"].split(".")[-1]
+
+        def get_output_dim(encoder_config):
+            encoder_type = encoder_config["_model_target_"].split(".")[-1]
+            if (encoder_type == "AutoEncoder" and encoder_config["encoder_name"] in ["lstm", "self-attention-lstm",
+                                                                                     "bi-encoder"]) or encoder_type == "NoPretrainedEncoder":
+                output_dim = 0
+                if encoder_config.get("lstm"):
+                    output_dim += encoder_config["lstm"]["output_dim"]
+                if encoder_config.get("attention"):
+                    output_dim += encoder_config["attention"]["output_dim"]
+                return output_dim
+            else:
+                return encoder_config["output_dim"]
+
+        if encoder_type == "BiEncoder":
+            output_dim = get_output_dim(encoder_config["intent_encoder"]) + \
+                         get_output_dim(encoder_config["slot_encoder"])
+        else:
+            output_dim = get_output_dim(encoder_config)
+        self.__dict__["model"]["encoder"]["output_dim"] = output_dim
+
+        # Set interaction
+        if "interaction" in self.__dict__["model"]["decoder"] and self.__dict__["model"]["decoder"]["interaction"].get(
+                "input_dim") is None:
+            self.__dict__["model"]["decoder"]["interaction"]["input_dim"] = output_dim
+            interaction_type = self.__dict__["model"]["decoder"]["interaction"]["_model_target_"].split(".")[-1]
+            if not ((encoder_type == "AutoEncoder" and encoder_config[
+                "encoder_name"] == "self-attention-lstm") or encoder_type == "SelfAttentionLSTMEncoder") and interaction_type != "BiModelWithoutDecoderInteraction":
+                output_dim = self.__dict__["model"]["decoder"]["interaction"]["output_dim"]
+
+        # Set classifier
+        if "slot_classifier" in self.__dict__["model"]["decoder"]:
+            if self.__dict__["model"]["decoder"]["slot_classifier"].get("input_dim") is None:
+                self.__dict__["model"]["decoder"]["slot_classifier"]["input_dim"] = output_dim
+            self.__dict__["model"]["decoder"]["slot_classifier"]["use_slot"] = True
+        if "intent_classifier" in self.__dict__["model"]["decoder"]:
+            if self.__dict__["model"]["decoder"]["intent_classifier"].get("input_dim") is None:
+                self.__dict__["model"]["decoder"]["intent_classifier"]["input_dim"] = output_dim
+            self.__dict__["model"]["decoder"]["intent_classifier"]["use_intent"] = True
+
+    def get_intent_label_num(self):
+        """ get the number of intent labels.
+        """
+        classifier_conf = self.__dict__["model"]["decoder"]["intent_classifier"]
+        return classifier_conf["intent_label_num"] if "intent_label_num" in classifier_conf else 0
+
+    def get_slot_label_num(self):
+        """ get the number of slot labels.
+        """
+        classifier_conf = self.__dict__["model"]["decoder"]["slot_classifier"]
+        return classifier_conf["slot_label_num"] if "slot_label_num" in classifier_conf else 0
+
+    def set_intent_label_num(self, intent_label_num):
+        """ set the number of intent labels.
+        
+        Args:
+            slot_label_num (int): the number of intent label
+        """
+        self.__dict__["base"]["intent_label_num"] = intent_label_num
+        self.__dict__["model"]["decoder"]["intent_classifier"]["intent_label_num"] = intent_label_num
+        if "interaction" in self.__dict__["model"]["decoder"]:
+
+            self.__dict__["model"]["decoder"]["interaction"]["intent_label_num"] = intent_label_num
+            if self.__dict__["model"]["decoder"]["interaction"]["_model_target_"].split(".")[
+                -1] == "StackInteraction":
+                self.__dict__["model"]["decoder"]["slot_classifier"]["input_dim"] += intent_label_num
+
+    
+    def set_slot_label_num(self, slot_label_num:int)->None:
+        """set the number of slot label
+        
+        Args:
+            slot_label_num (int): the number of slot label
+        """
+        self.__dict__["base"]["slot_label_num"] = slot_label_num
+        self.__dict__["model"]["decoder"]["slot_classifier"]["slot_label_num"] = slot_label_num
+        if "interaction" in self.__dict__["model"]["decoder"]:
+            self.__dict__["model"]["decoder"]["interaction"]["slot_label_num"] = slot_label_num
+
+    def set_vocab_size(self, vocab_size):
+        """set the size of vocabulary in non-pretrained tokenizer
+        Args:
+            slot_label_num (int): the number of slot label
+        """
+        encoder_type = self.__dict__["model"]["encoder"]["_model_target_"].split(".")[-1]
+        encoder_name = self.__dict__["model"]["encoder"].get("encoder_name")
+        if encoder_type == "BiEncoder" or (encoder_type == "AutoEncoder" and encoder_name == "bi-encoder"):
+            self.__dict__["model"]["encoder"]["intent_encoder"]["embedding"]["vocab_size"] = vocab_size
+            self.__dict__["model"]["encoder"]["slot_encoder"]["embedding"]["vocab_size"] = vocab_size
+        elif self.__dict__["model"]["encoder"].get("embedding"):
+            self.__dict__["model"]["encoder"]["embedding"]["vocab_size"] = vocab_size

common/loader.py

@@ -0,0 +1,333 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-02-07 19:26:06
+Description: all class for load data.
+
+'''
+import os
+import torch
+import json
+from datasets import load_dataset, Dataset
+from torch.utils.data import DataLoader
+
+from common.utils import InputData
+
+ABS_PATH=os.path.join(os.path.abspath(os.path.dirname(__file__)), "../")
+
+class DataFactory(object):
+    def __init__(self, tokenizer,use_multi_intent=False, to_lower_case=True):
+        """_summary_
+
+        Args:
+            tokenizer (Tokenizer): _description_
+            use_multi_intent (bool, optional): _description_. Defaults to False.
+        """
+        self.tokenizer = tokenizer
+        self.slot_label_list = []
+        self.intent_label_list = []
+        self.use_multi = use_multi_intent
+        self.to_lower_case = to_lower_case
+        self.slot_label_dict = None
+        self.intent_label_dict = None
+
+    def __is_supported_datasets(self, dataset_name:str)->bool:
+        return dataset_name.lower() in ["atis", "snips", "mix-atis", "mix-atis"]
+
+    def load_dataset(self, dataset_config, split="train"):
+        # TODO: 关闭use_auth_token
+        dataset_name = None
+        if split not in dataset_config:
+            dataset_name = dataset_config.get("dataset_name")
+        elif self.__is_supported_datasets(dataset_config[split]):
+            dataset_name = dataset_config[split].lower()
+        if dataset_name is not None:
+            return load_dataset("LightChen2333/OpenSLU", dataset_name, split=split, use_auth_token=True)
+        else:
+            data_file = dataset_config[split]
+            data_dict = {"text": [], "slot": [], "intent":[]}
+            with open(data_file, encoding="utf-8") as f:
+                for line in f:
+                    row = json.loads(line)
+                    data_dict["text"].append(row["text"])
+                    data_dict["slot"].append(row["slot"])
+                    data_dict["intent"].append(row["intent"])
+            return Dataset.from_dict(data_dict)
+
+    def update_label_names(self, dataset):
+        for intent_labels in dataset["intent"]:
+            if self.use_multi:
+                intent_label = intent_labels.split("#")
+            else:
+                intent_label = [intent_labels]
+            for x in intent_label:
+                if x not in self.intent_label_list:
+                    self.intent_label_list.append(x)
+        for slot_label in dataset["slot"]:
+            for x in slot_label:
+                if x not in self.slot_label_list:
+                    self.slot_label_list.append(x)
+        self.intent_label_dict = {key: index for index,
+                                  key in enumerate(self.intent_label_list)}
+        self.slot_label_dict = {key: index for index,
+                                key in enumerate(self.slot_label_list)}
+
+    def update_vocabulary(self, dataset):
+        if self.tokenizer.name_or_path in ["word_tokenizer"]:
+            for data in dataset:
+                self.tokenizer.add_instance(data["text"])
+
+    @staticmethod
+    def fast_align_data(text, padding_side="right"):
+        for i in range(len(text.input_ids)):
+            desired_output = []
+            for word_id in text.word_ids(i):
+                if word_id is not None:
+                    start, end = text.word_to_tokens(
+                        i, word_id, sequence_index=0 if padding_side == "right" else 1)
+                    if start == end - 1:
+                        tokens = [start]
+                    else:
+                        tokens = [start, end - 1]
+                    if len(desired_output) == 0 or desired_output[-1] != tokens:
+                        desired_output.append(tokens)
+            yield desired_output
+
+    def fast_align(self,
+                   batch,
+                   ignore_index=-100,
+                   device="cuda",
+                   config=None,
+                   enable_label=True,
+                   label2tensor=True):
+        if self.to_lower_case:
+            input_list = [[t.lower() for t in x["text"]] for x in batch]
+        else:
+            input_list = [x["text"] for x in batch]
+        text = self.tokenizer(input_list,
+                              return_tensors="pt",
+                              padding=True,
+                              is_split_into_words=True,
+                              truncation=True,
+                              **config).to(device)
+        if enable_label:
+            if label2tensor:
+
+                slot_mask = torch.ones_like(text.input_ids) * ignore_index
+                for i, offsets in enumerate(
+                        DataFactory.fast_align_data(text, padding_side=self.tokenizer.padding_side)):
+                    num = 0
+                    assert len(offsets) == len(batch[i]["text"])
+                    assert len(offsets) == len(batch[i]["slot"])
+                    for off in offsets:
+                        slot_mask[i][off[0]
+                                     ] = self.slot_label_dict[batch[i]["slot"][num]]
+                        num += 1
+                slot = slot_mask.clone()
+                attentin_id = 0 if self.tokenizer.padding_side == "right" else 1
+                for i, slot_batch in enumerate(slot):
+                    for j, x in enumerate(slot_batch):
+                        if x == ignore_index and text.attention_mask[i][j] == attentin_id and (text.input_ids[i][
+                                j] not in self.tokenizer.all_special_ids or text.input_ids[i][j] == self.tokenizer.unk_token_id):
+                            slot[i][j] = slot[i][j - 1]
+                slot = slot.to(device)
+                if not self.use_multi:
+                    intent = torch.tensor(
+                        [self.intent_label_dict[x["intent"]] for x in batch]).to(device)
+                else:
+                    one_hot = torch.zeros(
+                        (len(batch), len(self.intent_label_list)), dtype=torch.float)
+                    for index, b in enumerate(batch):
+                        for x in b["intent"].split("#"):
+                            one_hot[index][self.intent_label_dict[x]] = 1.
+                    intent = one_hot.to(device)
+            else:
+                slot_mask = None
+                slot = [['#' for _ in range(text.input_ids.shape[1])]
+                        for _ in range(text.input_ids.shape[0])]
+                for i, offsets in enumerate(DataFactory.fast_align_data(text)):
+                    num = 0
+                    for off in offsets:
+                        slot[i][off[0]] = batch[i]["slot"][num]
+                        num += 1
+                if not self.use_multi:
+                    intent = [x["intent"] for x in batch]
+                else:
+                    intent = [
+                        [x for x in b["intent"].split("#")] for b in batch]
+            return InputData((text, slot, intent))
+        else:
+            return InputData((text, None, None))
+
+    def general_align_data(self, split_text_list, raw_text_list, encoded_text):
+        for i in range(len(split_text_list)):
+            desired_output = []
+            jdx = 0
+            offset = encoded_text.offset_mapping[i].tolist()
+            split_texts = split_text_list[i]
+            raw_text = raw_text_list[i]
+            last = 0
+            temp_offset = []
+            for off in offset:
+                s, e = off
+                if len(temp_offset) > 0 and (e != 0 and last == s):
+                    len_1 = off[1] - off[0]
+                    len_2 = temp_offset[-1][1] - temp_offset[-1][0]
+                    if len_1 > len_2:
+                        temp_offset.pop(-1)
+                        temp_offset.append([0, 0])
+                        temp_offset.append(off)
+                    continue
+                temp_offset.append(off)
+                last = s
+            offset = temp_offset
+            for split_text in split_texts:
+                while jdx < len(offset) and offset[jdx][0] == 0 and offset[jdx][1] == 0:
+                    jdx += 1
+                if jdx == len(offset):
+                    continue
+                start_, end_ = offset[jdx]
+                tokens = None
+                if split_text == raw_text[start_:end_].strip():
+                    tokens = [jdx]
+                else:
+                    # Compute "xxx" -> "xx" "#x"
+                    temp_jdx = jdx
+                    last_str = raw_text[start_:end_].strip()
+                    while last_str != split_text and temp_jdx < len(offset) - 1:
+                        temp_jdx += 1
+                        last_str += raw_text[offset[temp_jdx]
+                                             [0]:offset[temp_jdx][1]].strip()
+
+                    if temp_jdx == jdx:
+                        raise ValueError("Illegal Input data")
+                    elif last_str == split_text:
+                        tokens = [jdx, temp_jdx]
+                        jdx = temp_jdx
+                    else:
+                        jdx -= 1
+                jdx += 1
+                if tokens is not None:
+                    desired_output.append(tokens)
+            yield desired_output
+
+    def general_align(self,
+                      batch,
+                      ignore_index=-100,
+                      device="cuda",
+                      config=None,
+                      enable_label=True,
+                      label2tensor=True,
+                      locale="en-US"):
+        if self.to_lower_case:
+            raw_data = [" ".join(x["text"]).lower() if locale not in ['ja-JP', 'zh-CN', 'zh-TW'] else "".join(x["text"]) for x in
+                    batch]
+            input_list = [[t.lower() for t in x["text"]] for x in batch]
+        else:
+            input_list = [x["text"] for x in batch]
+            raw_data = [" ".join(x["text"]) if locale not in ['ja-JP', 'zh-CN', 'zh-TW'] else "".join(x["text"]) for x in
+                        batch]
+        text = self.tokenizer(raw_data,
+                              return_tensors="pt",
+                              padding=True,
+                              truncation=True,
+                              return_offsets_mapping=True,
+                              **config).to(device)
+        if enable_label:
+            if label2tensor:
+                slot_mask = torch.ones_like(text.input_ids) * ignore_index
+                for i, offsets in enumerate(
+                        self.general_align_data(input_list, raw_data, encoded_text=text)):
+                    num = 0
+                    # if len(offsets) != len(batch[i]["text"]) or len(offsets) != len(batch[i]["slot"]):
+                    #     if
+                    for off in offsets:
+                        slot_mask[i][off[0]
+                                     ] = self.slot_label_dict[batch[i]["slot"][num]]
+                        num += 1
+                # slot = slot_mask.clone()
+                # attentin_id = 0 if self.tokenizer.padding_side == "right" else 1
+                # for i, slot_batch in enumerate(slot):
+                #     for j, x in enumerate(slot_batch):
+                #         if x == ignore_index and text.attention_mask[i][j] == attentin_id and text.input_ids[i][
+                #             j] not in self.tokenizer.all_special_ids:
+                #             slot[i][j] = slot[i][j - 1]
+                slot = slot_mask.to(device)
+                if not self.use_multi:
+                    intent = torch.tensor(
+                        [self.intent_label_dict[x["intent"]] for x in batch]).to(device)
+                else:
+                    one_hot = torch.zeros(
+                        (len(batch), len(self.intent_label_list)), dtype=torch.float)
+                    for index, b in enumerate(batch):
+                        for x in b["intent"].split("#"):
+                            one_hot[index][self.intent_label_dict[x]] = 1.
+                    intent = one_hot.to(device)
+            else:
+                slot_mask = None
+                slot = [['#' for _ in range(text.input_ids.shape[1])]
+                        for _ in range(text.input_ids.shape[0])]
+                for i, offsets in enumerate(self.general_align_data(input_list, raw_data, encoded_text=text)):
+                    num = 0
+                    for off in offsets:
+                        slot[i][off[0]] = batch[i]["slot"][num]
+                        num += 1
+                if not self.use_multi:
+                    intent = [x["intent"] for x in batch]
+                else:
+                    intent = [
+                        [x for x in b["intent"].split("#")] for b in batch]
+            return InputData((text, slot, intent))
+        else:
+            return InputData((text, None, None))
+
+    def batch_fn(self,
+                 batch,
+                 ignore_index=-100,
+                 device="cuda",
+                 config=None,
+                 align_mode="fast",
+                 enable_label=True,
+                 label2tensor=True):
+        if align_mode == "fast":
+            # try:
+            return self.fast_align(batch,
+                                   ignore_index=ignore_index,
+                                   device=device,
+                                   config=config,
+                                   enable_label=enable_label,
+                                   label2tensor=label2tensor)
+            # except:
+            #     return self.general_align(batch,
+            #                               ignore_index=ignore_index,
+            #                               device=device,
+            #                               config=config,
+            #                               enable_label=enable_label,
+            #                               label2tensor=label2tensor)
+        else:
+            return self.general_align(batch,
+                                      ignore_index=ignore_index,
+                                      device=device,
+                                      config=config,
+                                      enable_label=enable_label,
+                                      label2tensor=label2tensor)
+
+    def get_data_loader(self,
+                        dataset,
+                        batch_size,
+                        shuffle=False,
+                        device="cuda",
+                        enable_label=True,
+                        align_mode="fast",
+                        label2tensor=True, **config):
+        data_loader = DataLoader(dataset,
+                                 shuffle=shuffle,
+                                 batch_size=batch_size,
+                                 collate_fn=lambda x: self.batch_fn(x,
+                                                                    device=device,
+                                                                    config=config,
+                                                                    enable_label=enable_label,
+                                                                    align_mode=align_mode,
+                                                                    label2tensor=label2tensor))
+        return data_loader

common/logger.py

@@ -0,0 +1,197 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-02-02 16:29:13
+Description: log manager
+
+'''
+import json
+import os
+import time
+from common.config import Config
+
+def mkdirs(dir_names):
+    for dir_name in dir_names:
+        if not os.path.exists(dir_name):
+            os.mkdir(dir_name)
+
+
+
+class Logger():
+    """ logging infomation by [wandb, fitlog, local file]
+    """
+    def __init__(self, 
+                 logger_type: str,
+                 logger_name: str,
+                 logging_level="INFO",
+                 start_time='',
+                 accelerator=None):
+        """ create logger
+
+        Args:
+            logger_type (str): support type = ["wandb", "fitlog", "local"]
+            logger_name (str): logger name, means project name in wandb, and logging file name
+            logging_level (str, optional): logging level. Defaults to "INFO".
+            start_time (str, optional): start time string. Defaults to ''.
+        """
+        self.logger_type = logger_type
+        times = time.localtime()
+        self.output_dir = "logs/" + logger_name + "/" + str(times.tm_year) + start_time
+        self.accelerator = accelerator
+        self.logger_name = logger_name
+        if accelerator is not None:
+            from accelerate.logging import get_logger
+            self.logging = get_logger(logger_name)
+        else:
+            if self.logger_type == "wandb":
+                import wandb
+                self.logger = wandb
+                mkdirs(["logs", "logs/" + logger_name, self.output_dir])
+                self.logger.init(project=logger_name)
+            elif self.logger_type == "fitlog":
+                import fitlog
+                self.logger = fitlog
+                mkdirs(["logs", "logs/" + logger_name, self.output_dir])
+                self.logger.set_log_dir("logs/" + logger_name)
+            else:
+                mkdirs(["logs", "logs/" + logger_name, self.output_dir])
+                self.config_file = os.path.join(self.output_dir, "/config.jsonl")
+                with open(self.config_file, "w", encoding="utf8") as f:
+                    print(f"Config will be written to {self.config_file}")
+
+                self.loss_file = os.path.join(self.output_dir, "/loss.jsonl")
+                with open(self.loss_file, "w", encoding="utf8") as f:
+                    print(f"Loss Result will be written to {self.loss_file}")
+
+                self.metric_file = os.path.join(self.output_dir, "/metric.jsonl")
+                with open(self.metric_file, "w", encoding="utf8") as f:
+                    print(f"Metric Result will be written to {self.metric_file}")
+
+                self.other_log_file = os.path.join(self.output_dir, "/other_log.jsonl")
+                with open(self.other_log_file, "w", encoding="utf8") as f:
+                    print(f"Other Log Result will be written to {self.other_log_file}")
+            import logging
+            LOGGING_LEVEL_MAP = {
+                "CRITICAL": logging.CRITICAL,
+                "FATAL": logging.FATAL,
+                "ERROR": logging.ERROR,
+                "WARNING": logging.WARNING,
+                "WARN": logging.WARN,
+                "INFO": logging.INFO,
+                "DEBUG": logging.DEBUG,
+                "NOTSET": logging.NOTSET,
+            }
+            logging.basicConfig(format='[%(levelname)s - %(asctime)s]\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p',
+                            filename=os.path.join(self.output_dir, "log.log"), level=LOGGING_LEVEL_MAP[logging_level])
+            self.logging = logging
+
+    def set_config(self, config: Config):
+        """save config
+
+        Args:
+            config (Config): configuration object to save
+        """
+        if self.accelerator is not None:
+            self.accelerator.init_trackers(self.logger_name, config=config)
+        elif self.logger_type == "wandb":
+            self.logger.config.update(config)
+        elif self.logger_type == "fitlog":
+            self.logger.add_hyper(config)
+        else:
+            with open(self.config_file, "a", encoding="utf8") as f:
+                f.write(json.dumps(config) + "\n")
+
+    def log(self, data, step=0):
+        """log data and step
+
+        Args:
+            data (Any): data to log
+            step (int, optional): step num. Defaults to 0.
+        """
+        if self.accelerator is not None:
+            self.accelerator.log(data, step=0)
+        elif self.logger_type == "wandb":
+            self.logger.log(data, step=step)
+        elif self.logger_type == "fitlog":
+            self.logger.add_other({"data": data, "step": step})
+        else:
+            with open(self.other_log_file, "a", encoding="utf8") as f:
+                f.write(json.dumps({"data": data, "step": step}) + "\n")
+
+    def log_metric(self, metric, metric_split="dev", step=0):
+        """log metric
+
+        Args:
+            metric (Any): metric
+            metric_split (str, optional): dataset split. Defaults to 'dev'.
+            step (int, optional): step num. Defaults to 0.
+        """
+        if self.accelerator is not None:
+            self.accelerator.log({metric_split: metric}, step=step)
+        elif self.logger_type == "wandb":
+            self.logger.log({metric_split: metric}, step=step)
+        elif self.logger_type == "fitlog":
+            self.logger.add_metric({metric_split: metric}, step=step)
+        else:
+            with open(self.metric_file, "a", encoding="utf8") as f:
+                f.write(json.dumps({metric_split: metric, "step": step}) + "\n")
+
+    def log_loss(self, loss, loss_name="Loss", step=0):
+        """log loss
+
+        Args:
+            loss (Any): loss
+            loss_name (str, optional): loss description. Defaults to 'Loss'.
+            step (int, optional): step num. Defaults to 0.
+        """
+        if self.accelerator is not None:
+            self.accelerator.log({loss_name: loss}, step=step)
+        elif self.logger_type == "wandb":
+            self.logger.log({loss_name: loss}, step=step)
+        elif self.logger_type == "fitlog":
+            self.logger.add_loss(loss, name=loss_name, step=step)
+        else:
+            with open(self.loss_file, "a", encoding="utf8") as f:
+                f.write(json.dumps({loss_name: loss, "step": step}) + "\n")
+
+    def finish(self):
+        """finish logging
+        """
+        if self.logger_type == "fitlog":
+            self.logger.finish()
+
+    def info(self, message:str):
+        """ Log a message with severity 'INFO' in local file / console.
+        
+        Args:
+            message (str): message to log
+        """
+        self.logging.info(message)
+
+    def warning(self, message):
+        """ Log a message with severity 'WARNING' in local file / console.
+        
+        Args:
+            message (str): message to log
+        """
+        self.logging.warning(message)
+
+    def error(self, message):
+        """ Log a message with severity 'ERROR' in local file / console.
+        
+        Args:
+            message (str): message to log
+        """
+        self.logging.error(message)
+
+    def debug(self, message):
+        """ Log a message with severity 'DEBUG' in local file / console.
+        
+        Args:
+            message (str): message to log
+        """
+        self.logging.debug(message)
+
+    def critical(self, message):
+        self.logging.critical(message)

common/metric.py

@@ -0,0 +1,343 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-26 12:12:55
+Description: Metric calculation class
+
+'''
+from collections import Counter
+from typing import List, Dict
+
+import numpy as np
+from sklearn.metrics import f1_score
+
+from common.utils import InputData, OutputData
+
+
+class Evaluator(object):
+    """Evaluation metric funtions library class
+        supported metric:
+        - slot_f1
+        - intent_acc
+        - exactly_match_accuracy
+        - intent_f1 (defult "macro_intent_f1")
+            - macro_intent_f1
+            - micro_intent_f1=
+    """
+    @staticmethod
+    def exactly_match_accuracy(pred_slot: List[List[str or int]],
+                               real_slot: List[List[str or int]],
+                               pred_intent: List[List[str or int] or str or int],
+                               real_intent: List[List[str or int] or str or int]) -> float:
+        """Compute the accuracy based on the whole predictions of given sentence, including slot and intent.
+            (both support str or int index as the representation of slot and intent)
+        Args:
+            pred_slot (List[List[str or int]]): predicted sequence of slot list
+            real_slot (List[List[str or int]]): golden sequence of slot list.
+            pred_intent (List[List[str or int] or str or int]): golden intent list / golden multi intent list.
+            real_intent (List[List[str or int] or str or int]): predicted intent list / predicted multi intent list.
+
+        Returns:
+            float: exactly match accuracy score
+        """
+        total_count, correct_count = 0.0, 0.0
+        for p_slot, r_slot, p_intent, r_intent in zip(pred_slot, real_slot, pred_intent, real_intent):
+            if isinstance(p_intent, list):
+                p_intent, r_intent = set(p_intent), set(r_intent)
+            if p_slot == r_slot and p_intent == r_intent:
+                correct_count += 1.0
+            total_count += 1.0
+
+        return 1.0 * correct_count / total_count
+
+
+    @staticmethod
+    def intent_accuracy(pred_list: List, real_list: List) -> float:
+        """Get  intent accuracy measured by predictions and ground-trues. Support both multi intent and single intent.
+
+        Args:
+            pred_list (List): predicted intent list
+            real_list (List): golden intent list
+
+        Returns:
+            float: intent accuracy score
+        """
+        total_count, correct_count = 0.0, 0.0
+        for p_intent, r_intent in zip(pred_list, real_list):
+            if isinstance(p_intent, list):
+                p_intent, r_intent = set(p_intent), set(r_intent)
+            if p_intent == r_intent:
+                correct_count += 1.0
+            total_count += 1.0
+
+        return 1.0 * correct_count / total_count
+
+    @staticmethod
+    def intent_f1(pred_list: List[List[int]], real_list: List[List[int]], num_intent: int, average='macro') -> float:
+        """Get  intent accuracy measured by predictions and ground-trues. Support both multi intent and single intent.
+        (Only support multi intent now, but you can use [[intent1], [intent2], ...] to compute intent f1 in single intent)
+        Args:
+            pred_list (List[List[int]]): predicted multi intent list.
+            real_list (List[List[int]]): golden multi intent list.
+            num_intent (int)
+            average (str): support "micro" and "macro"
+
+        Returns:
+            float: intent accuracy score
+        """
+        return f1_score(Evaluator.__instance2onehot(num_intent, real_list),
+                        Evaluator.__instance2onehot(num_intent, pred_list),
+                        average=average,
+                        zero_division=0)
+
+    @staticmethod
+    def __multilabel2one_hot(labels, nums):
+        res = [0.] * nums
+        if len(labels) == 0:
+            return res
+        if isinstance(labels[0], list):
+            for label in labels[0]:
+                res[label] = 1.
+            return res
+        for label in labels:
+            res[label] = 1.
+        return res
+
+    @staticmethod
+    def __instance2onehot(num_intent, data):
+        res = []
+        for intents in data:
+            res.append(Evaluator.__multilabel2one_hot(intents, num_intent))
+        return np.array(res)
+
+    @staticmethod
+    def __startOfChunk(prevTag, tag, prevTagType, tagType, chunkStart=False):
+        if prevTag == 'B' and tag == 'B':
+            chunkStart = True
+        if prevTag == 'I' and tag == 'B':
+            chunkStart = True
+        if prevTag == 'O' and tag == 'B':
+            chunkStart = True
+        if prevTag == 'O' and tag == 'I':
+            chunkStart = True
+
+        if prevTag == 'E' and tag == 'E':
+            chunkStart = True
+        if prevTag == 'E' and tag == 'I':
+            chunkStart = True
+        if prevTag == 'O' and tag == 'E':
+            chunkStart = True
+        if prevTag == 'O' and tag == 'I':
+            chunkStart = True
+
+        if tag != 'O' and tag != '.' and prevTagType != tagType:
+            chunkStart = True
+        return chunkStart
+
+    @staticmethod
+    def __endOfChunk(prevTag, tag, prevTagType, tagType, chunkEnd=False):
+        if prevTag == 'B' and tag == 'B':
+            chunkEnd = True
+        if prevTag == 'B' and tag == 'O':
+            chunkEnd = True
+        if prevTag == 'I' and tag == 'B':
+            chunkEnd = True
+        if prevTag == 'I' and tag == 'O':
+            chunkEnd = True
+
+        if prevTag == 'E' and tag == 'E':
+            chunkEnd = True
+        if prevTag == 'E' and tag == 'I':
+            chunkEnd = True
+        if prevTag == 'E' and tag == 'O':
+            chunkEnd = True
+        if prevTag == 'I' and tag == 'O':
+            chunkEnd = True
+
+        if prevTag != 'O' and prevTag != '.' and prevTagType != tagType:
+            chunkEnd = True
+        return chunkEnd
+
+    @staticmethod
+    def __splitTagType(tag):
+        s = tag.split('-')
+        if len(s) > 2 or len(s) == 0:
+            raise ValueError('tag format wrong. it must be B-xxx.xxx')
+        if len(s) == 1:
+            tag = s[0]
+            tagType = ""
+        else:
+            tag = s[0]
+            tagType = s[1]
+        return tag, tagType
+
+    @staticmethod
+    def computeF1Score(correct_slots: List[List[str]], pred_slots: List[List[str]]) -> float:
+        """compute f1 score is modified from conlleval.pl
+
+        Args:
+            correct_slots (List[List[str]]): golden slot string list
+            pred_slots (List[List[str]]): predicted slot string list
+
+        Returns:
+            float: slot f1 score
+        """
+        correctChunk = {}
+        correctChunkCnt = 0.0
+        foundCorrect = {}
+        foundCorrectCnt = 0.0
+        foundPred = {}
+        foundPredCnt = 0.0
+        correctTags = 0.0
+        tokenCount = 0.0
+        for correct_slot, pred_slot in zip(correct_slots, pred_slots):
+            inCorrect = False
+            lastCorrectTag = 'O'
+            lastCorrectType = ''
+            lastPredTag = 'O'
+            lastPredType = ''
+            for c, p in zip(correct_slot, pred_slot):
+                correctTag, correctType = Evaluator.__splitTagType(c)
+                predTag, predType = Evaluator.__splitTagType(p)
+
+                if inCorrect == True:
+                    if Evaluator.__endOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) == True and \
+                            Evaluator.__endOfChunk(lastPredTag, predTag, lastPredType, predType) == True and \
+                            (lastCorrectType == lastPredType):
+                        inCorrect = False
+                        correctChunkCnt += 1.0
+                        if lastCorrectType in correctChunk:
+                            correctChunk[lastCorrectType] += 1.0
+                        else:
+                            correctChunk[lastCorrectType] = 1.0
+                    elif Evaluator.__endOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) != \
+                            Evaluator.__endOfChunk(lastPredTag, predTag, lastPredType, predType) or \
+                            (correctType != predType):
+                        inCorrect = False
+
+                if Evaluator.__startOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) == True and \
+                        Evaluator.__startOfChunk(lastPredTag, predTag, lastPredType, predType) == True and \
+                        (correctType == predType):
+                    inCorrect = True
+
+                if Evaluator.__startOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) == True:
+                    foundCorrectCnt += 1
+                    if correctType in foundCorrect:
+                        foundCorrect[correctType] += 1.0
+                    else:
+                        foundCorrect[correctType] = 1.0
+
+                if Evaluator.__startOfChunk(lastPredTag, predTag, lastPredType, predType) == True:
+                    foundPredCnt += 1.0
+                    if predType in foundPred:
+                        foundPred[predType] += 1.0
+                    else:
+                        foundPred[predType] = 1.0
+
+                if correctTag == predTag and correctType == predType:
+                    correctTags += 1.0
+
+                tokenCount += 1.0
+
+                lastCorrectTag = correctTag
+                lastCorrectType = correctType
+                lastPredTag = predTag
+                lastPredType = predType
+
+            if inCorrect == True:
+                correctChunkCnt += 1.0
+                if lastCorrectType in correctChunk:
+                    correctChunk[lastCorrectType] += 1.0
+                else:
+                    correctChunk[lastCorrectType] = 1.0
+
+        if foundPredCnt > 0:
+            precision = 1.0 * correctChunkCnt / foundPredCnt
+        else:
+            precision = 0
+
+        if foundCorrectCnt > 0:
+            recall = 1.0 * correctChunkCnt / foundCorrectCnt
+        else:
+            recall = 0
+
+        if (precision + recall) > 0:
+            f1 = (2.0 * precision * recall) / (precision + recall)
+        else:
+            f1 = 0
+
+        return f1
+
+    @staticmethod
+    def max_freq_predict(sample):
+        """Max frequency prediction.
+        """
+        predict = []
+        for items in sample:
+            predict.append(Counter(items).most_common(1)[0][0])
+        return predict
+
+    @staticmethod
+    def __token_map(indexes, token_label_map):
+        return [[token_label_map[idx] if idx in token_label_map else -1 for idx in index] for index in indexes]
+
+    @staticmethod
+    def compute_all_metric(inps: InputData,
+                           output: OutputData,
+                           intent_label_map: dict = None,
+                           metric_list: List=None)-> Dict:
+        """Auto compute all metric mentioned in 'metric_list'
+        
+        Args:
+            inps (InputData): input golden slot and intent labels
+            output (OutputData): output predicted slot and intent labels
+            intent_label_map (dict, Optional): dict like {"intent1": 0, "intent2": 1, ...},which aims to map intent string to index
+            metric_list (List): support metrics in ["slot_f1", "intent_acc", "intent_f1", "macro_intent_f1", "micro_intent_f1", "EMA"]
+
+        Returns:
+            Dict: all metric mentioned in 'metric_list', like {'EMA': 0.7, ...}
+            
+        
+        Example:
+            if compute slot metric:
+            
+                inps.slot = [["slot1", "slot2", ...], ...]; output.slot_ids=[["slot1", "slot2", ...], ...];
+                
+            if compute intent metric:
+            
+                [Multi Intent] inps.intent = [["intent1", "intent2", ...], ...]; output.intent_ids = [["intent1", "intent2", ...], ...] 
+                
+                [Single Intent] inps.intent = ["intent1", ...]; [Single Intent] output.intent_ids = ["intent1", ...]
+        """
+        if not metric_list:
+            metric_list = ["slot_f1", "intent_acc", "EMA"]
+        res_dict = {}
+        use_slot = output.slot_ids is not None and len(output.slot_ids) > 0
+        use_intent = output.intent_ids is not None and len(
+            output.intent_ids) > 0
+        if use_slot and "slot_f1" in metric_list:
+            res_dict["slot_f1"] = Evaluator.computeF1Score(
+                output.slot_ids, inps.slot)
+        if use_intent and "intent_acc" in metric_list:
+            res_dict["intent_acc"] = Evaluator.intent_accuracy(
+                output.intent_ids, inps.intent)
+            if isinstance(output.intent_ids[0], list):
+                if "intent_f1" in metric_list:
+                    res_dict["intent_f1"] = Evaluator.intent_f1(Evaluator.__token_map(output.intent_ids, intent_label_map),
+                                                                Evaluator.__token_map(
+                                                                    inps.intent, intent_label_map),
+                                                                len(intent_label_map.keys()))
+                elif "macro_intent_f1" in metric_list:
+                    res_dict["macro_intent_f1"] = Evaluator.intent_f1(Evaluator.__token_map(output.intent_ids, intent_label_map),
+                                                                      Evaluator.__token_map(inps.intent, intent_label_map),
+                                                                      len(intent_label_map.keys()), average="macro")
+                if "micro_intent_f1" in metric_list:
+                    res_dict["micro_intent_f1"] = Evaluator.intent_f1(Evaluator.__token_map(output.intent_ids, intent_label_map),
+                                                                      Evaluator.__token_map(inps.intent, intent_label_map),
+                                                                      len(intent_label_map.keys()), average="micro")
+
+        if use_slot and use_intent and "EMA" in metric_list:
+            res_dict["EMA"] = Evaluator.exactly_match_accuracy(output.slot_ids, inps.slot, output.intent_ids,
+                                                               inps.intent)
+        return res_dict

common/model_manager.py

@@ -0,0 +1,324 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-02-07 21:36:06
+Description: manage all process of model training and prediction.
+
+'''
+import os
+import random
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+
+from common import utils
+from common.loader import DataFactory
+from common.logger import Logger
+from common.metric import Evaluator
+from common.tokenizer import get_tokenizer, get_tokenizer_class, load_embedding
+from common.utils import InputData, instantiate
+from common.utils import OutputData
+from common.config import Config
+import dill
+
+
+class ModelManager(object):
+    def __init__(self, config: Config):
+        """create model manager by config
+
+        Args:
+            config (Config): configuration to manage all process in OpenSLU
+        """
+        # init config
+        self.config = config
+        self.__set_seed(self.config.base.get("seed"))
+        self.device = self.config.base.get("device")
+        
+        # enable accelerator
+        if "accelerator" in self.config and self.config["accelerator"].get("use_accelerator"):
+            from accelerate import Accelerator
+            self.accelerator = Accelerator(log_with="wandb")
+        else:
+            self.accelerator = None
+        if self.config.base.get("train"):
+            self.tokenizer = get_tokenizer(
+                self.config.tokenizer.get("_tokenizer_name_"))
+            self.logger = Logger(
+                "wandb", self.config.base["name"], start_time=config.start_time, accelerator=self.accelerator)
+
+            # init dataloader & load data
+            if self.config.base.get("save_dir"):
+                self.model_save_dir = self.config.base["save_dir"]
+            else:
+                if not os.path.exists("save/"):
+                    os.mkdir("save/")
+                self.model_save_dir = "save/" + config.start_time
+            if not os.path.exists(self.model_save_dir):
+                    os.mkdir(self.model_save_dir)
+            batch_size = self.config.base["batch_size"]
+            df = DataFactory(tokenizer=self.tokenizer,
+                             use_multi_intent=self.config.base.get("multi_intent"),
+                             to_lower_case=self.config.base.get("_to_lower_case_"))
+            train_dataset = df.load_dataset(self.config.dataset, split="train")
+
+            # update label and vocabulary
+            df.update_label_names(train_dataset)
+            df.update_vocabulary(train_dataset)
+
+            # init tokenizer config and dataloaders
+            tokenizer_config = {key: self.config.tokenizer[key]
+                                for key in self.config.tokenizer if key[0] != "_" and key[-1] != "_"}
+            self.train_dataloader = df.get_data_loader(train_dataset,
+                                                       batch_size,
+                                                       shuffle=True,
+                                                       device=self.device,
+                                                       enable_label=True,
+                                                       align_mode=self.config.tokenizer.get(
+                                                           "_align_mode_"),
+                                                       label2tensor=True,
+                                                       **tokenizer_config)
+            dev_dataset = df.load_dataset(
+                self.config.dataset, split="validation")
+            self.dev_dataloader = df.get_data_loader(dev_dataset,
+                                                     batch_size,
+                                                     shuffle=False,
+                                                     device=self.device,
+                                                     enable_label=True,
+                                                     align_mode=self.config.tokenizer.get(
+                                                         "_align_mode_"),
+                                                     label2tensor=False,
+                                                     **tokenizer_config)
+            df.update_vocabulary(dev_dataset)
+            # add intent label num and slot label num to config
+            if int(self.config.get_intent_label_num()) == 0 or int(self.config.get_slot_label_num()) == 0:
+                self.intent_list = df.intent_label_list
+                self.intent_dict = df.intent_label_dict
+                self.config.set_intent_label_num(len(self.intent_list))
+                self.slot_list = df.slot_label_list
+                self.slot_dict = df.slot_label_dict
+                self.config.set_slot_label_num(len(self.slot_list))
+                self.config.set_vocab_size(self.tokenizer.vocab_size)
+
+            # autoload embedding for non-pretrained encoder
+            if self.config["model"]["encoder"].get("embedding") and self.config["model"]["encoder"]["embedding"].get(
+                    "load_embedding_name"):
+                self.config["model"]["encoder"]["embedding"]["embedding_matrix"] = load_embedding(self.tokenizer,
+                                                                                                  self.config["model"][
+                                                                                                      "encoder"][
+                                                                                                      "embedding"].get(
+                                                                                                      "load_embedding_name"))
+            # fill template in config
+            self.config.autoload_template()
+            # save config
+            self.logger.set_config(self.config)
+
+            self.model = None
+            self.optimizer = None
+            self.total_step = None
+            self.lr_scheduler = None
+            if self.config.tokenizer.get("_tokenizer_name_") == "word_tokenizer":
+                self.tokenizer.save(os.path.join(self.model_save_dir, "tokenizer.json"))
+            utils.save_json(os.path.join(
+                self.model_save_dir, "label.json"), {"intent": self.intent_list,"slot": self.slot_list})
+        if self.config.base.get("test"):
+            self.test_dataset = df.load_dataset(
+                self.config.dataset, split="test")
+            self.test_dataloader = df.get_data_loader(self.test_dataset,
+                                                      batch_size,
+                                                      shuffle=False,
+                                                      device=self.device,
+                                                      enable_label=True,
+                                                      align_mode=self.config.tokenizer.get(
+                                                          "_align_mode_"),
+                                                      label2tensor=False,
+                                                      **tokenizer_config)
+
+    def init_model(self, model):
+        """init model, optimizer, lr_scheduler
+
+        Args:
+            model (Any): pytorch model
+        """
+        self.model = model
+        self.model.to(self.device)
+        if self.config.base.get("train"):
+            self.optimizer = instantiate(
+                self.config["optimizer"])(self.model.parameters())
+            self.total_step = int(self.config.base.get(
+                "epoch_num")) * len(self.train_dataloader)
+            self.lr_scheduler = instantiate(self.config["scheduler"])(
+                optimizer=self.optimizer,
+                num_training_steps=self.total_step
+            )
+        if self.accelerator is not None:
+            self.model, self.optimizer, self.train_dataloader, self.lr_scheduler = self.accelerator.prepare(
+                self.model, self.optimizer, self.train_dataloader, self.lr_scheduler)
+            if self.config.base.get("load_dir_path"):
+                self.accelerator.load_state(self.config.base.get("load_dir_path"))
+            # self.dev_dataloader = self.accelerator.prepare(self.dev_dataloader)
+
+    def eval(self, step: int, best_metric: float) -> float:
+        """ evaluation models.
+
+        Args:
+            step (int): which step the model has trained in
+            best_metric (float): last best metric value to judge whether to test or save model
+
+        Returns:
+            float: updated best metric value
+        """
+        # TODO: save dev
+        _, res = self.__evaluate(self.model, self.dev_dataloader)
+        self.logger.log_metric(res, metric_split="dev", step=step)
+        if res[self.config.base.get("best_key")] > best_metric:
+            best_metric = res[self.config.base.get("best_key")]
+            outputs, test_res = self.__evaluate(
+                self.model, self.test_dataloader)
+            if not os.path.exists(self.model_save_dir):
+                os.mkdir(self.model_save_dir)
+            if self.accelerator is None:
+                torch.save(self.model, os.path.join(
+                    self.model_save_dir, "model.pkl"))
+                torch.save(self.optimizer, os.path.join(
+                    self.model_save_dir, "optimizer.pkl"))
+                torch.save(self.lr_scheduler, os.path.join(
+                    self.model_save_dir, "lr_scheduler.pkl"), pickle_module=dill)
+                torch.save(step, os.path.join(
+                    self.model_save_dir, "step.pkl"))
+            else:
+                self.accelerator.wait_for_everyone()
+                unwrapped_model = self.accelerator.unwrap_model(self.model)
+                self.accelerator.save(unwrapped_model.state_dict(
+                ), os.path.join(self.model_save_dir, "model.pkl"))
+                self.accelerator.save_state(output_dir=self.model_save_dir)
+            outputs.save(self.model_save_dir, self.test_dataset)
+            self.logger.log_metric(test_res, metric_split="test", step=step)
+        return best_metric
+
+    def train(self) -> float:
+        """ train models.
+
+        Returns:
+            float: updated best metric value
+        """
+        step = 0
+        best_metric = 0
+        progress_bar = tqdm(range(self.total_step))
+        for _ in range(int(self.config.base.get("epoch_num"))):
+            for data in self.train_dataloader:
+                if step == 0:
+                    self.logger.info(data.get_item(
+                        0, tokenizer=self.tokenizer, intent_map=self.intent_list, slot_map=self.slot_list))
+                output = self.model(data)
+                if self.accelerator is not None and hasattr(self.model, "module"):
+                    loss, intent_loss, slot_loss = self.model.module.compute_loss(
+                        pred=output, target=data)
+                else:
+                    loss, intent_loss, slot_loss = self.model.compute_loss(
+                        pred=output, target=data)
+                self.logger.log_loss(loss, "Loss", step=step)
+                self.logger.log_loss(intent_loss, "Intent Loss", step=step)
+                self.logger.log_loss(slot_loss, "Slot Loss", step=step)
+                self.optimizer.zero_grad()
+
+                if self.accelerator is not None:
+                    self.accelerator.backward(loss)
+                else:
+                    loss.backward()
+                self.optimizer.step()
+                self.lr_scheduler.step()
+                if not self.config.base.get("eval_by_epoch") and step % self.config.base.get(
+                        "eval_step") == 0 and step != 0:
+                    best_metric = self.eval(step, best_metric)
+                step += 1
+                progress_bar.update(1)
+            if self.config.base.get("eval_by_epoch"):
+                best_metric = self.eval(step, best_metric)
+        self.logger.finish()
+        return best_metric
+
+    def __set_seed(self, seed_value: int):
+        """Manually set random seeds.
+
+        Args:
+            seed_value (int): random seed
+        """
+        random.seed(seed_value)
+        np.random.seed(seed_value)
+        torch.manual_seed(seed_value)
+        torch.random.manual_seed(seed_value)
+        os.environ['PYTHONHASHSEED'] = str(seed_value)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed(seed_value)
+            torch.cuda.manual_seed_all(seed_value)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = True
+        return
+
+    def __evaluate(self, model, dataloader):
+        model.eval()
+        inps = InputData()
+        outputs = OutputData()
+        for data in dataloader:
+            torch.cuda.empty_cache()
+            output = model(data)
+            if self.accelerator is not None and hasattr(self.model, "module"):
+                decode_output = model.module.decode(output, data)
+            else:
+                decode_output = model.decode(output, data)
+
+            decode_output.map_output(slot_map=self.slot_list,
+                                     intent_map=self.intent_list)
+            data, decode_output = utils.remove_slot_ignore_index(
+                data, decode_output, ignore_index="#")
+
+            inps.merge_input_data(data)
+            outputs.merge_output_data(decode_output)
+        if "metric" in self.config:
+            res = Evaluator.compute_all_metric(
+                inps, outputs, intent_label_map=self.intent_dict, metric_list=self.config.metric)
+        else:
+            res = Evaluator.compute_all_metric(
+                inps, outputs, intent_label_map=self.intent_dict)
+        model.train()
+        return outputs, res
+
+    def load(self):
+        
+        self.model = torch.load(os.path.join(self.config.base["model_dir"], "model.pkl"))
+        if self.config.tokenizer["_tokenizer_name_"] == "word_tokenizer":
+            self.tokenizer = get_tokenizer_class(self.config.tokenizer["_tokenizer_name_"]).from_file(
+                os.path.join(self.config.base["model_dir"], "tokenizer.json"))
+        else:
+            self.tokenizer = get_tokenizer(self.config.tokenizer["_tokenizer_name_"])
+        self.model.to(self.device)
+        label = utils.load_json(os.path.join(self.config.base["model_dir"], "label.json"))
+        self.intent_list = label["intent"]
+        self.slot_list = label["slot"]
+        self.data_factory=DataFactory(tokenizer=self.tokenizer,
+                             use_multi_intent=self.config.base.get("multi_intent"),
+                             to_lower_case=self.config.tokenizer.get("_to_lower_case_"))
+
+    def predict(self, text_data):
+        self.model.eval()
+        tokenizer_config = {key: self.config.tokenizer[key]
+                                for key in self.config.tokenizer if key[0] != "_" and key[-1] != "_"}
+        align_mode = self.config.tokenizer.get("_align_mode_")
+        inputs = self.data_factory.batch_fn(batch=[{"text": text_data.split(" ")}],
+                                          device=self.device,
+                                        config=tokenizer_config,
+                                        enable_label=False,
+                                        align_mode= align_mode if align_mode is not None else "general",
+                                        label2tensor=False)
+        output = self.model(inputs)
+        decode_output = self.model.decode(output, inputs)
+        decode_output.map_output(slot_map=self.slot_list,
+                                     intent_map=self.intent_list)
+        if self.config.base.get("multi_intent"):
+            intent = decode_output.intent_ids[0]
+        else:
+            intent = [decode_output.intent_ids[0]]
+        return {"intent": intent, "slot": decode_output.slot_ids[0], "text": self.tokenizer.decode(inputs.input_ids[0])}

common/tokenizer.py

@@ -0,0 +1,311 @@
+import json
+import os
+from collections import Counter
+from collections import OrderedDict
+from typing import List
+
+import torch
+from ordered_set import OrderedSet
+from transformers import AutoTokenizer
+
+from common.utils import download, unzip_file
+
+
+def get_tokenizer(tokenizer_name:str):
+    """auto get tokenizer
+
+    Args:
+        tokenizer_name (str): support "word_tokenizer" and other pretrained tokenizer in hugging face.
+
+    Returns:
+        Any: Tokenizer Object
+    """
+    if tokenizer_name == "word_tokenizer":
+        return WordTokenizer(tokenizer_name)
+    else:
+        return AutoTokenizer.from_pretrained(tokenizer_name)
+
+def get_tokenizer_class(tokenizer_name:str):
+    """auto get tokenizer class
+
+    Args:
+        tokenizer_name (str): support "word_tokenizer" and other pretrained tokenizer in hugging face.
+
+    Returns:
+        Any: Tokenizer Class
+    """
+    if tokenizer_name == "word_tokenizer":
+        return WordTokenizer
+    else:
+        return AutoTokenizer.from_pretrained
+
+BATCH_STATE = 1
+INSTANCE_STATE = 2
+
+
+class WordTokenizer(object):
+
+    def __init__(self, name):
+        self.__name = name
+        self.index2instance = OrderedSet()
+        self.instance2index = OrderedDict()
+        # Counter Object record the frequency
+        # of element occurs in raw text.
+        self.counter = Counter()
+
+        self.__sign_pad = "[PAD]"
+        self.add_instance(self.__sign_pad)
+        self.__sign_unk = "[UNK]"
+        self.add_instance(self.__sign_unk)
+
+    @property
+    def padding_side(self):
+        return "right"
+    @property
+    def all_special_ids(self):
+        return [self.unk_token_id, self.pad_token_id]
+
+    @property
+    def name_or_path(self):
+        return self.__name
+
+    @property
+    def vocab_size(self):
+        return len(self.instance2index)
+
+    @property
+    def pad_token_id(self):
+        return self.instance2index[self.__sign_pad]
+
+    @property
+    def unk_token_id(self):
+        return self.instance2index[self.__sign_unk]
+
+    def add_instance(self, instance):
+        """ Add instances to alphabet.
+
+        1, We support any iterative data structure which
+        contains elements of str type.
+
+        2, We will count added instances that will influence
+        the serialization of unknown instance.
+
+        Args:
+            instance: is given instance or a list of it.
+        """
+
+        if isinstance(instance, (list, tuple)):
+            for element in instance:
+                self.add_instance(element)
+            return
+
+        # We only support elements of str type.
+        assert isinstance(instance, str)
+
+        # count the frequency of instances.
+        self.counter[instance] += 1
+
+        if instance not in self.index2instance:
+            self.instance2index[instance] = len(self.index2instance)
+            self.index2instance.append(instance)
+
+    def __call__(self, instance,
+                 return_tensors="pt",
+                 is_split_into_words=True,
+                 padding=True,
+                 add_special_tokens=False,
+                 truncation=True,
+                 max_length=512,
+                 **config):
+        if isinstance(instance, (list, tuple)) and isinstance(instance[0], (str)) and is_split_into_words:
+            res = self.get_index(instance)
+            state = INSTANCE_STATE
+        elif isinstance(instance, str) and not is_split_into_words:
+            res = self.get_index(instance.split(" "))
+            state = INSTANCE_STATE
+        elif not is_split_into_words and isinstance(instance, (list, tuple)):
+            res = [self.get_index(ins.split(" ")) for ins in instance]
+            state = BATCH_STATE
+        else:
+            res = [self.get_index(ins) for ins in instance]
+            state = BATCH_STATE
+        res = [r[:max_length] if len(r) >= max_length else r for r in res]
+        pad_id = self.get_index(self.__sign_pad)
+        if padding and state == BATCH_STATE:
+            max_len = max([len(x) for x in instance])
+
+            for i in range(len(res)):
+                res[i] = res[i] + [pad_id] * (max_len - len(res[i]))
+        if return_tensors == "pt":
+            input_ids = torch.Tensor(res).long()
+            attention_mask = (input_ids != pad_id).long()
+        elif state == BATCH_STATE:
+            input_ids = res
+            attention_mask = [1 if r != pad_id else 0 for batch in res for r in batch]
+        else:
+            input_ids = res
+            attention_mask = [1 if r != pad_id else 0 for r in res]
+        return TokenizedData(input_ids, token_type_ids=attention_mask, attention_mask=attention_mask)
+
+    def get_index(self, instance):
+        """ Serialize given instance and return.
+
+        For unknown words, the return index of alphabet
+        depends on variable self.__use_unk:
+
+            1, If True, then return the index of "<UNK>";
+            2, If False, then return the index of the
+            element that hold max frequency in training data.
+
+        Args:
+            instance (Any): is given instance or a list of it.
+        Return:
+            Any: the serialization of query instance.
+        """
+
+        if isinstance(instance, (list, tuple)):
+            return [self.get_index(elem) for elem in instance]
+
+        assert isinstance(instance, str)
+
+        try:
+            return self.instance2index[instance]
+        except KeyError:
+            return self.instance2index[self.__sign_unk]
+
+    def decode(self, index):
+        """ Get corresponding instance of query index.
+
+        if index is invalid, then throws exception.
+
+        Args:
+            index (int): is query index, possibly iterable.
+        Returns:
+            is corresponding instance.
+        """
+
+        if isinstance(index, list):
+            return [self.decode(elem) for elem in index]
+        if isinstance(index, torch.Tensor):
+            index = index.tolist()
+            return self.decode(index)
+        return self.index2instance[index]
+
+    def save(self, path):
+        """ Save the content of alphabet to files.
+
+        There are two kinds of saved files:
+            1, The first is a list file, elements are
+            sorted by the frequency of occurrence.
+
+            2, The second is a dictionary file, elements
+            are sorted by it serialized index.
+
+        Args:
+            path (str): is the path to save object.
+        """
+        
+        with open(path, 'w', encoding="utf8") as fw:
+            fw.write(json.dumps({"name": self.__name, "token_map": self.instance2index}))
+    
+    @staticmethod
+    def from_file(path):
+        with open(path, 'r', encoding="utf8") as fw:
+            obj = json.load(fw)
+            tokenizer = WordTokenizer(obj["name"])
+            tokenizer.instance2index = OrderedDict(obj["token_map"])
+            tokenizer.counter = len(tokenizer.instance2index)
+            tokenizer.index2instance = OrderedSet(tokenizer.instance2index.keys())
+            return tokenizer
+    
+    def __len__(self):
+        return len(self.index2instance)
+
+    def __str__(self):
+        return 'Alphabet {} contains about {} words: \n\t{}'.format(self.name_or_path, len(self), self.index2instance)
+
+    def convert_tokens_to_ids(self, tokens):
+        """convert token sequence to intput ids sequence
+
+        Args:
+            tokens (Any): token sequence
+
+        Returns:
+            Any: intput ids sequence
+        """
+        try:
+            if isinstance(tokens, (list, tuple)):
+                return [self.instance2index[x] for x in tokens]
+            return self.instance2index[tokens]
+
+        except KeyError:
+            return self.instance2index[self.__sign_unk]
+
+
+class TokenizedData():
+    """tokenized output data with input_ids, token_type_ids, attention_mask
+    """
+    def __init__(self, input_ids, token_type_ids, attention_mask):
+        self.input_ids = input_ids
+        self.token_type_ids = token_type_ids
+        self.attention_mask = attention_mask
+
+    def word_ids(self, index: int) -> List[int or None]:
+        """ get word id list
+
+        Args:
+            index (int): word index in sequence
+        
+        Returns:
+            List[int or None]: word id list
+        """
+        return [j if self.attention_mask[index][j] != 0 else None for j, x in enumerate(self.input_ids[index])]
+
+    def word_to_tokens(self, index, word_id, **kwargs):
+        """map word and tokens
+
+        Args:
+            index (int): unused
+            word_id (int): word index in sequence
+        """
+        return (word_id, word_id + 1)
+
+    def to(self, device):
+        """set device
+
+        Args:
+            device (str): support ["cpu", "cuda"]
+        """
+        self.input_ids = self.input_ids.to(device)
+        self.token_type_ids = self.token_type_ids.to(device)
+        self.attention_mask = self.attention_mask.to(device)
+        return self
+
+
+def load_embedding(tokenizer: WordTokenizer, glove_name:str):
+    """ load embedding from standford server or local cache.
+
+    Args:
+        tokenizer (WordTokenizer): non-pretrained tokenizer
+        glove_name (str): _description_
+
+    Returns:
+        Any: word embedding
+    """
+    save_path = "save/" + glove_name + ".zip"
+    if not os.path.exists(save_path):
+        download("http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip#" + glove_name, save_path)
+        unzip_file(save_path, "save/" + glove_name)
+    dim = int(glove_name.split(".")[-2][:-1])
+    embedding_list = torch.rand((tokenizer.vocab_size, dim))
+    embedding_list[tokenizer.pad_token_id] = torch.zeros((1, dim))
+    with open("save/" + glove_name + "/" + glove_name, "r", encoding="utf8") as f:
+        for line in f.readlines():
+            datas = line.split(" ")
+            word = datas[0]
+            embedding = torch.Tensor([float(datas[i + 1]) for i in range(len(datas) - 1)])
+            tokenized = tokenizer.convert_tokens_to_ids(word)
+            if isinstance(tokenized, int) and tokenized != tokenizer.unk_token_id:
+                embedding_list[tokenized] = embedding
+
+    return embedding_list

common/utils.py

@@ -0,0 +1,489 @@
+import functools
+import importlib
+import json
+import os
+import tarfile
+from typing import List, Tuple
+import zipfile
+from collections import Callable
+from ruamel import yaml
+import requests
+import torch
+from torch.nn.utils.rnn import pad_sequence
+from tqdm import tqdm
+from torch import Tensor
+
+class InputData():
+    """input datas class
+    """
+    def __init__(self, inputs: List =None):
+        """init input datas class
+        
+        if inputs is None:
+            this class can be used to save all InputData in the history by 'merge_input_data(X:InputData)' 
+        else:
+            this class can be used for model input.
+        
+        Args:
+            inputs (List, optional): inputs with [tokenized_data, slot, intent]. Defaults to None.
+        """
+        if inputs == None:
+            self.slot = []
+            self.intent = []
+            self.input_ids = None
+            self.token_type_ids = None
+            self.attention_mask = None
+            self.seq_lens = None
+        else:
+            self.input_ids = inputs[0].input_ids
+            self.token_type_ids = None
+            if hasattr(inputs[0], "token_type_ids"):
+                self.token_type_ids = inputs[0].token_type_ids
+            self.attention_mask = inputs[0].attention_mask
+            if len(inputs)>=2:
+                self.slot = inputs[1]
+            if len(inputs)>=3:
+                self.intent = inputs[2]
+            self.seq_lens = self.attention_mask.sum(-1)
+
+    def get_inputs(self):
+        """ get tokenized_data
+
+        Returns:
+            dict: tokenized data
+        """
+        res = {
+            "input_ids": self.input_ids,
+            "attention_mask": self.attention_mask
+        }
+        if self.token_type_ids is not None:
+            res["token_type_ids"] = self.token_type_ids
+        return res
+
+    def merge_input_data(self, inp: "InputData"):
+        """merge another InputData object with slot and intent
+
+        Args:
+            inp (InputData): another InputData object
+        """
+        self.slot += inp.slot
+        self.intent += inp.intent
+
+    def get_slot_mask(self, ignore_index:int)->Tensor:
+        """get slot mask 
+
+        Args:
+            ignore_index (int): ignore index used in slot padding
+        
+        Returns:
+            Tensor: mask tensor
+        """
+        mask = self.slot != ignore_index
+        mask[:, 0] = torch.ones_like(mask[:, 0]).to(self.slot.device)
+        return mask
+
+    def get_item(self, index, tokenizer=None, intent_map=None, slot_map=None, ignore_index = -100):
+        res = {"input_ids": self.input_ids[index]}
+        if tokenizer is not None:
+            res["tokens"] = [tokenizer.decode(x) for x in self.input_ids[index]]
+        if intent_map is not None:
+            intents = self.intent.tolist()
+            if isinstance(intents[index], list):
+                res["intent"] = [intent_map[int(x)] for x in intents[index]]
+            else:
+                res["intent"] = intent_map[intents[index]]
+        if slot_map is not None:
+            res["slot"] = [slot_map[x] if x != ignore_index else "#" for x in self.slot.tolist()[index]]
+        return res
+
+class OutputData():
+    """output data class
+    """
+    def __init__(self, intent_ids=None, slot_ids=None):
+        """init output data class
+        
+        if intent_ids is None and slot_ids is None:
+            this class can be used to save all OutputData in the history by 'merge_output_data(X:OutputData)' 
+        else:
+            this class can be used to model output management.
+        
+        Args:
+            intent_ids (Any, optional): list(Tensor) of intent ids / logits / strings. Defaults to None.
+            slot_ids (Any, optional): list(Tensor) of slot ids / ids / strings. Defaults to None.
+        """
+        if intent_ids is None and slot_ids is None:
+            self.intent_ids = []
+            self.slot_ids = []
+        else:
+            if isinstance(intent_ids, ClassifierOutputData):
+                self.intent_ids = intent_ids.classifier_output
+            else:
+                self.intent_ids = intent_ids
+            if isinstance(slot_ids, ClassifierOutputData):
+                self.slot_ids = slot_ids.classifier_output
+            else:
+                self.slot_ids = slot_ids
+
+    def map_output(self, slot_map=None, intent_map=None):
+        """ map intent or slot ids to intent or slot string.
+
+        Args:
+            slot_map (dict, optional): slot id-to-string map. Defaults to None.
+            intent_map (dict, optional): intent id-to-string map. Defaults to None.
+        """
+        if self.slot_ids is not None:
+            if slot_map:
+                self.slot_ids = [[slot_map[x] if x >= 0 else "#" for x in sid] for sid in self.slot_ids]
+        if self.intent_ids is not None:
+            if intent_map:
+                self.intent_ids = [[intent_map[x] for x in sid] if isinstance(sid, list) else intent_map[sid] for sid in
+                                   self.intent_ids]
+
+    def merge_output_data(self, output:"OutputData"):
+        """merge another OutData object with slot and intent
+
+        Args:
+            output (OutputData): another OutputData object
+        """
+        if output.slot_ids is not None:
+            self.slot_ids += output.slot_ids
+        if output.intent_ids is not None:
+            self.intent_ids += output.intent_ids
+
+    def save(self, path:str, original_dataset=None):
+        """ save all OutputData in the history
+
+        Args:
+            path (str): save dir path
+            original_dataset(Iterable): original dataset
+        """
+        # with open(f"{path}/intent.jsonl", "w") as f:
+        #     for x in self.intent_ids:
+        #         f.write(json.dumps(x) + "\n")
+        with open(f"{path}/outputs.jsonl", "w") as f:
+            if original_dataset is not None:
+                for i, s, d in zip(self.intent_ids, self.slot_ids, original_dataset):
+                    f.write(json.dumps({"pred_intent": i, "pred_slot": s, "text": d["text"], "golden_intent":d["intent"], "golden_slot":d["slot"]}) + "\n")
+            else:
+                for i, s in zip(self.intent_ids, self.slot_ids):
+                    f.write(json.dumps({"pred_intent": i, "pred_slot": s}) + "\n")
+
+
+class HiddenData():
+    """Interactive data structure for all model components
+    """
+    def __init__(self, intent_hidden, slot_hidden):
+        """init hidden data structure
+
+        Args:
+            intent_hidden (Any): sentence-level or intent hidden state
+            slot_hidden (Any): token-level or slot hidden state
+        """
+        self.intent_hidden = intent_hidden
+        self.slot_hidden = slot_hidden
+        self.inputs = None
+        self.embedding = None
+
+    def get_intent_hidden_state(self):
+        """get intent hidden state
+
+        Returns:
+            Any: intent hidden state
+        """
+        return self.intent_hidden
+
+    def get_slot_hidden_state(self):
+        """get slot hidden state
+
+        Returns:
+            Any: slot hidden state
+        """
+        return self.slot_hidden
+
+    def update_slot_hidden_state(self, hidden_state):
+        """update slot hidden state
+
+        Args:
+            hidden_state (Any): slot hidden state to update
+        """
+        self.slot_hidden = hidden_state
+
+    def update_intent_hidden_state(self, hidden_state):
+        """update intent hidden state
+
+        Args:
+            hidden_state (Any): intent hidden state to update
+        """
+        self.intent_hidden = hidden_state
+
+    def add_input(self, inputs: InputData or "HiddenData"):
+        """add last model component input information to next model component
+
+        Args:
+            inputs (InputDataor or HiddenData): last model component input
+        """
+        self.inputs = inputs
+
+    def add_embedding(self, embedding):
+        self.embedding = embedding
+
+
+class ClassifierOutputData():
+    """Classifier output data structure of all classifier components
+    """
+    def __init__(self, classifier_output):
+        self.classifier_output = classifier_output
+        self.output_embedding = None
+
+def remove_slot_ignore_index(inputs:InputData, outputs:OutputData, ignore_index=-100):
+    """ remove padding or extra token in input id and output id
+
+    Args:
+        inputs (InputData): input data with input id
+        outputs (OutputData): output data with decoded output id
+        ignore_index (int, optional): ignore_index in input_ids. Defaults to -100.
+
+    Returns:
+        InputData: input data removed padding or extra token
+        OutputData: output data removed padding or extra token
+    """
+    for index, (inp_ss, out_ss) in enumerate(zip(inputs.slot, outputs.slot_ids)):
+        temp_inp = []
+        temp_out = []
+        for inp_s, out_s in zip(list(inp_ss), list(out_ss)):
+            if inp_s != ignore_index:
+                temp_inp.append(inp_s)
+                temp_out.append(out_s)
+
+        inputs.slot[index] = temp_inp
+        outputs.slot_ids[index] = temp_out
+    return inputs, outputs
+
+
+def pack_sequence(inputs:Tensor, seq_len:Tensor or List) -> Tensor:
+    """pack sequence data to packed data without padding.
+    
+    Args:
+        inputs (Tensor): list(Tensor) of packed sequence inputs
+        seq_len (Tensor or List): list(Tensor) of sequence length
+
+    Returns:
+        Tensor: packed inputs
+        
+    Examples:
+        inputs = [[x, y, z, PAD, PAD], [x, y, PAD, PAD, PAD]]
+        
+        seq_len = [3,2]
+        
+        return -> [x, y, z, x, y]
+    """
+    output = []
+    for index, batch in enumerate(inputs):
+        output.append(batch[:seq_len[index]])
+    return torch.cat(output, dim=0)
+
+
+def unpack_sequence(inputs:Tensor, seq_lens:Tensor or List, padding_value=0) -> Tensor:
+    """unpack sequence data.
+        
+    Args:
+        inputs (Tensor): list(Tensor) of packed sequence inputs
+        seq_lens (Tensor or List): list(Tensor) of sequence length
+        padding_value (int, optional): padding value. Defaults to 0.
+
+    Returns:
+        Tensor: unpacked inputs
+        
+    Examples:
+        inputs = [x, y, z, x, y]
+        
+        seq_len = [3,2]
+        
+        return -> [[x, y, z, PAD, PAD], [x, y, PAD, PAD, PAD]]
+    """
+    last_idx = 0
+    output = []
+    for _, seq_len in enumerate(seq_lens):
+        output.append(inputs[last_idx:last_idx + seq_len])
+        last_idx = last_idx + seq_len
+    return pad_sequence(output, batch_first=True, padding_value=padding_value)
+
+
+def get_dict_with_key_prefix(input_dict: dict, prefix=""):
+    res = {}
+    for t in input_dict:
+        res[t + prefix] = input_dict[t]
+    return res
+
+
+def download(url: str, fname: str):
+    """download file from url to fname
+
+    Args:
+        url (str): remote server url path
+        fname (str): local path to save
+    """
+    resp = requests.get(url, stream=True)
+    total = int(resp.headers.get('content-length', 0))
+    with open(fname, 'wb') as file, tqdm(
+            desc=fname,
+            total=total,
+            unit='iB',
+            unit_scale=True,
+            unit_divisor=1024,
+    ) as bar:
+        for data in resp.iter_content(chunk_size=1024):
+            size = file.write(data)
+            bar.update(size)
+
+
+def tar_gz_data(file_name:str):
+    """use "tar.gz" format to compress data
+
+    Args:
+        file_name (str): file path to tar
+    """
+    t = tarfile.open(f"{file_name}.tar.gz", "w:gz")
+
+    for root, dir, files in os.walk(f"{file_name}"):
+        print(root, dir, files)
+        for file in files:
+            fullpath = os.path.join(root, file)
+            t.add(fullpath)
+    t.close()
+
+
+def untar(fname:str, dirs:str):
+    """ uncompress "tar.gz" file
+
+    Args:
+        fname (str): file path to untar
+        dirs (str): target dir path
+    """
+    t = tarfile.open(fname)
+    t.extractall(path=dirs)
+
+
+def unzip_file(zip_src:str, dst_dir:str):
+    """ uncompress "zip" file
+
+    Args:
+        fname (str): file path to unzip
+        dirs (str): target dir path
+    """
+    r = zipfile.is_zipfile(zip_src)
+    if r:
+        if not os.path.exists(dst_dir):
+            os.mkdir(dst_dir)
+        fz = zipfile.ZipFile(zip_src, 'r')
+        for file in fz.namelist():
+            fz.extract(file, dst_dir)
+    else:
+        print('This is not zip')
+
+
+def find_callable(target: str) -> Callable:
+    """ find callable function / class to instantiate
+
+    Args:
+        target (str): class/module path
+
+    Raises:
+        e: can not import module
+
+    Returns:
+        Callable: return function / class
+    """
+    target_module_path, target_callable_path = target.rsplit(".", 1)
+    target_callable_paths = [target_callable_path]
+
+    target_module = None
+    while len(target_module_path):
+        try:
+            target_module = importlib.import_module(target_module_path)
+            break
+        except Exception as e:
+            raise e
+    target_callable = target_module
+    for attr in reversed(target_callable_paths):
+        target_callable = getattr(target_callable, attr)
+
+    return target_callable
+
+
+def instantiate(config, target="_model_target_", partial="_model_partial_"):
+    """ instantiate object by config.
+    
+    Modified from https://github.com/HIT-SCIR/ltp/blob/main/python/core/ltp_core/models/utils/instantiate.py.
+
+    Args:
+        config (Any): configuration
+        target (str, optional): key to assign the class to be instantiated. Defaults to "_model_target_".
+        partial (str, optional): key to judge object whether should be instantiated partially. Defaults to "_model_partial_".
+
+    Returns:
+        Any: instantiated object
+    """
+    if isinstance(config, dict) and target in config:
+        target_path = config.get(target)
+        target_callable = find_callable(target_path)
+
+        is_partial = config.get(partial, False)
+        target_args = {
+            key: instantiate(value)
+            for key, value in config.items()
+            if key not in [target, partial]
+        }
+
+        if is_partial:
+            return functools.partial(target_callable, **target_args)
+        else:
+            return target_callable(**target_args)
+    elif isinstance(config, dict):
+        return {key: instantiate(value) for key, value in config.items()}
+    else:
+        return config
+
+
+def load_yaml(file):
+    """ load data from yaml files.
+
+    Args:
+        file (str): yaml file path.
+
+    Returns:
+        Any: data
+    """
+    with open(file, encoding="utf-8") as stream:
+        try:
+            return yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            raise exc
+
+def from_configured(configure_name_or_file:str, model_class:Callable, config_prefix="./config/", **input_config):
+    """load module from pre-configured data
+
+    Args:
+        configure_name_or_file (str): config path -> {config_prefix}/{configure_name_or_file}.yaml
+        model_class (Callable): module class
+        config_prefix (str, optional): configuration root path. Defaults to "./config/".
+
+    Returns:
+        Any: instantiated object.
+    """
+    if os.path.exists(configure_name_or_file):
+        configure_file=configure_name_or_file
+    else:
+        configure_file= os.path.join(config_prefix, configure_name_or_file+".yaml")
+    config = load_yaml(configure_file)
+    config.update(input_config)
+    return model_class(**config)
+
+def save_json(file_path, obj):
+    with open(file_path, 'w', encoding="utf8") as fw:
+            fw.write(json.dumps(obj))
+            
+def load_json(file_path):
+    with open(file_path, 'r', encoding="utf8") as fw:
+        res =json.load(fw)
+    return res

model/__init__.py

@@ -0,0 +1,3 @@
+from model.open_slu_model import OpenSLUModel
+
+__all__ = ["OpenSLUModel"]

model/decoder/__init__.py

@@ -0,0 +1,5 @@
+from model.decoder.agif_decoder import AGIFDecoder
+from model.decoder.base_decoder import StackPropagationDecoder, BaseDecoder, DCANetDecoder
+from model.decoder.gl_gin_decoder import GLGINDecoder
+
+__all__ = ["StackPropagationDecoder", "BaseDecoder", "DCANetDecoder", "AGIFDecoder", "GLGINDecoder"]

model/decoder/agif_decoder.py

@@ -0,0 +1,16 @@
+from common.utils import HiddenData, OutputData
+from model.decoder.base_decoder import BaseDecoder
+
+
+class AGIFDecoder(BaseDecoder):
+    def forward(self, hidden: HiddenData, **kwargs):
+        # hidden = self.interaction(hidden)
+        pred_intent = self.intent_classifier(hidden)
+        intent_index = self.intent_classifier.decode(OutputData(pred_intent, None),
+                                                     return_list=False,
+                                                     return_sentence_level=True)
+        interact_args = {"intent_index": intent_index,
+                         "batch_size": pred_intent.classifier_output.shape[0],
+                         "intent_label_num": self.intent_classifier.config["intent_label_num"]}
+        pred_slot = self.slot_classifier(hidden, internal_interaction=self.interaction, **interact_args)
+        return OutputData(pred_intent, pred_slot)

model/decoder/base_decoder.py

@@ -0,0 +1,94 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-31 18:22:36
+Description: 
+
+'''
+from torch import nn
+
+from common.utils import HiddenData, OutputData, InputData
+
+
+class BaseDecoder(nn.Module):
+    """Base class for all decoder module. 
+    
+    Notice: t is often only necessary to change this module and its sub-modules
+    """
+    def __init__(self, intent_classifier, slot_classifier, interaction=None):
+        super().__init__()
+        self.intent_classifier = intent_classifier
+        self.slot_classifier = slot_classifier
+        self.interaction = interaction
+
+    def forward(self, hidden: HiddenData):
+        """forward
+
+        Args:
+            hidden (HiddenData): encoded data
+
+        Returns:
+            OutputData: prediction logits 
+        """
+        if self.interaction is not None:
+            hidden = self.interaction(hidden)
+        return OutputData(self.intent_classifier(hidden), self.slot_classifier(hidden))
+
+    def decode(self, output: OutputData, target: InputData = None):
+        """decode output logits
+
+        Args:
+            output (OutputData): output logits data
+            target (InputData, optional): input data with attention mask. Defaults to None.
+
+        Returns:
+            List: decoded sequence ids
+        """
+        return OutputData(self.intent_classifier.decode(output, target), self.slot_classifier.decode(output, target))
+
+    def compute_loss(self, pred: OutputData, target: InputData, compute_intent_loss=True, compute_slot_loss=True):
+        """compute loss.
+        Notice: can set intent and slot loss weight by adding 'weight' config item in corresponding classifier configuration.
+
+        Args:
+            pred (OutputData): output logits data
+            target (InputData): input golden data
+            compute_intent_loss (bool, optional): whether to compute intent loss. Defaults to True.
+            compute_slot_loss (bool, optional): whether to compute intent loss. Defaults to True.
+
+        Returns:
+            Tensor: loss result
+        """
+        intent_loss = self.intent_classifier.compute_loss(pred, target) if compute_intent_loss else None
+        slot_loss = self.slot_classifier.compute_loss(pred, target) if compute_slot_loss else None
+        slot_weight = self.slot_classifier.config.get("weight")
+        slot_weight = slot_weight if slot_weight is not None else 1.
+        intent_weight =  self.intent_classifier.config.get("weight")
+        intent_weight = intent_weight if intent_weight is not None else 1.
+        loss = 0
+        if intent_loss is not None:
+            loss += intent_loss * intent_weight
+        if slot_loss is not None:
+            loss += slot_loss * slot_weight
+        return loss, intent_loss, slot_loss
+
+
+class StackPropagationDecoder(BaseDecoder):
+
+    def forward(self, hidden: HiddenData):
+        # hidden = self.interaction(hidden)
+        pred_intent = self.intent_classifier(hidden)
+        # embedding = pred_intent.output_embedding if pred_intent.output_embedding is not None else pred_intent.classifier_output
+        # hidden.update_intent_hidden_state(torch.cat([hidden.get_slot_hidden_state(), embedding], dim=-1))
+        hidden = self.interaction(pred_intent, hidden)
+        pred_slot = self.slot_classifier(hidden)
+        return OutputData(pred_intent, pred_slot)
+
+class DCANetDecoder(BaseDecoder):
+
+    def forward(self, hidden: HiddenData):
+        if self.interaction is not None:
+            hidden = self.interaction(hidden, intent_emb=self.intent_classifier, slot_emb=self.slot_classifier)
+        return OutputData(self.intent_classifier(hidden), self.slot_classifier(hidden))
+

model/decoder/classifier.py

@@ -0,0 +1,321 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-31 20:07:00
+Description: 
+
+'''
+import random
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+from model.decoder import decoder_utils
+
+from torchcrf import CRF
+
+from common.utils import HiddenData, OutputData, InputData, ClassifierOutputData, unpack_sequence, pack_sequence, \
+    instantiate
+
+
+class BaseClassifier(nn.Module):
+    """Base class for all classifier module
+    """
+    def __init__(self, **config):
+        super().__init__()
+        self.config = config
+        if config.get("loss_fn"):
+            self.loss_fn = instantiate(config.get("loss_fn"))
+        else:
+            self.loss_fn = CrossEntropyLoss(ignore_index=self.config.get("ignore_index"))
+
+    def forward(self, *args, **kwargs):
+        raise NotImplementedError("No implemented classifier.")
+
+    def decode(self, output: OutputData,
+               target: InputData = None,
+               return_list=True,
+               return_sentence_level=None):
+        """decode output logits
+
+        Args:
+            output (OutputData): output logits data
+            target (InputData, optional): input data with attention mask. Defaults to None.
+            return_list (bool, optional):  if True return list else return torch Tensor.. Defaults to True.
+            return_sentence_level (_type_, optional): if True decode sentence level intent else decode token level intent. Defaults to None.
+
+        Returns:
+            List or Tensor: decoded sequence ids
+        """
+        if self.config.get("return_sentence_level") is not None and return_sentence_level is None:
+            return_sentence_level = self.config.get("return_sentence_level")
+        elif self.config.get("return_sentence_level") is None and return_sentence_level is None:
+            return_sentence_level = False
+        return decoder_utils.decode(output, target,
+                                    return_list=return_list,
+                                    return_sentence_level=return_sentence_level,
+                                    pred_type=self.config.get("mode"),
+                                    use_multi=self.config.get("use_multi"),
+                                    multi_threshold=self.config.get("multi_threshold"))
+
+    def compute_loss(self, pred: OutputData, target: InputData):
+        """compute loss
+
+        Args:
+            pred (OutputData): output logits data
+            target (InputData): input golden data
+
+        Returns:
+            Tensor: loss result
+        """
+        _CRF = None
+        if self.config.get("use_crf"):
+            _CRF = self.CRF
+        return decoder_utils.compute_loss(pred, target, criterion_type=self.config["mode"],
+                                          use_crf=_CRF is not None,
+                                          ignore_index=self.config["ignore_index"],
+                                          use_multi=self.config.get("use_multi"),
+                                          loss_fn=self.loss_fn,
+                                          CRF=_CRF)
+
+
+class LinearClassifier(BaseClassifier):
+    """
+    Decoder structure based on Linear.
+    """
+    def __init__(self, **config):
+        """Construction function for LinearClassifier
+        
+        Args:
+            config (dict):
+                input_dim (int): hidden state dim.
+                use_slot (bool): whether to classify slot label.
+                slot_label_num (int, optional): the number of slot label. Enabled if use_slot is True.
+                use_intent (bool): whether to classify intent label.
+                intent_label_num (int, optional): the number of intent label. Enabled if use_intent is True.
+                use_crf (bool): whether to use crf for slot.
+        """
+        super().__init__(**config)
+        self.config = config
+        if config.get("use_slot"):
+            self.slot_classifier = nn.Linear(config["input_dim"], config["slot_label_num"])
+            if self.config.get("use_crf"):
+                self.CRF = CRF(num_tags=config["slot_label_num"], batch_first=True)
+        if config.get("use_intent"):
+            self.intent_classifier = nn.Linear(config["input_dim"], config["intent_label_num"])
+
+    def forward(self, hidden: HiddenData):
+        if self.config.get("use_intent"):
+            return ClassifierOutputData(self.intent_classifier(hidden.get_intent_hidden_state()))
+        if self.config.get("use_slot"):
+            return ClassifierOutputData(self.slot_classifier(hidden.get_slot_hidden_state()))
+
+
+
+class AutoregressiveLSTMClassifier(BaseClassifier):
+    """
+    Decoder structure based on unidirectional LSTM.
+    """
+
+    def __init__(self, **config):
+        """ Construction function for Decoder.
+
+        Args:
+            config (dict):
+                input_dim (int): input dimension of Decoder. In fact, it's encoder hidden size.
+                use_slot (bool): whether to classify slot label.
+                slot_label_num (int, optional): the number of slot label. Enabled if use_slot is True.
+                use_intent (bool): whether to classify intent label.
+                intent_label_num (int, optional): the number of intent label. Enabled if use_intent is True.
+                use_crf (bool): whether to use crf for slot.
+                hidden_dim (int): hidden dimension of iterative LSTM.
+                embedding_dim (int): if it's not None, the input and output are relevant.
+                dropout_rate (float): dropout rate of network which is only useful for embedding.
+        """
+
+        super(AutoregressiveLSTMClassifier, self).__init__(**config)
+        if config.get("use_slot") and config.get("use_crf"):
+            self.CRF = CRF(num_tags=config["slot_label_num"], batch_first=True)
+        self.input_dim = config["input_dim"]
+        self.hidden_dim = config["hidden_dim"]
+        if config.get("use_intent"):
+            self.output_dim = config["intent_label_num"]
+        if config.get("use_slot"):
+            self.output_dim = config["slot_label_num"]
+        self.dropout_rate = config["dropout_rate"]
+        self.embedding_dim = config.get("embedding_dim")
+        self.force_ratio = config.get("force_ratio")
+        self.config = config
+        self.ignore_index = config.get("ignore_index") if config.get("ignore_index") is not None else -100
+        # If embedding_dim is not None, the output and input
+        # of this structure is relevant.
+        if self.embedding_dim is not None:
+            self.embedding_layer = nn.Embedding(self.output_dim, self.embedding_dim)
+            self.init_tensor = nn.Parameter(
+                torch.randn(1, self.embedding_dim),
+                requires_grad=True
+            )
+
+        # Make sure the input dimension of iterative LSTM.
+        if self.embedding_dim is not None:
+            lstm_input_dim = self.input_dim + self.embedding_dim
+        else:
+            lstm_input_dim = self.input_dim
+
+        # Network parameter definition.
+        self.dropout_layer = nn.Dropout(self.dropout_rate)
+        self.lstm_layer = nn.LSTM(
+            input_size=lstm_input_dim,
+            hidden_size=self.hidden_dim,
+            batch_first=True,
+            bidirectional=self.config["bidirectional"],
+            dropout=self.dropout_rate,
+            num_layers=self.config["layer_num"]
+        )
+        self.linear_layer = nn.Linear(
+            self.hidden_dim,
+            self.output_dim
+        )
+        # self.loss_fn = CrossEntropyLoss(ignore_index=self.ignore_index)
+
+    def forward(self, hidden: HiddenData, internal_interaction=None, **interaction_args):
+        """ Forward process for decoder.
+
+        :param internal_interaction:
+        :param hidden:
+        :return: is distribution of prediction labels.
+        """
+        input_tensor = hidden.slot_hidden
+        seq_lens = hidden.inputs.attention_mask.sum(-1).detach().cpu().tolist()
+        output_tensor_list, sent_start_pos = [], 0
+        input_tensor = pack_sequence(input_tensor, seq_lens)
+        forced_input = None
+        if self.training:
+            if random.random() < self.force_ratio:
+                if self.config["mode"]=="slot":
+
+                    forced_slot = pack_sequence(hidden.inputs.slot, seq_lens)
+                    temp_slot = []
+                    for index, x in enumerate(forced_slot):
+                        if index == 0:
+                            temp_slot.append(x.reshape(1))
+                        elif x == self.ignore_index:
+                            temp_slot.append(temp_slot[-1])
+                        else:
+                            temp_slot.append(x.reshape(1))
+                    forced_input = torch.cat(temp_slot, 0)
+                if self.config["mode"]=="token-level-intent":
+                    forced_intent = hidden.inputs.intent.unsqueeze(1).repeat(1, hidden.inputs.slot.shape[1])
+                    forced_input = pack_sequence(forced_intent, seq_lens)
+        if self.embedding_dim is None or forced_input is not None:
+
+            for sent_i in range(0, len(seq_lens)):
+                sent_end_pos = sent_start_pos + seq_lens[sent_i]
+
+                # Segment input hidden tensors.
+                seg_hiddens = input_tensor[sent_start_pos: sent_end_pos, :]
+
+                if self.embedding_dim is not None and forced_input is not None:
+                    if seq_lens[sent_i] > 1:
+                        seg_forced_input = forced_input[sent_start_pos: sent_end_pos]
+
+                        seg_forced_tensor = self.embedding_layer(seg_forced_input)[:-1]
+                        seg_prev_tensor = torch.cat([self.init_tensor, seg_forced_tensor], dim=0)
+                    else:
+                        seg_prev_tensor = self.init_tensor
+
+                    # Concatenate forced target tensor.
+                    combined_input = torch.cat([seg_hiddens, seg_prev_tensor], dim=1)
+                else:
+                    combined_input = seg_hiddens
+                dropout_input = self.dropout_layer(combined_input)
+                lstm_out, _ = self.lstm_layer(dropout_input.view(1, seq_lens[sent_i], -1))
+                if internal_interaction is not None:
+                    interaction_args["sent_id"] = sent_i
+                    lstm_out = internal_interaction(torch.transpose(lstm_out, 0, 1), **interaction_args)[:, 0]
+                linear_out = self.linear_layer(lstm_out.view(seq_lens[sent_i], -1))
+
+                output_tensor_list.append(linear_out)
+                sent_start_pos = sent_end_pos
+        else:
+            for sent_i in range(0, len(seq_lens)):
+                prev_tensor = self.init_tensor
+
+                # It's necessary to remember h and c state
+                # when output prediction every single step.
+                last_h, last_c = None, None
+
+                sent_end_pos = sent_start_pos + seq_lens[sent_i]
+                for word_i in range(sent_start_pos, sent_end_pos):
+                    seg_input = input_tensor[[word_i], :]
+                    combined_input = torch.cat([seg_input, prev_tensor], dim=1)
+                    dropout_input = self.dropout_layer(combined_input).view(1, 1, -1)
+                    if last_h is None and last_c is None:
+                        lstm_out, (last_h, last_c) = self.lstm_layer(dropout_input)
+                    else:
+                        lstm_out, (last_h, last_c) = self.lstm_layer(dropout_input, (last_h, last_c))
+
+                    if internal_interaction is not None:
+                        interaction_args["sent_id"] = sent_i
+                        lstm_out = internal_interaction(lstm_out, **interaction_args)[:, 0]
+
+                    lstm_out = self.linear_layer(lstm_out.view(1, -1))
+                    output_tensor_list.append(lstm_out)
+
+                    _, index = lstm_out.topk(1, dim=1)
+                    prev_tensor = self.embedding_layer(index).view(1, -1)
+                sent_start_pos = sent_end_pos
+        seq_unpacked = unpack_sequence(torch.cat(output_tensor_list, dim=0), seq_lens)
+        # TODO: 都支持softmax
+        if self.config.get("use_multi"):
+            pred_output = ClassifierOutputData(seq_unpacked)
+        else:
+            pred_output = ClassifierOutputData(F.log_softmax(seq_unpacked, dim=-1))
+        return pred_output
+
+
+class MLPClassifier(BaseClassifier):
+    """
+    Decoder structure based on MLP.
+    """
+    def __init__(self, **config):
+        """ Construction function for Decoder.
+
+        Args:
+            config (dict):
+                use_slot (bool): whether to classify slot label.
+                use_intent (bool): whether to classify intent label.
+                mlp (List):
+                
+                    - _model_target_: torch.nn.Linear
+                    
+                    in_features (int): input feature dim
+                      
+                    out_features (int): output feature dim
+                      
+                    -  _model_target_: torch.nn.LeakyReLU
+                    
+                    negative_slope: 0.2
+                    
+                    - ...
+        """
+        super(MLPClassifier, self).__init__(**config)
+        self.config = config
+        for i, x in enumerate(config["mlp"]):
+            if isinstance(x.get("in_features"), str):
+                config["mlp"][i]["in_features"] = self.config[x["in_features"][1:-1]]
+            if isinstance(x.get("out_features"), str):
+                config["mlp"][i]["out_features"] = self.config[x["out_features"][1:-1]]
+        mlp = [instantiate(x) for x in config["mlp"]]
+        self.seq = nn.Sequential(*mlp)
+
+
+    def forward(self, hidden: HiddenData):
+        if self.config.get("use_intent"):
+            res = self.seq(hidden.intent_hidden)
+        else:
+            res = self.seq(hidden.slot_hidden)
+        return ClassifierOutputData(res)

model/decoder/decoder_utils.py

@@ -0,0 +1,155 @@
+from typing import List
+import torch
+
+from common import utils
+from common.utils import OutputData, InputData
+from torch import Tensor
+
+def argmax_for_seq_len(inputs, seq_lens, padding_value=-100):
+    packed_inputs = utils.pack_sequence(inputs, seq_lens)
+    outputs = torch.argmax(packed_inputs, dim=-1, keepdim=True)
+    return utils.unpack_sequence(outputs, seq_lens, padding_value).squeeze(-1)
+
+
+def decode(output: OutputData,
+           target: InputData = None,
+           pred_type="slot",
+           multi_threshold=0.5,
+           ignore_index=-100,
+           return_list=True,
+           return_sentence_level=True,
+           use_multi=False,
+           use_crf=False,
+           CRF=None) -> List or Tensor:
+    """ decode output logits
+
+    Args:
+        output (OutputData): output logits data
+        target (InputData, optional): input data with attention mask. Defaults to None.
+        pred_type (str, optional): prediction type in ["slot", "intent", "token-level-intent"]. Defaults to "slot".
+        multi_threshold (float, optional): multi intent decode threshold. Defaults to 0.5.
+        ignore_index (int, optional): align and pad token with ignore index. Defaults to -100.
+        return_list (bool, optional): if True return list else return torch Tensor. Defaults to True.
+        return_sentence_level (bool, optional): if True decode sentence level intent else decode token level intent. Defaults to True.
+        use_multi (bool, optional): whether to decode to multi intent. Defaults to False.
+        use_crf (bool, optional): whether to use crf. Defaults to False.
+        CRF (CRF, optional): CRF function. Defaults to None.
+
+    Returns:
+        List or Tensor: decoded sequence ids
+    """
+    if pred_type == "slot":
+        inputs = output.slot_ids
+    else:
+        inputs = output.intent_ids
+
+    if pred_type == "slot":
+        if not use_multi:
+            if use_crf:
+                res = CRF.decode(inputs, mask=target.attention_mask)
+            else:
+                res = torch.argmax(inputs, dim=-1)
+        else:
+            raise NotImplementedError("Multi-slot prediction is not supported.")
+    elif pred_type == "intent":
+        if not use_multi:
+            res = torch.argmax(inputs, dim=-1)
+        else:
+            res = (torch.sigmoid(inputs) > multi_threshold).nonzero()
+            if return_list:
+                res_index = res.detach().cpu().tolist()
+                res_list = [[] for _ in range(len(target.seq_lens))]
+                for item in res_index:
+                    res_list[item[0]].append(item[1])
+                return res_list
+            else:
+                return res
+    elif pred_type == "token-level-intent":
+        if not use_multi:
+            res = torch.argmax(inputs, dim=-1)
+            if not return_sentence_level:
+                return res
+            if return_list:
+                res = res.detach().cpu().tolist()
+            attention_mask = target.attention_mask
+            for i in range(attention_mask.shape[0]):
+                temp = []
+                for j in range(attention_mask.shape[1]):
+                    if attention_mask[i][j] == 1:
+                        temp.append(res[i][j])
+                    else:
+                        break
+                res[i] = temp
+            return [max(it, key=lambda v: it.count(v)) for it in res]
+        else:
+            seq_lens = target.seq_lens
+
+            if not return_sentence_level:
+                token_res = torch.cat([
+                    torch.sigmoid(inputs[i, 0:seq_lens[i], :]) > multi_threshold
+                    for i in range(len(seq_lens))],
+                    dim=0)
+                return utils.unpack_sequence(token_res, seq_lens, padding_value=ignore_index)
+
+            intent_index_sum = torch.cat([
+                torch.sum(torch.sigmoid(inputs[i, 0:seq_lens[i], :]) > multi_threshold, dim=0).unsqueeze(0)
+                for i in range(len(seq_lens))],
+                dim=0)
+
+            res = (intent_index_sum > torch.div(seq_lens, 2, rounding_mode='floor').unsqueeze(1)).nonzero()
+            if return_list:
+                res_index = res.detach().cpu().tolist()
+                res_list = [[] for _ in range(len(seq_lens))]
+                for item in res_index:
+                    res_list[item[0]].append(item[1])
+                return res_list
+            else:
+                return res
+    else:
+        raise NotImplementedError("Prediction mode except ['slot','intent','token-level-intent'] is not supported.")
+    if return_list:
+        res = res.detach().cpu().tolist()
+    return res
+
+
+def compute_loss(pred: OutputData,
+                 target: InputData,
+                 criterion_type="slot",
+                 use_crf=False,
+                 ignore_index=-100,
+                 loss_fn=None,
+                 use_multi=False,
+                 CRF=None):
+    """ compute loss
+
+    Args:
+        pred (OutputData): output logits data
+        target (InputData): input golden data
+        criterion_type (str, optional): criterion type in ["slot", "intent", "token-level-intent"]. Defaults to "slot".
+        ignore_index (int, optional): compute loss with ignore index. Defaults to -100.
+        loss_fn (_type_, optional): loss function. Defaults to None.
+        use_crf (bool, optional): whether to use crf. Defaults to False.
+        CRF (CRF, optional): CRF function. Defaults to None.
+
+    Returns:
+        Tensor: loss result
+    """
+    if criterion_type == "slot":
+        if use_crf:
+            return -1 * CRF(pred.slot_ids, target.slot, target.get_slot_mask(ignore_index).byte())
+        else:
+            pred_slot = utils.pack_sequence(pred.slot_ids, target.seq_lens)
+            target_slot = utils.pack_sequence(target.slot, target.seq_lens)
+            return loss_fn(pred_slot, target_slot)
+    elif criterion_type == "token-level-intent":
+        # TODO: Two decode function
+        intent_target = target.intent.unsqueeze(1)
+        if not use_multi:
+            intent_target = intent_target.repeat(1, pred.intent_ids.shape[1])
+        else:
+            intent_target = intent_target.repeat(1, pred.intent_ids.shape[1], 1)
+        intent_pred = utils.pack_sequence(pred.intent_ids, target.seq_lens)
+        intent_target = utils.pack_sequence(intent_target, target.seq_lens)
+        return loss_fn(intent_pred, intent_target)
+    else:
+        return loss_fn(pred.intent_ids, target.intent)

model/decoder/gl_gin_decoder.py

@@ -0,0 +1,47 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from common.utils import HiddenData, OutputData, InputData
+from model.decoder import BaseDecoder
+from model.decoder.interaction.gl_gin_interaction import LSTMEncoder
+
+
+class IntentEncoder(nn.Module):
+    def __init__(self,input_dim, dropout_rate):
+        super().__init__()
+        self.dropout_rate = dropout_rate
+        self.__intent_lstm = LSTMEncoder(
+            input_dim,
+            input_dim,
+            dropout_rate
+        )
+
+    def forward(self, g_hiddens, seq_lens):
+        intent_lstm_out = self.__intent_lstm(g_hiddens, seq_lens)
+        return F.dropout(intent_lstm_out, p=self.dropout_rate, training=self.training)
+
+
+class GLGINDecoder(BaseDecoder):
+    def __init__(self, intent_classifier, slot_classifier, interaction=None, **config):
+        super().__init__(intent_classifier, slot_classifier, interaction)
+        self.config=config
+        self.intent_encoder = IntentEncoder(self.intent_classifier.config["input_dim"], self.config["dropout_rate"])
+
+    def forward(self, hidden: HiddenData, forced_slot=None, forced_intent=None, differentiable=None):
+        seq_lens = hidden.inputs.attention_mask.sum(-1)
+        intent_lstm_out = self.intent_encoder(hidden.slot_hidden, seq_lens)
+        hidden.update_intent_hidden_state(intent_lstm_out)
+        pred_intent = self.intent_classifier(hidden)
+        intent_index = self.intent_classifier.decode(OutputData(pred_intent, None),hidden.inputs,
+                                                     return_list=False,
+                                                     return_sentence_level=True)
+        slot_hidden = self.interaction(
+            hidden,
+            pred_intent=pred_intent,
+            intent_index=intent_index,
+        )
+        pred_slot = self.slot_classifier(slot_hidden)
+        num_intent = self.intent_classifier.config["intent_label_num"]
+        pred_slot = pred_slot.classifier_output[:, num_intent:]
+        return OutputData(pred_intent, F.log_softmax(pred_slot, dim=1))

model/decoder/interaction/__init__.py

@@ -0,0 +1,10 @@
+from model.decoder.interaction.agif_interaction import AGIFInteraction
+from model.decoder.interaction.base_interaction import BaseInteraction
+from model.decoder.interaction.bi_model_interaction import BiModelInteraction, BiModelWithoutDecoderInteraction
+from model.decoder.interaction.dca_net_interaction import DCANetInteraction
+from model.decoder.interaction.gl_gin_interaction import GLGINInteraction
+from model.decoder.interaction.slot_gated_interaction import SlotGatedInteraction
+from model.decoder.interaction.stack_interaction import StackInteraction
+
+__all__ = ["BaseInteraction", "BiModelInteraction", "BiModelWithoutDecoderInteraction", "DCANetInteraction",
+           "StackInteraction", "SlotGatedInteraction", "AGIFInteraction", "GLGINInteraction"]

model/decoder/interaction/agif_interaction.py

@@ -0,0 +1,132 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from model.decoder.interaction.base_interaction import BaseInteraction
+
+
+class GraphAttentionLayer(nn.Module):
+    """
+    Simple GAT layer, similar to https://arxiv.org/abs/1710.10903
+    """
+
+    def __init__(self, in_features, out_features, dropout, alpha, concat=True):
+        super(GraphAttentionLayer, self).__init__()
+        self.dropout = dropout
+        self.in_features = in_features
+        self.out_features = out_features
+        self.alpha = alpha
+        self.concat = concat
+
+        self.W = nn.Parameter(torch.zeros(size=(in_features, out_features)))
+        nn.init.xavier_uniform_(self.W.data, gain=1.414)
+        self.a = nn.Parameter(torch.zeros(size=(2 * out_features, 1)))
+        nn.init.xavier_uniform_(self.a.data, gain=1.414)
+
+        self.leakyrelu = nn.LeakyReLU(self.alpha)
+
+    def forward(self, input, adj):
+        h = torch.matmul(input, self.W)
+        B, N = h.size()[0], h.size()[1]
+
+        a_input = torch.cat([h.repeat(1, 1, N).view(B, N * N, -1), h.repeat(1, N, 1)], dim=2).view(B, N, -1,
+                                                                                                   2 * self.out_features)
+        e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(3))
+
+        zero_vec = -9e15 * torch.ones_like(e)
+        attention = torch.where(adj > 0, e, zero_vec)
+        attention = F.softmax(attention, dim=2)
+        attention = F.dropout(attention, self.dropout, training=self.training)
+        h_prime = torch.matmul(attention, h)
+
+        if self.concat:
+            return F.elu(h_prime)
+        else:
+            return h_prime
+
+
+class GAT(nn.Module):
+    def __init__(self, nfeat, nhid, nclass, dropout, alpha, nheads, nlayers=2):
+        """Dense version of GAT."""
+        super(GAT, self).__init__()
+        self.dropout = dropout
+        self.nlayers = nlayers
+        self.nheads = nheads
+        self.attentions = [GraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True) for _ in
+                           range(nheads)]
+        for i, attention in enumerate(self.attentions):
+            self.add_module('attention_{}'.format(i), attention)
+        if self.nlayers > 2:
+            for i in range(self.nlayers - 2):
+                for j in range(self.nheads):
+                    self.add_module('attention_{}_{}'.format(i + 1, j),
+                                    GraphAttentionLayer(nhid * nheads, nhid, dropout=dropout, alpha=alpha, concat=True))
+
+        self.out_att = GraphAttentionLayer(nhid * nheads, nclass, dropout=dropout, alpha=alpha, concat=False)
+
+    def forward(self, x, adj):
+        x = F.dropout(x, self.dropout, training=self.training)
+        input = x
+        x = torch.cat([att(x, adj) for att in self.attentions], dim=2)
+        if self.nlayers > 2:
+            for i in range(self.nlayers - 2):
+                temp = []
+                x = F.dropout(x, self.dropout, training=self.training)
+                cur_input = x
+                for j in range(self.nheads):
+                    temp.append(self.__getattr__('attention_{}_{}'.format(i + 1, j))(x, adj))
+                x = torch.cat(temp, dim=2) + cur_input
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = F.elu(self.out_att(x, adj))
+        return x + input
+
+
+def normalize_adj(mx):
+    """
+    Row-normalize matrix  D^{-1}A
+    torch.diag_embed: https://github.com/pytorch/pytorch/pull/12447
+    """
+    mx = mx.float()
+    rowsum = mx.sum(2)
+    r_inv = torch.pow(rowsum, -1)
+    r_inv[torch.isinf(r_inv)] = 0.
+    r_mat_inv = torch.diag_embed(r_inv, 0)
+    mx = r_mat_inv.matmul(mx)
+    return mx
+
+
+class AGIFInteraction(BaseInteraction):
+    def __init__(self, **config):
+        super().__init__(**config)
+        self.intent_embedding = nn.Parameter(
+            torch.FloatTensor(self.config["intent_label_num"], self.config["intent_embedding_dim"]))  # 191, 32
+        nn.init.normal_(self.intent_embedding.data)
+        self.adj = None
+        self.graph = GAT(
+            config["output_dim"],
+            config["hidden_dim"],
+            config["output_dim"],
+            config["dropout_rate"],
+            config["alpha"],
+            config["num_heads"],
+            config["num_layers"])
+
+    def generate_adj_gat(self, index, batch, intent_label_num):
+        intent_idx_ = [[torch.tensor(0)] for i in range(batch)]
+        for item in index:
+            intent_idx_[item[0]].append(item[1] + 1)
+        intent_idx = intent_idx_
+        self.adj = torch.cat([torch.eye(intent_label_num + 1).unsqueeze(0) for i in range(batch)])
+        for i in range(batch):
+            for j in intent_idx[i]:
+                self.adj[i, j, intent_idx[i]] = 1.
+        if self.config["row_normalized"]:
+            self.adj = normalize_adj(self.adj)
+        self.adj = self.adj.to(self.intent_embedding.device)
+
+    def forward(self, encode_hidden, **interaction_args):
+        if self.adj is None or interaction_args["sent_id"] == 0:
+            self.generate_adj_gat(interaction_args["intent_index"], interaction_args["batch_size"], interaction_args["intent_label_num"])
+        lstm_out = torch.cat((encode_hidden,
+                              self.intent_embedding.unsqueeze(0).repeat(encode_hidden.shape[0], 1, 1)), dim=1)
+        return self.graph(lstm_out, self.adj[interaction_args["sent_id"]])

model/decoder/interaction/base_interaction.py

@@ -0,0 +1,9 @@
+from torch import nn
+
+class BaseInteraction(nn.Module):
+    def __init__(self, **config):
+        super().__init__()
+        self.config = config
+
+    def forward(self, hidden1, hidden2):
+        NotImplementedError("no implemented")

model/decoder/interaction/bi_model_interaction.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from common.utils import HiddenData
+from model.decoder.interaction.base_interaction import BaseInteraction
+
+
+class BiModelInteraction(BaseInteraction):
+    def __init__(self, **config):
+        super().__init__(**config)
+        self.intent_lstm = nn.LSTM(input_size=self.config["input_dim"], hidden_size=self.config["output_dim"],
+                                   batch_first=True,
+                                   num_layers=1)
+        self.slot_lstm = nn.LSTM(input_size=self.config["input_dim"] + self.config["output_dim"],
+                                 hidden_size=self.config["output_dim"], num_layers=1)
+
+    def forward(self, encode_hidden: HiddenData, **kwargs):
+        slot_hidden = encode_hidden.get_slot_hidden_state()
+        intent_hidden_detached = encode_hidden.get_intent_hidden_state().clone().detach()
+        seq_lens = encode_hidden.inputs.attention_mask.sum(-1)
+        batch = slot_hidden.size(0)
+        length = slot_hidden.size(1)
+        dec_init_out = torch.zeros(batch, 1, self.config["output_dim"]).to(slot_hidden.device)
+        hidden_state = (torch.zeros(1, 1, self.config["output_dim"]).to(slot_hidden.device), torch.zeros(1, 1, self.config["output_dim"]).to(slot_hidden.device))
+        slot_hidden = torch.cat((slot_hidden, intent_hidden_detached), dim=-1).transpose(1,
+                                                                                         0)  # 50 x batch x feature_size
+        slot_drop = F.dropout(slot_hidden, self.config["dropout_rate"])
+        all_out = []
+        for i in range(length):
+            if i == 0:
+                out, hidden_state = self.slot_lstm(torch.cat((slot_drop[i].unsqueeze(1), dec_init_out), dim=-1),
+                                                   hidden_state)
+            else:
+                out, hidden_state = self.slot_lstm(torch.cat((slot_drop[i].unsqueeze(1), out), dim=-1), hidden_state)
+            all_out.append(out)
+        slot_output = torch.cat(all_out, dim=1)  # batch x 50 x feature_size
+
+        intent_hidden = torch.cat((encode_hidden.get_intent_hidden_state(),
+                                   encode_hidden.get_slot_hidden_state().clone().detach()),
+                                  dim=-1)
+        intent_drop = F.dropout(intent_hidden, self.config["dropout_rate"])
+        intent_lstm_output, _ = self.intent_lstm(intent_drop)
+        intent_output = F.dropout(intent_lstm_output, self.config["dropout_rate"])
+        output_list = []
+        for index, slen in enumerate(seq_lens):
+            output_list.append(intent_output[index, slen - 1, :].unsqueeze(0))
+
+        encode_hidden.update_intent_hidden_state(torch.cat(output_list, dim=0))
+        encode_hidden.update_slot_hidden_state(slot_output)
+
+        return encode_hidden
+
+
+class BiModelWithoutDecoderInteraction(BaseInteraction):
+    def forward(self, encode_hidden: HiddenData, **kwargs):
+        slot_hidden = encode_hidden.get_slot_hidden_state()
+        intent_hidden_detached = encode_hidden.get_intent_hidden_state().clone().detach()
+        seq_lens = encode_hidden.inputs.attention_mask.sum(-1)
+        slot_hidden = torch.cat((slot_hidden, intent_hidden_detached), dim=-1)  # 50 x batch x feature_size
+        slot_output = F.dropout(slot_hidden, self.config["dropout_rate"])
+
+        intent_hidden = torch.cat((encode_hidden.get_intent_hidden_state(),
+                                   encode_hidden.get_slot_hidden_state().clone().detach()),
+                                  dim=-1)
+        intent_output = F.dropout(intent_hidden, self.config["dropout_rate"])
+        output_list = []
+        for index, slen in enumerate(seq_lens):
+            output_list.append(intent_output[index, slen - 1, :].unsqueeze(0))
+
+        encode_hidden.update_intent_hidden_state(torch.cat(output_list, dim=0))
+        encode_hidden.update_slot_hidden_state(slot_output)
+
+        return encode_hidden

model/decoder/interaction/dca_net_interaction.py

@@ -0,0 +1,176 @@
+import math
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.nn import LayerNorm
+
+from common.utils import HiddenData
+from model.decoder.interaction import BaseInteraction
+
+
+class DCANetInteraction(BaseInteraction):
+    def __init__(self, **config):
+        super().__init__(**config)
+        self.I_S_Emb = Label_Attention()
+        self.T_block1 = I_S_Block(self.config["input_dim"], self.config["attention_dropout"], self.config["num_attention_heads"])
+        self.T_block2 = I_S_Block(self.config["input_dim"], self.config["attention_dropout"], self.config["num_attention_heads"])
+
+    def forward(self, encode_hidden: HiddenData, **kwargs):
+        mask = encode_hidden.inputs.attention_mask
+        H = encode_hidden.slot_hidden
+        H_I, H_S = self.I_S_Emb(H, H, kwargs["intent_emb"], kwargs["slot_emb"])
+        H_I, H_S = self.T_block1(H_I + H, H_S + H, mask)
+        H_I_1, H_S_1 = self.I_S_Emb(H_I, H_S, kwargs["intent_emb"], kwargs["slot_emb"])
+        H_I, H_S = self.T_block2(H_I + H_I_1, H_S + H_S_1, mask)
+        encode_hidden.update_intent_hidden_state(F.max_pool1d((H_I + H).transpose(1, 2), H_I.size(1)).squeeze(2))
+        encode_hidden.update_slot_hidden_state(H_S + H)
+        return encode_hidden
+
+
+class Label_Attention(nn.Module):
+    def __init__(self):
+        super(Label_Attention, self).__init__()
+
+    def forward(self, input_intent, input_slot, intent_emb, slot_emb):
+        self.W_intent_emb = intent_emb.intent_classifier.weight
+        self.W_slot_emb = slot_emb.slot_classifier.weight
+        intent_score = torch.matmul(input_intent, self.W_intent_emb.t())
+        slot_score = torch.matmul(input_slot, self.W_slot_emb.t())
+        intent_probs = nn.Softmax(dim=-1)(intent_score)
+        slot_probs = nn.Softmax(dim=-1)(slot_score)
+        intent_res = torch.matmul(intent_probs, self.W_intent_emb)
+        slot_res = torch.matmul(slot_probs, self.W_slot_emb)
+
+        return intent_res, slot_res
+
+
+class I_S_Block(nn.Module):
+    def __init__(self, hidden_size, attention_dropout, num_attention_heads):
+        super(I_S_Block, self).__init__()
+        self.I_S_Attention = I_S_SelfAttention(hidden_size, 2 * hidden_size, hidden_size, attention_dropout, num_attention_heads)
+        self.I_Out = SelfOutput(hidden_size, attention_dropout)
+        self.S_Out = SelfOutput(hidden_size, attention_dropout)
+        self.I_S_Feed_forward = Intermediate_I_S(hidden_size, hidden_size, attention_dropout)
+
+    def forward(self, H_intent_input, H_slot_input, mask):
+        H_slot, H_intent = self.I_S_Attention(H_intent_input, H_slot_input, mask)
+        H_slot = self.S_Out(H_slot, H_slot_input)
+        H_intent = self.I_Out(H_intent, H_intent_input)
+        H_intent, H_slot = self.I_S_Feed_forward(H_intent, H_slot)
+
+        return H_intent, H_slot
+
+
+class I_S_SelfAttention(nn.Module):
+    def __init__(self, input_size, hidden_size, out_size, attention_dropout, num_attention_heads):
+        super(I_S_SelfAttention, self).__init__()
+
+        self.num_attention_heads = num_attention_heads
+        self.attention_head_size = int(hidden_size / self.num_attention_heads)
+
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+        self.out_size = out_size
+        self.query = nn.Linear(input_size, self.all_head_size)
+        self.query_slot = nn.Linear(input_size, self.all_head_size)
+        self.key = nn.Linear(input_size, self.all_head_size)
+        self.key_slot = nn.Linear(input_size, self.all_head_size)
+        self.value = nn.Linear(input_size, self.out_size)
+        self.value_slot = nn.Linear(input_size, self.out_size)
+        self.dropout = nn.Dropout(attention_dropout)
+
+    def transpose_for_scores(self, x):
+        last_dim = int(x.size()[-1] / self.num_attention_heads)
+        new_x_shape = x.size()[:-1] + (self.num_attention_heads, last_dim)
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(self, intent, slot, mask):
+        extended_attention_mask = mask.unsqueeze(1).unsqueeze(2)
+
+        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+        attention_mask = (1.0 - extended_attention_mask) * -10000.0
+
+        mixed_query_layer = self.query(intent)
+        mixed_key_layer = self.key(slot)
+        mixed_value_layer = self.value(slot)
+
+        mixed_query_layer_slot = self.query_slot(slot)
+        mixed_key_layer_slot = self.key_slot(intent)
+        mixed_value_layer_slot = self.value_slot(intent)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+        query_layer_slot = self.transpose_for_scores(mixed_query_layer_slot)
+        key_layer = self.transpose_for_scores(mixed_key_layer)
+        key_layer_slot = self.transpose_for_scores(mixed_key_layer_slot)
+        value_layer = self.transpose_for_scores(mixed_value_layer)
+        value_layer_slot = self.transpose_for_scores(mixed_value_layer_slot)
+
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        # attention_scores_slot = torch.matmul(query_slot, key_slot.transpose(1,0))
+        attention_scores_slot = torch.matmul(query_layer_slot, key_layer_slot.transpose(-1, -2))
+        attention_scores_slot = attention_scores_slot / math.sqrt(self.attention_head_size)
+        attention_scores_intent = attention_scores + attention_mask
+
+        attention_scores_slot = attention_scores_slot + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs_slot = nn.Softmax(dim=-1)(attention_scores_slot)
+        attention_probs_intent = nn.Softmax(dim=-1)(attention_scores_intent)
+
+        attention_probs_slot = self.dropout(attention_probs_slot)
+        attention_probs_intent = self.dropout(attention_probs_intent)
+
+        context_layer_slot = torch.matmul(attention_probs_slot, value_layer_slot)
+        context_layer_intent = torch.matmul(attention_probs_intent, value_layer)
+
+        context_layer = context_layer_slot.permute(0, 2, 1, 3).contiguous()
+        context_layer_intent = context_layer_intent.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.out_size,)
+        new_context_layer_shape_intent = context_layer_intent.size()[:-2] + (self.out_size,)
+
+        context_layer = context_layer.view(*new_context_layer_shape)
+        context_layer_intent = context_layer_intent.view(*new_context_layer_shape_intent)
+        return context_layer, context_layer_intent
+
+
+class SelfOutput(nn.Module):
+    def __init__(self, hidden_size, hidden_dropout_prob):
+        super(SelfOutput, self).__init__()
+        self.dense = nn.Linear(hidden_size, hidden_size)
+        self.LayerNorm = LayerNorm(hidden_size, eps=1e-12)
+        self.dropout = nn.Dropout(hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class Intermediate_I_S(nn.Module):
+    def __init__(self, intermediate_size, hidden_size, attention_dropout):
+        super(Intermediate_I_S, self).__init__()
+        self.dense_in = nn.Linear(hidden_size * 6, intermediate_size)
+        self.intermediate_act_fn = nn.ReLU()
+        self.dense_out = nn.Linear(intermediate_size, hidden_size)
+        self.LayerNorm_I = LayerNorm(hidden_size, eps=1e-12)
+        self.LayerNorm_S = LayerNorm(hidden_size, eps=1e-12)
+        self.dropout = nn.Dropout(attention_dropout)
+
+    def forward(self, hidden_states_I, hidden_states_S):
+        hidden_states_in = torch.cat([hidden_states_I, hidden_states_S], dim=2)
+        batch_size, max_length, hidden_size = hidden_states_in.size()
+        h_pad = torch.zeros(batch_size, 1, hidden_size).to(hidden_states_I.device)
+        h_left = torch.cat([h_pad, hidden_states_in[:, :max_length - 1, :]], dim=1)
+        h_right = torch.cat([hidden_states_in[:, 1:, :], h_pad], dim=1)
+        hidden_states_in = torch.cat([hidden_states_in, h_left, h_right], dim=2)
+
+        hidden_states = self.dense_in(hidden_states_in)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        hidden_states = self.dense_out(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states_I_NEW = self.LayerNorm_I(hidden_states + hidden_states_I)
+        hidden_states_S_NEW = self.LayerNorm_S(hidden_states + hidden_states_S)
+        return hidden_states_I_NEW, hidden_states_S_NEW

model/decoder/interaction/gl_gin_interaction.py

@@ -0,0 +1,227 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
+
+from common.utils import HiddenData, ClassifierOutputData
+from model.decoder.interaction import BaseInteraction
+
+
+class LSTMEncoder(nn.Module):
+    """
+    Encoder structure based on bidirectional LSTM.
+    """
+
+    def __init__(self, embedding_dim, hidden_dim, dropout_rate):
+        super(LSTMEncoder, self).__init__()
+
+        # Parameter recording.
+        self.__embedding_dim = embedding_dim
+        self.__hidden_dim = hidden_dim // 2
+        self.__dropout_rate = dropout_rate
+
+        # Network attributes.
+        self.__dropout_layer = nn.Dropout(self.__dropout_rate)
+        self.__lstm_layer = nn.LSTM(
+            input_size=self.__embedding_dim,
+            hidden_size=self.__hidden_dim,
+            batch_first=True,
+            bidirectional=True,
+            dropout=self.__dropout_rate,
+            num_layers=1
+        )
+
+    def forward(self, embedded_text, seq_lens):
+        """ Forward process for LSTM Encoder.
+
+        (batch_size, max_sent_len)
+        -> (batch_size, max_sent_len, word_dim)
+        -> (batch_size, max_sent_len, hidden_dim)
+
+        :param embedded_text: padded and embedded input text.
+        :param seq_lens: is the length of original input text.
+        :return: is encoded word hidden vectors.
+        """
+
+        # Padded_text should be instance of LongTensor.
+        dropout_text = self.__dropout_layer(embedded_text)
+
+        # Pack and Pad process for input of variable length.
+        packed_text = pack_padded_sequence(dropout_text, seq_lens.cpu(), batch_first=True, enforce_sorted=False)
+        lstm_hiddens, (h_last, c_last) = self.__lstm_layer(packed_text)
+        padded_hiddens, _ = pad_packed_sequence(lstm_hiddens, batch_first=True)
+
+        return padded_hiddens
+
+
+class GraphAttentionLayer(nn.Module):
+    """
+    Simple GAT layer, similar to https://arxiv.org/abs/1710.10903
+    """
+
+    def __init__(self, in_features, out_features, dropout, alpha, concat=True):
+        super(GraphAttentionLayer, self).__init__()
+        self.dropout = dropout
+        self.in_features = in_features
+        self.out_features = out_features
+        self.alpha = alpha
+        self.concat = concat
+
+        self.W = nn.Parameter(torch.zeros(size=(in_features, out_features)))
+        nn.init.xavier_uniform_(self.W.data, gain=1.414)
+        self.a = nn.Parameter(torch.zeros(size=(2 * out_features, 1)))
+        nn.init.xavier_uniform_(self.a.data, gain=1.414)
+
+        self.leakyrelu = nn.LeakyReLU(self.alpha)
+
+    def forward(self, input, adj):
+        h = torch.matmul(input, self.W)
+        B, N = h.size()[0], h.size()[1]
+
+        a_input = torch.cat([h.repeat(1, 1, N).view(B, N * N, -1), h.repeat(1, N, 1)], dim=2).view(B, N, -1,
+                                                                                                   2 * self.out_features)
+        e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(3))
+
+        zero_vec = -9e15 * torch.ones_like(e)
+        attention = torch.where(adj > 0, e, zero_vec)
+        attention = F.softmax(attention, dim=2)
+        attention = F.dropout(attention, self.dropout, training=self.training)
+        h_prime = torch.matmul(attention, h)
+
+        if self.concat:
+            return F.elu(h_prime)
+        else:
+            return h_prime
+
+
+class GAT(nn.Module):
+    def __init__(self, nfeat, nhid, nclass, dropout, alpha, nheads, nlayers=2):
+        """Dense version of GAT."""
+        super(GAT, self).__init__()
+        self.dropout = dropout
+        self.nlayers = nlayers
+        self.nheads = nheads
+        self.attentions = [GraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True) for _ in
+                           range(nheads)]
+        for i, attention in enumerate(self.attentions):
+            self.add_module('attention_{}'.format(i), attention)
+        if self.nlayers > 2:
+            for i in range(self.nlayers - 2):
+                for j in range(self.nheads):
+                    self.add_module('attention_{}_{}'.format(i + 1, j),
+                                    GraphAttentionLayer(nhid * nheads, nhid, dropout=dropout, alpha=alpha, concat=True))
+
+        self.out_att = GraphAttentionLayer(nhid * nheads, nclass, dropout=dropout, alpha=alpha, concat=False)
+
+    def forward(self, x, adj):
+        x = F.dropout(x, self.dropout, training=self.training)
+        input = x
+        x = torch.cat([att(x, adj) for att in self.attentions], dim=2)
+        if self.nlayers > 2:
+            for i in range(self.nlayers - 2):
+                temp = []
+                x = F.dropout(x, self.dropout, training=self.training)
+                cur_input = x
+                for j in range(self.nheads):
+                    temp.append(self.__getattr__('attention_{}_{}'.format(i + 1, j))(x, adj))
+                x = torch.cat(temp, dim=2) + cur_input
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = F.elu(self.out_att(x, adj))
+        return x + input
+
+
+def normalize_adj(mx):
+    """
+    Row-normalize matrix  D^{-1}A
+    torch.diag_embed: https://github.com/pytorch/pytorch/pull/12447
+    """
+    mx = mx.float()
+    rowsum = mx.sum(2)
+    r_inv = torch.pow(rowsum, -1)
+    r_inv[torch.isinf(r_inv)] = 0.
+    r_mat_inv = torch.diag_embed(r_inv, 0)
+    mx = r_mat_inv.matmul(mx)
+    return mx
+
+
+class GLGINInteraction(BaseInteraction):
+    def __init__(self, **config):
+        super().__init__(**config)
+        self.intent_embedding = nn.Parameter(
+            torch.FloatTensor(self.config["intent_label_num"], self.config["intent_embedding_dim"]))  # 191, 32
+        nn.init.normal_(self.intent_embedding.data)
+        self.adj = None
+        self.__slot_lstm = LSTMEncoder(
+            self.config["input_dim"] + self.config["intent_label_num"],
+            config["output_dim"],
+            config["dropout_rate"]
+        )
+        self.__slot_graph = GAT(
+            config["output_dim"],
+            config["hidden_dim"],
+            config["output_dim"],
+            config["dropout_rate"],
+            config["alpha"],
+            config["num_heads"],
+            config["num_layers"])
+
+        self.__global_graph = GAT(
+            config["output_dim"],
+            config["hidden_dim"],
+            config["output_dim"],
+            config["dropout_rate"],
+            config["alpha"],
+            config["num_heads"],
+            config["num_layers"])
+
+    def generate_global_adj_gat(self, seq_len, index, batch, window):
+        global_intent_idx = [[] for i in range(batch)]
+        global_slot_idx = [[] for i in range(batch)]
+        for item in index:
+            global_intent_idx[item[0]].append(item[1])
+
+        for i, len in enumerate(seq_len):
+            global_slot_idx[i].extend(
+                list(range(self.config["intent_label_num"], self.config["intent_label_num"] + len)))
+
+        adj = torch.cat([torch.eye(self.config["intent_label_num"] + max(seq_len)).unsqueeze(0) for i in range(batch)])
+        for i in range(batch):
+            for j in global_intent_idx[i]:
+                adj[i, j, global_slot_idx[i]] = 1.
+                adj[i, j, global_intent_idx[i]] = 1.
+            for j in global_slot_idx[i]:
+                adj[i, j, global_intent_idx[i]] = 1.
+
+        for i in range(batch):
+            for j in range(self.config["intent_label_num"], self.config["intent_label_num"] + seq_len[i]):
+                adj[i, j, max(self.config["intent_label_num"], j - window):min(seq_len[i] + self.config["intent_label_num"], j + window + 1)] = 1.
+
+        if self.config["row_normalized"]:
+            adj = normalize_adj(adj)
+        adj = adj.to(self.intent_embedding.device)
+        return adj
+
+    def generate_slot_adj_gat(self, seq_len, batch, window):
+        slot_idx_ = [[] for i in range(batch)]
+        adj = torch.cat([torch.eye(max(seq_len)).unsqueeze(0) for i in range(batch)])
+        for i in range(batch):
+            for j in range(seq_len[i]):
+                adj[i, j, max(0, j - window):min(seq_len[i], j + window + 1)] = 1.
+        if self.config["row_normalized"]:
+            adj = normalize_adj(adj)
+        adj = adj.to(self.intent_embedding.device)
+        return adj
+
+    def forward(self, encode_hidden: HiddenData, pred_intent: ClassifierOutputData = None, intent_index=None):
+        seq_lens = encode_hidden.inputs.attention_mask.sum(-1)
+        slot_lstm_out = self.__slot_lstm(torch.cat([encode_hidden.slot_hidden, pred_intent.classifier_output], dim=-1),
+                                         seq_lens)
+        global_adj = self.generate_global_adj_gat(seq_lens, intent_index, len(seq_lens),
+                                                  self.config["slot_graph_window"])
+        slot_adj = self.generate_slot_adj_gat(seq_lens, len(seq_lens), self.config["slot_graph_window"])
+        batch = len(seq_lens)
+        slot_graph_out = self.__slot_graph(slot_lstm_out, slot_adj)
+        intent_in = self.intent_embedding.unsqueeze(0).repeat(batch, 1, 1)
+        global_graph_in = torch.cat([intent_in, slot_graph_out], dim=1)
+        encode_hidden.update_slot_hidden_state(self.__global_graph(global_graph_in, global_adj))
+        return encode_hidden

model/decoder/interaction/slot_gated_interaction.py

@@ -0,0 +1,59 @@
+import math
+
+import einops
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.nn import LayerNorm
+
+from common.utils import HiddenData
+from model.decoder.interaction import BaseInteraction
+
+
+class SlotGatedInteraction(BaseInteraction):
+    def __init__(self, **config):
+        super().__init__(**config)
+        self.intent_linear = nn.Linear(self.config["input_dim"],1, bias=False)
+        self.slot_linear1 = nn.Linear(self.config["input_dim"],1, bias=False)
+        self.slot_linear2 = nn.Linear(self.config["input_dim"],1, bias=False)
+        self.remove_slot_attn = self.config["remove_slot_attn"]
+        self.slot_gate = SlotGate(**config)
+
+    def forward(self, encode_hidden: HiddenData, **kwargs):
+        input_hidden = encode_hidden.get_slot_hidden_state()
+
+        seq_lens = encode_hidden.inputs.attention_mask.sum(-1)
+        output_list = []
+        for index, slen in enumerate(seq_lens):
+            output_list.append(input_hidden[index, slen - 1, :].unsqueeze(0))
+        intent_input = torch.cat(output_list, dim=0)
+        e_I = torch.tanh(self.intent_linear(intent_input)).squeeze(1)
+        alpha_I = einops.repeat(e_I, 'b -> b h', h=intent_input.shape[-1])
+        c_I = alpha_I * intent_input
+        intent_hidden = intent_input+c_I
+        if not self.remove_slot_attn:
+            # slot attention
+            h_k = einops.repeat(self.slot_linear1(input_hidden), 'b l h -> b l c h', c=input_hidden.shape[1])
+            h_i = einops.repeat(self.slot_linear2(input_hidden), 'b l h -> b l c h', c=input_hidden.shape[1]).transpose(1,2)
+            e_S = torch.tanh(h_k + h_i)
+            alpha_S = torch.softmax(e_S, dim=2).squeeze(3)
+            alpha_S = einops.repeat(alpha_S, 'b l1 l2 -> b l1 l2 h', h=input_hidden.shape[-1])
+            map_input_hidden = einops.repeat(input_hidden, 'b l h -> b l c h', c=input_hidden.shape[1])
+            c_S = torch.sum(alpha_S * map_input_hidden, dim=2)
+        else:
+            c_S = input_hidden
+        slot_hidden = input_hidden + c_S * self.slot_gate(c_S,c_I)
+        encode_hidden.update_intent_hidden_state(intent_hidden)
+        encode_hidden.update_slot_hidden_state(slot_hidden)
+        return encode_hidden
+
+class SlotGate(nn.Module):
+    def __init__(self, **config):
+        super().__init__()
+        self.linear = nn.Linear(config["input_dim"], config["output_dim"],bias=False)
+        self.v = nn.Parameter(torch.rand(size=[1]))
+
+    def forward(self, slot_context, intent_context):
+        intent_gate = self.linear(intent_context)
+        intent_gate = einops.repeat(intent_gate, 'b h -> b l h', l=slot_context.shape[1])
+        return self.v * torch.tanh(slot_context + intent_gate)

model/decoder/interaction/stack_interaction.py

@@ -0,0 +1,36 @@
+import os
+import torch
+from torch import nn
+
+from common import utils
+from common.utils import ClassifierOutputData, HiddenData
+from model.decoder.interaction.base_interaction import BaseInteraction
+
+
+class StackInteraction(BaseInteraction):
+    def __init__(self, **config):
+        super().__init__(**config)
+        self.intent_embedding = nn.Embedding(
+            self.config["intent_label_num"], self.config["intent_label_num"]
+        )
+        self.differentiable = config.get("differentiable")
+        self.intent_embedding.weight.data = torch.eye(
+            self.config["intent_label_num"])
+        self.intent_embedding.weight.requires_grad = False
+
+    def forward(self, intent_output: ClassifierOutputData, encode_hidden: HiddenData):
+        if not self.differentiable:
+            _, idx_intent = intent_output.classifier_output.topk(1, dim=-1)
+            feed_intent = self.intent_embedding(idx_intent.squeeze(2))
+        else:
+            feed_intent = intent_output.classifier_output
+        encode_hidden.update_slot_hidden_state(
+            torch.cat([encode_hidden.get_slot_hidden_state(), feed_intent], dim=-1))
+        return encode_hidden
+
+    @staticmethod
+    def from_configured(configure_name_or_file="stack-interaction", **input_config):
+        return utils.from_configured(configure_name_or_file,
+                                     model_class=StackInteraction,
+                                     config_prefix="./config/decoder/interaction",
+                                     **input_config)

model/encoder/__init__.py

@@ -0,0 +1,5 @@
+from model.encoder.pretrained_encoder import PretrainedEncoder
+from model.encoder.non_pretrained_encoder import NonPretrainedEncoder
+from model.encoder.base_encoder import BiEncoder
+from model.encoder.auto_encoder import AutoEncoder
+__all__ = ["PretrainedEncoder", "NonPretrainedEncoder", "AutoEncoder","BiEncoder"]

model/encoder/auto_encoder.py

@@ -0,0 +1,37 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-26 17:46:10
+Description: 
+
+'''
+from common.utils import InputData
+from model.encoder.base_encoder import BaseEncoder, BiEncoder
+from model.encoder.pretrained_encoder import PretrainedEncoder
+from model.encoder.non_pretrained_encoder import NonPretrainedEncoder
+
+class AutoEncoder(BaseEncoder):
+    
+    def __init__(self, **config):
+        """automatedly load encoder by 'encoder_name'
+        Args:
+            config (dict):
+                encoder_name (str): support ["lstm", "self-attention-lstm", "bi-encoder"] and other pretrained model in hugging face
+                **args (Any): other configuration items corresponding to each module.
+        """
+        super().__init__()
+        self.config = config
+        if config.get("encoder_name"):
+            encoder_name = config.get("encoder_name").lower()
+            if encoder_name in ["lstm", "self-attention-lstm"]:
+                self.__encoder = NonPretrainedEncoder(**config)
+            elif encoder_name == "bi-encoder":
+                self.__encoder= BiEncoder(self.__init__(**config["intent_encoder"]), self.__init__(**config["intent_encoder"]))
+            else:
+                self.__encoder = PretrainedEncoder(**config)
+        else:
+            raise ValueError("There is no Encoder Name in config.")
+
+    def forward(self, inputs: InputData):
+        return self.__encoder(inputs)

model/encoder/base_encoder.py

@@ -0,0 +1,41 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-26 17:25:17
+Description: Base encoder and bi encoder
+
+'''
+from torch import nn
+
+from common.utils import InputData
+
+
+class BaseEncoder(nn.Module):
+    """Base class for all encoder module
+    """
+    def __init__(self, **config):
+        super().__init__()
+        self.config = config
+        NotImplementedError("no implement")
+
+    def forward(self, inputs: InputData):
+        self.encoder(inputs.input_ids)
+
+
+class BiEncoder(nn.Module):
+    """Bi Encoder for encode intent and slot separately
+    """
+    def __init__(self, intent_encoder: BaseEncoder, slot_encoder: BaseEncoder, **config):
+        super().__init__()
+        self.intent_encoder = intent_encoder
+        self.slot_encoder = slot_encoder
+
+    def forward(self, inputs: InputData):
+        hidden_slot = self.slot_encoder(inputs)
+        hidden_intent = self.intent_encoder(inputs)
+        if not self.intent_encoder.config["return_sentence_level_hidden"]:
+            hidden_slot.update_intent_hidden_state(hidden_intent.get_slot_hidden_state())
+        else:
+            hidden_slot.update_intent_hidden_state(hidden_intent.get_intent_hidden_state())
+        return hidden_slot

model/encoder/non_pretrained_encoder.py

@@ -0,0 +1,212 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-30 15:00:29
+Description: non-pretrained encoder model
+
+'''
+import math
+
+import einops
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
+
+from common.utils import HiddenData, InputData
+from model.encoder.base_encoder import BaseEncoder
+
+class NonPretrainedEncoder(BaseEncoder):
+    """
+    Encoder structure based on bidirectional LSTM and self-attention.
+    """
+
+    def __init__(self, **config):
+        """ init non-pretrained encoder
+        
+        Args:
+            config (dict):
+                embedding (dict):
+                    dropout_rate        (float): dropout rate.
+                    load_embedding_name (str): None if not use pretrained embedding or embedding name like "glove.6B.300d.txt".
+                    embedding_matrix    (Tensor, Optional): embedding matrix tensor. Enabled if load_embedding_name is not None. 
+                    vocab_size          (str, Optional): vocabulary size. Enabled if load_embedding_name is None.
+                lstm (dict):
+                    output_dim    (int): lstm output dim.
+                    bidirectional (bool): if use BiLSTM or LSTM.
+                    layer_num     (int): number of layers.
+                    dropout_rate  (float): dropout rate.
+                attention (dict, Optional):
+                    dropout_rate (float): dropout rate.
+                    hidden_dim   (int): attention hidden dim.
+                    output_dim   (int): attention output dim.
+                unflat_attention (dict, optional): Enabled if attention is not None.
+                    dropout_rate (float): dropout rate.
+        """
+        super(NonPretrainedEncoder, self).__init__()
+        self.config = config
+        # Embedding Initialization
+        embed_config = config["embedding"]
+        self.__embedding_dim = embed_config["embedding_dim"]
+        if embed_config.get("load_embedding_name"):
+            self.__embedding_layer = nn.Embedding.from_pretrained(embed_config["embedding_matrix"], padding_idx=0)
+        else:
+            self.__embedding_layer = nn.Embedding(
+                embed_config["vocab_size"], self.__embedding_dim
+            )
+        self.__embedding_dropout_layer = nn.Dropout(embed_config["dropout_rate"])
+
+        # LSTM Initialization
+        lstm_config = config["lstm"]
+        self.__hidden_size = lstm_config["output_dim"]
+        self.__lstm_layer = nn.LSTM(
+            input_size=self.__embedding_dim,
+            hidden_size=lstm_config["output_dim"] // 2,
+            batch_first=True,
+            bidirectional=lstm_config["bidirectional"],
+            dropout=lstm_config["dropout_rate"],
+            num_layers=lstm_config["layer_num"]
+        )
+        if self.config.get("attention"):
+            # Attention Initialization
+            att_config = config["attention"]
+            self.__attention_dropout_layer = nn.Dropout(att_config["dropout_rate"])
+            self.__attention_layer = QKVAttention(
+                self.__embedding_dim, self.__embedding_dim, self.__embedding_dim,
+                att_config["hidden_dim"], att_config["output_dim"], att_config["dropout_rate"]
+            )
+            if self.config.get("unflat_attention"):
+                unflat_att_config = config["unflat_attention"]
+                self.__sentattention = UnflatSelfAttention(
+                    lstm_config["output_dim"] + att_config["output_dim"],
+                    unflat_att_config["dropout_rate"]
+                )
+
+    def forward(self, inputs: InputData):
+        """ Forward process for Non-Pretrained Encoder.
+
+        Args:
+            inputs: padded input ids, masks.
+        Returns:
+            encoded hidden vectors.
+        """
+
+        # LSTM Encoder
+        # Padded_text should be instance of LongTensor.
+        embedded_text = self.__embedding_layer(inputs.input_ids)
+        dropout_text = self.__embedding_dropout_layer(embedded_text)
+        seq_lens = inputs.attention_mask.sum(-1).detach().cpu()
+        # Pack and Pad process for input of variable length.
+        packed_text = pack_padded_sequence(dropout_text, seq_lens, batch_first=True, enforce_sorted=False)
+        lstm_hiddens, (h_last, c_last) = self.__lstm_layer(packed_text)
+        padded_hiddens, _ = pad_packed_sequence(lstm_hiddens, batch_first=True)
+
+        if self.config.get("attention"):
+            # Attention Encoder
+            dropout_text = self.__attention_dropout_layer(embedded_text)
+            attention_hiddens = self.__attention_layer(
+                dropout_text, dropout_text, dropout_text, mask=inputs.attention_mask
+            )
+
+            # Attention + LSTM
+            hiddens = torch.cat([attention_hiddens, padded_hiddens], dim=-1)
+            hidden = HiddenData(None, hiddens)
+            if self.config.get("return_with_input"):
+                hidden.add_input(inputs)
+            if self.config.get("return_sentence_level_hidden"):
+                if self.config.get("unflat_attention"):
+                    sentence = self.__sentattention(hiddens, seq_lens)
+                else:
+                    sentence = hiddens[:, 0, :]
+                hidden.update_intent_hidden_state(sentence)
+        else:
+            sentence_hidden = None
+            if self.config.get("return_sentence_level_hidden"):
+                sentence_hidden = torch.cat((h_last[-1], h_last[-1], c_last[-1], c_last[-2]), dim=-1)
+            hidden = HiddenData(sentence_hidden, padded_hiddens)
+            if self.config.get("return_with_input"):
+                hidden.add_input(inputs)
+
+        return hidden
+
+
+class QKVAttention(nn.Module):
+    """
+    Attention mechanism based on Query-Key-Value architecture. And
+    especially, when query == key == value, it's self-attention.
+    """
+
+    def __init__(self, query_dim, key_dim, value_dim, hidden_dim, output_dim, dropout_rate):
+        super(QKVAttention, self).__init__()
+
+        # Record hyper-parameters.
+        self.__query_dim = query_dim
+        self.__key_dim = key_dim
+        self.__value_dim = value_dim
+        self.__hidden_dim = hidden_dim
+        self.__output_dim = output_dim
+        self.__dropout_rate = dropout_rate
+
+        # Declare network structures.
+        self.__query_layer = nn.Linear(self.__query_dim, self.__hidden_dim)
+        self.__key_layer = nn.Linear(self.__key_dim, self.__hidden_dim)
+        self.__value_layer = nn.Linear(self.__value_dim, self.__output_dim)
+        self.__dropout_layer = nn.Dropout(p=self.__dropout_rate)
+
+    def forward(self, input_query, input_key, input_value, mask=None):
+        """ The forward propagation of attention.
+
+        Here we require the first dimension of input key
+        and value are equal.
+
+        Args:
+            input_query: is query tensor, (n, d_q)
+            input_key:  is key tensor, (m, d_k)
+            input_value:  is value tensor, (m, d_v)
+        
+        Returns:
+            attention based tensor, (n, d_h)
+        """
+
+        # Linear transform to fine-tune dimension.
+        linear_query = self.__query_layer(input_query)
+        linear_key = self.__key_layer(input_key)
+        linear_value = self.__value_layer(input_value)
+
+        score_tensor = torch.matmul(
+            linear_query,
+            linear_key.transpose(-2, -1)
+        ) / math.sqrt(self.__hidden_dim)
+        if mask is not None:
+            attn_mask = einops.repeat((mask == 0), "b l -> b l h", h=score_tensor.shape[-1])
+            score_tensor = score_tensor.masked_fill_(attn_mask, -float(1e20))
+        score_tensor = F.softmax(score_tensor, dim=-1)
+        forced_tensor = torch.matmul(score_tensor, linear_value)
+        forced_tensor = self.__dropout_layer(forced_tensor)
+
+        return forced_tensor
+
+
+class UnflatSelfAttention(nn.Module):
+    """
+    scores each element of the sequence with a linear layer and uses the normalized scores to compute a context over the sequence.
+    """
+
+    def __init__(self, d_hid, dropout=0.):
+        super().__init__()
+        self.scorer = nn.Linear(d_hid, 1)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, inp, lens):
+        batch_size, seq_len, d_feat = inp.size()
+        inp = self.dropout(inp)
+        scores = self.scorer(inp.contiguous().view(-1, d_feat)).view(batch_size, seq_len)
+        max_len = max(lens)
+        for i, l in enumerate(lens):
+            if l < max_len:
+                scores.data[i, l:] = -np.inf
+        scores = F.softmax(scores, dim=1)
+        context = scores.unsqueeze(2).expand_as(inp).mul(inp).sum(1)
+        return context

model/encoder/pretrained_encoder.py

@@ -0,0 +1,39 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-26 17:18:01
+Description: pretrained encoder model
+
+'''
+from transformers import AutoModel
+
+from common.utils import InputData, HiddenData
+from model.encoder.base_encoder import BaseEncoder
+
+
+class PretrainedEncoder(BaseEncoder):
+    def __init__(self, **config):
+        """ init pretrained encoder
+        
+        Args:
+            config (dict):
+                encoder_name (str): pretrained model name in hugging face.
+        """
+        super().__init__(**config)
+        self.encoder = AutoModel.from_pretrained(config["encoder_name"])
+
+    def forward(self, inputs: InputData):
+        output = self.encoder(**inputs.get_inputs())
+        hidden = HiddenData(None, output.last_hidden_state)
+        if self.config.get("return_with_input"):
+            hidden.add_input(inputs)
+        if self.config.get("return_sentence_level_hidden"):
+            padding_side = self.config.get("padding_side")
+            if hasattr(output, "pooler_output"):
+                hidden.update_intent_hidden_state(output.pooler_output)
+            elif padding_side is not None and padding_side == "left":
+                hidden.update_intent_hidden_state(output.last_hidden_state[:, -1])
+            else:
+                hidden.update_intent_hidden_state(output.last_hidden_state[:, 0])
+        return hidden

model/open_slu_model.py

@@ -0,0 +1,64 @@
+'''
+Author: Qiguang Chen
+Date: 2023-01-11 10:39:26
+LastEditors: Qiguang Chen
+LastEditTime: 2023-01-26 17:18:22
+Description: Root Model Module
+
+'''
+from torch import nn
+
+from common.utils import OutputData, InputData
+from model.decoder.base_decoder import BaseDecoder
+from model.encoder.base_encoder import BaseEncoder
+
+
+class OpenSLUModel(nn.Module):
+    def __init__(self, encoder: BaseEncoder, decoder:BaseDecoder, **config):
+        """Create model automatedly
+
+        Args:
+            encoder (BaseEncoder): encoder created by config
+            decoder (BaseDecoder): decoder created by config
+            config (dict): any other args
+        """
+        super().__init__()
+        self.encoder = encoder
+        self.decoder = decoder
+        self.config = config
+
+    def forward(self, inp: InputData) -> OutputData:
+        """ model forward
+
+        Args:
+            inp (InputData): input ids and other information
+
+        Returns:
+            OutputData: pred logits
+        """
+        return self.decoder(self.encoder(inp))
+
+    def decode(self, output: OutputData, target: InputData=None):
+        """ decode output
+
+        Args:
+            pred (OutputData): pred logits data
+            target (InputData): golden data
+
+        Returns: decoded ids
+        """
+        return self.decoder.decode(output, target)
+
+    def compute_loss(self, pred: OutputData, target: InputData, compute_intent_loss=True, compute_slot_loss=True):
+        """ compute loss
+
+        Args:
+            pred (OutputData): pred logits data
+            target (InputData): golden data
+            compute_intent_loss (bool, optional): whether to compute intent loss. Defaults to True.
+            compute_slot_loss (bool, optional): whether to compute slot loss. Defaults to True.
+
+        Returns: loss value
+        """
+        return self.decoder.compute_loss(pred, target, compute_intent_loss=compute_intent_loss,
+                                         compute_slot_loss=compute_slot_loss)

save/stack/label.json

@@ -0,0 +1 @@
+{"intent": ["atis_flight", "atis_airfare", "atis_airline", "atis_ground_service", "atis_quantity", "atis_city", "atis_flight#atis_airfare", "atis_abbreviation", "atis_aircraft", "atis_distance", "atis_ground_fare", "atis_capacity", "atis_flight_time", "atis_meal", "atis_aircraft#atis_flight#atis_flight_no", "atis_flight_no", "atis_restriction", "atis_airport", "atis_airline#atis_flight_no", "atis_cheapest", "atis_ground_service#atis_ground_fare"], "slot": ["O", "B-fromloc.city_name", "B-toloc.city_name", "B-round_trip", "I-round_trip", "B-cost_relative", "B-fare_amount", "I-fare_amount", "B-arrive_date.month_name", "B-arrive_date.day_number", "I-fromloc.city_name", "B-stoploc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time", "B-toloc.state_code", "I-toloc.city_name", "I-stoploc.city_name", "B-meal_description", "B-depart_date.month_name", "B-depart_date.day_number", "B-airline_name", "I-airline_name", "B-depart_time.period_of_day", "B-depart_date.day_name", "B-toloc.state_name", "B-depart_time.time_relative", "B-depart_time.time", "B-toloc.airport_name", "I-toloc.airport_name", "B-depart_date.date_relative", "B-or", "B-airline_code", "B-class_type", "I-class_type", "I-cost_relative", "I-depart_time.time", "B-fromloc.airport_name", "I-fromloc.airport_name", "B-city_name", "B-flight_mod", "B-meal", "B-economy", "B-fare_basis_code", "I-depart_date.day_number", "B-depart_date.today_relative", "B-flight_stop", "B-airport_code", "B-fromloc.state_name", "I-fromloc.state_name", "I-city_name", "B-connect", "B-arrive_date.day_name", "B-fromloc.state_code", "B-arrive_date.today_relative", "B-depart_date.year", "B-depart_time.start_time", "I-depart_time.start_time", "B-depart_time.end_time", "I-depart_time.end_time", "B-arrive_time.start_time", "B-arrive_time.end_time", "I-arrive_time.end_time", "I-flight_mod", "B-flight_days", "B-mod", "B-flight_number", "I-toloc.state_name", "B-meal_code", "I-meal_code", "B-airport_name", "I-airport_name", "I-flight_stop", "B-transport_type", "I-transport_type", "B-state_code", "B-aircraft_code", "B-toloc.country_name", "I-arrive_date.day_number", "B-toloc.airport_code", "B-return_date.date_relative", "I-return_date.date_relative", "B-flight_time", "I-economy", "B-fromloc.airport_code", "B-arrive_time.period_of_day", "B-depart_time.period_mod", "I-flight_time", "B-return_date.day_name", "B-arrive_date.date_relative", "B-restriction_code", "I-restriction_code", "B-arrive_time.period_mod", "I-arrive_time.period_of_day", "B-period_of_day", "B-stoploc.state_code", "I-depart_date.today_relative", "I-fare_basis_code", "I-arrive_time.start_time", "B-time", "B-today_relative", "I-today_relative", "B-state_name", "B-days_code", "I-depart_time.period_of_day", "I-arrive_time.time_relative", "B-time_relative", "I-time", "B-return_date.month_name", "B-return_date.day_number", "I-depart_time.time_relative", "B-stoploc.airport_name", "B-day_name", "B-month_name", "B-day_number", "B-return_time.period_mod", "B-return_time.period_of_day", "B-return_date.today_relative", "I-return_date.today_relative", "I-meal_description"]}

save/stack/model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9710de3d7d5c8a34fe55ef4dc36dc8a851863d1fb3bb14871d914a4e945c96ef
+size 5793644

save/stack/tokenizer.json

@@ -0,0 +1 @@
+{"name": "word_tokenizer", "token_map": {"[PAD]": 0, "[UNK]": 1, "i": 2, "want": 3, "to": 4, "fly": 5, "from": 6, "baltimore": 7, "dallas": 8, "round": 9, "trip": 10, "fares": 11, "philadelphia": 12, "less": 13, "than": 14, "1000": 15, "dollars": 16, "denver": 17, "pittsburgh": 18, "show": 19, "me": 20, "the": 21, "flights": 22, "arriving": 23, "on": 24, "june": 25, "fourteenth": 26, "what": 27, "are": 28, "which": 29, "depart": 30, "san": 31, "francisco": 32, "washington": 33, "via": 34, "indianapolis": 35, "and": 36, "arrive": 37, "by": 38, "9": 39, "pm": 40, "airlines": 41, "boston": 42, "dc": 43, "other": 44, "cities": 45, "i'm": 46, "looking": 47, "for": 48, "a": 49, "flight": 50, "charlotte": 51, "las": 52, "vegas": 53, "that": 54, "stops": 55, "in": 56, "st.": 57, "louis": 58, "hopefully": 59, "dinner": 60, "how": 61, "can": 62, "find": 63, "out": 64, "okay": 65, "then": 66, "i'd": 67, "like": 68, "travel": 69, "atlanta": 70, "september": 71, "fourth": 72, "all": 73, "cincinnati": 74, "us": 75, "air": 76, "diego": 77, "afternoon": 78, "what's": 79, "available": 80, "tuesday": 81, "leave": 82, "phoenix": 83, "paul": 84, "minnesota": 85, "after": 86, "noon": 87, "american": 88, "chicago": 89, "los": 90, "angeles": 91, "morning": 92, "types": 93, "of": 94, "ground": 95, "transportation": 96, "there": 97, "airport": 98, "next": 99, "two": 100, "days": 101, "nashville": 102, "jose": 103, "or": 104, "tacoma": 105, "does": 106, "continental": 107, "milwaukee": 108, "many": 109, "twa": 110, "have": 111, "business": 112, "class": 113, "first": 114, "least": 115, "expensive": 116, "one": 117, "way": 118, "fare": 119, "booking": 120, "classes": 121, "wednesday": 122, "nineteenth": 123, "july": 124, "fifth": 125, "7": 126, "please": 127, "list": 128, "departing": 129, "general": 130, "mitchell": 131, "international": 132, "time": 133, "zone": 134, "is": 135, "serves": 136, "meal": 137, "seattle": 138, "salt": 139, "lake": 140, "city": 141, "you": 142, "with": 143, "economy": 144, "leaving": 145, "miami": 146, "cleveland": 147, "give": 148, "between": 149, "their": 150, "cost": 151, "code": 152, "y": 153, "mean": 154, "could": 155, "tell": 156, "leaves": 157, "united": 158, "airline": 159, "over": 160, "departures": 161, "arrivals": 162, "earliest": 163, "latest": 164, "return": 165, "within": 166, "same": 167, "day": 168, "orlando": 169, "either": 170, "evening": 171, "thursday": 172, "originating": 173, "going": 174, "order": 175, "snack": 176, "do": 177, "at": 178, "645": 179, "am": 180, "into": 181, "atlanta's": 182, "friday": 183, "qx": 184, "would": 185, "information": 186, "but": 187, "stopover": 188, "some": 189, "oakland": 190, "monday": 191, "know": 192, "type": 193, "aircraft": 194, "used": 195, "detroit": 196, "twenty": 197, "eighth": 198, "petersburg": 199, "take": 200, "begins": 201, "lands": 202, "fort": 203, "worth": 204, "stop": 205, "tomorrow": 206, "noontime": 207, "around": 208, "need": 209, "northwest": 210, "toronto": 211, "memphis": 212, "thirtieth": 213, "nonstop": 214, "houston": 215, "august": 216, "twentieth": 217, "ewr": 218, "seventh": 219, "newark": 220, "11": 221, "lowest": 222, "delta": 223, "has": 224, "go": 225, "any": 226, "jet": 227, "mco": 228, "new": 229, "jersey": 230, "ontario": 231, "saturday": 232, "york": 233, "long": 234, "it": 235, "get": 236, "prices": 237, "an": 238, "inexpensive": 239, "breakfast": 240, "direct": 241, "what're": 242, "sunday": 243, "colorado": 244, "see": 245, "serving": 246, "before": 247, "o'clock": 248, "january": 249, "1992": 250, "10": 251, "2": 252, "445": 253, "515": 254, "6": 255, "8": 256, "coach": 257, "only": 258, "weekdays": 259, "la": 260, "3": 261, "my": 262, "choices": 263, "early": 264, "minneapolis": 265, "cheapest": 266, "flying": 267, "possible": 268, "daily": 269, "beach": 270, "stopping": 271, "kansas": 272, "night": 273, "serve": 274, "meals": 275, "heading": 276, "kind": 277, "once": 278, "mornings": 279, "ff": 280, "arrangements": 281, "served": 282, "canadian": 283, "california": 284, "about": 285, "530": 286, "kinds": 287, "requesting": 288, "landing": 289, "distance": 290, "downtown": 291, "love": 292, "field": 293, "this": 294, "tampa": 295, "florida": 296, "5": 297, "must": 298, "be": 299, "advertises": 300, "having": 301, "more": 302, "november": 303, "eleventh": 304, "services": 305, "qo": 306, "american's": 307, "last": 308, "using": 309, "dl": 310, "217": 311, "montreal": 312, "service": 313, "ticket": 314, "should": 315, "near": 316, "also": 317, "missouri": 318, "utah": 319, "interested": 320, "those": 321, "when": 322, "north": 323, "carolina": 324, "415": 325, "200": 326, "explain": 327, "codes": 328, "sd": 329, "d": 330, "trying": 331, "plane": 332, "flies": 333, "weekday": 334, "columbus": 335, "1991": 336, "carries": 337, "smallest": 338, "number": 339, "passengers": 340, "takeoffs": 341, "landings": 342, "book": 343, "shortest": 344, "both": 345, "nationair": 346, "823": 347, "guardia": 348, "as": 349, "well": 350, "sixteenth": 351, "rental": 352, "car": 353, "rates": 354, "through": 355, "boeing": 356, "757": 357, "limousine": 358, "listing": 359, "canada": 360, "much": 361, "71": 362, "airfare": 363, "12": 364, "third": 365, "seating": 366, "capacity": 367, "arrives": 368, "bwi": 369, "ninth": 370, "late": 371, "nevada": 372, "4": 373, "price": 374, "fifteenth": 375, "eighteenth": 376, "returning": 377, "following": 378, "capacities": 379, "planes": 380, "1145": 381, "use": 382, "burbank": 383, "may": 384, "america": 385, "west": 386, "now": 387, "eastern": 388, "825": 389, "555": 390, "area": 391, "schedule": 392, "dfw": 393, "these": 394, "connecting": 395, "make": 396, "connection": 397, "lunch": 398, "f": 399, "belong": 400, "most": 401, "tickets": 402, "logan": 403, "vicinity": 404, "210": 405, "wednesdays": 406, "thursdays": 407, "yes": 408, "will": 409, "continuing": 410, "1039": 411, "southwest": 412, "times": 413, "400": 414, "week": 415, "if": 416, "813": 417, "enroute": 418, "another": 419, "twelfth": 420, "turboprop": 421, "420": 422, "today": 423, "1": 424, "we're": 425, "westchester": 426, "county": 427, "various": 428, "airplanes": 429, "uses": 430, "yn": 431, "852": 432, "transport": 433, "display": 434, "under": 435, "500": 436, "airfares": 437, "back": 438, "hours": 439, "fn": 440, "options": 441, "december": 442, "second": 443, "april": 444, "ohio": 445, "departs": 446, "2153": 447, "schedules": 448, "who": 449, "restriction": 450, "ap": 451, "57": 452, "layover": 453, "abbreviation": 454, "stands": 455, "1291": 456, "324": 457, "again": 458, "offer": 459, "dc10": 460, "currently": 461, "represented": 462, "database": 463, "arizona": 464, "1505": 465, "sixth": 466, "3724": 467, "three": 468, "including": 469, "connections": 470, "numbers": 471, "six": 472, "1100": 473, "destination": 474, "838": 475, "no": 476, "h": 477, "traveling": 478, "ap57": 479, "far": 480, "lufthansa": 481, "abbreviations": 482, "such": 483, "aa": 484, "459": 485, "where": 486, "ua": 487, "281": 488, "your": 489, "texas": 490, "1500": 491, "bound": 492, "includes": 493, "right": 494, "airports": 495, "eight": 496, "sixteen": 497, "trips": 498, "seventeenth": 499, "thrift": 500, "delta's": 501, "departure": 502, "listed": 503, "1055": 504, "405": 505, "midnight": 506, "hi": 507, "630": 508, "question": 509, "live": 510, "stand": 511, "ten": 512, "people": 513, "during": 514, "2100": 515, "gets": 516, "just": 517, "philly": 518, "21": 519, "airplane": 520, "1765": 521, "iah": 522, "737": 523, "midwest": 524, "express": 525, "s": 526, "designate": 527, "747": 528, "650": 529, "goes": 530, "reaches": 531, "seventeen": 532, "sorry": 533, "anywhere": 534, "provided": 535, "d10": 536, "toward": 537, "preferably": 538, "rate": 539, "difference": 540, "q": 541, "b": 542, "ac": 543, "tower": 544, "tenth": 545, "hp": 546, "4400": 547, "georgia": 548, "offers": 549, "fine": 550, "201": 551, "343": 552, "october": 553, "ea": 554, "jfk": 555, "name": 556, "arrange": 557, "largest": 558, "connect": 559, "operating": 560, "sundays": 561, "720": 562, "land": 563, "final": 564, "don't": 565, "stopovers": 566, "total": 567, "friday's": 568, "755": 569, "cheap": 570, "sfo": 571, "thirty": 572, "across": 573, "continent": 574, "makes": 575, "1220": 576, "co": 577, "1209": 578, "wanted": 579, "1850": 580, "without": 581, "listings": 582, "local": 583, "wish": 584, "bring": 585, "up": 586, "home": 587, "417": 588, "approximately": 589, "actually": 590, "1200": 591, "230": 592, "819": 593, "serviced": 594, "928": 595, "reservation": 596, "limousines": 597, "taxi": 598, "fit": 599, "72s": 600, "352": 601, "1133": 602, "43": 603, "define": 604, "directly": 605, "m80": 606, "close": 607, "restrictions": 608, "430": 609, "718": 610, "hou": 611, "costs": 612, "466": 613, "march": 614, "1026": 615, "1024": 616, "different": 617, "rentals": 618, "each": 619, "arrival": 620, "say": 621, "mealtime": 622, "932": 623, "1115": 624, "1245": 625, "include": 626, "whether": 627, "offered": 628, "130": 629, "alaska": 630, "296": 631, "they": 632, "106": 633, "york's": 634, "497766": 635, "itinerary": 636, "coming": 637, "month": 638, "bur": 639, "travels": 640, "pennsylvania": 641, "usa": 642, "1288": 643, "c": 644, "names": 645, "sure": 646, "meaning": 647, "ap80": 648, "269": 649, "reservations": 650, "d9s": 651, "sunday's": 652, "f28": 653, "934": 654, "earlier": 655, "1017": 656, "date": 657, "thank": 658, "oak": 659, "atl": 660, "cp": 661, "3357": 662, "1045": 663, "limo": 664, "845": 665, "sometime": 666, "1222": 667, "i'll": 668, "tennessee": 669, "0900": 670, "hello": 671, "let": 672, "repeat": 673, "provide": 674, "still": 675, "along": 676, "operation": 677, "year": 678, "one's": 679, "great": 680, "too": 681, "nighttime": 682, "1300": 683, "saturdays": 684, "416": 685, "four": 686, "257": 687, "minimum": 688, "intercontinental": 689, "february": 690, "spend": 691, "lastest": 692, "thing": 693, "originate": 694, "describe": 695, "concerning": 696, "sa": 697, "help": 698, "1700": 699, "225": 700, "1158": 701, "equipment": 702, "let's": 703, "wednesday's": 704, "quebec": 705, "highest": 706, "starting": 707, "taking": 708, "311": 709, "1230": 710, "able": 711, "put": 712, "later": 713, "takes": 714, "amount": 715, "qw": 716, "seven": 717, "maximum": 718, "yyz": 719, "it's": 720, "80": 721, "place": 722, "equal": 723, "while": 724, "train": 725, "look": 726, "815": 727, "takeoff": 728, "plan": 729, "2134": 730, "297": 731, "323": 732, "229": 733, "329": 734, "runs": 735, "730": 736, "closest": 737, "dulles": 738, "73s": 739, "so": 740, "economic": 741, "single": 742, "supper": 743, "110": 744, "calling": 745, "1205": 746, "55": 747, "michigan": 748, "proper": 749, "regarding": 750, "seats": 751, "19": 752, "m": 753, "midway": 754, "besides": 755, "reverse": 756, "1993": 757, "402": 758, "level": 759, "reaching": 760, "771": 761, "straight": 762, "located": 763, "305": 764, "repeating": 765, "indiana": 766, "connects": 767, "beginning": 768, "staying": 769, "town": 770, "cars": 771, "nonstops": 772, "300": 773, "345": 774, "dinnertime": 775, "sort": 776, "route": 777, "j31": 778, "tuesdays": 779, "212": 780, "705": 781, "red": 782, "eye": 783, "laying": 784, "friends": 785, "visit": 786, "here": 787, "them": 788, "lives": 789, "rent": 790, "279": 791, "137338": 792, "transcontinental": 793, "trans": 794, "world": 795, "1030": 796, "1130": 797, "come": 798, "727": 799, "1020": 800, "505": 801, "that's": 802, "163": 803, "ls": 804, "greatest": 805, "i've": 806, "got": 807, "somebody": 808, "else": 809, "wants": 810, "charges": 811, "734": 812, "carried": 813, "thirteenth": 814, "making": 815, "733": 816, "everywhere": 817, "prefer": 818, "run": 819, "non": 820, "315": 821, "746": 822, "companies": 823, "buy": 824, "very": 825, "270": 826, "locate": 827, "hartfield": 828, "start": 829, "98": 830, "inform": 831, "oh": 832, "82": 833, "139": 834, "1600": 835, "eleven": 836, "ord": 837, "mia": 838, "qualify": 839, "doesn't": 840, "mondays": 841, "catch": 842, "priced": 843, "bna": 844, "being": 845, "working": 846, "scenario": 847, "767": 848, "1940": 849, "150": 850, "100": 851, "afternoons": 852, "provides": 853, "723": 854, "1110": 855, "symbols": 856, "grounds": 857, "nw": 858, "539": 859, "soon": 860, "thereafter": 861, "scheduled": 862, "instead": 863, "810": 864, "lester": 865, "pearson": 866, "stapleton": 867, "615": 868, "twelve": 869, "bay": 870, "sounds": 871, "o'hare": 872, "ap68": 873, "fridays": 874, "try": 875, "fifteen": 876, "nights": 877, "determine": 878, "hold": 879, "lax": 880, "seat": 881, "k": 882, "planning": 883, "discount": 884, "summer": 885, "cover": 886, "271": 887, "tonight": 888, "off": 889, "124": 890, "thanks": 891, "longest": 892, "kindly": 893, "afterwards": 894, "overnight": 895, "1083": 896, "428": 897, "anything": 898, "1059": 899}}