diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..c7d9f3332a950355d5a77d85000f05e6f45435ea --- /dev/null +++ b/.gitattributes @@ -0,0 +1,34 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..3185e0ad7ba1842ecd18d0570f1451c373a246c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,136 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +.idea/ +wandb/* +save/* +!save/.gitkeep +logs/* +!logs/.gitkeep +test +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +.vscode/ diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bbdfa8d934dea8ba991d7362c71766973217678f --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +--- +license: mit +title: OpenSLU +sdk: gradio +sdk_version: 3.18.0 +app_file: app.py +emoji: 🚀 +colorFrom: blue +colorTo: purple +pinned: false +tags: +- making-demos +duplicated_from: LightChen2333/OpenSLU +--- diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ + diff --git a/accelerate/config-old.yaml b/accelerate/config-old.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96c0db2acd15ccc84d88ae8e375303dcb31655ba --- /dev/null +++ b/accelerate/config-old.yaml @@ -0,0 +1,16 @@ +compute_environment: LOCAL_MACHINE +deepspeed_config: {} +distributed_type: MULTI_GPU +downcast_bf16: 'no' +fsdp_config: {} +gpu_ids: all +machine_rank: 0 +main_process_ip: null +main_process_port: 9001 +main_training_function: main +mixed_precision: 'no' +num_machines: 0 +num_processes: 2 +rdzv_backend: static +same_network: true +use_cpu: false diff --git a/accelerate/config.yaml b/accelerate/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4da53f316a9da379d601a447d54ea4644a3a9b8 --- /dev/null +++ b/accelerate/config.yaml @@ -0,0 +1,22 @@ +command_file: null +commands: null +compute_environment: LOCAL_MACHINE +deepspeed_config: {} +distributed_type: 'NO' +downcast_bf16: 'no' +dynamo_backend: 'NO' +fsdp_config: {} +gpu_ids: all +machine_rank: 0 +main_process_ip: null +main_process_port: null +main_training_function: main +megatron_lm_config: {} +mixed_precision: 'no' +num_machines: 1 +num_processes: 2 +rdzv_backend: static +same_network: true +tpu_name: null +tpu_zone: null +use_cpu: false diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..9392b7af72e5fb26f466f0e8ae47775778d4f705 --- /dev/null +++ b/app.py @@ -0,0 +1,63 @@ +''' +Author: Qiguang Chen +LastEditors: Qiguang Chen +Date: 2023-02-07 15:42:32 +LastEditTime: 2023-02-19 21:04:03 +Description: + +''' +import argparse +import gradio as gr + +from common.config import Config +from common.model_manager import ModelManager +from common.utils import str2bool + + +parser = argparse.ArgumentParser() +parser.add_argument('--config_path', '-cp', type=str, default="config/examples/from_pretrained.yaml") +parser.add_argument('--push_to_public', '-p', type=str2bool, nargs='?', + const=True, default=False, + help="Push to public network.") +args = parser.parse_args() +config = Config.load_from_yaml(args.config_path) +config.base["train"] = False +config.base["test"] = False + +model_manager = ModelManager(config) +model_manager.init_model() + + +def text_analysis(text): + print(text) + data = model_manager.predict(text) + html = """ + """ + html += """
Intent:""" + + for intent in data["intent"]: + html += """""" + html += """
Slot:""" + for t, slot in zip(data["text"], data["slot"]): + html += """""" + html+="
" + return html + + +demo = gr.Interface( + text_analysis, + gr.Textbox(placeholder="Enter sentence here..."), + ["html"], + examples=[ + ["i would like to find a flight from charlotte to las vegas that makes a stop in st louis"], + ], +) +if args.push_to_public: + demo.launch(share=True) +else: + demo.launch() diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/common/__init__.py @@ -0,0 +1 @@ + diff --git a/common/config.py b/common/config.py new file mode 100644 index 0000000000000000000000000000000000000000..9563ea6ffa6a75095e61a872db5b4fcd6f2e9d65 --- /dev/null +++ b/common/config.py @@ -0,0 +1,192 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-15 17:58:53 +Description: Configuration class to manage all process in OpenSLU like model construction, learning processing and so on. + +''' +import re + +from ruamel import yaml +import datetime + +class Config(dict): + def __init__(self, *args, **kwargs): + """ init with dict as args + """ + dict.__init__(self, *args, **kwargs) + self.__dict__ = self + self.start_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S%f') + if not self.model.get("_from_pretrained_"): + self.__autowired() + + @staticmethod + def load_from_yaml(file_path:str)->"Config": + """load config files with path + + Args: + file_path (str): yaml configuration file path. + + Returns: + Config: config object. + """ + with open(file_path) as stream: + try: + return Config(yaml.safe_load(stream)) + except yaml.YAMLError as exc: + print(exc) + + @staticmethod + def load_from_args(args)->"Config": + """ load args to replace item value in config files assigned with '--config_path' or '--model' + + Args: + args (Any): args with command line. + + Returns: + Config: _description_ + """ + if args.model is not None and args.dataset is not None: + args.config_path = f"config/reproduction/{args.dataset}/{args.model}.yaml" + config = Config.load_from_yaml(args.config_path) + if args.dataset is not None: + config.__update_dataset(args.dataset) + if args.device is not None: + config["base"]["device"] = args.device + if args.learning_rate is not None: + config["optimizer"]["lr"] = args.learning_rate + if args.epoch_num is not None: + config["base"]["epoch_num"] = args.epoch_num + return config + + def autoload_template(self): + """ search '{*}' template to excute as python code, support replace variable as any configure item + """ + self.__autoload_template(self.__dict__) + + def __get_autoload_value(self, matched): + keys = matched.group()[1:-1].split(".") + temp = self.__dict__ + for k in keys: + temp = temp[k] + return str(temp) + + def __autoload_template(self, config:dict): + for k in config: + if isinstance(config, dict): + sub_config = config[k] + elif isinstance(config, list): + sub_config = k + else: + continue + if isinstance(sub_config, dict) or isinstance(sub_config, list): + self.__autoload_template(sub_config) + if isinstance(sub_config, str) and "{" in sub_config and "}" in sub_config: + res = re.sub(r'{.*?}', self.__get_autoload_value, config[k]) + res_dict= {"res": None} + exec("res=" + res, res_dict) + config[k] = res_dict["res"] + + def __update_dataset(self, dataset_name): + if dataset_name is not None and isinstance(dataset_name, str): + self.__dict__["dataset"]["dataset_name"] = dataset_name + + def get_model_config(self): + return self.__dict__["model"] + + def __autowired(self): + # Set encoder + encoder_config = self.__dict__["model"]["encoder"] + encoder_type = encoder_config["_model_target_"].split(".")[-1] + + def get_output_dim(encoder_config): + encoder_type = encoder_config["_model_target_"].split(".")[-1] + if (encoder_type == "AutoEncoder" and encoder_config["encoder_name"] in ["lstm", "self-attention-lstm", + "bi-encoder"]) or encoder_type == "NoPretrainedEncoder": + output_dim = 0 + if encoder_config.get("lstm"): + output_dim += encoder_config["lstm"]["output_dim"] + if encoder_config.get("attention"): + output_dim += encoder_config["attention"]["output_dim"] + return output_dim + else: + return encoder_config["output_dim"] + + if encoder_type == "BiEncoder": + output_dim = get_output_dim(encoder_config["intent_encoder"]) + \ + get_output_dim(encoder_config["slot_encoder"]) + else: + output_dim = get_output_dim(encoder_config) + self.__dict__["model"]["encoder"]["output_dim"] = output_dim + + # Set interaction + if "interaction" in self.__dict__["model"]["decoder"] and self.__dict__["model"]["decoder"]["interaction"].get( + "input_dim") is None: + self.__dict__["model"]["decoder"]["interaction"]["input_dim"] = output_dim + interaction_type = self.__dict__["model"]["decoder"]["interaction"]["_model_target_"].split(".")[-1] + if not ((encoder_type == "AutoEncoder" and encoder_config[ + "encoder_name"] == "self-attention-lstm") or encoder_type == "SelfAttentionLSTMEncoder") and interaction_type != "BiModelWithoutDecoderInteraction": + output_dim = self.__dict__["model"]["decoder"]["interaction"]["output_dim"] + + # Set classifier + if "slot_classifier" in self.__dict__["model"]["decoder"]: + if self.__dict__["model"]["decoder"]["slot_classifier"].get("input_dim") is None: + self.__dict__["model"]["decoder"]["slot_classifier"]["input_dim"] = output_dim + self.__dict__["model"]["decoder"]["slot_classifier"]["use_slot"] = True + if "intent_classifier" in self.__dict__["model"]["decoder"]: + if self.__dict__["model"]["decoder"]["intent_classifier"].get("input_dim") is None: + self.__dict__["model"]["decoder"]["intent_classifier"]["input_dim"] = output_dim + self.__dict__["model"]["decoder"]["intent_classifier"]["use_intent"] = True + + def get_intent_label_num(self): + """ get the number of intent labels. + """ + classifier_conf = self.__dict__["model"]["decoder"]["intent_classifier"] + return classifier_conf["intent_label_num"] if "intent_label_num" in classifier_conf else 0 + + def get_slot_label_num(self): + """ get the number of slot labels. + """ + classifier_conf = self.__dict__["model"]["decoder"]["slot_classifier"] + return classifier_conf["slot_label_num"] if "slot_label_num" in classifier_conf else 0 + + def set_intent_label_num(self, intent_label_num): + """ set the number of intent labels. + + Args: + slot_label_num (int): the number of intent label + """ + self.__dict__["base"]["intent_label_num"] = intent_label_num + self.__dict__["model"]["decoder"]["intent_classifier"]["intent_label_num"] = intent_label_num + if "interaction" in self.__dict__["model"]["decoder"]: + + self.__dict__["model"]["decoder"]["interaction"]["intent_label_num"] = intent_label_num + if self.__dict__["model"]["decoder"]["interaction"]["_model_target_"].split(".")[ + -1] == "StackInteraction": + self.__dict__["model"]["decoder"]["slot_classifier"]["input_dim"] += intent_label_num + + + def set_slot_label_num(self, slot_label_num:int)->None: + """set the number of slot label + + Args: + slot_label_num (int): the number of slot label + """ + self.__dict__["base"]["slot_label_num"] = slot_label_num + self.__dict__["model"]["decoder"]["slot_classifier"]["slot_label_num"] = slot_label_num + if "interaction" in self.__dict__["model"]["decoder"]: + self.__dict__["model"]["decoder"]["interaction"]["slot_label_num"] = slot_label_num + + def set_vocab_size(self, vocab_size): + """set the size of vocabulary in non-pretrained tokenizer + Args: + slot_label_num (int): the number of slot label + """ + encoder_type = self.__dict__["model"]["encoder"]["_model_target_"].split(".")[-1] + encoder_name = self.__dict__["model"]["encoder"].get("encoder_name") + if encoder_type == "BiEncoder" or (encoder_type == "AutoEncoder" and encoder_name == "bi-encoder"): + self.__dict__["model"]["encoder"]["intent_encoder"]["embedding"]["vocab_size"] = vocab_size + self.__dict__["model"]["encoder"]["slot_encoder"]["embedding"]["vocab_size"] = vocab_size + elif self.__dict__["model"]["encoder"].get("embedding"): + self.__dict__["model"]["encoder"]["embedding"]["vocab_size"] = vocab_size diff --git a/common/global_pool.py b/common/global_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..c1f6e0db50fd1d1c6fbd4ae10658cbdb97de5494 --- /dev/null +++ b/common/global_pool.py @@ -0,0 +1,26 @@ +''' +Author: Qiguang Chen +LastEditors: Qiguang Chen +Date: 2023-02-12 14:35:37 +LastEditTime: 2023-02-12 14:37:40 +Description: + +''' +def _init(): + global _global_dict + _global_dict = {} + + +def set_value(key, value): + # set gobal value to object pool + _global_dict[key] = value + + +def get_value(key): + # get gobal value from object pool + try: + return _global_dict[key] + except: + print('读取' + key + '失败\r\n') + + \ No newline at end of file diff --git a/common/loader.py b/common/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..5b97680ae3ef9b5523cce591e30c520059ff36a8 --- /dev/null +++ b/common/loader.py @@ -0,0 +1,332 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-19 15:39:48 +Description: all class for load data. + +''' +import os +import torch +import json +from datasets import load_dataset, Dataset +from torch.utils.data import DataLoader + +from common.utils import InputData + +ABS_PATH=os.path.join(os.path.abspath(os.path.dirname(__file__)), "../") + +class DataFactory(object): + def __init__(self, tokenizer,use_multi_intent=False, to_lower_case=True): + """_summary_ + + Args: + tokenizer (Tokenizer): _description_ + use_multi_intent (bool, optional): _description_. Defaults to False. + """ + self.tokenizer = tokenizer + self.slot_label_list = [] + self.intent_label_list = [] + self.use_multi = use_multi_intent + self.to_lower_case = to_lower_case + self.slot_label_dict = None + self.intent_label_dict = None + + def __is_supported_datasets(self, dataset_name:str)->bool: + return dataset_name.lower() in ["atis", "snips", "mix-atis", "mix-atis"] + + def load_dataset(self, dataset_config, split="train"): + dataset_name = None + if split not in dataset_config: + dataset_name = dataset_config.get("dataset_name") + elif self.__is_supported_datasets(dataset_config[split]): + dataset_name = dataset_config[split].lower() + if dataset_name is not None: + return load_dataset("LightChen2333/OpenSLU", dataset_name, split=split) + else: + data_file = dataset_config[split] + data_dict = {"text": [], "slot": [], "intent":[]} + with open(data_file, encoding="utf-8") as f: + for line in f: + row = json.loads(line) + data_dict["text"].append(row["text"]) + data_dict["slot"].append(row["slot"]) + data_dict["intent"].append(row["intent"]) + return Dataset.from_dict(data_dict) + + def update_label_names(self, dataset): + for intent_labels in dataset["intent"]: + if self.use_multi: + intent_label = intent_labels.split("#") + else: + intent_label = [intent_labels] + for x in intent_label: + if x not in self.intent_label_list: + self.intent_label_list.append(x) + for slot_label in dataset["slot"]: + for x in slot_label: + if x not in self.slot_label_list: + self.slot_label_list.append(x) + self.intent_label_dict = {key: index for index, + key in enumerate(self.intent_label_list)} + self.slot_label_dict = {key: index for index, + key in enumerate(self.slot_label_list)} + + def update_vocabulary(self, dataset): + if self.tokenizer.name_or_path in ["word_tokenizer"]: + for data in dataset: + self.tokenizer.add_instance(data["text"]) + + @staticmethod + def fast_align_data(text, padding_side="right"): + for i in range(len(text.input_ids)): + desired_output = [] + for word_id in text.word_ids(i): + if word_id is not None: + start, end = text.word_to_tokens( + i, word_id, sequence_index=0 if padding_side == "right" else 1) + if start == end - 1: + tokens = [start] + else: + tokens = [start, end - 1] + if len(desired_output) == 0 or desired_output[-1] != tokens: + desired_output.append(tokens) + yield desired_output + + def fast_align(self, + batch, + ignore_index=-100, + device="cuda", + config=None, + enable_label=True, + label2tensor=True): + if self.to_lower_case: + input_list = [[t.lower() for t in x["text"]] for x in batch] + else: + input_list = [x["text"] for x in batch] + text = self.tokenizer(input_list, + return_tensors="pt", + padding=True, + is_split_into_words=True, + truncation=True, + **config).to(device) + if enable_label: + if label2tensor: + + slot_mask = torch.ones_like(text.input_ids) * ignore_index + for i, offsets in enumerate( + DataFactory.fast_align_data(text, padding_side=self.tokenizer.padding_side)): + num = 0 + assert len(offsets) == len(batch[i]["text"]) + assert len(offsets) == len(batch[i]["slot"]) + for off in offsets: + slot_mask[i][off[0] + ] = self.slot_label_dict[batch[i]["slot"][num]] + num += 1 + slot = slot_mask.clone() + attentin_id = 0 if self.tokenizer.padding_side == "right" else 1 + for i, slot_batch in enumerate(slot): + for j, x in enumerate(slot_batch): + if x == ignore_index and text.attention_mask[i][j] == attentin_id and (text.input_ids[i][ + j] not in self.tokenizer.all_special_ids or text.input_ids[i][j] == self.tokenizer.unk_token_id): + slot[i][j] = slot[i][j - 1] + slot = slot.to(device) + if not self.use_multi: + intent = torch.tensor( + [self.intent_label_dict[x["intent"]] for x in batch]).to(device) + else: + one_hot = torch.zeros( + (len(batch), len(self.intent_label_list)), dtype=torch.float) + for index, b in enumerate(batch): + for x in b["intent"].split("#"): + one_hot[index][self.intent_label_dict[x]] = 1. + intent = one_hot.to(device) + else: + slot_mask = None + slot = [['#' for _ in range(text.input_ids.shape[1])] + for _ in range(text.input_ids.shape[0])] + for i, offsets in enumerate(DataFactory.fast_align_data(text)): + num = 0 + for off in offsets: + slot[i][off[0]] = batch[i]["slot"][num] + num += 1 + if not self.use_multi: + intent = [x["intent"] for x in batch] + else: + intent = [ + [x for x in b["intent"].split("#")] for b in batch] + return InputData((text, slot, intent)) + else: + return InputData((text, None, None)) + + def general_align_data(self, split_text_list, raw_text_list, encoded_text): + for i in range(len(split_text_list)): + desired_output = [] + jdx = 0 + offset = encoded_text.offset_mapping[i].tolist() + split_texts = split_text_list[i] + raw_text = raw_text_list[i] + last = 0 + temp_offset = [] + for off in offset: + s, e = off + if len(temp_offset) > 0 and (e != 0 and last == s): + len_1 = off[1] - off[0] + len_2 = temp_offset[-1][1] - temp_offset[-1][0] + if len_1 > len_2: + temp_offset.pop(-1) + temp_offset.append([0, 0]) + temp_offset.append(off) + continue + temp_offset.append(off) + last = s + offset = temp_offset + for split_text in split_texts: + while jdx < len(offset) and offset[jdx][0] == 0 and offset[jdx][1] == 0: + jdx += 1 + if jdx == len(offset): + continue + start_, end_ = offset[jdx] + tokens = None + if split_text == raw_text[start_:end_].strip(): + tokens = [jdx] + else: + # Compute "xxx" -> "xx" "#x" + temp_jdx = jdx + last_str = raw_text[start_:end_].strip() + while last_str != split_text and temp_jdx < len(offset) - 1: + temp_jdx += 1 + last_str += raw_text[offset[temp_jdx] + [0]:offset[temp_jdx][1]].strip() + + if temp_jdx == jdx: + raise ValueError("Illegal Input data") + elif last_str == split_text: + tokens = [jdx, temp_jdx] + jdx = temp_jdx + else: + jdx -= 1 + jdx += 1 + if tokens is not None: + desired_output.append(tokens) + yield desired_output + + def general_align(self, + batch, + ignore_index=-100, + device="cuda", + config=None, + enable_label=True, + label2tensor=True, + locale="en-US"): + if self.to_lower_case: + raw_data = [" ".join(x["text"]).lower() if locale not in ['ja-JP', 'zh-CN', 'zh-TW'] else "".join(x["text"]) for x in + batch] + input_list = [[t.lower() for t in x["text"]] for x in batch] + else: + input_list = [x["text"] for x in batch] + raw_data = [" ".join(x["text"]) if locale not in ['ja-JP', 'zh-CN', 'zh-TW'] else "".join(x["text"]) for x in + batch] + text = self.tokenizer(raw_data, + return_tensors="pt", + padding=True, + truncation=True, + return_offsets_mapping=True, + **config).to(device) + if enable_label: + if label2tensor: + slot_mask = torch.ones_like(text.input_ids) * ignore_index + for i, offsets in enumerate( + self.general_align_data(input_list, raw_data, encoded_text=text)): + num = 0 + # if len(offsets) != len(batch[i]["text"]) or len(offsets) != len(batch[i]["slot"]): + # if + for off in offsets: + slot_mask[i][off[0] + ] = self.slot_label_dict[batch[i]["slot"][num]] + num += 1 + # slot = slot_mask.clone() + # attentin_id = 0 if self.tokenizer.padding_side == "right" else 1 + # for i, slot_batch in enumerate(slot): + # for j, x in enumerate(slot_batch): + # if x == ignore_index and text.attention_mask[i][j] == attentin_id and text.input_ids[i][ + # j] not in self.tokenizer.all_special_ids: + # slot[i][j] = slot[i][j - 1] + slot = slot_mask.to(device) + if not self.use_multi: + intent = torch.tensor( + [self.intent_label_dict[x["intent"]] for x in batch]).to(device) + else: + one_hot = torch.zeros( + (len(batch), len(self.intent_label_list)), dtype=torch.float) + for index, b in enumerate(batch): + for x in b["intent"].split("#"): + one_hot[index][self.intent_label_dict[x]] = 1. + intent = one_hot.to(device) + else: + slot_mask = None + slot = [['#' for _ in range(text.input_ids.shape[1])] + for _ in range(text.input_ids.shape[0])] + for i, offsets in enumerate(self.general_align_data(input_list, raw_data, encoded_text=text)): + num = 0 + for off in offsets: + slot[i][off[0]] = batch[i]["slot"][num] + num += 1 + if not self.use_multi: + intent = [x["intent"] for x in batch] + else: + intent = [ + [x for x in b["intent"].split("#")] for b in batch] + return InputData((text, slot, intent)) + else: + return InputData((text, None, None)) + + def batch_fn(self, + batch, + ignore_index=-100, + device="cuda", + config=None, + align_mode="fast", + enable_label=True, + label2tensor=True): + if align_mode == "fast": + # try: + return self.fast_align(batch, + ignore_index=ignore_index, + device=device, + config=config, + enable_label=enable_label, + label2tensor=label2tensor) + # except: + # return self.general_align(batch, + # ignore_index=ignore_index, + # device=device, + # config=config, + # enable_label=enable_label, + # label2tensor=label2tensor) + else: + return self.general_align(batch, + ignore_index=ignore_index, + device=device, + config=config, + enable_label=enable_label, + label2tensor=label2tensor) + + def get_data_loader(self, + dataset, + batch_size, + shuffle=False, + device="cuda", + enable_label=True, + align_mode="fast", + label2tensor=True, **config): + data_loader = DataLoader(dataset, + shuffle=shuffle, + batch_size=batch_size, + collate_fn=lambda x: self.batch_fn(x, + device=device, + config=config, + enable_label=enable_label, + align_mode=align_mode, + label2tensor=label2tensor)) + return data_loader diff --git a/common/logger.py b/common/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..9ac7bfa1a371ae23897c5716f2aeade5b2587d4a --- /dev/null +++ b/common/logger.py @@ -0,0 +1,237 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-19 22:05:49 +Description: log manager + +''' +import datetime +import json +import os +import time +from common.config import Config +import logging +import colorlog + +def mkdirs(dir_names): + for dir_name in dir_names: + if not os.path.exists(dir_name): + os.mkdir(dir_name) + + + +class Logger(): + """ logging infomation by [wandb, fitlog, local file] + """ + def __init__(self, + logger_type: str, + logger_name: str, + logging_level="INFO", + start_time='', + accelerator=None): + """ create logger + + Args: + logger_type (str): support type = ["wandb", "fitlog", "local"] + logger_name (str): logger name, means project name in wandb, and logging file name + logging_level (str, optional): logging level. Defaults to "INFO". + start_time (str, optional): start time string. Defaults to ''. + """ + self.logger_type = logger_type + times = time.localtime() + self.output_dir = "logs/" + logger_name + "/" + start_time + self.accelerator = accelerator + self.logger_name = logger_name + if accelerator is not None: + from accelerate.logging import get_logger + self.logging = get_logger(logger_name) + else: + if self.logger_type == "wandb": + import wandb + self.logger = wandb + mkdirs(["logs", "logs/" + logger_name, self.output_dir]) + self.logger.init(project=logger_name) + elif self.logger_type == "fitlog": + import fitlog + self.logger = fitlog + mkdirs(["logs", "logs/" + logger_name, self.output_dir]) + self.logger.set_log_dir("logs/" + logger_name) + else: + mkdirs(["logs", "logs/" + logger_name, self.output_dir]) + self.config_file = os.path.join(self.output_dir, "config.jsonl") + with open(self.config_file, "w", encoding="utf8") as f: + print(f"Config will be written to {self.config_file}") + + self.loss_file = os.path.join(self.output_dir, "loss.jsonl") + with open(self.loss_file, "w", encoding="utf8") as f: + print(f"Loss Result will be written to {self.loss_file}") + + self.metric_file = os.path.join(self.output_dir, "metric.jsonl") + with open(self.metric_file, "w", encoding="utf8") as f: + print(f"Metric Result will be written to {self.metric_file}") + + self.other_log_file = os.path.join(self.output_dir, "other_log.jsonl") + with open(self.other_log_file, "w", encoding="utf8") as f: + print(f"Other Log Result will be written to {self.other_log_file}") + + LOGGING_LEVEL_MAP = { + "CRITICAL": logging.CRITICAL, + "FATAL": logging.FATAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "WARN": logging.WARN, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, + "NOTSET": logging.NOTSET, + } + # logging.basicConfig(format='[%(levelname)s - %(asctime)s]\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', + # filename=os.path.join(self.output_dir, "log.log"), level=LOGGING_LEVEL_MAP[logging_level]) + + # logger = logging.getLogger() + # KZT = logging.StreamHandler() + # KZT.setLevel(logging.DEBUG) + # logger.addHandler(KZT) + + self.logging = self._get_logging_logger(logging_level) + + def _get_logging_logger(self, level="INFO"): + log_colors_config = { + 'DEBUG': 'cyan', + 'INFO': 'blue', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + } + + logger = logging.getLogger() + logger.setLevel(level) + + log_path = os.path.join(self.output_dir, "log.log") + + if not logger.handlers: + sh = logging.StreamHandler() + fh = logging.FileHandler(filename=log_path, mode='a', encoding="utf-8") + fmt = logging.Formatter( + fmt='[%(levelname)s - %(asctime)s]\t%(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p') + + sh_fmt = colorlog.ColoredFormatter( + fmt='%(log_color)s[%(levelname)s - %(asctime)s]\t%(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p', + log_colors=log_colors_config) + sh.setFormatter(fmt=sh_fmt) + fh.setFormatter(fmt=fmt) + logger.addHandler(sh) + logger.addHandler(fh) + return logger + + def set_config(self, config: Config): + """save config + + Args: + config (Config): configuration object to save + """ + if self.accelerator is not None: + self.accelerator.init_trackers(self.logger_name, config=config) + elif self.logger_type == "wandb": + self.logger.config.update(config) + elif self.logger_type == "fitlog": + self.logger.add_hyper(config) + else: + with open(self.config_file, "a", encoding="utf8") as f: + f.write(json.dumps(config) + "\n") + + def log(self, data, step=0): + """log data and step + + Args: + data (Any): data to log + step (int, optional): step num. Defaults to 0. + """ + if self.accelerator is not None: + self.accelerator.log(data, step=0) + elif self.logger_type == "wandb": + self.logger.log(data, step=step) + elif self.logger_type == "fitlog": + self.logger.add_other({"data": data, "step": step}) + else: + with open(self.other_log_file, "a", encoding="utf8") as f: + f.write(json.dumps({"data": data, "step": step}) + "\n") + + def log_metric(self, metric, metric_split="dev", step=0): + """log metric + + Args: + metric (Any): metric + metric_split (str, optional): dataset split. Defaults to 'dev'. + step (int, optional): step num. Defaults to 0. + """ + if self.accelerator is not None: + self.accelerator.log({metric_split: metric}, step=step) + elif self.logger_type == "wandb": + self.logger.log({metric_split: metric}, step=step) + elif self.logger_type == "fitlog": + self.logger.add_metric({metric_split: metric}, step=step) + else: + with open(self.metric_file, "a", encoding="utf8") as f: + f.write(json.dumps({metric_split: metric, "step": step}) + "\n") + + def log_loss(self, loss, loss_name="Loss", step=0): + """log loss + + Args: + loss (Any): loss + loss_name (str, optional): loss description. Defaults to 'Loss'. + step (int, optional): step num. Defaults to 0. + """ + if self.accelerator is not None: + self.accelerator.log({loss_name: loss}, step=step) + elif self.logger_type == "wandb": + self.logger.log({loss_name: loss}, step=step) + elif self.logger_type == "fitlog": + self.logger.add_loss(loss, name=loss_name, step=step) + else: + with open(self.loss_file, "a", encoding="utf8") as f: + f.write(json.dumps({loss_name: loss, "step": step}) + "\n") + + def finish(self): + """finish logging + """ + if self.logger_type == "fitlog": + self.logger.finish() + + def info(self, message:str): + """ Log a message with severity 'INFO' in local file / console. + + Args: + message (str): message to log + """ + self.logging.info(message) + + def warning(self, message): + """ Log a message with severity 'WARNING' in local file / console. + + Args: + message (str): message to log + """ + self.logging.warning(message) + + def error(self, message): + """ Log a message with severity 'ERROR' in local file / console. + + Args: + message (str): message to log + """ + self.logging.error(message) + + def debug(self, message): + """ Log a message with severity 'DEBUG' in local file / console. + + Args: + message (str): message to log + """ + self.logging.debug(message) + + def critical(self, message): + self.logging.critical(message) diff --git a/common/metric.py b/common/metric.py new file mode 100644 index 0000000000000000000000000000000000000000..f84fd67830f2b2da9e09a4f7cb67644a51f139dc --- /dev/null +++ b/common/metric.py @@ -0,0 +1,346 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-17 19:39:22 +Description: Metric calculation class + +''' +from collections import Counter +from typing import List, Dict + +import numpy as np +from sklearn.metrics import f1_score + +from common.utils import InputData, OutputData + + +class Evaluator(object): + """Evaluation metric funtions library class + supported metric: + - slot_f1 + - intent_acc + - exactly_match_accuracy + - intent_f1 (defult "macro_intent_f1") + - macro_intent_f1 + - micro_intent_f1= + """ + @staticmethod + def exactly_match_accuracy(pred_slot: List[List[str or int]], + real_slot: List[List[str or int]], + pred_intent: List[List[str or int] or str or int], + real_intent: List[List[str or int] or str or int]) -> float: + """Compute the accuracy based on the whole predictions of given sentence, including slot and intent. + (both support str or int index as the representation of slot and intent) + Args: + pred_slot (List[List[str or int]]): predicted sequence of slot list + real_slot (List[List[str or int]]): golden sequence of slot list. + pred_intent (List[List[str or int] or str or int]): golden intent list / golden multi intent list. + real_intent (List[List[str or int] or str or int]): predicted intent list / predicted multi intent list. + + Returns: + float: exactly match accuracy score + """ + total_count, correct_count = 0.0, 0.0 + for p_slot, r_slot, p_intent, r_intent in zip(pred_slot, real_slot, pred_intent, real_intent): + if isinstance(p_intent, list): + p_intent, r_intent = set(p_intent), set(r_intent) + if p_slot == r_slot and p_intent == r_intent: + correct_count += 1.0 + total_count += 1.0 + + return 1.0 * correct_count / total_count + + + @staticmethod + def intent_accuracy(pred_list: List, real_list: List) -> float: + """Get intent accuracy measured by predictions and ground-trues. Support both multi intent and single intent. + + Args: + pred_list (List): predicted intent list + real_list (List): golden intent list + + Returns: + float: intent accuracy score + """ + total_count, correct_count = 0.0, 0.0 + for p_intent, r_intent in zip(pred_list, real_list): + if isinstance(p_intent, list): + p_intent, r_intent = set(p_intent), set(r_intent) + if p_intent == r_intent: + correct_count += 1.0 + total_count += 1.0 + + return 1.0 * correct_count / total_count + + @staticmethod + def intent_f1(pred_list: List[List[int]], real_list: List[List[int]], num_intent: int, average='macro') -> float: + """Get intent accuracy measured by predictions and ground-trues. Support both multi intent and single intent. + (Only support multi intent now, but you can use [[intent1], [intent2], ...] to compute intent f1 in single intent) + Args: + pred_list (List[List[int]]): predicted multi intent list. + real_list (List[List[int]]): golden multi intent list. + num_intent (int) + average (str): support "micro" and "macro" + + Returns: + float: intent accuracy score + """ + return f1_score(Evaluator.__instance2onehot(num_intent, real_list), + Evaluator.__instance2onehot(num_intent, pred_list), + average=average, + zero_division=0) + + @staticmethod + def __multilabel2one_hot(labels, nums): + res = [0.] * nums + if len(labels) == 0: + return res + if isinstance(labels[0], list): + for label in labels[0]: + res[label] = 1. + return res + for label in labels: + res[label] = 1. + return res + + @staticmethod + def __instance2onehot(num_intent, data): + res = [] + for intents in data: + res.append(Evaluator.__multilabel2one_hot(intents, num_intent)) + return np.array(res) + + @staticmethod + def __startOfChunk(prevTag, tag, prevTagType, tagType, chunkStart=False): + if prevTag == 'B' and tag == 'B': + chunkStart = True + if prevTag == 'I' and tag == 'B': + chunkStart = True + if prevTag == 'O' and tag == 'B': + chunkStart = True + if prevTag == 'O' and tag == 'I': + chunkStart = True + + if prevTag == 'E' and tag == 'E': + chunkStart = True + if prevTag == 'E' and tag == 'I': + chunkStart = True + if prevTag == 'O' and tag == 'E': + chunkStart = True + if prevTag == 'O' and tag == 'I': + chunkStart = True + + if tag != 'O' and tag != '.' and prevTagType != tagType: + chunkStart = True + return chunkStart + + @staticmethod + def __endOfChunk(prevTag, tag, prevTagType, tagType, chunkEnd=False): + if prevTag == 'B' and tag == 'B': + chunkEnd = True + if prevTag == 'B' and tag == 'O': + chunkEnd = True + if prevTag == 'I' and tag == 'B': + chunkEnd = True + if prevTag == 'I' and tag == 'O': + chunkEnd = True + + if prevTag == 'E' and tag == 'E': + chunkEnd = True + if prevTag == 'E' and tag == 'I': + chunkEnd = True + if prevTag == 'E' and tag == 'O': + chunkEnd = True + if prevTag == 'I' and tag == 'O': + chunkEnd = True + + if prevTag != 'O' and prevTag != '.' and prevTagType != tagType: + chunkEnd = True + return chunkEnd + + @staticmethod + def __splitTagType(tag): + s = tag.split('-') + if len(s) > 2 or len(s) == 0: + raise ValueError('tag format wrong. it must be B-xxx.xxx') + if len(s) == 1: + tag = s[0] + tagType = "" + else: + tag = s[0] + tagType = s[1] + return tag, tagType + + @staticmethod + def computeF1Score(correct_slots: List[List[str]], pred_slots: List[List[str]]) -> float: + """compute f1 score is modified from conlleval.pl + + Args: + correct_slots (List[List[str]]): golden slot string list + pred_slots (List[List[str]]): predicted slot string list + + Returns: + float: slot f1 score + """ + correctChunk = {} + correctChunkCnt = 0.0 + foundCorrect = {} + foundCorrectCnt = 0.0 + foundPred = {} + foundPredCnt = 0.0 + correctTags = 0.0 + tokenCount = 0.0 + for correct_slot, pred_slot in zip(correct_slots, pred_slots): + inCorrect = False + lastCorrectTag = 'O' + lastCorrectType = '' + lastPredTag = 'O' + lastPredType = '' + for c, p in zip(correct_slot, pred_slot): + c = str(c) + p = str(p) + correctTag, correctType = Evaluator.__splitTagType(c) + predTag, predType = Evaluator.__splitTagType(p) + + if inCorrect == True: + if Evaluator.__endOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) == True and \ + Evaluator.__endOfChunk(lastPredTag, predTag, lastPredType, predType) == True and \ + (lastCorrectType == lastPredType): + inCorrect = False + correctChunkCnt += 1.0 + if lastCorrectType in correctChunk: + correctChunk[lastCorrectType] += 1.0 + else: + correctChunk[lastCorrectType] = 1.0 + elif Evaluator.__endOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) != \ + Evaluator.__endOfChunk(lastPredTag, predTag, lastPredType, predType) or \ + (correctType != predType): + inCorrect = False + + if Evaluator.__startOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) == True and \ + Evaluator.__startOfChunk(lastPredTag, predTag, lastPredType, predType) == True and \ + (correctType == predType): + inCorrect = True + + if Evaluator.__startOfChunk(lastCorrectTag, correctTag, lastCorrectType, correctType) == True: + foundCorrectCnt += 1 + if correctType in foundCorrect: + foundCorrect[correctType] += 1.0 + else: + foundCorrect[correctType] = 1.0 + + if Evaluator.__startOfChunk(lastPredTag, predTag, lastPredType, predType) == True: + foundPredCnt += 1.0 + if predType in foundPred: + foundPred[predType] += 1.0 + else: + foundPred[predType] = 1.0 + + if correctTag == predTag and correctType == predType: + correctTags += 1.0 + + tokenCount += 1.0 + + lastCorrectTag = correctTag + lastCorrectType = correctType + lastPredTag = predTag + lastPredType = predType + + if inCorrect == True: + correctChunkCnt += 1.0 + if lastCorrectType in correctChunk: + correctChunk[lastCorrectType] += 1.0 + else: + correctChunk[lastCorrectType] = 1.0 + + if foundPredCnt > 0: + precision = 1.0 * correctChunkCnt / foundPredCnt + else: + precision = 0 + + if foundCorrectCnt > 0: + recall = 1.0 * correctChunkCnt / foundCorrectCnt + else: + recall = 0 + + if (precision + recall) > 0: + f1 = (2.0 * precision * recall) / (precision + recall) + else: + f1 = 0 + + return f1 + + @staticmethod + def max_freq_predict(sample): + """Max frequency prediction. + """ + predict = [] + for items in sample: + predict.append(Counter(items).most_common(1)[0][0]) + return predict + + @staticmethod + def __token_map(indexes, token_label_map): + return [[token_label_map[idx] if idx in token_label_map else -1 for idx in index] for index in indexes] + + @staticmethod + def compute_all_metric(inps: InputData, + output: OutputData, + intent_label_map: dict = None, + metric_list: List=None)-> Dict: + """Auto compute all metric mentioned in 'metric_list' + + Args: + inps (InputData): input golden slot and intent labels + output (OutputData): output predicted slot and intent labels + intent_label_map (dict, Optional): dict like {"intent1": 0, "intent2": 1, ...},which aims to map intent string to index + metric_list (List): support metrics in ["slot_f1", "intent_acc", "intent_f1", "macro_intent_f1", "micro_intent_f1", "EMA"] + + Returns: + Dict: all metric mentioned in 'metric_list', like {'EMA': 0.7, ...} + + + Example: + if compute slot metric: + + inps.slot = [["slot1", "slot2", ...], ...]; output.slot_ids=[["slot1", "slot2", ...], ...]; + + if compute intent metric: + + [Multi Intent] inps.intent = [["intent1", "intent2", ...], ...]; output.intent_ids = [["intent1", "intent2", ...], ...] + + [Single Intent] inps.intent = ["intent1", ...]; [Single Intent] output.intent_ids = ["intent1", ...] + """ + if not metric_list: + metric_list = ["slot_f1", "intent_acc", "EMA"] + res_dict = {} + use_slot = output.slot_ids is not None and len(output.slot_ids) > 0 + use_intent = output.intent_ids is not None and len( + output.intent_ids) > 0 + if use_slot and "slot_f1" in metric_list: + + res_dict["slot_f1"] = Evaluator.computeF1Score( + output.slot_ids, inps.slot) + if use_intent and "intent_acc" in metric_list: + res_dict["intent_acc"] = Evaluator.intent_accuracy( + output.intent_ids, inps.intent) + if isinstance(output.intent_ids[0], list): + if "intent_f1" in metric_list: + res_dict["intent_f1"] = Evaluator.intent_f1(Evaluator.__token_map(output.intent_ids, intent_label_map), + Evaluator.__token_map( + inps.intent, intent_label_map), + len(intent_label_map.keys())) + elif "macro_intent_f1" in metric_list: + res_dict["macro_intent_f1"] = Evaluator.intent_f1(Evaluator.__token_map(output.intent_ids, intent_label_map), + Evaluator.__token_map(inps.intent, intent_label_map), + len(intent_label_map.keys()), average="macro") + if "micro_intent_f1" in metric_list: + res_dict["micro_intent_f1"] = Evaluator.intent_f1(Evaluator.__token_map(output.intent_ids, intent_label_map), + Evaluator.__token_map(inps.intent, intent_label_map), + len(intent_label_map.keys()), average="micro") + + if use_slot and use_intent and "EMA" in metric_list: + res_dict["EMA"] = Evaluator.exactly_match_accuracy(output.slot_ids, inps.slot, output.intent_ids, + inps.intent) + return res_dict diff --git a/common/model_manager.py b/common/model_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..c2969529bbccf0ee37ab24aff851e002ea5ba3bf --- /dev/null +++ b/common/model_manager.py @@ -0,0 +1,419 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-19 18:50:11 +Description: manage all process of model training and prediction. + +''' +import math +import os +import queue +import random + +import numpy as np +import torch +from tqdm import tqdm + + +from common import utils +from common.loader import DataFactory +from common.logger import Logger +from common.metric import Evaluator +from common.saver import Saver +from common.tokenizer import get_tokenizer, get_tokenizer_class, load_embedding +from common.utils import InputData, instantiate +from common.utils import OutputData +from common.config import Config +import dill +from common import global_pool +from tools.load_from_hugging_face import PreTrainedTokenizerForSLU, PretrainedModelForSLU +# from tools.hugging_face_parser import load_model, load_tokenizer + + +class ModelManager(object): + def __init__(self, config: Config): + """create model manager by config + + Args: + config (Config): configuration to manage all process in OpenSLU + """ + # init config + global_pool._init() + self.config = config + self.__set_seed(self.config.base.get("seed")) + self.device = self.config.base.get("device") + self.load_dir = self.config.model_manager.get("load_dir") + if self.config.get("logger") and self.config["logger"].get("logger_type"): + logger_type = self.config["logger"].get("logger_type") + else: + logger_type = "wandb" + # enable accelerator + if "accelerator" in self.config and self.config["accelerator"].get("use_accelerator"): + from accelerate import Accelerator + self.accelerator = Accelerator(log_with=logger_type) + else: + self.accelerator = None + self.tokenizer = None + self.saver = Saver(self.config.model_manager, start_time=self.config.start_time) + if self.config.base.get("train"): + self.model = None + self.optimizer = None + self.total_step = None + self.lr_scheduler = None + self.init_step = 0 + self.best_metric = 0 + self.logger = Logger(logger_type=logger_type, + logger_name=self.config.base["name"], + start_time=self.config.start_time, + accelerator=self.accelerator) + global_pool.set_value("logger", self.logger) + + def init_model(self): + """init model, optimizer, lr_scheduler + + Args: + model (Any): pytorch model + """ + self.prepared = False + if self.load_dir is not None: + self.load() + self.config.set_vocab_size(self.tokenizer.vocab_size) + self.init_data() + if self.config.base.get("train") and self.config.model_manager.get("load_train_state"): + train_state = torch.load(os.path.join( + self.load_dir, "train_state.pkl"), pickle_module=dill) + self.optimizer = instantiate( + self.config["optimizer"])(self.model.parameters()) + self.lr_scheduler = instantiate(self.config["scheduler"])( + optimizer=self.optimizer, + num_training_steps=self.total_step + ) + self.optimizer.load_state_dict(train_state["optimizer"]) + self.optimizer.zero_grad() + self.lr_scheduler.load_state_dict(train_state["lr_scheduler"]) + self.init_step = train_state["step"] + self.best_metric = train_state["best_metric"] + elif self.config.model.get("_from_pretrained_") and self.config.tokenizer.get("_from_pretrained_"): + self.from_pretrained() + self.config.set_vocab_size(self.tokenizer.vocab_size) + self.init_data() + else: + self.tokenizer = get_tokenizer( + self.config.tokenizer.get("_tokenizer_name_")) + self.init_data() + self.model = instantiate(self.config.model) + self.model.to(self.device) + if self.config.base.get("train"): + self.optimizer = instantiate( + self.config["optimizer"])(self.model.parameters()) + self.lr_scheduler = instantiate(self.config["scheduler"])( + optimizer=self.optimizer, + num_training_steps=self.total_step + ) + + + def init_data(self): + self.data_factory = DataFactory(tokenizer=self.tokenizer, + use_multi_intent=self.config.base.get("multi_intent"), + to_lower_case=self.config.tokenizer.get("_to_lower_case_")) + batch_size = self.config.base["batch_size"] + # init tokenizer config and dataloaders + tokenizer_config = {key: self.config.tokenizer[key] + for key in self.config.tokenizer if key[0] != "_" and key[-1] != "_"} + + if self.config.base.get("train"): + # init dataloader & load data + + + train_dataset = self.data_factory.load_dataset(self.config.dataset, split="train") + + # update label and vocabulary (ONLY SUPPORT FOR "word_tokenizer") + self.data_factory.update_label_names(train_dataset) + self.data_factory.update_vocabulary(train_dataset) + + + self.train_dataloader = self.data_factory.get_data_loader(train_dataset, + batch_size, + shuffle=True, + device=self.device, + enable_label=True, + align_mode=self.config.tokenizer.get( + "_align_mode_"), + label2tensor=True, + **tokenizer_config) + self.total_step = int(self.config.base.get("epoch_num")) * len(self.train_dataloader) + dev_dataset = self.data_factory.load_dataset(self.config.dataset, split="validation") + self.dev_dataloader = self.data_factory.get_data_loader(dev_dataset, + batch_size, + shuffle=False, + device=self.device, + enable_label=True, + align_mode=self.config.tokenizer.get( + "_align_mode_"), + label2tensor=False, + **tokenizer_config) + self.data_factory.update_vocabulary(dev_dataset) + self.intent_list = None + self.intent_dict = None + self.slot_list = None + self.slot_dict = None + # add intent label num and slot label num to config + if self.config.model["decoder"].get("intent_classifier") and int(self.config.get_intent_label_num()) == 0: + self.intent_list = self.data_factory.intent_label_list + self.intent_dict = self.data_factory.intent_label_dict + self.config.set_intent_label_num(len(self.intent_list)) + if self.config.model["decoder"].get("slot_classifier") and int(self.config.get_slot_label_num()) == 0: + self.slot_list = self.data_factory.slot_label_list + self.slot_dict = self.data_factory.slot_label_dict + self.config.set_slot_label_num(len(self.slot_list)) + + + + # autoload embedding for non-pretrained encoder + if self.config["model"]["encoder"].get("embedding") and self.config["model"]["encoder"]["embedding"].get( + "load_embedding_name"): + self.config["model"]["encoder"]["embedding"]["embedding_matrix"] = load_embedding(self.tokenizer, + self.config["model"][ + "encoder"][ + "embedding"].get( + "load_embedding_name")) + # fill template in config + self.config.autoload_template() + # save config + self.logger.set_config(self.config) + self.saver.save_tokenizer(self.tokenizer) + self.saver.save_label(self.intent_list, self.slot_list) + self.config.set_vocab_size(self.tokenizer.vocab_size) + + if self.config.base.get("test"): + self.test_dataset = self.data_factory.load_dataset(self.config.dataset, split="test") + self.test_dataloader = self.data_factory.get_data_loader(self.test_dataset, + batch_size, + shuffle=False, + device=self.device, + enable_label=True, + align_mode=self.config.tokenizer.get( + "_align_mode_"), + label2tensor=False, + **tokenizer_config) + + def eval(self, step: int, best_metric: float) -> float: + """ evaluation models. + + Args: + step (int): which step the model has trained in + best_metric (float): last best metric value to judge whether to test or save model + + Returns: + float: updated best metric value + """ + # TODO: save dev + _, res = self.__evaluate(self.model, self.dev_dataloader, mode="dev") + self.logger.log_metric(res, metric_split="dev", step=step) + if res[self.config.evaluator.get("best_key")] > best_metric: + best_metric = res[self.config.evaluator.get("best_key")] + train_state = { + "step": step, + "best_metric": best_metric, + "optimizer": self.optimizer.state_dict(), + "lr_scheduler": self.lr_scheduler.state_dict() + } + self.saver.save_model(self.model, train_state, self.accelerator) + if self.config.base.get("test"): + outputs, test_res = self.__evaluate(self.model, self.test_dataloader, mode="test") + self.saver.save_output(outputs, self.test_dataset) + self.logger.log_metric(test_res, metric_split="test", step=step) + return best_metric + + def train(self) -> float: + """ train models. + + Returns: + float: updated best metric value + """ + self.model.train() + if self.accelerator is not None: + self.total_step = math.ceil(self.total_step / self.accelerator.num_processes) + if self.optimizer is None: + self.optimizer = instantiate(self.config["optimizer"])(self.model.parameters()) + if self.lr_scheduler is None: + self.lr_scheduler = instantiate(self.config["scheduler"])( + optimizer=self.optimizer, + num_training_steps=self.total_step + ) + if not self.prepared and self.accelerator is not None: + self.model, self.optimizer, self.train_dataloader, self.lr_scheduler = self.accelerator.prepare( + self.model, self.optimizer, self.train_dataloader, self.lr_scheduler) + step = self.init_step + progress_bar = tqdm(range(self.total_step)) + progress_bar.update(self.init_step) + self.optimizer.zero_grad() + for _ in range(int(self.config.base.get("epoch_num"))): + for data in self.train_dataloader: + if step == 0: + self.logger.info(data.get_item( + 0, tokenizer=self.tokenizer, intent_map=self.intent_list, slot_map=self.slot_list)) + output = self.model(data) + if self.accelerator is not None and hasattr(self.model, "module"): + loss, intent_loss, slot_loss = self.model.module.compute_loss( + pred=output, target=data) + else: + loss, intent_loss, slot_loss = self.model.compute_loss( + pred=output, target=data) + self.logger.log_loss(loss, "Loss", step=step) + self.logger.log_loss(intent_loss, "Intent Loss", step=step) + self.logger.log_loss(slot_loss, "Slot Loss", step=step) + self.optimizer.zero_grad() + + if self.accelerator is not None: + self.accelerator.backward(loss) + else: + loss.backward() + self.optimizer.step() + self.lr_scheduler.step() + train_state = { + "step": step, + "best_metric": self.best_metric, + "optimizer": self.optimizer.state_dict(), + "lr_scheduler": self.lr_scheduler.state_dict() + } + if not self.saver.auto_save_step(self.model, train_state, self.accelerator): + if not self.config.evaluator.get("eval_by_epoch") and step % self.config.evaluator.get("eval_step") == 0 and step != 0: + self.best_metric = self.eval(step, self.best_metric) + step += 1 + progress_bar.update(1) + if self.config.evaluator.get("eval_by_epoch"): + self.best_metric = self.eval(step, self.best_metric) + self.logger.finish() + return self.best_metric + + def test(self): + return self.__evaluate(self.model, self.test_dataloader, mode="test") + + def __set_seed(self, seed_value: int): + """Manually set random seeds. + + Args: + seed_value (int): random seed + """ + random.seed(seed_value) + np.random.seed(seed_value) + torch.manual_seed(seed_value) + torch.random.manual_seed(seed_value) + os.environ['PYTHONHASHSEED'] = str(seed_value) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed_value) + torch.cuda.manual_seed_all(seed_value) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = True + return + + def __evaluate(self, model, dataloader, mode="dev"): + model.eval() + inps = InputData() + outputs = OutputData() + for data in dataloader: + torch.cuda.empty_cache() + output = model(data) + if self.accelerator is not None and hasattr(self.model, "module"): + decode_output = model.module.decode(output, data) + else: + decode_output = model.decode(output, data) + + decode_output.map_output(slot_map=self.slot_list, + intent_map=self.intent_list) + if self.config.model["decoder"].get("slot_classifier"): + data, decode_output = utils.remove_slot_ignore_index( + data, decode_output, ignore_index="#") + + inps.merge_input_data(data) + outputs.merge_output_data(decode_output) + if "metric" in self.config.evaluator: + res = Evaluator.compute_all_metric( + inps, outputs, intent_label_map=self.intent_dict, metric_list=self.config.evaluator["metric"]) + else: + res = Evaluator.compute_all_metric( + inps, outputs, intent_label_map=self.intent_dict) + self.logger.info(f"Best {mode} metric: "+str(res)) + model.train() + return outputs, res + + def load(self): + + if self.tokenizer is None: + with open(os.path.join(self.load_dir, "tokenizer.pkl"), 'rb') as f: + self.tokenizer = dill.load(f) + label = utils.load_json(os.path.join(self.load_dir, "label.json")) + if label["intent"] is None: + self.intent_list = None + self.intent_dict = None + else: + self.intent_list = label["intent"] + self.intent_dict = {x: i for i, x in enumerate(label["intent"])} + self.config.set_intent_label_num(len(self.intent_list)) + if label["slot"] is None: + self.slot_list = None + self.slot_dict = None + else: + self.slot_list = label["slot"] + self.slot_dict = {x: i for i, x in enumerate(label["slot"])} + self.config.set_slot_label_num(len(self.slot_list)) + self.config.set_vocab_size(self.tokenizer.vocab_size) + if self.accelerator is not None and self.load_dir is not None: + self.model = torch.load(os.path.join(self.load_dir, "model.pkl"), map_location=torch.device(self.device)) + self.prepared = True + self.accelerator.load_state(self.load_dir) + self.accelerator.prepare_model(self.model) + else: + self.model = torch.load(os.path.join( + self.load_dir, "model.pkl"), map_location=torch.device(self.device)) + # if self.config.tokenizer["_tokenizer_name_"] == "word_tokenizer": + # self.tokenizer = get_tokenizer_class(self.config.tokenizer["_tokenizer_name_"]).from_file(os.path.join(self.load_dir, "tokenizer.json")) + # else: + # self.tokenizer = get_tokenizer(self.config.tokenizer["_tokenizer_name_"]) + self.model.to(self.device) + + + def from_pretrained(self): + self.config.autoload_template() + model = PretrainedModelForSLU.from_pretrained(self.config.model["_from_pretrained_"]) + # model = load_model(self.config.model["_from_pretrained_"]) + self.model = model.model + if self.tokenizer is None: + self.tokenizer = PreTrainedTokenizerForSLU.from_pretrained( + self.config.tokenizer["_from_pretrained_"]) + self.config.tokenizer = model.config.tokenizer + # self.tokenizer = load_tokenizer(self.config.tokenizer["_from_pretrained_"]) + + self.model.to(self.device) + label = model.config._id2label + self.config.model = model.config.model + self.intent_list = label["intent"] + self.slot_list = label["slot"] + self.intent_dict = {x: i for i, x in enumerate(label["intent"])} + self.slot_dict = {x: i for i, x in enumerate(label["slot"])} + + def predict(self, text_data): + self.model.eval() + tokenizer_config = {key: self.config.tokenizer[key] + for key in self.config.tokenizer if key[0] != "_" and key[-1] != "_"} + align_mode = self.config.tokenizer.get("_align_mode_") + inputs = self.data_factory.batch_fn(batch=[{"text": text_data.split(" ")}], + device=self.device, + config=tokenizer_config, + enable_label=False, + align_mode=align_mode if align_mode is not None else "general", + label2tensor=False) + output = self.model(inputs) + decode_output = self.model.decode(output, inputs) + decode_output.map_output(slot_map=self.slot_list, + intent_map=self.intent_list) + if self.config.base.get("multi_intent"): + intent = decode_output.intent_ids[0] + else: + intent = [decode_output.intent_ids[0]] + input_ids = inputs.input_ids[0].tolist() + tokens = [self.tokenizer.decode(ids) for ids in input_ids] + slots = decode_output.slot_ids[0] + return {"intent": intent, "slot": slots, "text": tokens} diff --git a/common/saver.py b/common/saver.py new file mode 100644 index 0000000000000000000000000000000000000000..82536faeb764ce50bd28db893f693633d35c6310 --- /dev/null +++ b/common/saver.py @@ -0,0 +1,80 @@ +''' +Author: Qiguang Chen +LastEditors: Qiguang Chen +Date: 2023-02-12 22:23:58 +LastEditTime: 2023-02-19 14:14:56 +Description: + +''' +import json +import os +import queue +import shutil +import torch +import dill +from common import utils + + +class Saver(): + def __init__(self, config, start_time=None) -> None: + self.config = config + if self.config.get("save_dir"): + self.model_save_dir = self.config["save_dir"] + else: + if not os.path.exists("save/"): + os.mkdir("save/") + self.model_save_dir = "save/" + start_time + if not os.path.exists(self.model_save_dir): + os.mkdir(self.model_save_dir) + save_mode = config.get("save_mode") + self.save_mode = save_mode if save_mode is not None else "save-by-eval" + + max_save_num = self.config.get("max_save_num") + self.max_save_num = max_save_num if max_save_num is not None else 1 + self.save_pool = queue.Queue(maxsize=max_save_num) + + def save_tokenizer(self, tokenizer): + with open(os.path.join(self.model_save_dir, "tokenizer.pkl"), 'wb') as f: + dill.dump(tokenizer, f) + + def save_label(self, intent_list, slot_list): + utils.save_json(os.path.join(self.model_save_dir, "label.json"), {"intent": intent_list, "slot": slot_list}) + + + def save_model(self, model, train_state, accelerator=None): + step = train_state["step"] + if self.max_save_num != 1: + + model_save_dir =os.path.join(self.model_save_dir, str(step)) + if self.save_pool.full(): + delete_dir = self.save_pool.get() + shutil.rmtree(delete_dir) + self.save_pool.put(model_save_dir) + else: + self.save_pool.put(model_save_dir) + if not os.path.exists(model_save_dir): + os.mkdir(model_save_dir) + else: + model_save_dir = self.model_save_dir + if not os.path.exists(model_save_dir): + os.mkdir(model_save_dir) + if accelerator is None: + torch.save(model, os.path.join(model_save_dir, "model.pkl")) + torch.save(train_state, os.path.join(model_save_dir, "train_state.pkl"), pickle_module=dill) + else: + accelerator.wait_for_everyone() + unwrapped_model = accelerator.unwrap_model(model) + accelerator.save(unwrapped_model, os.path.join(model_save_dir, "model.pkl")) + accelerator.save_state(output_dir=model_save_dir) + + def auto_save_step(self, model, train_state, accelerator=None): + step = train_state["step"] + if self.save_mode == "save-by-step" and step % self.config.get("save_step")==0 and step != 0: + self.save_model(model, train_state, accelerator) + return True + else: + return False + + + def save_output(self, outputs, dataset): + outputs.save(self.model_save_dir, dataset) \ No newline at end of file diff --git a/common/tokenizer.py b/common/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..8b97e721b6da3cb94b18edaf5778c2c149412ff7 --- /dev/null +++ b/common/tokenizer.py @@ -0,0 +1,323 @@ +import json +import os +from collections import Counter +from collections import OrderedDict +from typing import List + +import torch +from ordered_set import OrderedSet +from transformers import AutoTokenizer + +from common.utils import download, unzip_file + + +def get_tokenizer(tokenizer_name:str): + """auto get tokenizer + + Args: + tokenizer_name (str): support "word_tokenizer" and other pretrained tokenizer in hugging face. + + Returns: + Any: Tokenizer Object + """ + if tokenizer_name == "word_tokenizer": + return WordTokenizer(tokenizer_name) + else: + return AutoTokenizer.from_pretrained(tokenizer_name) + +def get_tokenizer_class(tokenizer_name:str): + """auto get tokenizer class + + Args: + tokenizer_name (str): support "word_tokenizer" and other pretrained tokenizer in hugging face. + + Returns: + Any: Tokenizer Class + """ + if tokenizer_name == "word_tokenizer": + return WordTokenizer + else: + return AutoTokenizer.from_pretrained + +BATCH_STATE = 1 +INSTANCE_STATE = 2 + + +class WordTokenizer(object): + + def __init__(self, name): + self.__name = name + self.index2instance = OrderedSet() + self.instance2index = OrderedDict() + # Counter Object record the frequency + # of element occurs in raw text. + self.counter = Counter() + + self.__sign_pad = "[PAD]" + self.add_instance(self.__sign_pad) + self.__sign_unk = "[UNK]" + self.add_instance(self.__sign_unk) + + @property + def padding_side(self): + return "right" + @property + def all_special_ids(self): + return [self.unk_token_id, self.pad_token_id] + + @property + def name_or_path(self): + return self.__name + + @property + def vocab_size(self): + return len(self.instance2index) + + @property + def pad_token_id(self): + return self.instance2index[self.__sign_pad] + + @property + def unk_token_id(self): + return self.instance2index[self.__sign_unk] + + def add_instance(self, instance): + """ Add instances to alphabet. + + 1, We support any iterative data structure which + contains elements of str type. + + 2, We will count added instances that will influence + the serialization of unknown instance. + + Args: + instance: is given instance or a list of it. + """ + + if isinstance(instance, (list, tuple)): + for element in instance: + self.add_instance(element) + return + + # We only support elements of str type. + assert isinstance(instance, str) + + # count the frequency of instances. + # self.counter[instance] += 1 + + if instance not in self.index2instance: + self.instance2index[instance] = len(self.index2instance) + self.index2instance.append(instance) + + def __call__(self, instance, + return_tensors="pt", + is_split_into_words=True, + padding=True, + add_special_tokens=False, + truncation=True, + max_length=512, + **config): + if isinstance(instance, (list, tuple)) and isinstance(instance[0], (str)) and is_split_into_words: + res = self.get_index(instance) + state = INSTANCE_STATE + elif isinstance(instance, str) and not is_split_into_words: + res = self.get_index(instance.split(" ")) + state = INSTANCE_STATE + elif not is_split_into_words and isinstance(instance, (list, tuple)): + res = [self.get_index(ins.split(" ")) for ins in instance] + state = BATCH_STATE + else: + res = [self.get_index(ins) for ins in instance] + state = BATCH_STATE + res = [r[:max_length] if len(r) >= max_length else r for r in res] + pad_id = self.get_index(self.__sign_pad) + if padding and state == BATCH_STATE: + max_len = max([len(x) for x in instance]) + + for i in range(len(res)): + res[i] = res[i] + [pad_id] * (max_len - len(res[i])) + if return_tensors == "pt": + input_ids = torch.Tensor(res).long() + attention_mask = (input_ids != pad_id).long() + elif state == BATCH_STATE: + input_ids = res + attention_mask = [1 if r != pad_id else 0 for batch in res for r in batch] + else: + input_ids = res + attention_mask = [1 if r != pad_id else 0 for r in res] + return TokenizedData(input_ids, token_type_ids=attention_mask, attention_mask=attention_mask) + + def get_index(self, instance): + """ Serialize given instance and return. + + For unknown words, the return index of alphabet + depends on variable self.__use_unk: + + 1, If True, then return the index of ""; + 2, If False, then return the index of the + element that hold max frequency in training data. + + Args: + instance (Any): is given instance or a list of it. + Return: + Any: the serialization of query instance. + """ + + if isinstance(instance, (list, tuple)): + return [self.get_index(elem) for elem in instance] + + assert isinstance(instance, str) + + try: + return self.instance2index[instance] + except KeyError: + return self.instance2index[self.__sign_unk] + + def decode(self, index): + """ Get corresponding instance of query index. + + if index is invalid, then throws exception. + + Args: + index (int): is query index, possibly iterable. + Returns: + is corresponding instance. + """ + + if isinstance(index, list): + return [self.decode(elem) for elem in index] + if isinstance(index, torch.Tensor): + index = index.tolist() + return self.decode(index) + return self.index2instance[index] + + def decode_batch(self, index, **kargs): + """ Get corresponding instance of query index. + + if index is invalid, then throws exception. + + Args: + index (int): is query index, possibly iterable. + Returns: + is corresponding instance. + """ + return self.decode(index) + + def save(self, path): + """ Save the content of alphabet to files. + + There are two kinds of saved files: + 1, The first is a list file, elements are + sorted by the frequency of occurrence. + + 2, The second is a dictionary file, elements + are sorted by it serialized index. + + Args: + path (str): is the path to save object. + """ + + with open(path, 'w', encoding="utf8") as fw: + fw.write(json.dumps({"name": self.__name, "token_map": self.instance2index})) + + @staticmethod + def from_file(path): + with open(path, 'r', encoding="utf8") as fw: + obj = json.load(fw) + tokenizer = WordTokenizer(obj["name"]) + tokenizer.instance2index = OrderedDict(obj["token_map"]) + # tokenizer.counter = len(tokenizer.instance2index) + tokenizer.index2instance = OrderedSet(tokenizer.instance2index.keys()) + return tokenizer + + def __len__(self): + return len(self.index2instance) + + def __str__(self): + return 'Alphabet {} contains about {} words: \n\t{}'.format(self.name_or_path, len(self), self.index2instance) + + def convert_tokens_to_ids(self, tokens): + """convert token sequence to intput ids sequence + + Args: + tokens (Any): token sequence + + Returns: + Any: intput ids sequence + """ + try: + if isinstance(tokens, (list, tuple)): + return [self.instance2index[x] for x in tokens] + return self.instance2index[tokens] + + except KeyError: + return self.instance2index[self.__sign_unk] + + +class TokenizedData(): + """tokenized output data with input_ids, token_type_ids, attention_mask + """ + def __init__(self, input_ids, token_type_ids, attention_mask): + self.input_ids = input_ids + self.token_type_ids = token_type_ids + self.attention_mask = attention_mask + + def word_ids(self, index: int) -> List[int or None]: + """ get word id list + + Args: + index (int): word index in sequence + + Returns: + List[int or None]: word id list + """ + return [j if self.attention_mask[index][j] != 0 else None for j, x in enumerate(self.input_ids[index])] + + def word_to_tokens(self, index, word_id, **kwargs): + """map word and tokens + + Args: + index (int): unused + word_id (int): word index in sequence + """ + return (word_id, word_id + 1) + + def to(self, device): + """set device + + Args: + device (str): support ["cpu", "cuda"] + """ + self.input_ids = self.input_ids.to(device) + self.token_type_ids = self.token_type_ids.to(device) + self.attention_mask = self.attention_mask.to(device) + return self + + +def load_embedding(tokenizer: WordTokenizer, glove_name:str): + """ load embedding from standford server or local cache. + + Args: + tokenizer (WordTokenizer): non-pretrained tokenizer + glove_name (str): _description_ + + Returns: + Any: word embedding + """ + save_path = "save/" + glove_name + ".zip" + if not os.path.exists(save_path): + download("http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip#" + glove_name, save_path) + unzip_file(save_path, "save/" + glove_name) + dim = int(glove_name.split(".")[-2][:-1]) + embedding_list = torch.rand((tokenizer.vocab_size, dim)) + embedding_list[tokenizer.pad_token_id] = torch.zeros((1, dim)) + with open("save/" + glove_name + "/" + glove_name, "r", encoding="utf8") as f: + for line in f.readlines(): + datas = line.split(" ") + word = datas[0] + embedding = torch.Tensor([float(datas[i + 1]) for i in range(len(datas) - 1)]) + tokenized = tokenizer.convert_tokens_to_ids(word) + if isinstance(tokenized, int) and tokenized != tokenizer.unk_token_id: + embedding_list[tokenized] = embedding + + return embedding_list diff --git a/common/utils.py b/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8569f3b06d1054623990fe9431826ed7cedc98e4 --- /dev/null +++ b/common/utils.py @@ -0,0 +1,499 @@ +import functools +import importlib +import json +import os +import tarfile +from typing import List, Tuple +import zipfile +from collections import Callable +from ruamel import yaml +import requests +import torch +from torch.nn.utils.rnn import pad_sequence +from tqdm import tqdm +from torch import Tensor +import argparse +class InputData(): + """input datas class + """ + def __init__(self, inputs: List =None): + """init input datas class + + if inputs is None: + this class can be used to save all InputData in the history by 'merge_input_data(X:InputData)' + else: + this class can be used for model input. + + Args: + inputs (List, optional): inputs with [tokenized_data, slot, intent]. Defaults to None. + """ + if inputs == None: + self.slot = [] + self.intent = [] + self.input_ids = None + self.token_type_ids = None + self.attention_mask = None + self.seq_lens = None + else: + self.input_ids = inputs[0].input_ids + self.token_type_ids = None + if hasattr(inputs[0], "token_type_ids"): + self.token_type_ids = inputs[0].token_type_ids + self.attention_mask = inputs[0].attention_mask + if len(inputs)>=2: + self.slot = inputs[1] + if len(inputs)>=3: + self.intent = inputs[2] + self.seq_lens = self.attention_mask.sum(-1) + + def get_inputs(self): + """ get tokenized_data + + Returns: + dict: tokenized data + """ + res = { + "input_ids": self.input_ids, + "attention_mask": self.attention_mask + } + if self.token_type_ids is not None: + res["token_type_ids"] = self.token_type_ids + return res + + def merge_input_data(self, inp: "InputData"): + """merge another InputData object with slot and intent + + Args: + inp (InputData): another InputData object + """ + self.slot += inp.slot + self.intent += inp.intent + + def get_slot_mask(self, ignore_index:int)->Tensor: + """get slot mask + + Args: + ignore_index (int): ignore index used in slot padding + + Returns: + Tensor: mask tensor + """ + mask = self.slot != ignore_index + mask[:, 0] = torch.ones_like(mask[:, 0]).to(self.slot.device) + return mask + + def get_item(self, index, tokenizer=None, intent_map=None, slot_map=None, ignore_index = -100): + res = {"input_ids": self.input_ids[index]} + if tokenizer is not None: + res["tokens"] = [tokenizer.decode(x) for x in self.input_ids[index]] + if intent_map is not None: + intents = self.intent.tolist() + if isinstance(intents[index], list): + res["intent"] = [intent_map[int(x)] for x in intents[index]] + else: + res["intent"] = intent_map[intents[index]] + if slot_map is not None: + res["slot"] = [slot_map[x] if x != ignore_index else "#" for x in self.slot.tolist()[index]] + return res + +class OutputData(): + """output data class + """ + def __init__(self, intent_ids=None, slot_ids=None): + """init output data class + + if intent_ids is None and slot_ids is None: + this class can be used to save all OutputData in the history by 'merge_output_data(X:OutputData)' + else: + this class can be used to model output management. + + Args: + intent_ids (Any, optional): list(Tensor) of intent ids / logits / strings. Defaults to None. + slot_ids (Any, optional): list(Tensor) of slot ids / ids / strings. Defaults to None. + """ + if intent_ids is None and slot_ids is None: + self.intent_ids = [] + self.slot_ids = [] + else: + if isinstance(intent_ids, ClassifierOutputData): + self.intent_ids = intent_ids.classifier_output + else: + self.intent_ids = intent_ids + if isinstance(slot_ids, ClassifierOutputData): + self.slot_ids = slot_ids.classifier_output + else: + self.slot_ids = slot_ids + + def map_output(self, slot_map=None, intent_map=None): + """ map intent or slot ids to intent or slot string. + + Args: + slot_map (dict, optional): slot id-to-string map. Defaults to None. + intent_map (dict, optional): intent id-to-string map. Defaults to None. + """ + if self.slot_ids is not None: + if slot_map: + self.slot_ids = [[slot_map[x] if x >= 0 else "#" for x in sid] for sid in self.slot_ids] + if self.intent_ids is not None: + if intent_map: + self.intent_ids = [[intent_map[x] for x in sid] if isinstance(sid, list) else intent_map[sid] for sid in + self.intent_ids] + + def merge_output_data(self, output:"OutputData"): + """merge another OutData object with slot and intent + + Args: + output (OutputData): another OutputData object + """ + if output.slot_ids is not None: + self.slot_ids += output.slot_ids + if output.intent_ids is not None: + self.intent_ids += output.intent_ids + + def save(self, path:str, original_dataset=None): + """ save all OutputData in the history + + Args: + path (str): save dir path + original_dataset(Iterable): original dataset + """ + # with open(f"{path}/intent.jsonl", "w") as f: + # for x in self.intent_ids: + # f.write(json.dumps(x) + "\n") + with open(f"{path}/outputs.jsonl", "w") as f: + if original_dataset is not None: + for i, s, d in zip(self.intent_ids, self.slot_ids, original_dataset): + f.write(json.dumps({"pred_intent": i, "pred_slot": s, "text": d["text"], "golden_intent":d["intent"], "golden_slot":d["slot"]}) + "\n") + else: + for i, s in zip(self.intent_ids, self.slot_ids): + f.write(json.dumps({"pred_intent": i, "pred_slot": s}) + "\n") + + +class HiddenData(): + """Interactive data structure for all model components + """ + def __init__(self, intent_hidden, slot_hidden): + """init hidden data structure + + Args: + intent_hidden (Any): sentence-level or intent hidden state + slot_hidden (Any): token-level or slot hidden state + """ + self.intent_hidden = intent_hidden + self.slot_hidden = slot_hidden + self.inputs = None + self.embedding = None + + def get_intent_hidden_state(self): + """get intent hidden state + + Returns: + Any: intent hidden state + """ + return self.intent_hidden + + def get_slot_hidden_state(self): + """get slot hidden state + + Returns: + Any: slot hidden state + """ + return self.slot_hidden + + def update_slot_hidden_state(self, hidden_state): + """update slot hidden state + + Args: + hidden_state (Any): slot hidden state to update + """ + self.slot_hidden = hidden_state + + def update_intent_hidden_state(self, hidden_state): + """update intent hidden state + + Args: + hidden_state (Any): intent hidden state to update + """ + self.intent_hidden = hidden_state + + def add_input(self, inputs: InputData or "HiddenData"): + """add last model component input information to next model component + + Args: + inputs (InputDataor or HiddenData): last model component input + """ + self.inputs = inputs + + def add_embedding(self, embedding): + self.embedding = embedding + + +class ClassifierOutputData(): + """Classifier output data structure of all classifier components + """ + def __init__(self, classifier_output): + self.classifier_output = classifier_output + self.output_embedding = None + +def remove_slot_ignore_index(inputs:InputData, outputs:OutputData, ignore_index=-100): + """ remove padding or extra token in input id and output id + + Args: + inputs (InputData): input data with input id + outputs (OutputData): output data with decoded output id + ignore_index (int, optional): ignore_index in input_ids. Defaults to -100. + + Returns: + InputData: input data removed padding or extra token + OutputData: output data removed padding or extra token + """ + for index, (inp_ss, out_ss) in enumerate(zip(inputs.slot, outputs.slot_ids)): + temp_inp = [] + temp_out = [] + for inp_s, out_s in zip(list(inp_ss), list(out_ss)): + if inp_s != ignore_index: + temp_inp.append(inp_s) + temp_out.append(out_s) + + inputs.slot[index] = temp_inp + outputs.slot_ids[index] = temp_out + return inputs, outputs + + +def pack_sequence(inputs:Tensor, seq_len:Tensor or List) -> Tensor: + """pack sequence data to packed data without padding. + + Args: + inputs (Tensor): list(Tensor) of packed sequence inputs + seq_len (Tensor or List): list(Tensor) of sequence length + + Returns: + Tensor: packed inputs + + Examples: + inputs = [[x, y, z, PAD, PAD], [x, y, PAD, PAD, PAD]] + + seq_len = [3,2] + + return -> [x, y, z, x, y] + """ + output = [] + for index, batch in enumerate(inputs): + output.append(batch[:seq_len[index]]) + return torch.cat(output, dim=0) + + +def unpack_sequence(inputs:Tensor, seq_lens:Tensor or List, padding_value=0) -> Tensor: + """unpack sequence data. + + Args: + inputs (Tensor): list(Tensor) of packed sequence inputs + seq_lens (Tensor or List): list(Tensor) of sequence length + padding_value (int, optional): padding value. Defaults to 0. + + Returns: + Tensor: unpacked inputs + + Examples: + inputs = [x, y, z, x, y] + + seq_len = [3,2] + + return -> [[x, y, z, PAD, PAD], [x, y, PAD, PAD, PAD]] + """ + last_idx = 0 + output = [] + for _, seq_len in enumerate(seq_lens): + output.append(inputs[last_idx:last_idx + seq_len]) + last_idx = last_idx + seq_len + return pad_sequence(output, batch_first=True, padding_value=padding_value) + + +def get_dict_with_key_prefix(input_dict: dict, prefix=""): + res = {} + for t in input_dict: + res[t + prefix] = input_dict[t] + return res + + +def download(url: str, fname: str): + """download file from url to fname + + Args: + url (str): remote server url path + fname (str): local path to save + """ + resp = requests.get(url, stream=True) + total = int(resp.headers.get('content-length', 0)) + with open(fname, 'wb') as file, tqdm( + desc=fname, + total=total, + unit='iB', + unit_scale=True, + unit_divisor=1024, + ) as bar: + for data in resp.iter_content(chunk_size=1024): + size = file.write(data) + bar.update(size) + + +def tar_gz_data(file_name:str): + """use "tar.gz" format to compress data + + Args: + file_name (str): file path to tar + """ + t = tarfile.open(f"{file_name}.tar.gz", "w:gz") + + for root, dir, files in os.walk(f"{file_name}"): + print(root, dir, files) + for file in files: + fullpath = os.path.join(root, file) + t.add(fullpath) + t.close() + + +def untar(fname:str, dirs:str): + """ uncompress "tar.gz" file + + Args: + fname (str): file path to untar + dirs (str): target dir path + """ + t = tarfile.open(fname) + t.extractall(path=dirs) + + +def unzip_file(zip_src:str, dst_dir:str): + """ uncompress "zip" file + + Args: + fname (str): file path to unzip + dirs (str): target dir path + """ + r = zipfile.is_zipfile(zip_src) + if r: + if not os.path.exists(dst_dir): + os.mkdir(dst_dir) + fz = zipfile.ZipFile(zip_src, 'r') + for file in fz.namelist(): + fz.extract(file, dst_dir) + else: + print('This is not zip') + + +def find_callable(target: str) -> Callable: + """ find callable function / class to instantiate + + Args: + target (str): class/module path + + Raises: + e: can not import module + + Returns: + Callable: return function / class + """ + target_module_path, target_callable_path = target.rsplit(".", 1) + target_callable_paths = [target_callable_path] + + target_module = None + while len(target_module_path): + try: + target_module = importlib.import_module(target_module_path) + break + except Exception as e: + raise e + target_callable = target_module + for attr in reversed(target_callable_paths): + target_callable = getattr(target_callable, attr) + + return target_callable + + +def instantiate(config, target="_model_target_", partial="_model_partial_"): + """ instantiate object by config. + + Modified from https://github.com/HIT-SCIR/ltp/blob/main/python/core/ltp_core/models/utils/instantiate.py. + + Args: + config (Any): configuration + target (str, optional): key to assign the class to be instantiated. Defaults to "_model_target_". + partial (str, optional): key to judge object whether should be instantiated partially. Defaults to "_model_partial_". + + Returns: + Any: instantiated object + """ + if isinstance(config, dict) and target in config: + target_path = config.get(target) + target_callable = find_callable(target_path) + + is_partial = config.get(partial, False) + target_args = { + key: instantiate(value) + for key, value in config.items() + if key not in [target, partial] + } + + if is_partial: + return functools.partial(target_callable, **target_args) + else: + return target_callable(**target_args) + elif isinstance(config, dict): + return {key: instantiate(value) for key, value in config.items()} + else: + return config + + +def load_yaml(file): + """ load data from yaml files. + + Args: + file (str): yaml file path. + + Returns: + Any: data + """ + with open(file, encoding="utf-8") as stream: + try: + return yaml.safe_load(stream) + except yaml.YAMLError as exc: + raise exc + +def from_configured(configure_name_or_file:str, model_class:Callable, config_prefix="./config/", **input_config): + """load module from pre-configured data + + Args: + configure_name_or_file (str): config path -> {config_prefix}/{configure_name_or_file}.yaml + model_class (Callable): module class + config_prefix (str, optional): configuration root path. Defaults to "./config/". + + Returns: + Any: instantiated object. + """ + if os.path.exists(configure_name_or_file): + configure_file=configure_name_or_file + else: + configure_file= os.path.join(config_prefix, configure_name_or_file+".yaml") + config = load_yaml(configure_file) + config.update(input_config) + return model_class(**config) + +def save_json(file_path, obj): + with open(file_path, 'w', encoding="utf8") as fw: + fw.write(json.dumps(obj)) + +def load_json(file_path): + with open(file_path, 'r', encoding="utf8") as fw: + res =json.load(fw) + return res + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') \ No newline at end of file diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000000000000000000000000000000000000..995c429a5628b1da9aeb6ad6519a4f7cc91d29ab --- /dev/null +++ b/config/README.md @@ -0,0 +1,348 @@ +# Configuation + +## 1. Introduction + +Configuration is divided into fine-grained reusable modules: + +- `base`: basic configuration +- `logger`: logger setting +- `model_manager`: loading and saving model parameters +- `accelerator`: whether to enable multi-GPU +- `dataset`: dataset management +- `evaluator`: evaluation and metrics setting. +- `tokenizer`: Tokenizer initiation and tokenizing setting. +- `optimizer`: Optimizer initiation setting. +- `scheduler`: scheduler initiation setting. +- `model`: model construction setting. + +From Sec. 2 to Sec. 11, we will describe the configuration in detail. Or you can see [Examples](examples/README.md) for Quick Start. + +NOTE: `_*_` config are reserved fields in OpenSLU. + +## Configuration Item Script +In OpenSLU configuration, we support simple calculation script for each configuration item. For example, we can get `dataset_name` by using `{dataset.dataset_name}`, and fill its value into python script `'LightChen2333/agif-slu-' + '*'`.(Without '', `{dataset.dataset_name}` value will be treated as a variable). + +NOTE: each item with `{}` will be treated as python script. +```yaml +tokenizer: + _from_pretrained_: "'LightChen2333/agif-slu-' + '{dataset.dataset_name}'" # Support simple calculation script + +``` + +## `base` Config +```yaml +# `start_time` will generated automatically when start any config script, needless to be assigned. +# start_time: xxxxxxxx +base: + name: "OpenSLU" # project/logger name + multi_intent: false # whether to enable multi-intent setting + train: True # enable train else enable zero-shot + test: True # enable test during train. + device: cuda # device for cuda/cpu + seed: 42 # random seed + best_key: EMA # save model by which metric[intent_acc/slot_f1/EMA] + tokenizer_name: word_tokenizer # tokenizer: word_tokenizer for no pretrained model, else use [AutoTokenizer] tokenizer name + add_special_tokens: false # whether add [CLS], [SEP] special tokens + epoch_num: 300 # train epoch num +# eval_step: 280 # if eval_by_epoch = false and eval_step > 0, will evaluate model by steps + eval_by_epoch: true # evaluate model by epoch + batch_size: 16 # batch size +``` +## `logger` Config +```yaml +logger: + # `wandb` is supported both in single- multi-GPU, + # `tensorboard` is only supported in multi-GPU, + # and `fitlog` is only supported in single-GPU + logger_type: wandb +``` +## `model_manager` Config +```yaml +model_manager: + # if load_dir != `null`, OpenSLU will try to load checkpoint to continue training, + # if load_dir == `null`, OpenSLU will restart training. + load_dir: null + # The dir path to save model and training state. + # if save_dir == `null` model will be saved to `save/{start_time}` + save_dir: save/stack + # save_mode can be selected in [save-by-step, save-by-eval] + # `save-by-step` means save model only by {save_step} steps without evaluation. + # `save-by-eval` means save model by best validation performance + save_mode: save-by-eval + # save_step: 100 # only enabled when save_mode == `save-by-step` + max_save_num: 1 # The number of best models will be saved. +``` +## `accelerator` Config +```yaml +accelerator: + use_accelerator: false # will enable `accelerator` if use_accelerator is `true` +``` +## `dataset` Config +```yaml +dataset: + # support load model from hugging-face. + # dataset_name can be selected in [atis, snips, mix-atis, mix-snips] + dataset_name: atis + # support assign any one of dataset path and other dataset split is the same as split in `dataset_name` + # train: atis # support load model from hugging-face or assigned local data path. + # validation: {root}/ATIS/dev.jsonl + # test: {root}/ATIS/test.jsonl +``` +## `evaluator` Config +```yaml +evaluator: + best_key: EMA # the metric to judge the best model + eval_by_epoch: true # Evaluate after an epoch if `true`. + # Evaluate after {eval_step} steps if eval_by_epoch == `false`. + # eval_step: 1800 + # metric is supported the metric as below: + # - intent_acc + # - slot_f1 + # - EMA + # - intent_f1 + # - macro_intent_f1 + # - micro_intent_f1 + # NOTE: [intent_f1, macro_intent_f1, micro_intent_f1] is only supported in multi-intent setting. intent_f1 and macro_intent_f1 is the same metric. + metric: + - intent_acc + - slot_f1 + - EMA +``` +## `tokenizer` Config +```yaml +tokenizer: + # Init tokenizer. Support `word_tokenizer` and other tokenizers in huggingface. + _tokenizer_name_: word_tokenizer + # if `_tokenizer_name_` is not assigned, you can load pretrained tokenizer from hugging-face. + # _from_pretrained_: LightChen2333/stack-propagation-slu-atis + _padding_side_: right # the padding side of tokenizer, support [left/ right] + # Align mode between text and slot, support [fast/ general], + # `general` is supported in most tokenizer, `fast` is supported only in small portion of tokenizers. + _align_mode_: fast + _to_lower_case_: true + add_special_tokens: false # other tokenizer args, you can add other args to tokenizer initialization except `_*_` format args + max_length: 512 + +``` +## `optimizer` Config +```yaml +optimizer: + _model_target_: torch.optim.Adam # Optimizer class/ function return Optimizer object + _model_partial_: true # partial load configuration. Here will add model.parameters() to complete all Optimizer parameters + lr: 0.001 # learning rate + weight_decay: 1e-6 # weight decay +``` +## `scheduler` Config +```yaml +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true # partial load configuration. Here will add optimizer, num_training_steps to complete all Optimizer parameters + name : "linear" + num_warmup_steps: 0 +``` +## `model` Config +```yaml +model: + # _from_pretrained_: LightChen2333/stack-propagation-slu-atis # load model from hugging-face and is not need to assigned any parameters below. + _model_target_: model.OpenSLUModel # the general model class, can automatically build the model through configuration. + + encoder: + _model_target_: model.encoder.AutoEncoder # auto-encoder to autoload provided encoder model + encoder_name: self-attention-lstm # support [lstm/ self-attention-lstm] and other pretrained models those hugging-face supported + + embedding: # word embedding layer +# load_embedding_name: glove.6B.300d.txt # support autoload glove embedding. + embedding_dim: 256 # embedding dim + dropout_rate: 0.5 # dropout ratio after embedding + + lstm: + layer_num: 1 # lstm configuration + bidirectional: true + output_dim: 256 # module should set output_dim for autoload input_dim in next module. You can also set input_dim manually. + dropout_rate: 0.5 + + attention: # self-attention configuration + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.5 + + return_with_input: true # add inputs information, like attention_mask, to decoder module. + return_sentence_level_hidden: false # if return sentence representation to decoder module + + decoder: + _model_target_: model.decoder.StackPropagationDecoder # decoder name + interaction: + _model_target_: model.decoder.interaction.StackInteraction # interaction module name + differentiable: false # interaction module config + + intent_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier # intent classifier module name + layer_num: 1 + bidirectional: false + hidden_dim: 64 + force_ratio: 0.9 # teacher-force ratio + embedding_dim: 8 # intent embedding dim + ignore_index: -100 # ignore index to compute loss and metric + dropout_rate: 0.5 + mode: "token-level-intent" # decode mode, support [token-level-intent, intent, slot] + use_multi: "{base.multi_intent}" + return_sentence_level: true # whether to return sentence level prediction as decoded input + + slot_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: 64 + embedding_dim: 32 + ignore_index: -100 + dropout_rate: 0.5 + mode: "slot" + use_multi: false + return_sentence_level: false +``` + +## Implementing a New Model + +### 1. Interaction Re-Implement +Here we take `DCA-Net` as an example: + +In most cases, you just need to rewrite `Interaction` module: + +```python +from common.utils import HiddenData +from model.decoder.interaction import BaseInteraction +class DCANetInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.T_block1 = I_S_Block(self.config["output_dim"], self.config["attention_dropout"], self.config["num_attention_heads"]) + ... + + def forward(self, encode_hidden: HiddenData, **kwargs): + ... +``` + +and then you should configure your module: +```yaml +base: + ... + +optimizer: + ... + +scheduler: + ... + +model: + _model_target_: model.OpenSLUModel + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + load_embedding_name: glove.6B.300d.txt + embedding_dim: 300 + dropout_rate: 0.5 + + lstm: + dropout_rate: 0.5 + output_dim: 128 + layer_num: 2 + bidirectional: true + output_dim: "{model.encoder.lstm.output_dim}" + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.DCANetDecoder + interaction: + _model_target_: model.decoder.interaction.DCANetInteraction + output_dim: "{model.encoder.output_dim}" + attention_dropout: 0.5 + num_attention_heads: 8 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + input_dim: "{model.decoder.output_dim.output_dim}" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + input_dim: "{model.decoder.output_dim.output_dim}" + ignore_index: -100 +``` + +Oops, you finish all model construction. You can run script as follows to train model: +```shell +python run.py -cp config/dca_net.yaml [-ds atis] +``` +### 2. Decoder Re-Implement +Sometimes, `interaction then classification` order can not meet your needs. Therefore, you should simply rewrite decoder for flexible interaction order: + +Here, we take `stack-propagation` as an example: +1. We should rewrite interaction module for `stack-propagation` +```python +from common.utils import ClassifierOutputData, HiddenData +from model.decoder.interaction.base_interaction import BaseInteraction +class StackInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + ... + + def forward(self, intent_output: ClassifierOutputData, encode_hidden: HiddenData): + ... +``` +2. We should rewrite `StackPropagationDecoder` for stack-propagation interaction order: +```python +from common.utils import HiddenData, OutputData +class StackPropagationDecoder(BaseDecoder): + + def forward(self, hidden: HiddenData): + pred_intent = self.intent_classifier(hidden) + hidden = self.interaction(pred_intent, hidden) + pred_slot = self.slot_classifier(hidden) + return OutputData(pred_intent, pred_slot) +``` + +3. Then we can easily combine general model by `config/stack-propagation.yaml` configuration file: +```yaml +base: + ... + +... + +model: + _model_target_: model.OpenSLUModel + + encoder: + ... + + decoder: + _model_target_: model.decoder.StackPropagationDecoder + interaction: + _model_target_: model.decoder.interaction.StackInteraction + differentiable: false + + intent_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + ... # parameters needed __init__(*) + mode: "token-level-intent" + use_multi: false + return_sentence_level: true + + slot_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + ... # parameters needed __init__(*) + mode: "slot" + use_multi: false + return_sentence_level: false +``` +4. You can run script as follows to train model: +```shell +python run.py -cp config/stack-propagation.yaml +``` + + + diff --git a/config/app.yaml b/config/app.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9ecb57aa556caa885125c4b11e8aac67ceaa6b4 --- /dev/null +++ b/config/app.yaml @@ -0,0 +1,6 @@ +host: 127.0.0.1 +port: 7860 + +is_push_to_public: false +save-path: save/stack/outputs.jsonl +page-size: 2 \ No newline at end of file diff --git a/config/decoder/interaction/stack-propagation.yaml b/config/decoder/interaction/stack-propagation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e488a13bab125a850bc6220f1780d4a4c569724 --- /dev/null +++ b/config/decoder/interaction/stack-propagation.yaml @@ -0,0 +1 @@ +differentiable: false \ No newline at end of file diff --git a/config/examples/README.md b/config/examples/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aec8ce8006690c161333a3100dde4c1b7dab2cb5 --- /dev/null +++ b/config/examples/README.md @@ -0,0 +1,38 @@ +# Examples + +Here we introduce some usage of our famework by configuration. + +## Reload to train + +Firstly, you can run this script to train a `joint-bert` model: +```shell +python run.py -cp config/examples/normal.yaml +``` + +and you can use `kill` or `Ctrl+C` to kill the training process. + +Then, to reload model and continue training, you can run `reload_to_train.yaml` to reload checkpoint and training state. +```shell +python run.py -cp config/examples/reload_to_train.yaml +``` + +The main difference in `reload_to_train.yaml` is the `model_manager` configuration item: +```yaml +... +model_manager: + load_train_state: True # set to True + load_dir: save/joint_bert # not null + ... +... +``` + +## Load from Pre-finetuned model. +We upload all models to [LightChen2333](https://huggingface.co/LightChen2333). You can load those model by simple configuration. +In `from_pretrained.yaml` and `from_pretrained_multi.yaml`, we show two example scripts to load from hugging face in single- and multi-intent, respectively. The key configuration items are as below: +```yaml +tokenizer: + _from_pretrained_: "'LightChen2333/agif-slu-' + '{dataset.dataset_name}'" # Support simple calculation script + +model: + _from_pretrained_: "'LightChen2333/agif-slu-' + '{dataset.dataset_name}'" +``` diff --git a/config/examples/from_pretrained.yaml b/config/examples/from_pretrained.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbf7aaa3f6daf906460a96167637d32b95cb649d --- /dev/null +++ b/config/examples/from_pretrained.yaml @@ -0,0 +1,53 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + train: false + test: true + device: cpu + seed: 42 + epoch_num: 300 + batch_size: 16 + +logger: + logger_type: local # wandb is supported both in single- multi-GPU, tensorboard is only supported in multi-GPU, and fitlog is only supported in single-GPU + +model_manager: + load_dir: null + save_dir: save/joint_bert + save_mode: save-by-eval # save-by-step + # save_step: 100 + max_save_num: 1 + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +tokenizer: + _from_pretrained_: "'LightChen2333/joint-bert-slu-' + '{dataset.dataset_name}'" + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _from_pretrained_: "'LightChen2333/joint-bert-slu-' + '{dataset.dataset_name}'" \ No newline at end of file diff --git a/config/examples/from_pretrained_multi.yaml b/config/examples/from_pretrained_multi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbdb0f1a791193f60f3b60fc92696d2a52ee77dd --- /dev/null +++ b/config/examples/from_pretrained_multi.yaml @@ -0,0 +1,55 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + multi_intent: true + train: false + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + + +logger: + logger_type: wandb # wandb is supported both in single- multi-GPU, tensorboard is only supported in multi-GPU, and fitlog is only supported in single-GPU + +model_manager: + load_dir: null + save_dir: save/joint_bert + save_mode: save-by-eval # save-by-step + # save_step: 100 + max_save_num: 1 + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +tokenizer: + _from_pretrained_: "'LightChen2333/agif-slu-' + '{dataset.dataset_name}'" + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _from_pretrained_: "'LightChen2333/agif-slu-' + '{dataset.dataset_name}'" \ No newline at end of file diff --git a/config/examples/normal.yaml b/config/examples/normal.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27aa91d3683e94d7bb1b9f44787955b6860be185 --- /dev/null +++ b/config/examples/normal.yaml @@ -0,0 +1,70 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLU-test" + train: True + test: True + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 128 + +model_manager: + load_dir: null + save_dir: save/joint_bert + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: bert-base-uncased + _padding_side_: right + _align_mode_: general + add_special_tokens: true + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 4e-6 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: bert-base-uncased + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/examples/reload_to_train.yaml b/config/examples/reload_to_train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52b11f1c9178c8d69927a8557fd053fd81fba617 --- /dev/null +++ b/config/examples/reload_to_train.yaml @@ -0,0 +1,71 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLU-test" + train: True + test: True + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 128 + +model_manager: + load_train_state: True + load_dir: save/joint_bert + save_dir: save/joint_bert + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: bert-base-uncased + _padding_side_: right + _align_mode_: general + add_special_tokens: true + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 4e-6 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: bert-base-uncased + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/bi-model.yaml b/config/reproduction/atis/bi-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02e0a5cd430695a43c7264b41b3eb3a8cb1d1ecc --- /dev/null +++ b/config/reproduction/atis/bi-model.yaml @@ -0,0 +1,106 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/bi-model-atis + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.BiEncoder + intent_encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.4 + + lstm: + dropout_rate: 0.5 + output_dim: 256 + layer_num: 2 + bidirectional: true + + return_with_input: true + return_sentence_level_hidden: false + + slot_encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.4 + + lstm: + dropout_rate: 0.5 + output_dim: 256 + layer_num: 2 + bidirectional: true + + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.BaseDecoder +# teacher_forcing: true + interaction: + _model_target_: model.decoder.interaction.BiModelInteraction + output_dim: 256 + dropout_rate: 0.4 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/dca-net.yaml b/config/reproduction/atis/dca-net.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad58227493734467ba3a34bec44b49b9d0d362ab --- /dev/null +++ b/config/reproduction/atis/dca-net.yaml @@ -0,0 +1,88 @@ +device: "Tesla P100-PCIE-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/dca-net-atis + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + load_embedding_name: glove.6B.300d.txt + embedding_dim: 300 + dropout_rate: 0.5 + + lstm: + dropout_rate: 0.5 + output_dim: 128 + layer_num: 2 + bidirectional: true + output_dim: "{model.encoder.lstm.output_dim}" + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.DCANetDecoder + interaction: + _model_target_: model.decoder.interaction.DCANetInteraction + output_dim: "{model.encoder.output_dim}" + attention_dropout: 0.5 + num_attention_heads: 8 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + input_dim: "{model.encoder.output_dim}" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + input_dim: "{model.encoder.output_dim}" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/deberta.yaml b/config/reproduction/atis/deberta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9f6532787ba5e1e4d44e0a7bd2dbe32b0b73a60 --- /dev/null +++ b/config/reproduction/atis/deberta.yaml @@ -0,0 +1,67 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + +model_manager: + load_dir: null + save_dir: save/deberta-atis + +dataset: + dataset_name: atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +tokenizer: + _tokenizer_name_: microsoft/deberta-v3-base + _padding_side_: right + add_special_tokens: true + max_length: 512 + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 2e-5 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: microsoft/deberta-v3-base + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/electra.yaml b/config/reproduction/atis/electra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0273298d5c38219bb425d36d13887ad134c16b62 --- /dev/null +++ b/config/reproduction/atis/electra.yaml @@ -0,0 +1,67 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: True + test: True + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + +model_manager: + load_dir: null + save_dir: save/electra-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: google/electra-small-discriminator + _padding_side_: right + add_special_tokens: true + max_length: 512 + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 2e-5 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: google/electra-small-discriminator + output_dim: 256 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/joint-bert.yaml b/config/reproduction/atis/joint-bert.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87f6b2b5783d68f38e82833395ec1d1fbca54765 --- /dev/null +++ b/config/reproduction/atis/joint-bert.yaml @@ -0,0 +1,70 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: True + test: True + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 128 + +model_manager: + load_dir: null + save_dir: save/joint-bert-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: bert-base-uncased + _padding_side_: right + _align_mode_: general + add_special_tokens: true + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 4e-6 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: bert-base-uncased + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/roberta.yaml b/config/reproduction/atis/roberta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f68f97ad30263daa0d2ac046cef8fe9237c203a9 --- /dev/null +++ b/config/reproduction/atis/roberta.yaml @@ -0,0 +1,70 @@ +device: "Tesla V100-SXM2-16GB" #Useless info + +base: + name: "OpenSLUv1" + train: True + test: True + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + +model_manager: + load_dir: null + save_dir: save/roberta-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: roberta-base + _padding_side_: right + add_special_tokens: true + max_length: 512 + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 2e-5 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: roberta-base + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/slot-gated.yaml b/config/reproduction/atis/slot-gated.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd58f74119fd9723bbb10e77fd6a070ba6a70552 --- /dev/null +++ b/config/reproduction/atis/slot-gated.yaml @@ -0,0 +1,87 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/slot-gated-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.4 + + lstm: + dropout_rate: 0.5 + output_dim: 256 + layer_num: 2 + bidirectional: true + + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.BaseDecoder + + interaction: + _model_target_: model.decoder.interaction.SlotGatedInteraction + remove_slot_attn: false + output_dim: 256 + dropout_rate: 0.4 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/atis/stack-propagation.yaml b/config/reproduction/atis/stack-propagation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5041e664215dbbcca1f83fade44aba453dc6333 --- /dev/null +++ b/config/reproduction/atis/stack-propagation.yaml @@ -0,0 +1,109 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/stack-propagation-atis + save_mode: save-by-eval # save-by-step + # save_step: 100 + max_save_num: 1 + +accelerator: + use_accelerator: false + +dataset: + dataset_name: atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + _to_lower_case_: true + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.55 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.5 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.6 + + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.StackPropagationDecoder + interaction: + _model_target_: model.decoder.interaction.StackInteraction + differentiable: false + + intent_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: 64 + embedding_dim: 8 + ignore_index: -100 + dropout_rate: 0.5 + mode: "token-level-intent" + use_multi: false + return_sentence_level: true + + slot_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: 64 + embedding_dim: 32 + ignore_index: -100 + dropout_rate: 0.55 + mode: "slot" + use_multi: false + return_sentence_level: false \ No newline at end of file diff --git a/config/reproduction/mix-atis/agif.yaml b/config/reproduction/mix-atis/agif.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f13e65dc1bb42319aa7120dffbe36b0e9951257 --- /dev/null +++ b/config/reproduction/mix-atis/agif.yaml @@ -0,0 +1,133 @@ +device: "NVIDIA GeForce RTX 3080" + +base: + name: "OpenSLUv1" + multi_intent: true + train: true + test: true + device: cuda + seed: 42 + epoch_num: 100 + batch_size: 32 + ignore_index: -100 + +model_manager: + load_dir: null + save_dir: save/agif-mix-atis + +accelerator: + use_accelerator: false + +dataset: + dataset_name: mix-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - intent_f1 + - slot_f1 + - EMA + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 128 + dropout_rate: 0.4 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + + unflat_attention: + dropout_rate: 0.4 + output_dim: "{model.encoder.lstm.output_dim} + {model.encoder.attention.output_dim}" + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.AGIFDecoder +# teacher_forcing: true + interaction: + _model_target_: model.decoder.interaction.AGIFInteraction + intent_embedding_dim: 128 + input_dim: "{model.encoder.output_dim}" + hidden_dim: 128 + output_dim: "{model.decoder.interaction.intent_embedding_dim}" + dropout_rate: 0.4 + alpha: 0.2 + num_heads: 4 + num_layers: 2 + row_normalized: true + + intent_classifier: + _model_target_: model.decoder.classifier.MLPClassifier + mode: "intent" + mlp: + - _model_target_: torch.nn.Linear + in_features: "{model.encoder.output_dim}" + out_features: 256 + - _model_target_: torch.nn.LeakyReLU + negative_slope: 0.2 + - _model_target_: torch.nn.Linear + in_features: 256 + out_features: "{base.intent_label_num}" + dropout_rate: 0.4 + loss_fn: + _model_target_: torch.nn.BCEWithLogitsLoss + use_multi: "{base.multi_intent}" + multi_threshold: 0.5 + return_sentence_level: true + ignore_index: -100 + weight: 0.3 + + slot_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + mode: "slot" + input_dim: "{model.encoder.output_dim}" + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: "{model.decoder.interaction.intent_embedding_dim}" + embedding_dim: 128 +# loss_fn: +# _model_target_: torch.nn.NLLLoss + ignore_index: -100 + dropout_rate: 0.4 + use_multi: false + multi_threshold: 0.5 + return_sentence_level: false + weight: 0.7 \ No newline at end of file diff --git a/config/reproduction/mix-atis/gl-gin.yaml b/config/reproduction/mix-atis/gl-gin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a52a049c98e9dcdb068815d18ba6e3441dec343 --- /dev/null +++ b/config/reproduction/mix-atis/gl-gin.yaml @@ -0,0 +1,128 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + multi_intent: true + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + ignore_index: -100 + +model_manager: + load_dir: null + save_dir: save/gl-gin-mix-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - intent_f1 + - slot_f1 + - EMA + +dataset: + dataset_name: mix-atis + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 128 + dropout_rate: 0.4 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + output_dim: "{model.encoder.lstm.output_dim} + {model.encoder.attention.output_dim}" + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.GLGINDecoder + dropout_rate: 0.4 + interaction: + _model_target_: model.decoder.interaction.GLGINInteraction + intent_embedding_dim: 64 + input_dim: "{model.encoder.output_dim}" + hidden_dim: 256 + output_dim: "{model.decoder.interaction.intent_embedding_dim}" + dropout_rate: 0.4 + alpha: 0.2 + num_heads: 8 + num_layers: 2 + row_normalized: true + slot_graph_window: 1 + intent_label_num: "{base.intent_label_num}" + + intent_classifier: + _model_target_: model.decoder.classifier.MLPClassifier + mode: "token-level-intent" + mlp: + - _model_target_: torch.nn.Linear + in_features: "{model.encoder.output_dim}" + out_features: 256 + - _model_target_: torch.nn.LeakyReLU + negative_slope: 0.2 + - _model_target_: torch.nn.Linear + in_features: 256 + out_features: "{base.intent_label_num}" + loss_fn: + _model_target_: torch.nn.BCEWithLogitsLoss + dropout_rate: 0.4 + use_multi: "{base.multi_intent}" + multi_threshold: 0.5 + return_sentence_level: true + ignore_index: "{base.ignore_index}" + + slot_classifier: + _model_target_: model.decoder.classifier.MLPClassifier + mode: "slot" + mlp: + - _model_target_: torch.nn.Linear + in_features: "{model.decoder.interaction.output_dim}" + out_features: "{model.decoder.interaction.output_dim}" + - _model_target_: torch.nn.LeakyReLU + negative_slope: 0.2 + - _model_target_: torch.nn.Linear + in_features: "{model.decoder.interaction.output_dim}" + out_features: "{base.slot_label_num}" + ignore_index: "{base.ignore_index}" + dropout_rate: 0.4 + use_multi: false + multi_threshold: 0.5 + return_sentence_level: false \ No newline at end of file diff --git a/config/reproduction/mix-atis/vanilla.yaml b/config/reproduction/mix-atis/vanilla.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36ee8ed2b2b34133e0f4a6273617fade021ef03b --- /dev/null +++ b/config/reproduction/mix-atis/vanilla.yaml @@ -0,0 +1,95 @@ +base: + name: "OpenSLUv1" + multi_intent: true + train: true + test: true + device: cuda + seed: 42 + epoch_num: 100 + batch_size: 16 + ignore_index: -100 + +model_manager: + load_dir: null + save_dir: save/vanilla-mix-atis + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - intent_f1 + - slot_f1 + - EMA + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 128 + dropout_rate: 0.4 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + output_dim: "{model.encoder.lstm.output_dim} + {model.encoder.attention.output_dim}" + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.BaseDecoder + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + input_dim: "{model.encoder.output_dim}" + loss_fn: + _model_target_: torch.nn.BCEWithLogitsLoss + use_multi: "{base.multi_intent}" + multi_threshold: 0.5 + return_sentence_level: true + ignore_index: "{base.ignore_index}" + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + input_dim: "{model.encoder.output_dim}" + use_multi: false + multi_threshold: 0.5 + ignore_index: "{base.ignore_index}" + return_sentence_level: false \ No newline at end of file diff --git a/config/reproduction/mix-snips/agif.yaml b/config/reproduction/mix-snips/agif.yaml new file mode 100644 index 0000000000000000000000000000000000000000..877ad78ec112b088e192b6942fb940bb2f9af988 --- /dev/null +++ b/config/reproduction/mix-snips/agif.yaml @@ -0,0 +1,131 @@ +device: "Tesla P100-PCIE-16GB" + +base: + name: "OpenSLUv1" + multi_intent: true + train: true + test: true + device: cuda + seed: 42 + epoch_num: 50 + batch_size: 64 + ignore_index: -100 + +model_manager: + load_dir: null + save_dir: save/agif-mix-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - intent_f1 + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: mix-snips + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 128 + dropout_rate: 0.4 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + + unflat_attention: + dropout_rate: 0.4 + output_dim: "{model.encoder.lstm.output_dim} + {model.encoder.attention.output_dim}" + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.AGIFDecoder +# teacher_forcing: true + interaction: + _model_target_: model.decoder.interaction.AGIFInteraction + intent_embedding_dim: 128 + input_dim: "{model.encoder.output_dim}" + hidden_dim: 128 + output_dim: "{model.decoder.interaction.intent_embedding_dim}" + dropout_rate: 0.4 + alpha: 0.2 + num_heads: 4 + num_layers: 2 + row_normalized: true + + intent_classifier: + _model_target_: model.decoder.classifier.MLPClassifier + mode: "intent" + mlp: + - _model_target_: torch.nn.Linear + in_features: "{model.encoder.output_dim}" + out_features: 256 + - _model_target_: torch.nn.LeakyReLU + negative_slope: 0.2 + - _model_target_: torch.nn.Linear + in_features: 256 + out_features: "{base.intent_label_num}" + dropout_rate: 0.4 + loss_fn: + _model_target_: torch.nn.BCEWithLogitsLoss + use_multi: "{base.multi_intent}" + multi_threshold: 0.5 + return_sentence_level: true + ignore_index: -100 + weight: 0.3 + + slot_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + mode: "slot" + input_dim: "{model.encoder.output_dim}" + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: "{model.decoder.interaction.intent_embedding_dim}" + embedding_dim: 128 + ignore_index: -100 + dropout_rate: 0.4 + use_multi: false + multi_threshold: 0.5 + return_sentence_level: false + weight: 0.7 \ No newline at end of file diff --git a/config/reproduction/mix-snips/gl-gin.yaml b/config/reproduction/mix-snips/gl-gin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f90eec35696f1ba54ef2eb44705c5393db65e1f --- /dev/null +++ b/config/reproduction/mix-snips/gl-gin.yaml @@ -0,0 +1,131 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + multi_intent: true + train: true + test: true + device: cuda + seed: 42 + epoch_num: 50 + batch_size: 32 + ignore_index: -100 + + +model_manager: + load_dir: null + save_dir: save/gl-gin-mix-snips + +evaluator: + best_key: EMA + eval_by_epoch: false + eval_step: 1800 + metric: + - intent_acc + - intent_f1 + - slot_f1 + - EMA + +dataset: + dataset_name: mix-snips + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 128 + dropout_rate: 0.4 + + lstm: + layer_num: 2 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + output_dim: "{model.encoder.lstm.output_dim} + {model.encoder.attention.output_dim}" + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.GLGINDecoder + dropout_rate: 0.4 + interaction: + _model_target_: model.decoder.interaction.GLGINInteraction + intent_embedding_dim: 256 + input_dim: "{model.encoder.output_dim}" + hidden_dim: 256 + output_dim: "{model.decoder.interaction.intent_embedding_dim}" + dropout_rate: 0.4 + alpha: 0.2 + num_heads: 4 + num_layers: 2 + row_normalized: true + slot_graph_window: 1 + intent_label_num: "{base.intent_label_num}" + + intent_classifier: + _model_target_: model.decoder.classifier.MLPClassifier + mode: "token-level-intent" + mlp: + - _model_target_: torch.nn.Linear + in_features: "{model.encoder.output_dim}" + out_features: 256 + - _model_target_: torch.nn.LeakyReLU + negative_slope: 0.2 + - _model_target_: torch.nn.Linear + in_features: 256 + out_features: "{base.intent_label_num}" + loss_fn: + _model_target_: torch.nn.BCEWithLogitsLoss + dropout_rate: 0.4 + use_multi: "{base.multi_intent}" + multi_threshold: 0.5 + return_sentence_level: true + ignore_index: "{base.ignore_index}" + weight: 0.2 + + slot_classifier: + _model_target_: model.decoder.classifier.MLPClassifier + mode: "slot" + mlp: + - _model_target_: torch.nn.Linear + in_features: "{model.decoder.interaction.output_dim}" + out_features: "{model.decoder.interaction.output_dim}" + - _model_target_: torch.nn.LeakyReLU + negative_slope: 0.2 + - _model_target_: torch.nn.Linear + in_features: "{model.decoder.interaction.output_dim}" + out_features: "{base.slot_label_num}" + ignore_index: "{base.ignore_index}" + dropout_rate: 0.4 + use_multi: false + multi_threshold: 0.5 + weight: 0.8 + return_sentence_level: false \ No newline at end of file diff --git a/config/reproduction/mix-snips/vanilla.yaml b/config/reproduction/mix-snips/vanilla.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a7e8738e6c34b63ed8887796926648a1a45cece --- /dev/null +++ b/config/reproduction/mix-snips/vanilla.yaml @@ -0,0 +1,95 @@ +base: + name: "OpenSLUv1" + multi_intent: true + train: true + test: true + device: cuda + seed: 42 + epoch_num: 100 + batch_size: 16 + ignore_index: -100 + +model_manager: + load_dir: null + save_dir: save/vanilla-mix-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - intent_f1 + - slot_f1 + - EMA + +dataset: + dataset_name: atis + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 128 + dropout_rate: 0.4 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + output_dim: "{model.encoder.lstm.output_dim} + {model.encoder.attention.output_dim}" + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.BaseDecoder + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + input_dim: "{model.encoder.output_dim}" + loss_fn: + _model_target_: torch.nn.BCEWithLogitsLoss + use_multi: "{base.multi_intent}" + multi_threshold: 0.5 + return_sentence_level: true + ignore_index: "{base.ignore_index}" + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + input_dim: "{model.encoder.output_dim}" + use_multi: false + multi_threshold: 0.5 + ignore_index: "{base.ignore_index}" + return_sentence_level: false \ No newline at end of file diff --git a/config/reproduction/snips/bi-model.yaml b/config/reproduction/snips/bi-model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eb192aa2063904691d9539d73833a4a005581bf --- /dev/null +++ b/config/reproduction/snips/bi-model.yaml @@ -0,0 +1,104 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/bi-model-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.BiEncoder + intent_encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.5 + + lstm: + dropout_rate: 0.5 + output_dim: 256 + layer_num: 2 + bidirectional: true + + return_with_input: true + return_sentence_level_hidden: false + + slot_encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.5 + + lstm: + dropout_rate: 0.5 + output_dim: 256 + layer_num: 2 + bidirectional: true + + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.BaseDecoder + interaction: + _model_target_: model.decoder.interaction.BiModelInteraction + output_dim: 256 + dropout_rate: 0.5 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/dca_net.yaml b/config/reproduction/snips/dca_net.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d42d525ba6bf74e2d457687129a3c518b814843c --- /dev/null +++ b/config/reproduction/snips/dca_net.yaml @@ -0,0 +1,88 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/dca-net-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + load_embedding_name: glove.6B.300d.txt + embedding_dim: 300 + dropout_rate: 0.4 + + lstm: + dropout_rate: 0.4 + output_dim: 128 + layer_num: 2 + bidirectional: true + output_dim: "{model.encoder.lstm.output_dim}" + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.DCANetDecoder + interaction: + _model_target_: model.decoder.interaction.DCANetInteraction + output_dim: "{model.encoder.output_dim}" + attention_dropout: 0.4 + num_attention_heads: 8 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + input_dim: "{model.encoder.output_dim}" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + input_dim: "{model.encoder.output_dim}" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/deberta.yaml b/config/reproduction/snips/deberta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fd23a2b41754841352b315f205cde4c8588b342 --- /dev/null +++ b/config/reproduction/snips/deberta.yaml @@ -0,0 +1,70 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + +model_manager: + load_dir: null + save_dir: save/deberta-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: microsoft/deberta-v3-base + _padding_side_: right + add_special_tokens: true + max_length: 512 + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 2e-5 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: microsoft/deberta-v3-base + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/electra.yaml b/config/reproduction/snips/electra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c923723dfc47c3bfb4dadaf0a7735dd8723b321 --- /dev/null +++ b/config/reproduction/snips/electra.yaml @@ -0,0 +1,69 @@ +device: "Tesla V100-SXM2-16GB" +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + +model_manager: + load_dir: null + save_dir: save/electra-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: google/electra-small-discriminator + _padding_side_: right + add_special_tokens: true + max_length: 512 + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 2e-5 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: google/electra-small-discriminator + output_dim: 256 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/joint-bert.yaml b/config/reproduction/snips/joint-bert.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb0fbe2698186952e7c78fbde13620d1b7c0391e --- /dev/null +++ b/config/reproduction/snips/joint-bert.yaml @@ -0,0 +1,75 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 128 + +model_manager: + load_dir: null + save_dir: save/joint-bert-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +metric: + - intent_acc + - slot_f1 + - EMA + +tokenizer: + _tokenizer_name_: bert-base-uncased + _padding_side_: right + _align_mode_: general + add_special_tokens: true + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 4e-6 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: bert-base-uncased + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/roberta.yaml b/config/reproduction/snips/roberta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..965ec41ccffb10c7f8a01289aa1486ba128361e5 --- /dev/null +++ b/config/reproduction/snips/roberta.yaml @@ -0,0 +1,70 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 32 + +model_manager: + load_dir: null + save_dir: save/roberta-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: roberta-base + _padding_side_: right + add_special_tokens: true + max_length: 512 + +optimizer: + _model_target_: torch.optim.AdamW + _model_partial_: true + lr: 2e-5 + weight_decay: 1e-8 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.open_slu_model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: roberta-base + output_dim: 768 + return_with_input: true + return_sentence_level_hidden: true + + decoder: + _model_target_: model.decoder.base_decoder.BaseDecoder + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/slot-gated.yaml b/config/reproduction/snips/slot-gated.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52f9a06479821c42d5bf8fd2a2e144064da9dd9e --- /dev/null +++ b/config/reproduction/snips/slot-gated.yaml @@ -0,0 +1,87 @@ +device: "NVIDIA GeForce RTX 2080 Ti" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/slot-gated-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + ignore_index: -100 + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.4 + + lstm: + dropout_rate: 0.5 + output_dim: 256 + layer_num: 2 + bidirectional: true + + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.BaseDecoder + + interaction: + _model_target_: model.decoder.interaction.SlotGatedInteraction + remove_slot_attn: false + output_dim: 256 + dropout_rate: 0.4 + + intent_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "intent" + ignore_index: -100 + + slot_classifier: + _model_target_: model.decoder.classifier.LinearClassifier + mode: "slot" + ignore_index: -100 \ No newline at end of file diff --git a/config/reproduction/snips/stack-propagation.yaml b/config/reproduction/snips/stack-propagation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0039d929b242724b07121368c65a7b18cf2a7e54 --- /dev/null +++ b/config/reproduction/snips/stack-propagation.yaml @@ -0,0 +1,105 @@ +device: "Tesla V100-SXM2-16GB" + +base: + name: "OpenSLUv1" + train: true + test: true + device: cuda + seed: 42 + epoch_num: 300 + batch_size: 16 + +model_manager: + load_dir: null + save_dir: save/stack-propagation-snips + +evaluator: + best_key: EMA + eval_by_epoch: true + # eval_step: 1800 + metric: + - intent_acc + - slot_f1 + - EMA + +accelerator: + use_accelerator: false + +dataset: + dataset_name: snips + +tokenizer: + _tokenizer_name_: word_tokenizer + _padding_side_: right + _align_mode_: fast + add_special_tokens: false + max_length: 512 + +optimizer: + _model_target_: torch.optim.Adam + _model_partial_: true + lr: 0.001 + weight_decay: 1e-6 + +scheduler: + _model_target_: transformers.get_scheduler + _model_partial_: true + name : "linear" + num_warmup_steps: 0 + +model: + _model_target_: model.OpenSLUModel + + encoder: + _model_target_: model.encoder.AutoEncoder + encoder_name: self-attention-lstm + + embedding: + embedding_dim: 256 + dropout_rate: 0.4 + + lstm: + layer_num: 1 + bidirectional: true + output_dim: 256 + dropout_rate: 0.4 + + attention: + hidden_dim: 1024 + output_dim: 128 + dropout_rate: 0.4 + + return_with_input: true + return_sentence_level_hidden: false + + decoder: + _model_target_: model.decoder.StackPropagationDecoder + interaction: + _model_target_: model.decoder.interaction.StackInteraction + differentiable: false + + intent_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: 64 + embedding_dim: 8 + ignore_index: -100 + dropout_rate: 0.4 + mode: "token-level-intent" + use_multi: false + return_sentence_level: true + + slot_classifier: + _model_target_: model.decoder.classifier.AutoregressiveLSTMClassifier + layer_num: 1 + bidirectional: false + force_ratio: 0.9 + hidden_dim: 64 + embedding_dim: 32 + ignore_index: -100 + dropout_rate: 0.4 + mode: "slot" + use_multi: false + return_sentence_level: false \ No newline at end of file diff --git a/config/visual.yaml b/config/visual.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eb80b81b14be9602ef90c81d2af0f83e56e45f5 --- /dev/null +++ b/config/visual.yaml @@ -0,0 +1,6 @@ +host: 127.0.0.1 +port: 7861 + +is_push_to_public: true +output_path: save/stack/outputs.jsonl +page-size: 2 \ No newline at end of file diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..33d939509e11296578019006a5d2e1ea07bf1c1c --- /dev/null +++ b/model/__init__.py @@ -0,0 +1,3 @@ +from model.open_slu_model import OpenSLUModel + +__all__ = ["OpenSLUModel"] \ No newline at end of file diff --git a/model/decoder/__init__.py b/model/decoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..06a2ee86998009c2ce9105c5ecbab26b0fbb8425 --- /dev/null +++ b/model/decoder/__init__.py @@ -0,0 +1,5 @@ +from model.decoder.agif_decoder import AGIFDecoder +from model.decoder.base_decoder import StackPropagationDecoder, BaseDecoder, DCANetDecoder +from model.decoder.gl_gin_decoder import GLGINDecoder + +__all__ = ["StackPropagationDecoder", "BaseDecoder", "DCANetDecoder", "AGIFDecoder", "GLGINDecoder"] diff --git a/model/decoder/agif_decoder.py b/model/decoder/agif_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..a99ad55ae3bda76cd858f8567d48dcd5f41d9946 --- /dev/null +++ b/model/decoder/agif_decoder.py @@ -0,0 +1,16 @@ +from common.utils import HiddenData, OutputData +from model.decoder.base_decoder import BaseDecoder + + +class AGIFDecoder(BaseDecoder): + def forward(self, hidden: HiddenData, **kwargs): + # hidden = self.interaction(hidden) + pred_intent = self.intent_classifier(hidden) + intent_index = self.intent_classifier.decode(OutputData(pred_intent, None), + return_list=False, + return_sentence_level=True) + interact_args = {"intent_index": intent_index, + "batch_size": pred_intent.classifier_output.shape[0], + "intent_label_num": self.intent_classifier.config["intent_label_num"]} + pred_slot = self.slot_classifier(hidden, internal_interaction=self.interaction, **interact_args) + return OutputData(pred_intent, pred_slot) diff --git a/model/decoder/base_decoder.py b/model/decoder/base_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..2c8ae3baffea61879c8ca912134cb9ac02c4a82f --- /dev/null +++ b/model/decoder/base_decoder.py @@ -0,0 +1,107 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-01-31 18:22:36 +Description: + +''' +from torch import nn + +from common.utils import HiddenData, OutputData, InputData + + +class BaseDecoder(nn.Module): + """Base class for all decoder module. + + Notice: t is often only necessary to change this module and its sub-modules + """ + def __init__(self, intent_classifier=None, slot_classifier=None, interaction=None): + super().__init__() + self.intent_classifier = intent_classifier + self.slot_classifier = slot_classifier + self.interaction = interaction + + def forward(self, hidden: HiddenData): + """forward + + Args: + hidden (HiddenData): encoded data + + Returns: + OutputData: prediction logits + """ + if self.interaction is not None: + hidden = self.interaction(hidden) + intent = None + slot = None + if self.intent_classifier is not None: + intent = self.intent_classifier(hidden) + if self.slot_classifier is not None: + slot = self.slot_classifier(hidden) + return OutputData(intent, slot) + + def decode(self, output: OutputData, target: InputData = None): + """decode output logits + + Args: + output (OutputData): output logits data + target (InputData, optional): input data with attention mask. Defaults to None. + + Returns: + List: decoded sequence ids + """ + intent, slot = None, None + if self.intent_classifier is not None: + intent = self.intent_classifier.decode(output, target) + if self.slot_classifier is not None: + slot = self.slot_classifier.decode(output, target) + return OutputData(intent, slot) + + def compute_loss(self, pred: OutputData, target: InputData, compute_intent_loss=True, compute_slot_loss=True): + """compute loss. + Notice: can set intent and slot loss weight by adding 'weight' config item in corresponding classifier configuration. + + Args: + pred (OutputData): output logits data + target (InputData): input golden data + compute_intent_loss (bool, optional): whether to compute intent loss. Defaults to True. + compute_slot_loss (bool, optional): whether to compute intent loss. Defaults to True. + + Returns: + Tensor: loss result + """ + loss = 0 + intent_loss = None + slot_loss = None + if self.intent_classifier is not None: + intent_loss = self.intent_classifier.compute_loss(pred, target) if compute_intent_loss else None + intent_weight = self.intent_classifier.config.get("weight") + intent_weight = intent_weight if intent_weight is not None else 1. + loss += intent_loss * intent_weight + if self.slot_classifier is not None: + slot_loss = self.slot_classifier.compute_loss(pred, target) if compute_slot_loss else None + slot_weight = self.slot_classifier.config.get("weight") + slot_weight = slot_weight if slot_weight is not None else 1. + loss += slot_loss * slot_weight + return loss, intent_loss, slot_loss + + +class StackPropagationDecoder(BaseDecoder): + + def forward(self, hidden: HiddenData): + # hidden = self.interaction(hidden) + pred_intent = self.intent_classifier(hidden) + # embedding = pred_intent.output_embedding if pred_intent.output_embedding is not None else pred_intent.classifier_output + # hidden.update_intent_hidden_state(torch.cat([hidden.get_slot_hidden_state(), embedding], dim=-1)) + hidden = self.interaction(pred_intent, hidden) + pred_slot = self.slot_classifier(hidden) + return OutputData(pred_intent, pred_slot) + +class DCANetDecoder(BaseDecoder): + + def forward(self, hidden: HiddenData): + if self.interaction is not None: + hidden = self.interaction(hidden, intent_emb=self.intent_classifier, slot_emb=self.slot_classifier) + return OutputData(self.intent_classifier(hidden), self.slot_classifier(hidden)) + diff --git a/model/decoder/classifier.py b/model/decoder/classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..ed4fd55b69f34c0700c05cca1228ba327f905ac5 --- /dev/null +++ b/model/decoder/classifier.py @@ -0,0 +1,321 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-01-31 20:07:00 +Description: + +''' +import random + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import CrossEntropyLoss + +from model.decoder import decoder_utils + +from torchcrf import CRF + +from common.utils import HiddenData, OutputData, InputData, ClassifierOutputData, unpack_sequence, pack_sequence, \ + instantiate + + +class BaseClassifier(nn.Module): + """Base class for all classifier module + """ + def __init__(self, **config): + super().__init__() + self.config = config + if config.get("loss_fn"): + self.loss_fn = instantiate(config.get("loss_fn")) + else: + self.loss_fn = CrossEntropyLoss(ignore_index=self.config.get("ignore_index")) + + def forward(self, *args, **kwargs): + raise NotImplementedError("No implemented classifier.") + + def decode(self, output: OutputData, + target: InputData = None, + return_list=True, + return_sentence_level=None): + """decode output logits + + Args: + output (OutputData): output logits data + target (InputData, optional): input data with attention mask. Defaults to None. + return_list (bool, optional): if True return list else return torch Tensor.. Defaults to True. + return_sentence_level (_type_, optional): if True decode sentence level intent else decode token level intent. Defaults to None. + + Returns: + List or Tensor: decoded sequence ids + """ + if self.config.get("return_sentence_level") is not None and return_sentence_level is None: + return_sentence_level = self.config.get("return_sentence_level") + elif self.config.get("return_sentence_level") is None and return_sentence_level is None: + return_sentence_level = False + return decoder_utils.decode(output, target, + return_list=return_list, + return_sentence_level=return_sentence_level, + pred_type=self.config.get("mode"), + use_multi=self.config.get("use_multi"), + multi_threshold=self.config.get("multi_threshold")) + + def compute_loss(self, pred: OutputData, target: InputData): + """compute loss + + Args: + pred (OutputData): output logits data + target (InputData): input golden data + + Returns: + Tensor: loss result + """ + _CRF = None + if self.config.get("use_crf"): + _CRF = self.CRF + return decoder_utils.compute_loss(pred, target, criterion_type=self.config["mode"], + use_crf=_CRF is not None, + ignore_index=self.config["ignore_index"], + use_multi=self.config.get("use_multi"), + loss_fn=self.loss_fn, + CRF=_CRF) + + +class LinearClassifier(BaseClassifier): + """ + Decoder structure based on Linear. + """ + def __init__(self, **config): + """Construction function for LinearClassifier + + Args: + config (dict): + input_dim (int): hidden state dim. + use_slot (bool): whether to classify slot label. + slot_label_num (int, optional): the number of slot label. Enabled if use_slot is True. + use_intent (bool): whether to classify intent label. + intent_label_num (int, optional): the number of intent label. Enabled if use_intent is True. + use_crf (bool): whether to use crf for slot. + """ + super().__init__(**config) + self.config = config + if config.get("use_slot"): + self.slot_classifier = nn.Linear(config["input_dim"], config["slot_label_num"]) + if self.config.get("use_crf"): + self.CRF = CRF(num_tags=config["slot_label_num"], batch_first=True) + if config.get("use_intent"): + self.intent_classifier = nn.Linear(config["input_dim"], config["intent_label_num"]) + + def forward(self, hidden: HiddenData): + if self.config.get("use_intent"): + return ClassifierOutputData(self.intent_classifier(hidden.get_intent_hidden_state())) + if self.config.get("use_slot"): + return ClassifierOutputData(self.slot_classifier(hidden.get_slot_hidden_state())) + + + +class AutoregressiveLSTMClassifier(BaseClassifier): + """ + Decoder structure based on unidirectional LSTM. + """ + + def __init__(self, **config): + """ Construction function for Decoder. + + Args: + config (dict): + input_dim (int): input dimension of Decoder. In fact, it's encoder hidden size. + use_slot (bool): whether to classify slot label. + slot_label_num (int, optional): the number of slot label. Enabled if use_slot is True. + use_intent (bool): whether to classify intent label. + intent_label_num (int, optional): the number of intent label. Enabled if use_intent is True. + use_crf (bool): whether to use crf for slot. + hidden_dim (int): hidden dimension of iterative LSTM. + embedding_dim (int): if it's not None, the input and output are relevant. + dropout_rate (float): dropout rate of network which is only useful for embedding. + """ + + super(AutoregressiveLSTMClassifier, self).__init__(**config) + if config.get("use_slot") and config.get("use_crf"): + self.CRF = CRF(num_tags=config["slot_label_num"], batch_first=True) + self.input_dim = config["input_dim"] + self.hidden_dim = config["hidden_dim"] + if config.get("use_intent"): + self.output_dim = config["intent_label_num"] + if config.get("use_slot"): + self.output_dim = config["slot_label_num"] + self.dropout_rate = config["dropout_rate"] + self.embedding_dim = config.get("embedding_dim") + self.force_ratio = config.get("force_ratio") + self.config = config + self.ignore_index = config.get("ignore_index") if config.get("ignore_index") is not None else -100 + # If embedding_dim is not None, the output and input + # of this structure is relevant. + if self.embedding_dim is not None: + self.embedding_layer = nn.Embedding(self.output_dim, self.embedding_dim) + self.init_tensor = nn.Parameter( + torch.randn(1, self.embedding_dim), + requires_grad=True + ) + + # Make sure the input dimension of iterative LSTM. + if self.embedding_dim is not None: + lstm_input_dim = self.input_dim + self.embedding_dim + else: + lstm_input_dim = self.input_dim + + # Network parameter definition. + self.dropout_layer = nn.Dropout(self.dropout_rate) + self.lstm_layer = nn.LSTM( + input_size=lstm_input_dim, + hidden_size=self.hidden_dim, + batch_first=True, + bidirectional=self.config["bidirectional"], + dropout=self.dropout_rate, + num_layers=self.config["layer_num"] + ) + self.linear_layer = nn.Linear( + self.hidden_dim, + self.output_dim + ) + # self.loss_fn = CrossEntropyLoss(ignore_index=self.ignore_index) + + def forward(self, hidden: HiddenData, internal_interaction=None, **interaction_args): + """ Forward process for decoder. + + :param internal_interaction: + :param hidden: + :return: is distribution of prediction labels. + """ + input_tensor = hidden.slot_hidden + seq_lens = hidden.inputs.attention_mask.sum(-1).detach().cpu().tolist() + output_tensor_list, sent_start_pos = [], 0 + input_tensor = pack_sequence(input_tensor, seq_lens) + forced_input = None + if self.training: + if random.random() < self.force_ratio: + if self.config["mode"]=="slot": + + forced_slot = pack_sequence(hidden.inputs.slot, seq_lens) + temp_slot = [] + for index, x in enumerate(forced_slot): + if index == 0: + temp_slot.append(x.reshape(1)) + elif x == self.ignore_index: + temp_slot.append(temp_slot[-1]) + else: + temp_slot.append(x.reshape(1)) + forced_input = torch.cat(temp_slot, 0) + if self.config["mode"]=="token-level-intent": + forced_intent = hidden.inputs.intent.unsqueeze(1).repeat(1, hidden.inputs.slot.shape[1]) + forced_input = pack_sequence(forced_intent, seq_lens) + if self.embedding_dim is None or forced_input is not None: + + for sent_i in range(0, len(seq_lens)): + sent_end_pos = sent_start_pos + seq_lens[sent_i] + + # Segment input hidden tensors. + seg_hiddens = input_tensor[sent_start_pos: sent_end_pos, :] + + if self.embedding_dim is not None and forced_input is not None: + if seq_lens[sent_i] > 1: + seg_forced_input = forced_input[sent_start_pos: sent_end_pos] + + seg_forced_tensor = self.embedding_layer(seg_forced_input)[:-1] + seg_prev_tensor = torch.cat([self.init_tensor, seg_forced_tensor], dim=0) + else: + seg_prev_tensor = self.init_tensor + + # Concatenate forced target tensor. + combined_input = torch.cat([seg_hiddens, seg_prev_tensor], dim=1) + else: + combined_input = seg_hiddens + dropout_input = self.dropout_layer(combined_input) + lstm_out, _ = self.lstm_layer(dropout_input.view(1, seq_lens[sent_i], -1)) + if internal_interaction is not None: + interaction_args["sent_id"] = sent_i + lstm_out = internal_interaction(torch.transpose(lstm_out, 0, 1), **interaction_args)[:, 0] + linear_out = self.linear_layer(lstm_out.view(seq_lens[sent_i], -1)) + + output_tensor_list.append(linear_out) + sent_start_pos = sent_end_pos + else: + for sent_i in range(0, len(seq_lens)): + prev_tensor = self.init_tensor + + # It's necessary to remember h and c state + # when output prediction every single step. + last_h, last_c = None, None + + sent_end_pos = sent_start_pos + seq_lens[sent_i] + for word_i in range(sent_start_pos, sent_end_pos): + seg_input = input_tensor[[word_i], :] + combined_input = torch.cat([seg_input, prev_tensor], dim=1) + dropout_input = self.dropout_layer(combined_input).view(1, 1, -1) + if last_h is None and last_c is None: + lstm_out, (last_h, last_c) = self.lstm_layer(dropout_input) + else: + lstm_out, (last_h, last_c) = self.lstm_layer(dropout_input, (last_h, last_c)) + + if internal_interaction is not None: + interaction_args["sent_id"] = sent_i + lstm_out = internal_interaction(lstm_out, **interaction_args)[:, 0] + + lstm_out = self.linear_layer(lstm_out.view(1, -1)) + output_tensor_list.append(lstm_out) + + _, index = lstm_out.topk(1, dim=1) + prev_tensor = self.embedding_layer(index).view(1, -1) + sent_start_pos = sent_end_pos + seq_unpacked = unpack_sequence(torch.cat(output_tensor_list, dim=0), seq_lens) + # TODO: 都支持softmax + if self.config.get("use_multi"): + pred_output = ClassifierOutputData(seq_unpacked) + else: + pred_output = ClassifierOutputData(F.log_softmax(seq_unpacked, dim=-1)) + return pred_output + + +class MLPClassifier(BaseClassifier): + """ + Decoder structure based on MLP. + """ + def __init__(self, **config): + """ Construction function for Decoder. + + Args: + config (dict): + use_slot (bool): whether to classify slot label. + use_intent (bool): whether to classify intent label. + mlp (List): + + - _model_target_: torch.nn.Linear + + in_features (int): input feature dim + + out_features (int): output feature dim + + - _model_target_: torch.nn.LeakyReLU + + negative_slope: 0.2 + + - ... + """ + super(MLPClassifier, self).__init__(**config) + self.config = config + for i, x in enumerate(config["mlp"]): + if isinstance(x.get("in_features"), str): + config["mlp"][i]["in_features"] = self.config[x["in_features"][1:-1]] + if isinstance(x.get("out_features"), str): + config["mlp"][i]["out_features"] = self.config[x["out_features"][1:-1]] + mlp = [instantiate(x) for x in config["mlp"]] + self.seq = nn.Sequential(*mlp) + + + def forward(self, hidden: HiddenData): + if self.config.get("use_intent"): + res = self.seq(hidden.intent_hidden) + else: + res = self.seq(hidden.slot_hidden) + return ClassifierOutputData(res) diff --git a/model/decoder/decoder_utils.py b/model/decoder/decoder_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d61e729c336b919d4ff6d7ae5dfe76434bdef0fa --- /dev/null +++ b/model/decoder/decoder_utils.py @@ -0,0 +1,155 @@ +from typing import List +import torch + +from common import utils +from common.utils import OutputData, InputData +from torch import Tensor + +def argmax_for_seq_len(inputs, seq_lens, padding_value=-100): + packed_inputs = utils.pack_sequence(inputs, seq_lens) + outputs = torch.argmax(packed_inputs, dim=-1, keepdim=True) + return utils.unpack_sequence(outputs, seq_lens, padding_value).squeeze(-1) + + +def decode(output: OutputData, + target: InputData = None, + pred_type="slot", + multi_threshold=0.5, + ignore_index=-100, + return_list=True, + return_sentence_level=True, + use_multi=False, + use_crf=False, + CRF=None) -> List or Tensor: + """ decode output logits + + Args: + output (OutputData): output logits data + target (InputData, optional): input data with attention mask. Defaults to None. + pred_type (str, optional): prediction type in ["slot", "intent", "token-level-intent"]. Defaults to "slot". + multi_threshold (float, optional): multi intent decode threshold. Defaults to 0.5. + ignore_index (int, optional): align and pad token with ignore index. Defaults to -100. + return_list (bool, optional): if True return list else return torch Tensor. Defaults to True. + return_sentence_level (bool, optional): if True decode sentence level intent else decode token level intent. Defaults to True. + use_multi (bool, optional): whether to decode to multi intent. Defaults to False. + use_crf (bool, optional): whether to use crf. Defaults to False. + CRF (CRF, optional): CRF function. Defaults to None. + + Returns: + List or Tensor: decoded sequence ids + """ + if pred_type == "slot": + inputs = output.slot_ids + else: + inputs = output.intent_ids + + if pred_type == "slot": + if not use_multi: + if use_crf: + res = CRF.decode(inputs, mask=target.attention_mask) + else: + res = torch.argmax(inputs, dim=-1) + else: + raise NotImplementedError("Multi-slot prediction is not supported.") + elif pred_type == "intent": + if not use_multi: + res = torch.argmax(inputs, dim=-1) + else: + res = (torch.sigmoid(inputs) > multi_threshold).nonzero() + if return_list: + res_index = res.detach().cpu().tolist() + res_list = [[] for _ in range(len(target.seq_lens))] + for item in res_index: + res_list[item[0]].append(item[1]) + return res_list + else: + return res + elif pred_type == "token-level-intent": + if not use_multi: + res = torch.argmax(inputs, dim=-1) + if not return_sentence_level: + return res + if return_list: + res = res.detach().cpu().tolist() + attention_mask = target.attention_mask + for i in range(attention_mask.shape[0]): + temp = [] + for j in range(attention_mask.shape[1]): + if attention_mask[i][j] == 1: + temp.append(res[i][j]) + else: + break + res[i] = temp + return [max(it, key=lambda v: it.count(v)) for it in res] + else: + seq_lens = target.seq_lens + + if not return_sentence_level: + token_res = torch.cat([ + torch.sigmoid(inputs[i, 0:seq_lens[i], :]) > multi_threshold + for i in range(len(seq_lens))], + dim=0) + return utils.unpack_sequence(token_res, seq_lens, padding_value=ignore_index) + + intent_index_sum = torch.cat([ + torch.sum(torch.sigmoid(inputs[i, 0:seq_lens[i], :]) > multi_threshold, dim=0).unsqueeze(0) + for i in range(len(seq_lens))], + dim=0) + + res = (intent_index_sum > torch.div(seq_lens, 2, rounding_mode='floor').unsqueeze(1)).nonzero() + if return_list: + res_index = res.detach().cpu().tolist() + res_list = [[] for _ in range(len(seq_lens))] + for item in res_index: + res_list[item[0]].append(item[1]) + return res_list + else: + return res + else: + raise NotImplementedError("Prediction mode except ['slot','intent','token-level-intent'] is not supported.") + if return_list: + res = res.detach().cpu().tolist() + return res + + +def compute_loss(pred: OutputData, + target: InputData, + criterion_type="slot", + use_crf=False, + ignore_index=-100, + loss_fn=None, + use_multi=False, + CRF=None): + """ compute loss + + Args: + pred (OutputData): output logits data + target (InputData): input golden data + criterion_type (str, optional): criterion type in ["slot", "intent", "token-level-intent"]. Defaults to "slot". + ignore_index (int, optional): compute loss with ignore index. Defaults to -100. + loss_fn (_type_, optional): loss function. Defaults to None. + use_crf (bool, optional): whether to use crf. Defaults to False. + CRF (CRF, optional): CRF function. Defaults to None. + + Returns: + Tensor: loss result + """ + if criterion_type == "slot": + if use_crf: + return -1 * CRF(pred.slot_ids, target.slot, target.get_slot_mask(ignore_index).byte()) + else: + pred_slot = utils.pack_sequence(pred.slot_ids, target.seq_lens) + target_slot = utils.pack_sequence(target.slot, target.seq_lens) + return loss_fn(pred_slot, target_slot) + elif criterion_type == "token-level-intent": + # TODO: Two decode function + intent_target = target.intent.unsqueeze(1) + if not use_multi: + intent_target = intent_target.repeat(1, pred.intent_ids.shape[1]) + else: + intent_target = intent_target.repeat(1, pred.intent_ids.shape[1], 1) + intent_pred = utils.pack_sequence(pred.intent_ids, target.seq_lens) + intent_target = utils.pack_sequence(intent_target, target.seq_lens) + return loss_fn(intent_pred, intent_target) + else: + return loss_fn(pred.intent_ids, target.intent) diff --git a/model/decoder/gl_gin_decoder.py b/model/decoder/gl_gin_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..91906e94c77060d1909e98055d09d83bb96efc52 --- /dev/null +++ b/model/decoder/gl_gin_decoder.py @@ -0,0 +1,47 @@ +import torch +import torch.nn.functional as F +from torch import nn + +from common.utils import HiddenData, OutputData, InputData +from model.decoder import BaseDecoder +from model.decoder.interaction.gl_gin_interaction import LSTMEncoder + + +class IntentEncoder(nn.Module): + def __init__(self,input_dim, dropout_rate): + super().__init__() + self.dropout_rate = dropout_rate + self.__intent_lstm = LSTMEncoder( + input_dim, + input_dim, + dropout_rate + ) + + def forward(self, g_hiddens, seq_lens): + intent_lstm_out = self.__intent_lstm(g_hiddens, seq_lens) + return F.dropout(intent_lstm_out, p=self.dropout_rate, training=self.training) + + +class GLGINDecoder(BaseDecoder): + def __init__(self, intent_classifier, slot_classifier, interaction=None, **config): + super().__init__(intent_classifier, slot_classifier, interaction) + self.config=config + self.intent_encoder = IntentEncoder(self.intent_classifier.config["input_dim"], self.config["dropout_rate"]) + + def forward(self, hidden: HiddenData, forced_slot=None, forced_intent=None, differentiable=None): + seq_lens = hidden.inputs.attention_mask.sum(-1) + intent_lstm_out = self.intent_encoder(hidden.slot_hidden, seq_lens) + hidden.update_intent_hidden_state(intent_lstm_out) + pred_intent = self.intent_classifier(hidden) + intent_index = self.intent_classifier.decode(OutputData(pred_intent, None),hidden.inputs, + return_list=False, + return_sentence_level=True) + slot_hidden = self.interaction( + hidden, + pred_intent=pred_intent, + intent_index=intent_index, + ) + pred_slot = self.slot_classifier(slot_hidden) + num_intent = self.intent_classifier.config["intent_label_num"] + pred_slot = pred_slot.classifier_output[:, num_intent:] + return OutputData(pred_intent, F.log_softmax(pred_slot, dim=1)) \ No newline at end of file diff --git a/model/decoder/interaction/__init__.py b/model/decoder/interaction/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4e411af2bcfac5aeda51c746e5371f84300781e4 --- /dev/null +++ b/model/decoder/interaction/__init__.py @@ -0,0 +1,10 @@ +from model.decoder.interaction.agif_interaction import AGIFInteraction +from model.decoder.interaction.base_interaction import BaseInteraction +from model.decoder.interaction.bi_model_interaction import BiModelInteraction, BiModelWithoutDecoderInteraction +from model.decoder.interaction.dca_net_interaction import DCANetInteraction +from model.decoder.interaction.gl_gin_interaction import GLGINInteraction +from model.decoder.interaction.slot_gated_interaction import SlotGatedInteraction +from model.decoder.interaction.stack_interaction import StackInteraction + +__all__ = ["BaseInteraction", "BiModelInteraction", "BiModelWithoutDecoderInteraction", "DCANetInteraction", + "StackInteraction", "SlotGatedInteraction", "AGIFInteraction", "GLGINInteraction"] diff --git a/model/decoder/interaction/agif_interaction.py b/model/decoder/interaction/agif_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..6d5ef4d1e9bed3c27b190e4183783c622715544c --- /dev/null +++ b/model/decoder/interaction/agif_interaction.py @@ -0,0 +1,132 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from model.decoder.interaction.base_interaction import BaseInteraction + + +class GraphAttentionLayer(nn.Module): + """ + Simple GAT layer, similar to https://arxiv.org/abs/1710.10903 + """ + + def __init__(self, in_features, out_features, dropout, alpha, concat=True): + super(GraphAttentionLayer, self).__init__() + self.dropout = dropout + self.in_features = in_features + self.out_features = out_features + self.alpha = alpha + self.concat = concat + + self.W = nn.Parameter(torch.zeros(size=(in_features, out_features))) + nn.init.xavier_uniform_(self.W.data, gain=1.414) + self.a = nn.Parameter(torch.zeros(size=(2 * out_features, 1))) + nn.init.xavier_uniform_(self.a.data, gain=1.414) + + self.leakyrelu = nn.LeakyReLU(self.alpha) + + def forward(self, input, adj): + h = torch.matmul(input, self.W) + B, N = h.size()[0], h.size()[1] + + a_input = torch.cat([h.repeat(1, 1, N).view(B, N * N, -1), h.repeat(1, N, 1)], dim=2).view(B, N, -1, + 2 * self.out_features) + e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(3)) + + zero_vec = -9e15 * torch.ones_like(e) + attention = torch.where(adj > 0, e, zero_vec) + attention = F.softmax(attention, dim=2) + attention = F.dropout(attention, self.dropout, training=self.training) + h_prime = torch.matmul(attention, h) + + if self.concat: + return F.elu(h_prime) + else: + return h_prime + + +class GAT(nn.Module): + def __init__(self, nfeat, nhid, nclass, dropout, alpha, nheads, nlayers=2): + """Dense version of GAT.""" + super(GAT, self).__init__() + self.dropout = dropout + self.nlayers = nlayers + self.nheads = nheads + self.attentions = [GraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True) for _ in + range(nheads)] + for i, attention in enumerate(self.attentions): + self.add_module('attention_{}'.format(i), attention) + if self.nlayers > 2: + for i in range(self.nlayers - 2): + for j in range(self.nheads): + self.add_module('attention_{}_{}'.format(i + 1, j), + GraphAttentionLayer(nhid * nheads, nhid, dropout=dropout, alpha=alpha, concat=True)) + + self.out_att = GraphAttentionLayer(nhid * nheads, nclass, dropout=dropout, alpha=alpha, concat=False) + + def forward(self, x, adj): + x = F.dropout(x, self.dropout, training=self.training) + input = x + x = torch.cat([att(x, adj) for att in self.attentions], dim=2) + if self.nlayers > 2: + for i in range(self.nlayers - 2): + temp = [] + x = F.dropout(x, self.dropout, training=self.training) + cur_input = x + for j in range(self.nheads): + temp.append(self.__getattr__('attention_{}_{}'.format(i + 1, j))(x, adj)) + x = torch.cat(temp, dim=2) + cur_input + x = F.dropout(x, self.dropout, training=self.training) + x = F.elu(self.out_att(x, adj)) + return x + input + + +def normalize_adj(mx): + """ + Row-normalize matrix D^{-1}A + torch.diag_embed: https://github.com/pytorch/pytorch/pull/12447 + """ + mx = mx.float() + rowsum = mx.sum(2) + r_inv = torch.pow(rowsum, -1) + r_inv[torch.isinf(r_inv)] = 0. + r_mat_inv = torch.diag_embed(r_inv, 0) + mx = r_mat_inv.matmul(mx) + return mx + + +class AGIFInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.intent_embedding = nn.Parameter( + torch.FloatTensor(self.config["intent_label_num"], self.config["intent_embedding_dim"])) # 191, 32 + nn.init.normal_(self.intent_embedding.data) + self.adj = None + self.graph = GAT( + config["output_dim"], + config["hidden_dim"], + config["output_dim"], + config["dropout_rate"], + config["alpha"], + config["num_heads"], + config["num_layers"]) + + def generate_adj_gat(self, index, batch, intent_label_num): + intent_idx_ = [[torch.tensor(0)] for i in range(batch)] + for item in index: + intent_idx_[item[0]].append(item[1] + 1) + intent_idx = intent_idx_ + self.adj = torch.cat([torch.eye(intent_label_num + 1).unsqueeze(0) for i in range(batch)]) + for i in range(batch): + for j in intent_idx[i]: + self.adj[i, j, intent_idx[i]] = 1. + if self.config["row_normalized"]: + self.adj = normalize_adj(self.adj) + self.adj = self.adj.to(self.intent_embedding.device) + + def forward(self, encode_hidden, **interaction_args): + if self.adj is None or interaction_args["sent_id"] == 0: + self.generate_adj_gat(interaction_args["intent_index"], interaction_args["batch_size"], interaction_args["intent_label_num"]) + lstm_out = torch.cat((encode_hidden, + self.intent_embedding.unsqueeze(0).repeat(encode_hidden.shape[0], 1, 1)), dim=1) + return self.graph(lstm_out, self.adj[interaction_args["sent_id"]]) diff --git a/model/decoder/interaction/base_interaction.py b/model/decoder/interaction/base_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..e51d6abea6cc78f3a77d199bceb5b5e054f942fe --- /dev/null +++ b/model/decoder/interaction/base_interaction.py @@ -0,0 +1,9 @@ +from torch import nn + +class BaseInteraction(nn.Module): + def __init__(self, **config): + super().__init__() + self.config = config + + def forward(self, hidden1, hidden2): + NotImplementedError("no implemented") diff --git a/model/decoder/interaction/bi_model_interaction.py b/model/decoder/interaction/bi_model_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..09f5f52e7afd2de2255847c2578f44332be654e6 --- /dev/null +++ b/model/decoder/interaction/bi_model_interaction.py @@ -0,0 +1,74 @@ +import torch +import torch.nn.functional as F +from torch import nn + +from common.utils import HiddenData +from model.decoder.interaction.base_interaction import BaseInteraction + + +class BiModelInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.intent_lstm = nn.LSTM(input_size=self.config["input_dim"], hidden_size=self.config["output_dim"], + batch_first=True, + num_layers=1) + self.slot_lstm = nn.LSTM(input_size=self.config["input_dim"] + self.config["output_dim"], + hidden_size=self.config["output_dim"], num_layers=1) + + def forward(self, encode_hidden: HiddenData, **kwargs): + slot_hidden = encode_hidden.get_slot_hidden_state() + intent_hidden_detached = encode_hidden.get_intent_hidden_state().clone().detach() + seq_lens = encode_hidden.inputs.attention_mask.sum(-1) + batch = slot_hidden.size(0) + length = slot_hidden.size(1) + dec_init_out = torch.zeros(batch, 1, self.config["output_dim"]).to(slot_hidden.device) + hidden_state = (torch.zeros(1, 1, self.config["output_dim"]).to(slot_hidden.device), torch.zeros(1, 1, self.config["output_dim"]).to(slot_hidden.device)) + slot_hidden = torch.cat((slot_hidden, intent_hidden_detached), dim=-1).transpose(1, + 0) # 50 x batch x feature_size + slot_drop = F.dropout(slot_hidden, self.config["dropout_rate"]) + all_out = [] + for i in range(length): + if i == 0: + out, hidden_state = self.slot_lstm(torch.cat((slot_drop[i].unsqueeze(1), dec_init_out), dim=-1), + hidden_state) + else: + out, hidden_state = self.slot_lstm(torch.cat((slot_drop[i].unsqueeze(1), out), dim=-1), hidden_state) + all_out.append(out) + slot_output = torch.cat(all_out, dim=1) # batch x 50 x feature_size + + intent_hidden = torch.cat((encode_hidden.get_intent_hidden_state(), + encode_hidden.get_slot_hidden_state().clone().detach()), + dim=-1) + intent_drop = F.dropout(intent_hidden, self.config["dropout_rate"]) + intent_lstm_output, _ = self.intent_lstm(intent_drop) + intent_output = F.dropout(intent_lstm_output, self.config["dropout_rate"]) + output_list = [] + for index, slen in enumerate(seq_lens): + output_list.append(intent_output[index, slen - 1, :].unsqueeze(0)) + + encode_hidden.update_intent_hidden_state(torch.cat(output_list, dim=0)) + encode_hidden.update_slot_hidden_state(slot_output) + + return encode_hidden + + +class BiModelWithoutDecoderInteraction(BaseInteraction): + def forward(self, encode_hidden: HiddenData, **kwargs): + slot_hidden = encode_hidden.get_slot_hidden_state() + intent_hidden_detached = encode_hidden.get_intent_hidden_state().clone().detach() + seq_lens = encode_hidden.inputs.attention_mask.sum(-1) + slot_hidden = torch.cat((slot_hidden, intent_hidden_detached), dim=-1) # 50 x batch x feature_size + slot_output = F.dropout(slot_hidden, self.config["dropout_rate"]) + + intent_hidden = torch.cat((encode_hidden.get_intent_hidden_state(), + encode_hidden.get_slot_hidden_state().clone().detach()), + dim=-1) + intent_output = F.dropout(intent_hidden, self.config["dropout_rate"]) + output_list = [] + for index, slen in enumerate(seq_lens): + output_list.append(intent_output[index, slen - 1, :].unsqueeze(0)) + + encode_hidden.update_intent_hidden_state(torch.cat(output_list, dim=0)) + encode_hidden.update_slot_hidden_state(slot_output) + + return encode_hidden diff --git a/model/decoder/interaction/dca_net_interaction.py b/model/decoder/interaction/dca_net_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..021f01969044042c102a47bb1f4c9bea3880e9fa --- /dev/null +++ b/model/decoder/interaction/dca_net_interaction.py @@ -0,0 +1,176 @@ +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn import LayerNorm + +from common.utils import HiddenData +from model.decoder.interaction import BaseInteraction + + +class DCANetInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.I_S_Emb = Label_Attention() + self.T_block1 = I_S_Block(self.config["input_dim"], self.config["attention_dropout"], self.config["num_attention_heads"]) + self.T_block2 = I_S_Block(self.config["input_dim"], self.config["attention_dropout"], self.config["num_attention_heads"]) + + def forward(self, encode_hidden: HiddenData, **kwargs): + mask = encode_hidden.inputs.attention_mask + H = encode_hidden.slot_hidden + H_I, H_S = self.I_S_Emb(H, H, kwargs["intent_emb"], kwargs["slot_emb"]) + H_I, H_S = self.T_block1(H_I + H, H_S + H, mask) + H_I_1, H_S_1 = self.I_S_Emb(H_I, H_S, kwargs["intent_emb"], kwargs["slot_emb"]) + H_I, H_S = self.T_block2(H_I + H_I_1, H_S + H_S_1, mask) + encode_hidden.update_intent_hidden_state(F.max_pool1d((H_I + H).transpose(1, 2), H_I.size(1)).squeeze(2)) + encode_hidden.update_slot_hidden_state(H_S + H) + return encode_hidden + + +class Label_Attention(nn.Module): + def __init__(self): + super(Label_Attention, self).__init__() + + def forward(self, input_intent, input_slot, intent_emb, slot_emb): + self.W_intent_emb = intent_emb.intent_classifier.weight + self.W_slot_emb = slot_emb.slot_classifier.weight + intent_score = torch.matmul(input_intent, self.W_intent_emb.t()) + slot_score = torch.matmul(input_slot, self.W_slot_emb.t()) + intent_probs = nn.Softmax(dim=-1)(intent_score) + slot_probs = nn.Softmax(dim=-1)(slot_score) + intent_res = torch.matmul(intent_probs, self.W_intent_emb) + slot_res = torch.matmul(slot_probs, self.W_slot_emb) + + return intent_res, slot_res + + +class I_S_Block(nn.Module): + def __init__(self, hidden_size, attention_dropout, num_attention_heads): + super(I_S_Block, self).__init__() + self.I_S_Attention = I_S_SelfAttention(hidden_size, 2 * hidden_size, hidden_size, attention_dropout, num_attention_heads) + self.I_Out = SelfOutput(hidden_size, attention_dropout) + self.S_Out = SelfOutput(hidden_size, attention_dropout) + self.I_S_Feed_forward = Intermediate_I_S(hidden_size, hidden_size, attention_dropout) + + def forward(self, H_intent_input, H_slot_input, mask): + H_slot, H_intent = self.I_S_Attention(H_intent_input, H_slot_input, mask) + H_slot = self.S_Out(H_slot, H_slot_input) + H_intent = self.I_Out(H_intent, H_intent_input) + H_intent, H_slot = self.I_S_Feed_forward(H_intent, H_slot) + + return H_intent, H_slot + + +class I_S_SelfAttention(nn.Module): + def __init__(self, input_size, hidden_size, out_size, attention_dropout, num_attention_heads): + super(I_S_SelfAttention, self).__init__() + + self.num_attention_heads = num_attention_heads + self.attention_head_size = int(hidden_size / self.num_attention_heads) + + self.all_head_size = self.num_attention_heads * self.attention_head_size + self.out_size = out_size + self.query = nn.Linear(input_size, self.all_head_size) + self.query_slot = nn.Linear(input_size, self.all_head_size) + self.key = nn.Linear(input_size, self.all_head_size) + self.key_slot = nn.Linear(input_size, self.all_head_size) + self.value = nn.Linear(input_size, self.out_size) + self.value_slot = nn.Linear(input_size, self.out_size) + self.dropout = nn.Dropout(attention_dropout) + + def transpose_for_scores(self, x): + last_dim = int(x.size()[-1] / self.num_attention_heads) + new_x_shape = x.size()[:-1] + (self.num_attention_heads, last_dim) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, intent, slot, mask): + extended_attention_mask = mask.unsqueeze(1).unsqueeze(2) + + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + mixed_query_layer = self.query(intent) + mixed_key_layer = self.key(slot) + mixed_value_layer = self.value(slot) + + mixed_query_layer_slot = self.query_slot(slot) + mixed_key_layer_slot = self.key_slot(intent) + mixed_value_layer_slot = self.value_slot(intent) + + query_layer = self.transpose_for_scores(mixed_query_layer) + query_layer_slot = self.transpose_for_scores(mixed_query_layer_slot) + key_layer = self.transpose_for_scores(mixed_key_layer) + key_layer_slot = self.transpose_for_scores(mixed_key_layer_slot) + value_layer = self.transpose_for_scores(mixed_value_layer) + value_layer_slot = self.transpose_for_scores(mixed_value_layer_slot) + + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + # attention_scores_slot = torch.matmul(query_slot, key_slot.transpose(1,0)) + attention_scores_slot = torch.matmul(query_layer_slot, key_layer_slot.transpose(-1, -2)) + attention_scores_slot = attention_scores_slot / math.sqrt(self.attention_head_size) + attention_scores_intent = attention_scores + attention_mask + + attention_scores_slot = attention_scores_slot + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs_slot = nn.Softmax(dim=-1)(attention_scores_slot) + attention_probs_intent = nn.Softmax(dim=-1)(attention_scores_intent) + + attention_probs_slot = self.dropout(attention_probs_slot) + attention_probs_intent = self.dropout(attention_probs_intent) + + context_layer_slot = torch.matmul(attention_probs_slot, value_layer_slot) + context_layer_intent = torch.matmul(attention_probs_intent, value_layer) + + context_layer = context_layer_slot.permute(0, 2, 1, 3).contiguous() + context_layer_intent = context_layer_intent.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.out_size,) + new_context_layer_shape_intent = context_layer_intent.size()[:-2] + (self.out_size,) + + context_layer = context_layer.view(*new_context_layer_shape) + context_layer_intent = context_layer_intent.view(*new_context_layer_shape_intent) + return context_layer, context_layer_intent + + +class SelfOutput(nn.Module): + def __init__(self, hidden_size, hidden_dropout_prob): + super(SelfOutput, self).__init__() + self.dense = nn.Linear(hidden_size, hidden_size) + self.LayerNorm = LayerNorm(hidden_size, eps=1e-12) + self.dropout = nn.Dropout(hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class Intermediate_I_S(nn.Module): + def __init__(self, intermediate_size, hidden_size, attention_dropout): + super(Intermediate_I_S, self).__init__() + self.dense_in = nn.Linear(hidden_size * 6, intermediate_size) + self.intermediate_act_fn = nn.ReLU() + self.dense_out = nn.Linear(intermediate_size, hidden_size) + self.LayerNorm_I = LayerNorm(hidden_size, eps=1e-12) + self.LayerNorm_S = LayerNorm(hidden_size, eps=1e-12) + self.dropout = nn.Dropout(attention_dropout) + + def forward(self, hidden_states_I, hidden_states_S): + hidden_states_in = torch.cat([hidden_states_I, hidden_states_S], dim=2) + batch_size, max_length, hidden_size = hidden_states_in.size() + h_pad = torch.zeros(batch_size, 1, hidden_size).to(hidden_states_I.device) + h_left = torch.cat([h_pad, hidden_states_in[:, :max_length - 1, :]], dim=1) + h_right = torch.cat([hidden_states_in[:, 1:, :], h_pad], dim=1) + hidden_states_in = torch.cat([hidden_states_in, h_left, h_right], dim=2) + + hidden_states = self.dense_in(hidden_states_in) + hidden_states = self.intermediate_act_fn(hidden_states) + hidden_states = self.dense_out(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states_I_NEW = self.LayerNorm_I(hidden_states + hidden_states_I) + hidden_states_S_NEW = self.LayerNorm_S(hidden_states + hidden_states_S) + return hidden_states_I_NEW, hidden_states_S_NEW diff --git a/model/decoder/interaction/gl_gin_interaction.py b/model/decoder/interaction/gl_gin_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..d6e697866719d3e71fca9a1ec91cd95dce81b9c1 --- /dev/null +++ b/model/decoder/interaction/gl_gin_interaction.py @@ -0,0 +1,227 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + +from common.utils import HiddenData, ClassifierOutputData +from model.decoder.interaction import BaseInteraction + + +class LSTMEncoder(nn.Module): + """ + Encoder structure based on bidirectional LSTM. + """ + + def __init__(self, embedding_dim, hidden_dim, dropout_rate): + super(LSTMEncoder, self).__init__() + + # Parameter recording. + self.__embedding_dim = embedding_dim + self.__hidden_dim = hidden_dim // 2 + self.__dropout_rate = dropout_rate + + # Network attributes. + self.__dropout_layer = nn.Dropout(self.__dropout_rate) + self.__lstm_layer = nn.LSTM( + input_size=self.__embedding_dim, + hidden_size=self.__hidden_dim, + batch_first=True, + bidirectional=True, + dropout=self.__dropout_rate, + num_layers=1 + ) + + def forward(self, embedded_text, seq_lens): + """ Forward process for LSTM Encoder. + + (batch_size, max_sent_len) + -> (batch_size, max_sent_len, word_dim) + -> (batch_size, max_sent_len, hidden_dim) + + :param embedded_text: padded and embedded input text. + :param seq_lens: is the length of original input text. + :return: is encoded word hidden vectors. + """ + + # Padded_text should be instance of LongTensor. + dropout_text = self.__dropout_layer(embedded_text) + + # Pack and Pad process for input of variable length. + packed_text = pack_padded_sequence(dropout_text, seq_lens.cpu(), batch_first=True, enforce_sorted=False) + lstm_hiddens, (h_last, c_last) = self.__lstm_layer(packed_text) + padded_hiddens, _ = pad_packed_sequence(lstm_hiddens, batch_first=True) + + return padded_hiddens + + +class GraphAttentionLayer(nn.Module): + """ + Simple GAT layer, similar to https://arxiv.org/abs/1710.10903 + """ + + def __init__(self, in_features, out_features, dropout, alpha, concat=True): + super(GraphAttentionLayer, self).__init__() + self.dropout = dropout + self.in_features = in_features + self.out_features = out_features + self.alpha = alpha + self.concat = concat + + self.W = nn.Parameter(torch.zeros(size=(in_features, out_features))) + nn.init.xavier_uniform_(self.W.data, gain=1.414) + self.a = nn.Parameter(torch.zeros(size=(2 * out_features, 1))) + nn.init.xavier_uniform_(self.a.data, gain=1.414) + + self.leakyrelu = nn.LeakyReLU(self.alpha) + + def forward(self, input, adj): + h = torch.matmul(input, self.W) + B, N = h.size()[0], h.size()[1] + + a_input = torch.cat([h.repeat(1, 1, N).view(B, N * N, -1), h.repeat(1, N, 1)], dim=2).view(B, N, -1, + 2 * self.out_features) + e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(3)) + + zero_vec = -9e15 * torch.ones_like(e) + attention = torch.where(adj > 0, e, zero_vec) + attention = F.softmax(attention, dim=2) + attention = F.dropout(attention, self.dropout, training=self.training) + h_prime = torch.matmul(attention, h) + + if self.concat: + return F.elu(h_prime) + else: + return h_prime + + +class GAT(nn.Module): + def __init__(self, nfeat, nhid, nclass, dropout, alpha, nheads, nlayers=2): + """Dense version of GAT.""" + super(GAT, self).__init__() + self.dropout = dropout + self.nlayers = nlayers + self.nheads = nheads + self.attentions = [GraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True) for _ in + range(nheads)] + for i, attention in enumerate(self.attentions): + self.add_module('attention_{}'.format(i), attention) + if self.nlayers > 2: + for i in range(self.nlayers - 2): + for j in range(self.nheads): + self.add_module('attention_{}_{}'.format(i + 1, j), + GraphAttentionLayer(nhid * nheads, nhid, dropout=dropout, alpha=alpha, concat=True)) + + self.out_att = GraphAttentionLayer(nhid * nheads, nclass, dropout=dropout, alpha=alpha, concat=False) + + def forward(self, x, adj): + x = F.dropout(x, self.dropout, training=self.training) + input = x + x = torch.cat([att(x, adj) for att in self.attentions], dim=2) + if self.nlayers > 2: + for i in range(self.nlayers - 2): + temp = [] + x = F.dropout(x, self.dropout, training=self.training) + cur_input = x + for j in range(self.nheads): + temp.append(self.__getattr__('attention_{}_{}'.format(i + 1, j))(x, adj)) + x = torch.cat(temp, dim=2) + cur_input + x = F.dropout(x, self.dropout, training=self.training) + x = F.elu(self.out_att(x, adj)) + return x + input + + +def normalize_adj(mx): + """ + Row-normalize matrix D^{-1}A + torch.diag_embed: https://github.com/pytorch/pytorch/pull/12447 + """ + mx = mx.float() + rowsum = mx.sum(2) + r_inv = torch.pow(rowsum, -1) + r_inv[torch.isinf(r_inv)] = 0. + r_mat_inv = torch.diag_embed(r_inv, 0) + mx = r_mat_inv.matmul(mx) + return mx + + +class GLGINInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.intent_embedding = nn.Parameter( + torch.FloatTensor(self.config["intent_label_num"], self.config["intent_embedding_dim"])) # 191, 32 + nn.init.normal_(self.intent_embedding.data) + self.adj = None + self.__slot_lstm = LSTMEncoder( + self.config["input_dim"] + self.config["intent_label_num"], + config["output_dim"], + config["dropout_rate"] + ) + self.__slot_graph = GAT( + config["output_dim"], + config["hidden_dim"], + config["output_dim"], + config["dropout_rate"], + config["alpha"], + config["num_heads"], + config["num_layers"]) + + self.__global_graph = GAT( + config["output_dim"], + config["hidden_dim"], + config["output_dim"], + config["dropout_rate"], + config["alpha"], + config["num_heads"], + config["num_layers"]) + + def generate_global_adj_gat(self, seq_len, index, batch, window): + global_intent_idx = [[] for i in range(batch)] + global_slot_idx = [[] for i in range(batch)] + for item in index: + global_intent_idx[item[0]].append(item[1]) + + for i, len in enumerate(seq_len): + global_slot_idx[i].extend( + list(range(self.config["intent_label_num"], self.config["intent_label_num"] + len))) + + adj = torch.cat([torch.eye(self.config["intent_label_num"] + max(seq_len)).unsqueeze(0) for i in range(batch)]) + for i in range(batch): + for j in global_intent_idx[i]: + adj[i, j, global_slot_idx[i]] = 1. + adj[i, j, global_intent_idx[i]] = 1. + for j in global_slot_idx[i]: + adj[i, j, global_intent_idx[i]] = 1. + + for i in range(batch): + for j in range(self.config["intent_label_num"], self.config["intent_label_num"] + seq_len[i]): + adj[i, j, max(self.config["intent_label_num"], j - window):min(seq_len[i] + self.config["intent_label_num"], j + window + 1)] = 1. + + if self.config["row_normalized"]: + adj = normalize_adj(adj) + adj = adj.to(self.intent_embedding.device) + return adj + + def generate_slot_adj_gat(self, seq_len, batch, window): + slot_idx_ = [[] for i in range(batch)] + adj = torch.cat([torch.eye(max(seq_len)).unsqueeze(0) for i in range(batch)]) + for i in range(batch): + for j in range(seq_len[i]): + adj[i, j, max(0, j - window):min(seq_len[i], j + window + 1)] = 1. + if self.config["row_normalized"]: + adj = normalize_adj(adj) + adj = adj.to(self.intent_embedding.device) + return adj + + def forward(self, encode_hidden: HiddenData, pred_intent: ClassifierOutputData = None, intent_index=None): + seq_lens = encode_hidden.inputs.attention_mask.sum(-1) + slot_lstm_out = self.__slot_lstm(torch.cat([encode_hidden.slot_hidden, pred_intent.classifier_output], dim=-1), + seq_lens) + global_adj = self.generate_global_adj_gat(seq_lens, intent_index, len(seq_lens), + self.config["slot_graph_window"]) + slot_adj = self.generate_slot_adj_gat(seq_lens, len(seq_lens), self.config["slot_graph_window"]) + batch = len(seq_lens) + slot_graph_out = self.__slot_graph(slot_lstm_out, slot_adj) + intent_in = self.intent_embedding.unsqueeze(0).repeat(batch, 1, 1) + global_graph_in = torch.cat([intent_in, slot_graph_out], dim=1) + encode_hidden.update_slot_hidden_state(self.__global_graph(global_graph_in, global_adj)) + return encode_hidden diff --git a/model/decoder/interaction/slot_gated_interaction.py b/model/decoder/interaction/slot_gated_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..20735bdd69f0f2679a13a2d479d074045c62e039 --- /dev/null +++ b/model/decoder/interaction/slot_gated_interaction.py @@ -0,0 +1,59 @@ +import math + +import einops +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn import LayerNorm + +from common.utils import HiddenData +from model.decoder.interaction import BaseInteraction + + +class SlotGatedInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.intent_linear = nn.Linear(self.config["input_dim"],1, bias=False) + self.slot_linear1 = nn.Linear(self.config["input_dim"],1, bias=False) + self.slot_linear2 = nn.Linear(self.config["input_dim"],1, bias=False) + self.remove_slot_attn = self.config["remove_slot_attn"] + self.slot_gate = SlotGate(**config) + + def forward(self, encode_hidden: HiddenData, **kwargs): + input_hidden = encode_hidden.get_slot_hidden_state() + + seq_lens = encode_hidden.inputs.attention_mask.sum(-1) + output_list = [] + for index, slen in enumerate(seq_lens): + output_list.append(input_hidden[index, slen - 1, :].unsqueeze(0)) + intent_input = torch.cat(output_list, dim=0) + e_I = torch.tanh(self.intent_linear(intent_input)).squeeze(1) + alpha_I = einops.repeat(e_I, 'b -> b h', h=intent_input.shape[-1]) + c_I = alpha_I * intent_input + intent_hidden = intent_input+c_I + if not self.remove_slot_attn: + # slot attention + h_k = einops.repeat(self.slot_linear1(input_hidden), 'b l h -> b l c h', c=input_hidden.shape[1]) + h_i = einops.repeat(self.slot_linear2(input_hidden), 'b l h -> b l c h', c=input_hidden.shape[1]).transpose(1,2) + e_S = torch.tanh(h_k + h_i) + alpha_S = torch.softmax(e_S, dim=2).squeeze(3) + alpha_S = einops.repeat(alpha_S, 'b l1 l2 -> b l1 l2 h', h=input_hidden.shape[-1]) + map_input_hidden = einops.repeat(input_hidden, 'b l h -> b l c h', c=input_hidden.shape[1]) + c_S = torch.sum(alpha_S * map_input_hidden, dim=2) + else: + c_S = input_hidden + slot_hidden = input_hidden + c_S * self.slot_gate(c_S,c_I) + encode_hidden.update_intent_hidden_state(intent_hidden) + encode_hidden.update_slot_hidden_state(slot_hidden) + return encode_hidden + +class SlotGate(nn.Module): + def __init__(self, **config): + super().__init__() + self.linear = nn.Linear(config["input_dim"], config["output_dim"],bias=False) + self.v = nn.Parameter(torch.rand(size=[1])) + + def forward(self, slot_context, intent_context): + intent_gate = self.linear(intent_context) + intent_gate = einops.repeat(intent_gate, 'b h -> b l h', l=slot_context.shape[1]) + return self.v * torch.tanh(slot_context + intent_gate) diff --git a/model/decoder/interaction/stack_interaction.py b/model/decoder/interaction/stack_interaction.py new file mode 100644 index 0000000000000000000000000000000000000000..205072236141e74c45407530b4a5642cd65cecdc --- /dev/null +++ b/model/decoder/interaction/stack_interaction.py @@ -0,0 +1,36 @@ +import os +import torch +from torch import nn + +from common import utils +from common.utils import ClassifierOutputData, HiddenData +from model.decoder.interaction.base_interaction import BaseInteraction + + +class StackInteraction(BaseInteraction): + def __init__(self, **config): + super().__init__(**config) + self.intent_embedding = nn.Embedding( + self.config["intent_label_num"], self.config["intent_label_num"] + ) + self.differentiable = config.get("differentiable") + self.intent_embedding.weight.data = torch.eye( + self.config["intent_label_num"]) + self.intent_embedding.weight.requires_grad = False + + def forward(self, intent_output: ClassifierOutputData, encode_hidden: HiddenData): + if not self.differentiable: + _, idx_intent = intent_output.classifier_output.topk(1, dim=-1) + feed_intent = self.intent_embedding(idx_intent.squeeze(2)) + else: + feed_intent = intent_output.classifier_output + encode_hidden.update_slot_hidden_state( + torch.cat([encode_hidden.get_slot_hidden_state(), feed_intent], dim=-1)) + return encode_hidden + + @staticmethod + def from_configured(configure_name_or_file="stack-interaction", **input_config): + return utils.from_configured(configure_name_or_file, + model_class=StackInteraction, + config_prefix="./config/decoder/interaction", + **input_config) diff --git a/model/encoder/__init__.py b/model/encoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0f9039cc79440b3995ebbe5e670a32426ee8765d --- /dev/null +++ b/model/encoder/__init__.py @@ -0,0 +1,5 @@ +from model.encoder.pretrained_encoder import PretrainedEncoder +from model.encoder.non_pretrained_encoder import NonPretrainedEncoder +from model.encoder.base_encoder import BiEncoder +from model.encoder.auto_encoder import AutoEncoder +__all__ = ["PretrainedEncoder", "NonPretrainedEncoder", "AutoEncoder","BiEncoder"] \ No newline at end of file diff --git a/model/encoder/auto_encoder.py b/model/encoder/auto_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..9c15f4c7f341794e68a1882704fe181b0e58f824 --- /dev/null +++ b/model/encoder/auto_encoder.py @@ -0,0 +1,37 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-18 19:33:34 +Description: + +''' +from common.utils import InputData +from model.encoder.base_encoder import BaseEncoder, BiEncoder +from model.encoder.pretrained_encoder import PretrainedEncoder +from model.encoder.non_pretrained_encoder import NonPretrainedEncoder + +class AutoEncoder(BaseEncoder): + + def __init__(self, **config): + """automatedly load encoder by 'encoder_name' + Args: + config (dict): + encoder_name (str): support ["lstm", "self-attention-lstm", "bi-encoder"] and other pretrained model in hugging face + **args (Any): other configuration items corresponding to each module. + """ + super().__init__() + self.config = config + if config.get("encoder_name"): + encoder_name = config.get("encoder_name").lower() + if encoder_name in ["lstm", "self-attention-lstm"]: + self.__encoder = NonPretrainedEncoder(**config) + elif encoder_name == "bi-encoder": + self.__encoder= BiEncoder(self.__init__(**config["intent_encoder"]), self.__init__(**config["intent_encoder"])) + else: + self.__encoder = PretrainedEncoder(**config) + else: + raise ValueError("There is no Encoder Name in config.") + + def forward(self, inputs: InputData): + return self.__encoder(inputs) \ No newline at end of file diff --git a/model/encoder/base_encoder.py b/model/encoder/base_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..6fabf34b7f50602e7c7c50e2fa863398a4753e2b --- /dev/null +++ b/model/encoder/base_encoder.py @@ -0,0 +1,41 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-01-26 17:25:17 +Description: Base encoder and bi encoder + +''' +from torch import nn + +from common.utils import InputData + + +class BaseEncoder(nn.Module): + """Base class for all encoder module + """ + def __init__(self, **config): + super().__init__() + self.config = config + NotImplementedError("no implement") + + def forward(self, inputs: InputData): + self.encoder(inputs.input_ids) + + +class BiEncoder(nn.Module): + """Bi Encoder for encode intent and slot separately + """ + def __init__(self, intent_encoder: BaseEncoder, slot_encoder: BaseEncoder, **config): + super().__init__() + self.intent_encoder = intent_encoder + self.slot_encoder = slot_encoder + + def forward(self, inputs: InputData): + hidden_slot = self.slot_encoder(inputs) + hidden_intent = self.intent_encoder(inputs) + if not self.intent_encoder.config["return_sentence_level_hidden"]: + hidden_slot.update_intent_hidden_state(hidden_intent.get_slot_hidden_state()) + else: + hidden_slot.update_intent_hidden_state(hidden_intent.get_intent_hidden_state()) + return hidden_slot diff --git a/model/encoder/non_pretrained_encoder.py b/model/encoder/non_pretrained_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..f842b4bd2ca8c5e6eb9001a03e5f46ec98650e37 --- /dev/null +++ b/model/encoder/non_pretrained_encoder.py @@ -0,0 +1,212 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-17 21:08:19 +Description: non-pretrained encoder model + +''' +import math + +import einops +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + +from common.utils import HiddenData, InputData +from model.encoder.base_encoder import BaseEncoder + +class NonPretrainedEncoder(BaseEncoder): + """ + Encoder structure based on bidirectional LSTM and self-attention. + """ + + def __init__(self, **config): + """ init non-pretrained encoder + + Args: + config (dict): + embedding (dict): + dropout_rate (float): dropout rate. + load_embedding_name (str): None if not use pretrained embedding or embedding name like "glove.6B.300d.txt". + embedding_matrix (Tensor, Optional): embedding matrix tensor. Enabled if load_embedding_name is not None. + vocab_size (str, Optional): vocabulary size. Enabled if load_embedding_name is None. + lstm (dict): + output_dim (int): lstm output dim. + bidirectional (bool): if use BiLSTM or LSTM. + layer_num (int): number of layers. + dropout_rate (float): dropout rate. + attention (dict, Optional): + dropout_rate (float): dropout rate. + hidden_dim (int): attention hidden dim. + output_dim (int): attention output dim. + unflat_attention (dict, optional): Enabled if attention is not None. + dropout_rate (float): dropout rate. + """ + super(NonPretrainedEncoder, self).__init__() + self.config = config + # Embedding Initialization + embed_config = config["embedding"] + self.__embedding_dim = embed_config["embedding_dim"] + if embed_config.get("load_embedding_name") and embed_config.get("embedding_matrix"): + self.__embedding_layer = nn.Embedding.from_pretrained(embed_config["embedding_matrix"], padding_idx=0) + else: + self.__embedding_layer = nn.Embedding( + embed_config["vocab_size"], self.__embedding_dim + ) + self.__embedding_dropout_layer = nn.Dropout(embed_config["dropout_rate"]) + + # LSTM Initialization + lstm_config = config["lstm"] + self.__hidden_size = lstm_config["output_dim"] + self.__lstm_layer = nn.LSTM( + input_size=self.__embedding_dim, + hidden_size=lstm_config["output_dim"] // 2, + batch_first=True, + bidirectional=lstm_config["bidirectional"], + dropout=lstm_config["dropout_rate"], + num_layers=lstm_config["layer_num"] + ) + if self.config.get("attention"): + # Attention Initialization + att_config = config["attention"] + self.__attention_dropout_layer = nn.Dropout(att_config["dropout_rate"]) + self.__attention_layer = QKVAttention( + self.__embedding_dim, self.__embedding_dim, self.__embedding_dim, + att_config["hidden_dim"], att_config["output_dim"], att_config["dropout_rate"] + ) + if self.config.get("unflat_attention"): + unflat_att_config = config["unflat_attention"] + self.__sentattention = UnflatSelfAttention( + lstm_config["output_dim"] + att_config["output_dim"], + unflat_att_config["dropout_rate"] + ) + + def forward(self, inputs: InputData): + """ Forward process for Non-Pretrained Encoder. + + Args: + inputs: padded input ids, masks. + Returns: + encoded hidden vectors. + """ + + # LSTM Encoder + # Padded_text should be instance of LongTensor. + embedded_text = self.__embedding_layer(inputs.input_ids) + dropout_text = self.__embedding_dropout_layer(embedded_text) + seq_lens = inputs.attention_mask.sum(-1).detach().cpu() + # Pack and Pad process for input of variable length. + packed_text = pack_padded_sequence(dropout_text, seq_lens, batch_first=True, enforce_sorted=False) + lstm_hiddens, (h_last, c_last) = self.__lstm_layer(packed_text) + padded_hiddens, _ = pad_packed_sequence(lstm_hiddens, batch_first=True) + + if self.config.get("attention"): + # Attention Encoder + dropout_text = self.__attention_dropout_layer(embedded_text) + attention_hiddens = self.__attention_layer( + dropout_text, dropout_text, dropout_text, mask=inputs.attention_mask + ) + + # Attention + LSTM + hiddens = torch.cat([attention_hiddens, padded_hiddens], dim=-1) + hidden = HiddenData(None, hiddens) + if self.config.get("return_with_input"): + hidden.add_input(inputs) + if self.config.get("return_sentence_level_hidden"): + if self.config.get("unflat_attention"): + sentence = self.__sentattention(hiddens, seq_lens) + else: + sentence = hiddens[:, 0, :] + hidden.update_intent_hidden_state(sentence) + else: + sentence_hidden = None + if self.config.get("return_sentence_level_hidden"): + sentence_hidden = torch.cat((h_last[-1], h_last[-1], c_last[-1], c_last[-2]), dim=-1) + hidden = HiddenData(sentence_hidden, padded_hiddens) + if self.config.get("return_with_input"): + hidden.add_input(inputs) + + return hidden + + +class QKVAttention(nn.Module): + """ + Attention mechanism based on Query-Key-Value architecture. And + especially, when query == key == value, it's self-attention. + """ + + def __init__(self, query_dim, key_dim, value_dim, hidden_dim, output_dim, dropout_rate): + super(QKVAttention, self).__init__() + + # Record hyper-parameters. + self.__query_dim = query_dim + self.__key_dim = key_dim + self.__value_dim = value_dim + self.__hidden_dim = hidden_dim + self.__output_dim = output_dim + self.__dropout_rate = dropout_rate + + # Declare network structures. + self.__query_layer = nn.Linear(self.__query_dim, self.__hidden_dim) + self.__key_layer = nn.Linear(self.__key_dim, self.__hidden_dim) + self.__value_layer = nn.Linear(self.__value_dim, self.__output_dim) + self.__dropout_layer = nn.Dropout(p=self.__dropout_rate) + + def forward(self, input_query, input_key, input_value, mask=None): + """ The forward propagation of attention. + + Here we require the first dimension of input key + and value are equal. + + Args: + input_query: is query tensor, (n, d_q) + input_key: is key tensor, (m, d_k) + input_value: is value tensor, (m, d_v) + + Returns: + attention based tensor, (n, d_h) + """ + + # Linear transform to fine-tune dimension. + linear_query = self.__query_layer(input_query) + linear_key = self.__key_layer(input_key) + linear_value = self.__value_layer(input_value) + + score_tensor = torch.matmul( + linear_query, + linear_key.transpose(-2, -1) + ) / math.sqrt(self.__hidden_dim) + if mask is not None: + attn_mask = einops.repeat((mask == 0), "b l -> b l h", h=score_tensor.shape[-1]) + score_tensor = score_tensor.masked_fill_(attn_mask, -float(1e20)) + score_tensor = F.softmax(score_tensor, dim=-1) + forced_tensor = torch.matmul(score_tensor, linear_value) + forced_tensor = self.__dropout_layer(forced_tensor) + + return forced_tensor + + +class UnflatSelfAttention(nn.Module): + """ + scores each element of the sequence with a linear layer and uses the normalized scores to compute a context over the sequence. + """ + + def __init__(self, d_hid, dropout=0.): + super().__init__() + self.scorer = nn.Linear(d_hid, 1) + self.dropout = nn.Dropout(dropout) + + def forward(self, inp, lens): + batch_size, seq_len, d_feat = inp.size() + inp = self.dropout(inp) + scores = self.scorer(inp.contiguous().view(-1, d_feat)).view(batch_size, seq_len) + max_len = max(lens) + for i, l in enumerate(lens): + if l < max_len: + scores.data[i, l:] = -np.inf + scores = F.softmax(scores, dim=1) + context = scores.unsqueeze(2).expand_as(inp).mul(inp).sum(1) + return context \ No newline at end of file diff --git a/model/encoder/pretrained_encoder.py b/model/encoder/pretrained_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..3003ce20658050f6471ae9c828c1c69c3bf6abca --- /dev/null +++ b/model/encoder/pretrained_encoder.py @@ -0,0 +1,44 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-02-18 17:38:30 +Description: pretrained encoder model + +''' +from transformers import AutoModel, AutoConfig +from common import utils + +from common.utils import InputData, HiddenData +from model.encoder.base_encoder import BaseEncoder + + +class PretrainedEncoder(BaseEncoder): + def __init__(self, **config): + """ init pretrained encoder + + Args: + config (dict): + encoder_name (str): pretrained model name in hugging face. + """ + super().__init__(**config) + if self.config.get("_is_check_point_"): + self.encoder = utils.instantiate(config["pretrained_model"], target="_pretrained_model_target_") + # print(self.encoder) + else: + self.encoder = AutoModel.from_pretrained(config["encoder_name"]) + + def forward(self, inputs: InputData): + output = self.encoder(**inputs.get_inputs()) + hidden = HiddenData(None, output.last_hidden_state) + if self.config.get("return_with_input"): + hidden.add_input(inputs) + if self.config.get("return_sentence_level_hidden"): + padding_side = self.config.get("padding_side") + if hasattr(output, "pooler_output"): + hidden.update_intent_hidden_state(output.pooler_output) + elif padding_side is not None and padding_side == "left": + hidden.update_intent_hidden_state(output.last_hidden_state[:, -1]) + else: + hidden.update_intent_hidden_state(output.last_hidden_state[:, 0]) + return hidden diff --git a/model/open_slu_model.py b/model/open_slu_model.py new file mode 100644 index 0000000000000000000000000000000000000000..342c98968f58dd42ead03157a8309044c18aa04e --- /dev/null +++ b/model/open_slu_model.py @@ -0,0 +1,64 @@ +''' +Author: Qiguang Chen +Date: 2023-01-11 10:39:26 +LastEditors: Qiguang Chen +LastEditTime: 2023-01-26 17:18:22 +Description: Root Model Module + +''' +from torch import nn + +from common.utils import OutputData, InputData +from model.decoder.base_decoder import BaseDecoder +from model.encoder.base_encoder import BaseEncoder + + +class OpenSLUModel(nn.Module): + def __init__(self, encoder: BaseEncoder, decoder:BaseDecoder, **config): + """Create model automatedly + + Args: + encoder (BaseEncoder): encoder created by config + decoder (BaseDecoder): decoder created by config + config (dict): any other args + """ + super().__init__() + self.encoder = encoder + self.decoder = decoder + self.config = config + + def forward(self, inp: InputData) -> OutputData: + """ model forward + + Args: + inp (InputData): input ids and other information + + Returns: + OutputData: pred logits + """ + return self.decoder(self.encoder(inp)) + + def decode(self, output: OutputData, target: InputData=None): + """ decode output + + Args: + pred (OutputData): pred logits data + target (InputData): golden data + + Returns: decoded ids + """ + return self.decoder.decode(output, target) + + def compute_loss(self, pred: OutputData, target: InputData, compute_intent_loss=True, compute_slot_loss=True): + """ compute loss + + Args: + pred (OutputData): pred logits data + target (InputData): golden data + compute_intent_loss (bool, optional): whether to compute intent loss. Defaults to True. + compute_slot_loss (bool, optional): whether to compute slot loss. Defaults to True. + + Returns: loss value + """ + return self.decoder.compute_loss(pred, target, compute_intent_loss=compute_intent_loss, + compute_slot_loss=compute_slot_loss) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..da5624771ffd3f5386c2e5b6d3b32ebd5b0b074d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +torch +transformers==4.25.1 +ruamel-yaml==0.17.21 +accelerate==0.13.2 +dill==0.3.6 +einops==0.6.0 +wandb==0.13.8 +scikit-learn==1.2.0 +pytorch-crf==0.7.2 +ordered-set==4.1.0 +gradio==3.16.2 +flask==2.2.2 +datasets==2.8.0 +colorlog==6.7.0 \ No newline at end of file diff --git a/save/stack/label.json b/save/stack/label.json new file mode 100644 index 0000000000000000000000000000000000000000..ffa99c89b5e0e74872b92884d8988b37aed4d4c0 --- /dev/null +++ b/save/stack/label.json @@ -0,0 +1 @@ +{"intent": ["atis_flight", "atis_airfare", "atis_airline", "atis_ground_service", "atis_quantity", "atis_city", "atis_flight#atis_airfare", "atis_abbreviation", "atis_aircraft", "atis_distance", "atis_ground_fare", "atis_capacity", "atis_flight_time", "atis_meal", "atis_aircraft#atis_flight#atis_flight_no", "atis_flight_no", "atis_restriction", "atis_airport", "atis_airline#atis_flight_no", "atis_cheapest", "atis_ground_service#atis_ground_fare"], "slot": ["O", "B-fromloc.city_name", "B-toloc.city_name", "B-round_trip", "I-round_trip", "B-cost_relative", "B-fare_amount", "I-fare_amount", "B-arrive_date.month_name", "B-arrive_date.day_number", "I-fromloc.city_name", "B-stoploc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time", "B-toloc.state_code", "I-toloc.city_name", "I-stoploc.city_name", "B-meal_description", "B-depart_date.month_name", "B-depart_date.day_number", "B-airline_name", "I-airline_name", "B-depart_time.period_of_day", "B-depart_date.day_name", "B-toloc.state_name", "B-depart_time.time_relative", "B-depart_time.time", "B-toloc.airport_name", "I-toloc.airport_name", "B-depart_date.date_relative", "B-or", "B-airline_code", "B-class_type", "I-class_type", "I-cost_relative", "I-depart_time.time", "B-fromloc.airport_name", "I-fromloc.airport_name", "B-city_name", "B-flight_mod", "B-meal", "B-economy", "B-fare_basis_code", "I-depart_date.day_number", "B-depart_date.today_relative", "B-flight_stop", "B-airport_code", "B-fromloc.state_name", "I-fromloc.state_name", "I-city_name", "B-connect", "B-arrive_date.day_name", "B-fromloc.state_code", "B-arrive_date.today_relative", "B-depart_date.year", "B-depart_time.start_time", "I-depart_time.start_time", "B-depart_time.end_time", "I-depart_time.end_time", "B-arrive_time.start_time", "B-arrive_time.end_time", "I-arrive_time.end_time", "I-flight_mod", "B-flight_days", "B-mod", "B-flight_number", "I-toloc.state_name", "B-meal_code", "I-meal_code", "B-airport_name", "I-airport_name", "I-flight_stop", "B-transport_type", "I-transport_type", "B-state_code", "B-aircraft_code", "B-toloc.country_name", "I-arrive_date.day_number", "B-toloc.airport_code", "B-return_date.date_relative", "I-return_date.date_relative", "B-flight_time", "I-economy", "B-fromloc.airport_code", "B-arrive_time.period_of_day", "B-depart_time.period_mod", "I-flight_time", "B-return_date.day_name", "B-arrive_date.date_relative", "B-restriction_code", "I-restriction_code", "B-arrive_time.period_mod", "I-arrive_time.period_of_day", "B-period_of_day", "B-stoploc.state_code", "I-depart_date.today_relative", "I-fare_basis_code", "I-arrive_time.start_time", "B-time", "B-today_relative", "I-today_relative", "B-state_name", "B-days_code", "I-depart_time.period_of_day", "I-arrive_time.time_relative", "B-time_relative", "I-time", "B-return_date.month_name", "B-return_date.day_number", "I-depart_time.time_relative", "B-stoploc.airport_name", "B-day_name", "B-month_name", "B-day_number", "B-return_time.period_mod", "B-return_time.period_of_day", "B-return_date.today_relative", "I-return_date.today_relative", "I-meal_description"]} \ No newline at end of file diff --git a/save/stack/model.pkl b/save/stack/model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d5df9437eccce7b892438866d375928a90bc3a4f --- /dev/null +++ b/save/stack/model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9710de3d7d5c8a34fe55ef4dc36dc8a851863d1fb3bb14871d914a4e945c96ef +size 5793644 diff --git a/save/stack/outputs.jsonl b/save/stack/outputs.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3a89f32e47cd5c27584af95ab2b26cb3f3c88b98 --- /dev/null +++ b/save/stack/outputs.jsonl @@ -0,0 +1,893 @@ +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "to", "find", "a", "flight", "from", "charlotte", "to", "las", "vegas", "that", "makes", "a", "stop", "in", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["on", "april", "first", "i", "need", "a", "ticket", "from", "tacoma", "to", "san", "jose", "departing", "before", "7", "am"], "golden_intent": "atis_airfare", "golden_slot": ["O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["on", "april", "first", "i", "need", "a", "flight", "going", "from", "phoenix", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "traveling", "one", "way", "from", "phoenix", "to", "san", "diego", "on", "april", "first"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "orlando", "to", "salt", "lake", "city", "for", "april", "first", "on", "delta", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "toronto", "to", "newark", "one", "way", "leaving", "wednesday", "evening", "or", "thursday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["monday", "morning", "i", "would", "like", "to", "fly", "from", "columbus", "to", "indianapolis"], "golden_intent": "atis_flight", "golden_slot": ["B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["on", "wednesday", "april", "sixth", "i", "would", "like", "to", "fly", "from", "long", "beach", "to", "columbus", "after", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["after", "12", "pm", "on", "wednesday", "april", "sixth", "i", "would", "like", "to", "fly", "from", "long", "beach", "to", "columbus"], "golden_intent": "atis_flight", "golden_slot": ["B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["are", "there", "any", "flights", "from", "long", "beach", "to", "columbus", "on", "wednesday", "april", "sixth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["find", "a", "flight", "from", "memphis", "to", "tacoma", "dinner"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-meal_description"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["on", "next", "wednesday", "flight", "from", "kansas", "city", "to", "chicago", "should", "arrive", "in", "chicago", "around", "7", "pm", "return", "flight", "on", "thursday"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time", "O", "O", "O", "B-return_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "flight", "and", "prices", "kansas", "city", "to", "chicago", "on", "next", "wednesday", "arriving", "in", "chicago", "by", "7", "pm"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["flight", "on", "american", "from", "miami", "to", "chicago", "arrive", "in", "chicago", "about", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "I-toloc.city_name", "O"], "text": ["find", "flights", "arriving", "new", "york", "city", "next", "saturday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "B-arrive_date.date_relative", "B-arrive_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["find", "nonstop", "flights", "from", "salt", "lake", "city", "to", "new", "york", "on", "saturday", "april", "ninth"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O"], "text": ["show", "flights", "from", "burbank", "to", "milwaukee", "for", "today"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.today_relative"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "flights", "tomorrow", "evening", "from", "milwaukee", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-depart_date.today_relative", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "saturday", "evening", "from", "st.", "louis", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "burbank", "to", "st.", "louis", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "burbank", "to", "milwaukee", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "flights", "tuesday", "evening", "from", "milwaukee", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "wednesday", "evening", "from", "st.", "louis", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "travel", "from", "kansas", "city", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "flights", "travel", "from", "las", "vegas", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "travel", "from", "kansas", "city", "to", "los", "angeles", "on", "april", "ninth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "travel", "from", "las", "vegas", "to", "los", "angeles", "california", "and", "arrive", "on", "april", "ninth", "between", "4", "and", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name", "O", "O", "O", "B-arrive_date.month_name", "B-arrive_date.day_number", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "flights", "on", "us", "air", "go", "from", "orlando", "to", "cleveland"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "travel", "from", "cleveland", "to", "indianapolis", "on", "april", "fifth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O"], "text": ["which", "flights", "travel", "from", "indianapolis", "to", "san", "diego", "on", "april", "fifth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "go", "from", "cleveland", "to", "indianapolis", "on", "april", "fifth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "flights", "travel", "from", "nashville", "to", "tacoma"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["does", "tacoma", "airport", "offer", "transportation", "from", "the", "airport", "to", "the", "downtown", "area"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "B-airport_name", "I-airport_name", "O", "O", "O", "O", "O", "O", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "travel", "from", "tacoma", "to", "san", "jose"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O"], "text": ["what", "day", "of", "the", "week", "do", "flights", "from", "nashville", "to", "tacoma", "fly", "on"], "golden_intent": "atis_day_name", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "tacoma", "to", "san", "jose"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O"], "text": ["what", "days", "of", "the", "week", "do", "flights", "from", "san", "jose", "to", "nashville", "fly", "on"], "golden_intent": "atis_day_name", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "tacoma", "to", "san", "jose"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "that", "goes", "from", "boston", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["are", "there", "any", "flights", "from", "boston", "to", "orlando", "connecting", "in", "new", "york"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-connect", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["are", "there", "any", "flights", "from", "boston", "to", "orlando", "connecting", "in", "columbus"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-connect", "O", "B-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name"], "text": ["does", "this", "flight", "serve", "dinner"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-meal_description"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "charlotte", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "nonstop", "flight", "from", "miami", "to", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "nonstop", "flight", "from", "toronto", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "toronto", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "st.", "louis", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "on", "united", "airlines", "from", "la", "guardia", "to", "san", "jose"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "tampa", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "milwaukee", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "meals", "are", "served", "on", "american", "flight", "811", "from", "tampa", "to", "milwaukee"], "golden_intent": "atis_meal", "golden_slot": ["O", "B-meal", "O", "O", "O", "B-airline_name", "O", "B-flight_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "meals", "are", "served", "on", "american", "flight", "665", "673", "from", "milwaukee", "to", "seattle"], "golden_intent": "atis_meal", "golden_slot": ["O", "B-meal", "O", "O", "O", "B-airline_name", "O", "B-flight_number", "I-flight_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "memphis", "to", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "las", "vegas", "to", "ontario"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "ontario", "to", "memphis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "go", "from", "milwaukee", "to", "tampa", "and", "stop", "in", "nashville"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "leave", "newark", "after", "noon", "next", "saturday", "and", "arrive", "in", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-depart_time.time_relative", "B-depart_time.period_of_day", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "leave", "detroit", "and", "arrive", "at", "st.", "petersburg", "around", "9", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "on", "northwest", "airline", "leave", "detroit", "and", "arrive", "at", "st.", "petersburg"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "flights", "leave", "chicago", "next", "tuesday", "and", "arrive", "in", "detroit", "around", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["show", "me", "round", "trip", "flights", "from", "chicago", "to", "detroit", "leaving", "next", "tuesday", "and", "returning", "the", "day", "after"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "O", "B-return_date.date_relative", "I-return_date.date_relative"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "round", "trip", "flights", "leave", "chicago", "next", "tuesday", "around", "6", "pm", "and", "arrive", "in", "detroit"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "round", "trip", "flights", "leave", "chicago", "next", "tuesday", "and", "arrive", "in", "detroit", "around", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "leave", "on", "monday", "from", "montreal", "and", "arrive", "in", "chicago", "in", "the", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_date.day_name", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "leave", "chicago", "on", "april", "twelfth", "and", "arrive", "in", "indianapolis", "in", "the", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "leave", "on", "wednesday", "april", "thirteenth", "from", "indianapolis", "and", "arrive", "in", "montreal", "in", "the", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "leave", "april", "twelfth", "from", "indianapolis", "and", "arrive", "in", "montreal", "around", "10", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["i'd", "like", "to", "go", "from", "long", "beach", "to", "st.", "louis", "and", "i'd", "like", "to", "stop", "in", "dallas", "i'd", "also", "like", "to", "have", "lunch", "during", "my", "flight"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-stoploc.city_name", "O", "O", "O", "O", "O", "B-meal_description", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["next", "wednesday", "i", "would", "like", "to", "leave", "kansas", "city", "on", "a", "trip", "to", "chicago", "which", "arrives", "in", "chicago", "around", "7", "pm"], "golden_intent": "atis_flight", "golden_slot": ["B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "to", "return", "from", "chicago", "around", "7", "pm", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "to", "leave", "this", "afternoon", "on", "an", "american", "flight", "from", "cincinnati", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.today_relative", "B-depart_time.period_of_day", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["on", "sunday", "evening", "i", "would", "like", "to", "leave", "montreal", "quebec", "on", "a", "flight", "to", "san", "diego", "california"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "on", "sunday", "from", "montreal", "quebec", "to", "san", "diego", "california"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.day_name", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["on", "tuesday", "are", "the", "flights", "from", "san", "diego", "california", "to", "indianapolis", "indiana", "i", "would", "like", "the", "flight", "to", "be", "in", "the", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "B-toloc.state_name", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["on", "thursday", "morning", "i", "would", "like", "a", "nonstop", "flight", "from", "indianapolis", "to", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["on", "friday", "morning", "i", "would", "like", "to", "fly", "from", "toronto", "to", "montreal"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "an", "early", "morning", "flight", "today", "from", "los", "angeles", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_time.period_of_day", "B-depart_time.period_of_day", "O", "B-depart_date.today_relative", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["on", "wednesday", "night", "i", "would", "like", "a", "flight", "from", "charlotte", "to", "newark"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["on", "friday", "night", "i", "would", "like", "a", "flight", "from", "newark", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["find", "a", "flight", "from", "tampa", "to", "montreal", "by", "way", "of", "new", "york"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "miami", "florida", "to", "las", "vegas", "nevada", "arriving", "before", "4", "o'clock", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "las", "vegas", "to", "michigan"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "detroit", "michigan", "to", "st.", "petersburg", "arriving", "before", "10", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "st.", "petersburg", "to", "miami", "on", "thursday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "san", "diego", "to", "toronto", "on", "alaska", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "find", "the", "flights", "from", "columbus", "to", "houston", "with", "a", "layover", "in", "nashville", "tomorrow"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-stoploc.city_name", "B-depart_date.today_relative"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O"], "text": ["please", "give", "me", "the", "flights", "from", "nashville", "to", "houston", "nonstop", "with", "dinner", "served"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-flight_stop", "O", "B-meal_description", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "find", "flights", "available", "from", "kansas", "city", "to", "newark"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "find", "a", "flight", "that", "goes", "from", "kansas", "city", "to", "newark", "to", "orlando", "back", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "kansas", "city", "to", "newark"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "newark", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "find", "a", "flight", "from", "orlando", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "to", "fly", "from", "columbus", "to", "phoenix", "through", "cincinnati", "in", "the", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-stoploc.city_name", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["i", "would", "like", "to", "know", "what", "airports", "are", "in", "los", "angeles"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name"], "text": ["does", "the", "airport", "at", "burbank", "have", "a", "flight", "that", "comes", "in", "from", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "arrive", "in", "burbank", "from", "kansas", "city", "on", "saturdays", "in", "the", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-arrive_date.day_name", "O", "O", "B-arrive_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "arrive", "in", "burbank", "from", "las", "vegas", "on", "saturday", "april", "twenty", "third", "in", "the", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "are", "available", "from", "orlando", "to", "cleveland", "that", "arrive", "around", "10", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-depart_time.period_of_day"], "text": ["what", "flights", "are", "available", "from", "indianapolis", "to", "san", "diego", "on", "april", "twenty", "first", "in", "the", "late", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "O", "O", "B-depart_time.period_of_day", "I-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["what", "flights", "leave", "cleveland", "going", "to", "indianapolis", "on", "april", "twenty", "first", "in", "the", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "flights", "are", "available", "on", "april", "twenty", "first", "in", "the", "morning", "from", "nashville", "to", "tacoma"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "O", "O", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "are", "available", "from", "tacoma", "to", "san", "jose", "in", "the", "morning", "on", "april", "twenty", "second"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_time.period_of_day", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["which", "flights", "are", "available", "from", "san", "jose", "to", "nashville", "leaving", "in", "the", "morning", "on", "april", "twenty", "three"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-depart_time.period_of_day", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "most", "expensive", "one", "way", "fare", "between", "detroit", "and", "westchester", "county"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "I-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "airlines", "fly", "between", "detroit", "and", "westchester", "county"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "departure", "times", "from", "detroit", "to", "westchester", "county"], "golden_intent": "atis_flight_time", "golden_slot": ["O", "O", "O", "B-flight_time", "I-flight_time", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "latest", "flight", "from", "baltimore", "to", "oakland", "that", "serves", "dinner"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-meal_description"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O"], "text": ["what", "is", "the", "earliest", "flight", "between", "baltimore", "and", "oakland", "that", "serves", "breakfast"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-meal_description"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name"], "text": ["to", "what", "cities", "from", "boston", "does", "america", "west", "fly", "first", "class"], "golden_intent": "atis_city", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-airline_name", "I-airline_name", "O", "B-class_type", "I-class_type"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "airline", "flies", "from", "boston", "to", "san", "diego"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "latest", "breakfast", "flight", "from", "dallas", "to", "tampa"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "B-meal_description", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "lufthansa", "flights", "from", "seattle", "to", "boston", "with", "stopovers", "in", "minneapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "flights", "from", "seattle", "to", "boston", "with", "stopovers", "in", "minneapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["list", "philadelphia", "to", "san", "francisco", "flights", "with", "stopovers", "in", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "B-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O"], "text": ["show", "me", "the", "connecting", "flights", "between", "boston", "and", "denver", "and", "the", "types", "of", "aircraft", "used"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-connect", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "the", "morning", "flights", "from", "philadelphia", "to", "fort", "worth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "kansas", "city", "to", "st.", "paul"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "northwest", "flight", "608", "from", "kansas", "city", "to", "st.", "paul"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "O", "B-flight_number", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "indianapolis", "to", "charlotte", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "ground", "transportation", "between", "the", "charlotte", "airport", "charlotte", "airport", "and", "downtown", "charlotte"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "B-city_name", "I-city_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "charlotte", "to", "minneapolis", "that", "leave", "at", "2", "pm", "or", "later", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-depart_time.time", "I-depart_time.time", "O", "B-depart_time.time_relative", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "charlotte", "to", "minneapolis", "on", "tuesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "direct", "flights", "from", "charlotte", "to", "minneapolis", "on", "tuesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-connect", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["show", "me", "flight", "us", "1500", "on", "monday", "from", "charlotte", "to", "minneapolis", "please"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_code", "B-flight_number", "O", "B-depart_date.day_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "minneapolis", "to", "indianapolis", "on", "tuesday", "that", "leave", "after", "2", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "from", "minneapolis", "to", "indiana"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "flights", "in", "from", "minneapolis", "to", "indianapolis", "on", "tuesday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "flight", "us", "1207", "from", "indianapolis", "to", "charlotte", "on", "monday", "and", "flight", "us", "1500", "from", "charlotte", "to", "minneapolis", "on", "monday", "and", "flight", "twa", "639", "from", "minneapolis", "to", "indianapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_code", "B-flight_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "O", "O", "B-flight_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "O", "B-airline_code", "B-flight_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "las", "vegas", "to", "new", "york", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "different", "airlines", "go", "from", "las", "vegas", "to", "new", "york", "city"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "on", "america", "west", "and", "twa", "from", "las", "vegas", "to", "jfk", "on", "a", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-airline_code", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.airport_code", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "tacoma", "to", "miami", "that", "leave", "after", "6", "pm", "tomorrow"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "B-depart_date.today_relative"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "to", "fly", "from", "san", "diego", "to", "houston", "on", "june", "tenth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["is", "there", "an", "american", "airlines", "flight", "from", "houston", "to", "newark", "on", "june", "tenth", "after", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["is", "there", "an", "american", "airlines", "flight", "from", "houston", "to", "newark", "on", "june", "tenth", "after", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "to", "get", "from", "cincinnati", "to", "denver", "on", "june", "sixth", "by", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-arrive_date.month_name", "B-arrive_date.day_number", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what's", "the", "ground", "transportation", "in", "denver"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what's", "the", "fare", "for", "a", "taxi", "to", "denver"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "O", "O", "O", "O", "B-transport_type", "O", "B-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "are", "the", "fares", "for", "ground", "transportation", "in", "denver"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "to", "fly", "from", "denver", "to", "westchester", "county", "on", "june", "seventh", "after", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what's", "the", "ground", "transportation", "in", "westchester", "county"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "to", "take", "a", "united", "airlines", "flight", "on", "june", "eighth", "from", "westchester", "county", "to", "cincinnati", "after", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "united", "airlines", "flights", "on", "june", "eighth", "go", "from", "westchester", "county", "to", "cincinnati"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["on", "june", "eighth", "what", "flights", "go", "from", "westchester", "county", "to", "cincinnati"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["does", "us", "air", "fly", "from", "cincinnati", "to", "denver", "on", "june", "sixth"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "cincinnati", "to", "denver", "on", "june", "sixth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "denver", "to", "westchester", "county", "on", "june", "seventh"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["list", "the", "flights", "from", "westchester", "county", "to", "cincinnati", "on", "june", "eighth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "cincinnati", "to", "westchester", "county", "on", "june", "sixth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "westchester", "county", "to", "denver", "on", "june", "seventh"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["list", "the", "flights", "from", "denver", "to", "cincinnati", "on", "june", "eighth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "denver", "to", "cincinnati", "on", "june", "sixth", "after", "4", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "cincinnati", "to", "westchester", "county", "on", "june", "seventh"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "westchester", "county", "to", "cincinnati", "on", "june", "seventh", "leaving", "after", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "airlines", "off", "from", "love", "field", "between", "6", "and", "10", "am", "on", "june", "sixth"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-depart_time.start_time", "O", "B-depart_time.end_time", "I-depart_time.end_time", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "flights", "arrive", "at", "love", "field", "on", "june", "sixth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-toloc.airport_name", "I-toloc.airport_name", "O", "B-arrive_date.month_name", "B-arrive_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "montreal", "to", "philly", "before", "9", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "the", "flights", "from", "cleveland", "to", "memphis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "the", "flights", "from", "memphis", "to", "cleveland"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "denver", "to", "baltimore", "arriving", "on", "july", "first"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_date.month_name", "B-arrive_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "dallas", "to", "baltimore", "arriving", "july", "first"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-arrive_date.month_name", "B-arrive_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "pittsburgh", "to", "baltimore", "arriving", "on", "july", "first"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_date.month_name", "B-arrive_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O"], "text": ["list", "the", "flights", "on", "canadian", "airlines", "international"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "I-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "canadian", "airlines", "international", "flights", "use", "j31"], "golden_intent": "atis_quantity", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "I-airline_name", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "canadian", "airlines", "international", "flights", "use", "aircraft", "320"], "golden_intent": "atis_quantity", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "I-airline_name", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "canadian", "airlines", "flights", "use", "aircraft", "dh8"], "golden_intent": "atis_quantity", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "flights", "on", "american", "airlines", "which", "fly", "from", "st.", "petersburg", "to", "ontario", "canada", "with", "a", "stopover", "in", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.country_name", "O", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["show", "me", "the", "flights", "on", "american", "airlines", "which", "go", "from", "st.", "petersburg", "to", "ontario", "california", "by", "way", "of", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["which", "airport", "is", "closest", "to", "ontario", "california"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-mod", "O", "B-city_name", "B-state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "from", "houston", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "from", "orlando", "to", "houston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "flights", "from", "detroit", "to", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "round", "trip", "coach", "fare", "from", "las", "vegas", "to", "detroit"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "B-class_type", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "round", "trip", "coach", "fare", "on", "twa", "from", "las", "vegas", "to", "detroit"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "B-class_type", "O", "O", "B-airline_code", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "delta", "flights", "which", "serve", "a", "snack", "to", "coach", "passengers"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "O", "O", "O", "O", "B-meal_description", "O", "B-compartment", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "is", "meal", "code", "sb"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-meal_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "meals", "are", "available", "on", "dl", "468", "which", "al", "arrives", "in", "san", "francisco", "at", "950", "am"], "golden_intent": "atis_meal", "golden_slot": ["O", "B-meal", "O", "O", "O", "B-airline_code", "B-flight_number", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "delta", "flights", "from", "tampa", "to", "san", "francisco"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name"], "text": ["show", "me", "delta", "flight", "486"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "O", "B-flight_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "the", "tower", "air", "flights", "on", "mondays"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "all", "tower", "air", "flights", "with", "meals"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-meal"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name"], "text": ["what", "flights", "depart", "from", "baltimore"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_date.day_name"], "text": ["what", "flights", "depart", "from", "baltimore", "and", "arrive", "at", "san", "francisco", "on", "a", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-arrive_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["what", "flights", "leave", "from", "cincinnati", "in", "the", "morning", "and", "arrive", "in", "tampa"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-depart_time.period_of_day", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "depart", "from", "tampa", "and", "arrive", "in", "cincinnati", "in", "the", "evening"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "depart", "from", "tampa", "in", "the", "early", "evening", "and", "arrive", "in", "cincinnati"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-depart_time.period_of_day", "B-depart_time.period_of_day", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "depart", "from", "philadelphia", "and", "arrive", "in", "atlanta"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["which", "flights", "depart", "from", "a", "atlanta", "and", "arrive", "in", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "depart", "from", "toronto", "and", "arrive", "in", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "depart", "from", "new", "york", "and", "arrive", "in", "los", "angeles", "after", "10", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "far", "is", "new", "york's", "la", "guardia", "from", "downtown"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "B-city_name", "I-city_name", "B-airport_name", "I-airport_name", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["how", "far", "is", "toronto", "international", "from", "downtown"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "B-airport_name", "I-airport_name", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "far", "is", "los", "angeles", "international", "from", "downtown"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "B-airport_name", "I-airport_name", "I-airport_name", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-fromloc.city_name"], "text": ["how", "far", "is", "san", "francisco", "international", "from", "downtown"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "B-airport_name", "I-airport_name", "I-airport_name", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["how", "much", "is", "the", "limousine", "service", "in", "boston"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "O", "O", "O", "B-transport_type", "O", "O", "B-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O"], "text": ["how", "much", "is", "a", "limousine", "service", "in", "la", "guardia"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "O", "O", "O", "B-transport_type", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "much", "is", "a", "limousine", "service", "in", "toronto", "international"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "O", "O", "O", "B-transport_type", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "much", "is", "limousine", "service", "in", "los", "angeles"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "O", "O", "B-transport_type", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "airlines", "fly", "between", "washington", "dc", "and", "columbus", "ohio"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "flights", "are", "there", "between", "washington", "dc", "and", "columbus", "ohio"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "between", "washington", "dc", "and", "columbus", "ohio"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "fares", "for", "all", "flights", "between", "washington", "and", "columbus"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["at", "the", "charlotte", "airport", "how", "many", "different", "types", "of", "aircraft", "are", "there", "for", "us", "air"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "B-city_name", "I-city_name", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "us", "air", "flights", "arriving", "in", "charlotte", "on", "saturday", "at", "1", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_date.day_name", "O", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "first", "class", "round", "trip", "airfare", "from", "india", "indianapolis", "to", "memphis"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-class_type", "I-class_type", "B-round_trip", "I-round_trip", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "all", "flights", "from", "memphis", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "and", "their", "fares", "from", "indianapolis", "to", "memphis", "on", "a", "monday", "morning"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "and", "their", "fares", "from", "memphis", "to", "miami", "on", "a", "wednesday", "evening"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "all", "flights", "and", "their", "fares", "for", "all", "flights", "between", "miami", "and", "indianapolis"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "all", "flights", "from", "cleveland", "to", "kansas", "city", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "all", "flights", "from", "kansas", "city", "to", "cleveland"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "all", "flights", "from", "cleveland", "to", "nashville"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "all", "flights", "from", "nashville", "to", "cleveland", "on", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["list", "all", "sunday", "flights", "from", "cleveland", "to", "nashville", "and", "their", "fares"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["what", "airlines", "are", "departing", "from", "baltimore"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "airlines", "fly", "from", "baltimore", "to", "san", "francisco"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["how", "long", "does", "a", "flight", "from", "baltimore", "to", "san", "francisco", "take"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["which", "flights", "are", "leaving", "from", "kansas", "city", "to", "atlanta", "early", "monday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.period_mod", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "flights", "are", "leaving", "atlanta", "and", "arriving", "in", "st.", "louis", "close", "to", "230", "pm", "on", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "B-arrive_time.time_relative", "O", "B-arrive_time.time", "I-arrive_time.time", "O", "B-arrive_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "flights", "from", "st.", "louis", "to", "st.", "paul", "which", "depart", "after", "10", "am", "thursday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "O", "O", "O", "O"], "text": ["list", "flights", "from", "st.", "paul", "to", "kansas", "city", "friday", "in", "the", "evening", "with", "a", "meal", "included"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-meal", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "early", "morning", "flights", "from", "cincinnati", "to", "tampa"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_time.period_of_day", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "early", "evening", "flights", "from", "tampa", "to", "cincinnati"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_time.period_of_day", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "evening", "flights", "from", "tampa", "to", "cincinnati"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "philadelphia", "to", "atlanta", "friday", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "atlanta", "to", "toronto", "friday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "toronto", "to", "washington", "dc", "saturday", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "philadelphia", "saturday", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "direct", "flights", "from", "new", "york", "city", "to", "los", "angeles", "after", "10", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-connect", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "the", "airfare", "for", "american", "airlines", "flight", "19", "from", "jfk", "to", "lax"], "golden_intent": "atis_airfare#atis_flight", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-flight_number", "O", "B-fromloc.airport_code", "O", "B-toloc.airport_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "is", "fare", "code", "m"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["list", "the", "distance", "in", "miles", "from", "boston", "airport", "to", "downtown", "boston"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "distance", "in", "miles", "from", "new", "york's", "la", "guardia", "airport", "to", "downtown", "new", "york", "city"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.airport_name", "I-fromloc.airport_name", "I-fromloc.airport_name", "O", "O", "B-city_name", "I-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["list", "the", "distance", "in", "miles", "from", "toronto", "international", "airport", "to", "downtown", "toronto"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "I-fromloc.airport_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["list", "the", "distance", "in", "miles", "from", "san", "francisco", "international", "airport", "to", "san", "francisco", "downtown"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "I-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "limousine", "rates", "for", "the", "city", "of", "boston"], "golden_intent": "atis_ground_fare", "golden_slot": ["O", "B-transport_type", "O", "O", "O", "B-city_name", "I-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "american", "airlines", "flights", "from", "houston", "to", "milwaukee", "departing", "friday", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "houston", "to", "milwaukee", "friday", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "american", "airlines", "flights", "from", "milwaukee", "to", "san", "jose", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "american", "airlines", "flights", "from", "san", "jose", "to", "dallas", "friday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "dallas", "to", "houston", "arriving", "sunday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-arrive_date.day_name", "B-arrive_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "airlines", "flying", "from", "seattle", "to", "salt", "lake", "city"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "for", "aircraft", "l10"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "for", "delta", "be1"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airline_name", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "seattle", "to", "salt", "lake", "city", "on", "delta", "l10"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-airline_name", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "seattle", "to", "salt", "lake", "city", "on", "delta", "be1"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-airline_name", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "boston", "to", "pittsburgh", "daily"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-flight_days"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "pittsburgh", "to", "newark", "daily"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-flight_days"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "newark", "to", "boston", "daily"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-flight_days"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "us", "air", "flights", "leaving", "saturday", "from", "charlotte", "airport", "at", "1", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-depart_date.day_name", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "us", "air", "flights", "leaving", "saturday", "from", "charlotte", "airport", "around", "1", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-depart_date.day_name", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "first", "class", "airfare", "round", "trip", "from", "indianapolis", "to", "memphis"], "golden_intent": "atis_airfare", "golden_slot": ["O", "B-class_type", "I-class_type", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "is", "fare", "code", "f"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "memphis", "to", "miami", "wednesday", "evening"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "miami", "to", "indianapolis", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "ontario", "california", "to", "orlando", "florida"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "ontario", "california", "to", "salt", "lake", "city", "utah"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "ontario", "california", "to", "salt", "lake", "city", "utah", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "salt", "lake", "city", "utah", "to", "phoenix", "arizona", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "B-toloc.state_name", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "salt", "lake", "city", "to", "phoenix", "arizona", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "phoenix", "arizona", "to", "ontario", "california", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "B-toloc.state_name", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "airlines", "fly", "from", "baltimore", "to", "san", "francisco"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "fare", "for", "a", "first", "class", "round", "trip", "ticket", "from", "detroit", "to", "st.", "petersburg"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-class_type", "I-class_type", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "airfare", "for", "a", "round", "trip", "first", "class", "ticket", "from", "detroit", "to", "st.", "petersburg"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-round_trip", "I-round_trip", "B-class_type", "I-class_type", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O"], "text": ["kansas", "city", "to", "atlanta", "monday", "morning", "flights"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["monday", "morning", "flights", "from", "atlanta", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O"], "text": ["kansas", "city", "to", "atlanta", "monday", "morning", "flights"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O"], "text": ["atlanta", "to", "st.", "louis", "tuesday", "before", "230", "pm", "flights"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["st.", "louis", "to", "st.", "paul", "thursday", "after", "10", "am"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["st.", "paul", "to", "kansas", "city", "friday", "night"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["cleveland", "to", "kansas", "city", "arrive", "monday", "before", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-arrive_date.day_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["kansas", "city", "to", "cleveland", "flight", "arrive", "wednesday", "before", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_date.day_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["cleveland", "to", "nashville", "flight", "friday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["nashville", "to", "cleveland", "sunday", "before", "9"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["B-fromloc.city_name", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["first", "class", "flights", "pittsburgh", "to", "newark", "monday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["B-class_type", "I-class_type", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["flights", "newark", "to", "los", "angeles", "wednesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["los", "angeles", "to", "minneapolis", "thursday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O"], "text": ["minneapolis", "to", "pittsburgh", "flight"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O"], "text": ["minneapolis", "to", "pittsburgh", "first", "class", "flight"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-class_type", "I-class_type", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "flights", "leaving", "from", "milwaukee", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "does", "hp", "stand", "for"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-airline_code", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "ontario", "to", "tacoma"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "minneapolis", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "salt", "lake", "city", "to", "cincinnati"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "to", "see", "flights", "from", "cincinnati", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i'd", "like", "flights", "from", "new", "york", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "miami", "to", "new", "york"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "leaving", "san", "francisco", "for", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "san", "diego", "to", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "san", "diego", "to", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "las", "vegas", "to", "san", "francisco"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "bn", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "to", "have", "the", "airline", "that", "flies", "toronto", "detroit", "and", "st.", "louis"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "toronto", "to", "detroit"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "detroit", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "toronto", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "flights", "from", "san", "francisco", "to", "long", "beach"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "flights", "leaving", "san", "francisco", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "san", "francisco", "to", "st.", "petersburg"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "a", "one", "way", "flight", "from", "milwaukee", "to", "orlando", "leaving", "wednesday", "afternoon", "after", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "one", "way", "flights", "from", "milwaukee", "to", "orlando", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "flights", "from", "columbus", "to", "chicago", "first", "class", "that", "leave", "before", "10", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-class_type", "I-class_type", "O", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "round", "trip", "between", "st.", "petersburg", "and", "detroit", "that", "arrives", "before", "7", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "nonstop", "flights", "from", "kansas", "city", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "is", "airline", "wn"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "first", "class", "round", "trip", "from", "new", "york", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-class_type", "I-class_type", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["now", "show", "me", "all", "the", "round", "trips", "from", "new", "york", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "one", "way", "flight", "from", "san", "francisco", "to", "houston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["now", "show", "me", "the", "cheapest", "one", "way", "flight", "from", "houston", "to", "boston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "round", "trip", "fares", "from", "houston", "to", "boston"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "round", "trip", "fares", "from", "san", "francisco", "to", "houston"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O"], "text": ["show", "me", "the", "cheapest", "round", "trip", "fare", "from", "san", "francisco", "to", "houston", "on", "february", "twenty", "eighth", "1994"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "B-depart_date.year"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O"], "text": ["show", "me", "the", "cheapest", "one", "way", "fare", "from", "san", "francisco", "to", "houston", "on", "february", "twenty", "eighth", "1994"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "B-depart_date.year"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["now", "show", "me", "ground", "transportation", "in", "houston", "on", "monday", "afternoon"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-city_name", "O", "B-day_name", "B-period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["now", "show", "me", "one", "way", "flights", "from", "houston", "to", "boston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["and", "now", "show", "me", "ground", "transportation", "that", "i", "could", "get", "in", "boston", "late", "night"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-city_name", "B-period_of_day", "B-period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "airlines", "that", "have", "flights", "between", "toronto", "and", "detroit", "between", "detroit", "and", "st.", "louis", "and", "between", "st.", "louis", "and", "toronto"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O"], "text": ["show", "me", "round", "trip", "fares", "from", "toronto", "to", "detroit", "on", "delta", "northwest", "us", "air", "and", "united", "airlines"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name", "B-airline_name", "B-airline_name", "I-airline_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "O"], "text": ["show", "me", "flights", "between", "detroit", "and", "st.", "louis", "on", "delta", "northwest", "us", "air", "and", "united", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-airline_name", "B-airline_name", "B-airline_name", "I-airline_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "flights", "from", "montreal", "to", "orlando", "and", "long", "beach"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "flights", "from", "montreal", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "on", "friday", "afternoon", "in", "june", "from", "new", "york", "to", "cleveland"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-depart_date.month_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "new", "york", "to", "los", "angeles", "on", "saturday", "evening", "on", "us", "air"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "a", "red", "eye", "flight", "from", "new", "york", "to", "los", "angeles", "on", "saturday", "evening", "on", "us", "air"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "I-flight_mod", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "a", "flight", "from", "new", "york", "to", "los", "angeles", "on", "saturday", "morning", "on", "us", "air"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "san", "francisco", "to", "milwaukee", "on", "monday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "does", "ua", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-airline_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "milwaukee", "to", "washington", "dc", "on", "monday", "night"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["how", "about", "flights", "from", "milwaukee", "to", "washington", "dc", "on", "tuesday", "mornings"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "meals", "are", "there", "on", "flight", "382", "from", "milwaukee", "to", "washington", "dc", "on", "tuesday", "morning"], "golden_intent": "atis_meal", "golden_slot": ["O", "B-meal", "O", "O", "O", "O", "B-flight_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'll", "need", "to", "rent", "a", "car", "in", "washington", "dc"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-transport_type", "O", "B-city_name", "B-state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["can", "i", "get", "a", "flight", "on", "tuesday", "night", "from", "washington", "dc", "to", "oakland"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["how", "about", "from", "dc", "to", "oakland", "on", "wednesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.state_code", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["how", "much", "does", "it", "cost", "to", "fly", "on", "twa", "from", "columbus", "to", "milwaukee"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-airline_code", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "does", "q", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["how", "much", "does", "it", "cost", "to", "fly", "from", "columbus", "to", "st.", "louis", "round", "trip", "on", "twa"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-round_trip", "I-round_trip", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["what's", "the", "cheapest", "flight", "from", "columbus", "to", "st.", "louis", "round", "trip", "on", "twa"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-round_trip", "I-round_trip", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what's", "the", "cheapest", "round", "trip", "flight", "on", "twa", "from", "columbus", "to", "st.", "paul"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-airline_code", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "to", "fly", "from", "milwaukee", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["can", "i", "get", "the", "shortest", "flight", "from", "milwaukee", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "shortest", "flight", "from", "milwaukee", "to", "long", "beach"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "does", "m", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "does", "ap", "57", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-restriction_code", "I-restriction_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "shortest", "flight", "from", "milwaukee", "to", "st.", "petersburg"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "shortest", "flight", "from", "milwaukee", "to", "long", "beach"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "shortest", "flight", "from", "milwaukee", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "does", "ap", "20", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-restriction_code", "I-restriction_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["can", "i", "get", "a", "flight", "today", "from", "san", "francisco", "to", "detroit", "michigan"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.today_relative", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["what's", "the", "cheapest", "flight", "from", "san", "francisco", "to", "detroit", "today"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.today_relative"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "want", "to", "fly", "from", "san", "francisco", "to", "milwaukee", "and", "from", "milwaukee", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what's", "the", "cheapest", "flight", "from", "san", "francisco", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "to", "rent", "a", "car", "in", "milwaukee"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-transport_type", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what's", "the", "cheapest", "flight", "tomorrow", "from", "milwaukee", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-cost_relative", "O", "B-depart_date.today_relative", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what", "ground", "transportation", "is", "available", "at", "denver"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what's", "the", "cheapest", "flight", "from", "san", "francisco", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["what", "flights", "leave", "from", "cleveland", "and", "go", "to", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "nonstop", "flights", "from", "st.", "petersburg", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "us"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "flights", "between", "toronto", "and", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O"], "text": ["what", "is", "phl"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-airport_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O"], "text": ["what", "is", "mci"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-airport_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "flights", "between", "oakland", "and", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "does", "not", "sa", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-mod", "B-days_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "earliest", "daily", "flight", "between", "oakland", "and", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "B-flight_days", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "dl"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "latest", "daily", "flight", "between", "oakland", "and", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "B-flight_days", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "between", "los", "angeles", "and", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["what", "ground", "transportation", "is", "available", "from", "dallas", "fort", "worth", "airport", "to", "downtown", "dallas"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "I-airport_name", "I-airport_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "passengers", "can", "an", "l1011", "aircraft", "hold"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "B-aircraft_code", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "a", "dc9"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "between", "dallas", "and", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "ground", "transportation", "is", "available", "between", "phoenix", "airport", "and", "downtown", "phoenix"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "for", "the", "aircraft", "m80"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-fromloc.city_name", "O"], "text": ["are", "there", "any", "flights", "between", "dallas", "and", "phoenix", "using", "a", "dc10", "aircraft"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-aircraft_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "aa"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "between", "milwaukee", "and", "indiana"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "between", "milwaukee", "and", "pittsburgh"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["what", "ground", "transportation", "is", "available", "between", "pittsburgh", "airport", "and", "downtown", "pittsburgh"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "flights", "between", "pittsburgh", "and", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["what", "ground", "transportation", "is", "available", "between", "dca", "and", "downtown", "washington"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_code", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "between", "dca", "and", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airport_code", "O", "B-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "ground", "transportation", "is", "available", "between", "milwaukee", "airport", "and", "downtown", "milwaukee"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["determine", "the", "type", "of", "aircraft", "used", "on", "a", "flight", "from", "cleveland", "to", "dallas", "that", "leaves", "before", "noon"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O"], "text": ["find", "a", "flight", "between", "st.", "petersburg", "and", "charlotte", "the", "flight", "should", "leave", "in", "the", "afternoon", "and", "arrive", "as", "soon", "after", "5", "pm", "as", "possible", "it", "should", "be", "a", "nonstop", "flight"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-depart_time.period_of_day", "O", "O", "O", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time", "O", "O", "O", "O", "O", "O", "B-flight_stop", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "a", "flight", "on", "delta", "airlines", "from", "toronto", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "a", "flight", "on", "american", "airlines", "from", "toronto", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "a", "flight", "from", "toronto", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "oakland", "to", "salt", "lake", "city", "leaving", "after", "1700", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "oakland", "to", "salt", "lake", "city", "leaving", "after", "midnight", "thursday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.period_of_day", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "between", "phoenix", "and", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "las", "vegas", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "milwaukee", "to", "washington", "dc", "before", "1200"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "B-depart_time.time_relative", "B-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "pittsburgh", "leaving", "after", "1800"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "pittsburgh"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "between", "pittsburgh", "and", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name"], "text": ["i'd", "like", "a", "flight", "to", "san", "diego", "from", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "B-fromloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i'd", "like", "to", "fly", "from", "cleveland", "to", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "to", "fly", "from", "washington", "dc", "to", "phoenix", "arizona"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "phoenix", "to", "atlanta"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "to", "fly", "from", "atlanta", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "to", "fly", "from", "san", "diego", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "to", "fly", "from", "orlando", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "kansas", "city", "to", "minneapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "san", "diego", "to", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "washington", "dc", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O"], "text": ["i", "need", "a", "round", "trip", "flight", "from", "san", "diego", "to", "washington", "dc", "and", "the", "fares"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "round", "trip", "from", "atlanta", "to", "washington", "dc", "and", "the", "fares", "leaving", "in", "the", "morning"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "O", "O", "O", "O", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "round", "trip", "from", "phoenix", "to", "washington", "dc", "and", "the", "fare", "leaving", "in", "the", "morning"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code", "O", "O", "O", "O", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "lowest", "fare", "for", "a", "flight", "from", "washington", "dc", "to", "boston"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "lowest", "fare", "from", "washington", "dc", "to", "montreal"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "lowest", "fare", "from", "toronto", "to", "washington", "dc"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "a", "flight", "from", "montreal", "to", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "a", "flight", "from", "nashville", "to", "seattle", "that", "arrives", "no", "later", "than", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.time_relative", "I-arrive_time.time_relative", "I-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "a", "flight", "from", "memphis", "to", "seattle", "that", "arrives", "no", "later", "than", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.time_relative", "I-arrive_time.time_relative", "I-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "indianapolis", "to", "seattle", "arriving", "in", "seattle", "at", "1205", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "want", "a", "flight", "round", "trip", "from", "memphis", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O"], "text": ["i", "want", "to", "fly", "from", "nashville", "to", "seattle", "and", "i", "want", "the", "cheapest", "fare", "round", "trip"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-cost_relative", "O", "B-round_trip", "I-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O"], "text": ["i", "want", "to", "fly", "from", "memphis", "to", "seattle", "round", "trip", "with", "the", "cheapest", "fare"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "O", "O", "B-cost_relative", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O"], "text": ["i", "want", "to", "fly", "from", "indianapolis", "to", "seattle", "round", "trip", "with", "the", "cheapest", "fare"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "O", "O", "B-cost_relative", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "flights", "from", "orlando", "to", "philadelphia"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "flights", "from", "san", "francisco", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "flights", "from", "milwaukee", "to", "philadelphia"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "flights", "from", "philadelphia", "to", "san", "francisco"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["please", "show", "ground", "transportation", "to", "milwaukee"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "flights", "from", "san", "francisco", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "houston", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "houston", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "phoenix", "to", "houston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "newark", "to", "houston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "denver", "to", "houston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "st.", "petersburg", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "flights", "from", "orlando", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "kansas", "city", "to", "minneapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "kansas", "city", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "flights", "from", "minneapolis", "to", "kansas", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "flights", "from", "kansas", "city", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "boston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "fares", "from", "washington", "dc", "to", "montreal"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "montreal"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O"], "text": ["list", "fares", "from", "washington", "dc", "to", "toronto", "that", "should", "be", "good"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "O", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "fares", "from", "washington", "dc", "to", "boston"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "montreal"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "washington", "dc", "to", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "toronto", "to", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "oakland", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "flights", "go", "from", "dallas", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "flights", "go", "from", "phoenix", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "an", "early", "flight", "from", "milwaukee", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "types", "of", "ground", "transportation", "are", "available", "in", "denver"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "flights", "go", "from", "denver", "to", "st.", "louis", "on", "tuesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["is", "ground", "transportation", "available", "in", "st.", "louis"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "to", "fly", "from", "st.", "louis", "to", "milwaukee", "on", "wednesday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flights", "from", "washington", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flights", "from", "atlanta", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flights", "from", "san", "diego", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "flight", "information", "from", "phoenix", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["could", "i", "have", "flight", "information", "on", "flights", "from", "salt", "lake", "city", "to", "phoenix", "please"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["could", "i", "have", "flight", "information", "on", "flights", "from", "pittsburgh", "to", "phoenix", "please"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "information", "on", "flights", "leaving", "from", "washington", "dc", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["i", "need", "information", "on", "flights", "from", "washington", "to", "boston", "that", "leave", "on", "a", "saturday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "the", "flights", "from", "washington", "to", "montreal", "on", "a", "saturday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "the", "fares", "on", "flights", "from", "washington", "to", "toronto", "on", "a", "saturday"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["i", "want", "to", "go", "from", "boston", "to", "washington", "on", "a", "saturday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["i", "need", "a", "flight", "from", "cleveland", "to", "dallas", "that", "leaves", "before", "noon", "see", "if", "too", "much", "information"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.time", "O", "O", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "fares", "from", "washington", "to", "boston"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "saturday", "fares", "from", "washington", "to", "boston"], "golden_intent": "atis_airfare", "golden_slot": ["O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "fares", "from", "washington", "to", "montreal"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "saturday", "fares", "from", "washington", "to", "montreal"], "golden_intent": "atis_airfare", "golden_slot": ["O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "saturday", "fares", "from", "washington", "to", "toronto"], "golden_intent": "atis_airfare", "golden_slot": ["O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "the", "saturday", "fare", "from", "washington", "to", "toronto"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "saturday", "flights", "from", "washington", "to", "boston"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "saturday", "flights", "from", "boston", "to", "washington"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "flights", "from", "milwaukee", "to", "dtw"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.airport_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "milwaukee", "to", "detroit"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "flights", "from", "detroit", "to", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "flights", "from", "toronto", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "last", "flight", "from", "oakland", "to", "salt", "lake", "city", "on", "wednesday", "or", "first", "flight", "from", "oakland", "to", "salt", "lake", "city", "on", "thursday"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-or", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "first", "flight", "from", "oakland", "to", "salt", "lake", "city", "on", "thursday"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "last", "flight", "from", "oakland", "to", "salt", "lake", "city", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "last", "wednesday", "flight", "from", "oakland", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-flight_mod", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O"], "text": ["get", "flight", "from", "toronto", "to", "san", "diego", "stopping", "at", "dtw"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-stoploc.airport_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["get", "flights", "between", "st.", "petersburg", "and", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "milwaukee", "to", "indianapolis", "leaving", "monday", "before", "9", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "departing", "from", "milwaukee", "to", "indianapolis", "leaving", "monday", "before", "8", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["is", "there", "ground", "transportation", "available", "at", "the", "indianapolis", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "flight", "information", "for", "a", "flight", "departing", "from", "indianapolis", "to", "cleveland", "departing", "tuesday", "at", "noon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "information", "for", "a", "flight", "departing", "from", "cleveland", "to", "milwaukee", "wednesday", "after", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "information", "for", "flights", "departing", "from", "cleveland", "going", "back", "to", "milwaukee", "wednesday", "after", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "information", "for", "flights", "departing", "from", "cleveland", "to", "milwaukee", "on", "wednesday", "after", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "information", "for", "flights", "departing", "from", "cleveland", "to", "milwaukee", "on", "wednesday", "after", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "denver", "to", "salt", "lake", "city", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["is", "there", "ground", "transportation", "available", "at", "the", "denver", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "and", "airline", "information", "for", "a", "flight", "from", "denver", "to", "salt", "lake", "city", "on", "monday", "departing", "after", "5", "pm"], "golden_intent": "atis_flight#atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["is", "there", "ground", "transportation", "available", "at", "the", "salt", "lake", "city", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "I-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "salt", "lake", "city", "to", "phoenix", "departing", "wednesday", "after", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["is", "there", "ground", "transportation", "available", "at", "the", "phoenix", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "oakland", "to", "salt", "lake", "city", "on", "wednesday", "departing", "after", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "and", "fare", "information", "for", "thursday", "departing", "prior", "to", "9", "am", "from", "oakland", "going", "to", "salt", "lake", "city"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-depart_date.day_name", "O", "B-depart_time.time_relative", "I-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "and", "fare", "information", "departing", "from", "oakland", "to", "salt", "lake", "city", "on", "thursday", "before", "8", "am"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "numbers", "and", "airlines", "for", "flights", "departing", "from", "oakland", "to", "salt", "lake", "city", "on", "thursday", "departing", "before", "8", "am"], "golden_intent": "atis_flight_no#atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flight", "numbers", "for", "those", "flights", "departing", "on", "thursday", "before", "8", "am", "from", "oakland", "going", "to", "salt", "lake", "city"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O"], "text": ["list", "airports", "in", "arizona", "nevada", "and", "california", "please"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-state_name", "I-state_name", "O", "B-state_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["list", "california", "nevada", "arizona", "airports"], "golden_intent": "atis_airport", "golden_slot": ["O", "B-state_name", "B-state_name", "B-state_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["list", "the", "arizona", "airport"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "B-state_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O"], "text": ["list", "california", "airports"], "golden_intent": "atis_airport", "golden_slot": ["O", "B-state_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "las", "vegas", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O"], "text": ["list", "california", "airports"], "golden_intent": "atis_airport", "golden_slot": ["O", "B-state_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O"], "text": ["list", "airports"], "golden_intent": "atis_airport", "golden_slot": ["O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "wednesday", "night", "flights", "from", "oakland", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "oakland", "to", "salt", "lake", "city", "before", "6", "am", "thursday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["which", "airlines", "fly", "between", "toronto", "and", "san", "diego"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "afternoon", "flights", "between", "st.", "petersburg", "and", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O"], "text": ["what", "is", "tpa"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-airport_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "cleveland", "to", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "only", "the", "flights", "from", "cleveland", "to", "dallas", "that", "leave", "before", "noon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "type", "of", "aircraft", "are", "flying", "from", "cleveland", "to", "dallas", "before", "noon"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "information", "on", "flights", "from", "indianapolis", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "memphis", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "ticket", "from", "nashville", "to", "seattle"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "ticket", "from", "nashville", "tennessee", "to", "seattle"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "flight", "information", "from", "milwaukee", "to", "tampa"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["i", "need", "to", "rent", "a", "car", "at", "tampa"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-transport_type", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "daily", "flight", "from", "st.", "louis", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_days", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "flights", "departing", "from", "oakland", "and", "arriving", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "information", "on", "flights", "from", "toronto", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "information", "on", "flights", "from", "toronto", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "a", "flight", "from", "toronto", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "information", "on", "flights", "between", "st.", "petersburg", "and", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["i", "need", "the", "flight", "numbers", "of", "flights", "leaving", "from", "cleveland", "and", "arriving", "at", "dallas"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["which", "flights", "go", "from", "new", "york", "to", "miami", "and", "back"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "qo", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "flights", "from", "milwaukee", "to", "orlando", "one", "way"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "the", "abbreviation", "us", "stand", "for"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "B-airline_code", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "a", "one", "way", "ticket", "from", "milwaukee", "to", "orlando", "either", "wednesday", "evening", "or", "thursday", "morning"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "B-or", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "flights", "from", "milwaukee", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "f", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "h", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "y", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["what", "are", "restrictions", "ap", "57"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "B-restriction_code", "I-restriction_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["please", "show", "me", "first", "class", "flights", "from", "indianapolis", "to", "memphis", "one", "way", "leaving", "before", "10", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-class_type", "I-class_type", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["now", "show", "me", "all", "round", "trip", "flights", "from", "burbank", "to", "seattle", "that", "arrive", "before", "7", "pm", "in", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["round", "trip", "flights", "from", "orlando", "to", "montreal", "please"], "golden_intent": "atis_flight", "golden_slot": ["B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "dl"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "delta", "airlines", "flights", "from", "montreal", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["show", "me", "all", "flights", "from", "orlando", "to", "montreal", "please"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["which", "airline", "is", "kw"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["please", "list", "all", "flights", "from", "new", "york", "to", "miami", "any", "any", "type", "of", "class"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "bh", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["show", "me", "a", "return", "flight", "from", "miami", "to", "jfk", "please"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.airport_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "bh", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "bh", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "bh", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "bh", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "one", "way", "flights", "from", "milwaukee", "to", "orlando", "after", "6", "pm", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "from", "indianapolis", "to", "memphis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "round", "trip", "flights", "from", "burbank", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "round", "trip", "flights", "from", "orlando", "to", "montreal"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "nonstop", "flights", "from", "montreal", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "round", "trips", "between", "montreal", "and", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "round", "trip", "flights", "from", "montreal", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "one", "way", "flights", "from", "montreal", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "one", "way", "flights", "from", "orlando", "to", "montreal"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "cheapest", "economy", "flights", "from", "miami", "to", "new", "york"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-economy", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["kansas", "city", "to", "las", "vegas", "economy"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-economy"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["kansas", "city", "to", "las", "vegas", "economy"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-economy"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "hp"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["ground", "transportation", "in", "las", "vegas"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["ground", "transportation", "for", "las", "vegas"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["las", "vegas", "to", "baltimore", "economy"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-economy"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["las", "vegas", "to", "baltimore", "economy"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-economy"]} +{"pred_intent": "atis_flight", "pred_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["baltimore", "to", "kansas", "city", "economy"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-economy"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "us"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["which", "airline", "is", "us"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["which", "airline", "is", "us"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["which", "airline", "is", "us"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["which", "airline", "is", "us"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["columbus", "to", "chicago", "one", "way", "before", "10", "am"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "hp"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name"], "text": ["st.", "petersburg", "to", "detroit"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["from", "milwaukee", "to", "orlando", "one", "way", "after", "5", "pm", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["and", "from", "milwaukee", "to", "atlanta", "before", "10", "am", "daily"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "B-flight_days"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "yx"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "flights", "from", "san", "jose", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "flights", "from", "san", "jose", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "hp"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["show", "me", "ground", "transportation", "in", "phoenix"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "flights", "from", "phoenix", "to", "fort", "worth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["show", "me", "ground", "transportation", "in", "fort", "worth"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "flights", "from", "fort", "worth", "to", "san", "jose"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["show", "me", "first", "class", "flights", "from", "new", "york", "to", "miami", "round", "trip"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-class_type", "I-class_type", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["show", "me", "first", "class", "flights", "from", "new", "york", "to", "miami", "round", "trip"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-class_type", "I-class_type", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "round", "trip", "flights", "from", "new", "york", "to", "miami", "nonstop"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-flight_stop"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "round", "trip", "flights", "from", "miami", "to", "new", "york", "nonstop"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-flight_stop"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O"], "text": ["show", "me", "one", "way", "flights", "from", "indianapolis", "to", "memphis", "before", "10", "am", "on", "any", "day"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "f", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "round", "trip", "flights", "from", "burbank", "to", "tacoma"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "the", "restriction", "ap58", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-restriction_code", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "does", "fare", "code", "h", "mean"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "B-fare_basis_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "as"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "as"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "as"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what", "airline", "is", "as", "as", "in", "sam"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "nonstop", "flights", "from", "st.", "petersburg", "to", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "nonstop", "flights", "from", "toronto", "to", "st.", "petersburg"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "nonstop", "flights", "and", "fares", "from", "toronto", "to", "st.", "petersburg"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "nonstop", "flights", "from", "toronto", "to", "st.", "petersburg"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "weekday", "flights", "from", "milwaukee", "to", "orlando", "one", "way"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "weekday", "flights", "from", "milwaukee", "to", "orlando", "one", "way"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "airline", "is", "hp"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "chicago", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "chicago", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "kansas", "city", "to", "denver"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "denver", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "phoenix", "to", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "las", "vegas", "to", "san", "diego"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "chicago", "to", "kansas", "city", "in", "the", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "houston", "to", "san", "jose"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "houston", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "milwaukee", "to", "san", "jose", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "san", "jose", "to", "dallas", "on", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "dallas", "to", "houston"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "distance", "from", "airports", "to", "downtown", "in", "new", "york"], "golden_intent": "atis_distance", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "airports", "in", "new", "york"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "airports", "in", "new", "york"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name"], "text": ["list", "airports", "in", "la"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O"], "text": ["list", "airports"], "golden_intent": "atis_airport", "golden_slot": ["O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name"], "text": ["list", "airports", "in", "la"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name"], "text": ["list", "airports", "in", "la"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-toloc.city_name"], "text": ["list", "the", "airports", "in", "la"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-toloc.city_name"], "text": ["list", "la"], "golden_intent": "atis_city", "golden_slot": ["O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-toloc.city_name"], "text": ["list", "la"], "golden_intent": "atis_city", "golden_slot": ["O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "new", "york", "to", "la"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "la", "guardia", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "la", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "ontario", "california", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "ontario", "california", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "indianapolis", "to", "memphis", "with", "fares", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "indianapolis", "to", "memphis", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "memphis", "to", "miami", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "miami", "to", "indianapolis", "on", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "charlotte", "on", "saturday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "type", "of", "aircraft", "for", "all", "flights", "from", "charlotte"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["list", "flights", "and", "fares", "from", "tacoma", "to", "orlando", "round", "trip", "leaving", "saturday", "returning", "next", "saturday"], "golden_intent": "atis_flight#atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-round_trip", "I-round_trip", "O", "B-depart_date.day_name", "O", "B-return_date.date_relative", "B-return_date.day_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "class", "is", "fare", "code", "q"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "O", "O", "B-booking_class"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "O"], "text": ["list", "flights", "from", "orlando", "to", "tacoma", "on", "saturday", "of", "fare", "basis", "code", "of", "q"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "O", "O", "O", "O", "B-fare_basis_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "airfares", "for", "first", "class", "round", "trip", "from", "detroit", "to", "st.", "petersburg"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-class_type", "I-class_type", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "coach", "round", "trip", "airfare", "from", "detroit", "to", "st.", "petersburg"], "golden_intent": "atis_airfare", "golden_slot": ["O", "B-class_type", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "pittsburgh", "to", "newark", "on", "monday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "minneapolis", "to", "pittsburgh", "on", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "before", "9", "am", "from", "cincinnati", "to", "tampa"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "cincinnati", "to", "tampa", "before", "noon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "flights", "from", "tampa", "to", "cincinnati", "after", "3", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "airlines", "that", "fly", "from", "seattle", "to", "salt", "lake", "city"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "delta", "flights", "from", "seattle", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["list", "seating", "capacities", "of", "delta", "flights", "from", "seattle", "to", "salt", "lake", "city"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O"], "text": ["list", "delta", "flights", "from", "seattle", "to", "salt", "lake", "city", "with", "aircraft", "type"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what", "ground", "transportation", "is", "there", "in", "baltimore"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["list", "ground", "transportation", "in", "baltimore"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "flights", "from", "baltimore", "to", "san", "francisco", "on", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "los", "angeles", "to", "pittsburgh", "on", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "pittsburgh", "to", "los", "angeles", "thursday", "evening"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "round", "trip", "flights", "from", "cleveland", "to", "miami", "next", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "fares", "for", "round", "trip", "flights", "from", "cleveland", "to", "miami", "next", "wednesday"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "flights", "and", "fares", "for", "a", "trip", "to", "cleveland", "from", "miami", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "fares", "from", "miami", "to", "cleveland", "next", "sunday"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "milwaukee", "to", "phoenix", "on", "saturday", "or", "sunday", "on", "american", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-or", "B-depart_date.day_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "phoenix", "to", "milwaukee", "on", "wednesday", "evening"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "phoenix", "to", "milwaukee", "on", "wednesday", "on", "american", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "phoenix", "to", "milwaukee", "on", "american", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "phoenix", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "meal", "flights", "departing", "early", "saturday", "morning", "from", "chicago", "to", "seattle", "nonstop"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-meal", "O", "O", "B-depart_time.period_mod", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-flight_stop"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O"], "text": ["give", "me", "the", "flights", "from", "chicago", "to", "seattle", "saturday", "morning", "that", "have", "meals"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "B-meal"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["give", "me", "flights", "from", "seattle", "to", "chicago", "that", "have", "meals", "on", "continental"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-meal", "O", "B-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "seattle", "to", "chicago", "that", "have", "meals", "on", "continental", "saturday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-meal", "O", "B-airline_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-depart_time.period_of_day", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "chicago", "to", "seattle", "on", "continental", "that", "have", "meals", "early", "saturday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name", "O", "O", "B-meal", "B-depart_time.period_mod", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-depart_time.period_of_day", "O", "B-toloc.city_name"], "text": ["give", "me", "a", "combination", "of", "continental", "flights", "from", "chicago", "to", "seattle", "that", "have", "meals", "early", "saturday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-meal", "B-depart_time.period_mod", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "saturday", "morning", "flights", "with", "meals", "from", "chicago", "to", "minneapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "B-meal", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "saturday", "morning", "flights", "on", "continental", "that", "have", "meals", "from", "chicago", "to", "minneapolis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "B-airline_name", "O", "O", "B-meal", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "O", "O", "O"], "text": ["give", "me", "the", "saturday", "morning", "flights", "from", "chicago", "to", "st.", "paul", "on", "continental", "that", "have", "meals"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-airline_name", "O", "O", "B-meal"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "new", "york", "to", "las", "vegas", "nonstop"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-flight_stop"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "memphis", "to", "las", "vegas", "nonstop"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-flight_stop"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "the", "cheapest", "round", "trip", "flights", "from", "indianapolis", "to", "orlando", "around", "december", "twenty", "fifth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "friday", "flight", "from", "newark", "to", "tampa"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "need", "a", "sunday", "flight", "from", "tampa", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["give", "me", "a", "flight", "from", "charlotte", "to", "baltimore", "on", "tuesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"], "text": ["can", "i", "have", "a", "morning", "flight", "from", "baltimore", "to", "newark", "please"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["cheapest", "round", "trip", "fare", "from", "or", "indianapolis", "to", "orlando", "on", "december", "twenty", "fifth"], "golden_intent": "atis_airfare", "golden_slot": ["B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["cheapest", "one", "way", "fare", "from", "indianapolis", "to", "orlando", "on", "december", "twenty", "seventh"], "golden_intent": "atis_airfare", "golden_slot": ["B-cost_relative", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flight", "number", "from", "dallas", "to", "houston"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flight", "number", "from", "houston", "to", "dallas"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["saturday", "flight", "on", "american", "airlines", "from", "milwaukee", "to", "phoenix"], "golden_intent": "atis_flight", "golden_slot": ["B-depart_date.day_name", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flight", "numbers", "on", "american", "airlines", "from", "phoenix", "to", "milwaukee"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["flight", "numbers", "from", "chicago", "to", "seattle"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["flight", "numbers", "from", "chicago", "to", "seattle", "on", "continental"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["flight", "numbers", "from", "seattle", "to", "chicago", "on", "continental"], "golden_intent": "atis_flight_no", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["is", "there", "a", "fare", "from", "pittsburgh", "to", "cleveland", "under", "200", "dollars"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-cost_relative", "B-fare_amount", "I-fare_amount"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["how", "much", "is", "coach", "flight", "from", "pittsburgh", "to", "atlanta"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-class_type", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["newark", "to", "tampa", "on", "friday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["tampa", "to", "charlotte", "sunday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["charlotte", "to", "baltimore", "on", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["baltimore", "to", "newark", "wednesday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["dallas", "to", "houston", "after", "1201", "am"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "O"], "text": ["houston", "to", "dallas", "before", "midnight"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["indianapolis", "to", "orlando", "december", "twenty", "seventh"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name", "O", "O", "O"], "text": ["cheapest", "fare", "from", "indianapolis", "to", "orlando", "on", "the", "twenty", "seventh", "of", "december"], "golden_intent": "atis_airfare", "golden_slot": ["B-cost_relative", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_number", "I-depart_date.day_number", "O", "B-depart_date.month_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["cheapest", "fare", "round", "trip", "from", "indianapolis", "to", "orlando", "on", "december", "twenty", "seventh"], "golden_intent": "atis_airfare", "golden_slot": ["B-cost_relative", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["cleveland", "to", "miami", "on", "wednesday", "arriving", "before", "4", "pm"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["miami", "to", "cleveland", "sunday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["new", "york", "city", "to", "las", "vegas", "and", "memphis", "to", "las", "vegas", "on", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["new", "york", "city", "to", "las", "vegas", "and", "memphis", "to", "las", "vegas", "on", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["new", "york", "to", "las", "vegas", "sunday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["memphis", "to", "las", "vegas", "sunday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["new", "york", "to", "las", "vegas", "on", "sunday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["chicago", "to", "seattle", "saturday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["chicago", "to", "las", "vegas", "saturday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["los", "angeles", "to", "pittsburgh", "afternoon", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.period_of_day", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["los", "angeles", "to", "pittsburgh", "afternoon", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.period_of_day", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["pittsburgh", "to", "los", "angeles", "thursday", "evening"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["milwaukee", "to", "phoenix", "on", "saturday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["phoenix", "to", "milwaukee", "on", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["phoenix", "to", "milwaukee", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["a", "flight", "from", "baltimore", "to", "san", "francisco", "arriving", "between", "5", "and", "8", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-flight", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name"], "text": ["how", "many", "northwest", "flights", "leave", "st.", "paul"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name"], "text": ["how", "many", "northwest", "flights", "leave", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "flights", "does", "northwest", "have", "leaving", "dulles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what", "cities", "does", "northwest", "fly", "out", "of"], "golden_intent": "atis_city", "golden_slot": ["O", "O", "O", "B-airline_name", "O", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O"], "text": ["list", "the", "cities", "from", "which", "northwest", "flies"], "golden_intent": "atis_city", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["what", "cities", "does", "northwest", "fly", "to"], "golden_intent": "atis_city", "golden_slot": ["O", "O", "O", "B-airline_name", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "connecting", "flight", "from", "dallas", "to", "san", "francisco", "leaving", "after", "4", "o'clock"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-connect", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "all", "the", "flights", "from", "dallas", "to", "san", "francisco"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["tell", "me", "again", "the", "morning", "flights", "on", "american", "airlines", "from", "philadelphia", "to", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["tell", "me", "the", "flights", "that", "leave", "philadelphia", "and", "go", "to", "dallas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "is", "a", "d9s"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what", "type", "of", "plane", "is", "a", "d9s"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["what", "is", "a", "d9s"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["show", "me", "the", "airports", "serviced", "by", "tower", "air"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "first", "class", "and", "coach", "flights", "between", "jfk", "and", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-class_type", "I-class_type", "O", "B-class_type", "O", "O", "B-fromloc.airport_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "first", "class", "and", "coach", "flights", "from", "kennedy", "airport", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-class_type", "I-class_type", "O", "B-class_type", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "first", "class", "and", "coach", "flights", "from", "jfk", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-class_type", "I-class_type", "O", "B-class_type", "O", "O", "B-fromloc.airport_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["are", "meals", "ever", "served", "on", "tower", "air"], "golden_intent": "atis_meal", "golden_slot": ["O", "B-meal", "O", "O", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["are", "snacks", "served", "on", "tower", "air"], "golden_intent": "atis_meal", "golden_slot": ["O", "B-meal_description", "O", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "delta", "airlines", "flights", "from", "jfk", "to", "miami"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.airport_code", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "delta", "airlines", "from", "boston", "to", "salt", "lake"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "delta", "airlines", "flights", "from", "boston", "to", "salt", "lake"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "delta", "airlines", "flights", "from", "boston", "to", "salt", "lake", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "fares", "for", "flights", "between", "boston", "and", "washington", "dc"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "least", "expensive", "fare", "from", "boston", "to", "salt", "lake", "city"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "I-cost_relative", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "lowest", "fares", "from", "washington", "dc", "to", "salt", "lake", "city"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "B-fromloc.state_code", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "lowest", "fare", "from", "bwi", "to", "salt", "lake", "city"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "B-cost_relative", "O", "O", "B-fromloc.airport_code", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_airfare", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O"], "text": ["show", "me", "the", "cost", "of", "a", "first", "class", "ticket", "from", "detroit", "to", "las", "vegas", "and", "back"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-class_type", "I-class_type", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-round_trip"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "earliest", "arriving", "flight", "from", "boston", "to", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "I-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "is", "the", "earliest", "arriving", "flight", "between", "boston", "and", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "I-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what's", "the", "earliest", "arriving", "flight", "between", "boston", "and", "washington", "dc"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-flight_mod", "I-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "earliest", "arriving", "flight", "from", "houston", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "I-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "is", "the", "earliest", "arriving", "flight", "from", "houston", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_mod", "I-flight_mod", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "between", "houston", "and", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "between", "houston", "and", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "the", "flights", "from", "houston", "to", "orlando"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "leaving", "denver", "between", "8", "pm", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-depart_time.start_time", "I-depart_time.start_time", "O", "B-depart_time.end_time", "I-depart_time.end_time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "on", "the", "aircraft", "733"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "a", "72s"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "the", "aircraft", "72s"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "the", "aircraft", "m80"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "the", "type", "of", "aircraft", "m80"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "an", "m80"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name"], "text": ["what", "airlines", "serve", "denver"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-fromloc.city_name"], "text": ["list", "the", "airlines", "with", "flights", "to", "or", "from", "denver"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name"], "text": ["what", "airlines", "fly", "into", "denver"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "arriving", "in", "denver", "between", "8", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "capacity", "of", "the", "73s"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O"], "text": ["what", "is", "73s"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "seating", "capacity", "on", "the", "aircraft", "73s"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "a", "757"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "people", "will", "a", "757", "hold"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "B-aircraft_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["how", "many", "passengers", "can", "fly", "on", "a", "757"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "of", "the", "daily", "flights", "arriving", "in", "denver", "between", "8", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-flight_days", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "of", "the", "daily", "flights", "arriving", "in", "denver", "from", "8", "to", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-flight_days", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "all", "of", "the", "daily", "flights", "arriving", "in", "denver", "between", "8", "pm", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-flight_days", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.start_time", "I-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "the", "757"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["tell", "me", "about", "the", "m80", "aircraft"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "B-aircraft_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O"], "text": ["tell", "me", "about", "the", "m80", "aircraft"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "B-aircraft_code", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["tell", "me", "about", "the", "type", "of", "aircraft", "called", "an", "m80"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "the", "733"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "of", "the", "m80"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "is", "the", "seating", "capacity", "on", "the", "aircraft", "m80"], "golden_intent": "atis_capacity", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-aircraft_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "arriving", "or", "leaving", "denver", "between", "8", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-city_name", "O", "B-depart_time.start_time", "O", "B-depart_time.end_time", "I-depart_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "arriving", "in", "denver", "between", "8", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "all", "flights", "on", "all", "types", "of", "aircraft", "arriving", "in", "denver", "between", "8", "and", "9", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-arrive_time.start_time", "O", "B-arrive_time.end_time", "I-arrive_time.end_time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "all", "flights", "from", "nashville", "to", "memphis", "on", "monday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "the", "flights", "from", "nashville", "to", "memphis", "on", "monday", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["is", "there", "ground", "transportation", "from", "the", "memphis", "airport", "into", "town", "when", "if", "i", "arrive", "at", "842", "in", "the", "morning"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_date.day_name", "I-toloc.city_name"], "text": ["please", "list", "the", "flights", "from", "memphis", "to", "new", "york", "city", "on", "a", "monday", "night"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O"], "text": ["what", "is", "cvg"], "golden_intent": "atis_abbreviation", "golden_slot": ["O", "O", "B-airport_code"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "O"], "text": ["what", "ground", "transportation", "is", "available", "from", "la", "guardia", "airport", "into", "new", "york", "city"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "I-airport_name", "O", "B-city_name", "I-city_name", "I-city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name"], "text": ["is", "there", "ground", "transportation", "from", "lga", "into", "new", "york", "city"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-airport_code", "O", "B-city_name", "I-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name"], "text": ["please", "list", "the", "ground", "transportation", "from", "lga", "into", "new", "york", "city"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airport_code", "O", "B-city_name", "I-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name"], "text": ["please", "list", "ground", "transportation", "from", "ewr", "into", "new", "york", "city"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "B-airport_code", "O", "B-city_name", "I-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["show", "me", "the", "morning", "flights", "from", "memphis", "to", "new", "york", "city"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["give", "me", "the", "flights", "from", "new", "york", "city", "to", "nashville", "leaving", "after", "5", "pm", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O"], "text": ["tell", "me", "about", "the", "ground", "transportation", "from", "nashville", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "nonstop", "flights", "from", "cincinnati", "to", "charlotte", "leaving", "after", "noon", "and", "arriving", "before", "7", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_time.time_relative", "B-depart_time.period_of_day", "O", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["how", "many", "flights", "does", "alaska", "airlines", "have", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["list", "the", "alaska", "airline", "flights", "from", "burbank", "to", "anywhere"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name", "O", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "the", "alaska", "airline", "flights", "from", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "B-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O"], "text": ["which", "airline", "is", "as"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "B-airline_code"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["list", "the", "alaska", "airlines", "flights", "arriving", "in", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "the", "alaska", "airlines", "flights", "a", "departing", "from", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O", "O", "O", "O", "B-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O"], "text": ["list", "all", "alaska", "airlines", "flights"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "B-airline_name", "I-airline_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name"], "text": ["list", "all", "flights", "departing", "from", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "flights", "from", "indianapolis", "to", "memphis", "that", "leave", "before", "noon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["list", "the", "cheapest", "fare", "from", "charlotte", "to", "las", "vegas"], "golden_intent": "atis_airfare", "golden_slot": ["O", "O", "B-cost_relative", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["i", "want", "a", "flight", "from", "los", "angeles", "to", "charlotte", "early", "in", "the", "morning"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.period_mod", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "a", "morning", "flight", "from", "charlotte", "to", "newark"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "a", "morning", "flight", "from", "newark", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "an", "evening", "flight", "from", "newark", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "that", "leaves", "on", "sunday", "from", "montreal", "quebec", "to", "san", "diego", "california"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-depart_date.day_name", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "on", "tuesday", "which", "leaves", "from", "san", "diego", "to", "indianapolis", "indiana", "and", "that", "leaves", "in", "the", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-depart_date.day_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name", "O", "O", "O", "O", "O", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "to", "leave", "thursday", "morning", "from", "indianapolis", "to", "toronto"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "on", "friday", "morning", "from", "toronto", "to", "montreal"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "a", "flight", "from", "cincinnati", "to", "burbank", "on", "american"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name"], "text": ["what", "type", "of", "aircraft", "is", "used", "for", "the", "american", "flight", "leaving", "at", "419", "pm"], "golden_intent": "atis_aircraft", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-airline_name", "O", "O", "O", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["i", "need", "a", "flight", "leaving", "kansas", "city", "to", "chicago", "leaving", "next", "wednesday", "and", "returning", "the", "following", "day"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "O", "B-return_date.date_relative", "I-return_date.date_relative", "I-return_date.date_relative"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "flights", "go", "from", "long", "beach", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "memphis", "to", "las", "vegas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "las", "vegas", "to", "ontario"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "ontario", "to", "memphis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_ground_service", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["what", "type", "of", "ground", "transportation", "is", "there", "at", "the", "las", "vegas", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-airport_name", "I-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["is", "there", "taxi", "service", "at", "the", "ontario", "airport"], "golden_intent": "atis_ground_service", "golden_slot": ["O", "O", "B-transport_type", "I-transport_type", "O", "O", "B-airport_name", "I-airport_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "tampa", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "milwaukee", "to", "seattle"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "la", "guardia", "to", "san", "jose", "on", "united"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.airport_name", "I-fromloc.airport_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "on", "mondays", "that", "travel", "from", "charlotte", "north", "carolina", "to", "phoenix", "arizona"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.day_name", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "I-fromloc.state_name", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "phoenix", "arizona", "to", "st.", "paul", "minnesota", "on", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "on", "thursday", "leaving", "from", "st.", "paul", "minnesota", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.day_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.state_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "st.", "louis", "to", "charlotte", "north", "carolina", "leaving", "on", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name", "I-toloc.state_name", "O", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "from", "boston", "to", "orlando", "that", "stop", "in", "new", "york"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "morning", "flight", "from", "burbank", "to", "milwaukee", "on", "next", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-depart_time.period_of_day", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.date_relative", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["how", "about", "a", "flight", "from", "milwaukee", "to", "st.", "louis", "that", "leaves", "monday", "night"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["and", "a", "flight", "from", "st.", "louis", "to", "burbank", "that", "leaves", "tuesday", "afternoon"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["how", "about", "a", "flight", "leaving", "tuesday", "night", "from", "st.", "louis", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.day_name", "B-depart_time.period_of_day", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "from", "salt", "lake", "to", "newark", "airport", "that", "arrives", "on", "saturday", "before", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.airport_name", "I-toloc.airport_name", "O", "O", "O", "B-arrive_date.day_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["i'd", "like", "a", "flight", "from", "cincinnati", "to", "newark", "airport", "that", "arrives", "on", "saturday", "before", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.airport_name", "I-toloc.airport_name", "O", "O", "O", "B-arrive_date.day_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "need", "a", "flight", "on", "american", "airlines", "from", "miami", "to", "chicago", "that", "arrives", "around", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-airline_name", "I-airline_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O"], "text": ["i", "need", "a", "flight", "from", "memphis", "to", "tacoma", "that", "goes", "through", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "flights", "between", "cincinnati", "and", "san", "jose", "california", "which", "leave", "after", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-toloc.state_name", "O", "O", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["what", "are", "the", "nonstop", "flights", "between", "san", "jose", "and", "houston", "texas"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "nonstop", "flights", "between", "houston", "and", "memphis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-flight_stop", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "flights", "between", "memphis", "and", "cincinnati", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["what", "are", "the", "american", "flights", "from", "newark", "to", "nashville"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-toloc.city_name"], "text": ["the", "flights", "from", "ontario", "to", "westchester", "that", "stop", "in", "chicago"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "B-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "the", "flights", "from", "los", "angeles", "to", "charlotte"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["please", "list", "the", "flights", "from", "charlotte", "to", "newark"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "the", "flights", "from", "newark", "to", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "the", "flights", "from", "cincinnati", "to", "burbank", "on", "american", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "give", "me", "the", "flights", "from", "kansas", "city", "to", "chicago", "on", "june", "sixteenth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "give", "me", "the", "flights", "from", "chicago", "to", "kansas", "city", "on", "june", "seventeenth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "all", "the", "flights", "from", "kansas", "city", "to", "chicago", "on", "june", "sixteenth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["please", "list", "all", "the", "flights", "from", "chicago", "to", "kansas", "city", "on", "june", "seventeenth"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i'd", "like", "to", "travel", "from", "burbank", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["can", "you", "find", "me", "a", "flight", "from", "salt", "lake", "city", "to", "new", "york", "city", "next", "saturday", "before", "arriving", "before", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name", "B-arrive_time.time_relative", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["can", "you", "find", "me", "another", "flight", "from", "cincinnati", "to", "new", "york", "on", "saturday", "before", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["can", "you", "list", "all", "of", "the", "delta", "flights", "from", "salt", "lake", "city", "to", "new", "york", "next", "saturday", "arriving", "before", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-airline_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "to", "fly", "from", "miami", "to", "chicago", "on", "on", "american", "airlines", "arriving", "around", "5", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-airline_name", "I-airline_name", "O", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "to", "travel", "from", "kansas", "city", "to", "chicago", "next", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_date.date_relative", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "a", "round", "trip", "flight", "from", "kansas", "city", "to", "chicago", "on", "wednesday", "may", "twenty", "sixth", "arriving", "at", "7", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_date.month_name", "B-depart_date.day_number", "I-depart_date.day_number", "O", "O", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["yes", "i'd", "like", "to", "find", "a", "flight", "from", "memphis", "to", "tacoma", "stopping", "in", "los", "angeles"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["find", "flight", "from", "san", "diego", "to", "phoenix", "on", "monday", "am"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name", "B-depart_time.period_of_day"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "flight", "from", "phoenix", "to", "detroit", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "flight", "from", "detroit", "to", "san", "diego", "on", "tuesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "flight", "from", "cincinnati", "to", "san", "jose", "on", "monday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "flight", "from", "san", "jose", "to", "houston", "on", "wednesday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "flight", "from", "houston", "to", "memphis", "on", "friday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "flight", "from", "memphis", "to", "cincinnati", "on", "sunday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name"], "text": ["find", "american", "flight", "from", "newark", "to", "nashville", "around", "630", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "B-airline_name", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-depart_time.time_relative", "B-depart_time.time", "I-depart_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "text": ["please", "find", "a", "flight", "round", "trip", "from", "los", "angeles", "to", "tacoma", "washington", "with", "a", "stopover", "in", "san", "francisco", "not", "exceeding", "the", "price", "of", "300", "dollars", "for", "june", "tenth", "1993"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "B-toloc.state_name", "O", "O", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name", "B-cost_relative", "I-cost_relative", "O", "O", "O", "B-fare_amount", "I-fare_amount", "O", "B-depart_date.month_name", "B-depart_date.day_number", "B-depart_date.year"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["are", "there", "any", "flights", "on", "june", "tenth", "from", "burbank", "to", "tacoma"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-depart_date.month_name", "B-depart_date.day_number", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "O", "O"], "text": ["please", "find", "a", "flight", "from", "ontario", "to", "westchester", "that", "makes", "a", "stop", "in", "chicago", "on", "may", "seventeenth", "one", "way", "with", "dinner"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "O", "O", "O", "B-stoploc.city_name", "O", "B-depart_date.month_name", "B-depart_date.day_number", "B-round_trip", "I-round_trip", "O", "B-meal_description"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["like", "to", "book", "a", "flight", "from", "burbank", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["show", "me", "all", "the", "flights", "from", "burbank", "to", "milwaukee"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["find", "me", "all", "the", "flights", "from", "milwaukee", "to", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["now", "show", "me", "all", "the", "flights", "from", "st.", "louis", "to", "burbank"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-city_name", "I-city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["is", "there", "one", "airline", "that", "flies", "from", "burbank", "to", "milwaukee", "milwaukee", "to", "st.", "louis", "and", "from", "st.", "louis", "to", "burbank"], "golden_intent": "atis_airline", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["find", "me", "all", "the", "round", "trip", "flights", "from", "burbank", "to", "milwaukee", "stopping", "in", "st.", "louis"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "O", "B-stoploc.city_name", "I-stoploc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "to", "book", "two", "flights", "to", "westchester", "county"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i", "want", "to", "book", "a", "flight", "from", "salt", "lake", "city", "to", "westchester", "county"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O"], "text": ["tell", "me", "all", "the", "airports", "near", "westchester", "county"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-city_name", "I-city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-fromloc.city_name", "O", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "to", "book", "a", "flight", "from", "cincinnati", "to", "new", "york", "city", "on", "united", "airlines", "for", "next", "saturday"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "B-airline_name", "I-airline_name", "O", "B-depart_date.date_relative", "B-depart_date.day_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O"], "text": ["tell", "me", "all", "the", "airports", "in", "the", "new", "york", "city", "area"], "golden_intent": "atis_airport", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-city_name", "I-city_name", "I-city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "O", "B-depart_time.period_of_day"], "text": ["please", "find", "all", "the", "flights", "from", "cincinnati", "to", "any", "airport", "in", "the", "new", "york", "city", "area", "that", "arrive", "next", "saturday", "before", "6", "pm"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O", "O", "O", "B-arrive_date.date_relative", "B-arrive_date.day_name", "B-arrive_time.time_relative", "B-arrive_time.time", "I-arrive_time.time"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name", "O", "O", "B-toloc.city_name", "O", "O", "O"], "text": ["find", "me", "a", "flight", "from", "cincinnati", "to", "any", "airport", "in", "the", "new", "york", "city", "area"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "O", "O", "O", "O", "B-toloc.city_name", "I-toloc.city_name", "I-toloc.city_name", "O"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-toloc.city_name", "I-toloc.city_name"], "text": ["i'd", "like", "to", "fly", "from", "miami", "to", "chicago", "on", "american", "airlines"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name", "O", "B-airline_name", "I-airline_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["i", "would", "like", "to", "book", "a", "round", "trip", "flight", "from", "kansas", "city", "to", "chicago"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "B-round_trip", "I-round_trip", "O", "O", "B-fromloc.city_name", "I-fromloc.city_name", "O", "B-toloc.city_name"]} +{"pred_intent": "atis_flight", "pred_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"], "text": ["find", "me", "a", "flight", "that", "flies", "from", "memphis", "to", "tacoma"], "golden_intent": "atis_flight", "golden_slot": ["O", "O", "O", "O", "O", "O", "O", "B-fromloc.city_name", "O", "B-toloc.city_name"]} diff --git a/save/stack/tokenizer.json b/save/stack/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a7176287ec199af7f53971a8e4d1215d48aaa1 --- /dev/null +++ b/save/stack/tokenizer.json @@ -0,0 +1 @@ +{"name": "word_tokenizer", "token_map": {"[PAD]": 0, "[UNK]": 1, "i": 2, "want": 3, "to": 4, "fly": 5, "from": 6, "baltimore": 7, "dallas": 8, "round": 9, "trip": 10, "fares": 11, "philadelphia": 12, "less": 13, "than": 14, "1000": 15, "dollars": 16, "denver": 17, "pittsburgh": 18, "show": 19, "me": 20, "the": 21, "flights": 22, "arriving": 23, "on": 24, "june": 25, "fourteenth": 26, "what": 27, "are": 28, "which": 29, "depart": 30, "san": 31, "francisco": 32, "washington": 33, "via": 34, "indianapolis": 35, "and": 36, "arrive": 37, "by": 38, "9": 39, "pm": 40, "airlines": 41, "boston": 42, "dc": 43, "other": 44, "cities": 45, "i'm": 46, "looking": 47, "for": 48, "a": 49, "flight": 50, "charlotte": 51, "las": 52, "vegas": 53, "that": 54, "stops": 55, "in": 56, "st.": 57, "louis": 58, "hopefully": 59, "dinner": 60, "how": 61, "can": 62, "find": 63, "out": 64, "okay": 65, "then": 66, "i'd": 67, "like": 68, "travel": 69, "atlanta": 70, "september": 71, "fourth": 72, "all": 73, "cincinnati": 74, "us": 75, "air": 76, "diego": 77, "afternoon": 78, "what's": 79, "available": 80, "tuesday": 81, "leave": 82, "phoenix": 83, "paul": 84, "minnesota": 85, "after": 86, "noon": 87, "american": 88, "chicago": 89, "los": 90, "angeles": 91, "morning": 92, "types": 93, "of": 94, "ground": 95, "transportation": 96, "there": 97, "airport": 98, "next": 99, "two": 100, "days": 101, "nashville": 102, "jose": 103, "or": 104, "tacoma": 105, "does": 106, "continental": 107, "milwaukee": 108, "many": 109, "twa": 110, "have": 111, "business": 112, "class": 113, "first": 114, "least": 115, "expensive": 116, "one": 117, "way": 118, "fare": 119, "booking": 120, "classes": 121, "wednesday": 122, "nineteenth": 123, "july": 124, "fifth": 125, "7": 126, "please": 127, "list": 128, "departing": 129, "general": 130, "mitchell": 131, "international": 132, "time": 133, "zone": 134, "is": 135, "serves": 136, "meal": 137, "seattle": 138, "salt": 139, "lake": 140, "city": 141, "you": 142, "with": 143, "economy": 144, "leaving": 145, "miami": 146, "cleveland": 147, "give": 148, "between": 149, "their": 150, "cost": 151, "code": 152, "y": 153, "mean": 154, "could": 155, "tell": 156, "leaves": 157, "united": 158, "airline": 159, "over": 160, "departures": 161, "arrivals": 162, "earliest": 163, "latest": 164, "return": 165, "within": 166, "same": 167, "day": 168, "orlando": 169, "either": 170, "evening": 171, "thursday": 172, "originating": 173, "going": 174, "order": 175, "snack": 176, "do": 177, "at": 178, "645": 179, "am": 180, "into": 181, "atlanta's": 182, "friday": 183, "qx": 184, "would": 185, "information": 186, "but": 187, "stopover": 188, "some": 189, "oakland": 190, "monday": 191, "know": 192, "type": 193, "aircraft": 194, "used": 195, "detroit": 196, "twenty": 197, "eighth": 198, "petersburg": 199, "take": 200, "begins": 201, "lands": 202, "fort": 203, "worth": 204, "stop": 205, "tomorrow": 206, "noontime": 207, "around": 208, "need": 209, "northwest": 210, "toronto": 211, "memphis": 212, "thirtieth": 213, "nonstop": 214, "houston": 215, "august": 216, "twentieth": 217, "ewr": 218, "seventh": 219, "newark": 220, "11": 221, "lowest": 222, "delta": 223, "has": 224, "go": 225, "any": 226, "jet": 227, "mco": 228, "new": 229, "jersey": 230, "ontario": 231, "saturday": 232, "york": 233, "long": 234, "it": 235, "get": 236, "prices": 237, "an": 238, "inexpensive": 239, "breakfast": 240, "direct": 241, "what're": 242, "sunday": 243, "colorado": 244, "see": 245, "serving": 246, "before": 247, "o'clock": 248, "january": 249, "1992": 250, "10": 251, "2": 252, "445": 253, "515": 254, "6": 255, "8": 256, "coach": 257, "only": 258, "weekdays": 259, "la": 260, "3": 261, "my": 262, "choices": 263, "early": 264, "minneapolis": 265, "cheapest": 266, "flying": 267, "possible": 268, "daily": 269, "beach": 270, "stopping": 271, "kansas": 272, "night": 273, "serve": 274, "meals": 275, "heading": 276, "kind": 277, "once": 278, "mornings": 279, "ff": 280, "arrangements": 281, "served": 282, "canadian": 283, "california": 284, "about": 285, "530": 286, "kinds": 287, "requesting": 288, "landing": 289, "distance": 290, "downtown": 291, "love": 292, "field": 293, "this": 294, "tampa": 295, "florida": 296, "5": 297, "must": 298, "be": 299, "advertises": 300, "having": 301, "more": 302, "november": 303, "eleventh": 304, "services": 305, "qo": 306, "american's": 307, "last": 308, "using": 309, "dl": 310, "217": 311, "montreal": 312, "service": 313, "ticket": 314, "should": 315, "near": 316, "also": 317, "missouri": 318, "utah": 319, "interested": 320, "those": 321, "when": 322, "north": 323, "carolina": 324, "415": 325, "200": 326, "explain": 327, "codes": 328, "sd": 329, "d": 330, "trying": 331, "plane": 332, "flies": 333, "weekday": 334, "columbus": 335, "1991": 336, "carries": 337, "smallest": 338, "number": 339, "passengers": 340, "takeoffs": 341, "landings": 342, "book": 343, "shortest": 344, "both": 345, "nationair": 346, "823": 347, "guardia": 348, "as": 349, "well": 350, "sixteenth": 351, "rental": 352, "car": 353, "rates": 354, "through": 355, "boeing": 356, "757": 357, "limousine": 358, "listing": 359, "canada": 360, "much": 361, "71": 362, "airfare": 363, "12": 364, "third": 365, "seating": 366, "capacity": 367, "arrives": 368, "bwi": 369, "ninth": 370, "late": 371, "nevada": 372, "4": 373, "price": 374, "fifteenth": 375, "eighteenth": 376, "returning": 377, "following": 378, "capacities": 379, "planes": 380, "1145": 381, "use": 382, "burbank": 383, "may": 384, "america": 385, "west": 386, "now": 387, "eastern": 388, "825": 389, "555": 390, "area": 391, "schedule": 392, "dfw": 393, "these": 394, "connecting": 395, "make": 396, "connection": 397, "lunch": 398, "f": 399, "belong": 400, "most": 401, "tickets": 402, "logan": 403, "vicinity": 404, "210": 405, "wednesdays": 406, "thursdays": 407, "yes": 408, "will": 409, "continuing": 410, "1039": 411, "southwest": 412, "times": 413, "400": 414, "week": 415, "if": 416, "813": 417, "enroute": 418, "another": 419, "twelfth": 420, "turboprop": 421, "420": 422, "today": 423, "1": 424, "we're": 425, "westchester": 426, "county": 427, "various": 428, "airplanes": 429, "uses": 430, "yn": 431, "852": 432, "transport": 433, "display": 434, "under": 435, "500": 436, "airfares": 437, "back": 438, "hours": 439, "fn": 440, "options": 441, "december": 442, "second": 443, "april": 444, "ohio": 445, "departs": 446, "2153": 447, "schedules": 448, "who": 449, "restriction": 450, "ap": 451, "57": 452, "layover": 453, "abbreviation": 454, "stands": 455, "1291": 456, "324": 457, "again": 458, "offer": 459, "dc10": 460, "currently": 461, "represented": 462, "database": 463, "arizona": 464, "1505": 465, "sixth": 466, "3724": 467, "three": 468, "including": 469, "connections": 470, "numbers": 471, "six": 472, "1100": 473, "destination": 474, "838": 475, "no": 476, "h": 477, "traveling": 478, "ap57": 479, "far": 480, "lufthansa": 481, "abbreviations": 482, "such": 483, "aa": 484, "459": 485, "where": 486, "ua": 487, "281": 488, "your": 489, "texas": 490, "1500": 491, "bound": 492, "includes": 493, "right": 494, "airports": 495, "eight": 496, "sixteen": 497, "trips": 498, "seventeenth": 499, "thrift": 500, "delta's": 501, "departure": 502, "listed": 503, "1055": 504, "405": 505, "midnight": 506, "hi": 507, "630": 508, "question": 509, "live": 510, "stand": 511, "ten": 512, "people": 513, "during": 514, "2100": 515, "gets": 516, "just": 517, "philly": 518, "21": 519, "airplane": 520, "1765": 521, "iah": 522, "737": 523, "midwest": 524, "express": 525, "s": 526, "designate": 527, "747": 528, "650": 529, "goes": 530, "reaches": 531, "seventeen": 532, "sorry": 533, "anywhere": 534, "provided": 535, "d10": 536, "toward": 537, "preferably": 538, "rate": 539, "difference": 540, "q": 541, "b": 542, "ac": 543, "tower": 544, "tenth": 545, "hp": 546, "4400": 547, "georgia": 548, "offers": 549, "fine": 550, "201": 551, "343": 552, "october": 553, "ea": 554, "jfk": 555, "name": 556, "arrange": 557, "largest": 558, "connect": 559, "operating": 560, "sundays": 561, "720": 562, "land": 563, "final": 564, "don't": 565, "stopovers": 566, "total": 567, "friday's": 568, "755": 569, "cheap": 570, "sfo": 571, "thirty": 572, "across": 573, "continent": 574, "makes": 575, "1220": 576, "co": 577, "1209": 578, "wanted": 579, "1850": 580, "without": 581, "listings": 582, "local": 583, "wish": 584, "bring": 585, "up": 586, "home": 587, "417": 588, "approximately": 589, "actually": 590, "1200": 591, "230": 592, "819": 593, "serviced": 594, "928": 595, "reservation": 596, "limousines": 597, "taxi": 598, "fit": 599, "72s": 600, "352": 601, "1133": 602, "43": 603, "define": 604, "directly": 605, "m80": 606, "close": 607, "restrictions": 608, "430": 609, "718": 610, "hou": 611, "costs": 612, "466": 613, "march": 614, "1026": 615, "1024": 616, "different": 617, "rentals": 618, "each": 619, "arrival": 620, "say": 621, "mealtime": 622, "932": 623, "1115": 624, "1245": 625, "include": 626, "whether": 627, "offered": 628, "130": 629, "alaska": 630, "296": 631, "they": 632, "106": 633, "york's": 634, "497766": 635, "itinerary": 636, "coming": 637, "month": 638, "bur": 639, "travels": 640, "pennsylvania": 641, "usa": 642, "1288": 643, "c": 644, "names": 645, "sure": 646, "meaning": 647, "ap80": 648, "269": 649, "reservations": 650, "d9s": 651, "sunday's": 652, "f28": 653, "934": 654, "earlier": 655, "1017": 656, "date": 657, "thank": 658, "oak": 659, "atl": 660, "cp": 661, "3357": 662, "1045": 663, "limo": 664, "845": 665, "sometime": 666, "1222": 667, "i'll": 668, "tennessee": 669, "0900": 670, "hello": 671, "let": 672, "repeat": 673, "provide": 674, "still": 675, "along": 676, "operation": 677, "year": 678, "one's": 679, "great": 680, "too": 681, "nighttime": 682, "1300": 683, "saturdays": 684, "416": 685, "four": 686, "257": 687, "minimum": 688, "intercontinental": 689, "february": 690, "spend": 691, "lastest": 692, "thing": 693, "originate": 694, "describe": 695, "concerning": 696, "sa": 697, "help": 698, "1700": 699, "225": 700, "1158": 701, "equipment": 702, "let's": 703, "wednesday's": 704, "quebec": 705, "highest": 706, "starting": 707, "taking": 708, "311": 709, "1230": 710, "able": 711, "put": 712, "later": 713, "takes": 714, "amount": 715, "qw": 716, "seven": 717, "maximum": 718, "yyz": 719, "it's": 720, "80": 721, "place": 722, "equal": 723, "while": 724, "train": 725, "look": 726, "815": 727, "takeoff": 728, "plan": 729, "2134": 730, "297": 731, "323": 732, "229": 733, "329": 734, "runs": 735, "730": 736, "closest": 737, "dulles": 738, "73s": 739, "so": 740, "economic": 741, "single": 742, "supper": 743, "110": 744, "calling": 745, "1205": 746, "55": 747, "michigan": 748, "proper": 749, "regarding": 750, "seats": 751, "19": 752, "m": 753, "midway": 754, "besides": 755, "reverse": 756, "1993": 757, "402": 758, "level": 759, "reaching": 760, "771": 761, "straight": 762, "located": 763, "305": 764, "repeating": 765, "indiana": 766, "connects": 767, "beginning": 768, "staying": 769, "town": 770, "cars": 771, "nonstops": 772, "300": 773, "345": 774, "dinnertime": 775, "sort": 776, "route": 777, "j31": 778, "tuesdays": 779, "212": 780, "705": 781, "red": 782, "eye": 783, "laying": 784, "friends": 785, "visit": 786, "here": 787, "them": 788, "lives": 789, "rent": 790, "279": 791, "137338": 792, "transcontinental": 793, "trans": 794, "world": 795, "1030": 796, "1130": 797, "come": 798, "727": 799, "1020": 800, "505": 801, "that's": 802, "163": 803, "ls": 804, "greatest": 805, "i've": 806, "got": 807, "somebody": 808, "else": 809, "wants": 810, "charges": 811, "734": 812, "carried": 813, "thirteenth": 814, "making": 815, "733": 816, "everywhere": 817, "prefer": 818, "run": 819, "non": 820, "315": 821, "746": 822, "companies": 823, "buy": 824, "very": 825, "270": 826, "locate": 827, "hartfield": 828, "start": 829, "98": 830, "inform": 831, "oh": 832, "82": 833, "139": 834, "1600": 835, "eleven": 836, "ord": 837, "mia": 838, "qualify": 839, "doesn't": 840, "mondays": 841, "catch": 842, "priced": 843, "bna": 844, "being": 845, "working": 846, "scenario": 847, "767": 848, "1940": 849, "150": 850, "100": 851, "afternoons": 852, "provides": 853, "723": 854, "1110": 855, "symbols": 856, "grounds": 857, "nw": 858, "539": 859, "soon": 860, "thereafter": 861, "scheduled": 862, "instead": 863, "810": 864, "lester": 865, "pearson": 866, "stapleton": 867, "615": 868, "twelve": 869, "bay": 870, "sounds": 871, "o'hare": 872, "ap68": 873, "fridays": 874, "try": 875, "fifteen": 876, "nights": 877, "determine": 878, "hold": 879, "lax": 880, "seat": 881, "k": 882, "planning": 883, "discount": 884, "summer": 885, "cover": 886, "271": 887, "tonight": 888, "off": 889, "124": 890, "thanks": 891, "longest": 892, "kindly": 893, "afterwards": 894, "overnight": 895, "1083": 896, "428": 897, "anything": 898, "1059": 899}} \ No newline at end of file diff --git a/static/css/style.css b/static/css/style.css new file mode 100644 index 0000000000000000000000000000000000000000..4298d946ab63c672517d16cde93535f0553f520b --- /dev/null +++ b/static/css/style.css @@ -0,0 +1,98 @@ +.card { + --phoenix-card-spacer-y: 1.5rem; + --phoenix-card-spacer-x: 1.5rem; + --phoenix-card-title-spacer-y: 1rem; + --phoenix-card-border-width: 1px; + --phoenix-card-border-color: var(--phoenix-gray-200); + --phoenix-card-border-radius: 0.5rem; + --phoenix-card-box-shadow: ; + --phoenix-card-inner-border-radius: calc(0.5rem - 1px); + --phoenix-card-cap-padding-y: 1.5rem; + --phoenix-card-cap-padding-x: 1.5rem; + --phoenix-card-cap-bg: var(--phoenix-card-cap-bg); + --phoenix-card-cap-color: ; + --phoenix-card-height: ; + --phoenix-card-color: ; + --phoenix-card-bg: #fff; + --phoenix-card-img-overlay-padding: 1rem; + --phoenix-card-group-margin: 1rem; + position: relative; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + -ms-flex-direction: column; + flex-direction: column; + min-width: 0; + height: var(--phoenix-card-height); + word-wrap: break-word; + background-color: var(--phoenix-card-bg); + background-clip: border-box; + border: var(--phoenix-card-border-width) solid var(--phoenix-card-border-color); + border-radius: var(--phoenix-card-border-radius); + -webkit-box-shadow: var(--phoenix-card-box-shadow); + box-shadow: var(--phoenix-card-box-shadow); +} +.h-100 { + height: 100% !important; +} +.card-body { + -webkit-box-flex: 1; + -ms-flex: 1 1 auto; + flex: 1 1 auto; + padding: var(--phoenix-card-spacer-y) var(--phoenix-card-spacer-x); + color: var(--phoenix-card-color); +} + +.justify-content-between { + -webkit-box-pack: justify !important; + -ms-flex-pack: justify !important; + justify-content: space-between !important; +} +.d-flex { + display: -webkit-box !important; + display: -ms-flexbox !important; + display: flex !important; +} +.pt-3 { + padding-top: 1rem !important; +} +.mb-2 { + margin-bottom: 0.5rem !important; +} +.align-items-center { + -webkit-box-align: center !important; + -ms-flex-align: center !important; + align-items: center !important; +} +.bullet-item { + height: 0.5rem; + width: 1rem; + border-radius: 2px; +} +.bg-primary { + --phoenix-bg-opacity: 1; + background-color: rgba(var(--phoenix-primary-rgb), var(--phoenix-bg-opacity)) !important; +} +.me-2 { + margin-right: 0.5rem !important; +} +.flex-1 { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; +} +.text-900 { + --phoenix-text-opacity: 1; + color: rgba(var(--phoenix-900-rgb), var(--phoenix-text-opacity)) !important; +} +.fw-semi-bold { + font-weight: 600 !important; +} +.mb-0 { + margin-bottom: 0 !important; +} +h6, .h6 { + font-size: 0.8rem; +} \ No newline at end of file diff --git a/static/favicon.ico b/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..7d2b5ffd61175d0dc0450166381b093d890bbfee Binary files /dev/null and b/static/favicon.ico differ diff --git a/static/template/application.html b/static/template/application.html new file mode 100644 index 0000000000000000000000000000000000000000..c92797bd17d9fcb5238854e32c29ce47db5e0834 --- /dev/null +++ b/static/template/application.html @@ -0,0 +1,97 @@ + + + + + + + + + + + + + +
+
+
Input Sample +
+
+
+ +
+
+ +
+
+
+
Prediction Result +
+
+
+
+ + + + + \ No newline at end of file diff --git a/static/template/visualization.html b/static/template/visualization.html new file mode 100644 index 0000000000000000000000000000000000000000..ededde14d239f1dcb1487ff584f5039d5772a037 --- /dev/null +++ b/static/template/visualization.html @@ -0,0 +1,453 @@ + + + + + OpenSLU Visual Analysis + + + + + + + + + + +
+
+ +
+
+
+
+ Error Intent Label Distribution +
+
+
+ +
+
+
+
+ Error Slot Label Distribution +
+
+
+
+
+
+
+
+
+

+

+
+
+
+
+
+ + + + +
+
+
+
Instance Analysis +
+
+ {% for example in examples %} + Intent: + {% for e in example["intent"] %} + {% if e["intent"] == e["pred_intent"] %} + + + {% else %} + + {% endif %} + {% endfor %} + +
+ Slot: + {% for e in example["slot"] %} + {% if e["slot"] == e["pred_slot"] %} + + + {% else %} + + {% endif %} + {% endfor %} +

+ {% endfor %} +
+
+
+
+
Page Index:
+
+
+ +
+
+
+ + +
+ +
+
+ + +
+
+ + + + + \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ + diff --git a/tools/load_from_hugging_face.py b/tools/load_from_hugging_face.py new file mode 100644 index 0000000000000000000000000000000000000000..ad50ed015d0d883e42bad22d3cf1f58c1d7fe103 --- /dev/null +++ b/tools/load_from_hugging_face.py @@ -0,0 +1,71 @@ +''' +Author: Qiguang Chen +LastEditors: Qiguang Chen +Date: 2023-02-13 10:44:39 +LastEditTime: 2023-02-14 10:28:43 +Description: + +''' + +import os +import dill +from common import utils +from common.utils import InputData, download +from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizer + + +# parser = argparse.ArgumentParser() +# parser.add_argument('--config_path', '-cp', type=str, default="config/reproduction/atis/joint_bert.yaml") +# args = parser.parse_args() +# config = Config.load_from_yaml(args.config_path) +# config.base["train"] = False +# config.base["test"] = False + +# model_manager = ModelManager(config) +# model_manager.load() + + +class PretrainedConfigForSLU(PretrainedConfig): + def __init__(self, **kargs) -> None: + super().__init__(**kargs) + +# pretrained_config = PretrainedConfigForSLU() +# # pretrained_config.push_to_hub("xxxx") + + +class PretrainedModelForSLU(PreTrainedModel): + def __init__(self, config: PretrainedConfig, *inputs, **kwargs) -> None: + super().__init__(config, *inputs, **kwargs) + self.config_class = config + self.model = utils.instantiate(config.model) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + cls.config_class = PretrainedConfigForSLU + return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + +class PreTrainedTokenizerForSLU(PreTrainedTokenizer): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + dir_names = f"save/{pretrained_model_name_or_path}".split("/") + dir_name = "" + for name in dir_names: + dir_name += name+"/" + if not os.path.exists(dir_name): + os.mkdir(dir_name) + cache_path = f"./save/{pretrained_model_name_or_path}/tokenizer.pkl" + if not os.path.exists(cache_path): + download(f"https://huggingface.co/{pretrained_model_name_or_path}/resolve/main/tokenizer.pkl", cache_path) + with open(cache_path, "rb") as f: + tokenizer = dill.load(f) + return tokenizer + + +# pretrained_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") +# pretrained_tokenizer = PreTrainedTokenizerForSLU.from_pretrained("LightChen2333/joint-bert-slu-atis") +# test_model = PretrainedModelForSLU.from_pretrained("LightChen2333/joint-bert-slu-atis") +# print(test_model(InputData([pretrained_tokenizer("I want to go to Beijing !")]))) \ No newline at end of file diff --git a/tools/parse_to_hugging_face.py b/tools/parse_to_hugging_face.py new file mode 100644 index 0000000000000000000000000000000000000000..4231d89f997ea42f16cc54b1489ef5b5a42c076d --- /dev/null +++ b/tools/parse_to_hugging_face.py @@ -0,0 +1,86 @@ +''' +Author: Qiguang Chen +LastEditors: Qiguang Chen +Date: 2023-02-13 10:44:39 +LastEditTime: 2023-02-19 15:45:08 +Description: + +''' + +import argparse +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import dill + +from common.config import Config +from common.model_manager import ModelManager +from transformers import PretrainedConfig, PreTrainedModel, AutoModel, AutoTokenizer, PreTrainedTokenizer + +class PretrainedConfigForSLUToSave(PretrainedConfig): + def __init__(self, **kargs) -> None: + cfg = model_manager.config + kargs["name_or_path"] = cfg.base["name"] + kargs["return_dict"] = False + kargs["is_decoder"] = True + kargs["_id2label"] = {"intent": model_manager.intent_list, "slot": model_manager.slot_list} + kargs["_label2id"] = {"intent": model_manager.intent_dict, "slot": model_manager.slot_dict} + kargs["_num_labels"] = {"intent": len(model_manager.intent_list), "slot": len(model_manager.slot_list)} + kargs["tokenizer_class"] = cfg.base["name"] + kargs["vocab_size"] = model_manager.tokenizer.vocab_size + kargs["model"] = cfg.model + kargs["model"]["decoder"]["intent_classifier"]["intent_label_num"] = len(model_manager.intent_list) + kargs["model"]["decoder"]["slot_classifier"]["slot_label_num"] = len(model_manager.slot_list) + kargs["tokenizer"] = cfg.tokenizer + len(model_manager.slot_list) + super().__init__(**kargs) + +class PretrainedModelForSLUToSave(PreTrainedModel): + def __init__(self, config: PretrainedConfig, *inputs, **kwargs) -> None: + super().__init__(config, *inputs, **kwargs) + self.model = model_manager.model + self.config_class = config + + +class PreTrainedTokenizerForSLUToSave(PreTrainedTokenizer): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.tokenizer = model_manager.tokenizer + + # @overload + def save_vocabulary(self, save_directory: str, filename_prefix = None): + if filename_prefix is not None: + path = os.path.join(save_directory, filename_prefix+"-tokenizer.pkl") + else: + path = os.path.join(save_directory, "tokenizer.pkl") + # tokenizer_name=model_manager.config.tokenizer.get("_tokenizer_name_") + # if tokenizer_name == "word_tokenizer": + # self.tokenizer.save(path) + # else: + # torch.save() + with open(path,'wb') as f: + dill.dump(self.tokenizer,f) + return (path,) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--config_path', '-cp', type=str, required=True) + parser.add_argument('--output_path', '-op', type=str, default="save/temp") + args = parser.parse_args() + config = Config.load_from_yaml(args.config_path) + config.base["train"] = False + config.base["test"] = False + if config.model_manager["load_dir"] is None: + config.model_manager["load_dir"] = config.model_manager["save_dir"] + model_manager = ModelManager(config) + model_manager.load() + model_manager.config.autoload_template() + + pretrained_config = PretrainedConfigForSLUToSave() + pretrained_model= PretrainedModelForSLUToSave(pretrained_config) + pretrained_model.save_pretrained(args.output_path) + + pretrained_tokenizer = PreTrainedTokenizerForSLUToSave() + pretrained_tokenizer.save_pretrained(args.output_path) diff --git a/tools/visualization.py b/tools/visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..9f9f18f26e5fb86f46c8209f2515125d2a6ffb31 --- /dev/null +++ b/tools/visualization.py @@ -0,0 +1,163 @@ +''' +Author: Qiguang Chen +LastEditors: Qiguang Chen +Date: 2023-01-23 17:26:47 +LastEditTime: 2023-02-14 20:07:02 +Description: + +''' +import argparse +import os +import signal +import sys + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import time +from gradio import networking +from common.utils import load_yaml, str2bool +import json +import threading + +from flask import Flask, request, render_template, render_template_string + + +def get_example(start, end, predict_data_file_path): + data_list = [] + with open(predict_data_file_path, "r", encoding="utf8") as f1: + for index, line1 in enumerate(f1): + if index < start: + continue + if index > end: + break + line1 = json.loads(line1.strip()) + obj = {"text": line1["text"]} + obj["intent"] = [{"intent": line1["golden_intent"], + "pred_intent": line1["pred_intent"]}] + obj["slot"] = [{"text": t, "pred_slot": ps, "slot": s} for t, s, ps in zip( + line1["text"], line1["pred_slot"], line1["golden_slot"])] + data_list.append(obj) + return data_list + + +def analysis(predict_data_file_path): + intent_dict = {} + slot_dict = {} + sample_num = 0 + with open(predict_data_file_path, "r", encoding="utf8") as f1: + for index, line1 in enumerate(f1): + sample_num += 1 + line1 = json.loads(line1.strip()) + for s, ps in zip(line1["golden_slot"], line1["pred_slot"]): + if s not in slot_dict: + slot_dict[s] = {"_error_": 0, "_total_": 0} + if s != ps: + slot_dict[s]["_error_"] += 1 + if ps not in slot_dict[s]: + slot_dict[s][ps] = 0 + slot_dict[s][ps] += 1 + slot_dict[s]["_total_"] += 1 + for i, pi in zip([line1["golden_intent"]], [line1["pred_intent"]]): + if i not in intent_dict: + intent_dict[i] = {"_error_": 0, "_total_": 0} + if i != pi: + intent_dict[i]["_error_"] += 1 + if pi not in intent_dict[i]: + intent_dict[i][pi] = 0 + intent_dict[i][pi] += 1 + intent_dict[i]["_total_"] += 1 + intent_dict_list = [{"value": intent_dict[name]["_error_"], "name": name} for name in intent_dict] + + for intent in intent_dict_list: + temp_intent = sorted( + intent_dict[intent["name"]].items(), key=lambda d: d[1], reverse=True) + # [:7] + temp_intent = [[key, value] for key, value in temp_intent] + intent_dict[intent["name"]] = temp_intent + slot_dict_list = [{"value": slot_dict[name]["_error_"], "name": name} for name in slot_dict] + + for slot in slot_dict_list: + temp_slot = sorted( + slot_dict[slot["name"]].items(), key=lambda d: d[1], reverse=True) + temp_slot = [[key, value] for key, value in temp_slot] + slot_dict[slot["name"]] = temp_slot + return intent_dict_list, slot_dict_list, intent_dict, slot_dict, sample_num + +parser = argparse.ArgumentParser() +parser.add_argument('--config_path', '-cp', type=str, default="config/visual.yaml") +parser.add_argument('--output_path', '-op', type=str, default=None) +parser.add_argument('--push_to_public', '-p', type=str2bool, nargs='?', + const=True, default=None, + help="Push to public network.(Higher priority than config file)") +args = parser.parse_args() +button_html = "" +config = load_yaml(args.config_path) +if args.output_path is not None: + config["output_path"] = args.output_path +if args.push_to_public is not None: + config["is_push_to_public"] = args.push_to_public +intent_dict_list, slot_dict_list, intent_dict, slot_dict, sample_num = analysis(config["output_path"]) +PAGE_SIZE = config["page-size"] +PAGE_NUM = int(sample_num / PAGE_SIZE) + 1 + +app = Flask(__name__, template_folder="static//template") + + +@app.route("/") +def hello(): + page = request.args.get('page') + if page is None: + page = 0 + page = int(page) if int(page) >= 0 else 0 + init_index = page*PAGE_SIZE + examples = get_example(init_index, init_index + + PAGE_SIZE - 1, config["output_path"]) + return render_template('visualization.html', + examples=examples, + intent_dict_list=intent_dict_list, + slot_dict_list=slot_dict_list, + intent_dict=intent_dict, + slot_dict=slot_dict, + page=page) + +thread_lock_1 = False + + + + +class PushToPublicThread(): + def __init__(self, config) -> None: + self.thread = threading.Thread(target=self.push_to_public, args=(config,)) + self.thread_lock_2 = False + self.thread.daemon = True + + def start(self): + self.thread.start() + + def push_to_public(self, config): + print("Push visualization results to public by Gradio....") + print("Push to URL: ", networking.setup_tunnel(config["host"], str(config["port"]))) + print("This share link expires in 72 hours. And do not close this process for public sharing.") + while not self.thread_lock_2: + continue + + def exit(self, signum, frame): + self.thread_lock_2 = True + print("Exit..") + os._exit(0) + # exit() +if __name__ == '__main__': + + if config["is_push_to_public"]: + + thread_1 = threading.Thread(target=lambda: app.run( + config["host"], config["port"])) + thread_1.start() + thread_2 = PushToPublicThread(config) + signal.signal(signal.SIGINT, thread_2.exit) + signal.signal(signal.SIGTERM, thread_2.exit) + thread_2.start() + while True: + time.sleep(1) + else: + app.run(config["host"], config["port"])