Init

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Tianbao Xie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,6 +1,6 @@
 ---
 title: Binder
-emoji: 👀
+emoji: 🔗
 colorFrom: green
 colorTo: green
 sdk: streamlit

app.py

@@ -0,0 +1,228 @@
+import json
+import os
+
+import pandas as pd
+import streamlit as st
+import argparse
+import traceback
+from typing import Dict
+import requests
+from utils.utils import load_data_split
+from nsql.database import NeuralDB
+from nsql.nsql_exec import NSQLExecutor
+from nsql.nsql_exec_python import NPythonExecutor
+from generation.generator import Generator
+import time
+
+ROOT_DIR = os.path.join(os.path.dirname(__file__), "./")
+EXAMPLE_TABLES = {
+    "Estonia men's national volleyball team": (558, "what are the total number of players from france?"),
+    "Highest mountain peaks of California": (5, "which is the lowest mountain?"),
+    "2010–11 UAB Blazers men's basketball team": (1, "how many players come from alabama?"),
+    "1999 European Tour": (209, "how many consecutive times was south africa the host country?"),
+    "Nissan SR20DET": (438, "which car is the only one with more than 230 hp?"),
+}
+
+
+@st.cache
+def load_data():
+    return load_data_split("missing_squall", "validation")
+
+
+@st.cache
+def get_key():
+    # print the public IP of the demo machine
+    ip = requests.get('https://checkip.amazonaws.com').text.strip()
+    print(ip)
+
+    URL = "http://54.242.37.195:20217/api/predict"
+    # The springboard machine we built to protect the key, 20217 is the birthday of Tianbao's girlfriend
+    # we will only let the demo machine have the access to the keys
+
+    one_key = requests.post(url=URL, json={"data": "Hi, binder server. Give me a key!"}).json()['data'][0]
+    return one_key
+
+
+def read_markdown(path):
+    with open(path, "r") as f:
+        output = f.read()
+    st.markdown(output, unsafe_allow_html=True)
+
+
+def generate_binder_program(_args, _generator, _data_item):
+    n_shots = _args.n_shots
+    few_shot_prompt = _generator.build_few_shot_prompt_from_file(
+        file_path=_args.prompt_file,
+        n_shots=n_shots
+    )
+    generate_prompt = _generator.build_generate_prompt(
+        data_item=_data_item,
+        generate_type=(_args.generate_type,)
+    )
+    prompt = few_shot_prompt + "\n\n" + generate_prompt
+
+    # Ensure the input length fit Codex max input tokens by shrinking the n_shots
+    max_prompt_tokens = _args.max_api_total_tokens - _args.max_generation_tokens
+    from transformers import AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=os.path.join(ROOT_DIR, "utils", "gpt2"))
+    while len(tokenizer.tokenize(prompt)) >= max_prompt_tokens:  # TODO: Add shrink rows
+        n_shots -= 1
+        assert n_shots >= 0
+        few_shot_prompt = _generator.build_few_shot_prompt_from_file(
+            file_path=_args.prompt_file,
+            n_shots=n_shots
+        )
+        prompt = few_shot_prompt + "\n\n" + generate_prompt
+
+    response_dict = _generator.generate_one_pass(
+        prompts=[("0", prompt)],  # the "0" is the place taker, take effect only when there are multi threads
+        verbose=_args.verbose
+    )
+    print(response_dict)
+    return response_dict["0"][0][0]
+
+
+# Set up
+parser = argparse.ArgumentParser()
+
+parser.add_argument('--prompt_file', type=str, default='templates/prompts/prompt_wikitq_v3.txt')
+# Binder program generation options
+parser.add_argument('--prompt_style', type=str, default='create_table_select_3_full_table',
+                    choices=['create_table_select_3_full_table',
+                             'create_table_select_full_table',
+                             'create_table_select_3',
+                             'create_table',
+                             'create_table_select_3_full_table_w_all_passage_image',
+                             'create_table_select_3_full_table_w_gold_passage_image',
+                             'no_table'])
+parser.add_argument('--generate_type', type=str, default='nsql',
+                    choices=['nsql', 'sql', 'answer', 'npython', 'python'])
+parser.add_argument('--n_shots', type=int, default=14)
+parser.add_argument('--seed', type=int, default=42)
+
+# Codex options
+# todo: Allow adjusting Codex parameters
+parser.add_argument('--engine', type=str, default="code-davinci-002")
+parser.add_argument('--max_generation_tokens', type=int, default=512)
+parser.add_argument('--max_api_total_tokens', type=int, default=8001)
+parser.add_argument('--temperature', type=float, default=0.)
+parser.add_argument('--sampling_n', type=int, default=1)
+parser.add_argument('--top_p', type=float, default=1.0)
+parser.add_argument('--stop_tokens', type=str, default='\n\n',
+                    help='Split stop tokens by ||')
+parser.add_argument('--qa_retrieve_pool_file', type=str, default='templates/qa_retrieve_pool.json')
+
+# debug options
+parser.add_argument('-v', '--verbose', action='store_false')
+args = parser.parse_args()
+keys = [get_key()]
+
+# The title
+st.markdown("# Binder Playground")
+
+# Summary about Binder
+read_markdown('resources/summary.md')
+
+# Introduction of Binder
+# todo: Write Binder introduction here
+# read_markdown('resources/introduction.md')
+st.image('resources/intro.png')
+
+# Upload tables/Switch tables
+
+st.markdown('### Try Binder!')
+col1, _ = st.columns(2)
+with col1:
+    selected_table_title = st.selectbox(
+        "Select an example table",
+        (
+            "Estonia men's national volleyball team",
+            "Highest mountain peaks of California",
+            "2010–11 UAB Blazers men's basketball team",
+            "1999 European Tour",
+            "Nissan SR20DET",
+        )
+    )
+
+# Here we just use ourselves'
+data_items = load_data()
+data_item = data_items[EXAMPLE_TABLES[selected_table_title][0]]
+table = data_item['table']
+header, rows, title = table['header'], table['rows'], table['page_title']
+db = NeuralDB(
+    [{"title": title, "table": table}])  # todo: try to cache this db instead of re-creating it again and again.
+df = db.get_table_df()
+st.markdown("Title: {}".format(title))
+st.dataframe(df)
+
+# Let user input the question
+question = st.text_input(
+    "Ask a question about the table:",
+    value=EXAMPLE_TABLES[selected_table_title][1]
+)
+with col1:
+    # todo: Why selecting language will flush the page?
+    selected_language = st.selectbox(
+        "Select a programming language",
+        ("SQL", "Python"),
+    )
+if selected_language == 'SQL':
+    args.prompt_file = 'templates/prompts/prompt_wikitq_v3.txt'
+    args.generate_type = 'nsql'
+elif selected_language == 'Python':
+    args.prompt_file = 'templates/prompts/prompt_wikitq_python_simplified_v4.txt'
+    args.generate_type = 'npython'
+else:
+    raise ValueError(f'{selected_language} language is not supported.')
+button = st.button("Generate program")
+if not button:
+    st.stop()
+
+# Generate Binder Program
+generator = Generator(args, keys=keys)
+with st.spinner("Generating program ..."):
+    binder_program = generate_binder_program(args, generator,
+                                             {"question": question, "table": db.get_table_df(), "title": title})
+
+
+# Do execution
+st.markdown("#### Binder program")
+if selected_language == 'SQL':
+    with st.container():
+        st.write(binder_program)
+    executor = NSQLExecutor(args, keys=keys)
+elif selected_language == 'Python':
+    st.code(binder_program, language='python')
+    executor = NPythonExecutor(args, keys=keys)
+    db = db.get_table_df()
+else:
+    raise ValueError(f'{selected_language} language is not supported.')
+try:
+    os.makedirs('tmp_for_vis/', exist_ok=True)
+    with st.spinner("Executing program ..."):
+        exec_answer = executor.nsql_exec(binder_program, db)
+    # todo: Make it more pretty!
+    # todo: Do we need vis for Python?
+    if selected_language == 'SQL':
+        with open("tmp_for_vis/tmp_for_vis_steps.txt", "r") as f:
+            steps = json.load(f)
+        st.markdown("#### Steps & Intermediate results")
+        for i, step in enumerate(steps):
+            st.markdown(step)
+            st.text("↓")
+            with st.spinner('...'):
+                time.sleep(1)
+            with open("tmp_for_vis/result_step_{}.txt".format(i), "r") as f:
+                result_in_this_step = json.load(f)
+            if isinstance(result_in_this_step, Dict):
+                st.dataframe(pd.DataFrame(pd.DataFrame(result_in_this_step["rows"], columns=result_in_this_step["header"])))
+            else:
+                st.markdown(result_in_this_step)
+            st.text("↓")
+    elif selected_language == 'Python':
+        pass
+    if isinstance(exec_answer, list) and len(exec_answer) == 1:
+        exec_answer = exec_answer[0]
+    st.markdown(f'Execution answer: {exec_answer}')
+except Exception as e:
+    traceback.print_exc()

datasets/missing_squall.py

@@ -0,0 +1,133 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Datasets Authors, The Google AI Language Team Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The WikiTableQuestions dataset is for the task of question answering on semi-structured HTML tables"""
+
+import json
+import os
+import datasets
+from utils.wtq.utils import _load_table_w_page as _load_table
+
+# Find for instance the citation on arxiv or on the dataset repo/website
+_CITATION = """\
+@inproceedings{pasupat-liang-2015-compositional,
+    title = "Compositional Semantic Parsing on Semi-Structured Tables",
+    author = "Pasupat, Panupong  and
+      Liang, Percy",
+    booktitle = "Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
+    month = jul,
+    year = "2015",
+    address = "Beijing, China",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/P15-1142",
+    doi = "10.3115/v1/P15-1142",
+    pages = "1470--1480",
+}
+"""
+
+_DESCRIPTION = """\
+Two important aspects of semantic parsing for question answering are the breadth of the knowledge source and the depth of
+logical compositionality. While existing work trades off one aspect for another, this paper simultaneously makes progress 
+on both fronts through a new task: answering complex questions on semi-structured tables using question-answer pairs as 
+supervision. The central challenge arises from two compounding factors: the broader domain results in an open-ended set 
+of relations, and the deeper compositionality results in a combinatorial explosion in the space of logical forms. We 
+propose a logical-form driven parsing algorithm guided by strong typing constraints and show that it obtains significant
+ improvements over natural baselines. For evaluation, we created a new dataset of 22,033 complex questions on Wikipedia
+  tables, which is made publicly available.
+"""
+
+_HOMEPAGE = "https://ppasupat.github.io/WikiTableQuestions/"
+
+_LICENSE = "CC-BY-SA-4.0 License"
+
+_URL = "https://github.com/ppasupat/WikiTableQuestions/archive/refs/heads/master.zip"
+_SQUALL_URL = "https://github.com/tzshi/squall/archive/refs/heads/main.zip"
+
+
+class WikiTableQuestion(datasets.GeneratorBasedBuilder):
+    """The WikiTableQuestions dataset"""
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "id": datasets.Value("string"),
+                    "question": datasets.Value("string"),
+                    "table_id": datasets.Value("string"),
+                    "table": {"page_title": datasets.Value("string"),
+                              "header": datasets.features.Sequence(datasets.Value("string")),
+                              "rows": datasets.features.Sequence(datasets.features.Sequence(datasets.Value("string")))},
+                    "answer_text": datasets.features.Sequence(datasets.Value("string")),
+                }
+            ),
+            supervised_keys=None,
+            homepage=_HOMEPAGE,
+            license=_LICENSE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        """Returns SplitGenerators."""
+        data_dir = os.path.join(dl_manager.download_and_extract(_URL), 'WikiTableQuestions-master')
+        squall_dir = os.path.join(dl_manager.download_and_extract(_SQUALL_URL), 'squall-main')
+
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                gen_kwargs={"filepath": os.path.join(data_dir, "data/random-split-1-train.tsv"),
+                            "data_dir": data_dir,
+                            "squall_path": os.path.join(squall_dir, "data/squall.json")},
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                gen_kwargs={"filepath": os.path.join(data_dir, "data/random-split-1-dev.tsv"),
+                            "data_dir": data_dir,
+                            "squall_path": os.path.join(squall_dir, "data/squall.json")},
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.TEST,
+                gen_kwargs={"filepath": os.path.join(data_dir, "data/pristine-unseen-tables.tsv"),
+                            "data_dir": data_dir,
+                            "squall_path": os.path.join(squall_dir, "data/squall.json")},
+            ),
+
+        ]
+
+    def _generate_examples(self, filepath, data_dir, squall_path):
+        """Yields examples."""
+        squall_id_list = []
+        with open(squall_path) as f:
+            squall_data = json.load(f)
+            for squall_item in squall_data:
+                squall_id_list.append(squall_item["nt"])
+        # data_id, question, table_id, gold_result_str
+        with open(filepath, encoding="utf-8") as f:
+            for idx, line in enumerate(f):
+                # skip the header
+                if idx == 0:
+                    continue
+                data_id, question, table_id, gold_result_str = line.strip("\n").split("\t")
+                if data_id not in squall_id_list:
+                    gold_result = gold_result_str.split('|')
+                    yield idx, {
+                        "id": data_id,
+                        "question": question,
+                        "table_id": table_id,
+                        "table": _load_table(os.path.join(data_dir, table_id.replace('.csv', '.tsv'))),
+                        # convert the .csv postfix to .tsv, for easier read-in
+                        "answer_text": gold_result,
+                    }
+                else:
+                    continue

demos/get_key.py

@@ -0,0 +1,10 @@
+import requests
+
+
+def get_key():
+    URL = "http://54.242.37.195:20217/api/predict"
+    # The springboard machine we built to protect the key, 20217 is the birthday of Tianbao's girlfriend
+    # we will only let the demo machine have the access to the keys
+
+    one_key = requests.post(url=URL, json={"data": "Hi, binder server. Give me a key!"}).json()['data'][0]
+    return one_key

generation/generator.py

@@ -0,0 +1,180 @@
+"""
+Generate nsql and questions.
+"""
+
+from typing import Dict, List, Union, Tuple
+import openai
+import time
+
+from generation.prompt import PromptBuilder
+
+
+class Generator(object):
+    """
+    Codex generation wrapper.
+    """
+
+    def __init__(self, args, keys=None):
+        self.args = args
+        self.keys = keys
+        self.current_key_id = 0
+
+        # if the args provided, will initialize with the prompt builder for full usage
+        self.prompt_builder = PromptBuilder(args) if args else None
+
+    def prompt_row_truncate(
+            self,
+            prompt: str,
+            num_rows_to_remain: int,
+            table_end_token: str = '*/',
+    ):
+        """
+        Fit prompt into max token limits by row truncation.
+        """
+        table_end_pos = prompt.rfind(table_end_token)
+        assert table_end_pos != -1
+        prompt_part1, prompt_part2 = prompt[:table_end_pos], prompt[table_end_pos:]
+        prompt_part1_lines = prompt_part1.split('\n')[::-1]
+        trunc_line_index = None
+        for idx, line in enumerate(prompt_part1_lines):
+            if '\t' not in line:
+                continue
+            row_id = int(line.split('\t')[0])
+            if row_id <= num_rows_to_remain:
+                trunc_line_index = idx
+                break
+        new_prompt_part1 = '\n'.join(prompt_part1_lines[trunc_line_index:][::-1])
+        prompt = new_prompt_part1 + '\n' + prompt_part2
+        return prompt
+
+    def build_few_shot_prompt_from_file(
+            self,
+            file_path: str,
+            n_shots: int
+    ):
+        """
+        Build few-shot prompt for generation from file.
+        """
+        with open(file_path, 'r') as f:
+            lines = f.readlines()
+        few_shot_prompt_list = []
+        one_shot_prompt = ''
+        last_line = None
+        for line in lines:
+            if line == '\n' and last_line == '\n':
+                few_shot_prompt_list.append(one_shot_prompt)
+                one_shot_prompt = ''
+            else:
+                one_shot_prompt += line
+            last_line = line
+        few_shot_prompt_list.append(one_shot_prompt)
+        few_shot_prompt_list = few_shot_prompt_list[:n_shots]
+        few_shot_prompt_list[-1] = few_shot_prompt_list[
+            -1].strip()  # It is essential for prompting to remove extra '\n'
+        few_shot_prompt = '\n'.join(few_shot_prompt_list)
+        return few_shot_prompt
+
+    def build_generate_prompt(
+            self,
+            data_item: Dict,
+            generate_type: Tuple
+    ):
+        """
+        Build the generate prompt
+        """
+        return self.prompt_builder.build_generate_prompt(
+            **data_item,
+            generate_type=generate_type
+        )
+
+    def generate_one_pass(
+            self,
+            prompts: List[Tuple],
+            verbose: bool = False
+    ):
+        """
+        Generate one pass with codex according to the generation phase.
+        """
+        result_idx_to_eid = []
+        for p in prompts:
+            result_idx_to_eid.extend([p[0]] * self.args.sampling_n)
+        prompts = [p[1] for p in prompts]
+
+        start_time = time.time()
+
+        result = self._call_codex_api(
+            engine=self.args.engine,
+            prompt=prompts,
+            max_tokens=self.args.max_generation_tokens,
+            temperature=self.args.temperature,
+            top_p=self.args.top_p,
+            n=self.args.sampling_n,
+            stop=self.args.stop_tokens
+        )
+        print(f'Openai api one inference time: {time.time() - start_time}')
+
+        if verbose:
+            print('\n', '*' * 20, 'Codex API Call', '*' * 20)
+            for prompt in prompts:
+                print(prompt)
+                print('\n')
+            print('- - - - - - - - - - ->>')
+
+        # parse api results
+        response_dict = dict()
+        for idx, g in enumerate(result['choices']):
+            try:
+                text = g['text']
+                logprob = sum(g['logprobs']['token_logprobs'])
+                eid = result_idx_to_eid[idx]
+                eid_pairs = response_dict.get(eid, None)
+                if eid_pairs is None:
+                    eid_pairs = []
+                    response_dict[eid] = eid_pairs
+                eid_pairs.append((text, logprob))
+
+                if verbose:
+                    print(text)
+
+            except ValueError as e:
+                if verbose:
+                    print('----------- Error Msg--------')
+                    print(e)
+                    print(text)
+                    print('-----------------------------')
+                pass
+
+        return response_dict
+
+    def _call_codex_api(
+            self,
+            engine: str,
+            prompt: Union[str, List],
+            max_tokens,
+            temperature: float,
+            top_p: float,
+            n: int,
+            stop: List[str]
+    ):
+        start_time = time.time()
+        result = None
+        while result is None:
+            try:
+                key = self.keys[self.current_key_id]
+                self.current_key_id = (self.current_key_id + 1) % len(self.keys)
+                result = openai.Completion.create(
+                    engine=engine,
+                    prompt=prompt,
+                    api_key=key,
+                    max_tokens=max_tokens,
+                    temperature=temperature,
+                    top_p=top_p,
+                    n=n,
+                    stop=stop,
+                    logprobs=1
+                )
+                print('Openai api inference time:', time.time() - start_time)
+                return result
+            except Exception as e:
+                print(e, 'Retry.')
+                time.sleep(5)

generation/prompt.py

@@ -0,0 +1,499 @@
+"""
+Build NSQL generation prompt.
+Two main parts:
+1) PromptBuilder makes prompt for calling codex to generate NSQL(Binder-SQL).
+2) OpenAIQAPromptBuilder makes prompt for calling codex to generate QA answers.
+"""
+
+import random
+from typing import Dict, Tuple
+import pandas as pd
+import copy
+
+from utils.errors import DuplicateColumnsError
+from utils.mmqa.image_stuff import get_caption_map
+from retrieval.retrieve_pool import QAItem
+
+from utils.normalizer import prepare_df_for_neuraldb_from_table
+
+
+def _create_table_prompt(df: pd.DataFrame, title: str):
+    """
+    Return the CREATE TABLE clause as prompt.
+    """
+    string = "CREATE TABLE {}(\n".format(title)
+    for header in df.columns:
+        column_type = 'text'
+        try:
+            if df[header].dtype == 'int64':
+                column_type = 'int'
+            elif df[header].dtype == 'float64':
+                column_type = 'real'
+            elif df[header].dtype == 'datetime64':
+                column_type = 'datetime'
+        except AttributeError as e:
+            raise DuplicateColumnsError(e)
+
+        string += '\t{} {},\n'.format(header, column_type)
+    string = string.rstrip(',\n') + ')\n'
+    return string
+
+
+class PromptBuilder(object):
+    def __init__(self, args):
+        self.args = args
+        self.prompt_style = args.prompt_style
+        random.seed(args.seed)
+
+    def _select_x_prompt(self, df: pd.DataFrame, num_rows: int,
+                         few_shot_demonstration=True):
+        """
+        Return the first X rows table contents as prompt.
+        """
+        if self.prompt_style == 'create_table_select_full_table':
+            string = '/*\nAll rows of the table:\nSELECT * FROM w;\n'
+        elif self.prompt_style == 'create_table_select_3':
+            string = '/*\n{} example rows:\nSELECT * FROM w LIMIT {};\n'.format(num_rows, num_rows)
+        elif self.prompt_style == 'create_table_select_3_hidden':
+            string = '/*\n{} example rows:\n'.format(num_rows)
+        elif few_shot_demonstration is True and self.prompt_style in \
+                ["create_table_select_3_full_table",
+                 "create_table_select_3_full_table_w_gold_passage_image",
+                 "create_table_select_3_full_table_w_all_passage_image"]:
+            string = '/*\n{} example rows:\nSELECT * FROM w LIMIT {};\n'.format(num_rows, num_rows)
+        elif few_shot_demonstration is False and self.prompt_style in \
+                ["create_table_select_3_full_table",
+                 "create_table_select_3_full_table_w_gold_passage_image",
+                 "create_table_select_3_full_table_w_all_passage_image"]:
+            string = '/*\nAll rows of the table:\nSELECT * FROM w;\n'
+        else:
+            raise ValueError(f"Select x prompt style {self.prompt_style} is not supported.")
+
+        for column_id, header in enumerate(df.columns):
+            string += str(header)
+            if column_id != len(df.columns) - 1:
+                string += '\t'
+        string += '\n'
+
+        for row_id, row in df.iloc[:num_rows].iterrows():
+            for column_id, header in enumerate(df.columns):
+                string += str(row[header])
+                if column_id != len(df.columns) - 1:
+                    string += '\t'
+            string += '\n'
+        string += '*/\n'
+
+        return string
+
+    def _passage_prompt(self, passages, only_title, db_style_prompt=True):
+        """
+        Return the passage prompt.
+        """
+        if not db_style_prompt:
+            string = "Passages: "
+            for passage in passages:
+                if only_title:
+                    string += passage['title'] + ';; '
+                else:
+                    string += passage['title'] + f" ({passage['text']})" + ';; '
+            string = string.rstrip(';; ')
+            string += '\n'
+            return string
+        else:
+            if len(passages) == 0:
+                return ""
+            passage_table_prompt = ""
+            _header = []
+            _rows = [[]]
+            for passage in passages:
+                _header.append(passage['title'])
+                _rows[0].append(passage['text'])
+            passage_table = prepare_df_for_neuraldb_from_table({"header": _header, "rows": _rows})
+            passage_table_prompt += _create_table_prompt(passage_table, "Passages")
+            if not only_title:
+                passage_table_prompt += self._select_x_prompt(
+                    df=passage_table,
+                    num_rows=passage_table.shape[0]
+                )
+            return passage_table_prompt
+
+    def _image_prompt(self, images, only_title, db_style_prompt=True):
+        """
+        Return the image prompt.
+        """
+        if not db_style_prompt:
+            string = "Images: "
+            for image in images:
+                if only_title:
+                    string += image['title'] + ';;'
+                else:
+                    string += image['title'] + f" ({image['caption']})" + ';; '
+            string = string.rstrip(';; ')
+            string += '\n'
+            return string
+        else:
+            if len(images) == 0:
+                return ""
+            image_table_prompt = ""
+            _header = []
+            _rows = [[]]
+            for image in images:
+                _header.append(image['title'])
+                _rows[0].append(image['caption'])
+            image_table = prepare_df_for_neuraldb_from_table({"header": _header, "rows": _rows})
+            image_table_prompt += _create_table_prompt(image_table, "Images")
+            if not only_title:
+                image_table_prompt += self._select_x_prompt(
+                    df=image_table,
+                    num_rows=image_table.shape[0]
+                )
+            return image_table_prompt
+
+    def _pick_target_columns(self, df, strategy):
+        """
+        Pick the controllable target columns for generation.
+        """
+        if strategy == 'random':
+            return random.choice(list(df.columns) + ['*'])
+        elif strategy == 'traverse':
+            raise NotImplementedError
+        else:
+            return ValueError
+
+    def _pick_operators(self, df, strategy):
+        """
+        Pick the controllable operators for generation.
+        """
+        candidate_operators = ['none', 'count', 'max', 'min', 'sum']
+        if strategy == 'random':
+            return random.choice(candidate_operators)
+        elif strategy == 'traverse':
+            raise NotImplementedError
+        else:
+            return ValueError
+
+    def _pick_nested_levels(self, df, strategy):
+        """
+        Pick the controllable(maybe) nested levels for generation.
+        """
+        if strategy == 'fixed':
+            return 2
+        elif strategy == 'random':
+            raise NotImplementedError
+        elif strategy == 'traverse':
+            raise NotImplementedError
+        else:
+            raise ValueError
+
+    def build_one_shot_prompt(
+            self,
+            prompt_type: Tuple,
+            table: pd.DataFrame,
+            question: str,
+            answer_text: str,
+            nsql: str,
+            passages: Dict = None,
+            images: Dict = None,
+            title: str = None,
+            only_title: bool = False,
+            **kwargs
+    ):
+        """
+        Build one-shot prompt with table-question-nsql.
+        """
+        one_shot_prompt = ""
+        if self.prompt_style == 'create_table_select_full_table':
+            one_shot_prompt += _create_table_prompt(table, title)
+            one_shot_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=table.shape[0]
+            )
+        elif self.prompt_style in ['create_table_select_3_full_table', 'create_table_select_3']:
+            one_shot_prompt += _create_table_prompt(table, title)
+            one_shot_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=3,
+            )
+        elif self.prompt_style == 'create_table':
+            one_shot_prompt += _create_table_prompt(table, title)
+        elif self.prompt_style == 'no_table':
+            # No table input, to test Codex QA with only internal knowledge
+            pass
+        elif self.prompt_style in ['create_table_select_3_full_table_w_all_passage_image']:
+            assert passages is not None and images is not None
+            one_shot_prompt += _create_table_prompt(table, title)
+            one_shot_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=3,
+            )
+            all_passages, all_images = [], []
+            caption_map = get_caption_map()
+
+            for passage_idx in range(len(passages['id'])):
+                all_passages.append({
+                    'id': passages['id'][passage_idx],
+                    'title': passages['title'][passage_idx],
+                    'url': passages['url'][passage_idx],
+                    'text': passages['text'][passage_idx]
+                })
+
+            for image_idx in range(len(images['id'])):
+                all_images.append({
+                    "id": images['id'][image_idx],
+                    "title": images['title'][image_idx],
+                    "url": images['url'][image_idx],
+                    "path": images['path'][image_idx],
+                    "pic": images['pic'][image_idx],
+                    "caption": caption_map[images['id'][image_idx]]
+                })
+
+            one_shot_prompt += self._passage_prompt(
+                passages=all_passages,
+                only_title=only_title
+            )
+            one_shot_prompt += self._image_prompt(
+                images=all_images,
+                only_title=only_title
+            )
+        else:
+            raise ValueError('{} is not supported.'.format(self.prompt_style))
+
+        # question and nsql pairs
+        if prompt_type == ('question', 'nsql'):
+            one_shot_prompt += 'Q: {}\n'.format(question)
+            one_shot_prompt += 'NeuralSQL: {}\n'.format(nsql)
+        elif prompt_type == ('question', 'sql'):
+            one_shot_prompt += 'Q: {}\n'.format(question)
+            one_shot_prompt += 'SQL: {}\n'.format(nsql)
+        elif prompt_type == ('question', 'answer'):
+            one_shot_prompt += 'Q: {}\n'.format(question)
+            one_shot_prompt += 'A: {}\n'.format(', '.join(answer_text))
+        else:
+            raise ValueError(f'Prompt type {prompt_type} is not supported.')
+
+        return one_shot_prompt
+
+    def build_generate_prompt(
+            self,
+            generate_type: Tuple,
+            table: pd.DataFrame,
+            question: str = None,
+            passages: Dict = None,
+            images: Dict = None,
+            title: str = None,
+            only_title: bool = False,
+            supporting_context: Dict = None,
+            **kwargs
+    ):
+        """
+        Build the prompt of the generation sample.
+        """
+        generate_prompt = ""
+
+        # task instruction
+        if generate_type == ('answer',):
+            generate_prompt += """\n-- Answer the question based on the given table below.\n\n"""
+        elif generate_type == ('nsql',):
+            generate_prompt += """\n-- Parse the question into NeuralSQL based on the given table below.\n\n"""
+        elif generate_type == ('sql',):
+            generate_prompt += """\n-- Parse the question into SQL based on the given table below.\n\n"""
+        elif generate_type == ('npython',):
+            generate_prompt += """\n-- Parse the question into NeuralPython based on the given table below.\n\n"""
+        elif generate_type == ('python',):
+            generate_prompt += """\n-- Parse the question into Python based on the given table below.\n\n"""
+        else:
+            generate_prompt += """\n-- Generate NeuralSQL and question pairs based on the given table below.\n\n"""
+
+        # table prompt
+        if self.prompt_style in ['create_table_select_full_table', 'create_table_select_3_full_table']:
+            generate_prompt += _create_table_prompt(table, title)
+            generate_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=table.shape[0],
+                few_shot_demonstration=False
+            )
+        elif self.prompt_style in ['create_table_select_3']:
+            generate_prompt += _create_table_prompt(table, title)
+            generate_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=3,
+                few_shot_demonstration=False
+            )
+        elif self.prompt_style == 'create_table':
+            generate_prompt += _create_table_prompt(table, title)
+        elif self.prompt_style == 'no_table':
+            # No table input, to test Codex QA with only internal knowledge
+            pass
+        elif self.prompt_style in ['create_table_select_3_full_table_w_all_passage_image']:
+            assert passages is not None and images is not None
+            generate_prompt += _create_table_prompt(table, title)
+            generate_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=table.shape[0],
+                few_shot_demonstration=False
+            )
+            all_passages, all_images = [], []
+            caption_map = get_caption_map()
+
+            for passage_idx in range(len(passages['id'])):
+                all_passages.append({
+                    'id': passages['id'][passage_idx],
+                    'title': passages['title'][passage_idx],
+                    'url': passages['url'][passage_idx],
+                    'text': passages['text'][passage_idx]
+                })
+
+            for image_idx in range(len(images['id'])):
+                all_images.append({
+                    "id": images['id'][image_idx],
+                    "title": images['title'][image_idx],
+                    "url": images['url'][image_idx],
+                    "path": images['path'][image_idx],
+                    "pic": images['pic'][image_idx],
+                    "caption": caption_map[images['id'][image_idx]]
+                })
+
+            generate_prompt += self._passage_prompt(
+                passages=all_passages,
+                only_title=only_title
+            )
+            generate_prompt += self._image_prompt(
+                images=all_images,
+                only_title=only_title
+            )
+        elif self.prompt_style in ['create_table_select_3_full_table_w_gold_passage_image']:
+            assert passages is not None and images is not None
+            generate_prompt += _create_table_prompt(table, title)
+            generate_prompt += self._select_x_prompt(
+                df=table,
+                num_rows=table.shape[0],
+                few_shot_demonstration=False
+            )
+            gold_passages, gold_images = [], []
+            caption_map = get_caption_map()
+            for doc_id, doc_part in zip(supporting_context['doc_id'], supporting_context['doc_part']):
+                if doc_part == 'text':
+                    passage_idx = passages['id'].index(doc_id)
+                    gold_passages.append({
+                        'id': passages['id'][passage_idx],
+                        'title': passages['title'][passage_idx],
+                        'url': passages['url'][passage_idx],
+                        'text': passages['text'][passage_idx]
+                    })
+                elif doc_part == 'image':
+                    image_idx = images['id'].index(doc_id)
+                    gold_images.append({
+                        "id": images['id'][image_idx],
+                        "title": images['title'][image_idx],
+                        "url": images['url'][image_idx],
+                        "path": images['path'][image_idx],
+                        "pic": images['pic'][image_idx],
+                        "caption": caption_map[doc_id]
+                    })
+            generate_prompt += self._passage_prompt(
+                passages=gold_passages,
+                only_title=only_title
+            )
+            generate_prompt += self._image_prompt(
+                images=gold_images,
+                only_title=only_title
+            )
+        else:
+            raise ValueError('{} is not supported.'.format(self.prompt_style))
+
+        # determine the target to generate
+        if generate_type == ('answer',):
+            generate_prompt += 'Q: {}\n'.format(question)
+            generate_prompt += 'A: '
+        elif generate_type == ('nsql',):
+            generate_prompt += 'Q: {}\n'.format(question)
+            generate_prompt += 'NeuralSQL: '
+        elif generate_type == ('sql',):
+            generate_prompt += 'Q: {}\n'.format(question)
+            generate_prompt += 'SQL: '
+        elif generate_type == ('npython',):
+            generate_prompt += 'Q: {}\n'.format(question)
+            generate_prompt += 'NeuralPython: '
+        elif generate_type == ('python',):
+            generate_prompt += 'Q: {}\n'.format(question)
+            generate_prompt += 'Python: '
+        else:
+            raise ValueError(f'Generate type {generate_type} is not supported.')
+
+        return generate_prompt
+
+
+class OpenAIQAPromptBuilder(object):
+    @staticmethod
+    def table2codex_prompt(table, table_title=None, drop_row_id=True, ):
+        _table = copy.deepcopy(table)
+        header = _table['header']
+        rows = _table['rows']
+        if drop_row_id:
+            if header[0] == "row_id":
+                header = header[1:]
+                rows = [_row[1:] for _row in rows]
+        prompt_str = 'Table: {}\n'.format(table_title) if table_title else ''
+        prompt_str += "/*\n"
+        prompt_str += "\t".join(header) + "\n"
+        prompt_str += '\n'.join(["\t".join([str(cell) for cell in row]) for row in rows]) + "\n"
+        prompt_str += "*/"
+        return prompt_str
+
+    @staticmethod
+    def build_one_shot_prompt(
+            item: QAItem,
+            answer_split_token: str = ';',
+            verbose: bool = False,
+            prompting_method='new_db',
+            db_mapping_token="😅"
+    ) -> str:
+        """
+        Build one-shot QA prompt.
+        """
+        assert prompting_method in ['basic', 'new_db']
+        qa_type, qa_question = item.qa_question.split('@')
+        prompt = ''
+        db_prompt = OpenAIQAPromptBuilder.table2codex_prompt(item.table, item.title)
+        prompt += "Give a database as shown below:\n{}\n\n".format(db_prompt)
+
+        if prompting_method == 'basic':
+            if qa_type == "map":
+                prompt += "Q: Answer question \"{}\" row by row.".format(qa_question)
+                assert answer_split_token is not None
+                prompt += " The answer should be a list split by '{}' and have {} items in total.".format(
+                    answer_split_token, len(item.table['rows']))
+                prompt += "\nA: {}\n\n".format(f'{answer_split_token}'.join(item.qa_answer))
+            elif qa_type == "ans":
+                prompt += "Q: Answer question \"{}\" for the table.".format(qa_question)
+                prompt += " "
+                prompt += "\nA: {}\n\n".format(f'{answer_split_token}'.join(item.qa_answer))
+            else:
+                raise ValueError("The QA type is not supported!")
+
+            return prompt
+
+        elif prompting_method == "new_db":
+            if qa_type == "map":
+                prompt += "Q: Answer question \"{}\" row by row.".format(qa_question)
+                assert answer_split_token is not None
+                db_prompt_lines = db_prompt.split("\n")[2:-1]  # skip Title, /*, and */
+                db_prompt_lines_with_answer = []
+                db_prompt_lines_with_answer.append("/*")
+                db_prompt_lines_with_answer.append(db_prompt_lines[0])
+                assert len(db_prompt_lines[1:]) == len(
+                    item.qa_answer), "answer items and table rows must be in the same number, check annotations"
+                for db_prompt_line, qa_answer_item in zip(db_prompt_lines[1:], item.qa_answer):
+                    db_prompt_lines_with_answer.append(
+                        "{}{}{}".format(db_prompt_line, db_mapping_token, qa_answer_item))
+                db_prompt_lines_with_answer.append("*/")
+                prompt += "\n{}\n".format("\n".join(db_prompt_lines_with_answer))
+
+            elif qa_type == "ans":
+                prompt += "Q: Answer question \"{}\" for the table.".format(qa_question)
+                prompt += " "
+                prompt += "\nA: {}\n".format(f'{answer_split_token}'.join(item.qa_answer))
+            else:
+                raise ValueError("The QA type is not supported!")
+
+            return prompt

nsql/database.py

@@ -0,0 +1,202 @@
+import copy
+import os
+import sqlite3
+import records
+import sqlalchemy
+import pandas as pd
+from typing import Dict, List
+import uuid
+
+from utils.normalizer import convert_df_type, prepare_df_for_neuraldb_from_table
+from utils.mmqa.image_stuff import get_caption
+
+
+def check_in_and_return(key: str, source: dict):
+    # `` wrapped means as a whole
+    if key.startswith("`") and key.endswith("`"):
+        key = key[1:-1]
+    if key in source.keys():
+        return source[key]
+    else:
+        for _k, _v in source.items():
+            if _k.lower() == key.lower():
+                return _v
+        raise ValueError("{} not in {}".format(key, source))
+
+
+class NeuralDB(object):
+    def __init__(self, tables: List[Dict[str, Dict]], passages=None, images=None):
+        self.raw_tables = copy.deepcopy(tables)
+        self.passages = {}
+        self.images = {}
+        self.image_captions = {}
+        self.passage_linker = {}  # The links from cell value to passage
+        self.image_linker = {}  # The links from cell value to images
+
+        # Get passages
+        if passages:
+            for passage in passages:
+                title, passage_content = passage['title'], passage['text']
+                self.passages[title] = passage_content
+
+        # Get images
+        if images:
+            for image in images:
+                _id, title, picture = image['id'], image['title'], image['pic']
+                self.images[title] = picture
+                self.image_captions[title] = get_caption(_id)
+
+        # Link grounding resources from other modalities(passages, images).
+        if self.raw_tables[0]['table'].get('rows_with_links', None):
+            rows = self.raw_tables[0]['table']['rows']
+            rows_with_links = self.raw_tables[0]['table']['rows_with_links']
+
+            link_title2cell_map = {}
+            for row_id in range(len(rows)):
+                for col_id in range(len(rows[row_id])):
+                    cell = rows_with_links[row_id][col_id]
+                    for text, title, url in zip(cell[0], cell[1], cell[2]):
+                        text = text.lower().strip()
+                        link_title2cell_map[title] = text
+
+            # Link Passages
+            for passage in passages:
+                title, passage_content = passage['title'], passage['text']
+                linked_cell = link_title2cell_map.get(title, None)
+                if linked_cell:
+                    self.passage_linker[linked_cell] = title
+
+            # Images
+            for image in images:
+                title, picture = image['title'], image['pic']
+                linked_cell = link_title2cell_map.get(title, None)
+                if linked_cell:
+                    self.image_linker[linked_cell] = title
+
+        for table_info in tables:
+            table_info['table'] = prepare_df_for_neuraldb_from_table(table_info['table'])
+
+        self.tables = tables
+
+        # Connect to SQLite database
+        self.tmp_path = "tmp"
+        os.makedirs(self.tmp_path, exist_ok=True)
+        # self.db_path = os.path.join(self.tmp_path, '{}.db'.format(hash(time.time())))
+        self.db_path = os.path.join(self.tmp_path, '{}.db'.format(uuid.uuid4()))
+        self.sqlite_conn = sqlite3.connect(self.db_path)
+
+        # Create DB
+        assert len(tables) >= 1, "DB has no table inside"
+        table_0 = tables[0]
+        if len(tables) > 1:
+            raise ValueError("More than one table not support yet.")
+        else:
+            table_0["table"].to_sql("w", self.sqlite_conn)
+            self.table_name = "w"
+            self.table_title = table_0.get('title', None)
+
+        # Records conn
+        self.db = records.Database('sqlite:///{}'.format(self.db_path))
+        self.records_conn = self.db.get_connection()
+
+    def __str__(self):
+        return str(self.execute_query("SELECT * FROM {}".format(self.table_name)))
+
+    def get_table(self, table_name=None):
+        table_name = self.table_name if not table_name else table_name
+        sql_query = "SELECT * FROM {}".format(table_name)
+        _table = self.execute_query(sql_query)
+        return _table
+
+    def get_header(self, table_name=None):
+        _table = self.get_table(table_name)
+        return _table['header']
+
+    def get_rows(self, table_name):
+        _table = self.get_table(table_name)
+        return _table['rows']
+
+    def get_table_df(self):
+        return self.tables[0]['table']
+
+    def get_table_raw(self):
+        return self.raw_tables[0]['table']
+
+    def get_table_title(self):
+        return self.tables[0]['title']
+
+    def get_passages_titles(self):
+        return list(self.passages.keys())
+
+    def get_images_titles(self):
+        return list(self.images.keys())
+
+    def get_passage_by_title(self, title: str):
+        return check_in_and_return(title, self.passages)
+
+    def get_image_by_title(self, title):
+        return check_in_and_return(title, self.images)
+
+    def get_image_caption_by_title(self, title):
+        return check_in_and_return(title, self.image_captions)
+
+    def get_image_linker(self):
+        return copy.deepcopy(self.image_linker)
+
+    def get_passage_linker(self):
+        return copy.deepcopy(self.passage_linker)
+
+    def execute_query(self, sql_query: str):
+        """
+        Basic operation. Execute the sql query on the database we hold.
+        @param sql_query:
+        @return:
+        """
+        # When the sql query is a column name (@deprecated: or a certain value with '' and "" surrounded).
+        if len(sql_query.split(' ')) == 1 or (sql_query.startswith('`') and sql_query.endswith('`')):
+            col_name = sql_query
+            new_sql_query = r"SELECT row_id, {} FROM {}".format(col_name, self.table_name)
+            # Here we use a hack that when a value is surrounded by '' or "", the sql will return a column of the value,
+            # while for variable, no ''/"" surrounded, this sql will query for the column.
+            out = self.records_conn.query(new_sql_query)
+        # When the sql query wants all cols or col_id, which is no need for us to add 'row_id'.
+        elif sql_query.lower().startswith("select *") or sql_query.startswith("select col_id"):
+            out = self.records_conn.query(sql_query)
+        else:
+            try:
+                # SELECT row_id in addition, needed for result and old table alignment.
+                new_sql_query = "SELECT row_id, " + sql_query[7:]
+                out = self.records_conn.query(new_sql_query)
+            except sqlalchemy.exc.OperationalError as e:
+                # Execute normal SQL, and in this case the row_id is actually in no need.
+                out = self.records_conn.query(sql_query)
+
+        results = out.all()
+        unmerged_results = []
+        merged_results = []
+
+        headers = out.dataset.headers
+        for i in range(len(results)):
+            unmerged_results.append(list(results[i].values()))
+            merged_results.extend(results[i].values())
+
+        return {"header": headers, "rows": unmerged_results}
+
+    def add_sub_table(self, sub_table, table_name=None, verbose=True):
+        """
+        Add sub_table into the table.
+        @return:
+        """
+        table_name = self.table_name if not table_name else table_name
+        sql_query = "SELECT * FROM {}".format(table_name)
+        oring_table = self.execute_query(sql_query)
+        old_table = pd.DataFrame(oring_table["rows"], columns=oring_table["header"])
+        # concat the new column into old table
+        sub_table_df_normed = convert_df_type(pd.DataFrame(data=sub_table['rows'], columns=sub_table['header']))
+        new_table = old_table.merge(sub_table_df_normed,
+                                    how='left', on='row_id')  # do left join
+        new_table.to_sql(table_name, self.sqlite_conn, if_exists='replace',
+                         index=False)
+        if verbose:
+            print("Insert column(s) {} (dtypes: {}) into table.\n".format(', '.join([_ for _ in sub_table['header']]),
+                                                                          sub_table_df_normed.dtypes))

nsql/nsql_exec.py

@@ -0,0 +1,161 @@
+import json
+from typing import List, Dict
+from nsql.qa_module.openai_qa import OpenAIQAModel
+from nsql.qa_module.vqa import vqa_call
+from nsql.database import NeuralDB
+from nsql.parser import get_cfg_tree, get_steps, remove_duplicate, TreeNode, parse_question_paras, nsql_role_recognize, \
+    extract_answers
+
+
+class NSQLExecutor(object):
+    def __init__(self, args, keys=None):
+        self.new_col_name_id = 0
+        self.qa_model = OpenAIQAModel(args, keys)
+
+    def generate_new_col_names(self, number):
+        col_names = ["col_{}".format(i) for i in range(self.new_col_name_id, self.new_col_name_id + number)]
+        self.new_col_name_id += number
+        return col_names
+
+    def sql_exec(self, sql: str, db: NeuralDB, verbose=True):
+        if verbose:
+            print("Exec SQL '{}' with additional row_id on {}".format(sql, db))
+        result = db.execute_query(sql)
+        return result
+
+    def nsql_exec(self, stamp, nsql: str, db: NeuralDB, verbose=True):
+        steps = []
+        root_node = get_cfg_tree(nsql)  # Parse execution tree from nsql.
+        get_steps(root_node, steps)  # Flatten the execution tree and get the steps.
+        steps = remove_duplicate(steps)  # Remove the duplicate steps.
+        if verbose:
+            print("Steps:", [s.rename for s in steps])
+            with open("tmp_for_vis/{}_tmp_for_vis_steps.txt".format(stamp), "w") as f:
+                json.dump([s.rename for s in steps], f)
+        col_idx = 0
+        for step in steps:
+            # All steps should be formatted as 'QA()' except for last step which could also be normal SQL.
+            assert isinstance(step, TreeNode), "step must be treenode"
+            nsql = step.rename
+            if nsql.startswith('QA('):
+                question, sql_s = parse_question_paras(nsql, self.qa_model)
+                sql_executed_sub_tables = []
+
+                # Execute all SQLs and get the results as parameters
+                for sql_item in sql_s:
+                    role, sql_item = nsql_role_recognize(sql_item,
+                                                         db.get_header(),
+                                                         db.get_passages_titles(),
+                                                         db.get_images_titles())
+                    if role in ['col', 'complete_sql']:
+                        sql_executed_sub_table = self.sql_exec(sql_item, db, verbose=verbose)
+                        sql_executed_sub_tables.append(sql_executed_sub_table)
+                    elif role == 'val':
+                        val = eval(sql_item)
+                        sql_executed_sub_tables.append({
+                            "header": ["row_id", "val"],
+                            "rows": [["0", val]]
+                        })
+                    elif role == 'passage_title_and_image_title':
+                        sql_executed_sub_tables.append({
+                            "header": ["row_id", "{}".format(sql_item)],
+                            "rows": [["0", db.get_passage_by_title(sql_item) +
+                                      db.get_image_caption_by_title(sql_item)
+                                      # "{} (The answer of '{}' is {})".format(
+                                      #     sql_item,
+                                      #     # Add image qa result as backup info
+                                      #     question[len("***@"):],
+                                      #     vqa_call(question=question[len("***@"):],
+                                      #              image_path=db.get_image_by_title(sql_item)))
+                                      ]]
+                        })
+                    elif role == 'passage_title':
+                        sql_executed_sub_tables.append({
+                            "header": ["row_id", "{}".format(sql_item)],
+                            "rows": [["0", db.get_passage_by_title(sql_item)]]
+                        })
+                    elif role == 'image_title':
+                        sql_executed_sub_tables.append({
+                            "header": ["row_id", "{}".format(sql_item)],
+                            "rows": [["0", db.get_image_caption_by_title(sql_item)]],
+                            # "rows": [["0", "{} (The answer of '{}' is {})".format(
+                            #         sql_item,
+                            #         # Add image qa result as backup info
+                            #         question[len("***@"):],
+                            #         vqa_call(question=question[len("***@"):],
+                            #                  image_path=db.get_image_by_title(sql_item)))]],
+                        })
+
+                # If the sub_tables to execute with link, append it to the cell.
+                passage_linker = db.get_passage_linker()
+                image_linker = db.get_image_linker()
+                for _sql_executed_sub_table in sql_executed_sub_tables:
+                    for i in range(len(_sql_executed_sub_table['rows'])):
+                        for j in range(len(_sql_executed_sub_table['rows'][i])):
+                            _cell = _sql_executed_sub_table['rows'][i][j]
+                            if _cell in passage_linker.keys():
+                                _sql_executed_sub_table['rows'][i][j] += " ({})".format(
+                                    # Add passage text as backup info
+                                    db.get_passage_by_title(passage_linker[_cell]))
+
+                            if _cell in image_linker.keys():
+                                _sql_executed_sub_table['rows'][i][j] += " ({})".format(
+                                    # Add image caption as backup info
+                                    db.get_image_caption_by_title(image_linker[_cell]))
+                                # _sql_executed_sub_table['rows'][i][j] += " (The answer of '{}' is {})".format(
+                                #     # Add image qa result as backup info
+                                #     question[len("***@"):],
+                                #     vqa_call(question=question[len("***@"):],
+                                #              image_path=db.get_image_by_title(image_linker[_cell])))
+                                pass
+
+                if question.lower().startswith("map@"):
+                    # When the question is a type of mapping, we return the mapped column.
+                    question = question[len("map@"):]
+                    if step.father:
+                        step.rename_father_col(col_idx=col_idx)
+                        sub_table: Dict = self.qa_model.qa(question,
+                                                           sql_executed_sub_tables,
+                                                           table_title=db.table_title,
+                                                           qa_type="map",
+                                                           new_col_name_s=step.produced_col_name_s,
+                                                           verbose=verbose)
+                        with open("tmp_for_vis/{}_result_step_{}.txt".format(stamp, steps.index(step)), "w") as f:
+                            json.dump(sub_table, f)
+                        db.add_sub_table(sub_table, verbose=verbose)
+                        col_idx += 1
+                    else:  # This step is the final step
+                        sub_table: Dict = self.qa_model.qa(question,
+                                                           sql_executed_sub_tables,
+                                                           table_title=db.table_title,
+                                                           qa_type="map",
+                                                           new_col_name_s=["col_{}".format(col_idx)],
+                                                           verbose=verbose)
+                        with open("tmp_for_vis/{}_result_step_{}.txt".format(stamp, steps.index(step)), "w") as f:
+                            json.dump(sub_table, f)
+                        return extract_answers(sub_table)
+
+                elif question.lower().startswith("ans@"):
+                    # When the question is a type of answering, we return an answer list.
+                    question = question[len("ans@"):]
+                    answer: List = self.qa_model.qa(question,
+                                                    sql_executed_sub_tables,
+                                                    table_title=db.table_title,
+                                                    qa_type="ans",
+                                                    verbose=verbose)
+                    with open("tmp_for_vis/{}_result_step_{}.txt".format(stamp, steps.index(step)), "w") as f:
+                        json.dump(answer, f)
+                    if step.father:
+                        step.rename_father_val(answer)
+                    else:  # This step is the final step
+                        return answer
+                else:
+                    raise ValueError(
+                        "Except for operators or NL question must start with 'map@' or 'ans@'!, check '{}'".format(
+                            question))
+
+            else:
+                sub_table = self.sql_exec(nsql, db, verbose=verbose)
+                with open("tmp_for_vis/{}_result_step_{}.txt".format(stamp, steps.index(step)), "w") as f:
+                    json.dump(sub_table, f)
+                return extract_answers(sub_table)

nsql/nsql_exec_python.py

@@ -0,0 +1,129 @@
+# For sync the envs.
+import random
+import json
+import pandas as pd
+import pickle
+from nsql.qa_module.openai_qa import OpenAIQAModel
+import os
+import time
+from subprocess import PIPE, Popen
+import uuid
+
+
+# For Python execution.
+class NPythonExecutor(object):
+    def __init__(self, args, keys=None):
+        self.new_col_name_id = 0
+        self.qa_model = OpenAIQAModel(args, keys)
+
+    def nsql_exec(self, nsql: str, db: pd.DataFrame, verbose=True):
+        # Add import part
+        import_part = """import sys
+import random
+import json
+import pandas as pd
+import pickle
+import numpy as np
+import copy
+import os
+import time
+sys.path.append('./')
+from collections.abc import Iterable
+from nsql.qa_module.openai_qa import OpenAIQAModel
+from nsql.database import NeuralDB
+verbose = {}""".format(str(verbose))
+
+        # Add qa_map function
+        qa_map_function_part = """def qa_map(db: pd.DataFrame, question, columns):
+    new_db = NeuralDB([{"title": "", "table": {"header": db.columns.values.tolist(), "rows": db.values.tolist()}}])
+    sql_executed_sub_tables = []
+    for column in columns:
+        column = f"`{column}`"
+        sql_executed_sub_tables.append(new_db.execute_query(column))
+        sub_table = qa_model.qa(question,
+                                          sql_executed_sub_tables,
+                                          table_title=new_db.table_title,
+                                          qa_type="map",
+                                          new_col_name_s=[question],
+                                          verbose=verbose)
+        new_db.add_sub_table(sub_table, verbose=verbose)
+    table = new_db.get_table()
+    return pd.DataFrame(table["rows"], columns=table["header"])"""
+
+        # Add qa_ans function
+        qa_ans_function_part = """def qa_ans(db: pd.DataFrame, question, columns):
+    new_db = NeuralDB([{"title": "", "table": {"header": db.columns.values.tolist(), "rows": db.values.tolist()}}])
+    sql_executed_sub_tables = []
+    for column in columns:
+        column = f"`{column}`"
+        sql_executed_sub_tables.append(new_db.execute_query(column))
+        answer = qa_model.qa(question,sql_executed_sub_tables,table_title=new_db.table_title,qa_type="ans",verbose=verbose)
+    return answer"""
+
+        # Convert np number type to python type
+        convert_part = """def nested_to_python_number(x):
+    if isinstance(x, np.int64):
+        return int(x)
+    if isinstance(x, np.float64):
+        return float(x)
+    if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
+        return [nested_to_python_number(d) for d in x]
+    return x"""
+        # The prediction is a neural-python.
+
+        # Add main function
+        tmp_root_path = "tmp_python"
+        os.makedirs(tmp_root_path, exist_ok=True)
+        # Save the db
+        db_file_path = '{}.db'.format(format(uuid.uuid4()))
+        db_path = os.path.join(tmp_root_path, db_file_path)
+        with open(db_path, "wb") as f:
+            pickle.dump(db, f)
+
+        # Save the qa_model
+        model_file_path = '{}.model'.format(format(uuid.uuid4()))
+        model_path = os.path.join(tmp_root_path, model_file_path)
+        with open(model_path, "wb") as f:
+            pickle.dump(self.qa_model, f)
+
+        # Set the result path
+        result_file_path = '{}.json'.format(format(uuid.uuid4()))
+        result_path = os.path.join(tmp_root_path, result_file_path)
+
+        # Read it and call solve function
+        main_part = """if __name__ == '__main__':
+    with open("{}", "rb") as f:
+        db = pickle.load(f)
+    with open("{}", "rb") as f:
+        qa_model = pickle.load(f)
+    result = solve(db)
+    result = nested_to_python_number(result)
+    with open("{}", "w") as f:
+        json.dump(result, f)""".format(db_path, model_path, result_path)
+
+        # Concat the code and execute the python
+        all_code = "{}\n\n{}\n\n{}\n\n{}\n\n".format(import_part, qa_map_function_part, qa_ans_function_part,
+                                                     convert_part) + nsql + "\n\n" + main_part
+
+        if verbose:
+            print("----> Code <----")
+            print(all_code)
+
+        python_file_path = '{}.py'.format(format(uuid.uuid4()))
+        python_path = os.path.join(tmp_root_path, python_file_path)
+        with open(python_path, "w") as f:
+            f.write(all_code)
+
+        p = Popen("python " + python_path, shell=True, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = p.communicate()
+
+        # Error in execution so that we didn't get result.
+        if not os.path.exists(result_path):
+            print("stderr: ", stderr)
+            raise ValueError("Error execution!")
+
+        # Read the result
+        with open(result_path, "r") as f:
+            result = json.load(f)
+
+        return result

nsql/parser.py

@@ -0,0 +1,179 @@
+from typing import List
+import re
+import sqlparse
+
+
+class TreeNode(object):
+    def __init__(self, name=None, father=None):
+        self.name: str = name
+        self.rename: str = name
+        self.father: TreeNode = father
+        self.children: List = []
+        self.produced_col_name_s = None
+
+    def __eq__(self, other):
+        return self.rename == other.rename
+
+    def __hash__(self):
+        return hash(self.rename)
+
+    def set_name(self, name):
+        self.name = name
+        self.rename = name
+
+    def add_child(self, child):
+        self.children.append(child)
+        child.father = self
+
+    def rename_father_col(self, col_idx: int, col_prefix: str = "col_"):
+        new_col_name = "{}{}".format(col_prefix, col_idx)
+        self.father.rename = self.father.rename.replace(self.name, "{}".format(new_col_name))
+        self.produced_col_name_s = [new_col_name]  # fixme when multiple outputs for a qa func
+
+    def rename_father_val(self, val_names):
+        if len(val_names) == 1:
+            val_name = val_names[0]
+            new_val_equals_str = "'{}'".format(val_name) if isinstance(convert_type(val_name), str) else "{}".format(
+                val_name)
+        else:
+            new_val_equals_str = '({})'.format(', '.join(["'{}'".format(val_name) for val_name in val_names]))
+        self.father.rename = self.father.rename.replace(self.name, new_val_equals_str)
+
+
+def get_cfg_tree(nsql: str):
+    """
+    Parse QA() into a tree for execution guiding.
+    @param nsql:
+    @return:
+    """
+
+    stack: List = []  # Saving the state of the char.
+    expression_stack: List = []  # Saving the state of the expression.
+    current_tree_node = TreeNode(name=nsql)
+
+    for idx in range(len(nsql)):
+        if nsql[idx] == "(":
+            stack.append(idx)
+            if idx > 1 and nsql[idx - 2:idx + 1] == "QA(" and idx - 2 != 0:
+                tree_node = TreeNode()
+                current_tree_node.add_child(tree_node)
+                expression_stack.append(current_tree_node)
+                current_tree_node = tree_node
+        elif nsql[idx] == ")":
+            left_clause_idx = stack.pop()
+            if idx > 1 and nsql[left_clause_idx - 2:left_clause_idx + 1] == "QA(" and left_clause_idx - 2 != 0:
+                # the QA clause
+                nsql_span = nsql[left_clause_idx - 2:idx + 1]
+                current_tree_node.set_name(nsql_span)
+                current_tree_node = expression_stack.pop()
+
+    return current_tree_node
+
+
+def get_steps(tree_node: TreeNode, steps: List):
+    """Pred-Order Traversal"""
+    for child in tree_node.children:
+        get_steps(child, steps)
+    steps.append(tree_node)
+
+
+def parse_question_paras(nsql: str, qa_model):
+    # We assume there's no nested qa inside when running this func
+    nsql = nsql.strip(" ;")
+    assert nsql[:3] == "QA(" and nsql[-1] == ")", "must start with QA( symbol and end with )"
+    assert not "QA" in nsql[2:-1],  "must have no nested qa inside"
+
+    # Get question and the left part(paras_raw_str)
+    all_quote_idx = [i.start() for i in re.finditer('\"', nsql)]
+    question = nsql[all_quote_idx[0] + 1: all_quote_idx[1]]
+    paras_raw_str = nsql[all_quote_idx[1] + 1:-1].strip(" ;")
+
+    # Split Parameters(SQL/column/value) from all parameters.
+    paras = [_para.strip(' ;') for _para in sqlparse.split(paras_raw_str)]
+    return question, paras
+
+
+def convert_type(value):
+    try:
+        return eval(value)
+    except Exception as e:
+        return value
+
+
+def nsql_role_recognize(nsql_like_str, all_headers, all_passage_titles, all_image_titles):
+    """Recognize role. (SQL/column/value) """
+    orig_nsql_like_str = nsql_like_str
+
+    # strip the first and the last '`'
+    if nsql_like_str.startswith('`') and nsql_like_str.endswith('`'):
+        nsql_like_str = nsql_like_str[1:-1]
+
+    # Case 1: if col in header, it is column type.
+    if nsql_like_str in all_headers or nsql_like_str in list(map(lambda x: x.lower(), all_headers)):
+        return 'col', orig_nsql_like_str
+
+    # fixme: add case when the this nsql_like_str both in table headers, images title and in passages title.
+    # Case 2.1: if it is title of certain passage.
+    if (nsql_like_str.lower() in list(map(lambda x: x.lower(), all_passage_titles))) \
+            and (nsql_like_str.lower() in list(map(lambda x: x.lower(), all_image_titles))):
+        return "passage_title_and_image_title", orig_nsql_like_str
+    else:
+        try:
+            nsql_like_str_evaled = str(eval(nsql_like_str))
+            if (nsql_like_str_evaled.lower() in list(map(lambda x: x.lower(), all_passage_titles))) \
+                    and (nsql_like_str_evaled.lower() in list(map(lambda x: x.lower(), all_image_titles))):
+                return "passage_title_and_image_title", nsql_like_str_evaled
+        except:
+            pass
+
+    # Case 2.2: if it is title of certain passage.
+    if nsql_like_str.lower() in list(map(lambda x: x.lower(), all_passage_titles)):
+        return "passage_title", orig_nsql_like_str
+    else:
+        try:
+            nsql_like_str_evaled = str(eval(nsql_like_str))
+            if nsql_like_str_evaled.lower() in list(map(lambda x: x.lower(), all_passage_titles)):
+                return "passage_title", nsql_like_str_evaled
+        except:
+            pass
+
+    # Case 2.3: if it is title of certain picture.
+    if nsql_like_str.lower() in list(map(lambda x: x.lower(), all_image_titles)):
+        return "image_title", orig_nsql_like_str
+    else:
+        try:
+            nsql_like_str_evaled = str(eval(nsql_like_str))
+            if nsql_like_str_evaled.lower() in list(map(lambda x: x.lower(), all_image_titles)):
+                return "image_title", nsql_like_str_evaled
+        except:
+            pass
+
+    # Case 4: if it can be parsed by eval(), it is value type.
+    try:
+        eval(nsql_like_str)
+        return 'val', orig_nsql_like_str
+    except Exception as e:
+        pass
+
+    # Case 5: else it should be the sql, if it isn't, exception will be raised.
+    return 'complete_sql', orig_nsql_like_str
+
+
+def remove_duplicate(original_list):
+    no_duplicate_list = []
+    [no_duplicate_list.append(i) for i in original_list if i not in no_duplicate_list]
+    return no_duplicate_list
+
+
+def extract_answers(sub_table):
+    if not sub_table or sub_table['header'] is None:
+        return []
+    answer = []
+    if 'row_id' in sub_table['header']:
+        for _row in sub_table['rows']:
+            answer.extend(_row[1:])
+        return answer
+    else:
+        for _row in sub_table['rows']:
+            answer.extend(_row)
+        return answer

nsql/qa_module/openai_qa.py

@@ -0,0 +1,196 @@
+import os
+import random
+
+from generation.prompt import OpenAIQAPromptBuilder
+from generation.generator import Generator
+from retrieval.retriever import OpenAIQARetriever
+from retrieval.retrieve_pool import OpenAIQARetrievePool, QAItem
+
+num_parallel_prompts = 10
+num_qa_shots = 8
+infinite_rows_len = 50  # If the table contain rows larger than this number, it will be handled rows by rows.
+max_tokens = 1024
+ROOT_DIR = os.path.join(os.path.dirname(__file__), "../../")
+
+
+class OpenAIQAModel(object):
+    def __init__(self, args, keys=None):
+        super().__init__()
+
+        # Prepare keys
+        self.key_current_id = 0
+        self.keys = keys
+        random.seed(42)
+        random.shuffle(self.keys)
+
+        retrieve_pool = OpenAIQARetrievePool(
+            data_path=os.path.join(ROOT_DIR, args.qa_retrieve_pool_file)
+        )
+        self.retriever = OpenAIQARetriever(retrieve_pool)
+        self.generator = Generator(args=None, keys=self.keys)  # Just to use its call api function
+
+        self.prompting_method = 'new_db'
+        self.answer_split_token: str = ';'
+        self.db_mapping_token = "\t"
+
+    def call_openai_api_completion(self, prompt):
+        completion = self.generator._call_codex_api(engine="code-davinci-002",
+                                                    prompt=prompt,
+                                                    max_tokens=max_tokens,
+                                                    temperature=0,
+                                                    top_p=1,
+                                                    n=1,
+                                                    stop=["\n\n"])
+        return completion
+
+    def call_openai_for_completion_text(self, prompt, openai_usage_type="completion"):
+        if openai_usage_type == "completion":
+            completion = self.call_openai_api_completion(prompt)
+            return completion.choices[0].text
+        else:
+            raise ValueError("The model usage type '{}' doesn't exists!".format(openai_usage_type))
+
+    @staticmethod
+    def merge_tables(tables, by='row_id'):
+        assert len(set([len(_table['rows']) for _table in tables])) == 1, "Tables must have the same rows!"
+        merged_header = [by]
+        by_idx = tables[0]['header'].index(by)
+        merged_rows = [[_row[by_idx]] for _row in tables[0]['rows']]
+
+        for _table in tables:
+            header, rows = _table['header'], _table['rows']
+            for col_idx, col in enumerate(header):
+                if col == by:
+                    continue
+                if col in merged_header:
+                    # When the column is duplicate, and postfix _0, _1 etc.
+                    col = "{}_{}".format(col, merged_header.count(col))
+                merged_header.append(col)
+                for i, row in enumerate(rows):
+                    merged_rows[i].append(row[col_idx])
+        return {"header": merged_header, "rows": merged_rows}
+
+    def wrap_with_prompt_for_table_qa(self,
+                                      question,
+                                      sub_table,
+                                      table_title=None,
+                                      answer_split_token=None,
+                                      qa_type="ans",
+                                      prompting_method="new_db",
+                                      db_mapping_token="😅",
+                                      verbose=True):
+        prompt = "Question Answering Over Database:\n\n"
+        if qa_type in ['map', 'ans'] and num_qa_shots > 0:
+            query_item = QAItem(qa_question=question, table=sub_table, title=table_title)
+            retrieved_items = self.retriever.retrieve(item=query_item, num_shots=num_qa_shots, qa_type=qa_type)
+            few_shot_prompt_list = []
+            for item in retrieved_items:
+                one_shot_prompt = OpenAIQAPromptBuilder.build_one_shot_prompt(
+                    item=item,
+                    answer_split_token=answer_split_token,
+                    verbose=verbose,
+                    prompting_method=prompting_method,
+                    db_mapping_token=db_mapping_token
+                )
+                few_shot_prompt_list.append(one_shot_prompt)
+            few_shot_prompt = '\n'.join(few_shot_prompt_list[:num_qa_shots])
+            prompt = few_shot_prompt
+
+        prompt += "\nGive a database as shown below:\n{}\n\n".format(
+            OpenAIQAPromptBuilder.table2codex_prompt(sub_table, table_title)
+        )
+
+        if qa_type == "map":
+            prompt += "Q: Answer question \"{}\" row by row.".format(question)
+            assert answer_split_token is not None
+            if prompting_method == "basic":
+                prompt += " The answer should be a list split by '{}' and have {} items in total.".format(
+                    answer_split_token, len(sub_table['rows']))
+
+        elif qa_type == "ans":
+            prompt += "Q: Answer question \"{}\" for the table.".format(question)
+            prompt += " "
+        else:
+            raise ValueError("The QA type is not supported!")
+
+        prompt += "\n"
+        if qa_type == "map":
+            if prompting_method == "basic":
+                prompt += "A:"
+        elif qa_type == "ans":
+            prompt += "A:"
+
+        return prompt
+
+    def qa(self, question, sub_tables, qa_type: str, verbose: bool = True, **args):
+        # If it is not a problem API can handle, answer it with a QA model.
+        merged_table = OpenAIQAModel.merge_tables(sub_tables)
+        if verbose:
+            print("Make Question {} on {}".format(question, merged_table))
+        if qa_type == "map":
+            # Map: col(s) -question> one col
+
+            # Make model make a QA towards a sub-table
+            # col(s) -> one col, all QA in one time
+            def do_map(_table):
+                _prompt = self.wrap_with_prompt_for_table_qa(question,
+                                                             _table,
+                                                             args['table_title'],
+                                                             self.answer_split_token,
+                                                             qa_type,
+                                                             prompting_method=self.prompting_method,
+                                                             db_mapping_token=self.db_mapping_token,
+                                                             verbose=verbose)
+                completion_str = self.call_openai_for_completion_text(_prompt).lower().strip(' []')
+
+                if verbose:
+                    print(f'QA map@ input:\n{_prompt}')
+                    print(f'QA map@ output:\n{completion_str}')
+
+                if self.prompting_method == "basic":
+                    answers = [_answer.strip(" '").lower() for _answer in
+                               completion_str.split(self.answer_split_token)]
+                elif self.prompting_method == "new_db":
+                    answers = [line.split(self.db_mapping_token)[-1] for line in completion_str.split("\n")[2:-1]]
+                else:
+                    raise ValueError("No such prompting methods: '{}'! ".format(self.prompting_method))
+                return answers
+
+            # Handle infinite rows, rows by rows.
+            answers = []
+            rows_len = len(merged_table['rows'])
+            run_times = int(rows_len / infinite_rows_len) if rows_len % infinite_rows_len == 0 else int(
+                rows_len / infinite_rows_len) + 1
+
+            for run_idx in range(run_times):
+                _table = {
+                    "header": merged_table['header'],
+                    "rows": merged_table['rows'][run_idx * infinite_rows_len:]
+                } if run_idx == run_times - 1 else \
+                    {
+                        "header": merged_table['header'],
+                        "rows": merged_table['rows'][run_idx * infinite_rows_len:(run_idx + 1) * infinite_rows_len]
+                    }
+
+                answers.extend(do_map(_table))
+            if verbose:
+                print("The map@ openai answers are {}".format(answers))
+            # Add row_id in addition for finding to corresponding rows.
+            return {"header": ['row_id'] + args['new_col_name_s'],
+                    "rows": [[row[0], answer] for row, answer in zip(merged_table['rows'], answers)]}
+        elif qa_type == "ans":
+            # Ans: col(s) -question> answer
+            prompt = self.wrap_with_prompt_for_table_qa(question,
+                                                        merged_table,
+                                                        args['table_title'],
+                                                        prompting_method=self.prompting_method,
+                                                        verbose=verbose)
+            answers = [self.call_openai_for_completion_text(prompt).lower().strip(' []')]
+
+            if verbose:
+                print(f'QA ans@ input:\n{prompt}')
+                print(f'QA ans@ output:\n{answers}')
+
+            return answers
+        else:
+            raise ValueError("Please choose from map and ans in the qa usage!!")

nsql/qa_module/vqa.py

@@ -0,0 +1,10 @@
+import requests
+import base64
+import time
+
+
+def vqa_call(question, image_path, api_url='https://hf.space/embed/OFA-Sys/OFA-vqa/+/api/predict/'):
+    with open(image_path, "rb") as f:
+        base64_data = base64.b64encode(f.read())
+    base64_data_to_send = "data:image/{};base64,{}".format(image_path.split(".")[-1], str(base64_data)[2:-1])
+    return requests.post(url=api_url, json={"data": [base64_data_to_send, question]}).json()['data'][0]

requirements.txt

@@ -0,0 +1,209 @@
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: osx-64
+aiohttp=3.8.1=pypi_0
+aiosignal=1.2.0=pypi_0
+altair=4.2.0=pypi_0
+anyio=3.6.1=pypi_0
+appnope=0.1.2=pypi_0
+argon2-cffi=21.3.0=pypi_0
+argon2-cffi-bindings=21.2.0=pypi_0
+async-generator=1.10=pypi_0
+async-timeout=4.0.2=pypi_0
+asynctest=0.13.0=pypi_0
+attrs=21.4.0=pypi_0
+backcall=0.2.0=pypi_0
+backoff=2.0.1=pypi_0
+backports-zoneinfo=0.2.1=pypi_0
+bcrypt=4.0.0=pypi_0
+beautifulsoup4=4.10.0=pypi_0
+bleach=4.1.0=pypi_0
+blinker=1.4=pypi_0
+brotlipy=0.7.0=py37h9ed2024_1003
+ca-certificates=2022.4.26=hecd8cb5_0
+cached-property=1.5.2=pypi_0
+cachetools=5.0.0=pypi_0
+certifi=2022.6.15=py37hecd8cb5_0
+cffi=1.15.0=py37hc55c11b_1
+charset-normalizer=2.0.12=pypi_0
+click=8.0.4=pypi_0
+cryptography=37.0.1=py37hf6deb26_0
+cssselect=1.1.0=pypi_0
+cycler=0.11.0=pypi_0
+datasets=1.14.0=pypi_0
+datedelta=1.4=pypi_0
+debugpy=1.6.0=pypi_0
+decorator=5.1.1=pypi_0
+defusedxml=0.7.1=pypi_0
+dill=0.3.4=pypi_0
+docopt=0.6.2=pypi_0
+emoji=1.7.0=pypi_0
+entrypoints=0.4=pypi_0
+et-xmlfile=1.1.0=pypi_0
+fastapi=0.85.0=pypi_0
+ffmpy=0.3.0=pypi_0
+filelock=3.6.0=pypi_0
+fonttools=4.37.4=pypi_0
+frozenlist=1.3.0=pypi_0
+fsspec=2022.2.0=pypi_0
+fuzzywuzzy=0.18.0=pypi_0
+gitdb=4.0.9=pypi_0
+gitpython=3.1.27=pypi_0
+gradio=3.4.0=pypi_0
+grapheme=0.6.0=pypi_0
+greenlet=1.1.2=pypi_0
+h11=0.12.0=pypi_0
+h5py=3.6.0=pypi_0
+httpcore=0.15.0=pypi_0
+httpx=0.23.0=pypi_0
+huggingface-hub=0.0.19=pypi_0
+idna=3.3=pyhd3eb1b0_0
+importlib-metadata=4.11.3=pypi_0
+importlib-resources=5.6.0=pypi_0
+ipykernel=6.9.2=pypi_0
+ipython=7.32.0=pypi_0
+ipython-genutils=0.2.0=pypi_0
+ipywidgets=7.7.0=pypi_0
+jdcal=1.4.1=pypi_0
+jedi=0.18.1=pypi_0
+jinja2=3.1.1=pypi_0
+joblib=1.1.0=pypi_0
+jsonschema=4.4.0=pypi_0
+jupyter-client=7.1.2=pypi_0
+jupyter-core=4.9.2=pypi_0
+jupyterlab-pygments=0.1.2=pypi_0
+jupyterlab-widgets=1.1.0=pypi_0
+kiwisolver=1.4.4=pypi_0
+libcxx=12.0.0=h2f01273_0
+libffi=3.3=hb1e8313_2
+linkify-it-py=1.0.3=pypi_0
+lxml=4.8.0=pypi_0
+markdown-it-py=2.1.0=pypi_0
+markupsafe=2.1.1=pypi_0
+matplotlib=3.5.3=pypi_0
+matplotlib-inline=0.1.3=pypi_0
+mdit-py-plugins=0.3.1=pypi_0
+mdurl=0.1.2=pypi_0
+mistune=0.8.4=pypi_0
+multidict=6.0.2=pypi_0
+multipledispatch=0.6.0=pypi_0
+multiprocess=0.70.12.2=pypi_0
+nbclient=0.5.13=pypi_0
+nbconvert=6.4.5=pypi_0
+nbformat=5.2.0=pypi_0
+ncurses=6.3=hca72f7f_2
+nest-asyncio=1.5.4=pypi_0
+nltk=3.6.2=pypi_0
+notebook=6.4.10=pypi_0
+numpy=1.21.5=pypi_0
+openai=0.20.0=pypi_0
+openpyxl=2.4.11=pypi_0
+openssl=1.1.1o=hca72f7f_0
+orjson=3.8.0=pypi_0
+outcome=1.1.0=pypi_0
+packaging=21.3=pypi_0
+pandas=1.3.5=pypi_0
+pandas-stubs=1.2.0.58=pypi_0
+pandocfilters=1.5.0=pypi_0
+paramiko=2.11.0=pypi_0
+parso=0.8.3=pypi_0
+pexpect=4.8.0=pypi_0
+pickleshare=0.7.5=pypi_0
+pillow=9.0.1=pypi_0
+pip=21.2.2=py37hecd8cb5_0
+prometheus-client=0.13.1=pypi_0
+prompt-toolkit=3.0.28=pypi_0
+protobuf=3.19.4=pypi_0
+psutil=5.9.0=pypi_0
+ptyprocess=0.7.0=pypi_0
+pyarrow=7.0.0=pypi_0
+pycparser=2.21=pyhd3eb1b0_0
+pycryptodome=3.15.0=pypi_0
+pydantic=1.10.2=pypi_0
+pydeck=0.7.1=pypi_0
+pydub=0.25.1=pypi_0
+pygments=2.11.2=pypi_0
+pyhtml2pdf=0.0.3=pypi_0
+pympler=1.0.1=pypi_0
+pynacl=1.5.0=pypi_0
+pyopenssl=22.0.0=pyhd3eb1b0_0
+pyparsing=3.0.7=pypi_0
+pyquery=1.4.3=pypi_0
+pyrsistent=0.18.1=pypi_0
+pysocks=1.7.1=py37hecd8cb5_0
+python=3.7.11=h88f2d9e_0
+python-dateutil=2.8.2=pypi_0
+python-levenshtein=0.12.2=pypi_0
+python-multipart=0.0.5=pypi_0
+pytz=2022.1=pypi_0
+pytz-deprecation-shim=0.1.0.post0=pypi_0
+pyyaml=6.0=pypi_0
+pyzmq=22.3.0=pypi_0
+readline=8.1.2=hca72f7f_1
+recognizers-text=1.0.2a2=pypi_0
+recognizers-text-choice=1.0.2a2=pypi_0
+recognizers-text-date-time=1.0.2a2=pypi_0
+recognizers-text-number=1.0.2a2=pypi_0
+recognizers-text-number-with-unit=1.0.2a2=pypi_0
+recognizers-text-sequence=1.0.2a2=pypi_0
+recognizers-text-suite=1.0.2a2=pypi_0
+records=0.5.3=pypi_0
+regex=2022.3.15=pypi_0
+requests=2.28.0=pypi_0
+rfc3986=1.5.0=pypi_0
+sacremoses=0.0.49=pypi_0
+scikit-learn=1.0.2=pypi_0
+scipy=1.7.3=pypi_0
+selenium=4.1.3=pypi_0
+semver=2.13.0=pypi_0
+send2trash=1.8.0=pypi_0
+sentencepiece=0.1.97=pypi_0
+setuptools=58.0.4=py37hecd8cb5_0
+six=1.16.0=pypi_0
+smmap=5.0.0=pypi_0
+sniffio=1.2.0=pypi_0
+sortedcontainers=2.4.0=pypi_0
+soupsieve=2.3.1=pypi_0
+sqlalchemy=1.4.36=pypi_0
+sqlite=3.37.2=h707629a_0
+sqlparse=0.4.2=pypi_0
+stanza=1.4.2=pypi_0
+starlette=0.20.4=pypi_0
+streamlit=1.8.0=pypi_0
+tablib=3.2.1=pypi_0
+terminado=0.13.3=pypi_0
+testpath=0.6.0=pypi_0
+threadpoolctl=3.1.0=pypi_0
+timeout-decorator=0.5.0=pypi_0
+tk=8.6.11=h7bc2e8c_0
+tokenizers=0.10.3=pypi_0
+toml=0.10.2=pypi_0
+toolz=0.11.2=pypi_0
+torch=1.12.1=pypi_0
+tornado=6.1=pypi_0
+tqdm=4.63.1=pypi_0
+traitlets=5.1.1=pypi_0
+transformers=4.12.2=pypi_0
+trio=0.20.0=pypi_0
+trio-websocket=0.9.2=pypi_0
+typing-extensions=4.1.1=pypi_0
+tzdata=2022.1=pypi_0
+tzlocal=4.1=pypi_0
+uc-micro-py=1.0.1=pypi_0
+urllib3=1.26.9=py37hecd8cb5_0
+uvicorn=0.18.3=pypi_0
+validators=0.18.2=pypi_0
+watchdog=2.1.7=pypi_0
+wcwidth=0.2.5=pypi_0
+webdriver-manager=3.5.4=pypi_0
+webencodings=0.5.1=pypi_0
+websockets=10.3=pypi_0
+wheel=0.37.1=pyhd3eb1b0_0
+widgetsnbextension=3.6.0=pypi_0
+word2number=1.1=pypi_0
+wsproto=1.1.0=pypi_0
+xxhash=3.0.0=pypi_0
+xz=5.2.5=h1de35cc_0
+yarl=1.7.2=pypi_0
+zipp=3.7.0=pypi_0
+zlib=1.2.11=h4dc903c_4

resources/introduction.md

@@ -0,0 +1,2 @@
+## Introduction
+[placeholder, mainly introduce Figure1(better the gif version)]

resources/summary.md

@@ -0,0 +1,4 @@
+Binder is a training-free neural-symbolic framework that maps the task input to a program, which
++ Allows binding a unified API of language model(LM) functionalities to a programming language.
++ Adopts Codex as both the program parser and the underlying LM of API calls.
++ Requires only a dozen of in-context exemplar annotations.

retrieval/retrieve_pool.py

@@ -0,0 +1,55 @@
+"""
+Retrieval pool of candidates
+"""
+from dataclasses import dataclass
+from typing import List, Dict
+import json
+
+
+class OpenAIQARetrievePool(object):
+    def __init__(self, data_path):
+        with open(data_path, 'r') as f:
+            data = json.load(f)
+        self.data = []
+        for d in data:
+            if isinstance(d['qa_column'], List):
+                d['qa_column'] = '|'.join(d['qa_column'])
+            qa_item = QAItem(
+                id=d['id'],
+                qa_question=d['qa_question'],
+                qa_column=d['qa_column'],
+                qa_answer=d['qa_answer'],
+                table=d['table'],
+                title=d['title']
+            )
+            self.data.append(qa_item)
+
+        self.pointer = 0
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        pointer = self.pointer
+        if pointer < len(self):
+            self.pointer += 1
+            return self.data[pointer]
+        else:
+            self.pointer = 0
+            raise StopIteration
+
+    def __getitem__(self, item):
+        return self.data[item]
+
+    def __len__(self):
+        return len(self.data)
+
+
+@dataclass
+class QAItem(object):
+    id: int = None
+    qa_question: str = None
+    qa_column: str = None
+    qa_answer: str = None
+    table: Dict = None
+    title: str = None

retrieval/retriever.py

@@ -0,0 +1,95 @@
+"""
+Retriever to retrieve relevant examples from annotations.
+"""
+
+import copy
+from typing import Dict, List, Tuple, Any
+import nltk
+from nltk.stem import SnowballStemmer
+from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
+
+from utils.normalizer import normalize
+from retrieval.retrieve_pool import OpenAIQARetrievePool, QAItem
+
+
+class OpenAIQARetriever(object):
+    def __init__(self, retrieve_pool: OpenAIQARetrievePool):
+        self.retrieve_pool = retrieve_pool
+
+    @staticmethod
+    def _string_bleu(q1: str, q2: str, stop_words=None, stemmer=None):
+        """
+        BLEU score.
+        """
+        q1, q2 = normalize(q1), normalize(q2)
+        reference = [[tk for tk in nltk.word_tokenize(q1)]]
+        candidate = [tk for tk in nltk.word_tokenize(q2)]
+        if stemmer is not None:
+            reference = [[stemmer.stem(tk) for tk in reference[0]]]
+            candidate = [stemmer.stem(tk) for tk in candidate]
+
+        chencherry_smooth = SmoothingFunction()  # bleu smooth to avoid hard behaviour when no ngram overlaps
+        bleu_score = sentence_bleu(
+            reference,
+            candidate,
+            weights=(0.25, 0.3, 0.3, 0.15),
+            smoothing_function=chencherry_smooth.method1
+        )
+        return bleu_score
+
+    def _qh2qh_similarity(
+            self,
+            item: QAItem,
+            num_retrieve_samples: int,
+            score_func: str,
+            qa_type: str,
+            weight_h: float = 0.2,
+            verbose: bool = False
+    ):
+        """
+        Retrieve top K nsqls based on query&header to query&header similarities.
+        """
+        q = item.qa_question
+        header_wo_row_id = copy.copy(item.table['header'])
+        header_wo_row_id.remove('row_id')
+        h = ' '.join(header_wo_row_id)
+        stemmer = SnowballStemmer('english')
+        if score_func == 'bleu':
+            retrieve_q_list = [(d, self._string_bleu(q, d.qa_question.split('@')[1], stemmer=stemmer))
+                               for d in self.retrieve_pool if d.qa_question.split('@')[0] == qa_type]
+            retrieve_h_list = [(d, self._string_bleu(h, ' '.join(d.table['header']), stemmer=stemmer))
+                               for d in self.retrieve_pool if d.qa_question.split('@')[0] == qa_type]
+            retrieve_list = [(retrieve_q_list[idx][0], retrieve_q_list[idx][1] + weight_h * retrieve_h_list[idx][1])
+                             for idx in range(len(retrieve_q_list))]
+        else:
+            raise ValueError
+        retrieve_list = sorted(retrieve_list, key=lambda x: x[1], reverse=True)
+        retrieve_list = list(map(lambda x: x[0], retrieve_list))[:num_retrieve_samples]
+
+        if verbose:
+            print(retrieve_list)
+
+        return retrieve_list
+
+    def retrieve(
+            self,
+            item: QAItem,
+            num_shots: int,
+            method: str = 'qh2qh_bleu',
+            qa_type: str = 'map',
+            verbose: bool = False
+    ) -> List[QAItem]:
+        """
+        Retrieve a list of relevant QA samples.
+        """
+        if method == 'qh2qh_bleu':
+            retrieved_items = self._qh2qh_similarity(
+                item=item,
+                num_retrieve_samples=num_shots,
+                score_func='bleu',
+                qa_type=qa_type,
+                verbose=verbose
+            )
+            return retrieved_items
+        else:
+            raise ValueError(f'Retrieve method {method} is not supported.')

templates/mmqa_qa_retrieve_pool_v2.json

@@ -0,0 +1,1600 @@
+[
+    {
+        "id": 15127,
+        "qa_question": "map@Is its title screen featuring 5 women and 1 man?",
+        "qa_column": [
+            "Title"
+        ],
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "Title"
+            ],
+            "rows": [
+                [
+                    "Endurance (an image of the endurance logo.)"
+                ],
+                [
+                    "Jack & Bobby (a poster with a boy and jack & bobby words.)"
+                ],
+                [
+                    "Into the West (indian silhouette and a man holding a gun and a woman.)"
+                ],
+                [
+                    "Without a Trace (a black and white photo of a city without a trade)"
+                ],
+                [
+                    "CSI: Miami (a sign that says csi miami on a glass window)"
+                ],
+                [
+                    "Rizzoli & Isles (how to draw the numbers of the 100th anniversary of birth of a child)"
+                ],
+                [
+                    "The Lying Game (a group of people standing next to each other)"
+                ],
+                [
+                    "NCIS (a close up of the ncis logo on a computer screen)"
+                ],
+                [
+                    "Scream Queens (the logo for scream queens is shown in red)"
+                ],
+                [
+                    "All Hail King Julien: Exiled (all hail king julien movie poster)"
+                ],
+                [
+                    "All Hail King Julien (all hail king julien movie poster)"
+                ]
+            ]
+        },
+        "title": "Glen Powell (Television)"
+    },
+    {
+        "id": 16173,
+        "qa_question": "map@Does its logo has stars?",
+        "qa_column": [
+            "Team"
+        ],
+        "qa_answer": [
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "Team"
+            ],
+            "rows": [
+                [
+                    "Spokane Chiefs (an image of the boston spokane chiefs team logo, a alphabet s with wing)"
+                ],
+                [
+                    "Tri-City Americans (the logo of the tri-City americans team, three stars in the middle, with wing of american red and blue)"
+                ],
+                [
+                    "Kamloops Blazers (logo of kamloops blazers, a alphabet B contain a orange fire inside)"
+                ],
+                [
+                    "Spokane Chiefs (an image of the boston spokane chiefs team logo, a alphabet s with wing)"
+                ],
+                [
+                    "Lethbridge Hurricanes (the logo of lethbridge hurricanes, a red alphabet capital h with a red and black ring around it)"
+                ],
+                [
+                    "Medicine Hat Tigers (the logo of medicine hat tigers, a tiger head)"
+                ],
+                [
+                    "Lethbridge Hurricanes (the logo of lethbridge hurricanes, a red alphabet capital h with a red and black ring around it)"
+                ],
+                [
+                    "Tri-City Americans (the logo of the tri-City americans team, three stars in the middle, with wing of american red and blue)"
+                ],
+                [
+                    "Prince Albert Raiders (the logo of prince albert raiders, the capital fa form as a shield, and a sword go through its middle, slogan raiders on the top of the shield)"
+                ],
+                [
+                    "Swift Current Broncos (the logo of the swift current broncos team, a silhouette of a bull on a blue background passes through a green arch and the letters SC with a slogan consisting of uppercase blue Broncos)"
+                ]
+            ]
+        },
+        "title": "1990\u201391 WHL season (Scoring leaders)"
+    },
+    {
+        "id": 17770,
+        "qa_question": "map@Is it the city where 1988 Olympics were held?",
+        "qa_column": [
+            "Location"
+        ],
+        "qa_answer": [
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "Location"
+            ],
+            "rows": [
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    "Seoul, South Korea"
+                ],
+                [
+                    "Melbourne, Australia"
+                ],
+                [
+                    "Gold Coast, Australia (a beach with a city skyline in the background)"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    "Stockholm, Sweden"
+                ],
+                [
+                    "Hiroshima, Japan"
+                ],
+                [
+                    ""
+                ],
+                [
+                    "Seoul, South Korea"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    "Seoul, South Korea"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    ""
+                ],
+                [
+                    "Yokohama, Japan"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    "Seoul, South Korea"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    "Yokohama, Japan"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    ""
+                ],
+                [
+                    "Busan, South Korea"
+                ],
+                [
+                    "Yokohama, Japan"
+                ],
+                [
+                    "Tokyo, Japan"
+                ],
+                [
+                    ""
+                ]
+            ]
+        },
+        "title": "Virgil Kalakoda (Kickboxing record)"
+    },
+    {
+        "id": 14982,
+        "qa_question": "map@Does poster have a silhouette on it?",
+        "qa_column": [
+            "Title"
+        ],
+        "qa_answer": [
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "Title"
+            ],
+            "rows": [
+                [
+                    "Ladybug Ladybug ()"
+                ],
+                [
+                    "Bonnie and Clyde (a poster for the movie starring a man and a woman on a motorcycle)"
+                ],
+                [
+                    "Rachel, Rachel (a poster of a girl with a pink doll)"
+                ],
+                [
+                    "Don't Drink the Water (a poster for the movie they re caught in a security leak, a group of people block the grim reaper pouring from the giant water tap)"
+                ],
+                [
+                    "Watermelon Man (a very funny thing happened to jeff gerber advertising poster, a black man's head aside by a watermelon with american sign)"
+                ],
+                [
+                    "I Walk the Line (a poster for the movie walk the line, a man is holding on to a woman with blond hair)"
+                ]
+            ]
+        },
+        "title": "Estelle Parsons (Kickboxing record)"
+    },
+    {
+        "id": 561,
+        "qa_question": "map@What sport is shown in the logo?",
+        "qa_column": [
+            "Club"
+        ],
+        "qa_answer": [
+            "/",
+            "/",
+            "/",
+            "soccer",
+            "soccer"
+        ],
+        "table": {
+            "header": [
+                "Club"
+            ],
+            "rows": [
+                [
+                    "Bolton Wanderers (the symbol for peace with a red ribbon in front of it)"
+                ],
+                [
+                    "St Johnstone (loan)"
+                ],
+                [
+                    "Macclesfield Town (loan)"
+                ],
+                [
+                    "Crawley Town (A shield with an alternating grid of red and white, with a soccer on it.)"
+                ],
+                [
+                    "Kilmarnock (the logo for the manhattan football league 150 years ago, soccer, shield, hands with two fingers)"
+                ]
+            ]
+        },
+        "title": "Mark Connolly (Career statistics)"
+    },
+    {
+        "id": 2488,
+        "qa_question": "map@Does it has an instrument in its cover?",
+        "qa_column": [
+            "Title"
+        ],
+        "qa_answer": [
+            "yes",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "Title"
+            ],
+            "rows": [
+                [
+                    "Gettin' to It (a man playing a cello in front of a car)"
+                ],
+                [
+                    "Number Two Express (a man standing next to a cello)"
+                ],
+                [
+                    "A Family Affair (a man wearing sunglasses standing in front of a wall)"
+                ],
+                [
+                    "SciFi (a man on a screen with a glass window)"
+                ],
+                [
+                    "The Philadelphia Experiment (a poster for the philadelphia experiment with three people looking at a city)"
+                ],
+                [
+                    "Vertical Vision (a man in the middle of a guitar's string)"
+                ],
+                [
+                    "Live at Tonic (a man is standing on the sidewalk with a cello)"
+                ]
+            ]
+        },
+        "title": "Christian McBride (Discography  |  As leader)"
+    },
+    {
+        "id": 20323,
+        "qa_question": "map@Is it features a man on its logo?",
+        "qa_column": [
+            "Title"
+        ],
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "Title"
+            ],
+            "rows": [
+                [
+                    "Guiding Light (a neon sign that says guildlight in the dark)"
+                ],
+                [
+                    "Hack (a man in a car with his face on a screen)"
+                ],
+                [
+                    "Everwood (a neon sign that says everwood in front of mountains)"
+                ],
+                [
+                    "Law & Order: Criminal Intent (a neon law  order criminal intent sign on a dark)"
+                ],
+                [
+                    "Deception (a man standing in the darkness)"
+                ]
+            ]
+        },
+        "title": "Jess Weixler (Television)"
+    },
+    {
+        "id": 7157,
+        "qa_question": "map@Does it has a flower on its logo?",
+        "qa_column": [
+            "Title"
+        ],
+        "qa_answer": [
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "Title"
+            ],
+            "rows": [
+                [
+                    "Aldershot"
+                ],
+                [
+                    "Barrow (the logo of the barrow afc and the bee logo, with flower and football and submarine on the shield)"
+                ],
+                [
+                    "Bradford City (the logo of the bcaf afc announces its new logo, a rooster on a sheild)"
+                ],
+                [
+                    "Brentford (the logo for the brentford football club, a bee in the red circle)"
+                ],
+                [
+                    "Chester (a crest of a lion wearing a crown and a laurel wreath)"
+                ],
+                [
+                    "Crewe Alexandra (the logo of the glen alexandra football club, a red dragon among golden ears of wheat)"
+                ],
+                [
+                    "Darlington (the logo for the quakers, an orange and white shield, unfolding a tractor and a top hat)"
+                ],
+                [
+                    "Halifax Town (the logo of the shaymen football club, a blue striped and white shield with a floral pattern, the uppercase ntfc letters nesting a football)"
+                ]
+            ]
+        },
+        "title": "1962\u201363 Football League Cup (First round | Ties)"
+    },
+    {
+        "id": 21652,
+        "qa_question": "map@Is he/she has blonde hair that is curly in the back and has sideburns?",
+        "qa_column": [
+            "Player"
+        ],
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "Player"
+            ],
+            "rows": [
+                [
+                    "David Duval (a man wearing a hat and sunglasses and a shirt)"
+                ],
+                [
+                    "Niclas Fasth (a man standing in front of a cricket field with blue shirt and gold chain)"
+                ],
+                [
+                    "Darren Clarke (a man wearing a hat smiles at the camera)"
+                ],
+                [
+                    "Ernie Els (a man in a black shirt holding a golf club)"
+                ],
+                [
+                    "Miguel \u00c1ngel Jim\u00e9nez (a man wearing a baseball cap and a blue shirt is looking off into the distance)"
+                ]
+            ]
+        },
+        "title": "2001 Open Championship (Final round)"
+    },
+    {
+        "id": 9492,
+        "qa_question": "ans@Did the team that has a logo shaped like a quarter note with wings and is listed in the leading goaltenders in the 2008-09 National Hockey League season make the playoffs for the Stanley Cup?",
+        "qa_column": [
+            "St. Louis Blues"
+        ],
+        "qa_answer": [
+            "no"
+        ],
+        "table": {
+            "header": [
+                "St. Louis Blues"
+            ],
+            "rows": [
+                [
+                    "The team is named after the famous W.C. Handy song \"Saint Louis Blues.\" The franchise was founded in 1967 as an expansion team during the league's 1967 NHL Expansion, which expanded the league from six teams to twelve. The Blues are the oldest active NHL team never to have won the Stanley Cup, although they played in the Stanley Cup Finals three times in 1968, 1969 and 1970.St. Louis Blues. The logo has wings on it."
+                ]
+            ]
+        },
+        "title": "2008\u201309 NHL season"
+    },
+    {
+        "id": 5555,
+        "qa_question": "map@Is the film poster with a male reaching his hand out?",
+        "qa_column": "film",
+        "qa_answer": [
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "film"
+            ],
+            "rows": [
+                [
+                    "sethu ( a naked man and a beautiful woman )"
+                ],
+                [
+                    "nandha ( two men are staring at someone )"
+                ],
+                [
+                    "pithamagan ( a man is reaching the hand out )"
+                ],
+                [
+                    "maayavi ( a man is smiling and a woman is shocked )"
+                ],
+                [
+                    "naan kadavul ( a man is running in the sea )"
+                ]
+            ]
+        },
+        "title": "Bala (director)"
+    },
+    {
+        "id": 4456,
+        "qa_question": "ans@Which resort was it filmed at?",
+        "qa_column": [
+            "title",
+            "Couples Retreat"
+        ],
+        "qa_answer": [
+            "the St. Regis Bora Bora Resort, Bora Bora in French Polynesia"
+        ],
+        "table": {
+            "header": [
+                "title",
+                "Couples Retreat"
+            ],
+            "rows": [
+                [
+                    "couples retreat (The primary location for filming was at the St. Regis Bora Bora Resort, Bora Bora in French Polynesia.[citation needed] Other filming locations include Los Angeles, Universal Studios and O'Hare International Airport.) (A man and a woman are standing in the water.)"
+                ],
+                [
+                    "The primary location for filming was at the St. Regis Bora Bora Resort, Bora Bora in French Polynesia.[citation needed] Other filming locations include Los Angeles, Universal Studios and O'Hare International Airport. water is listed (or ranked) 2 on the list what to watch if you love life"
+                ]
+            ]
+        },
+        "title": "Justin Deeley"
+    },
+    {
+        "id": 13437,
+        "qa_question": "map@Was he/she playing the violin?",
+        "qa_column": "artist",
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "artist"
+            ],
+            "rows": [
+                [
+                    "tina turner ( a singer is singing on the stage )"
+                ],
+                [
+                    "soundgarden ( a band is playing the song )"
+                ],
+                [
+                    "live ( a band is playing with a naked lead singer )"
+                ],
+                [
+                    "r.e.m. ( a bald singer is singing in the dark )"
+                ],
+                [
+                    "tom petty ( a man is playing the guitar )"
+                ],
+                [
+                    "bruce springsteen ( a man is playing the guitar )"
+                ],
+                [
+                    "tracy bonham ( a woman is playing the violin )"
+                ]
+            ]
+        },
+        "title": "Bala (director)"
+    },
+    {
+        "id": 10968,
+        "qa_question": "ans@When was the last time the player in the 2nd round of the Players 2013 championship when the score was 67-67=134 was number 1?",
+        "qa_column": [
+            "player",
+            "List of World Number One male golfers"
+        ],
+        "qa_answer": [
+            "2009"
+        ],
+        "table": {
+            "header": [
+                "player",
+                "List of World Number One male golfers"
+            ],
+            "rows": [
+                [
+                    "tiger woods ( this is a close up of golfer.)"
+                ],
+                [
+                    "The following is a list of the 20 golfers who have risen to the top of the Official World Golf Ranking. As of January 21, 2018, Dustin Johnson is the number one ranked golfer. Tiger Woods has spent the most consecutive weeks (281) and most total weeks (683) in that position. Three golfers have spent an entire calendar year atop the rankings: Nick Faldo (1993), Greg Norman (1996), and Woods (2000, 2001, 2002, 2003, 2006, 2007, 2008, 2009)."
+                ]
+            ]
+        },
+        "title": "2013 Players Championship"
+    },
+    {
+        "id": 9184,
+        "qa_question": "ans@When did this season of the show where Jerrika Hinton plays Stephanie Edwards start?",
+        "qa_column": [
+            "title",
+            "Grey's Anatomy"
+        ],
+        "qa_answer": [
+            "September 28, 2017"
+        ],
+        "table": {
+            "header": [
+                "title",
+                "Grey's Anatomy"
+            ],
+            "rows": [
+                [
+                    "grey's anatomy (On February 10, 2017, ABC renewed Grey's Anatomy for a fourteenth season, which premiered on September 28, 2017. The series' success catapulted such long-running cast members as Pompeo, Dempsey, and Oh to worldwide recognition; they were among the top five highest-earning television actors in 2013.)"
+                ],
+                [
+                    "On February 10, 2017, ABC renewed Grey's Anatomy for a fourteenth season, which premiered on September 28, 2017. The series' success catapulted such long-running cast members as Pompeo, Dempsey, and Oh to worldwide recognition; they were among the top five highest-earning television actors in 2013."
+                ]
+            ]
+        },
+        "title": "Jerrika Hinton"
+    },
+    {
+        "id": 13726,
+        "qa_question": "ans@The place where the he first international earth summit held has how many photos in its collage?",
+        "qa_column": [
+            "Earth Summit",
+            "Rio de Janeiro"
+        ],
+        "qa_answer": [
+            "six"
+        ],
+        "table": {
+            "header": [
+                "Earth Summit",
+                "Rio de Janeiro"
+            ],
+            "rows": [
+                [
+                    "The United Nations Conference on Environment and Development (UNCED), also known as the Rio de Janeiro Earth Summit, the Rio Summit, the Rio Conference, and the Earth Summit (Portuguese: ECO92), was a major United Nations conference held in Rio de Janeiro from 3 to 14 June 1992."
+                ],
+                [
+                    "six views of city rio de janeiro, brazil"
+                ]
+            ]
+        },
+        "title": "2014\u201315 NBB season"
+    },
+    {
+        "id": 14059,
+        "qa_question": "ans@What color is the Young Artist Award?",
+        "qa_column": [
+            "Young Artist Award"
+        ],
+        "qa_answer": [
+            "gold"
+        ],
+        "table": {
+            "header": [
+                "Young Artist Award"
+            ],
+            "rows": [
+                [
+                    "The Young Artist Award (originally known as the Youth in Film Award) is an accolade presented by the Young Artist Association, a non-profit organization founded in 1978 to honor excellence of youth performers, and to provide scholarships for young artists who may be physically disabled or financially unstable. a gold sculpture holding a star."
+                ]
+            ]
+        },
+        "title": "Jenna Boyd"
+    },
+    {
+        "id": 6031,
+        "qa_question": "map@Does the video game produced by Mark Meer feature a view of Earth from space on its cover?",
+        "qa_column": "title",
+        "qa_answer": [
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "title"
+            ],
+            "rows": [
+                [
+                    "baldur's gate ii: shadows of amn \uff08 card of shadows of amn \uff09"
+                ],
+                [
+                    "baldur's gate ii: throne of bhaal ( card of throne of bhaal )"
+                ],
+                [
+                    "Mass Effect 3 ( mass effect 3 with a view of the earth)"
+                ],
+                [
+                    "baldur's gate: enhanced edition ( a sword at the bottom )"
+                ],
+                [
+                    "baldur's gate: enhanced edition ( a man versus a dragon )"
+                ],
+                [
+                    "dragon age: inquisition ( a man is battling with a dragon)"
+                ]
+            ]
+        },
+        "title": "Mark Meer"
+    },
+    {
+        "id": 21257,
+        "qa_question": "ans@The location with two tall narrow pointed towers near the waterfront, in Individual podiums of Riitta-Liisa Roponen, is?",
+        "qa_column": [
+            "Lahti"
+        ],
+        "qa_answer": [
+            "Lahti"
+        ],
+        "table": {
+            "header": [
+                "Lahti"
+            ],
+            "rows": [
+                [
+                    "a photo of lake geneva at night and at night in lahti"
+                ]
+            ]
+        },
+        "title": "Riitta-Liisa Roponen"
+    },
+    {
+        "id": 8555,
+        "qa_question": "ans@Was the film in which the role of Shannon was played by Madeline Carroll based on a true story?",
+        "qa_column": [
+            "title",
+            "I Can Only Imagine (film)"
+        ],
+        "qa_answer": [
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "title",
+                "I Can Only Imagine (film)"
+            ],
+            "rows": [
+                [
+                    "i can only imagine (I Can Only Imagine is a 2018 American Christian drama film directed by the Erwin Brothers and written by Alex Cramer, Jon Erwin, and Brent McCorkle, based on the story behind the MercyMe song of the same name, the best-selling Christian single of all time. The film stars J. Michael Finley as Bart Millard, the lead singer who wrote the song about his relationship with his father (Dennis Quaid). Madeline Carroll, Priscilla Shirer, Cloris Leachman, and Trace Adkins also star.) ( the only machine is listed (or ranked) 10 on the list the best movies and tv)"
+                ],
+                [
+                    "I Can Only Imagine is a 2018 American Christian drama film directed by the Erwin Brothers and written by Alex Cramer, Jon Erwin, and Brent McCorkle, based on the story behind the MercyMe song of the same name, the best-selling Christian single of all time. The film stars J. Michael Finley as Bart Millard, the lead singer who wrote the song about his relationship with his father (Dennis Quaid). Madeline Carroll, Priscilla Shirer, Cloris Leachman, and Trace Adkins also star. the only machine is listed (or ranked) 10 on the list the best movies and tv"
+                ]
+            ]
+        },
+        "title": "Madeline Carroll"
+    },
+    {
+        "id": 3145,
+        "qa_question": "ans@In fourth round proper of 1957\u201358 FA Cup, who got promoted to premier league this year and was the Home team when the Score was 4\u20131?",
+        "qa_column": [
+            "home team"
+        ],
+        "qa_answer": [
+            "Cardiff City"
+        ],
+        "table": {
+            "header": [
+                "home team"
+            ],
+            "rows": [
+                [
+                    "darlington ( railroads of the quakers logo)",
+                    "cardiff city"
+                ]
+            ]
+        },
+        "title": "1957\u201358 FA Cup"
+    },
+    {
+        "id": 2936,
+        "qa_question": "ans@Who does the voice in the show?",
+        "qa_column": [
+            "title",
+            "the Belko Experiment"
+        ],
+        "qa_answer": [
+            "gregg henry"
+        ],
+        "table": {
+            "header": [
+                "title",
+                "the Belko Experiment"
+            ],
+            "rows": [
+                [
+                    "the belko experiment (In a rage, Mike kills Barry with a tape dispenser. The building is then unsealed, as he is the last survivor, and the soldiers escort him to the hangar next door. There, he meets the owner of the voice (Gregg Henry), who introduces himself as a social scientist who believes that discoveries about human nature can only come from placing people in extreme environments. As he and his colleagues begin to question Mike about his emotional and mental state, Mike notices a panel of switches that correspond to the eighty employees. Having planted the trackers that Marty collected on the soldiers and the Voice, he flips every switch except his own. The trackers explode, killing the soldiers and wounding the Voice, before Mike grabs a gun and kills the remaining scientists. The Voice attempts to reason with Mike but Mike kills him. He then leaves the warehouse in a state of shock. The view zooms out to reveal that Mike is one of many sole survivors from similar experiments, being watched by another group through security cameras. A new voice states \"end stage one\" and \"commence stage two.\") ( actor in a still from the movie)"
+                ],
+                [
+                    "In a rage, Mike kills Barry with a tape dispenser. The building is then unsealed, as he is the last survivor, and the soldiers escort him to the hangar next door. There, he meets the owner of the voice (Gregg Henry), who introduces himself as a social scientist who believes that discoveries about human nature can only come from placing people in extreme environments. As he and his colleagues begin to question Mike about his emotional and mental state, Mike notices a panel of switches that correspond to the eighty employees. Having planted the trackers that Marty collected on the soldiers and the Voice, he flips every switch except his own. The trackers explode, killing the soldiers and wounding the Voice, before Mike grabs a gun and kills the remaining scientists. The Voice attempts to reason with Mike but Mike kills him. He then leaves the warehouse in a state of shock. The view zooms out to reveal that Mike is one of many sole survivors from similar experiments, being watched by another group through security cameras. A new voice states \"end stage one\" and \"commence stage two.\" actor in a still from the movie"
+                ]
+            ]
+        },
+        "title": "Sean Gunn"
+    },
+    {
+        "id": 23229,
+        "qa_question": "ans@in Tournaments of 1998 Ladies European Tour, does Spain share a land border with the location that has a star in the exact center of the flag?",
+        "qa_column": [
+            "Morocco\u2013Spain border",
+            "Morocco"
+        ],
+        "qa_answer": [
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "Morocco\u2013Spain border",
+                "Morocco"
+            ],
+            "rows": [
+                [
+                    "The Morocco-Spain border is located along the Plazas de soberan\u00eda, Ceuta, Melilla, and Albor\u00e1n Island along the north coast of Morocco."
+                ],
+                [
+                    "flag of the islamic republic of morocco with a star in the center"
+                ]
+            ]
+        },
+        "title": "1998 Ladies European Tour"
+    },
+    {
+        "id": 14260,
+        "qa_question": "map@Is there a wildcat on the logo?",
+        "qa_column": "wnba team",
+        "qa_answer": [
+            "no",
+            "no",
+            "yes",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "wnba team"
+            ],
+            "rows": [
+                [
+                    "houston comets"
+                ],
+                [
+                    "san antonio silver stars ( san antonio starts logo with a boom )"
+                ],
+                [
+                    "minnesota lynx ( a logo with wildcat of minnesota lynx )"
+                ],
+                [
+                    "phoenix mercury ( the phoenix mercury logo with the sun )"
+                ]
+            ]
+        },
+        "title": "2006 WNBA draft"
+    },
+    {
+        "id": 2376,
+        "qa_question": "ans@How many times did the opponent that have a bird on their logo, been in the super bowl?",
+        "qa_column": [
+            "History of the Philadelphia Eagles",
+            "Philadelphia Eagles"
+        ],
+        "qa_answer": [
+            "3"
+        ],
+        "table": {
+            "header": [
+                "History of the Philadelphia Eagles",
+                "Philadelphia Eagles"
+            ],
+            "rows": [
+                [
+                    "The history of the Philadelphia Eagles begins in 1933. In their history, the Eagles have appeared in the Super Bowl three times, losing in their first two appearances but winning the third, in 2018. They won three NFL Championships, the precursor to the Super Bowl, in four appearances."
+                ],
+                [
+                    "the logo with an eagle on it"
+                ]
+            ]
+        },
+        "title": "1951 Green Bay Packers season"
+    },
+    {
+        "id": 200,
+        "qa_question": "ans@This song released on September 29, 1983 and inspired a hit song by Usher was written by who?",
+        "qa_column": [
+            "Uptown Girl",
+            "Can't Stop Won't Stop (Usher song)"
+        ],
+        "qa_answer": [
+            "Billy Joel"
+        ],
+        "table": {
+            "header": [
+                "Uptown Girl",
+                "Can't Stop Won't Stop (Usher song)"
+            ],
+            "rows": [
+                [
+                    "\"Uptown Girl\" is a song written and performed by American musician Billy Joel. It was released on September 29, 1983, on his ninth studio album An Innocent Man (1983). The lyrics describe a working-class \"downtown man\" attempting to woo a wealthy \"uptown girl.\"",
+                    "\"Can't Stop Won't Stop\" is a song recorded by American recording artist Usher for his seventh studio album Looking 4 Myself (2012). Written and produced by Will \"will.i.am\" Adams and Keith Harris, the song contains an interpolation of the bridge to Billy Joel's 1983 hit single \"Uptown Girl\". Musically, \"Can't Stop Won't Stop\" is a eurodance and dance-pop song that incorporates elements of dubstep."
+                ]
+            ]
+        },
+        "title": "PNC Park"
+    },
+    {
+        "id": 300,
+        "qa_question": "ans@Oliver Mellor played Dr. Matt Carter on the TV show that had Tracy Barlow kill who?",
+        "qa_column": [
+            "title",
+            "Charlie Stubbs (Coronation Street)"
+        ],
+        "qa_answer": [
+            "Charlie"
+        ],
+        "table": {
+            "header": [
+                "title",
+                "Charlie Stubbs (Coronation Street)"
+            ],
+            "rows": [
+                [
+                    "coronation street( a screenshot of the coronation street soundtrack.)",
+                    "In 2005, Charlie began a relationship with Tracy Barlow (Kate Ford). He convinced her to move in with him and later in February 2006, manipulated her into having her daughter Amy (Amber Chadwick) move in with her parents. In turn, Tracy began to manipulate Charlie. She pretended to be pregnant and used the money he gave her for an abortion to buy expensive shoes and used her \"grief\" to have him allow Amy to move back in. When Shelley visited before her mother\u2019s marriage to Fred Elliott (John Savident), she and Charlie had a one-night stand. She told Tracy about their night of passion, who accused her of lying. Shelley later revealed that she was pregnant with Charlie\u2019s baby but didn\u2019t allow Charlie to have anything to do with the baby, and left. He and Tracy briefly split but reconciled. Charlie later began an affair with Maria Sutherland (Samia Smith), who was renting his flat. When David Platt (Jack P. Shepherd) discovered the affair he tried to blackmail Charlie, threatening to reveal the affair to Tracy. Charlie retaliated by trying to drown David in the bath. When Tracy eventually found out about the affair, they split once more. Tracy began to plot revenge against Charlie and pretended to make amends with Charlie. She pretended he was abusing her to the point of burning herself with an iron to make it look like Charlie was responsible for her injuries. Charlie eventually realized his partner was seeking revenge and when he was about to tell her their relationship was over, she insisted on performing a lap dance for him. She hit him round the head with a heavy ornament, and he later died in hospital. She claimed she\u2019d killed him in self-defence but the court found her guilty and she was given a life sentence."
+                ]
+            ]
+        },
+        "title": "Oliver Mellor"
+    },
+    {
+        "id": 350,
+        "qa_question": "ans@corn beans and squash the three most important crops of the wampanoag were also known as?",
+        "qa_column": [
+            "Wampanoag"
+        ],
+        "qa_answer": [
+            "three sisters"
+        ],
+        "table": {
+            "header": [
+                "Wampanoag"
+            ],
+            "rows": [
+                [
+                    "Traditionally Wampanoag people have been semi-sedentary, with seasonal movements between fixed sites in present-day southern New England. The men often traveled far north and south along the Eastern seaboard for seasonal fishing expeditions, and sometimes stayed in those distant locations for weeks and months at a time. The women cultivated varieties of the \"three sisters\" (the intercropping of maize, climbing beans, and squash) as the staples of their diet, supplemented by fish and game caught by the men. Each community had authority over a well-defined territory from which the people derived their livelihood through a seasonal round of fishing, planting, harvesting, and hunting. Because southern New England was thickly populated by indigenous peoples, hunting grounds had strictly defined boundaries."
+                ]
+            ]
+        },
+        "title": "Peter Egan"
+    },
+    {
+        "id": 402,
+        "qa_question": "ans@who played the part of the Cowardly Lion?",
+        "qa_column": [
+            "title",
+            "Bert Lahr"
+        ],
+        "qa_answer": [
+            "Bert Lahr"
+        ],
+        "table": {
+            "header": [
+                "title",
+                "Bert Lahr"
+            ],
+            "rows": [
+                [
+                    "the wizard of oz( theatrical poster for silent film.)",
+                    "Bert Lahr (August 13, 1895 \u2013 December 4, 1967) was an American actor, particularly of stage and film, and comedian. Lahr is known for his role as the Cowardly Lion, as well as his counterpart Kansas farmworker Zeke, in The Wizard of Oz (1939). He was well known for his explosive humor, but also adapted well to dramatic roles and his work in burlesque, vaudeville, and on Broadway."
+                ]
+            ]
+        },
+        "title": "1980 in home video"
+    },
+    {
+        "id": 410,
+        "qa_question": "ans@what is the city that was the center of imperial life in the roman empire in the early fifth century?",
+        "qa_column": [
+            "Imperial fora"
+        ],
+        "qa_answer": [
+            "Rome"
+        ],
+        "table": {
+            "header": [
+                "Imperial fora"
+            ],
+            "rows": [
+                [
+                    "The Imperial Fora (\"Fori Imperiali \" in Italian) are a series of monumental \"fora\" (public squares), constructed in Rome over a period of one and a half centuries, between 46 BC and 113 AD. The forums were the center of the Roman Republic and of the Roman Empire."
+                ]
+            ]
+        },
+        "title": "List of newspapers in Italy"
+    },
+    {
+        "id": 2005,
+        "qa_question": "ans@which one has a rooster on his logo?",
+        "qa_column": [
+            "college"
+        ],
+        "qa_answer": [
+            "South Carolina"
+        ],
+        "table": {
+            "header": [
+                "college"
+            ],
+            "rows": [
+                [
+                    "byu( this is the new yankee stadium that will be used at the super bowl of the)"
+                ],
+                [
+                    "south carolina( i pinned this because i think this is what the logo would look like on sports equipment business)"
+                ]
+            ]
+        },
+        "title": "2013 Detroit Lions season"
+    },
+    {
+        "id": 2015,
+        "qa_question": "ans@who is the guy in call me maybe video?",
+        "qa_column": [
+            "Call Me Maybe"
+        ],
+        "qa_answer": [
+            "Holden Nowell"
+        ],
+        "table": {
+            "header": [
+                "Call Me Maybe"
+            ],
+            "rows": [
+                [
+                    "The video begins with Jepsen spying on her attractive tattooed neighbour (Holden Nowell) as he is working on his lawn. As he takes his shirt off and notices she is staring at him, Jepsen slips on her high heels and falls below her window. She is reading the books Love at First Sight (Men In Uniform) by B.J. Daniels and Skylar's Outlaw by Linda Warren. The scene then cuts to her garage, where she is rehearsing the track with her band. Following the rehearsals, her bandmates push her to go and wash her car, where she tries to gain her neighbour's attention with various provocative poses only to fall from the hood of the car. She is briefly knocked out from the fall, during which she dreams of a romance novel-type encounter with her crush against the backdrop of Peggys Cove. As she comes to, the neighbour then helps her get up, and watches the band rehearse the track again. After turning and writing down her telephone number, Jepsen sees her neighbour pass one of her male bandmates (Tavish Crowe) his own number, indicating he doesn't like women at all and is gay, where the very end shows that Jepsen is taken aback by this. The video received three nominations on the 2012 MuchMusic Video Awards in the categories of UR Fave Video, Pop Video of the Year, and Video of the Year. wallpaper with a well dressed girl titled pop artist"
+                ]
+            ]
+        },
+        "title": "The Voice UK (series 2)"
+    },
+    {
+        "id": 410,
+        "qa_question": "ans@the building in the top right has what at its top?",
+        "qa_column": [
+            "val"
+        ],
+        "qa_answer": [
+            "Dome"
+        ],
+        "table": {
+            "header": [
+                "val"
+            ],
+            "rows": [
+                [
+                    "rome( visit the vatican city and sistine chapel in rome with a dome.)"
+                ]
+            ]
+        },
+        "title": "List of newspapers in Italy"
+    },
+    {
+        "id": 100,
+        "qa_question": "map@Has crossed swords on its logo?",
+        "qa_column": "signed from",
+        "qa_answer": [
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "signed from"
+            ],
+            "rows": [
+                [
+                    "penrith panthers (Logo of Penrith Panthers, a cartoon picture of a black leopard)"
+                ],
+                [
+                    "south sydney rabbitohs (Logo of South Sydney Rabbitohs, A white rabbit jumps on an oval pattern with a green frame on a red background )"
+                ],
+                [
+                    "gold coast titans (Logo of Gold Coast Titans, a warrior in golden armor holding two swords in a cross shape)"
+                ],
+                [
+                    "salford red devils"
+                ],
+                [
+                    "leigh centurions (Logo of Leigh Centurions, a warrior wearing a half-moon helmet and holding a red-edged shield)"
+                ],
+                [
+                    "castleford tigers (Logo of Castleford Tigers, the head of a roaring tiger in the middle)"
+                ],
+                [
+                    "sale sharks (Logo of Sale Sharks, a shark in deep blue)"
+                ],
+                [
+                    "l\u00e9zignan sangliers (Logo of L\u00e9zignan Sangliers, a pink and green logo with name on the right and a windmill-like logo on the left)"
+                ],
+                [
+                    "leigh centurions (Logo of Leigh Centurions, a warrior wearing a half-moon helmet and holding a red-edged shield)"
+                ]
+            ]
+        },
+        "title": "2018 Warrington Wolves season (Transfers | In)"
+    },
+    {
+        "id": 150,
+        "qa_question": "map@In which year did the San Francisco 49ers move to their new stadium?",
+        "qa_column": "game site",
+        "qa_answer": [
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "1971",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/",
+            "/"
+        ],
+        "table": {
+            "header": [
+                "game site"
+            ],
+            "rows": [
+                [
+                    "edward jones dome"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "mercedes-benz superdome (The logo of mercedes-benz superdome, in a beautiful font)"
+                ],
+                [
+                    "raymond james stadium (The logo of raymond james stadium, in a beautiful font)"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "candlestick park (Candlestick Park was an outdoor sports and entertainment stadium in the West Coast of the United States, located in San Francisco, in the Bayview Heights area. The stadium was originally the home of Major League Baseball's San Francisco Giants, who played there from 1960 until moving into Pacific Bell Park (since renamed AT&T Park) in 2000. It was also the home field of the San Francisco 49ers of the National Football League from 1971 through 2013. The 49ers moved to Levi's Stadium in Santa Clara for the 2014 season.)"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "bye"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "everbank field"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "lincoln financial field"
+                ],
+                [
+                    "university of phoenix stadium"
+                ],
+                [
+                    "lp field"
+                ],
+                [
+                    "centurylink field (The logo of CenturyLink Field, on the left is a minimalist icon of a stadium, on the right is a text logo)"
+                ],
+                [
+                    "university of phoenix stadium"
+                ]
+            ]
+        },
+        "title": "2013 Arizona Cardinals season (Regular season)"
+    },
+    {
+        "id": 250,
+        "qa_question": "map@what is the person behind holding?",
+        "qa_column": "driver",
+        "qa_answer": [
+            "/",
+            "water",
+            "/",
+            "/",
+            "/",
+            "/",
+            "camera",
+            "/",
+            "/",
+            "/"
+        ],
+        "table": {
+            "header": [
+                "driver"
+            ],
+            "rows": [
+                [
+                    "dale earnhardt, jr. (r)"
+                ],
+                [
+                    "jeff burton (A man with a sunglasses and racing jacket just finish the game. The man behind him is holding a bottle of water.)"
+                ],
+                [
+                    "bobby labonte (A man with a sunglasses and racing jacket is waving his hands on the left and grabbing a coke on the right.)"
+                ],
+                [
+                    "rusty wallace (A man with a sunglasses and racing jacket is smiling, a man is behind him trying to push something.)"
+                ],
+                [
+                    "kevin lepage (A man with sunglasses and black racing jacket is smiling.)"
+                ],
+                [
+                    "jeremy mayfield (A man with a cap and racing jacket is cheering.)"
+                ],
+                [
+                    "dale earnhardt (A man with mustache is wearing a racing jacket, sunglasses, smiling. The man behind him is holding a camera and taking photo.)"
+                ],
+                [
+                    "terry labonte (A man with mustache is wearing a yellow racing jacket and smiling.)"
+                ],
+                [
+                    "tony stewart (A man with beard, grey hair and sunglasses is looking towards the left side of the picture.)"
+                ],
+                [
+                    "ricky rudd (A man in racing jacket and sunglasses gets out of his car before the race.)"
+                ]
+            ]
+        },
+        "title": "2000 DirecTV 500 (Top 10 results)"
+    },
+    {
+        "id": 450,
+        "qa_question": "map@Has a ship in logo or Charlotte Knights?",
+        "qa_column": "team",
+        "qa_answer": [
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "team"
+            ],
+            "rows": [
+                [
+                    "scranton/wilkes-barre railriders (An angry porcupine riding a railroad)"
+                ],
+                [
+                    "syracuse mets (An icon with orange words on a blue background)"
+                ],
+                [
+                    "charlotte knights (A knight's helmet surrounded by a crescent moon with the word art of Charlotte Knights under it and a crown above it)"
+                ],
+                [
+                    "durham bulls (A puffing bull is crossing the blue letter D)"
+                ],
+                [
+                    "gwinnett stripers (A green fish with an open mouth is swimming over the letter G of the word art for \"Gwinnett Stripers\")"
+                ],
+                [
+                    "norfolk tides (A toothy seahorse holding a sea god's fork stands in the middle of a circle with the word Norfolk Tides)"
+                ],
+                [
+                    "columbus clippers (A ship sails on word art on Columbus Clippers)"
+                ],
+                [
+                    "indianapolis indians (An indian style octagon pattern)"
+                ],
+                [
+                    "louisville bats (A flying bat clutching a baseball bat, flying over a baseball as a background and Louisville Bats as a circled icon)"
+                ],
+                [
+                    "toledo mud hens (A chicken in a baseball cap with the letter T is waving a baseball bat with word art by Toledo Mud Hens)"
+                ]
+            ]
+        },
+        "title": "International League (Current teams)"
+    },
+    {
+        "id": 142,
+        "qa_question": "map@What is the time span?",
+        "qa_column": "Term",
+        "qa_answer": [
+            "5",
+            "5",
+            "11",
+            "/",
+            "1",
+            "6",
+            "3",
+            "9",
+            "4",
+            "3",
+            "/",
+            "11",
+            "5",
+            "4",
+            "3",
+            "/"
+        ],
+        "table": {
+            "header": [
+                "Term"
+            ],
+            "rows": [
+                [
+                    "1859\u20131864"
+                ],
+                [
+                    "1864\u20131869"
+                ],
+                [
+                    "1869\u20131880"
+                ],
+                [
+                    "Term"
+                ],
+                [
+                    "1894\u20131895"
+                ],
+                [
+                    "1895\u20131901"
+                ],
+                [
+                    "1901\u20131904"
+                ],
+                [
+                    "1904\u20131913"
+                ],
+                [
+                    "1913\u20131917"
+                ],
+                [
+                    "1917\u20131920"
+                ],
+                [
+                    "Term"
+                ],
+                [
+                    "1927\u20131938"
+                ],
+                [
+                    "1938\u20131943"
+                ],
+                [
+                    "1943\u20131947"
+                ],
+                [
+                    "1947\u20131950"
+                ],
+                [
+                    "Term"
+                ]
+            ]
+        },
+        "title": "Electoral district of Lachlan"
+    },
+    {
+        "id": 145,
+        "qa_question": "map@Is the date during in 1900's?",
+        "qa_column": "Created",
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "yes",
+            "no",
+            "yes",
+            "yes",
+            "yes",
+            "yes",
+            "yes",
+            "yes",
+            "yes",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "Created"
+            ],
+            "rows": [
+                [
+                    "29 October 1874"
+                ],
+                [
+                    "2 January 1857"
+                ],
+                [
+                    "26 October 1874"
+                ],
+                [
+                    "15 July 1949"
+                ],
+                [
+                    "4 August 1821"
+                ],
+                [
+                    "24 April 1940"
+                ],
+                [
+                    "2 January 1857"
+                ],
+                [
+                    "3 March 1970"
+                ],
+                [
+                    "12 December 1961"
+                ],
+                [
+                    "6 January 1965"
+                ],
+                [
+                    "16 March 1964"
+                ],
+                [
+                    "13 December 1963"
+                ],
+                [
+                    "6 February 1962"
+                ],
+                [
+                    "16 August 1921"
+                ],
+                [
+                    "2 January 1857"
+                ]
+            ]
+        },
+        "title": "List of districts of Lima"
+    },
+    {
+        "id": 155,
+        "qa_question": "map@Is the time less than a week?",
+        "qa_column": "Length of use",
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes",
+            "no",
+            "no",
+            "no",
+            "no",
+            "yes"
+        ],
+        "table": {
+            "header": [
+                "Length of use"
+            ],
+            "rows": [
+                [
+                    "14 days"
+                ],
+                [
+                    "10 days"
+                ],
+                [
+                    "21 days"
+                ],
+                [
+                    "7 days"
+                ],
+                [
+                    "10 days"
+                ],
+                [
+                    "10 days"
+                ],
+                [
+                    "Daily"
+                ],
+                [
+                    "14 days"
+                ],
+                [
+                    "10 days"
+                ],
+                [
+                    "14 days"
+                ],
+                [
+                    "20 days"
+                ],
+                [
+                    "2 hours"
+                ]
+            ]
+        },
+        "title": "Crest Whitestrips"
+    },
+    {
+        "id": 578,
+        "qa_question": "map@Is this player from Norway?",
+        "qa_column": "Player",
+        "qa_answer": [
+            "no",
+            "no",
+            "no",
+            "no",
+            "no",
+            "no"
+        ],
+        "table": {
+            "header": [
+                "Player"
+            ],
+            "rows": [
+                [
+                    "Raymond van Barneveld"
+                ],
+                [
+                    "Raymond van Barneveld"
+                ],
+                [
+                    "Adrian Lewis"
+                ],
+                [
+                    "Dean Winstanley"
+                ],
+                [
+                    "Michael van Gerwen"
+                ],
+                [
+                    "Terry Jenkins"
+                ]
+            ]
+        },
+        "title": "PDC World Darts Championship"
+    }
+]

templates/prompts/prompt_mmqa_v2.txt

@@ -0,0 +1,523 @@
+Generate SQL given the question, table, passages, image captions to answer the question correctly.
+If question-relevant column(s) contents are not suitable for SQL comparisons or calculations, map it to a new column with clean content by a new grammar QA("map@").
+If mapping to a new column still can not answer the question with valid SQL, turn to an end-to-end solution by a new grammar QA("ans@"). This grammar aims to solve all the rest of complex questions or tables or passages or image captions.
+
+CREATE TABLE Dutch Ruppersberger (Electoral history)(
+	row_id int,
+	year int,
+	office text,
+	election text,
+	filledcolumnname real,
+	subject text,
+	party text,
+	votes text,
+	% text,
+	filledcolumnname_2 real,
+	opponent text,
+	party_2 text,
+	votes_2 text,
+	%_2 text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	office	election	filledcolumnname	subject	party	votes	%	filledcolumnname_2	opponent	party_2	votes_2	%_2
+0	1994	baltimore county executive	general	nan	dutch ruppersberger	democratic	n/a	n/a	nan	n/a	n/a	n/a	n/a
+1	1998	baltimore county executive	general	nan	dutch ruppersberger	democratic	166482	70.47	nan	john j. bishop	republican	69449	29.4
+2	2002	none	general	nan	dutch ruppersberger	democratic	105718	54.16	nan	helen delich bentley	republican	88954	45.57
+*/
+Q: What year was Elizabeth Matory the opponent of Charles Albert Ruppersberger?
+NeuralSQL: SELECT year FROM w WHERE opponent = 'elizabeth matory'
+
+
+CREATE TABLE Virtual Console (Titles)(
+	row_id int,
+	system text,
+	japan int,
+	[[list of virtual console games for wii u (north america)|north  america]] real,
+	pal region - europe real,
+	pal region - australia real)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	system	japan	[[list of virtual console games for wii u (north america)|north  america]]	pal region - europe	pal region - australia
+0	nes/famicom	148	94.0	89.0	89.0
+1	super nes/super famicom	101	51.0	49.0	49.0
+2	nintendo 64	22	21.0	21.0	21.0
+*/
+Q: Which system has a lower number for Japan of the virtual console systems: Game Boy Advance or the Japan-only console MSX?
+NeuralSQL: SELECT system FROM w WHERE system IN ('game boy advance', 'msx (japan only)') ORDER BY japan LIMIT 1
+
+
+CREATE TABLE 2018 Warrington Wolves season (Transfers | In)(
+	row_id int,
+	player text,
+	signed from text,
+	contract length text,
+	announced text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	player	signed from	contract length	announced
+0	sitaleki akauola	penrith panthers	p2y	2017-08-01 00:00:00
+1	bryson goodwin	south sydney rabbitohs	p2y	2017-10-01 00:00:00
+2	tyrone roberts	gold coast titans	p3y	2017-10-01 00:00:00
+*/
+CREATE TABLE Images(
+	row_id int,
+	gold coast titans text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	gold coast titans
+0	a logo for the golden knights is painted on the beach.
+*/
+Q: What player was transferred from the team that has crossed swords on its logo to the Warrington Wolves in the 2018 season?
+NeuralSQL: SELECT player FROM w WHERE QA("map@Has crossed swords on its logo?"; `signed from`) = 'yes'
+
+
+CREATE TABLE 2013 Arizona Cardinals season (Regular season)(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	result text,
+	record text,
+	game site text,
+	nfl.com recap text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	result	record	game site	nfl.com recap
+0	1	september 8	at st. louis rams	l 24–27	0–1	edward jones dome	[http://www.nfl.com/gamecenter/2013090810/2013/reg1/cardinals@rams recap]
+1	2	september 15	detroit lions	w 25–21	1–1	university of phoenix stadium	[http://www.nfl.com/gamecenter/2013091509/2013/reg2/lions@cardinals recap]
+2	3	september 22	at new orleans saints	l 7–31	1–2	mercedes-benz superdome	[http://www.nfl.com/gamecenter/2013092207/2013/reg3/cardinals@saints recap]
+*/
+CREATE TABLE Passages(
+	row_id int,
+	candlestick park text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	candlestick park
+0	candlestick park was an outdoor sports and entertainment stadium in the west coast of the united states, located in san francisco, in the bayview heights area. the stadium was originally the home of major league baseball's san francisco giants, who played there from 1960 until moving into pacific bell park (since renamed at&t park) in 2000. it was also the home field of the san francisco 49ers of the national football league from 1971 through 2013. the 49ers moved to levi's stadium in santa clara for the 2014 season.
+*/
+Q: In which year did the San Francisco 49ers move to their new stadium, which was the location that the Arizona Cardinals lost a 2013 regular season game by the score of 20 to 32?
+NeuralSQL: SELECT QA("map@In which year did the San Francisco 49ers move to their new stadium?"; `game site`) FROM w WHERE opponent LIKE '%san francisco 49ers%' AND result = 'l 20–32'
+
+
+CREATE TABLE PNC Park (Concerts)(
+	row_id int,
+	date text,
+	artist text,
+	opening act(s) text,
+	tour / concert name text,
+	attendance text,
+	revenue text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	artist	opening act(s)	tour / concert name	attendance	revenue	notes
+0	2003-08-06 00:00:00	bruce springsteen & the e street band	—	the rising tour	42301 / 48074	$3137575	none
+1	2005-06-26 00:00:00	jimmy buffett	—	a salty piece of land tour	—	—	sonny landreth and jake shimabukuro were special guests http://www.buffettworld.com/archives/2005-a-salty-piece-of-land/6-26/
+2	2005-09-28 00:00:00	the rolling stones	pearl jam	a bigger bang	—	—	none
+*/
+CREATE TABLE Passages(
+	row_id int,
+	can't stop won't stop (usher song) text,
+	uptown girl text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	can't stop won't stop (usher song)	uptown girl
+0	"can't stop won't stop" is a song recorded by american recording artist usher for his seventh studio album looking 4 myself (2012). written and produced by will "will.i.am" adams and keith harris, the song contains an interpolation of the bridge to billy joel's 1983 hit single "uptown girl". musically, "can't stop won't stop" is a eurodance and dance-pop song that incorporates elements of dubstep.	"uptown girl" is a song written and performed by american musician billy joel. it was released on 1983-9-29, on his ninth studio album an innocent man (1983). the lyrics describe a working-class "downtown man" attempting to woo a wealthy "uptown girl."
+*/
+Q: This song released on September 29, 1983 and inspired a hit song by Usher was written by who?
+NeuralSQL: QA("ans@This song released on September 29, 1983 and inspired a hit song by Usher was written by who?"; Uptown Girl; Can't Stop Won't Stop (Usher song) )
+
+
+CREATE TABLE 2000 DirecTV 500 (Top 10 results)(
+	row_id int,
+	pos int,
+	grid int,
+	car number (no.) int,
+	driver text,
+	team text,
+	manufacturer text,
+	laps completed (laps) int,
+	points int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	pos	grid	car number (no.)	driver	team	manufacturer	laps completed (laps)	points
+0	1	4	8	dale earnhardt, jr. (r)	dale earnhardt, inc.	chevrolet	334	185
+1	2	37	99	jeff burton	roush racing	ford	334	175
+2	3	14	18	bobby labonte	joe gibbs racing	pontiac	334	170
+*/
+CREATE TABLE Images(
+	row_id int,
+	dale earnhardt text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	dale earnhardt
+0	a man wearing a number of neckties and a mustache.
+*/
+Q: The 2000 DirecTv 500 Top 10 Driver with 146 points has a person behind them holding what?
+NeuralSQL: SELECT QA("map@what is the person behind holding?"; driver) FROM w WHERE points = 146
+
+
+CREATE TABLE Oliver Mellor (Credits | Television)(
+	row_id int,
+	year text,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	2006	the royal	dr. guy fitzgerald	none
+1	2006	hollyoaks: in the city	detective monroe	3 episodes
+2	2006	doctor who	matt	episode "army of ghosts"
+*/
+CREATE TABLE Passages(
+	row_id int,
+	charlie stubbs (coronation street) text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	charlie stubbs (coronation street)
+0	in 2005, charlie began a relationship with tracy barlow (kate ford). he convinced her to move in with him and later in february 2006, manipulated her into having her daughter amy (amber chadwick) move in with her parents. in turn, tracy began to manipulate charlie. she pretended to be pregnant and used the money he gave her for an abortion to buy expensive shoes and used her "grief" to have him allow amy to move back in. when shelley visited before her mother’s marriage to fred elliott (john savident), she and charlie had a 1-tni stand. she told tracy about their tni of passion, who accused her of lying. shelley later revealed that she was pregnant with charlie’s baby but didn’t allow charlie to have anything to do with the baby, and left. he and tracy briefly split but reconciled. charlie later began an affair with maria sutherland (samia smith), who was renting his flat. when david platt (jack p. shepherd) discovered the affair he tried to blackmail charlie, threatening to reveal the affair to tracy. charlie retaliated by trying to drown david in the bath. when tracy eventually found out about the affair, they split once more. tracy began to plot revenge against charlie and pretended to make amends with charlie. she pretended he was abusing her to the point of burning herself with an iron to make it look like charlie was responsible for her injuries. charlie eventually realized his partner was seeking revenge and when he was about to tell her their relationship was over, she insisted on performing a lap dance for him. she hit him round the head with a heavy ornament, and he later died in hospital. she claimed she’d killed him in self-defence but the court found her guilty and she was given a life sentence.
+*/
+Q: Oliver Mellor played Dr. Matt Carter on the TV show that had Tracy Barlow kill who?
+NeuralSQL: QA("ans@Oliver Mellor played Dr. Matt Carter on the TV show that had Tracy Barlow kill who?"; SELECT title FROM w WHERE role = 'dr. matt carter'; Charlie Stubbs (Coronation Street))
+
+
+CREATE TABLE Peter Egan (Filmography)(
+	row_id int,
+	year text,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	1971	1 brief su	bill denton	none
+1	1971	elizabeth r	earl of southampton	episode: "sweet englands pride"
+2	1973	the hireling	captain hugh cantrip	none
+*/
+CREATE TABLE Passages(
+	row_id int,
+	wampanoag text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	wampanoag
+0	traditionally wampanoag people have been semi-sedentary, with seasonal movements between fixed sites in present-day southern new england. the men often traveled far north and south along the eastern seaboard for seasonal fishing expeditions, and sometimes stayed in those distant locations for weeks and months at a time. the women cultivated varieties of the "3 sisters" (the intercropping of maize, climbing beans, and squash) as the staples of their diet, supplemented by fish and game caught by the men. each community had authority over a well-defined territory from which the people derived their livelihood through a seasonal round of fishing, planting, harvesting, and hunting. because southern new england was thickly populated by indigenous peoples, hunting grounds had strictly defined boundaries.
+*/
+Q: corn beans and squash the three most important crops of the wampanoag were also known as
+NeuralSQL: QA("ans@corn beans and squash the three most important crops of the wampanoag were also known as?"; Wampanoag)
+
+
+CREATE TABLE 1980 in home video (Movie releases)(
+	row_id int,
+	u.s./canada release date text,
+	title text,
+	studio text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	u.s./canada release date	title	studio	notes
+0	january 1	the muppet movie	magnetic video	betamax release laserdisc release   vhs release
+1	march 4	20000 leagues under the sea	walt disney home entertainment	betamax release vhs release
+2	march 4	the apple dumpling gang	walt disney home entertainment	betamax release vhs release
+*/
+CREATE TABLE Passages(
+	row_id int,
+	bert lahr text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	bert lahr
+0	bert lahr ((1895-8-131967-12-4,p26410d)) was an american actor, particularly of stage and film, and comedian. lahr is known for his role as the cowardly lion, as well as his counterpart kansas farmworker zeke, in the wizard of oz (1939). he was well known for his explosive humor, but also adapted well to dramatic roles and his work in burlesque, vaudeville, and on broadway.
+*/
+Q: In the 1980 movie that was put out by the MGM/CBS Home Video studio, who played the part of the Cowardly Lion?
+NeuralSQL: QA("ans@who played the part of the Cowardly Lion?"; SELECT title FROM w WHERE studio = 'mgm/cbs home video';  Bert Lahr)
+
+
+CREATE TABLE List of newspapers in Italy (National daily newspapers)(
+	row_id int,
+	newspaper text,
+	circulation text,
+	headquarters text,
+	est. int,
+	political alignment text,
+	nameplate text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	newspaper	circulation	headquarters	est.	political alignment	nameplate
+0	corriere della sera	242684	milan	1876	centrism	200x200px
+1	la repubblica	198835	rome	1976	social democracy	150x150px
+2	la gazzetta dello sport	161796	milan	1896	—	200x200px
+*/
+CREATE TABLE Passages(
+	row_id int,
+	early middle ages text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	early middle ages
+0	for almost p1000y, rome was the most politically important, richest and largest city in europe. around 100 ce, it had a population of about 450000, and declined to a mere 20000 during the early middle ages, reducing the sprawling city to groups of inhabited buildings interspersed among large areas of ruins and vegetation.
+*/
+CREATE TABLE Images(
+	row_id int,
+	rome text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	rome
+0	a series of photographs showing a colorful scene.
+*/
+Q: In the city that was the center of imperial life in the roman empire in the early fifth century, the building in the top right has what at its top?
+NeuralSQL: QA("ans@he building in the top right has what at its top?"; QA("ans@what is the city that was the center of imperial life in the roman empire in the early fifth century?"; Imperial fora))
+
+
+CREATE TABLE International League (Current teams)(
+	row_id int,
+	division text,
+	team text,
+	founded int,
+	mlb affiliation text,
+	affiliated int,
+	city text,
+	stadium text,
+	capacity int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	division	team	founded	mlb affiliation	affiliated	city	stadium	capacity
+0	north	buffalo bisons	1985	toronto blue jays	2013	buffalo, new york	sahlen field	16600
+1	north	lehigh valley ironpigs	2008	philadelphia phillies	2007	allentown, pennsylvania	coca-cola park	10100
+2	north	pawtucket red sox	1973	boston red sox	1970	pawtucket, rhode island	mccoy stadium	10031
+*/
+CREATE TABLE Images(
+	row_id int,
+	columbus clippers text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	columbus clippers
+0	a large blue and white clock on the side of a building.
+*/
+Q: Was the Team that has a ship in logo or Charlotte Knights, the one with earlier affiliation in Current teams of International League?
+NeuralSQL: SELECT team FROM w WHERE team = 'charlotte knights' OR QA("map@Has a ship in logo or Charlotte Knights?"; team) = 'yes' ORDER BY founded LIMIT 1
+
+
+CREATE TABLE Warren Burton (Filmography)(
+	row_id int,
+	year int,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	1976	baby blue marine	second serviceman	none
+1	1977	chatterbox	tv reporter	none
+2	1977	the world's greatest lover	ludwig	none
+*/
+CREATE TABLE Images(
+	row_id int,
+	green lantern (film) text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	green lantern (film)
+0	a picture of a green and white costume and glasses.
+*/
+Q: How many people are on the poster for Green Lantern (film)?
+NeuralSQL: QA("ans@How many people are on the poster for Green Lantern (film)?";  Green Lantern (film))
+
+
+CREATE TABLE One Hour Photo (Accolades)(
+	row_id int,
+	award text,
+	category text,
+	recipients text,
+	result real)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	award	category	recipients	result
+0	critics' choice movie awards	best actor	robin williams	nan
+1	dallas–fort worth film critics association	best actor	robin williams	nan
+2	online film critics society	best actor	robin williams	nan
+*/
+CREATE TABLE Images(
+	row_id int,
+	saturn award text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	saturn award
+0	a man in a suit and tie holding a glass.
+*/
+Q: What is he holding in Saturn Award?
+NeuralSQL: QA("ans@What is he holding?"; Saturn Award)
+
+
+CREATE TABLE 2013 Detroit Lions season (2013 Draft class)(
+	row_id int,
+	draft order - round int,
+	draft order - choice int,
+	draft order - overall int,
+	player name text,
+	position text,
+	height text,
+	weight text,
+	college text,
+	contract text,
+	notes text,
+	source text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	draft order - round	draft order - choice	draft order - overall	player name	position	height	weight	college	contract	notes	source
+0	1	5	5	ezekiel ansah	defensive end	6ft 5 in	271lbs	byu	p5y /	none	[http://www.mlive.com/lions/index.ssf/2013-4/detroit_lions_select_ezekiel_a.html detroit lions select ezekiel ansah in first round of 2013 nfl draft] mlive.com, 2013-4-26
+1	2	4	36	darius slay	defensive back	6ft 1 in	190lbs	mississippi state	p4y /	none	[http://www.mlive.com/lions/index.ssf/2013-4/detroit_lions_select_mississip.html detroit lions select mississippi state cb darius slay in second round of 2013 nfl draft] mlive.com, 2013-4-27
+2	3	3	65	larry warford	offensive lineman	6ft 3 in	343lbs	kentucky	p4y /	none	[http://www.mlive.com/lions/index.ssf/2013-4/detroit_lions_fill_massive_nee.html detroit lions fill massive need with massive guard prospect larry warford] mlive.com, 2013-4-27
+*/
+CREATE TABLE Images(
+	row_id int,
+	south carolina gamecocks football text,
+	seattle seahawks text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	south carolina gamecocks football	seattle seahawks
+0	a group of people standing next to each other.	a large green and white bird with numbers.
+*/
+Q: What educational institution has a rooster on its logo and was the school listed in the 2013 Detroit Lions draft class for the defensive end player position?
+NeuralSQL: QA("ans@which one has a rooster on his logo?"; SELECT college FROM w WHERE position='defensive end')
+
+
+CREATE TABLE Melia Kreiling (Filmography |  Film roles)(
+	row_id int,
+	year int,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	2012	suspension of disbelief	juliette	none
+1	2013	company of heroes	kestrel	direct-to-video film
+2	2013	leopard	kara	none
+*/
+CREATE TABLE Passages(
+	row_id int,
+	list of marvel cinematic universe films text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	list of marvel cinematic universe films
+0	the first film in the marvel cinematic universe was iron man (2008), which was distributed by paramount pictures. paramount also distributed iron man 2 (2010), thor (2011) and captain america: the first avenger (2011), while universal pictures distributed the incredible hulk (2008). walt disney studios motion pictures began distributing the films with the 2012 crossover film the avengers, which concluded phase 1 of the franchise. phase 2 includes iron man 3 (2013), thor: the dark world (2013), captain america: wi soldier (2014), guardians of the galaxy (2014), avengers: age of ultron (2015), and ant-man (2015).
+*/
+Q: What was Melia Kreiling's role in the film that is the next Marvel movie after 'Captain America the Winter Soldier'?
+NeuralSQL: SELECT role FROM w WHERE title = QA("ans@which is the next  Marvel movie after 'Captain America the Winter Soldier'?"; List of Marvel Cinematic Universe films)
+
+
+CREATE TABLE 2006 Grand Prix of Portland (Qualifying results)(
+	row_id int,
+	pos int,
+	nat real,
+	name text,
+	team text,
+	qual 1 text,
+	qual 2 text,
+	best text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	pos	nat	name	team	qual 1	qual 2	best
+0	1	nan	bruno junqueira	newman/haas racing	59.576	57.631	57.631
+1	2	nan	a. j. allmendinger	forsythe racing	58.378	57.639	57.639
+2	3	nan	sébastien bourdais	newman/haas racing	58.464	57.646	57.646
+*/
+CREATE TABLE Passages(
+	row_id int,
+	jtg daugherty racing text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	jtg daugherty racing
+0	jtg daugherty racing (formerly st motorsports and jtg racing) is an american professional stock car racing team that currently competes in the monster energy nascar cup series. the team is owned by former advertising executive tad geschickter and his wife jodi, along with current espn analyst brad daugherty. the team formerly had alliances with wood brothers racing, then michael waltrip racing, and currently has a technical alliance with richard childress racing. the team currently fields the no. 37 cottonelle chevrolet ss driven by roush development driver chris buescher and the no. 47 clorox/bush's/scott products chevrolet ss driven by a. j. allmendinger in the monster energy nascar cup series.
+*/
+Q: The driver of Nascar number 47 qualified for the 2006 Grand Prix of Portland for which team?
+NeuralSQL: SELECT name FROM w WHERE team = QA("ans@which driver is number 47?"; JTG Daugherty Racing)
+
+
+CREATE TABLE List of churches in Copenhagen ([[Amager]])(
+	row_id int,
+	name text,
+	denomination text,
+	year int,
+	coordinates real,
+	image text,
+	refs real)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	name	denomination	year	coordinates	image	refs
+0	all saints' church	church of denmark	1932	nan	150px	nan
+1	dragør church	church of denmark	1885	nan	150px	nan
+2	hans tausen's church	church of denmark	1924	nan	150px	nan
+*/
+CREATE TABLE Images(
+	row_id int,
+	all saints' church, copenhagen text,
+	dragør church text,
+	nathanael's church text,
+	st. anne's church, copenhagen text,
+	sundby church text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	all saints' church, copenhagen	dragør church	nathanael's church	st. anne's church, copenhagen	sundby church
+0	 type of place of worship	 church of the holy trinity	 church of the holy trinity	 the building where the hotel is located	 a red brick church with a steeple and a flagpole in front of it.
+*/
+Q: Among Copenhagen churches on the "Amager" list, which have spires and are affiliated with the Church of Denmark denomination?
+NeuralSQL: SELECT name FROM w WHERE denomination = 'church of denmark' AND QA("map@does it have spires?"; name) = 'yes'
+
+
+CREATE TABLE Final Straw Tour (UK Tour (Leg III))(
+	row_id int,
+	date text,
+	city text,
+	country text,
+	venue text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	city	country	venue
+0	support acts: terra diablo & astrid	support acts: terra diablo & astrid	support acts: terra diablo & astrid	support acts: terra diablo & astrid
+1	2004-3-2	newcastle	england	newcastle university
+2	2004-3-3	liverpool	england	carling academy
+*/
+CREATE TABLE Images(
+	row_id int,
+	oxford text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	oxford
+0	 a guide to the city of edinburgh
+*/
+Q: The final straw tour held leg 3 of the UK tour on March 13, 2004 in this city with how many views on the bottom?
+NeuralSQL: SELECT QA("map@how many views on the bottom?"; city) FROM w WHERE date = '2004-3-13'

templates/prompts/prompt_mmqa_v2_Qa.txt

@@ -0,0 +1,521 @@
+Generate answer given the question, table, passages, image captions to answer the question correctly.
+
+CREATE TABLE Dutch Ruppersberger (Electoral history)(
+	row_id int,
+	year int,
+	office text,
+	election text,
+	filledcolumnname real,
+	subject text,
+	party text,
+	votes text,
+	% text,
+	filledcolumnname_2 real,
+	opponent text,
+	party_2 text,
+	votes_2 text,
+	%_2 text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	office	election	filledcolumnname	subject	party	votes	%	filledcolumnname_2	opponent	party_2	votes_2	%_2
+0	1994	baltimore county executive	general	nan	dutch ruppersberger	democratic	n/a	n/a	nan	n/a	n/a	n/a	n/a
+1	1998	baltimore county executive	general	nan	dutch ruppersberger	democratic	166482	70.47	nan	john j. bishop	republican	69449	29.4
+2	2002	none	general	nan	dutch ruppersberger	democratic	105718	54.16	nan	helen delich bentley	republican	88954	45.57
+*/
+Q: What year was Elizabeth Matory the opponent of Charles Albert Ruppersberger?
+A: 2018
+
+
+CREATE TABLE Virtual Console (Titles)(
+	row_id int,
+	system text,
+	japan int,
+	[[list of virtual console games for wii u (north america)|north  america]] real,
+	pal region - europe real,
+	pal region - australia real)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	system	japan	[[list of virtual console games for wii u (north america)|north  america]]	pal region - europe	pal region - australia
+0	nes/famicom	148	94.0	89.0	89.0
+1	super nes/super famicom	101	51.0	49.0	49.0
+2	nintendo 64	22	21.0	21.0	21.0
+*/
+Q: Which system has a lower number for Japan of the virtual console systems: Game Boy Advance or the Japan-only console MSX?
+A: msx (japan only)
+
+
+CREATE TABLE 2018 Warrington Wolves season (Transfers | In)(
+	row_id int,
+	player text,
+	signed from text,
+	contract length text,
+	announced text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	player	signed from	contract length	announced
+0	sitaleki akauola	penrith panthers	p2y	2017-08-01 00:00:00
+1	bryson goodwin	south sydney rabbitohs	p2y	2017-10-01 00:00:00
+2	tyrone roberts	gold coast titans	p3y	2017-10-01 00:00:00
+*/
+CREATE TABLE Images(
+	row_id int,
+	gold coast titans text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	gold coast titans
+0	a logo for the golden knights is painted on the beach.
+*/
+Q: What player was transferred from the team that has crossed swords on its logo to the Warrington Wolves in the 2018 season?
+A: tyrone roberts
+
+
+CREATE TABLE 2013 Arizona Cardinals season (Regular season)(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	result text,
+	record text,
+	game site text,
+	nfl.com recap text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	result	record	game site	nfl.com recap
+0	1	september 8	at st. louis rams	l 24–27	0–1	edward jones dome	[http://www.nfl.com/gamecenter/2013090810/2013/reg1/cardinals@rams recap]
+1	2	september 15	detroit lions	w 25–21	1–1	university of phoenix stadium	[http://www.nfl.com/gamecenter/2013091509/2013/reg2/lions@cardinals recap]
+2	3	september 22	at new orleans saints	l 7–31	1–2	mercedes-benz superdome	[http://www.nfl.com/gamecenter/2013092207/2013/reg3/cardinals@saints recap]
+*/
+CREATE TABLE Passages(
+	row_id int,
+	candlestick park text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	candlestick park
+0	candlestick park was an outdoor sports and entertainment stadium in the west coast of the united states, located in san francisco, in the bayview heights area. the stadium was originally the home of major league baseball's san francisco giants, who played there from 1960 until moving into pacific bell park (since renamed at&t park) in 2000. it was also the home field of the san francisco 49ers of the national football league from 1971 through 2013. the 49ers moved to levi's stadium in santa clara for the 2014 season.
+*/
+Q: In which year did the San Francisco 49ers move to their new stadium, which was the location that the Arizona Cardinals lost a 2013 regular season game by the score of 20 to 32?
+A: 1971
+
+
+CREATE TABLE PNC Park (Concerts)(
+	row_id int,
+	date text,
+	artist text,
+	opening act(s) text,
+	tour / concert name text,
+	attendance text,
+	revenue text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	artist	opening act(s)	tour / concert name	attendance	revenue	notes
+0	2003-08-06 00:00:00	bruce springsteen & the e street band	—	the rising tour	42301 / 48074	$3137575	none
+1	2005-06-26 00:00:00	jimmy buffett	—	a salty piece of land tour	—	—	sonny landreth and jake shimabukuro were special guests http://www.buffettworld.com/archives/2005-a-salty-piece-of-land/6-26/
+2	2005-09-28 00:00:00	the rolling stones	pearl jam	a bigger bang	—	—	none
+*/
+CREATE TABLE Passages(
+	row_id int,
+	can't stop won't stop (usher song) text,
+	uptown girl text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	can't stop won't stop (usher song)	uptown girl
+0	"can't stop won't stop" is a song recorded by american recording artist usher for his seventh studio album looking 4 myself (2012). written and produced by will "will.i.am" adams and keith harris, the song contains an interpolation of the bridge to billy joel's 1983 hit single "uptown girl". musically, "can't stop won't stop" is a eurodance and dance-pop song that incorporates elements of dubstep.	"uptown girl" is a song written and performed by american musician billy joel. it was released on 1983-9-29, on his ninth studio album an innocent man (1983). the lyrics describe a working-class "downtown man" attempting to woo a wealthy "uptown girl."
+*/
+Q: This song released on September 29, 1983 and inspired a hit song by Usher was written by who?
+A: billy joel
+
+
+CREATE TABLE 2000 DirecTV 500 (Top 10 results)(
+	row_id int,
+	pos int,
+	grid int,
+	car number (no.) int,
+	driver text,
+	team text,
+	manufacturer text,
+	laps completed (laps) int,
+	points int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	pos	grid	car number (no.)	driver	team	manufacturer	laps completed (laps)	points
+0	1	4	8	dale earnhardt, jr. (r)	dale earnhardt, inc.	chevrolet	334	185
+1	2	37	99	jeff burton	roush racing	ford	334	175
+2	3	14	18	bobby labonte	joe gibbs racing	pontiac	334	170
+*/
+CREATE TABLE Images(
+	row_id int,
+	dale earnhardt text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	dale earnhardt
+0	a man wearing a number of neckties and a mustache.
+*/
+Q: The 2000 DirecTv 500 Top 10 Driver with 146 points has a person behind them holding what?
+A: camera
+
+
+CREATE TABLE Oliver Mellor (Credits | Television)(
+	row_id int,
+	year text,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	2006	the royal	dr. guy fitzgerald	none
+1	2006	hollyoaks: in the city	detective monroe	3 episodes
+2	2006	doctor who	matt	episode "army of ghosts"
+*/
+CREATE TABLE Passages(
+	row_id int,
+	charlie stubbs (coronation street) text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	charlie stubbs (coronation street)
+0	in 2005, charlie began a relationship with tracy barlow (kate ford). he convinced her to move in with him and later in february 2006, manipulated her into having her daughter amy (amber chadwick) move in with her parents. in turn, tracy began to manipulate charlie. she pretended to be pregnant and used the money he gave her for an abortion to buy expensive shoes and used her "grief" to have him allow amy to move back in. when shelley visited before her mother’s marriage to fred elliott (john savident), she and charlie had a 1-tni stand. she told tracy about their tni of passion, who accused her of lying. shelley later revealed that she was pregnant with charlie’s baby but didn’t allow charlie to have anything to do with the baby, and left. he and tracy briefly split but reconciled. charlie later began an affair with maria sutherland (samia smith), who was renting his flat. when david platt (jack p. shepherd) discovered the affair he tried to blackmail charlie, threatening to reveal the affair to tracy. charlie retaliated by trying to drown david in the bath. when tracy eventually found out about the affair, they split once more. tracy began to plot revenge against charlie and pretended to make amends with charlie. she pretended he was abusing her to the point of burning herself with an iron to make it look like charlie was responsible for her injuries. charlie eventually realized his partner was seeking revenge and when he was about to tell her their relationship was over, she insisted on performing a lap dance for him. she hit him round the head with a heavy ornament, and he later died in hospital. she claimed she’d killed him in self-defence but the court found her guilty and she was given a life sentence.
+*/
+Q: Oliver Mellor played Dr. Matt Carter on the TV show that had Tracy Barlow kill who?
+A: charlie
+
+
+CREATE TABLE Peter Egan (Filmography)(
+	row_id int,
+	year text,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	1971	1 brief su	bill denton	none
+1	1971	elizabeth r	earl of southampton	episode: "sweet englands pride"
+2	1973	the hireling	captain hugh cantrip	none
+*/
+CREATE TABLE Passages(
+	row_id int,
+	wampanoag text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	wampanoag
+0	traditionally wampanoag people have been semi-sedentary, with seasonal movements between fixed sites in present-day southern new england. the men often traveled far north and south along the eastern seaboard for seasonal fishing expeditions, and sometimes stayed in those distant locations for weeks and months at a time. the women cultivated varieties of the "3 sisters" (the intercropping of maize, climbing beans, and squash) as the staples of their diet, supplemented by fish and game caught by the men. each community had authority over a well-defined territory from which the people derived their livelihood through a seasonal round of fishing, planting, harvesting, and hunting. because southern new england was thickly populated by indigenous peoples, hunting grounds had strictly defined boundaries.
+*/
+Q: corn beans and squash the three most important crops of the wampanoag were also known as
+A: three sisters
+
+
+CREATE TABLE 1980 in home video (Movie releases)(
+	row_id int,
+	u.s./canada release date text,
+	title text,
+	studio text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	u.s./canada release date	title	studio	notes
+0	january 1	the muppet movie	magnetic video	betamax release laserdisc release   vhs release
+1	march 4	20000 leagues under the sea	walt disney home entertainment	betamax release vhs release
+2	march 4	the apple dumpling gang	walt disney home entertainment	betamax release vhs release
+*/
+CREATE TABLE Passages(
+	row_id int,
+	bert lahr text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	bert lahr
+0	bert lahr ((1895-8-131967-12-4,p26410d)) was an american actor, particularly of stage and film, and comedian. lahr is known for his role as the cowardly lion, as well as his counterpart kansas farmworker zeke, in the wizard of oz (1939). he was well known for his explosive humor, but also adapted well to dramatic roles and his work in burlesque, vaudeville, and on broadway.
+*/
+Q: In the 1980 movie that was put out by the MGM/CBS Home Video studio, who played the part of the Cowardly Lion?
+A: bert lahr
+
+
+CREATE TABLE List of newspapers in Italy (National daily newspapers)(
+	row_id int,
+	newspaper text,
+	circulation text,
+	headquarters text,
+	est. int,
+	political alignment text,
+	nameplate text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	newspaper	circulation	headquarters	est.	political alignment	nameplate
+0	corriere della sera	242684	milan	1876	centrism	200x200px
+1	la repubblica	198835	rome	1976	social democracy	150x150px
+2	la gazzetta dello sport	161796	milan	1896	—	200x200px
+*/
+CREATE TABLE Passages(
+	row_id int,
+	early middle ages text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	early middle ages
+0	for almost p1000y, rome was the most politically important, richest and largest city in europe. around 100 ce, it had a population of about 450000, and declined to a mere 20000 during the early middle ages, reducing the sprawling city to groups of inhabited buildings interspersed among large areas of ruins and vegetation.
+*/
+CREATE TABLE Images(
+	row_id int,
+	rome text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	rome
+0	a series of photographs showing a colorful scene.
+*/
+Q: In the city that was the center of imperial life in the roman empire in the early fifth century, the building in the top right has what at its top?
+A: dome
+
+
+CREATE TABLE International League (Current teams)(
+	row_id int,
+	division text,
+	team text,
+	founded int,
+	mlb affiliation text,
+	affiliated int,
+	city text,
+	stadium text,
+	capacity int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	division	team	founded	mlb affiliation	affiliated	city	stadium	capacity
+0	north	buffalo bisons	1985	toronto blue jays	2013	buffalo, new york	sahlen field	16600
+1	north	lehigh valley ironpigs	2008	philadelphia phillies	2007	allentown, pennsylvania	coca-cola park	10100
+2	north	pawtucket red sox	1973	boston red sox	1970	pawtucket, rhode island	mccoy stadium	10031
+*/
+CREATE TABLE Images(
+	row_id int,
+	columbus clippers text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	columbus clippers
+0	a large blue and white clock on the side of a building.
+*/
+Q: Was the Team that has a ship in logo or Charlotte Knights, the one with earlier affiliation in Current teams of International League?
+A: charlotte knights
+
+
+CREATE TABLE Warren Burton (Filmography)(
+	row_id int,
+	year int,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	1976	baby blue marine	second serviceman	none
+1	1977	chatterbox	tv reporter	none
+2	1977	the world's greatest lover	ludwig	none
+*/
+CREATE TABLE Images(
+	row_id int,
+	green lantern (film) text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	green lantern (film)
+0	a picture of a green and white costume and glasses.
+*/
+Q: How many people are on the poster for Green Lantern (film)?
+A: 4
+
+
+CREATE TABLE One Hour Photo (Accolades)(
+	row_id int,
+	award text,
+	category text,
+	recipients text,
+	result real)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	award	category	recipients	result
+0	critics' choice movie awards	best actor	robin williams	nan
+1	dallas–fort worth film critics association	best actor	robin williams	nan
+2	online film critics society	best actor	robin williams	nan
+*/
+CREATE TABLE Images(
+	row_id int,
+	saturn award text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	saturn award
+0	a man in a suit and tie holding a glass.
+*/
+Q: What is he holding in Saturn Award?
+A: trophy
+
+
+CREATE TABLE 2013 Detroit Lions season (2013 Draft class)(
+	row_id int,
+	draft order - round int,
+	draft order - choice int,
+	draft order - overall int,
+	player name text,
+	position text,
+	height text,
+	weight text,
+	college text,
+	contract text,
+	notes text,
+	source text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	draft order - round	draft order - choice	draft order - overall	player name	position	height	weight	college	contract	notes	source
+0	1	5	5	ezekiel ansah	defensive end	6ft 5 in	271lbs	byu	p5y /	none	[http://www.mlive.com/lions/index.ssf/2013-4/detroit_lions_select_ezekiel_a.html detroit lions select ezekiel ansah in first round of 2013 nfl draft] mlive.com, 2013-4-26
+1	2	4	36	darius slay	defensive back	6ft 1 in	190lbs	mississippi state	p4y /	none	[http://www.mlive.com/lions/index.ssf/2013-4/detroit_lions_select_mississip.html detroit lions select mississippi state cb darius slay in second round of 2013 nfl draft] mlive.com, 2013-4-27
+2	3	3	65	larry warford	offensive lineman	6ft 3 in	343lbs	kentucky	p4y /	none	[http://www.mlive.com/lions/index.ssf/2013-4/detroit_lions_fill_massive_nee.html detroit lions fill massive need with massive guard prospect larry warford] mlive.com, 2013-4-27
+*/
+CREATE TABLE Images(
+	row_id int,
+	south carolina gamecocks football text,
+	seattle seahawks text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	south carolina gamecocks football	seattle seahawks
+0	a group of people standing next to each other.	a large green and white bird with numbers.
+*/
+Q: What educational institution has a rooster on its logo and was the school listed in the 2013 Detroit Lions draft class for the defensive end player position?
+A: south carolina
+
+
+CREATE TABLE Melia Kreiling (Filmography |  Film roles)(
+	row_id int,
+	year int,
+	title text,
+	role text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	title	role	notes
+0	2012	suspension of disbelief	juliette	none
+1	2013	company of heroes	kestrel	direct-to-video film
+2	2013	leopard	kara	none
+*/
+CREATE TABLE Passages(
+	row_id int,
+	list of marvel cinematic universe films text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	list of marvel cinematic universe films
+0	the first film in the marvel cinematic universe was iron man (2008), which was distributed by paramount pictures. paramount also distributed iron man 2 (2010), thor (2011) and captain america: the first avenger (2011), while universal pictures distributed the incredible hulk (2008). walt disney studios motion pictures began distributing the films with the 2012 crossover film the avengers, which concluded phase 1 of the franchise. phase 2 includes iron man 3 (2013), thor: the dark world (2013), captain america: wi soldier (2014), guardians of the galaxy (2014), avengers: age of ultron (2015), and ant-man (2015).
+*/
+Q: What was Melia Kreiling's role in the film that is the next Marvel movie after 'Captain America the Winter Soldier'?
+A: bereet
+
+
+CREATE TABLE 2006 Grand Prix of Portland (Qualifying results)(
+	row_id int,
+	pos int,
+	nat real,
+	name text,
+	team text,
+	qual 1 text,
+	qual 2 text,
+	best text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	pos	nat	name	team	qual 1	qual 2	best
+0	1	nan	bruno junqueira	newman/haas racing	59.576	57.631	57.631
+1	2	nan	a. j. allmendinger	forsythe racing	58.378	57.639	57.639
+2	3	nan	sébastien bourdais	newman/haas racing	58.464	57.646	57.646
+*/
+CREATE TABLE Passages(
+	row_id int,
+	jtg daugherty racing text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	jtg daugherty racing
+0	jtg daugherty racing (formerly st motorsports and jtg racing) is an american professional stock car racing team that currently competes in the monster energy nascar cup series. the team is owned by former advertising executive tad geschickter and his wife jodi, along with current espn analyst brad daugherty. the team formerly had alliances with wood brothers racing, then michael waltrip racing, and currently has a technical alliance with richard childress racing. the team currently fields the no. 37 cottonelle chevrolet ss driven by roush development driver chris buescher and the no. 47 clorox/bush's/scott products chevrolet ss driven by a. j. allmendinger in the monster energy nascar cup series.
+*/
+Q: The driver of Nascar number 47 qualified for the 2006 Grand Prix of Portland for which team?
+A: forsythe racing
+
+
+CREATE TABLE List of churches in Copenhagen ([[Amager]])(
+	row_id int,
+	name text,
+	denomination text,
+	year int,
+	coordinates real,
+	image text,
+	refs real)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	name	denomination	year	coordinates	image	refs
+0	all saints' church	church of denmark	1932	nan	150px	nan
+1	dragør church	church of denmark	1885	nan	150px	nan
+2	hans tausen's church	church of denmark	1924	nan	150px	nan
+*/
+CREATE TABLE Images(
+	row_id int,
+	all saints' church, copenhagen text,
+	dragør church text,
+	nathanael's church text,
+	st. anne's church, copenhagen text,
+	sundby church text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	all saints' church, copenhagen	dragør church	nathanael's church	st. anne's church, copenhagen	sundby church
+0	 type of place of worship	 church of the holy trinity	 church of the holy trinity	 the building where the hotel is located	 a red brick church with a steeple and a flagpole in front of it.
+*/
+Q: Among Copenhagen churches on the "Amager" list, which have spires and are affiliated with the Church of Denmark denomination?
+A: all saints' church | nathanael's church | dragør church | sundby church
+
+
+CREATE TABLE Final Straw Tour (UK Tour (Leg III))(
+	row_id int,
+	date text,
+	city text,
+	country text,
+	venue text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	city	country	venue
+0	support acts: terra diablo & astrid	support acts: terra diablo & astrid	support acts: terra diablo & astrid	support acts: terra diablo & astrid
+1	2004-3-2	newcastle	england	newcastle university
+2	2004-3-3	liverpool	england	carling academy
+*/
+CREATE TABLE Images(
+	row_id int,
+	oxford text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	oxford
+0	 a guide to the city of edinburgh
+*/
+Q: The final straw tour held leg 3 of the UK tour on March 13, 2004 in this city with how many views on the bottom?
+A: three

templates/prompts/prompt_qa_balanced.txt

@@ -0,0 +1,275 @@
+Generate answer given the question and table to answer the question correctly.
+
+CREATE TABLE Fabrice Santoro(
+	row_id int,
+	name text,
+	1989 text,
+	1990 text,
+	1991 text,
+	1992 text,
+	1993 text,
+	1994 text,
+	1995 text,
+	1996 text,
+	1997 text,
+	1998 text,
+	1999 text,
+	2000 text,
+	2001 text,
+	2002 text,
+	2003 text,
+	2004 text,
+	2005 text,
+	2006 text,
+	2007 text,
+	2008 text,
+	2009 text,
+	2010 text,
+	career\nsr text,
+	career\nwin-loss text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	name	1989	1990	1991	1992	1993	1994	1995	1996	1997	1998	1999	2000	2001	2002	2003	2004	2005	2006	2007	2008	2009	2010	career\nsr	career\nwin-loss
+0	australian open	a	a	1r	a	2r	3r	2r	1r	a	3r	4r	1r	2r	1r	3r	2r	1r	qf	3r	2r	3r	1r	0 / 18	22–18
+1	french open	1r	2r	4r	1r	1r	3r	1r	a	1r	3r	1r	2r	4r	2r	2r	3r	1r	1r	1r	2r	1r	a	0 / 20	17–20
+2	wimbledon	a	1r	a	a	a	a	1r	a	1r	a	2r	2r	3r	2r	2r	2r	2r	2r	2r	1r	2r	a	0 / 14	11–14
+3	us open	a	3r	1r	2r	1r	a	1r	a	1r	3r	3r	1r	2r	1r	2r	3r	2r	1r	2r	1r	1r	a	0 / 18	13–18
+4	grand slam sr	0 / 1	0 / 3	0 / 3	0 / 2	0 / 3	0 / 2	0 / 4	0 / 1	0 / 3	0 / 3	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 4	0 / 1	0 / 70	n/a
+5	grand slam win-loss	0–1	3–3	3–3	1–2	1–3	4–2	1–4	0–1	0–3	6–3	6–4	2–4	7–4	2–4	5–4	6–4	2–4	5–4	4–4	2–4	3–4	0–1	n/a	63–70
+6	indian wells	nme	a	3r	1r	qf	3r	2r	a	a	1r	a	3r	2r	3r	1r	1r	4r	1r	a	a	a	a	0 / 13	16–13
+7	miami	nme	2r	2r	1r	3r	a	a	a	a	4r	3r	2r	4r	2r	a	1r	a	2r	3r	3r	2r	a	0 / 14	15–14
+8	monte carlo	nme	1r	2r	2r	1r	a	3r	3r	sf	qf	a	2r	1r	1r	1r	3r	2r	1r	a	1r	a	a	0 / 16	17–16
+9	rome	nme	a	qf	3r	3r	a	3r	a	2r	1r	3r	3r	2r	1r	1r	a	2r	3r	a	1r	a	a	0 / 14	18–14
+10	hamburg	nme	2r	a	a	a	a	1r	a	a	qf	2r	1r	qf	1r	1r	a	1r	1r	a	a	nme	nme	0 / 10	8–10
+11	canada	nme	a	a	a	a	a	a	a	qf	2r	qf	1r	sf	qf	1r	qf	1r	1r	a	a	a	a	0 / 10	17–10
+12	cincinnati	nme	a	a	a	a	a	a	a	2r	1r	2r	qf	2r	1r	2r	qf	2r	1r	a	a	a	a	0 / 10	11–10
+13	stuttgart/madrid	nme	a	a	a	a	a	a	a	3r	1r	2r	2r	1r	sf	a	a	a	a	1r	a	a	a	0 / 7	8–7
+14	paris	nme	1r	1r	1r	a	a	a	a	2r	2r	2r	qf	2r	1r	2r	a	2r	1r	3r	a	1r	a	0 / 14	10–14
+15	masters series sr	n/a	0 / 4	0 / 5	0 / 5	0 / 4	0 / 1	0 / 4	0 / 1	0 / 6	0 / 9	0 / 7	0 / 9	0 / 9	0 / 9	0 / 7	0 / 5	0 / 7	0 / 8	0 / 3	0 / 3	0 / 2	0 / 0	0 / 108	n/a
+16	p1y win-loss	n/a	2–4	7–5	3–5	6–4	2–1	5–4	2–1	12–6	10–9	10–7	12–9	13–9	9–9	2–7	8–5	7–7	3–8	4–3	2–3	1–2	0–0	n/a	120–108
+17	year end ranking	235	62	43	43	55	46	102	118	29	41	34	31	22	35	62	52	58	52	37	52	68	–	n/a	none
+*/
+Q: did he win more at the australian open or indian wells?
+A: australian open
+
+
+CREATE TABLE Matthew Morrison(
+	row_id int,
+	year int,
+	title text,
+	role text,
+	notes text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	year	title	role	notes
+0	1999	bob rizzo's simply funk with suzanne	[himself]	as matthew j. morrison
+1	2003	marci x	boyz r us	as matthew j. morrison
+2	2005	once upon a mattress	sir harry	none
+3	2006	blinders	scott	none
+4	2007	music and lyrics	ray	none
+5	2007	dan in real life	policeman	none
+6	2007	i think i love my wife	salesman #2	none
+7	2011	the muppets	mahna mahna host	none
+8	2012	what to expect when you're expecting	evan	none
+*/
+Q: what movies was morrison involved with in 2007?
+A: music and lyrics, dan in real life, i think i love my wife
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result/score text,
+	record text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	week	date	opponent	time	game site	tv	result/score	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	rca dome	nbc	l 41 – 10	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	raymond james stadium	fox	l 31 – 14	0–2
+2	3	2007-9-24	tennessee titans	t20:30 edt	louisiana superdome	espn	l 31 – 14	0–3
+3	4	bye	bye	bye	bye	bye	bye	none
+4	5	2007-10-7	carolina panthers	t13:0 edt	louisiana superdome	fox	l 16 – 13	0–4
+5	6	2007-10-14	seattle seahawks	t20:15 edt	qwest field	nbc	w 28 – 17	1–4
+6	7	2007-10-21	atlanta falcons	t13:0 edt	louisiana superdome	fox	w 22 – 16	2–4
+7	8	2007-10-28	san francisco 49ers	t16:15 edt	monster park	fox	w 31 – 10	3–4
+8	9	2007-11-4	jacksonville jaguars	t13:0 est	louisiana superdome	cbs	w 41 – 24	4–4
+9	10	2007-11-11	st. louis rams	t13:0 est	louisiana superdome	fox	l 37 – 29	4–5
+10	11	2007-11-18	houston texans	t13:0 est	reliant stadium	fox	l 23 – 10	4–6
+11	12	2007-11-25	carolina panthers	t13:0 est	bank of america stadium	fox	w 31 – 6	5–6
+12	13	2007-12-2	tampa bay buccaneers	t13:0 est	louisiana superdome	fox	l 27 – 23	5–7
+13	14	2007-12-10	atlanta falcons	t20:30 est	georgia dome	espn	w 34 – 14	6–7
+14	15	2007-12-16	arizona cardinals	t13:0 est	louisiana superdome	fox	w 31–24	7–7
+15	16	2007-12-23	philadelphia eagles	t13:0 est	louisiana superdome	fox	l 38–23	7–8
+16	17	2007-12-30	chicago bears	t13:0 est	soldier field	fox	l 33–25	7–9
+*/
+Q: what number of games were lost at home?
+A: 5
+
+
+CREATE TABLE Demographics of Alaska(
+	row_id int,
+	by race text,
+	white text,
+	black text,
+	aian* text,
+	asian text,
+	nhpi* text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	by race	white	black	aian*	asian	nhpi*
+0	2000 (total population)	75.43%	4.46%	19.06%	5.24%	0.88%
+1	2000 (hispanic only)	3.42%	0.33%	0.45%	0.16%	0.06%
+2	2005 (total population)	74.71%	4.72%	18.77%	5.9%	0.88%
+3	2005 (hispanic only)	4.32%	0.38%	0.48%	0.19%	0.05%
+4	growth 2000–5 (total population)	4.85%	12.03%	4.27%	19.23%	5.35%
+5	growth 2000–5 (non-hispanic only)	3.49%	11.3%	4.02%	18.96%	5.86%
+6	growth 2000–5 (hispanic only)	33.56%	21.02%	14.52%	27.89%	-1.95%
+*/
+Q: which hispanic population had the greatest growth from 2000 to 2005?
+A: white
+
+
+CREATE TABLE Highest mountain peaks of California(
+	row_id int,
+	rank int,
+	mountain peak text,
+	mountain range text,
+	elevation text,
+	prominence text,
+	isolation text,
+	location text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	rank	mountain peak	mountain range	elevation	prominence	isolation	location
+0	1	mount whitney	sierra nevada	14505 ft; 4421 m	10080 ft; 3072 m	1646 mi; 2649 km	36°34′43″n 118°17′31″w / 36.5786°n 118.292°w
+1	2	mount williamson	sierra nevada	14379 ft; 4383 m	1677 ft; 511 m	5.4 mi; 8.7 km	36°39′21″n 118°18′40″w / 36.6559°n 118.3111°w
+2	3	white mountain peak	white mountains	14252 ft; 4344 m	7196 ft; 2193 m	67 mi; 109 km	37°38′3″n 118°15′21″w / 37.6341°n 118.2557°w
+3	4	north palisade	sierra nevada	14248 ft; 4343 m	2894 ft; 882 m	32 mi; 52 km	37°5′39″n 118°30′52″w / 37.0943°n 118.5145°w
+4	5	mount shasta	cascade range	14179 ft; 4322 m	9832 ft; 2997 m	335 mi; 539 km	41°24′33″n 122°11′42″w / 41.4092°n 122.1949°w
+5	6	mount humphreys	sierra nevada	13992 ft; 4265 m	2563 ft; 781 m	15 mi; 24 km	37°16′14″n 118°40′23″w / 37.2705°n 118.673°w
+6	7	mount keith	sierra nevada	13982 ft; 4262 m	1936 ft; 590 m	3.1 mi; 5 km	36°42′0″n 118°20′37″w / 36.7001°n 118.3436°w
+7	8	mount darwin	sierra nevada	13837 ft; 4218 m	1891 ft; 576 m	7 mi; 11 km	37°10′1″n 118°40′20″w / 37.1669°n 118.6721°w
+8	9	mount kaweah	sierra nevada	13807 ft; 4209 m	2027 ft; 618 m	11 mi; 17 km	36°31′34″n 118°28′43″w / 36.5261°n 118.4785°w
+9	10	mount morgan	sierra nevada	13758 ft; 4193 m	2648 ft; 807 m	10 mi; 16 km	37°24′19″n 118°43′58″w / 37.4053°n 118.7329°w
+10	11	mount gabb	sierra nevada	13747 ft; 4190 m	2601 ft; 793 m	4.3 mi; 6.9 km	37°22′37″n 118°48′9″w / 37.3769°n 118.8025°w
+11	12	mount tom	sierra nevada	13657 ft; 4163 m	1992 ft; 607 m	4.8 mi; 7.7 km	37°22′34″n 119°10′44″w / 37.3762°n 119.1789°w
+12	13	mount dubois	white mountains	13565 ft; 4135 m	2339 ft; 713 m	10 mi; 16 km	37°47′0″n 118°20′36″w / 37.7834°n 118.3432°w
+13	14	mount pinchot	sierra nevada	13500 ft; 4115 m	2110 ft; 643 m	4.7 mi; 7.6 km	36°56′50″n 118°24′19″w / 36.9473°n 118.4054°w
+14	15	red slate mountain	sierra nevada	13162 ft; 4012 m	1736 ft; 529 m	8 mi; 13 km	37°30′27″n 118°52′9″w / 37.5075°n 118.8693°w
+15	16	mount ritter	sierra nevada	13149 ft; 4008 m	3990 ft; 1216 m	22 mi; 35 km	37°41′21″n 119°11′59″w / 37.6891°n 119.1996°w
+*/
+Q: which mountain peak has a prominence more than 10,000 ft?
+A: mount whitney
+
+
+CREATE TABLE Daegu FC(
+	row_id int,
+	season int,
+	division int,
+	tms. int,
+	pos. int,
+	fa cup text,
+	afc cl real)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	season	division	tms.	pos.	fa cup	afc cl
+0	2003	1	12	11	quarter final	nan
+1	2004	1	13	10	round of 32	nan
+2	2005	1	13	8	quarter final	nan
+3	2006	1	14	7	quarter final	nan
+4	2007	1	14	12	round of 16	nan
+5	2008	1	14	11	semi final	nan
+6	2009	1	15	15	quarter-final	nan
+7	2010	1	15	15	round of 32	nan
+8	2011	1	16	12	round of 32	nan
+9	2012	1	16	10	round of 16	nan
+*/
+Q: how far did they make it in the fa cup after 2009?
+A: round of 16
+
+CREATE TABLE Portugal in the Eurovision Song Contest 1979(
+	row_id int,
+	draw int,
+	artist text,
+	song text,
+	points int,
+	place text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	draw	artist	song	points	place
+0	1	gonzaga coutinho	"tema para um homem só"	102	5th
+1	2	pedro osório s.a.r.l.	"uma canção comercial"	123	3rd
+2	3	concha	"qualquer dia, quem diria"	78	6th
+3	4	gabriela schaaf	"eu só quero"	132	2nd
+4	5	tózé brito	"novo canto português"	110	4th
+5	6	teresa silva carvalho	"cantemos até ser dia"	52	9th
+6	7	florência	"o combóio do tua"	63	8th
+7	8	manuel josé soares	"quando chego a casa"	76	7th
+8	9	manuela bravo	"sobe, sobe, balão sobe"	149	1st
+*/
+Q: who was the last draw?
+A: manuela bravo
+
+
+CREATE TABLE List of spans(
+	row_id int,
+	tramway text,
+	country text,
+	city text,
+	height of pylons text,
+	span width,\nleaning straight line text,
+	span width,\nhorizontal measurement text,
+	height of cable over ground text,
+	year of inauguration text,
+	notes text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	tramway	country	city	height of pylons	span width,\nleaning straight line	span width,\nhorizontal measurement	height of cable over ground	year of inauguration	notes
+0	peak 2 peak gondola	canada	whistler	65m	3024 m	3019 m	436 m	2008	3s aerial tramway constructed by doppelmayr
+1	hut of regensburg material transport aerial railway	austria	falbeson	?	?	?	430 m	?	none
+2	vanoise express	france	vanoise	none	1850 m	1800 m	380 m	2003	none
+3	aiguille du midi	france	chamonix	none	2867 m	2500 m	?	1955	2nd section
+4	vallee blanche aerial tramway	france	mont blanc	none	2831 m, 1684 m	span is almost horizontal	appr. 300 m	1958	rock anchored support structure
+5	3s aerial tramway	austria	kitzbühel	0 m, 80m	2507 m	?	400 m	2004	none
+6	sandia peak tramway	usa	albuquerque	70.7 m, 21.33 m	2353 m	?	274 m	1966	none
+7	feldmoos-chli-titlis aerial tramway	switzerland	titlis	37.6 m	3476.2 m	?	?	1979	temp. site tramway, demolished in 1986
+*/
+Q: was the sandia peak tramway innagurate before or after the 3s aerial tramway?
+A: before
+
+
+CREATE TABLE Płock Governorate(
+	row_id int,
+	language text,
+	number int,
+	percentage (%) text,
+	males int,
+	females int)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	language	number	percentage (%)	males	females
+0	polish	447685	80.86	216794	230891
+1	yiddish	51215	9.25	24538	26677
+2	german	35931	6.49	17409	18522
+3	russian	15137	2.73	13551	1586
+4	ukrainian	2350	0.42	2302	48
+5	other	1285	0.23	1041	244
+6	persons; that didn't name; their native language	27	>0.01	14	13
+7	total	553633	100	275652	277981
+*/
+Q: how many male and female german speakers are there?
+A: 35931

templates/prompts/prompt_qa_balanced_no_table_input.txt

@@ -0,0 +1,36 @@
+Generate answer to answer the question correctly.
+
+Q: did he win more at the australian open or indian wells?
+A: australian open
+
+
+Q: what movies was morrison involved with in 2007?
+A: music and lyrics, dan in real life, i think i love my wife
+
+
+Q: what number of games were lost at home?
+A: 5
+
+
+Q: which hispanic population had the greatest growth from 2000 to 2005?
+A: white
+
+
+Q: which mountain peak has a prominence more than 10,000 ft?
+A: mount whitney
+
+
+Q: how far did they make it in the fa cup after 2009?
+A: round of 16
+
+
+Q: who was the last draw?
+A: manuela bravo
+
+
+Q: was the sandia peak tramway innagurate before or after the 3s aerial tramway?
+A: before
+
+
+Q: how many male and female german speakers are there?
+A: 35931

templates/prompts/prompt_tab_fact_puresql_v2.txt

@@ -0,0 +1,277 @@
+Generate SQL given the statement and table to verify the statement correctly.
+
+CREATE TABLE turkish cup(
+	row_id int,
+	round text,
+	clubs remaining int,
+	clubs involved int,
+	winners from previous round real,
+	new entries this round real,
+	leagues entering at this round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	round	clubs remaining	clubs involved	winners from previous round	new entries this round	leagues entering at this round
+0	first round	156	86	nan	86.0	tff third league & turkish regional amateur league
+1	second round	113	108	43.0	65.0	süper lig & tff first league & tff second league
+2	third round	59	54	54.0	nan	none
+*/
+Q: during the 3rd round of the turkish cup , there be no new entry during that stage
+SQL: SELECT (SELECT `new entries this round` FROM w WHERE round = 'third round') IS NULL
+
+
+CREATE TABLE turkish cup(
+	row_id int,
+	round text,
+	clubs remaining int,
+	clubs involved int,
+	winners from previous round real,
+	new entries this round real,
+	leagues entering at this round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	round	clubs remaining	clubs involved	winners from previous round	new entries this round	leagues entering at this round
+0	first round	156	86	nan	86.0	tff third league
+1	second round	113	108	43.0	65.0	süper ligs
+2	third round	59	54	54.0	nan	none
+*/
+Q: süper lig be the league to win a round in the turkish cup with 110 clubs
+SQL: SELECT (SELECT clubs FROM w WHERE `leagues entering at this round` = 'süper ligs') = 110
+
+
+CREATE TABLE turkish cup(
+	row_id int,
+	round text,
+	clubs remaining int,
+	clubs involved int,
+	winners from previous round real,
+	new entries this round real,
+	leagues entering at this round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	round	clubs remaining	clubs involved	winners from previous round	new entries this round	leagues entering at this round
+0	first round	156	86	nan	86.0	tff third league & turkish regional amateur league
+1	second round	113	108	43.0	65.0	süper lig & tff first league & tff second league
+2	third round	59	54	54.0	nan	none
+*/
+Q: the lowest number of new entry conclude a round in the turkish cup be 5
+SQL: SELECT (SELECT MIN(`new entries this round`) FROM w) = 5
+
+
+CREATE TABLE cultural interest fraternities and sororities(
+	row_id int,
+	letters text,
+	organization text,
+	nickname text,
+	founding time text,
+	founding university text,
+	type text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	letters	organization	nickname	founding time	founding university	type
+0	αεπ	alpha epsilon pi 1	aepi	1913-11-07 00:00:00	new york university	fraternity
+1	αεφ	alpha epsilon phi 2	aephi	1909-10-24 00:00:00	barnard college	sorority
+2	σαεπ	sigma alpha epsilon pi 3	sigma	1998-10-01 00:00:00	university of california , davis	sorority
+*/
+Q: 4 of the cultural interest fraternity and sorority be fraternity while 3 be a sorority
+SQL: SELECT (SELECT (SELECT COUNT(*) FROM w WHERE type = 'fraternity') = 4) AND (SELECT (SELECT COUNT(*) FROM w WHERE type = 'sorority') = 3)
+
+
+CREATE TABLE british records in athletics(
+	row_id int,
+	event text,
+	data text,
+	athlete text,
+	date text,
+	place text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	event	data	athlete	date	place
+0	5 km	t19:29	andi drake	1990-05-27 00:00:00	norway
+1	5 miles	32:38 +	ian mccombie	1985-03-23 00:00:00	united kingdom
+2	10 km	40:17	chris maddocks	1989-04-30 00:00:00	united kingdom
+*/
+Q: there be 8 different event that take place within the united kingdom
+SQL: SELECT (SELECT COUNT(place) FROM w WHERE place = 'united kingdom') = 8
+
+
+CREATE TABLE jeev milkha singh(
+	row_id int,
+	tournament text,
+	wins int,
+	top - 10 int,
+	top - 25 int,
+	events int,
+	cuts made int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	tournament	wins	top - 10	top - 25	events	cuts made
+0	masters tournament	0	0	1	3	2
+1	us open	0	0	0	4	3
+2	the open championship	0	0	0	2	1
+*/
+Q: the number of cut made in the pga championship tournament be smaller than the number of event
+SQL: SELECT (SELECT `cuts made` FROM w WHERE tournament = 'pga championship') < (SELECT events FROM w WHERE tournament = 'pga championship')
+
+
+CREATE TABLE 2008 women 's british open(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score int,
+	to par int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par
+0	1	juli inkster	united states	65	7
+1	t2	momoko ueda	japan	66	6
+2	t2	laura diaz	united states	66	6
+*/
+Q: the 3 player from japan have the same score
+SQL: SELECT (SELECT COUNT(DISTINCT score) FROM w WHERE country = 'japan' GROUP BY score) = 1
+
+
+CREATE TABLE espn sunday night football results (1987 - 2005)(
+	row_id int,
+	date text,
+	visiting team text,
+	final score text,
+	host team text,
+	stadium text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	visiting team	final score	host team	stadium
+0	new year eve	indianapolis colts	24 - 7	baltimore ravens	m&t bank stadium
+1	new year eve	kansas city chiefs	23 - 17	oakland raiders	mcafee coliseum
+2	new year's day	new york giants	23 - 45	san diego chargers	qualcomm stadium
+*/
+Q: the hosting team be the new york giant on new year even and the st louis ram on new year 's day
+SQL: SELECT (SELECT (SELECT `host team` FROM w WHERE date = 'new year eve') = 'new york giant') AND (SELECT (SELECT `host team` FROM w WHERE date = 'new year's day') = 'st louis ram')
+
+
+CREATE TABLE 2008 women 's british open(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score text,
+	to par int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par
+0	t1	yuri fudoh	japan	134	10
+1	t1	jiyai shin	south korea	134	10
+2	3	juli inkster	united states 135	9
+*/
+Q: kristie kerr , tie for 4th place , finish the round 1 stroke under lorena ochoa of mexico
+SQL: SELECT (SELECT (SELECT score FROM w WHERE player = 'cristie kerr') < (SELECT score FROM w WHERE player = 'lorena ochoa' AND country = 'mexico')) AND (SELECT (SELECT place FROM w WHERE player = 'cristie kerr') = "t4")
+
+
+CREATE TABLE connecticut public radio(
+	row_id int,
+	call sign text,
+	frequency text,
+	city of license text,
+	facility id int,
+	erp / power w int,
+	height m ( ft ) real,
+	class text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	call sign	frequency	city of license	facility id	erp / power w	height m ( ft )	class
+0	waic	91.9 fm	springfield , ma	1749	230	nan	b1
+1	wedw - fm	88.5 fm	stamford , ct	13619	2000	nan	a
+2	wnpr	90.5 fm ( hd ) connecticut public radio	meriden , ct	13627	18500	nan	b
+*/
+Q: there be 3 station with a call sign number in the 90s
+SQL: SELECT (SELECT COUNT(*) FROM w WHERE frequency > 90 GROUP BY `call sign`) = 3
+
+
+CREATE TABLE 2003 chicago white sox season(
+	row_id int,
+	date text,
+	opponent text,
+	score text,
+	loss text,
+	time text,
+	att int,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	opponent	score	loss	time	att	record
+0	august 1	mariners	12 - 1	garcía (9 - 11)	2:52	39337	58 - 51
+1	august 2	mariners	0 - 10	wright (0 - 5)	2:22	45719	58 - 52
+2	august 3	mariners	2 - 8	buehrle (9 - 11)	2:57	45632	58 - 53
+*/
+Q: the 2003 chicago white sox game play on 26th august be longer than the game play on 24th august
+SQL: SELECT (SELECT time FROM w WHERE date = 'august 26') > (SELECT time FROM w WHERE date = 'august 24')
+
+
+CREATE TABLE 1987 masters tournament(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score text,
+	to par text,
+	money text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par	money
+0	t1	larry mize	united states	70 + 72 + 72 + 71 = 285	-3	playoff
+1	t1	bernhard langer	spain	73 + 71 + 70 + 71 = 285	-3	playoff
+2	t1	greg norman	australia	73 + 74 + 66 + 72 = 285	-3	playoff
+*/
+Q: bernhard m. langer have more point than roger maltbie during the 1987 master tournament
+SQL: SELECT (SELECT score FROM w WHERE player = 'bernhard langer') > (SELECT score FROM w WHERE player = 'roger maltbie')
+
+
+CREATE TABLE 1987 masters tournament(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score text,
+	to par text,
+	money text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par	money
+0	t1	larry mize	united states	70 + 72 + 72 + 71 = 285	-3	playoff
+1	t1	seve ballesteros	spain	73 + 71 + 70 + 71 = 285	-3	playoff
+2	t1	greg norman	australia	73 + 74 + 66 + 72 = 285	-3	playoff
+*/
+Q: most of the people who play for the 1987 master tournament be spanish
+SQL: SELECT (SELECT(SELECT COUNT(*) FROM w WHERE country = 'spain') / (SELECT COUNT(*) FROM w)) > 0.5
+
+
+CREATE TABLE 1976 world junior figure skating championships(
+	row_id int,
+	rank int,
+	name text,
+	nation text,
+	points real,
+	places int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	rank	name	nation	points	places
+0	1	sherri baier / robin cowan	canada	128.39	9
+1	2	lorene mitchell / donald mitchell	united states	124.94	16
+2	3	elizabeth cain / peter cain	australia	116.67	33
+*/
+Q: 2 of the 7 top - ranked figure skate team be from france
+SQL: SELECT (SELECT (SELECT COUNT(*) FROM w) = 7) AND (SELECT (SELECT COUNT(*) FROM w WHERE nation = 'france') = 2)

templates/prompts/prompt_tab_fact_sqllike_v3.txt

@@ -0,0 +1,358 @@
+Generate SQL given the statement and table to verify the statement correctly.
+If statement-relevant column(s) contents are not suitable for SQL comparisons or calculations, map it to a new column with clean content by a new grammar QA("map@").
+If mapping to a new column still can not answer the statement with valid SQL, turn to an end-to-end solution by a new grammar QA("ans@"). This grammar aims to solve all the rest of complex statements or tables.
+
+CREATE TABLE jason chambers(
+	row_id int,
+	res text,
+	record text,
+	opponent text,
+	method text,
+	event text,
+	round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	res	record	opponent	method	event	round
+0	win	18 - 5 - 2	dan new	submission (rear naked choke)	tfc - power fights	1
+1	win	17 - 5 - 2	rene gonzalez	decision (split)	mainstream mma - cold war	n / a
+2	loss	16 - 5 - 2	tristan yunker	submission ( armbar )	tfc 7 - total fight challenge 7	1
+*/
+Q: in mac - midwest absolute challenge , the player be defeat by dan spychalski in 1 round
+NeuralSQL: SELECT (SELECT opponent, round FROM w WHERE event = "mac - midwest absolute challenge")=("dan spychalski", 1)
+
+
+CREATE TABLE 1943 vfl season(
+	row_id int,
+	home team text,
+	home team score text,
+	away team text,
+	away team score text,
+	venue text,
+	crowd int,
+	date text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	home team	home team score	away team	away team score	venue	crowd	date
+0	footscray	10.11 (71)	south melbourne	6.14 (50)	western oval	7500	1943-06-26 00:00:00
+1	collingwood	10.21 (81)	melbourne	13.9 (87)	victoria park	5000	1943-06-26 00:00:00
+2	carlton	15.16 (106)	fitzroy	9.13 (67)	princes park	12000	1943-06-26 00:00:00
+*/
+Q: western oval be the venue when the home team footscray score 10.11 (71)
+NeuralSQL: SELECT (SELECT venue FROM w WHERE `home team`="footscray" AND `home team score`="10.11 (71)") = "western oval"
+
+
+CREATE TABLE 2005 pba draft(
+	row_id int,
+	pick int,
+	player text,
+	country of origin text,
+	pba team text,
+	college text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	pick	player	country of origin	pba team	college
+0	1	jay washington	united states	air21 express	eckerd
+1	2	alex cabagnot	united states	sta lucia realtors	hawaii - hilo
+2	3	dennis miranda	philippines	coca - cola tigers	feu
+*/
+Q: leo najorda be from philippine
+NeuralSQL: SELECT (SELECT `country of origin` FROM w WHERE player = "leo najorda")="philippines"
+
+
+CREATE TABLE none(
+	row_id int,
+	event text,
+	long course / short course text,
+	year set int,
+	time text,
+	meet text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	event	long course / short course	year set	time	meet
+0	100 m freestyle	long course	2007	54.08	2007 fina world aquatic championships
+1	200 m individual medley	long course	2011	2:11.23	2011 canadian world championship trials
+2	4 x 100 m medley relay	long course	2010	3:38.14	2010 pan pacific championships
+*/
+Q: in 2009 the record be set in 7:51:80
+NeuralSQL: SELECT (SELECT  time FROM w WHERE `year set` = 2009)="7:51.8"
+
+
+CREATE TABLE turkish cup(
+	row_id int,
+	round text,
+	clubs remaining int,
+	clubs involved int,
+	winners from previous round real,
+	new entries this round real,
+	leagues entering at this round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	round	clubs remaining	clubs involved	winners from previous round	new entries this round	leagues entering at this round
+0	first round	156	86	nan	86.0	tff third league & turkish regional amateur league
+1	second round	113	108	43.0	65.0	süper lig & tff first league & tff second league
+2	third round	59	54	54.0	nan	none
+*/
+Q: during the 3rd round of the turkish cup , there be no new entry during that stage
+NeuralSQL: SELECT (SELECT `new entries this round` FROM w WHERE round = 'third round') IS NULL
+
+
+CREATE TABLE turkish cup(
+	row_id int,
+	round text,
+	clubs remaining int,
+	clubs involved int,
+	winners from previous round real,
+	new entries this round real,
+	leagues entering at this round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	round	clubs remaining	clubs involved	winners from previous round	new entries this round	leagues entering at this round
+0	first round	156	86	nan	86.0	tff third league & turkish regional amateur league
+1	second round	113	108	43.0	65.0	süper ligs & tff first league & tff second league
+2	third round	59	54	54.0	nan	none
+*/
+Q: süper lig be the most common league to win a round in the turkish cup
+NeuralSQL: SELECT QA("ans@what is the most common league?"; `leagues entering at this round`) = 'süper ligs'
+
+
+CREATE TABLE turkish cup(
+	row_id int,
+	round text,
+	clubs remaining int,
+	clubs involved int,
+	winners from previous round real,
+	new entries this round real,
+	leagues entering at this round text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	round	clubs remaining	clubs involved	winners from previous round	new entries this round	leagues entering at this round
+0	first round	156	86	nan	86.0	tff third league & turkish regional amateur league
+1	second round	113	108	43.0	65.0	süper lig & tff first league & tff second league
+2	third round	59	54	54.0	nan	none
+*/
+Q: the lowest number of new entry conclude a round in the turkish cup be 5
+NeuralSQL: SELECT (SELECT MIN(`new entries this round`) FROM w) = 5
+
+
+CREATE TABLE cultural interest fraternities and sororities(
+	row_id int,
+	letters text,
+	organization text,
+	nickname text,
+	founding time text,
+	founding university text,
+	type text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	letters	organization	nickname	founding time	founding university	type
+0	αεπ	alpha epsilon pi 1	aepi	1913-11-07 00:00:00	new york university	fraternity
+1	αεφ	alpha epsilon phi 2	aephi	1909-10-24 00:00:00	barnard college	sorority
+2	σαεπ	sigma alpha epsilon pi 3	sigma	1998-10-01 00:00:00	university of california , davis	sorority
+*/
+Q: 4 of the cultural interest fraternity and sorority be fraternity while 3 be a sorority
+NeuralSQL: SELECT (SELECT (SELECT COUNT(*) FROM w WHERE type = 'fraternity') = 4) AND (SELECT (SELECT COUNT(*) FROM w WHERE type = 'sorority') = 3)
+
+
+CREATE TABLE british records in athletics(
+	row_id int,
+	event text,
+	data text,
+	athlete text,
+	date text,
+	place text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	event	data	athlete	date	place
+0	5 km	t19:29	andi drake	1990-05-27 00:00:00	søfteland , norway
+1	5 miles	32:38 +	ian mccombie	1985-03-23 00:00:00	york , united kingdom
+2	10 km	40:17	chris maddocks	1989-04-30 00:00:00	burrator , united kingdom
+*/
+Q: there be 8 different event that take place within the united kingdom
+NeuralSQL: SELECT (SELECT COUNT(place) FROM w WHERE QA("map@is it in united kingdom?"; place) = 'yes') = 8
+
+
+CREATE TABLE jeev milkha singh(
+	row_id int,
+	tournament text,
+	wins int,
+	top - 10 int,
+	top - 25 int,
+	events int,
+	cuts made int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	tournament	wins	top - 10	top - 25	events	cuts made
+0	masters tournament	0	0	1	3	2
+1	us open	0	0	0	4	3
+2	the open championship	0	0	0	2	1
+*/
+Q: the number of cut made in the pga championship tournament be smaller than the number of event
+NeuralSQL: SELECT (SELECT `cuts made` FROM w WHERE tournament = 'pga championship') < (SELECT events FROM w WHERE tournament = 'pga championship')
+
+
+CREATE TABLE 2008 women 's british open(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score int,
+	to par int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par
+0	1	juli inkster	united states	65	7
+1	t2	momoko ueda	japan	66	6
+2	t2	laura diaz	united states	66	6
+*/
+Q: the 3 player from japan have the same score
+NeuralSQL: SELECT (SELECT COUNT(DISTINCT score) FROM w WHERE country = 'japan' GROUP BY score) = 1
+
+
+CREATE TABLE espn sunday night football results (1987 - 2005)(
+	row_id int,
+	date text,
+	visiting team text,
+	final score text,
+	host team text,
+	stadium text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	visiting team	final score	host team	stadium
+0	september 11	indianapolis colts	24 - 7	baltimore ravens	m&t bank stadium
+1	september 18	kansas city chiefs	23 - 17	oakland raiders	mcafee coliseum
+2	september 25	new york giants	23 - 45	san diego chargers	qualcomm stadium
+*/
+Q: the hosting team be the new york giant on new year even and the st louis ram on new year 's day
+NeuralSQL: SELECT (SELECT (SELECT `host team` FROM w WHERE QA("map@is it new year even?"; date) = 'yes') = 'new york giant') AND (SELECT (SELECT `host team` FROM w WHERE QA("map@is it new year's day?"; date) = 'yes') = 'st louis ram')
+
+
+CREATE TABLE 2008 women 's british open(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score text,
+	to par int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par
+0	t1	yuri fudoh	japan	66 + 68 = 134	10
+1	t1	jiyai shin	south korea	66 + 68 = 134	10
+2	3	juli inkster	united states	65 + 70 = 135	9
+*/
+Q: kristie kerr , tie for 4th place , finish the round 1 stroke under lorena ochoa of mexico
+NeuralSQL: SELECT (SELECT (SELECT QA("map@what is the derived score?"; score) FROM w WHERE player = 'cristie kerr') < (SELECT QA("map@what is the derived score?"; score) FROM w WHERE player = 'lorena ochoa' AND country = 'mexico')) AND (SELECT (SELECT place FROM w WHERE player = 'cristie kerr') = "t4")
+
+
+CREATE TABLE connecticut public radio(
+	row_id int,
+	call sign text,
+	frequency text,
+	city of license text,
+	facility id int,
+	erp / power w int,
+	height m ( ft ) real,
+	class text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	call sign	frequency	city of license	facility id	erp / power w	height m ( ft )	class
+0	waic	91.9 fm	springfield , ma	1749	230	nan	b1
+1	wedw - fm	88.5 fm	stamford , ct	13619	2000	nan	a
+2	wnpr	90.5 fm ( hd ) connecticut public radio	meriden , ct	13627	18500	nan	b
+*/
+Q: there be 3 station with a call sign number in the 90s
+NeuralSQL: SELECT (SELECT COUNT(*) FROM w WHERE QA("map@is it in 90s?"; frequency) = 'yes' GROUP BY `call sign`) = 3
+
+
+CREATE TABLE 2003 chicago white sox season(
+	row_id int,
+	date text,
+	opponent text,
+	score text,
+	loss text,
+	time text,
+	att int,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	date	opponent	score	loss	time	att	record
+0	august 1	mariners	12 - 1	garcía (9 - 11)	2:52	39337	58 - 51
+1	august 2	mariners	0 - 10	wright (0 - 5)	2:22	45719	58 - 52
+2	august 3	mariners	2 - 8	buehrle (9 - 11)	2:57	45632	58 - 53
+*/
+Q: the 2003 chicago white sox game play on 26th august be longer than the game play on 24th august
+NeuralSQL: SELECT (SELECT time FROM w WHERE date = 'august 26') > (SELECT time FROM w WHERE date = 'august 24')
+
+
+CREATE TABLE 1987 masters tournament(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score text,
+	to par text,
+	money text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par	money
+0	t1	larry mize	united states	70 + 72 + 72 + 71 = 285	-3	playoff
+1	t1	bernhard langer	spain	73 + 71 + 70 + 71 = 285	-3	playoff
+2	t1	greg norman	australia	73 + 74 + 66 + 72 = 285	-3	playoff
+*/
+Q: bernhard m. langer have more point than roger maltbie during the 1987 master tournament
+NeuralSQL: SELECT (SELECT QA("map@what is the total score?"; score) FROM w WHERE player = 'bernhard langer') > (SELECT QA("map@what is the total score?"; score) FROM w WHERE player = 'roger maltbie')
+
+
+CREATE TABLE 1987 masters tournament(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score text,
+	to par text,
+	money text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	place	player	country	score	to par	money
+0	t1	larry mize	united states	70 + 72 + 72 + 71 = 285	-3	playoff
+1	t1	seve ballesteros	spain	73 + 71 + 70 + 71 = 285	-3	playoff
+2	t1	greg norman	australia	73 + 74 + 66 + 72 = 285	-3	playoff
+*/
+Q: most of the people who play for the 1987 master tournament be spanish
+NeuralSQL: SELECT (SELECT(SELECT COUNT(*) FROM w WHERE country = 'spain') / (SELECT COUNT(*) FROM w)) > 0.5
+
+
+CREATE TABLE 1976 world junior figure skating championships(
+	row_id int,
+	rank int,
+	name text,
+	nation text,
+	points real,
+	places int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	rank	name	nation	points	places
+0	1	sherri baier / robin cowan	canada	128.39	9
+1	2	lorene mitchell / donald mitchell	united states	124.94	16
+2	3	elizabeth cain / peter cain	australia	116.67	33
+*/
+Q: 2 of the 7 top - ranked figure skate team be from france
+NeuralSQL: SELECT (SELECT (SELECT COUNT(*) FROM w) = 7) AND (SELECT (SELECT COUNT(*) FROM w WHERE nation = 'france') = 2)

templates/prompts/prompt_tab_fact_word.txt

@@ -0,0 +1,228 @@
+Generate answer given the question and table to answer the question correctly.
+
+CREATE TABLE 1981 oakland raiders season(
+	row_id int,
+	round int,
+	overall int,
+	player text,
+	position text,
+	college text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	round	overall	player	position	college
+0	1	21	ted watts	cb	texas tech
+1	1	23	curt marsh	ot	washington
+2	2	48	howie long	de	villanova
+3	4	111	johnny robinson	dt	louisiana tech
+4	5	118	james davis	cb	southern
+5	9	248	curt mohl	ot	ucla
+6	10	276	frank hawkins	hb	nevada
+7	11	304	chester willis	hb	auburn
+8	12	332	phil nelson	te	delaware
+*/
+Q: the oakland raider drafter more defensive player than offensive player in the 1981 nlf draft
+A: entailed
+
+
+CREATE TABLE 1981 oakland raiders season(
+	row_id int,
+	round int,
+	overall int,
+	player text,
+	position text,
+	college text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	round	overall	player	position	college
+0	1	21	ted watts	cb	texas tech
+1	1	23	curt marsh	ot	washington
+2	2	48	howie long	de	villanova
+3	4	111	johnny robinson	dt	louisiana tech
+4	5	118	james davis	cb	southern
+5	9	248	curt mohl	ot	ucla
+6	10	276	frank hawkins	hb	nevada
+7	11	304	chester willis	hb	auburn
+8	12	332	phil nelson	te	delaware
+*/
+Q: the raider pick up halfback in back to back round in the 1981 nfl draft
+A: refuted
+
+
+CREATE TABLE wake forest demon deacons football , 1980 - 89(
+	row_id int,
+	date text,
+	opponent text,
+	location text,
+	result text,
+	attendance int)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	date	opponent	location	result	attendance
+0	1988-09-03 00:00:00	villanova	villanova stadium villanova , pa	w 31 - 11	11624
+1	1988-09-10 00:00:00	illinois state	groves stadium winston - salem , nc	w 35 - 0	22250
+2	1988-09-17 00:00:00	north carolina state	carter - finley stadium raleigh , nc	l 6 - 14	48000
+3	1988-09-24 00:00:00	19 michigan	michigan stadium ann arbor , mi	l 9 - 19	102776
+4	1988-10-08 00:00:00	north carolina	groves stadium winston - salem , nc	w 42 - 24	33500
+5	1988-10-15 00:00:00	maryland	byrd stadium college park , md	w 27 - 24	41278
+6	1988-10-22 00:00:00	virginia	groves stadium winston - salem , nc	l 14 - 34	21300
+7	1988-10-29 00:00:00	15 clemson	groves stadium winston - salem , nc	l 21 - 38	27300
+8	1988-11-05 00:00:00	duke	wallace wade stadium durham , nc	w 35 - 16	35500
+9	1988-11-12 00:00:00	georgia tech	groves stadium winston - salem , nc	w 28 - 24	21500
+10	1988-11-19 00:00:00	appalachian state	groves stadium winston - salem , nc	t 34 - 34	21050
+*/
+Q: the demon deacon finish the season with a 6 - 4 - 1 record
+A: refuted
+
+
+CREATE TABLE wake forest demon deacons football , 1980 - 89(
+	row_id int,
+	date text,
+	opponent text,
+	location text,
+	result text,
+	attendance int)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	date	opponent	location	result	attendance
+0	1988-09-03 00:00:00	villanova	villanova stadium villanova , pa	w 31 - 11	11624
+1	1988-09-10 00:00:00	illinois state	groves stadium winston - salem , nc	w 35 - 0	22250
+2	1988-09-17 00:00:00	north carolina state	carter - finley stadium raleigh , nc	l 6 - 14	48000
+3	1988-09-24 00:00:00	19 michigan	michigan stadium ann arbor , mi	l 9 - 19	102776
+4	1988-10-08 00:00:00	north carolina	groves stadium winston - salem , nc	w 42 - 24	33500
+5	1988-10-15 00:00:00	maryland	byrd stadium college park , md	w 27 - 24	41278
+6	1988-10-22 00:00:00	virginia	groves stadium winston - salem , nc	l 14 - 34	21300
+7	1988-10-29 00:00:00	15 clemson	groves stadium winston - salem , nc	l 21 - 38	27300
+8	1988-11-05 00:00:00	duke	wallace wade stadium durham , nc	w 35 - 16	35500
+9	1988-11-12 00:00:00	georgia tech	groves stadium winston - salem , nc	w 28 - 24	21500
+10	1988-11-19 00:00:00	appalachian state	groves stadium winston - salem , nc	t 34 - 34	21050
+*/
+Q: wake forest lose the only 5 game they play versus rank opponent
+A: entailed
+
+
+CREATE TABLE 2008 women 's british open(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score int,
+	to par int)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	place	player	country	score	to par
+0	1	juli inkster	united states	65	7
+1	t2	momoko ueda	japan	66	6
+2	t2	laura diaz	united states	66	6
+3	t2	ji young oh	south korea	66	6
+4	t2	yuri fudoh	japan	66	6
+5	t2	johanna head	england	66	6
+6	t2	stacy prammanasudh	united states	66	6
+7	t2	jiyai shin	south korea	66	6
+8	t9	kristy mcpherson	united states	67	5
+9	t9	karen stupples	england	67	5
+10	t9	rebecca hudson	england	67	5
+11	t9	sherri steinhauer	united states	67	5
+*/
+Q: 7 of the player have an identical score in the 2008 woman 's british open
+A: refuted
+
+
+CREATE TABLE 2008 women 's british open(
+	row_id int,
+	place text,
+	player text,
+	country text,
+	score int,
+	to par int)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	place	player	country	score	to par
+0	1	juli inkster	united states	65	7
+1	t2	momoko ueda	japan	66	6
+2	t2	laura diaz	united states	66	6
+3	t2	ji young oh	south korea	66	6
+4	t2	yuri fudoh	japan	66	6
+5	t2	johanna head	england	66	6
+6	t2	stacy prammanasudh	united states	66	6
+7	t2	jiyai shin	south korea	66	6
+8	t9	kristy mcpherson	united states	67	5
+9	t9	karen stupples	england	67	5
+10	t9	rebecca hudson	england	67	5
+11	t9	sherri steinhauer	united states	67	5
+*/
+Q: there be 3 player total from the united state
+A: entailed
+
+
+CREATE TABLE espn sunday night football results (1987 - 2005)(
+	row_id int,
+	date text,
+	visiting team text,
+	final score text,
+	host team text,
+	stadium text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	date	visiting team	final score	host team	stadium
+0	september 11	indianapolis colts	24 - 7	baltimore ravens	m&t bank stadium
+1	september 18	kansas city chiefs	23 - 17	oakland raiders	mcafee coliseum
+2	september 25	new york giants	23 - 45	san diego chargers	qualcomm stadium
+3	october 2	san francisco 49ers	14 - 31	arizona cardinals	estadio azteca
+4	october 9	cincinnati bengals	20 - 23	jacksonville jaguars	alltel stadium
+5	october 16	houston texans	10 - 42	seattle seahawks	qwest field
+6	october 30	buffalo bills	16 - 21	new england patriots	gillette stadium
+7	november 6	philadelphia eagles	10 - 17	washington redskins	fedex field
+8	november 13	cleveland browns	21 - 34	pittsburgh steelers	heinz field
+9	november 20	kansas city chiefs	45 - 17	houston texans	reliant stadium
+10	november 27	new orleans saints	21 - 19	new york jets	giants stadium
+11	december 4	oakland raiders	10 - 34	san diego chargers	qualcomm stadium
+12	december 11	detroit lions	13 - 16	green bay packers	lambeau field
+13	december 17	denver broncos	28 - 17	buffalo bills	ralph wilson stadium
+14	december 18	atlanta falcons	3 - 16	chicago bears	soldier field
+15	december 25	minnesota vikings	23 - 30	baltimore ravens	m&t bank stadium
+16	december 31	new york giants	30 - 21	oakland raiders	mcafee coliseum
+17	january 1 (2006)	st louis rams	20 - 10	dallas cowboys	texas stadium
+*/
+Q: the visiting team be the new york giant on new year 's eve , and st louis ram on new year 's day
+A: refuted
+
+
+CREATE TABLE espn sunday night football results (1987 - 2005)(
+	row_id int,
+	date text,
+	visiting team text,
+	final score text,
+	host team text,
+	stadium text)
+/*
+All rows of the table:
+SELECT * FROM w;
+row_id	date	visiting team	final score	host team	stadium
+0	september 11	indianapolis colts	24 - 7	baltimore ravens	m&t bank stadium
+1	september 18	kansas city chiefs	23 - 17	oakland raiders	mcafee coliseum
+2	september 25	new york giants	23 - 45	san diego chargers	qualcomm stadium
+3	october 2	san francisco 49ers	14 - 31	arizona cardinals	estadio azteca
+4	october 9	cincinnati bengals	20 - 23	jacksonville jaguars	alltel stadium
+5	october 16	houston texans	10 - 42	seattle seahawks	qwest field
+6	october 30	buffalo bills	16 - 21	new england patriots	gillette stadium
+7	november 6	philadelphia eagles	10 - 17	washington redskins	fedex field
+8	november 13	cleveland browns	21 - 34	pittsburgh steelers	heinz field
+9	november 20	kansas city chiefs	45 - 17	houston texans	reliant stadium
+10	november 27	new orleans saints	21 - 19	new york jets	giants stadium
+11	december 4	oakland raiders	10 - 34	san diego chargers	qualcomm stadium
+12	december 11	detroit lions	13 - 16	green bay packers	lambeau field
+13	december 17	denver broncos	28 - 17	buffalo bills	ralph wilson stadium
+14	december 18	atlanta falcons	3 - 16	chicago bears	soldier field
+15	december 25	minnesota vikings	23 - 30	baltimore ravens	m&t bank stadium
+16	december 31	new york giants	30 - 21	oakland raiders	mcafee coliseum
+17	january 1 (2006)	st louis rams	20 - 10	dallas cowboys	texas stadium
+*/
+Q: the indianapolis colt and kansas city chief be the only visit team in september
+A: entailed

templates/prompts/prompt_wikitq_puresql_v3.txt

@@ -0,0 +1,287 @@
+Generate SQL given the question and table to answer the question correctly.
+
+CREATE TABLE Fabrice Santoro(
+	row_id int,
+	name text,
+	2001 text,
+	2002 text,
+	2003 text,
+	2004 text,
+	2005 text,
+	2006 text,
+	2007 text,
+	2008 text,
+	2009 text,
+	2010 text,
+	career\nsr text,
+	wins int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	name	2001	2002	2003	2004	2005	2006	2007	2008	2009	2010	career\nsr	wins
+0	australian open	2r	1r	3r	2r	1r	qf	3r	2r	3r	1r	0 / 18	22
+1	french open	4r	2r	2r	3r	1r	1r	1r	2r	1r	a	0 / 20	17
+2	wimbledon	3r	2r	2r	2r	2r	2r	2r	1r	2r	a	0 / 14	11
+*/
+Q: did he win more at the australian open or indian wells?
+SQL: SELECT name FROM w WHERE name IN ('australian open', 'indian wells') ORDER BY wins DESC LIMIT 1
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result text,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	time	game site	tv	result	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	rca dome	nbc	l	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	raymond james stadium	fox	l	0–2
+2	3	2007-9-24	tennessee titans	t20:30 edt	louisiana superdome	espn	l	0–3
+*/
+Q: what number of games were lost at home?
+SQL: SELECT COUNT(*) FROM w WHERE result = 'l' AND `game site` = 'louisiana superdome'
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result/score text,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	time	game site	tv	result/score	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	away	nbc	loss	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	home	fox	win	1-1
+2	3	2007-9-24	tennessee titans	t20:30 edt	away	espn	loss	1-2
+*/
+Q: what number of games were lost at home?
+SQL: SELECT COUNT(*) FROM w WHERE `result/score` = 'loss' AND `game site` = 'home'
+
+
+CREATE TABLE Electricity in Sri Lanka(
+	row_id int,
+	filledcolumnname text,
+	2005 int,
+	2006 int,
+	2007 int,
+	2008 int,
+	2009 int,
+	2010 int,
+	2011 int,
+	2012 int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	filledcolumnname	2005	2006	2007	2008	2009	2010	2011	2012
+0	hydro power	1293	1316	1326	1357	1379	1382	1401	1584
+1	thermal	1155	1155	1155	1285	1290	1390	1690	1638
+2	other renewables	3	3	3	3	15	45	50	90
+*/
+Q: did the hydro power increase or decrease from 2010 to 2012?
+SQL: SELECT CASE WHEN (SELECT `2010` FROM w WHERE filledcolumnname = 'hydro power') < (SELECT `2012` FROM w WHERE filledcolumnname = 'hydro power') THEN 'increase' ELSE 'decrease' END
+
+
+CREATE TABLE Portugal in the Eurovision Song Contest 1979(
+	row_id int,
+	draw int,
+	artist text,
+	song text,
+	points int,
+	place text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	draw	artist	song	points	place
+0	1	gonzaga coutinho	"tema para um homem só"	102	5th
+1	2	pedro osório s.a.r.l.	"uma canção comercial"	123	3rd
+2	3	concha	"qualquer dia, quem diria"	78	6th
+*/
+Q: who was the last draw?
+SQL: SELECT `artist` FROM w ORDER by `draw` desc LIMIT 1
+
+
+CREATE TABLE GER Class N31(
+	row_id int,
+	year int,
+	order text,
+	quantity int,
+	ger nos. text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	order	quantity	ger nos.
+0	1893	n31	1	999
+1	1893	h33	10	979
+2	1894	l33	10	989
+*/
+Q: which had more ger numbers, 1898 or 1893?
+SQL: SELECT `year` FROM w WHERE `year` IN ( '1898' , '1893' ) GROUP by `year` ORDER by SUM (`ger nos.`) desc LIMIT 1
+
+
+CREATE TABLE List of spans(
+	row_id int,
+	tramway text,
+	country text,
+	city text,
+	height of pylons text,
+	span width,\nleaning straight line text,
+	span width,\nhorizontal measurement text,
+	height of cable over ground text,
+	year of inauguration text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	tramway	country	city	height of pylons	span width,\nleaning straight line	span width,\nhorizontal measurement	height of cable over ground	year of inauguration	notes
+0	peak 2 peak gondola	canada	whistler	65m	3024 m	3019 m	436 m	2008	3s aerial tramway constructed by doppelmayr
+1	hut of regensburg material transport aerial railway	austria	falbeson	?	?	?	430 m	?	none
+2	vanoise express	france	vanoise	none	1850 m	1800 m	380 m	2003	none
+*/
+Q: was the sandia peak tramway innagurate before or after the 3s aerial tramway?
+SQL: SELECT ( SELECT `year of inauguration` FROM w WHERE `tramway` = 'sandia peak tramway' ) < ( SELECT `year of inauguration` FROM w WHERE `tramway` = '3s aerial tramway' )
+
+
+CREATE TABLE World Artistic Gymnastics Championships – Women's floor(
+	id int,
+	year int,
+	location text,
+	gold text,
+	silver text,
+	bronze text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+id	year	location	gold	silver	bronze
+0	1950	basel	helena rakoczy	tereza kočiš	stefania reindlova
+1	1954	rome	tamara manina	eva bosáková	maria gorokovskaya
+2	1958	moscow	eva bosáková	larisa latynina	keiko tanaka
+*/
+Q: where were the championships held before the 1962 prague championships?
+SQL: SELECT `location` FROM w WHERE `year` < 1962 ORDER by `year` desc LIMIT 1
+
+
+CREATE TABLE WSL World Heavyweight Championship(
+	id int,
+	wrestler: text,
+	times: text,
+	date: text,
+	location: text,
+	notes: text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+id	wrestler:	times:	date:	location:	notes:
+0	jonnie stewart	1	1996-6-6	rochester, minnesota	defeated larry gligorovich to win the awa superstars of wrestling world heavyweight championship.
+1	king kong bundy	1	1999-3-31	oshkosh, wisconsin	later stripped of the title by owner dale gagne.
+2	the patriot; (danny dominion)	1	2000-7-29	pine bluff, arkansas	defeated dale gagne in an impromptu match to win the title.
+*/
+Q: when did steve corino win his first wsl title?
+SQL: SELECT `date:` FROM w WHERE `wrestler:` = 'steve corino' ORDER by `date:` LIMIT 1
+
+
+CREATE TABLE Płock Governorate(
+	row_id int,
+	language text,
+	number int,
+	percentage (%) text,
+	males int,
+	females int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	language	number	percentage (%)	males	females
+0	polish	447685	80.86	216794	230891
+1	yiddish	51215	9.25	24538	26677
+2	german	35931	6.49	17409	18522
+*/
+Q: how many male and female german speakers are there?
+SQL: SELECT `males` + `females` FROM w WHERE `language` = 'german'
+
+
+CREATE TABLE Shikoku Pilgrimage(
+	row_id int,
+	no. int,
+	temple text,
+	honzon (main image) text,
+	city/town/village text,
+	prefecture text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	no.	temple	honzon (main image)	city/town/village	prefecture
+0	1	ryōzen-ji (霊山寺)	shaka nyorai	naruto	tokushima prefecture
+1	2	gokuraku-ji (極楽寺)	amida nyorai	naruto	tokushima prefecture
+2	3	konsen-ji (金泉寺)	shaka nyorai	itano	tokushima prefecture
+*/
+Q: what is the difference in the number of temples between imabari and matsuyama?
+SQL: SELECT abs ( ( SELECT COUNT ( `temple` ) FROM w WHERE `city/town/village` = 'imabari' ) - ( SELECT COUNT ( `temple` ) FROM w WHERE `city/town/village` = 'matsuyama' ) )
+
+
+CREATE TABLE Athletics at the 2001 Goodwill Games – Results(
+	row_id int,
+	rank real,
+	name text,
+	nationality text,
+	time text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	rank	name	nationality	time
+0	nan	brahim boulami	morocco	2022-07-17 08:17:43
+1	nan	reuben kosgei	kenya	2022-07-17 08:18:37
+2	nan	stephen cherono	kenya	2022-07-17 08:19:58
+*/
+Q: what counties had the least participants for the race?
+SQL: SELECT `nationality` FROM w GROUP by `nationality` having COUNT ( `name` ) = ( SELECT COUNT ( `name` ) FROM w GROUP by `nationality` ORDER by COUNT ( `name` ) asc LIMIT 1 )
+
+
+CREATE TABLE Saint Helena, Ascension and Tristan da Cunha(
+	row_id int,
+	administrative\narea text,
+	area\nkm2 real,
+	area\nsq mi int,
+	population int,
+	administrative\ncentre text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	administrative\narea	area\nkm2	area\nsq mi	population	administrative\ncentre
+0	saint helena	122.0	47	5809	jamestown
+1	ascension island	91.0	35	1532	georgetown
+2	tristan da cunha	184.0	71	388	edinburgh of the 7 seas
+*/
+Q: is the are of saint helena more than that of nightingale island?
+SQL: SELECT ( SELECT `area\\nkm2` FROM w WHERE `administrative\\narea` = 'saint helena' ) > ( SELECT `area\\nkm2` FROM w WHERE `administrative\\narea` = 'nightingale island' )
+
+
+CREATE TABLE The Boys (comics)(
+	row_id int,
+	# int,
+	title text,
+	tpb isbn text,
+	tpb release date text,
+	tpb page number int,
+	collected material text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	#	title	tpb isbn	tpb release date	tpb page number	collected material
+0	1	the name of the game	isbn 91-33-30546-3	2007-06-01 00:00:00	152	the boys #1-6
+1	2	get some	isbn 1-933305-68-1	2008-03-01 00:00:00	192	the boys #7–14
+2	3	good for the soul	isbn 1-933305-92-4	2008-10-01 00:00:00	192	the boys #15-22
+*/
+Q: what title appears before "the self-preservation society"?
+SQL: SELECT `title` FROM w WHERE row_id = ( SELECT row_id FROM w WHERE `title` = 'the self-preservation society' ) - 1

templates/prompts/prompt_wikitq_python_simplified_v4.txt

@@ -0,0 +1,426 @@
+Generate Python given the question and table to answer the question correctly.
+If question-relevant column(s) contents require external knowledge or unsupported Python grammar, map it to a new column by calling function qa_map(table, question, column(s)).
+The `qa_map()` function definition is listed to help know its functionality better:
+<code>
+def qa_map(db: pd.DataFrame, question: str, columns: List[str]) -> pd.DataFrame:
+    qa_model = OpenAIQAModel()
+    new_db = NeuralDB([{"title": "", "table": {"header": db.columns.values.tolist(), "rows": db.values.tolist()}}])
+    sql_executed_sub_tables = []
+    for column in columns:
+        column = f"`{column}`"
+        sql_executed_sub_tables.append(new_db.execute_query(column))
+        sub_table = qa_model.qa(question, sql_executed_sub_tables,)
+        new_db.add_subtable(sub_table, verbose=verbose)
+    table = new_db.get_table()
+    return pd.DataFrame(table["rows"], columns=table["header"])
+</code>
+
+Here are some examples.
+
+CREATE TABLE Fabrice Santoro(
+	row_id int,
+	name text,
+	2001 text,
+	2002 text,
+	2003 text,
+	2004 text,
+	2005 text,
+	2006 text,
+	2007 text,
+	2008 text,
+	2009 text,
+	2010 text,
+	career\nsr text,
+	career\nwin-loss text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	name	2001	2002	2003	2004	2005	2006	2007	2008	2009	2010	career\nsr	career\nwin-loss
+0	australian open	2r	1r	3r	2r	1r	qf	3r	2r	3r	1r	0 / 18	22–18
+1	french open	4r	2r	2r	3r	1r	1r	1r	2r	1r	a	0 / 20	17–20
+2	wimbledon	3r	2r	2r	2r	2r	2r	2r	1r	2r	a	0 / 14	11–14
+*/
+Q: did he win more at the australian open or indian wells?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    table = qa_map(table, "how many wins?", ["career\\nwin-loss"])
+    sub_table = table[(table['name'] == 'australian open') | (table['name'] == 'indian wells')]
+    tmp = [(x, y) for x, y in zip(sub_table['name'], sub_table['how many wins?'])]
+    tmp = sorted(tmp, key=lambda x: x[1], reverse=True)
+    result = list(map(lambda x: x[0], tmp))[0]
+    return result
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result text,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	time	game site	tv	result	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	rca dome	nbc	l	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	raymond james stadium	fox	l	0–2
+2	3	2007-9-24	tennessee titans	t20:30 edt	louisiana superdome	espn	l	0–3
+*/
+Q: what number of games were lost at home?
+NeuralNeuralPython:
+def solve(table: pd.DataFrame):
+    sub_table = table[(table['result'] == 'l') & (table['game site'] == 'louisiana superdome')]
+    result = len(sub_table)
+    return result
+
+
+CREATE TABLE Electricity in Sri Lanka(
+	row_id int,
+	filledcolumnname text,
+	2005 int,
+	2006 int,
+	2007 int,
+	2008 int,
+	2009 int,
+	2010 int,
+	2011 int,
+	2012 int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	filledcolumnname	2005	2006	2007	2008	2009	2010	2011	2012
+0	hydro power	1293	1316	1326	1357	1379	1382	1401	1584
+1	thermal	1155	1155	1155	1285	1290	1390	1690	1638
+2	other renewables	3	3	3	3	15	45	50	90
+*/
+Q: did the hydro power increase or decrease from 2010 to 2012?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    result = table[table['filledcolumnname'] == 'hydro power']['2010'].values[0] - table[table['filledcolumnname'] == 'hydro power']['2012'].values[0]
+    if result > 0:
+        return 'decrease'
+    else:
+        return 'increase'
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result/score text,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	time	game site	tv	result/score	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	rca dome	nbc	l 41 – 10	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	raymond james stadium	fox	l 31 – 14	0–2
+2	3	2007-9-24	tennessee titans	t20:30 edt	louisiana superdome	espn	l 31 – 14	0–3
+*/
+Q: what number of games were lost at home?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    table = qa_map(table, "is it a loss?", ["result/score"])
+    table = qa_map(table, "is it the home court of is it the home court of New Orleans Saints?", ["game site"])
+    sub_table = table[(table['is it a loss?'] == 'yes') & (table['is it the home court of is it the home court of New Orleans Saints?'] == 'yes')]
+    result = len(sub_table)
+    return result
+
+
+CREATE TABLE Portugal in the Eurovision Song Contest 1979(
+	row_id int,
+	draw int,
+	artist text,
+	song text,
+	points int,
+	place text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	draw	artist	song	points	place
+0	1	gonzaga coutinho	"tema para um homem só"	102	5th
+1	2	pedro osório s.a.r.l.	"uma canção comercial"	123	3rd
+2	3	concha	"qualquer dia, quem diria"	78	6th
+*/
+Q: who was the last draw in the table?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    sub_table = table
+    tmp = [(x, y) for x, y in zip(sub_table['artist'], sub_table['row_id'])]
+    tmp = sorted(tmp, key=lambda x: x[1], reverse=True)
+    result = list(map(lambda x: x[0], tmp))[0]
+    return result
+
+
+CREATE TABLE Highest mountain peaks of California(
+	row_id int,
+	rank int,
+	mountain peak text,
+	mountain range text,
+	elevation text,
+	prominence text,
+	isolation text,
+	location text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	rank	mountain peak	mountain range	elevation	prominence	isolation	location
+0	1	mount whitney	sierra nevada	14505 ft; 4421 m	10080 ft; 3072 m	1646 mi; 2649 km	36°34′43″n 118°17′31″w / 36.5786°n 118.292°w
+1	2	mount williamson	sierra nevada	14379 ft; 4383 m	1677 ft; 511 m	5.4 mi; 8.7 km	36°39′21″n 118°18′40″w / 36.6559°n 118.3111°w
+2	3	white mountain peak	white mountains	14252 ft; 4344 m	7196 ft; 2193 m	67 mi; 109 km	37°38′3″n 118°15′21″w / 37.6341°n 118.2557°w
+*/
+Q: which mountain peak has a prominence more than 10,000 ft?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    table = qa_map(table, "how many feet is the prominence?", ["prominence"])
+    sub_table = table[(table['how many feet is the prominence?'] > 10000)]
+    result = [x for x in sub_table['mountain peak']]
+    return result
+
+
+CREATE TABLE List of spans(
+	row_id int,
+	tramway text,
+	country text,
+	city text,
+	height of pylons text,
+	span width,\nleaning straight line text,
+	span width,\nhorizontal measurement text,
+	height of cable over ground text,
+	year of inauguration text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	tramway	country	city	height of pylons	span width,\nleaning straight line	span width,\nhorizontal measurement	notes
+0	peak 2 peak gondola	canada	whistler	65m	3024 m	3019 m	436 m	2008	3s aerial tramway constructed by doppelmayr
+1	hut of regensburg material transport aerial railway	austria	falbeson	?	?	?	430 m	?	none
+2	vanoise express	france	vanoise	none	1850 m	1800 m	380 m	2003	none
+*/
+Q: was the sandia peak tramway innagurate before or after the 3s aerial tramway?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    result = table[table['tramway'] == 'sandia peak tramway']['year of inauguration'].values[0] - table[table['tramway'] == '3s aerial tramway']['year of inauguration'].values[0]
+    if result > 0:
+        return 'after'
+    else:
+        return 'before'
+
+
+CREATE TABLE WSL World Heavyweight Championship(
+	id int,
+	wrestler: text,
+	times: text,
+	date: text,
+	location: text,
+	notes: text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+id	wrestler:	times:	date:	location:	notes:
+0	jonnie stewart	1	1996-6-6	rochester, minnesota	defeated larry gligorovich to win the awa superstars of wrestling world heavyweight championship.
+1	king kong bundy	1	1999-3-31	oshkosh, wisconsin	later stripped of the title by owner dale gagne.
+2	the patriot; (danny dominion)	1	2000-7-29	pine bluff, arkansas	defeated dale gagne in an impromptu match to win the title.
+*/
+Q: when did steve corino win his first wsl title?
+NeuralPython:
+    sub_table = table[(table['wrestler:'] == 'steve corino')]
+    tmp = [x for x in sub_table['date:']]
+    tmp = sorted(tmp, reverse=False)
+    result = tmp[0]
+    return result
+
+
+CREATE TABLE Shikoku Pilgrimage(
+	row_id int,
+	no. int,
+	temple text,
+	honzon (main image) text,
+	city/town/village text,
+	prefecture text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	no.	temple	honzon (main image)	city/town/village	prefecture
+0	1	ryōzen-ji (霊山寺)	shaka nyorai	naruto	tokushima prefecture
+1	2	gokuraku-ji (極楽寺)	amida nyorai	naruto	tokushima prefecture
+2	3	konsen-ji (金泉寺)	shaka nyorai	itano	tokushima prefecture
+*/
+Q: what is the difference in the number of temples between imabari and matsuyama?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    sub_table1 = table[(table['city/town/village'] == 'imabari')]
+    sub_table2 = table[(table['city/town/village'] == 'matsuyama')]
+    result = math.abs(len(sub_table1) - len(sub_table2))
+    return result
+
+
+CREATE TABLE FC Seoul in Asian football(
+	row_id int,
+	# int,
+	season int,
+	competition text,
+	date text,
+	round text,
+	opponent text,
+	h / a text,
+	result text,
+	scorer (s) text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	#	season	competition	date	round	opponent	h / a	result	scorer (s)
+0	35	2011	afc; champions league	2011-03-02 00:00:00	group stage	al-ain	a	1–0	s : dejan damjanović
+1	36	2011	afc; champions league	2011-03-15 00:00:00	group stage	hangzhou greentown	h	3–0	s : dejan damjanović, ou kyoung-jun, mauricio molina
+2	37	2011	afc; champions league	2011-04-06 00:00:00	group stage	nagoya grampus	a	1–1	s : choi hyun-tae; n : kensuke nagai
+*/
+Q: how many consecutive games did dejan damjanovic score a goal in during the 2013 season?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    result = 0
+    consecutive_flag = False
+    for row_id, row in table.iterrows():
+        if row['season'] == 2013 and row['scorer (s)'] == 'dejan damjanović':
+            result += 1
+            consecutive_flag = True
+        elif consecutive_flag:
+            break
+    return result
+
+
+CREATE TABLE Płock Governorate(
+	row_id int,
+	language text,
+	number int,
+	percentage (%) text,
+	males int,
+	females int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	language	number	percentage (%)	males	females
+0	polish	447685	80.86	216794	230891
+1	yiddish	51215	9.25	24538	26677
+2	german	35931	6.49	17409	18522
+*/
+Q: how many male and female german speakers are there?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    sub_table = table[(table['language'] == 'german')]
+    result = sum([x + y for x, y in zip(sub_table['males'], sub_table['females'])])
+    return result
+
+
+CREATE TABLE Saint Helena, Ascension and Tristan da Cunha(
+	row_id int,
+	administrative\narea text,
+	area\nkm2 real,
+	area\nsq mi int,
+	population int,
+	administrative\ncentre text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	administrative\narea	area\nkm2	area\nsq mi	population	administrative\ncentre
+0	saint helena	122.0	47	5809	jamestown
+1	ascension island	91.0	35	1532	georgetown
+2	tristan da cunha	184.0	71	388	edinburgh of the 7 seas
+*/
+Q: is the are of saint helena more than that of nightingale island?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    result = table[table['administrative\\narea'] == 'saint helena']['area\\nkm2'].values[0] - table[table['administrative\\narea'] == 'nightingale island']['area\\nkm2'].values[0]
+    if result > 0:
+        return 'yes'
+    else:
+        return 'no'
+
+
+CREATE TABLE List of political parties in Japan(
+	row_id int,
+	party text,
+	diet representation\nrepresentatives int,
+	diet representation\ncouncillors int,
+	party leader(s) text,
+	comments text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	party	diet representation\nrepresentatives	diet representation\ncouncillors	party leader(s)	comments
+0	your party (yp); minna no tō みんなの党; ("everybody's party")	18	18	yoshimi watanabe reps.	conservative liberalism, neoliberalism, economic liberalism, libertarianism, anti-nuclear
+1	japanese communist party (jcp); nihon kyōsan-tō 日本共産党	8	11	kazuo shii reps.	the japanese communist party is japan's oldest party. it was formed in 1922 as an underground organization in the empire of japan, but was legalized after world war ii during the occupation. it used to be a communist party, but the party has past_ref shifted to a socialist party.
+2	people's life party (plp); seikatsu no tō 生活の党	7	2	ichirō ozawa reps.	life party was founded by ichirō ozawa and 14 other diet members who were in the 2022-7-4 party of japan after a leadership dispute between ozawa and yukiko kada.
+*/
+Q: what party is listed previous to the new renaissance party?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    result = []
+    for row_id, row in table.iterrows():
+        if row['party'] == 'new renaissance party':
+            result.append(table.iloc[row_id - 1]['party'])
+    return result
+
+
+CREATE TABLE 1975 24 Hours of Le Mans(
+	row_id int,
+	pos int,
+	class text,
+	no int,
+	team text,
+	drivers text,
+	chassis text,
+	engine text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	pos	class	no	team	drivers	chassis	engine
+0	32	s; 2	27	société roc	laurent ferrier;  xavier lapeyre;  christian ethuin	lola t294	roc-simca 2.0l i4
+1	33	s; 2	29	société roc	pierre-marie painvin;  franz hummel	lola t292	roc-simca 2.0l i4
+2	34	s; 3	3	christian poirot	christian poirot;  gérard cuynet;  guillermo ortega;  jean-claude lagniez	porsche 454	porsche 3.0l flat-8
+*/
+Q: which country has the most teams on the list?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    table = qa_map(table, "what is the country?", ["team"])
+    result = table['what is the country?'].value_counts().idxmax()
+    return result
+
+
+CREATE TABLE Fabrice Santoro(
+	row_id int,
+	name text,
+	2001 text,
+	2002 text,
+	2003 text,
+	2004 text,
+	2005 text,
+	2006 text,
+	2007 text,
+	2008 text,
+	2009 text,
+	2010 text,
+	career\nsr text,
+	wins int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	name	2001	2002	2003	2004	2005	2006	2007	2008	2009	2010	career\nsr	wins
+0	australian open	2r	1r	3r	2r	1r	qf	3r	2r	3r	1r	0 / 18	22
+1	french open	4r	2r	2r	3r	1r	1r	1r	2r	1r	a	0 / 20	17
+2	wimbledon	3r	2r	2r	2r	2r	2r	2r	1r	2r	a	0 / 14	11
+*/
+Q: did he win more at the australian open or indian wells?
+NeuralPython:
+def solve(table: pd.DataFrame):
+    sub_table = table[(table['name'] == 'australian open') | (table['name'] == 'indian wells')]
+    tmp = [(x, y) for x, y in zip(sub_table['name'], sub_table['wins'])]
+    tmp = sorted(tmp, key=lambda x: x[1], reverse=True)
+    result = list(map(lambda x: x[0], tmp))[0]
+    return result

templates/prompts/prompt_wikitq_v3.txt

@@ -0,0 +1,295 @@
+Generate SQL given the question and table to answer the question correctly.
+If question-relevant column(s) contents are not suitable for SQL comparisons or calculations, map it to a new column with clean content by a new grammar QA("map@").
+If mapping to a new column still can not answer the question with valid SQL, turn to an end-to-end solution by a new grammar QA("ans@"). This grammar aims to solve all the rest of complex questions or tables.
+
+CREATE TABLE Fabrice Santoro(
+	row_id int,
+	name text,
+	2001 text,
+	2002 text,
+	2003 text,
+	2004 text,
+	2005 text,
+	2006 text,
+	2007 text,
+	2008 text,
+	2009 text,
+	2010 text,
+	career\nsr text,
+	career\nwin-loss text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	name	2001	2002	2003	2004	2005	2006	2007	2008	2009	2010	career\nsr	career\nwin-loss
+0	australian open	2r	1r	3r	2r	1r	qf	3r	2r	3r	1r	0 / 18	22–18
+1	french open	4r	2r	2r	3r	1r	1r	1r	2r	1r	a	0 / 20	17–20
+2	wimbledon	3r	2r	2r	2r	2r	2r	2r	1r	2r	a	0 / 14	11–14
+*/
+Q: did he win more at the australian open or indian wells?
+NeuralSQL: SELECT name FROM w WHERE name IN ('australian open', 'indian wells') ORDER BY QA("map@how many wins?"; `career\nwin-loss`) DESC LIMIT 1
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result/score text,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	time	game site	tv	result/score	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	rca dome	nbc	l 41 – 10	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	raymond james stadium	fox	l 31 – 14	0–2
+2	3	2007-9-24	tennessee titans	t20:30 edt	louisiana superdome	espn	l 31 – 14	0–3
+*/
+Q: what number of games were lost at home?
+NeuralSQL: SELECT COUNT(*) FROM w WHERE QA("map@is it a loss?"; `result/score`) = 'yes' AND QA("map@is it the home court of New Orleans Saints?"; `game site`) = 'yes'
+
+
+CREATE TABLE 2007 New Orleans Saints season(
+	row_id int,
+	week int,
+	date text,
+	opponent text,
+	time text,
+	game site text,
+	tv text,
+	result/score text,
+	record text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	week	date	opponent	time	game site	tv	result/score	record
+0	1	2007-9-6	indianapolis colts	t20:30 edt	away	nbc	loss	0–1
+1	2	2007-9-16	tampa bay buccaneers	t13:0 edt	home	fox	win	1-1
+2	3	2007-9-24	tennessee titans	t20:30 edt	away	espn	loss	1-2
+*/
+Q: what number of games were lost at home?
+NeuralSQL: SELECT COUNT(*) FROM w WHERE `result/score` = 'loss' AND `game site` = 'home'
+
+
+CREATE TABLE Demographics of Alaska(
+	row_id int,
+	by race text,
+	white text,
+	black text,
+	aian* text,
+	asian text,
+	nhpi* text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	by race	white	black	aian*	asian	nhpi*
+0	2000 (total population)	75.43%	4.46%	19.06%	5.24%	0.88%
+1	2000 (hispanic only)	3.42%	0.33%	0.45%	0.16%	0.06%
+2	2005 (total population)	74.71%	4.72%	18.77%	5.9%	0.88%
+*/
+Q: which hispanic population had the greatest growth from 2000 to 2005?
+NeuralSQL: QA("ans@which race had the greatest value?"; SELECT white, black,  `aian*`, asian, `nhpi*` FROM w WHERE `by race` = 'growth 2000–5 (hispanic only)')
+
+
+CREATE TABLE Highest mountain peaks of California(
+	row_id int,
+	rank int,
+	mountain peak text,
+	mountain range text,
+	elevation text,
+	prominence text,
+	isolation text,
+	location text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	rank	mountain peak	mountain range	elevation	prominence	isolation	location
+0	1	mount whitney	sierra nevada	14505 ft; 4421 m	10080 ft; 3072 m	1646 mi; 2649 km	36°34′43″n 118°17′31″w / 36.5786°n 118.292°w
+1	2	mount williamson	sierra nevada	14379 ft; 4383 m	1677 ft; 511 m	5.4 mi; 8.7 km	36°39′21″n 118°18′40″w / 36.6559°n 118.3111°w
+2	3	white mountain peak	white mountains	14252 ft; 4344 m	7196 ft; 2193 m	67 mi; 109 km	37°38′3″n 118°15′21″w / 37.6341°n 118.2557°w
+*/
+Q: which mountain peak has a prominence more than 10,000 ft?
+NeuralSQL: SELECT `mountain peak` FROM w WHERE QA("map@prominence in ft?"; prominence) > 10000
+
+
+CREATE TABLE Daegu FC(
+	row_id int,
+	season int,
+	division int,
+	tms. int,
+	pos. int,
+	fa cup text,
+	afc cl text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	season	division	tms.	pos.	fa cup	afc cl
+0	2003	1	12	11	quarter final	none
+1	2004	1	13	10	round of 32	none
+2	2005	1	13	8	quarter final	none
+*/
+Q: how far did they make it in the fa cup after 2009?
+NeuralSQL: QA("ans@how far did they make?"; SELECT `fa cup` FROM w WHERE season > 2009)
+
+
+CREATE TABLE Electricity in Sri Lanka(
+	row_id int,
+	filledcolumnname text,
+	2005 int,
+	2006 int,
+	2007 int,
+	2008 int,
+	2009 int,
+	2010 int,
+	2011 int,
+	2012 int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	filledcolumnname	2005	2006	2007	2008	2009	2010	2011	2012
+0	hydro power	1293	1316	1326	1357	1379	1382	1401	1584
+1	thermal	1155	1155	1155	1285	1290	1390	1690	1638
+2	other renewables	3	3	3	3	15	45	50	90
+*/
+Q: did the hydro power increase or decrease from 2010 to 2012?
+NeuralSQL: SELECT CASE WHEN (SELECT `2010` FROM w WHERE filledcolumnname = 'hydro power') < (SELECT `2012` FROM w WHERE filledcolumnname = 'hydro power') THEN 'increase' ELSE 'decrease' END
+
+
+CREATE TABLE List of political parties in Japan(
+	row_id int,
+	party text,
+	diet representation\nrepresentatives int,
+	diet representation\ncouncillors int,
+	party leader(s) text,
+	comments text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	party	diet representation\nrepresentatives	diet representation\ncouncillors	party leader(s)	comments
+0	your party (yp); minna no tō みんなの党; ("everybody's party")	18	18	yoshimi watanabe reps.	conservative liberalism, neoliberalism, economic liberalism, libertarianism, anti-nuclear
+1	japanese communist party (jcp); nihon kyōsan-tō 日本共産党	8	11	kazuo shii reps.	the japanese communist party is japan's oldest party. it was formed in 1922 as an underground organization in the empire of japan, but was legalized after world war ii during the occupation. it used to be a communist party, but the party has past_ref shifted to a socialist party.
+2	people's life party (plp); seikatsu no tō 生活の党	7	2	ichirō ozawa reps.	life party was founded by ichirō ozawa and 14 other diet members who were in the 2022-7-4 party of japan after a leadership dispute between ozawa and yukiko kada.
+*/
+Q: what party is listed previous to the new renaissance party?
+NeuralSQL: SELECT QA("map@what is party name?"; party) FROM w WHERE row_id = (SELECT row_id FROM w WHERE QA("map@what is party name?"; party) = 'new renaissance party') - 1
+
+
+CREATE TABLE FC Seoul in Asian football(
+	row_id int,
+	# int,
+	season int,
+	competition text,
+	date text,
+	round text,
+	opponent text,
+	h / a text,
+	result text,
+	scorer (s) text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	#	season	competition	date	round	opponent	h / a	result	scorer (s)
+0	35	2011	afc; champions league	2011-03-02 00:00:00	group stage	al-ain	a	1–0	s : dejan damjanović
+1	36	2011	afc; champions league	2011-03-15 00:00:00	group stage	hangzhou greentown	h	3–0	s : dejan damjanović, ou kyoung-jun, mauricio molina
+2	37	2011	afc; champions league	2011-04-06 00:00:00	group stage	nagoya grampus	a	1–1	s : choi hyun-tae; n : kensuke nagai
+*/
+Q: how many consecutive games did dejan damjanovic score a goal in during the 2013 season?
+NeuralSQL: QA("ans@how many consecutive games did dejan damjanovic score a goal?"; SELECT `scorer (s)` FROM w WHERE season = 2013)
+
+
+CREATE TABLE Electoral district of Lachlan(
+	row_id int,
+	member text,
+	party text,
+	term text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	member	party	term
+0	john ryan	none	1859–1864
+1	james martin	none	1864–1869
+2	james watson	none	1869–1880
+*/
+Q: of the members of the third incarnation of the lachlan, who served the longest?
+NeuralSQL: SELECT member FROM w ORDER BY QA("map@how long does it last?"; term) DESC LIMIT 1
+
+
+CREATE TABLE Portugal in the Eurovision Song Contest 1979(
+	row_id int,
+	draw int,
+	artist text,
+	song text,
+	points int,
+	place text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	draw	artist	song	points	place
+0	1	gonzaga coutinho	"tema para um homem só"	102	5th
+1	2	pedro osório s.a.r.l.	"uma canção comercial"	123	3rd
+2	3	concha	"qualquer dia, quem diria"	78	6th
+*/
+Q: who was the last draw?
+NeuralSQL: SELECT `artist` FROM w ORDER by `draw` desc LIMIT 1
+
+
+CREATE TABLE GER Class N31(
+	row_id int,
+	year int,
+	order text,
+	quantity int,
+	ger nos. text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	year	order	quantity	ger nos.
+0	1893	n31	1	999
+1	1893	h33	10	979
+2	1894	l33	10	989
+*/
+Q: which had more ger numbers, 1898 or 1893?
+NeuralSQL: SELECT `year` FROM w WHERE `year` IN ( '1898' , '1893' ) GROUP by `year` ORDER by SUM (`ger nos.`) desc LIMIT 1
+
+
+CREATE TABLE List of spans(
+	row_id int,
+	tramway text,
+	country text,
+	city text,
+	height of pylons text,
+	span width,\nleaning straight line text,
+	span width,\nhorizontal measurement text,
+	height of cable over ground text,
+	year of inauguration text,
+	notes text)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	tramway	country	city	height of pylons	span width,\nleaning straight line	span width,\nhorizontal measurement	height of cable over ground	year of inauguration	notes
+0	peak 2 peak gondola	canada	whistler	65m	3024 m	3019 m	436 m	2008	3s aerial tramway constructed by doppelmayr
+1	hut of regensburg material transport aerial railway	austria	falbeson	?	?	?	430 m	?	none
+2	vanoise express	france	vanoise	none	1850 m	1800 m	380 m	2003	none
+*/
+Q: was the sandia peak tramway innagurate before or after the 3s aerial tramway?
+NeuralSQL: SELECT ( SELECT `year of inauguration` FROM w WHERE `tramway` = 'sandia peak tramway' ) < ( SELECT `year of inauguration` FROM w WHERE `tramway` = '3s aerial tramway' )
+
+
+CREATE TABLE Płock Governorate(
+	row_id int,
+	language text,
+	number int,
+	percentage (%) text,
+	males int,
+	females int)
+/*
+3 example rows:
+SELECT * FROM w LIMIT 3;
+row_id	language	number	percentage (%)	males	females
+0	polish	447685	80.86	216794	230891
+1	yiddish	51215	9.25	24538	26677
+2	german	35931	6.49	17409	18522
+*/
+Q: how many male and female german speakers are there?
+NeuralSQL: SELECT `males` + `females` FROM w WHERE `language` = 'german'

templates/qa_retrieve_pool.json

@@ -0,0 +1,3885 @@
+[
+  {
+    "id": 142,
+    "qa_question": "map@What is the time span?",
+    "qa_column": "Term",
+    "qa_answer": [
+      "5",
+      "5",
+      "11",
+      "/",
+      "1",
+      "6",
+      "3",
+      "9",
+      "4",
+      "3",
+      "/",
+      "11",
+      "5",
+      "4",
+      "3",
+      "/"
+    ],
+    "table": {
+      "header": [
+        "Term"
+      ],
+      "rows": [
+        [
+          "1859\u20131864"
+        ],
+        [
+          "1864\u20131869"
+        ],
+        [
+          "1869\u20131880"
+        ],
+        [
+          "Term"
+        ],
+        [
+          "1894\u20131895"
+        ],
+        [
+          "1895\u20131901"
+        ],
+        [
+          "1901\u20131904"
+        ],
+        [
+          "1904\u20131913"
+        ],
+        [
+          "1913\u20131917"
+        ],
+        [
+          "1917\u20131920"
+        ],
+        [
+          "Term"
+        ],
+        [
+          "1927\u20131938"
+        ],
+        [
+          "1938\u20131943"
+        ],
+        [
+          "1943\u20131947"
+        ],
+        [
+          "1947\u20131950"
+        ],
+        [
+          "Term"
+        ]
+      ]
+    },
+    "title": "Electoral district of Lachlan"
+  },
+  {
+    "id": 145,
+    "qa_question": "map@Is the date during in 1900's?",
+    "qa_column": "Created",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Created"
+      ],
+      "rows": [
+        [
+          "29 October 1874"
+        ],
+        [
+          "2 January 1857"
+        ],
+        [
+          "26 October 1874"
+        ],
+        [
+          "15 July 1949"
+        ],
+        [
+          "4 August 1821"
+        ],
+        [
+          "24 April 1940"
+        ],
+        [
+          "2 January 1857"
+        ],
+        [
+          "3 March 1970"
+        ],
+        [
+          "12 December 1961"
+        ],
+        [
+          "6 January 1965"
+        ],
+        [
+          "16 March 1964"
+        ],
+        [
+          "13 December 1963"
+        ],
+        [
+          "6 February 1962"
+        ],
+        [
+          "16 August 1921"
+        ],
+        [
+          "2 January 1857"
+        ]
+      ]
+    },
+    "title": "List of districts of Lima"
+  },
+  {
+    "id": 155,
+    "qa_question": "map@Is the time less than a week?",
+    "qa_column": "Length of use",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Length of use"
+      ],
+      "rows": [
+        [
+          "14 days"
+        ],
+        [
+          "10 days"
+        ],
+        [
+          "21 days"
+        ],
+        [
+          "7 days"
+        ],
+        [
+          "10 days"
+        ],
+        [
+          "10 days"
+        ],
+        [
+          "Daily"
+        ],
+        [
+          "14 days"
+        ],
+        [
+          "10 days"
+        ],
+        [
+          "14 days"
+        ],
+        [
+          "20 days"
+        ],
+        [
+          "2 hours"
+        ]
+      ]
+    },
+    "title": "Crest Whitestrips"
+  },
+  {
+    "id": 157,
+    "qa_question": "map@Is the eliminated time the first?",
+    "qa_column": "Eliminated",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Eliminated"
+      ],
+      "rows": [
+        [
+          "Winner"
+        ],
+        [
+          "Second Place"
+        ],
+        [
+          "Third Place"
+        ],
+        [
+          "Week 10"
+        ],
+        [
+          "Week 9"
+        ],
+        [
+          "Week 7"
+        ],
+        [
+          "Week 6"
+        ],
+        [
+          "Week 5"
+        ],
+        [
+          "Week 4 & Week 8(Winner of Star Salvation)"
+        ],
+        [
+          "Week 3"
+        ],
+        [
+          "Week 2"
+        ]
+      ]
+    },
+    "title": "Food Network Star"
+  },
+  {
+    "id": 200,
+    "qa_question": "map@Is it in march 1982?",
+    "qa_column": "Date",
+    "qa_answer": [
+      "no",
+      "yes",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Date"
+      ],
+      "rows": [
+        [
+          "January 26, 1982"
+        ],
+        [
+          "March 3, 1982"
+        ],
+        [
+          "March 21, 1982"
+        ],
+        [
+          "May 5, 1982"
+        ],
+        [
+          "May 19, 1982"
+        ],
+        [
+          "May 27, 1982"
+        ],
+        [
+          "June 14, 1982"
+        ],
+        [
+          "June 18, 1982"
+        ],
+        [
+          "June 23, 1982"
+        ],
+        [
+          "July 2, 1982"
+        ]
+      ]
+    },
+    "title": "1982 in Brazilian football"
+  },
+  {
+    "id": 578,
+    "qa_question": "map@Is this player from Norway?",
+    "qa_column": "Player",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Player"
+      ],
+      "rows": [
+        [
+          "Raymond van Barneveld"
+        ],
+        [
+          "Raymond van Barneveld"
+        ],
+        [
+          "Adrian Lewis"
+        ],
+        [
+          "Dean Winstanley"
+        ],
+        [
+          "Michael van Gerwen"
+        ],
+        [
+          "Terry Jenkins"
+        ]
+      ]
+    },
+    "title": "PDC World Darts Championship"
+  },
+  {
+    "id": 624,
+    "qa_question": "map@Is it designated?",
+    "qa_column": "Artist(s)",
+    "qa_answer": [
+      "no",
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Artist(s)"
+      ],
+      "rows": [
+        [
+          "various artists"
+        ],
+        [
+          "Robin"
+        ],
+        [
+          "various artists"
+        ],
+        [
+          "Robin"
+        ],
+        [
+          "Adele"
+        ],
+        [
+          "Jukka Poika"
+        ],
+        [
+          "Jesse Kaikuranta"
+        ],
+        [
+          "Chisu"
+        ],
+        [
+          "Juha Tapio"
+        ]
+      ]
+    },
+    "title": "List of number-one albums of 2012 (Finland)"
+  },
+  {
+    "id": 548,
+    "qa_question": "map@What is the winner's name?",
+    "qa_column": "Winner",
+    "qa_answer": [
+      "Jacinto Sicam",
+      "Romeo Bonzo",
+      "Ruben Carino",
+      "Pepito Calip",
+      "Rolando Pagnanawon",
+      "Reynaldo Dequito",
+      "Armando Catalan",
+      "Gerardo Igos",
+      "Manuel Buenaventura",
+      "Bernardo Llentada",
+      "Renato Dolosa",
+      "Carlo Guieb",
+      "Carlo Guieb",
+      "Renato Dolosa",
+      "Victor Espiritu",
+      "Wong Kam-po",
+      "Warren Davadilla"
+    ],
+    "table": {
+      "header": [
+        "Winner"
+      ],
+      "rows": [
+        [
+          "Jacinto Sicam\u00a0(PHI)"
+        ],
+        [
+          "Romeo Bonzo\u00a0(PHI)"
+        ],
+        [
+          "Ruben Carino\u00a0(PHI)"
+        ],
+        [
+          "Pepito Calip\u00a0(PHI)"
+        ],
+        [
+          "Rolando Pagnanawon\u00a0(PHI)"
+        ],
+        [
+          "Reynaldo Dequito\u00a0(PHI)"
+        ],
+        [
+          "Armando Catalan\u00a0(PHI)"
+        ],
+        [
+          "Gerardo Igos\u00a0(PHI)"
+        ],
+        [
+          "Manuel Buenaventura\u00a0(PHI)"
+        ],
+        [
+          "Bernardo Llentada\u00a0(PHI)"
+        ],
+        [
+          "Renato Dolosa\u00a0(PHI)"
+        ],
+        [
+          "Carlo Guieb\u00a0(PHI)"
+        ],
+        [
+          "Carlo Guieb\u00a0(PHI)"
+        ],
+        [
+          "Renato Dolosa\u00a0(PHI)"
+        ],
+        [
+          "Victor Espiritu\u00a0(PHI)"
+        ],
+        [
+          "Wong Kam-po\u00a0(HKG)"
+        ],
+        [
+          "Warren Davadilla\u00a0(PHI)"
+        ]
+      ]
+    },
+    "title": "Le Tour de Filipinas"
+  },
+  {
+    "id": 149,
+    "qa_question": "map@What is the last name of the whole name?",
+    "qa_column": "Performer",
+    "qa_answer": [
+      "Segura",
+      "Kosta",
+      "Wang",
+      "Peretti",
+      "D'Elia",
+      "Bargatze",
+      "Kondabolu",
+      "Whitehall",
+      "Jackson",
+      "Kinane",
+      "Fulchiron",
+      "Vecchione",
+      "Klein",
+      "Katz",
+      "Larson"
+    ],
+    "table": {
+      "header": [
+        "Performer"
+      ],
+      "rows": [
+        [
+          "Tom Segura"
+        ],
+        [
+          "Michael Kosta"
+        ],
+        [
+          "Sheng Wang"
+        ],
+        [
+          "Chelsea Peretti"
+        ],
+        [
+          "Chris D'Elia"
+        ],
+        [
+          "Nate Bargatze"
+        ],
+        [
+          "Hari Kondabolu"
+        ],
+        [
+          "Jack Whitehall"
+        ],
+        [
+          "Al Jackson"
+        ],
+        [
+          "Kyle Kinane"
+        ],
+        [
+          "Matt Fulchiron"
+        ],
+        [
+          "Mike Vecchione"
+        ],
+        [
+          "Jessi Klein"
+        ],
+        [
+          "Louis Katz"
+        ],
+        [
+          "Jay Larson"
+        ]
+      ]
+    },
+    "title": "List of Comedy Central Presents episodes"
+  },
+  {
+    "id": 159,
+    "qa_question": "map@What is the name outside paratheses?",
+    "qa_column": "Name",
+    "qa_answer": [
+      "Harriet Churchill",
+      "Henrietta Churchill",
+      "Anne Churchill",
+      "John Churchill",
+      "Elizabeth Churchill",
+      "Mary Churchill"
+    ],
+    "table": {
+      "header": [
+        "Name"
+      ],
+      "rows": [
+        [
+          "Harriet Churchill"
+        ],
+        [
+          "Henrietta Churchill (later Godolphin), 2nd Duchess of Marlborough in her own right"
+        ],
+        [
+          "Anne Churchill (later Spencer)"
+        ],
+        [
+          "John Churchill, Marquess of Blandford"
+        ],
+        [
+          "Elizabeth Churchill (later Egerton)"
+        ],
+        [
+          "Mary Churchill (later Montagu)"
+        ]
+      ]
+    },
+    "title": "Sarah Churchill, Duchess of Marlborough"
+  },
+  {
+    "id": 208,
+    "qa_question": "map@Did Kansas State win?",
+    "qa_column": "Winning team",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Winning team"
+      ],
+      "rows": [
+        [
+          "Kansas 16"
+        ],
+        [
+          "Kansas 34"
+        ],
+        [
+          "Kansas 41"
+        ],
+        [
+          "Kansas 28"
+        ],
+        [
+          "Kansas State 6"
+        ],
+        [
+          "Kansas 29"
+        ],
+        [
+          "Kansas 12"
+        ],
+        [
+          "Kansas 5"
+        ],
+        [
+          "Kansas 6"
+        ],
+        [
+          "Kansas 19"
+        ],
+        [
+          "Kansas 26"
+        ],
+        [
+          "Kansas 27"
+        ],
+        [
+          "Kansas 19"
+        ],
+        [
+          "Kansas 0"
+        ],
+        [
+          "Kansas 9"
+        ],
+        [
+          "Kansas 13"
+        ],
+        [
+          "Kansas 16"
+        ],
+        [
+          "Kansas 14"
+        ],
+        [
+          "Kansas 21"
+        ],
+        [
+          "Kansas 7"
+        ],
+        [
+          "Kansas 0"
+        ],
+        [
+          "Kansas State 6"
+        ],
+        [
+          "Kansas State 14"
+        ]
+      ]
+    },
+    "title": "Kansas\u2013Kansas State football rivalry"
+  },
+  {
+    "id": 224,
+    "qa_question": "map@Is the last name start with 'b'?",
+    "qa_column": "Name",
+    "qa_answer": [
+      "yes",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Name"
+      ],
+      "rows": [
+        [
+          "Yelizaveta Bryzhina"
+        ],
+        [
+          "Myriam Soumar\u00e9"
+        ],
+        [
+          "Ksenija Balta"
+        ],
+        [
+          "Niamh Whelan"
+        ],
+        [
+          "Sabina Veit"
+        ],
+        [
+          "Elin Backman"
+        ]
+      ]
+    },
+    "title": "2010 European Athletics Championships \u2013 Women's 200 metres"
+  },
+  {
+    "id": 528,
+    "qa_question": "map@Return the km value.",
+    "qa_column": "Isolation",
+    "qa_answer": [
+      "2649",
+      "8.70",
+      "109",
+      "52",
+      "539",
+      "24",
+      "5",
+      "11",
+      "17",
+      "16",
+      "6.9",
+      "7.7",
+      "16",
+      "7.6",
+      "13"
+    ],
+    "table": {
+      "header": [
+        "Isolation"
+      ],
+      "rows": [
+        [
+          "1,646\u00a0mi\\n2,649\u00a0km"
+        ],
+        [
+          "5.4\u00a0mi\\n8.7\u00a0km"
+        ],
+        [
+          "67\u00a0mi\\n109\u00a0km"
+        ],
+        [
+          "32\u00a0mi\\n52\u00a0km"
+        ],
+        [
+          "335\u00a0mi\\n539\u00a0km"
+        ],
+        [
+          "15\u00a0mi\\n24\u00a0km"
+        ],
+        [
+          "3.1\u00a0mi\\n5.0\u00a0km"
+        ],
+        [
+          "7\u00a0mi\\n11\u00a0km"
+        ],
+        [
+          "11\u00a0mi\\n17\u00a0km"
+        ],
+        [
+          "10\u00a0mi\\n16\u00a0km"
+        ],
+        [
+          "4.3\u00a0mi\\n6.9\u00a0km"
+        ],
+        [
+          "4.8\u00a0mi\\n7.7\u00a0km"
+        ],
+        [
+          "10\u00a0mi\\n16\u00a0km"
+        ],
+        [
+          "4.7\u00a0mi\\n7.6\u00a0km"
+        ],
+        [
+          "8\u00a0mi\\n13\u00a0km"
+        ]
+      ]
+    },
+    "title": "Highest mountain peaks of California"
+  },
+  {
+    "id": 212,
+    "qa_question": "map@Can it go faster than 450km/h?",
+    "qa_column": "Top speed (km/h)",
+    "qa_answer": [
+      "/",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "/",
+      "no",
+      "no",
+      "no",
+      "no",
+      "/",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Top speed (km/h)"
+      ],
+      "rows": [
+        [
+          ""
+        ],
+        [
+          "90 (1971)"
+        ],
+        [
+          "164 (October 1971)"
+        ],
+        [
+          "140 (September 1972)"
+        ],
+        [
+          "160 / 230 (1974)\u00a0?"
+        ],
+        [
+          "250 (end 1973), 253.2 (21 November 1977)"
+        ],
+        [
+          "401.3 (1974)"
+        ],
+        [
+          ""
+        ],
+        [
+          "36 (or 40\u00a0?)"
+        ],
+        [
+          "75"
+        ],
+        [
+          "302 (1984), 355 (1985), 392 (1987), 406 (1987), 412.6 (January 1988)"
+        ],
+        [
+          "436 (1989), 450 (17 June 1993)"
+        ],
+        [
+          ""
+        ],
+        [
+          "501 (12 November 2003)"
+        ]
+      ]
+    },
+    "title": "Transrapid"
+  },
+  {
+    "id": 223,
+    "qa_question": "map@What is the price money?",
+    "qa_column": "Tournament",
+    "qa_answer": [
+      "10000",
+      "10000",
+      "25000",
+      "25000",
+      "10000",
+      "25000",
+      "10000",
+      "10000",
+      "10000",
+      "10000"
+    ],
+    "table": {
+      "header": [
+        "Tournament"
+      ],
+      "rows": [
+        [
+          "$10,000 Bournemouth, Great Britain"
+        ],
+        [
+          "$10,000 Hatfield, Great Britain"
+        ],
+        [
+          "$25,000 Mount Gambier, Australia"
+        ],
+        [
+          "$25,000 Sunderland, Great Britain"
+        ],
+        [
+          "$10,000 Tipton, Great Britain"
+        ],
+        [
+          "$25,000 Felixstowe, Great Britain"
+        ],
+        [
+          "$10,000 Frinton, Great Britain"
+        ],
+        [
+          "$10,000 Wrexham, Great Britain"
+        ],
+        [
+          "$10,000 Cumberland (London),\\nGreat Britain"
+        ],
+        [
+          "$10,000 Mollerusa, Spain"
+        ]
+      ]
+    },
+    "title": "Jane O'Donoghue"
+  },
+  {
+    "id": 227,
+    "qa_question": "map@What is the height in inch?",
+    "qa_column": "Height",
+    "qa_answer": [
+      "77",
+      "78",
+      "74",
+      "79",
+      "74",
+      "80",
+      "77",
+      "82",
+      "80",
+      "78",
+      "82",
+      "76"
+    ],
+    "table": {
+      "header": [
+        "Height"
+      ],
+      "rows": [
+        [
+          "6'5\""
+        ],
+        [
+          "6'6\""
+        ],
+        [
+          "6'2\""
+        ],
+        [
+          "6'7\""
+        ],
+        [
+          "6'2\""
+        ],
+        [
+          "6'8\""
+        ],
+        [
+          "6'5\""
+        ],
+        [
+          "6'10\""
+        ],
+        [
+          "6'8\""
+        ],
+        [
+          "6'6\""
+        ],
+        [
+          "6'10\""
+        ],
+        [
+          "6'4\""
+        ]
+      ]
+    },
+    "title": "2009\u201310 Fresno State Bulldogs men's basketball team"
+  },
+  {
+    "id": 579,
+    "qa_question": "map@Is it US Bank Plaza?",
+    "qa_column": "Name",
+    "qa_answer": [
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Name"
+      ],
+      "rows": [
+        [
+          "Zions Bank Building\\n\\n\\n\\n\\nZions Bank Building in Downtown Boise, Idaho"
+        ],
+        [
+          "Boise Airport Air Traffic Control Tower"
+        ],
+        [
+          "US Bank Plaza\\n\\n\\n\\n\\nUS Bank Building in Downtown Boise"
+        ],
+        [
+          "One Capital Center\\n\\n\\n\\n\\nOne Capital Center in Downtown Boise, Idaho"
+        ],
+        [
+          "Idaho State Capitol\\n\\n\\n\\n\\nIdaho State Capitol; Boise, Idaho"
+        ],
+        [
+          "The Grove Hotel\\n\\n\\n\\n\\nGrove Hotel in Downtown Boise, Idaho"
+        ],
+        [
+          "The Aspen\\n\\n\\n\\n\\nAspen Loft Building in Downtown Boise, Idaho"
+        ],
+        [
+          "Wells Fargo Building\\n\\n\\n\\n\\nWells Fargo Building in Downtown Boise, Idaho"
+        ],
+        [
+          "Banner Bank Building\\n\\n\\n\\n\\nBanner Bank Building in Downtown Boise, Idaho"
+        ],
+        [
+          "Key Tower\\n\\n\\n\\n\\nKey Bank Building in Downtown Boise, Idaho"
+        ],
+        [
+          "Bronco Stadium\\n\\n\\n\\n\\nBronco stadium in Boise, Idaho"
+        ],
+        [
+          "Hoff Building\\n\\n\\n\\n\\nHoff Building in Downtown Boise, Idaho"
+        ],
+        [
+          "Chase Tower Plaza\\n\\n\\n\\n\\nChase Building in Downtown Boise, Idaho"
+        ]
+      ]
+    },
+    "title": "List of tallest buildings in Boise"
+  },
+  {
+    "id": 160,
+    "qa_question": "map@is this model crest whitestrip classic?",
+    "qa_column": "Model",
+    "qa_answer": [
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Model"
+      ],
+      "rows": [
+        [
+          "Crest Whitestrips Classic\\npreviously Crest Whitestrips"
+        ],
+        [
+          "Crest Whitestrips Professional"
+        ],
+        [
+          "Crest Whitestrips Supreme"
+        ],
+        [
+          "Crest Whitestrips Premium"
+        ],
+        [
+          "Crest Whitestrips Pro\\npreviously Crest Whitestrips Premium Plus"
+        ],
+        [
+          "Crest Whitestrips Renewal"
+        ],
+        [
+          "Crest Whitestrips Daily Multicare"
+        ],
+        [
+          "Crest Whitestrips Advanced Seal"
+        ],
+        [
+          "Crest Whitestrips 3D Vivid"
+        ],
+        [
+          "Crest Whitestrips 3D Advanced Vivid"
+        ],
+        [
+          "Crest Whitestrips 3D Professional Effects"
+        ],
+        [
+          "Crest 3D White 2 Hour Express"
+        ]
+      ]
+    },
+    "title": "Crest Whitestrips"
+  },
+  {
+    "id": 216,
+    "qa_question": "map@What is the certification?",
+    "qa_column": "Certifications\\n(sales threshold)",
+    "qa_answer": [
+      "Platinum",
+      "Platinum",
+      "Platinum",
+      "Gold",
+      "Gold"
+    ],
+    "table": {
+      "header": [
+        "Certifications\\n(sales threshold)"
+      ],
+      "rows": [
+        [
+          "RIAA: Platinum\\nMC: Gold"
+        ],
+        [
+          "RIAA: Platinum"
+        ],
+        [
+          "RIAA: Platinum"
+        ],
+        [
+          "RIAA: Gold"
+        ],
+        [
+          "RIAA: Gold"
+        ]
+      ]
+    },
+    "title": "Michael W. Smith discography"
+  },
+  {
+    "id": 220,
+    "qa_question": "map@Contain Promotion of the Chernobyl Program?",
+    "qa_column": "Projects and activities they support",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "/"
+    ],
+    "table": {
+      "header": [
+        "Projects and activities they support"
+      ],
+      "rows": [
+        [
+          "Dialogue among Civilizations"
+        ],
+        [
+          "Construction of Knowledge Societies"
+        ],
+        [
+          "Promotion and safeguarding of intangible cultural heritage, especially oral traditions and expressions"
+        ],
+        [
+          "Promotion of ballet dancing (Programme of Intangible Heritage)"
+        ],
+        [
+          "Peace"
+        ],
+        [
+          "Education of young people through seminars, science conferences and projects in the field"
+        ],
+        [
+          "Campaign against HIV/AIDS, human rights, Cultural Heritage"
+        ],
+        [
+          "Education and Social Inclusion"
+        ],
+        [
+          "Fundraising for children in distress and victims of war"
+        ],
+        [
+          "Promotion of the Chernobyl Program, creation of the Six Flags of Tolerance in 1995 and distribution in UNESCO Member States"
+        ],
+        [
+          "Promotion of women's rights, especially for women in the Mediterranean; Environment issues"
+        ],
+        [
+          "Protection of children and the family, empowerment of women and girls in Africa"
+        ],
+        [
+          ""
+        ]
+      ]
+    },
+    "title": "UNESCO Goodwill Ambassador"
+  },
+  {
+    "id": 226,
+    "qa_question": "map@Is this party the Green Wind Party?",
+    "qa_column": "Party",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Party"
+      ],
+      "rows": [
+        [
+          "Your Party (YP)\\nMinna no T\u014d \u307f\u3093\u306a\u306e\u515a\\n(\"Everybody's Party\")"
+        ],
+        [
+          "Japanese Communist Party (JCP)\\nNihon Ky\u014dsan-t\u014d \u65e5\u672c\u5171\u7523\u515a"
+        ],
+        [
+          "People's Life Party (PLP)\\nSeikatsu no T\u014d \u751f\u6d3b\u306e\u515a"
+        ],
+        [
+          "Social Democratic Party (SDP)\\nShakai Minshu-t\u014d \u793e\u4f1a\u6c11\u4e3b\u515a"
+        ],
+        [
+          "Green Wind\\nMidori no Kaze \u307f\u3069\u308a\u306e\u98a8"
+        ],
+        [
+          "New Party Daichi \u2013 True Democrats\\nShint\u014d Daichi \u2013 Shinminshu \u65b0\u515a\u5927\u5730\u30fb\u771f\u6c11\u4e3b"
+        ],
+        [
+          "New Renaissance Party (NRP)\\nShint\u014d Kaikaku \u65b0\u515a\u6539\u9769\\n(\"New Reform Party\")"
+        ]
+      ]
+    },
+    "title": "List of political parties in Japan"
+  },
+  {
+    "id": 229,
+    "qa_question": "map@Is the language derived from a country?",
+    "qa_column": "Language",
+    "qa_answer": [
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "yes",
+      "/",
+      "/"
+    ],
+    "table": {
+      "header": [
+        "Language"
+      ],
+      "rows": [
+        [
+          "Polish"
+        ],
+        [
+          "Yiddish"
+        ],
+        [
+          "German"
+        ],
+        [
+          "Russian"
+        ],
+        [
+          "Ukrainian"
+        ],
+        [
+          "Other"
+        ],
+        [
+          "Persons\\nthat didn't name\\ntheir native language"
+        ]
+      ]
+    },
+    "title": "P\u0142ock Governorate"
+  },
+  {
+    "id": 206,
+    "qa_question": "map@What is the year?",
+    "qa_column": "Notes",
+    "qa_answer": [
+      "1933",
+      "1933",
+      "1941",
+      "1941",
+      "1932",
+      "1932",
+      "1932",
+      "1932",
+      "1932",
+      "1932",
+      "1929",
+      "1929"
+    ],
+    "table": {
+      "header": [
+        "Notes"
+      ],
+      "rows": [
+        [
+          "retired 1933"
+        ],
+        [
+          "retired 1933"
+        ],
+        [
+          "became inspection car in 1932 retired 1941"
+        ],
+        [
+          "became inspection car in 1932 retired 1941"
+        ],
+        [
+          "retired 1932"
+        ],
+        [
+          "retired 1932"
+        ],
+        [
+          "retired 1932 preserved Western Railway Museum"
+        ],
+        [
+          "retired 1932"
+        ],
+        [
+          "retired 1932"
+        ],
+        [
+          "converted to express trailer in 1919 retired 1932"
+        ],
+        [
+          "retired 1929"
+        ],
+        [
+          "retired 1929"
+        ]
+      ]
+    },
+    "title": "Petaluma and Santa Rosa Railroad"
+  },
+  {
+    "id": 613,
+    "qa_question": "map@What is the month?",
+    "qa_column": "Dates",
+    "qa_answer": [
+      "17 Jan",
+      "24 Jan",
+      "31 Jan",
+      "7 Feb",
+      "14 Feb",
+      "20 Feb",
+      "28 Feb",
+      "7 Mar",
+      "14 Mar",
+      "28 Mar",
+      "11 Apr",
+      "18 Apr",
+      "25 Apr",
+      "2 May",
+      "9 May",
+      "16 May",
+      "24 May",
+      "31 May",
+      "6 Jun"
+    ],
+    "table": {
+      "header": [
+        "Dates"
+      ],
+      "rows": [
+        [
+          "14\u201317\u00a0Jan"
+        ],
+        [
+          "21\u201324\u00a0Jan"
+        ],
+        [
+          "28\u201331\u00a0Jan"
+        ],
+        [
+          "4\u20137\u00a0Feb"
+        ],
+        [
+          "11\u201314\u00a0Feb"
+        ],
+        [
+          "17\u201320\u00a0Feb"
+        ],
+        [
+          "24\u201328\u00a0Feb"
+        ],
+        [
+          "4\u20137\u00a0Mar"
+        ],
+        [
+          "11\u201314\u00a0Mar"
+        ],
+        [
+          "25\u201328\u00a0Mar"
+        ],
+        [
+          "8\u201311\u00a0Apr"
+        ],
+        [
+          "15\u201318\u00a0Apr"
+        ],
+        [
+          "22\u201325\u00a0Apr"
+        ],
+        [
+          "29\u00a0Apr\u20132\u00a0May"
+        ],
+        [
+          "6\u20139\u00a0May"
+        ],
+        [
+          "13\u201316\u00a0May"
+        ],
+        [
+          "21\u201324\u00a0May"
+        ],
+        [
+          "28\u201331\u00a0May"
+        ],
+        [
+          "3\u20136\u00a0Jun"
+        ]
+      ]
+    },
+    "title": "1999 European Tour"
+  },
+  {
+    "id": 590,
+    "qa_question": "map@Is it below 2:01",
+    "qa_column": "Time",
+    "qa_answer": [
+      "yes",
+      "yes",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Time"
+      ],
+      "rows": [
+        [
+          "2:00.06"
+        ],
+        [
+          "2:00.11"
+        ],
+        [
+          "2:00.77"
+        ],
+        [
+          "2:01.39"
+        ],
+        [
+          "2:01.53"
+        ],
+        [
+          "2:01.86"
+        ],
+        [
+          "2:02.64"
+        ]
+      ]
+    },
+    "title": "Athletics at the 2003 Summer Universiade \u2013 Women's 800 metres"
+  },
+  {
+    "id": 543,
+    "qa_question": "map@Is it april fools day?",
+    "qa_column": "Date signed",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "/",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Date signed"
+      ],
+      "rows": [
+        [
+          "March 29"
+        ],
+        [
+          "August 11"
+        ],
+        [
+          "March 27"
+        ],
+        [
+          "April 1"
+        ],
+        [
+          "April 10"
+        ],
+        [
+          "March 25"
+        ],
+        [
+          "June 25"
+        ],
+        [
+          "\u2013"
+        ],
+        [
+          "March 13"
+        ],
+        [
+          "March 15"
+        ],
+        [
+          "March 15"
+        ],
+        [
+          "March 19"
+        ],
+        [
+          "July 29"
+        ]
+      ]
+    },
+    "title": "2013 Chicago Bears season"
+  },
+  {
+    "id": 207,
+    "qa_question": "map@What is the whole points scored?",
+    "qa_column": "Result",
+    "qa_answer": [
+      "8",
+      "5",
+      "5",
+      "18",
+      "5",
+      "5",
+      "5",
+      "7",
+      "7",
+      "13",
+      "/",
+      "22",
+      "15",
+      "16",
+      "29",
+      "17",
+      "4",
+      "10",
+      "8"
+    ],
+    "table": {
+      "header": [
+        "Result"
+      ],
+      "rows": [
+        [
+          "2\u20133\\n3\u20130"
+        ],
+        [
+          "2\u20133"
+        ],
+        [
+          "5\u20130"
+        ],
+        [
+          "0\u20130\\n5\u20130\\n13\u20130"
+        ],
+        [
+          "2\u20133 (AET)"
+        ],
+        [
+          "1\u20130\\n0\u20131\\n2\u20131"
+        ],
+        [
+          "1\u20134"
+        ],
+        [
+          "1\u20130\\n3\u20130\\n0\u20133"
+        ],
+        [
+          "0\u20137"
+        ],
+        [
+          "0\u20132\\n0\u20130\\n6\u20130\\n1\u20134"
+        ],
+        [
+          "not played"
+        ],
+        [
+          "0\u20131 1\u20130\\n12\u20130 8\u20130"
+        ],
+        [
+          "1\u20132 1\u20130\\n7\u20130 4\u20130"
+        ],
+        [
+          "0\u20135\\n2\u20134\\n1\u20134"
+        ],
+        [
+          "21\u20130\\n8\u20130"
+        ],
+        [
+          "17\u20130"
+        ],
+        [
+          "3\u20131"
+        ],
+        [
+          "1\u20131\\n1\u20133\\n1\u20133"
+        ],
+        [
+          "0\u20133\\n1\u20131\\n1\u20132"
+        ]
+      ]
+    },
+    "title": "Australia women's national association football team"
+  },
+  {
+    "id": 0,
+    "qa_question": "map@Did Kansas State lost with 0 points?",
+    "qa_column": "Losing team",
+    "qa_answer": [
+      "yes",
+      "yes",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "no",
+      "no",
+      "yes",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Losing team"
+      ],
+      "rows": [
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 4"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas 4"
+        ],
+        [
+          "Kansas State 10"
+        ],
+        [
+          "Kansas State 6"
+        ],
+        [
+          "Kansas State 3"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 6"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 7"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 7"
+        ],
+        [
+          "Kansas State 3"
+        ],
+        [
+          "Kansas State 0"
+        ],
+        [
+          "Kansas State 7"
+        ]
+      ]
+    },
+    "title": "Kansas\u2013Kansas State football rivalry"
+  },
+  {
+    "id": 5,
+    "qa_question": "map@What is the value of in feet?",
+    "qa_column": "Prominence",
+    "qa_answer": [
+      "10080",
+      "1677",
+      "7196",
+      "2894",
+      "9832",
+      "2563",
+      "1936",
+      "1891",
+      "2027",
+      "2648",
+      "2601",
+      "1992",
+      "2339",
+      "2110",
+      "1736"
+    ],
+    "table": {
+      "header": [
+        "Prominence"
+      ],
+      "rows": [
+        [
+          "10,080\u00a0ft\\n3072\u00a0m"
+        ],
+        [
+          "1,677\u00a0ft\\n511\u00a0m"
+        ],
+        [
+          "7,196\u00a0ft\\n2193\u00a0m"
+        ],
+        [
+          "2,894\u00a0ft\\n882\u00a0m"
+        ],
+        [
+          "9,832\u00a0ft\\n2997\u00a0m"
+        ],
+        [
+          "2,563\u00a0ft\\n781\u00a0m"
+        ],
+        [
+          "1,936\u00a0ft\\n590\u00a0m"
+        ],
+        [
+          "1,891\u00a0ft\\n576\u00a0m"
+        ],
+        [
+          "2,027\u00a0ft\\n618\u00a0m"
+        ],
+        [
+          "2,648\u00a0ft\\n807\u00a0m"
+        ],
+        [
+          "2,601\u00a0ft\\n793\u00a0m"
+        ],
+        [
+          "1,992\u00a0ft\\n607\u00a0m"
+        ],
+        [
+          "2,339\u00a0ft\\n713\u00a0m"
+        ],
+        [
+          "2,110\u00a0ft\\n643\u00a0m"
+        ],
+        [
+          "1,736\u00a0ft\\n529\u00a0m"
+        ]
+      ]
+    },
+    "title": "Highest mountain peaks of California"
+  },
+  {
+    "id": 12,
+    "qa_question": "map@What is the height in ft before / ?",
+    "qa_column": "Height\\nft / m",
+    "qa_answer": [
+      "629",
+      "555",
+      "530",
+      "512",
+      "503",
+      "485",
+      "464",
+      "456",
+      "438",
+      "408",
+      "366",
+      "357",
+      "350",
+      "348",
+      "317",
+      "314",
+      "302",
+      "286",
+      "280",
+      "267",
+      "260",
+      "260",
+      "256",
+      "253",
+      "243",
+      "226",
+      "212",
+      "202",
+      "200"
+    ],
+    "table": {
+      "header": [
+        "Height\\nft / m"
+      ],
+      "rows": [
+        [
+          "629 / 192"
+        ],
+        [
+          "555 / 169"
+        ],
+        [
+          "530 / 162"
+        ],
+        [
+          "512 / 156"
+        ],
+        [
+          "503 / 153"
+        ],
+        [
+          "485 / 148"
+        ],
+        [
+          "464 / 141"
+        ],
+        [
+          "456 / 139"
+        ],
+        [
+          "438 / 134"
+        ],
+        [
+          "408 / 124"
+        ],
+        [
+          "366 / 112"
+        ],
+        [
+          "357 / 109"
+        ],
+        [
+          "350 / 107"
+        ],
+        [
+          "348 / 106"
+        ],
+        [
+          "317 / 97"
+        ],
+        [
+          "314 / 96"
+        ],
+        [
+          "302 / 92"
+        ],
+        [
+          "286 / 87"
+        ],
+        [
+          "280 / 85"
+        ],
+        [
+          "267 / 81"
+        ],
+        [
+          "260 / 79"
+        ],
+        [
+          "260 / 79"
+        ],
+        [
+          "256 / 78"
+        ],
+        [
+          "253 / 77"
+        ],
+        [
+          "243 / 74"
+        ],
+        [
+          "226 / 69"
+        ],
+        [
+          "212 / 64.6"
+        ],
+        [
+          "202 / 59.4"
+        ],
+        [
+          "200 / 57.9"
+        ]
+      ]
+    },
+    "title": "List of tallest buildings in Columbus, Ohio"
+  },
+  {
+    "id": 13,
+    "qa_question": "map@The number of series?",
+    "qa_column": "DVD Title",
+    "qa_answer": [
+      "1",
+      "2",
+      "3",
+      "4",
+      "5",
+      "6",
+      "7",
+      "8",
+      "9",
+      "10",
+      "11",
+      "12",
+      "/",
+      "/",
+      "/",
+      "/"
+    ],
+    "table": {
+      "header": [
+        "DVD Title"
+      ],
+      "rows": [
+        [
+          "Complete Series 1"
+        ],
+        [
+          "Complete Series 2"
+        ],
+        [
+          "Complete Series 3"
+        ],
+        [
+          "Complete Series 4"
+        ],
+        [
+          "Complete Series 5"
+        ],
+        [
+          "Complete Series 6"
+        ],
+        [
+          "Complete Series 7"
+        ],
+        [
+          "Complete Series 8"
+        ],
+        [
+          "Complete Series 9"
+        ],
+        [
+          "Complete Series 10"
+        ],
+        [
+          "Complete Series 11"
+        ],
+        [
+          "Complete Series 12"
+        ],
+        [
+          "The Christmas Specials"
+        ],
+        [
+          "The Complete Collection"
+        ],
+        [
+          "Two Ronnies In Australia"
+        ],
+        [
+          "The Best of...Volume 1"
+        ]
+      ]
+    },
+    "title": "The Two Ronnies"
+  },
+  {
+    "id": 15,
+    "qa_question": "map@return the number before '-",
+    "qa_column": "W\u2013L",
+    "qa_answer": [
+      "3",
+      "5",
+      "9",
+      "2",
+      "19",
+      "0"
+    ],
+    "table": {
+      "header": [
+        "W\u2013L"
+      ],
+      "rows": [
+        [
+          "3\u20135"
+        ],
+        [
+          "5\u20135"
+        ],
+        [
+          "9\u20134"
+        ],
+        [
+          "2\u20134"
+        ],
+        [
+          "19\u201318"
+        ],
+        [
+          "0\u20132"
+        ]
+      ]
+    },
+    "title": "\u0141ukasz Kubot"
+  },
+  {
+    "id": 16,
+    "qa_question": "map@Is it from alabama?",
+    "qa_column": "Hometown",
+    "qa_answer": [
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      " no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Hometown"
+      ],
+      "rows": [
+        [
+          "Chicago, IL, U.S."
+        ],
+        [
+          "Oklahoma City, OK, U.S."
+        ],
+        [
+          "Montgomery, AL, U.S."
+        ],
+        [
+          "Greenville, MS, U.S."
+        ],
+        [
+          "Stone Mountain, GA, U.S."
+        ],
+        [
+          "Douala, Cameroon"
+        ],
+        [
+          "San Antonio, TX, U.S."
+        ],
+        [
+          "Orlando, FL, U.S."
+        ],
+        [
+          "Conway, AR, U.S."
+        ],
+        [
+          "London, England, U.K."
+        ],
+        [
+          "Nashville, TN, U.S."
+        ],
+        [
+          "Wichita, KS, U.S."
+        ]
+      ]
+    },
+    "title": "2010\u201311 UAB Blazers men's basketball team"
+  },
+  {
+    "id": 17,
+    "qa_question": "map@value of elevation in feet",
+    "qa_column": "Elevation",
+    "qa_answer": [
+      "14505",
+      "14379",
+      "14252",
+      "14248",
+      "14179",
+      "13992",
+      "13982",
+      "13837",
+      "13807",
+      "13758",
+      "13747",
+      "13657",
+      "13565",
+      "13500",
+      "13162"
+    ],
+    "table": {
+      "header": [
+        "Elevation"
+      ],
+      "rows": [
+        [
+          "14,505\u00a0ft\\n4421\u00a0m"
+        ],
+        [
+          "14,379\u00a0ft\\n4383\u00a0m"
+        ],
+        [
+          "14,252\u00a0ft\\n4344\u00a0m"
+        ],
+        [
+          "14,248\u00a0ft\\n4343\u00a0m"
+        ],
+        [
+          "14,179\u00a0ft\\n4322\u00a0m"
+        ],
+        [
+          "13,992\u00a0ft\\n4265\u00a0m"
+        ],
+        [
+          "13,982\u00a0ft\\n4262\u00a0m"
+        ],
+        [
+          "13,837\u00a0ft\\n4218\u00a0m"
+        ],
+        [
+          "13,807\u00a0ft\\n4209\u00a0m"
+        ],
+        [
+          "13,758\u00a0ft\\n4193\u00a0m"
+        ],
+        [
+          "13,747\u00a0ft\\n4190\u00a0m"
+        ],
+        [
+          "13,657\u00a0ft\\n4163\u00a0m"
+        ],
+        [
+          "13,565\u00a0ft\\n4135\u00a0m"
+        ],
+        [
+          "13,500\u00a0ft\\n4115\u00a0m"
+        ],
+        [
+          "13,162\u00a0ft\\n4012\u00a0m"
+        ]
+      ]
+    },
+    "title": "Highest mountain peaks of California"
+  },
+  {
+    "id": 19,
+    "qa_question": "map@What is his/her country?",
+    "qa_column": "Driver",
+    "qa_answer": [
+      "uk",
+      "us",
+      "uk",
+      "australia",
+      "south africa",
+      "new zealand",
+      "uk",
+      "uk",
+      "us",
+      "netherlands",
+      "switzerland",
+      "belgium",
+      "scotland",
+      "france",
+      "uk",
+      "us",
+      "uk",
+      "us",
+      "uk"
+    ],
+    "table": {
+      "header": [
+        "Driver"
+      ],
+      "rows": [
+        [
+          "Jim Clark"
+        ],
+        [
+          "Richie Ginther"
+        ],
+        [
+          "Graham Hill"
+        ],
+        [
+          "Jack Brabham"
+        ],
+        [
+          "Tony Maggs"
+        ],
+        [
+          "Bruce McLaren"
+        ],
+        [
+          "Mike Hailwood"
+        ],
+        [
+          "Ian Burgess"
+        ],
+        [
+          "Peter Revson"
+        ],
+        [
+          "Carel Godin de Beaufort"
+        ],
+        [
+          "Jo Siffert"
+        ],
+        [
+          "Andr\u00e9 Pilette"
+        ],
+        [
+          "Innes Ireland"
+        ],
+        [
+          "Bernard Collomb"
+        ],
+        [
+          "Ian Raby"
+        ],
+        [
+          "Dan Gurney"
+        ],
+        [
+          "Mike Beckwith"
+        ],
+        [
+          "Masten Gregory"
+        ],
+        [
+          "Trevor Taylor"
+        ]
+      ]
+    },
+    "title": "1963 International Gold Cup"
+  },
+  {
+    "id": 23,
+    "qa_question": "map@What is the number?",
+    "qa_column": "Serial format",
+    "qa_answer": [
+      "12345",
+      "123",
+      "123",
+      "123",
+      "123",
+      "123",
+      "123",
+      "123",
+      "123"
+    ],
+    "table": {
+      "header": [
+        "Serial format"
+      ],
+      "rows": [
+        [
+          "A-12345"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "ABC-123"
+        ],
+        [
+          "123\u00b7ABC"
+        ]
+      ]
+    },
+    "title": "Vehicle registration plates of Arizona"
+  },
+  {
+    "id": 28,
+    "qa_question": "map@Is sabaru impreza?",
+    "qa_column": "Car",
+    "qa_answer": [
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Car"
+      ],
+      "rows": [
+        [
+          "Toyota Celica ST 185"
+        ],
+        [
+          "Toyota Supra"
+        ],
+        [
+          "Subaru Impreza"
+        ],
+        [
+          "Toyota Supra"
+        ],
+        [
+          "Mitsubishi Lancer Evo 4"
+        ],
+        [
+          "-"
+        ],
+        [
+          "Subaru Impreza WRX"
+        ],
+        [
+          "Mitsubishi Lancer Evo 4"
+        ],
+        [
+          "Mitsubishi Lancer Evo 4"
+        ],
+        [
+          "Subaru Impreza N10"
+        ],
+        [
+          "Subaru Impreza N8"
+        ],
+        [
+          "Subaru Impreza N10"
+        ],
+        [
+          "Subaru Impreza N10"
+        ],
+        [
+          "Subaru Impreza N8"
+        ],
+        [
+          "Subaru Impreza N10"
+        ],
+        [
+          "Subaru Impreza N8"
+        ],
+        [
+          "Subaru Impreza N12"
+        ]
+      ]
+    },
+    "title": "Pearl of Africa Rally"
+  },
+  {
+    "id": 30,
+    "qa_question": "map@When did it scrapped",
+    "qa_column": "Fate",
+    "qa_answer": [
+      "1972",
+      "1974",
+      "1959",
+      "1960",
+      "1969",
+      "1966",
+      "1960",
+      "1966",
+      "1964",
+      "1970",
+      "1959",
+      "1963",
+      "1960",
+      "1960",
+      "1971",
+      "1963",
+      "1960",
+      "1977",
+      "1966"
+    ],
+    "table": {
+      "header": [
+        "Fate"
+      ],
+      "rows": [
+        [
+          "Scrapped in 1972"
+        ],
+        [
+          "Scrapped in 1974"
+        ],
+        [
+          "Scrapped in 1959"
+        ],
+        [
+          "Scrapped in 1960"
+        ],
+        [
+          "Scrapped in 1969"
+        ],
+        [
+          "Scrapped in 1966"
+        ],
+        [
+          "Scrapped in 1960"
+        ],
+        [
+          "Scrapped in 1966"
+        ],
+        [
+          "Scrapped in 1964"
+        ],
+        [
+          "Scrapped in 1970"
+        ],
+        [
+          "Scrapped in 1959"
+        ],
+        [
+          "Scrapped in 1963"
+        ],
+        [
+          "Sold as oil hulk in 1960"
+        ],
+        [
+          "Scrapped in 1960"
+        ],
+        [
+          "Scrapped in 1971"
+        ],
+        [
+          "Scrapped in 1963"
+        ],
+        [
+          "Scrapped in 1960"
+        ],
+        [
+          "Scrapped in 1977"
+        ],
+        [
+          "Scrapped in 1966"
+        ]
+      ]
+    },
+    "title": "Wave-class oiler"
+  },
+  {
+    "id": 31,
+    "qa_question": "map@What is the max number instead of year?",
+    "qa_column": "Top speed (km/h)",
+    "qa_answer": [
+      "/",
+      "90",
+      "164",
+      "140",
+      "160",
+      "250",
+      "401.3",
+      "/",
+      "36",
+      "75",
+      "412.6",
+      "450",
+      "/",
+      "501"
+    ],
+    "table": {
+      "header": [
+        "Top speed (km/h)"
+      ],
+      "rows": [
+        [
+          ""
+        ],
+        [
+          "90 (1971)"
+        ],
+        [
+          "164 (October 1971)"
+        ],
+        [
+          "140 (September 1972)"
+        ],
+        [
+          "160 / 230 (1974)\u00a0?"
+        ],
+        [
+          "250 (end 1973), 253.2 (21 November 1977)"
+        ],
+        [
+          "401.3 (1974)"
+        ],
+        [
+          ""
+        ],
+        [
+          "36 (or 40\u00a0?)"
+        ],
+        [
+          "75"
+        ],
+        [
+          "302 (1984), 355 (1985), 392 (1987), 406 (1987), 412.6 (January 1988)"
+        ],
+        [
+          "436 (1989), 450 (17 June 1993)"
+        ],
+        [
+          ""
+        ],
+        [
+          "501 (12 November 2003)"
+        ]
+      ]
+    },
+    "title": "Transrapid"
+  },
+  {
+    "id": 44,
+    "qa_question": "map@Is it in united states?",
+    "qa_column": "Location",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "france",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Location"
+      ],
+      "rows": [
+        [
+          "Basel"
+        ],
+        [
+          "Rome"
+        ],
+        [
+          "Moscow"
+        ],
+        [
+          "Prague"
+        ],
+        [
+          "Dortmund"
+        ],
+        [
+          "Ljubljana"
+        ],
+        [
+          "Varna"
+        ],
+        [
+          "Strasbourg"
+        ],
+        [
+          "Fort Worth"
+        ],
+        [
+          "Moscow"
+        ],
+        [
+          "Budapest"
+        ],
+        [
+          "Montreal"
+        ],
+        [
+          "Rotterdam"
+        ],
+        [
+          "Stuttgart"
+        ],
+        [
+          "Indianapolis"
+        ],
+        [
+          "Paris"
+        ],
+        [
+          "Birmingham"
+        ],
+        [
+          "Brisbane"
+        ],
+        [
+          "Sabae"
+        ],
+        [
+          "San Juan"
+        ],
+        [
+          "Lausanne"
+        ],
+        [
+          "Tianjin"
+        ],
+        [
+          "Ghent"
+        ],
+        [
+          "Debrecen"
+        ],
+        [
+          "Anaheim"
+        ],
+        [
+          "Melbourne"
+        ],
+        [
+          "Aarhus"
+        ],
+        [
+          "Stuttgart"
+        ],
+        [
+          "London"
+        ],
+        [
+          "Rotterdam"
+        ],
+        [
+          "Tokyo"
+        ],
+        [
+          "Antwerp"
+        ],
+        [
+          "Nanning"
+        ]
+      ]
+    },
+    "title": "World Artistic Gymnastics Championships \u2013 Women's floor"
+  },
+  {
+    "id": 65,
+    "qa_question": "map@Which country is the company from?",
+    "qa_column": "Company (country)",
+    "qa_answer": [
+      "us",
+      "us",
+      "uk",
+      "us",
+      "us",
+      "us",
+      "france",
+      "italy",
+      "us",
+      "us",
+      "france",
+      "china",
+      "us",
+      "us",
+      "france",
+      "us",
+      "uk",
+      "us",
+      "us",
+      "us",
+      "us"
+    ],
+    "table": {
+      "header": [
+        "Company (country)"
+      ],
+      "rows": [
+        [
+          "Lockheed Martin"
+        ],
+        [
+          "Boeing"
+        ],
+        [
+          "BAE Systems"
+        ],
+        [
+          "General Dynamics"
+        ],
+        [
+          "Raytheon"
+        ],
+        [
+          "Northrop Grumman"
+        ],
+        [
+          "Airbus Group"
+        ],
+        [
+          "Finmeccanica"
+        ],
+        [
+          "L-3 Communications"
+        ],
+        [
+          "United Technologies Corporation"
+        ],
+        [
+          "Thales Group"
+        ],
+        [
+          "SAIC"
+        ],
+        [
+          "Huntington Ingalls Industries"
+        ],
+        [
+          "Honeywell"
+        ],
+        [
+          "SAFRAN"
+        ],
+        [
+          "Computer Sciences Corp."
+        ],
+        [
+          "Rolls-Royce"
+        ],
+        [
+          "United Aircraft Corporation"
+        ],
+        [
+          "Oshkosh Corporation"
+        ],
+        [
+          "General Electric"
+        ],
+        [
+          "ITT Corp."
+        ]
+      ]
+    },
+    "title": "List of defense contractors"
+  },
+  {
+    "id": 73,
+    "qa_question": "map@Is the result a win?",
+    "qa_column": "Result",
+    "qa_answer": [
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes"
+    ],
+    "table": {
+      "header": [
+        "Result"
+      ],
+      "rows": [
+        [
+          "W\u00a044\u20136"
+        ],
+        [
+          "L\u00a014\u201324"
+        ],
+        [
+          "W\u00a025\u201320"
+        ],
+        [
+          "W\u00a029\u201322"
+        ],
+        [
+          "W\u00a034\u201327"
+        ],
+        [
+          "L\u00a010\u201341"
+        ],
+        [
+          "L\u00a010\u201313"
+        ],
+        [
+          "L\u00a06\u201320"
+        ],
+        [
+          "L\u00a017\u201337"
+        ],
+        [
+          "W\u00a031\u201317"
+        ]
+      ]
+    },
+    "title": "1987 Oregon Ducks football team"
+  },
+  {
+    "id": 88,
+    "qa_question": "map@NA inside?",
+    "qa_column": "Regions",
+    "qa_answer": [
+      "no",
+      "yes",
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Regions"
+      ],
+      "rows": [
+        [
+          "JP"
+        ],
+        [
+          "JP, NA"
+        ],
+        [
+          "JP, NA"
+        ],
+        [
+          "JP"
+        ],
+        [
+          "JP, NA"
+        ],
+        [
+          "JP, NA"
+        ],
+        [
+          "JP"
+        ],
+        [
+          "JP, NA"
+        ],
+        [
+          "JP"
+        ],
+        [
+          "JP"
+        ],
+        [
+          "JP"
+        ],
+        [
+          "JP"
+        ]
+      ]
+    },
+    "title": "Super Chinese"
+  },
+  {
+    "id": 103,
+    "qa_question": "map@Is he/she a brazilian?",
+    "qa_column": "Gold",
+    "qa_answer": [
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      " no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "/"
+    ],
+    "table": {
+      "header": [
+        "Gold"
+      ],
+      "rows": [
+        [
+          "Helena Rakoczy"
+        ],
+        [
+          "Tamara Manina"
+        ],
+        [
+          "Eva Bos\u00e1kov\u00e1"
+        ],
+        [
+          "Larisa Latynina"
+        ],
+        [
+          "Natalia Kutchinskaya"
+        ],
+        [
+          "Ludmila Turicheva"
+        ],
+        [
+          "Ludmila Turicheva"
+        ],
+        [
+          "Nellie Kim\\n Elena Mukhina"
+        ],
+        [
+          "Emilia Eberle"
+        ],
+        [
+          "Natalia Ilienko"
+        ],
+        [
+          "Ekaterina Szabo"
+        ],
+        [
+          "Oksana Omelianchik"
+        ],
+        [
+          "Daniela Siliva\u015f\\n Elena Shushunova"
+        ],
+        [
+          "Daniela Siliva\u015f\\n Svetlana Boguinskaya"
+        ],
+        [
+          "Cristina Bonta\u015f\\n Oksana Chusovitina"
+        ],
+        [
+          "Kim Zmeskal"
+        ],
+        [
+          "Shannon Miller"
+        ],
+        [
+          "Dina Kochetkova"
+        ],
+        [
+          "Gina Gogean"
+        ],
+        [
+          "Kui Yuan-Yuan\\n Gina Gogean"
+        ],
+        [
+          "Gina Gogean"
+        ],
+        [
+          "Andreea R\u0103ducan"
+        ],
+        [
+          "Andreea R\u0103ducan"
+        ],
+        [
+          "Elena G\u00f3mez"
+        ],
+        [
+          "Daiane Dos Santos"
+        ],
+        [
+          "Alicia Sacramone"
+        ],
+        [
+          "Cheng Fei"
+        ],
+        [
+          "Shawn Johnson"
+        ],
+        [
+          "Elizabeth Tweddle"
+        ],
+        [
+          "Lauren Mitchell"
+        ],
+        [
+          "Ksenia Afanasyeva"
+        ],
+        [
+          "Simone Biles"
+        ],
+        [
+          ""
+        ]
+      ]
+    },
+    "title": "World Artistic Gymnastics Championships \u2013 Women's floor"
+  },
+  {
+    "id": 124,
+    "qa_question": "map@What is the country?",
+    "qa_column": "Team",
+    "qa_answer": [
+      "switzerland",
+      "italy",
+      "united states",
+      "switzerland",
+      "italy",
+      "germany",
+      "austria",
+      "germany",
+      "spain",
+      "austria",
+      "norway",
+      "great britain",
+      "sweden",
+      "romania",
+      "poland",
+      "sweden",
+      "great britain",
+      "france",
+      "united states",
+      "romania"
+    ],
+    "table": {
+      "header": [
+        "Team"
+      ],
+      "rows": [
+        [
+          "Switzerland\u00a0(SUI) Switzerland I"
+        ],
+        [
+          "Italy\u00a0(ITA) Italy II"
+        ],
+        [
+          "United States\u00a0(USA) USA I"
+        ],
+        [
+          "Switzerland\u00a0(SUI) Switzerland II"
+        ],
+        [
+          "Italy\u00a0(ITA) Italy I"
+        ],
+        [
+          "Germany\u00a0(GER) Germany I"
+        ],
+        [
+          "Austria\u00a0(AUT) Austria II"
+        ],
+        [
+          "Germany\u00a0(GER) Germany II"
+        ],
+        [
+          "Spain\u00a0(ESP) Spain I"
+        ],
+        [
+          "Austria\u00a0(AUT) Austria I"
+        ],
+        [
+          "Norway\u00a0(NOR) Norway I"
+        ],
+        [
+          "Great Britain\u00a0(GBR) Great Britain I"
+        ],
+        [
+          "Sweden\u00a0(SWE) Sweden II"
+        ],
+        [
+          "Romania\u00a0(ROU) Romania I"
+        ],
+        [
+          "Poland\u00a0(POL) Poland I"
+        ],
+        [
+          "Sweden\u00a0(SWE) Sweden I"
+        ],
+        [
+          "Great Britain\u00a0(GBR) Great Britain II"
+        ],
+        [
+          "France\u00a0(FRA) France I"
+        ],
+        [
+          "United States\u00a0(USA) USA II"
+        ],
+        [
+          "Romania\u00a0(ROU) Romania II"
+        ]
+      ]
+    },
+    "title": "Bobsleigh at the 1956 Winter Olympics \u2013 Four-man"
+  },
+  {
+    "id": 127,
+    "qa_question": "map@What is the last name?",
+    "qa_column": "Menteri Besar",
+    "qa_answer": [
+      "Mohamed",
+      "Mahbob",
+      "Jaafar",
+      "Jaafar",
+      "Yusof",
+      "Majid",
+      "Jaafar",
+      "Mohamed"
+    ],
+    "table": {
+      "header": [
+        "Menteri Besar"
+      ],
+      "rows": [
+        [
+          "Jaafar Mohamed"
+        ],
+        [
+          "Mohamed Mahbob"
+        ],
+        [
+          "Abdullah Jaafar"
+        ],
+        [
+          "Mustapha Jaafar"
+        ],
+        [
+          "Abdul Hamid Yusof"
+        ],
+        [
+          "Ungku Abdul Aziz Abdul Majid"
+        ],
+        [
+          "Onn Jaafar"
+        ],
+        [
+          "Syed Abdul Kadir Mohamed"
+        ]
+      ]
+    },
+    "title": "Menteri Besar of Johor"
+  },
+  {
+    "id": 131,
+    "qa_question": "map@Is it a tie competition?",
+    "qa_column": "Place",
+    "qa_answer": [
+      "yes",
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "yes",
+      "yes",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Place"
+      ],
+      "rows": [
+        [
+          "4th (tie)"
+        ],
+        [
+          "10th (tie)"
+        ],
+        [
+          "8th"
+        ],
+        [
+          "1st"
+        ],
+        [
+          "12th"
+        ],
+        [
+          "2nd"
+        ],
+        [
+          "4th (tie)"
+        ],
+        [
+          "10th (tie)"
+        ],
+        [
+          "9th"
+        ],
+        [
+          "6th"
+        ],
+        [
+          "7th"
+        ]
+      ]
+    },
+    "title": "Israel in the Eurovision Song Contest 1986"
+  },
+  {
+    "id": 134,
+    "qa_question": "map@Is it a win?",
+    "qa_column": "Result",
+    "qa_answer": [
+      "yes",
+      "yes",
+      "no",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "yes",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Result"
+      ],
+      "rows": [
+        [
+          "W\u00a010-0"
+        ],
+        [
+          "W\u00a038-7"
+        ],
+        [
+          "L\u00a028-29"
+        ],
+        [
+          "W\u00a035-7"
+        ],
+        [
+          "W\u00a046-0"
+        ],
+        [
+          "W\u00a027-6"
+        ],
+        [
+          "W\u00a035-15"
+        ],
+        [
+          "W\u00a042-0"
+        ],
+        [
+          "W\u00a035-0"
+        ],
+        [
+          "W\u00a035-7"
+        ],
+        [
+          "L\u00a06-14"
+        ]
+      ]
+    },
+    "title": "1977 Ohio State Buckeyes football team"
+  },
+  {
+    "id": 138,
+    "qa_question": "map@What is the album name?",
+    "qa_column": "Album",
+    "qa_answer": [
+      "Go West Young Man",
+      "Change Your World",
+      "I'll Lead You Home",
+      "Live the Life",
+      "Christmastime"
+    ],
+    "table": {
+      "header": [
+        "Album"
+      ],
+      "rows": [
+        [
+          "Go West Young Man\\n\\nReleased: October 1, 1990\\nLabel: Reunion\\nFormat: CD"
+        ],
+        [
+          "Change Your World\\n\\nReleased: September 1, 1992\\nLabel: Reunion\\nFormat: CD"
+        ],
+        [
+          "I'll Lead You Home\\n\\nReleased: August 29, 1995\\nLabel: Reunion\\nFormat: CD"
+        ],
+        [
+          "Live the Life\\n\\nReleased: April 28, 1998\\nLabel: Reunion\\nFormat: CD"
+        ],
+        [
+          "Christmastime\\n\\nReleased: October 13, 1998\\nLabel: Reunion\\nFormat: CD"
+        ]
+      ]
+    },
+    "title": "Michael W. Smith discography"
+  },
+  {
+    "id": 139,
+    "qa_question": "map@What is the number in km?",
+    "qa_column": "Event",
+    "qa_answer": [
+      "20",
+      "20",
+      "20",
+      "50",
+      "20",
+      "50",
+      "5",
+      "50",
+      "50",
+      "50",
+      "50",
+      "20",
+      "50",
+      "50",
+      "50",
+      "20",
+      "50",
+      "20",
+      "20",
+      "50",
+      "50",
+      "20",
+      "50",
+      "50"
+    ],
+    "table": {
+      "header": [
+        "Event"
+      ],
+      "rows": [
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "5000 m walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "20\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "20,000 m walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ],
+        [
+          "50\u00a0km walk"
+        ]
+      ]
+    },
+    "title": "Robert Korzeniowski"
+  },
+  {
+    "id": 140,
+    "qa_question": "map@Is the etymology a battle?",
+    "qa_column": "Etymology",
+    "qa_answer": [
+      "yes",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no",
+      "no"
+    ],
+    "table": {
+      "header": [
+        "Etymology"
+      ],
+      "rows": [
+        [
+          "The Battle of Alamance which was derived from the local Native American word meaning \"blue clay\" found in the Great Alamance Creek"
+        ],
+        [
+          "William J. Alexander, member of the legislature and Speaker of the North Carolina House of Commons"
+        ],
+        [
+          "Derived from a corruption of the Delaware Indian name for the Allegheny and Ohio Rivers and is said to have meant \"a fine stream\""
+        ],
+        [
+          "George, Lord Anson (1697\u20131762), a celebrated English admiral who circumnavigated the globe"
+        ],
+        [
+          "Samuel Ashe (1725\u20131813), a Revolutionary patriot, superior court judge, and governor of North Carolina"
+        ],
+        [
+          "Waightstill Avery (1741\u20131821), a soldier of the Revolution and Attorney General of North Carolina"
+        ],
+        [
+          "Henry Somerset, Duke of Beaufort, who in 1709 became one of the Lords Proprietor"
+        ],
+        [
+          "James or Henry Bertie, two Lords Proprietor of colonial North Carolina"
+        ],
+        [
+          "Martin Bladen, a member of the Board of Trade"
+        ],
+        [
+          "George I of Great Britain (1660\u20131727), Duke of Brunswick and L\u00fcneburg"
+        ],
+        [
+          "Edward Buncombe, a Revolutionary soldier, who was wounded and captured at the Battle of Germantown, and died a paroled prisoner in Philadelphia"
+        ],
+        [
+          "Thomas Burke (1747\u20131783), a member of the Continental Congress and governor of North Carolina"
+        ],
+        [
+          "Stephen Cabarrus (1754\u20131808), member of the legislature and Speaker of the North Carolina House of Commons"
+        ],
+        [
+          "Joseph Caldwell (1773\u20131835), the first president of the University of North Carolina"
+        ],
+        [
+          "Charles Pratt, 1st Earl Camden (1714\u20131794), who opposed the taxation of the American colonists"
+        ],
+        [
+          "John Carteret, 2nd Earl Granville (1690\u20131763), who inherited one-eighth share in the Province of Carolina through his great-grandfather George Carteret"
+        ],
+        [
+          "Richard Caswell (1729\u20131789), member of the first Continental Congress and first governor of North Carolina after the Declaration of Independence"
+        ],
+        [
+          "Catawba Nation"
+        ],
+        [
+          "William Pitt, 1st Earl of Chatham (1708\u20131778), Secretary of State during the French and Indian War and was later Prime Minister of Great Britain"
+        ],
+        [
+          "Cherokee Nation"
+        ],
+        [
+          "Chowan Native American tribe"
+        ],
+        [
+          "Henry Clay (1777\u20131852), statesman and orator who represented Kentucky in both the House of Representatives and Senate"
+        ],
+        [
+          "Benjamin Cleveland (1738\u20131806), a colonel in the American Revolutionary War who took part in the Battle of Kings Mountain"
+        ],
+        [
+          "Christopher Columbus (1451\u20131507), navigator, explorer, and one of the first Europeans to explore the Americas"
+        ],
+        [
+          "William Craven, 1st Earl of Craven (1608\u20131697), who was a Lords Proprietor of colonial North Carolina"
+        ],
+        [
+          "Prince William, Duke of Cumberland (1721\u20131765), a military leader and son of George II"
+        ],
+        [
+          "Traditionally said to be an American Indian word for wild geese, also rendered \"Coratank\""
+        ],
+        [
+          "Virginia Dare (b. 1587), the first child born of English parents in America"
+        ],
+        [
+          "William Lee Davidson (1746\u20131781), an American Revolutionary War general who was mortally wounded at Cowan's Ford"
+        ],
+        [
+          "William Richardson Davie (1756\u20131820), a member of the Federal Convention and governor of North Carolina"
+        ],
+        [
+          "Thomas Hay, Viscount Dupplin (1710\u20131787), who was the 9th Earl of Kinnoull"
+        ]
+      ]
+    },
+    "title": "List of counties in North Carolina"
+  },
+  {
+    "id": 142,
+    "qa_question": "map@What is the time span?",
+    "qa_column": "Term",
+    "qa_answer": [
+      "5",
+      "5",
+      "11",
+      "/",
+      "1",
+      "6",
+      "3",
+      "9",
+      "4",
+      "3",
+      "/",
+      "11",
+      "5",
+      "4",
+      "3",
+      "/"
+    ],
+    "table": {
+      "header": [
+        "Term"
+      ],
+      "rows": [
+        [
+          "1859\u20131864"
+        ],
+        [
+          "1864\u20131869"
+        ],
+        [
+          "1869\u20131880"
+        ],
+        [
+          "Term"
+        ],
+        [
+          "1894\u20131895"
+        ],
+        [
+          "1895\u20131901"
+        ],
+        [
+          "1901\u20131904"
+        ],
+        [
+          "1904\u20131913"
+        ],
+        [
+          "1913\u20131917"
+        ],
+        [
+          "1917\u20131920"
+        ],
+        [
+          "Term"
+        ],
+        [
+          "1927\u20131938"
+        ],
+        [
+          "1938\u20131943"
+        ],
+        [
+          "1943\u20131947"
+        ],
+        [
+          "1947\u20131950"
+        ],
+        [
+          "Term"
+        ]
+      ]
+    },
+    "title": "Electoral district of Lachlan"
+  },
+  {
+    "id": 112,
+    "qa_question": "map@What is his/her country?",
+    "qa_column": "Rider",
+    "qa_answer": [
+      "italy",
+      "spain",
+      "finland",
+      "us",
+      "spain",
+      "italy",
+      "japan",
+      "spain",
+      "thailand",
+      "italy",
+      "czech republic",
+      "italy",
+      "hungray",
+      "czech republic",
+      "italy",
+      "spain",
+      "indonesia",
+      "japan",
+      "italy",
+      "spain",
+      "italy",
+      "spain",
+      "italy"
+    ],
+    "table": {
+      "header": [
+        "Rider"
+      ],
+      "rows": [
+        [
+          "Marco Simoncelli"
+        ],
+        [
+          "\u00c1lvaro Bautista"
+        ],
+        [
+          "Mika Kallio"
+        ],
+        [
+          "Julian Simon"
+        ],
+        [
+          "Alex Debon"
+        ],
+        [
+          "Roberto Locatelli"
+        ],
+        [
+          "Yuki Takahashi"
+        ],
+        [
+          "Aleix Espargaro"
+        ],
+        [
+          "Ratthapark Wilairot"
+        ],
+        [
+          "Fabrizio Lai"
+        ],
+        [
+          "Karel Abraham"
+        ],
+        [
+          "Alex Baldolini"
+        ],
+        [
+          "Imre Toth"
+        ],
+        [
+          "Lukas Pesek"
+        ],
+        [
+          "Simone Grotzkyj"
+        ],
+        [
+          "Daniel Arcas"
+        ],
+        [
+          "Doni Tata Pradita"
+        ],
+        [
+          "Hiroshi Aoyama"
+        ],
+        [
+          "Mattia Pasini"
+        ],
+        [
+          "H\u00e9ctor Faubel"
+        ],
+        [
+          "Federico Sandi"
+        ],
+        [
+          "Manuel Hernandez"
+        ],
+        [
+          "Stefano Bianco"
+        ]
+      ]
+    },
+    "title": "2008 Australian motorcycle Grand Prix"
+  }
+]

utils/errors.py

@@ -0,0 +1,4 @@
+class DuplicateColumnsError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+

utils/evaluator.py

@@ -0,0 +1,105 @@
+import re
+
+from utils.normalizer import str_normalize
+from utils.wtq.evaluator import to_value_list, check_denotation
+from utils.mmqa.evaluator import acc
+
+
+class Evaluator:
+    def __init__(self):
+        pass
+
+    def evaluate(
+            self,
+            pred_answer,
+            gold_answer,
+            dataset,
+            allow_semantic=True,
+            question=None
+    ):
+        if dataset == 'wikitq':
+            return self.eval_ex_match(pred_answer, gold_answer, allow_semantic, question)
+        elif dataset == 'tab_fact':
+            return self.eval_tabfact_match(pred_answer, gold_answer)
+        elif dataset == 'mmqa':
+            # For more metrics on MMQA,
+            # please use the utils/mmqa/eval_mmqa.py to call official on all prediction data
+            return self.eval_mmqa_match(pred_answer, gold_answer)
+        else:
+            raise ValueError(f'{dataset} evaluator is not supported.')
+
+    def eval_ex_match(self, pred, gold, allow_semantic=True, question=None):
+        pred = [str(p).lower().strip() for p in pred]
+        gold = [str(g).lower().strip() for g in gold]
+
+        if not allow_semantic:
+            # WikiTQ eval w. string normalization using recognizer
+            pred = [str_normalize(span) for span in pred]
+            gold = [str_normalize(span) for span in gold]
+            pred = to_value_list(pred)
+            gold = to_value_list(gold)
+            return check_denotation(pred, gold)
+        else:
+            assert isinstance(question, str)
+            question = re.sub('\s+', ' ', question).strip().lower()
+            pred = [str_normalize(span) for span in pred]
+            gold = [str_normalize(span) for span in gold]
+            pred = sorted(list(set(pred)))
+            gold = sorted(list(set(gold)))
+            # (1) 0 matches 'no', 1 matches 'yes'; 0 matches 'more', 1 matches 'less', etc.
+            if len(pred) == 1 and len(gold) == 1:
+                if (pred[0] == '0' and gold[0] == 'no') \
+                        or (pred[0] == '1' and gold[0] == 'yes'):
+                    return True
+                question_tokens = question.split()
+                try:
+                    pos_or = question_tokens.index('or')
+                    token_before_or, token_after_or = question_tokens[pos_or - 1], question_tokens[pos_or + 1]
+                    if (pred[0] == '0' and gold[0] == token_after_or) \
+                            or (pred[0] == '1' and gold[0] == token_before_or):
+                        return True
+                except Exception as e:
+                    pass
+            # (2) Number value (allow units) and Date substring match
+            if len(pred) == 1 and len(gold) == 1:
+                NUMBER_UNITS_PATTERN = re.compile('^\$*[+-]?([0-9]*[.])?[0-9]+(\s*%*|\s+\w+)$')
+                DATE_PATTERN = re.compile('[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}\s*([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})?')
+                DURATION_PATTERN = re.compile('(P|PT)(\d+)(Y|M|D|H|S)')
+                p, g = pred[0], gold[0]
+                # Restore `duration` type, e.g., from 'P3Y' -> '3'
+                if re.match(DURATION_PATTERN, p):
+                    p = re.match(DURATION_PATTERN, p).group(2)
+                if re.match(DURATION_PATTERN, g):
+                    g = re.match(DURATION_PATTERN, g).group(2)
+                match = False
+                num_flag, date_flag = False, False
+                # Number w. unit match after string normalization.
+                # Either pred or gold being number w. units suffices it.
+                if re.match(NUMBER_UNITS_PATTERN, p) or re.match(NUMBER_UNITS_PATTERN, g):
+                    num_flag = True
+                # Date match after string normalization.
+                # Either pred or gold being date suffices it.
+                if re.match(DATE_PATTERN, p) or re.match(DATE_PATTERN, g):
+                    date_flag = True
+                if num_flag:
+                    p_set, g_set = set(p.split()), set(g.split())
+                    if p_set.issubset(g_set) or g_set.issubset(p_set):
+                        match = True
+                if date_flag:
+                    p_set, g_set = set(p.replace('-', ' ').split()), set(g.replace('-', ' ').split())
+                    if p_set.issubset(g_set) or g_set.issubset(p_set):
+                        match = True
+                if match:
+                    return True
+            pred = to_value_list(pred)
+            gold = to_value_list(gold)
+            return check_denotation(pred, gold)
+
+    def eval_tabfact_match(self, pred, gold):
+        if isinstance(pred, list):
+            pred = pred[0]
+        pred, gold = str(pred), str(gold)
+        return pred == gold
+
+    def eval_mmqa_match(self, pred_answer, gold_answer):
+        return acc(pred_answer, gold_answer)

utils/gpt2/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "resid_pdrop": 0.1,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "vocab_size": 50257
+}

utils/matcher.py

@@ -0,0 +1,65 @@
+from fuzzywuzzy import fuzz
+import pandas as pd
+import string
+
+from utils.normalizer import str_normalize
+
+
+class Matcher(object):
+    def __init__(self):
+        pass
+
+    def match_sentence_with_table(self, sent: str, df: pd.DataFrame, fuzz_threshold=100):
+        phrase2matched_cells = dict()
+        sent = str_normalize(sent)
+        sent = sent.strip(string.punctuation)
+        for ngram in range(5, 0, -1):
+            ngram_tokens_list = self._create_ngram_list(sent.split(), ngram)
+            for row_id, row in df.iterrows():
+                for col_id, cell in enumerate(row):
+                    if df.columns[col_id] == 'row_id':
+                        continue
+                    cell = str(cell)
+                    for ngram_phrase in ngram_tokens_list:
+                        fuzz_score = fuzz.ratio(ngram_phrase, cell)
+                        if fuzz_score >= fuzz_threshold:
+                            if ngram_phrase not in phrase2matched_cells:
+                                phrase2matched_cells[ngram_phrase] = []
+                            phrase2matched_cells[ngram_phrase].append((cell, fuzz_score, (row_id, col_id)))
+        # Remove non-longest phrase
+        phrases = list(phrase2matched_cells.keys())
+        for phrase in phrases:
+            for other_phrase in phrases:
+                if phrase != other_phrase and phrase in other_phrase:
+                    del phrase2matched_cells[phrase]
+                    break
+        # Sort by fuzzy score
+        for matched_cells in phrase2matched_cells.values():
+            matched_cells.sort(key=lambda x: x[1], reverse=True)
+
+        return phrase2matched_cells
+
+    def match_phrase_with_table(self, phrase: str, df: pd.DataFrame, fuzz_threshold=70):
+        matched_cells = []
+        for row_id, row in df.iterrows():
+            for col_id, cell in enumerate(row):
+                cell = str(cell)
+                fuzz_score = fuzz.ratio(phrase, cell)
+                # if fuzz_score == 100:
+                #     matched_cells = [(cell, fuzz_score, (row_id, col_id))]
+                #     return matched_cells
+                if fuzz_score >= fuzz_threshold:
+                    matched_cells.append((cell, fuzz_score, (row_id, col_id)))
+        # Sort by fuzzy score
+        matched_cells.sort(key=lambda x: x[1], reverse=True)
+        return matched_cells
+
+    def _create_ngram_list(self, input_list, ngram_num):
+        ngram_list = []
+        if len(input_list) <= ngram_num:
+            ngram_list.extend(input_list)
+        else:
+            for tmp in zip(*[input_list[i:] for i in range(ngram_num)]):
+                tmp = " ".join(tmp)
+                ngram_list.append(tmp)
+        return ngram_list

utils/normalizer.py

@@ -0,0 +1,498 @@
+from typing import List, Dict
+import pandas as pd
+import recognizers_suite
+from recognizers_suite import Culture
+import re
+import unicodedata
+from fuzzywuzzy import fuzz
+
+from utils.sql.extraction_from_sql import *
+from utils.sql.all_keywords import ALL_KEY_WORDS
+
+culture = Culture.English
+
+
+def str_normalize(user_input, recognition_types=None):
+    """A string normalizer which recognize and normalize value based on recognizers_suite"""
+    user_input = str(user_input)
+    user_input = user_input.replace("\\n", "; ")
+
+    def replace_by_idx_pairs(orig_str, strs_to_replace, idx_pairs):
+        assert len(strs_to_replace) == len(idx_pairs)
+        last_end = 0
+        to_concat = []
+        for idx_pair, str_to_replace in zip(idx_pairs, strs_to_replace):
+            to_concat.append(orig_str[last_end:idx_pair[0]])
+            to_concat.append(str_to_replace)
+            last_end = idx_pair[1]
+        to_concat.append(orig_str[last_end:])
+        return ''.join(to_concat)
+
+    if recognition_types is None:
+        recognition_types = ["datetime",
+                             "number",
+                             # "ordinal",
+                             # "percentage",
+                             # "age",
+                             # "currency",
+                             # "dimension",
+                             # "temperature",
+                             ]
+
+    for recognition_type in recognition_types:
+        if re.match("\d+/\d+", user_input):
+            # avoid calculating str as 1991/92
+            continue
+        recognized_list = getattr(recognizers_suite, "recognize_{}".format(recognition_type))(user_input,
+                                                                                              culture)  # may match multiple parts
+        strs_to_replace = []
+        idx_pairs = []
+        for recognized in recognized_list:
+            if not recognition_type == 'datetime':
+                recognized_value = recognized.resolution['value']
+                if str(recognized_value).startswith("P"):
+                    # if the datetime is a period:
+                    continue
+                else:
+                    strs_to_replace.append(recognized_value)
+                    idx_pairs.append((recognized.start, recognized.end + 1))
+            else:
+                if recognized.resolution:  # in some cases, this variable could be none.
+                    if len(recognized.resolution['values']) == 1:
+                        strs_to_replace.append(
+                            recognized.resolution['values'][0]['timex'])  # We use timex as normalization
+                        idx_pairs.append((recognized.start, recognized.end + 1))
+
+        if len(strs_to_replace) > 0:
+            user_input = replace_by_idx_pairs(user_input, strs_to_replace, idx_pairs)
+
+    if re.match("(.*)-(.*)-(.*) 00:00:00", user_input):
+        user_input = user_input[:-len("00:00:00") - 1]
+        # '2008-04-13 00:00:00' -> '2008-04-13'
+    return user_input
+
+
+def prepare_df_for_neuraldb_from_table(table: Dict, add_row_id=True, normalize=True, lower_case=True):
+    header, rows = table['header'], table['rows']
+    if add_row_id and 'row_id' not in header:
+        header = ["row_id"] + header
+        rows = [["{}".format(i)] + row for i, row in enumerate(rows)]
+    if normalize:
+        df = convert_df_type(pd.DataFrame(data=rows, columns=header), lower_case=lower_case)
+    else:
+        df = pd.DataFrame(data=rows, columns=header)
+
+    return df
+
+
+def convert_df_type(df: pd.DataFrame, lower_case=True):
+    """
+    A simple converter of dataframe data type from string to int/float/datetime.
+    """
+
+    def get_table_content_in_column(table):
+        if isinstance(table, pd.DataFrame):
+            header = table.columns.tolist()
+            rows = table.values.tolist()
+        else:
+            # Standard table dict format
+            header, rows = table['header'], table['rows']
+        all_col_values = []
+        for i in range(len(header)):
+            one_col_values = []
+            for _row in rows:
+                one_col_values.append(_row[i])
+            all_col_values.append(one_col_values)
+        return all_col_values
+
+    # Rename empty columns
+    new_columns = []
+    for idx, header in enumerate(df.columns):
+        if header == '':
+            new_columns.append('FilledColumnName')  # Fixme: give it a better name when all finished!
+        else:
+            new_columns.append(header)
+    df.columns = new_columns
+
+    # Rename duplicate columns
+    new_columns = []
+    for idx, header in enumerate(df.columns):
+        if header in new_columns:
+            new_header, suffix = header, 2
+            while new_header in new_columns:
+                new_header = header + '_' + str(suffix)
+                suffix += 1
+            new_columns.append(new_header)
+        else:
+            new_columns.append(header)
+    df.columns = new_columns
+
+    # Recognize null values like "-"
+    null_tokens = ['', '-', '/']
+    for header in df.columns:
+        df[header] = df[header].map(lambda x: str(None) if x in null_tokens else x)
+
+    # Convert the null values in digit column to "NaN"
+    all_col_values = get_table_content_in_column(df)
+    for col_i, one_col_values in enumerate(all_col_values):
+        all_number_flag = True
+        for row_i, cell_value in enumerate(one_col_values):
+            try:
+                float(cell_value)
+            except Exception as e:
+                if not cell_value in [str(None), str(None).lower()]:
+                    # None or none
+                    all_number_flag = False
+        if all_number_flag:
+            _header = df.columns[col_i]
+            df[_header] = df[_header].map(lambda x: "NaN" if x in [str(None), str(None).lower()] else x)
+
+    # Normalize cell values.
+    for header in df.columns:
+        df[header] = df[header].map(lambda x: str_normalize(x))
+
+    # Strip the mis-added "01-01 00:00:00"
+    all_col_values = get_table_content_in_column(df)
+    for col_i, one_col_values in enumerate(all_col_values):
+        all_with_00_00_00 = True
+        all_with_01_00_00_00 = True
+        all_with_01_01_00_00_00 = True
+        for row_i, cell_value in enumerate(one_col_values):
+            if not str(cell_value).endswith(" 00:00:00"):
+                all_with_00_00_00 = False
+            if not str(cell_value).endswith("-01 00:00:00"):
+                all_with_01_00_00_00 = False
+            if not str(cell_value).endswith("-01-01 00:00:00"):
+                all_with_01_01_00_00_00 = False
+        if all_with_01_01_00_00_00:
+            _header = df.columns[col_i]
+            df[_header] = df[_header].map(lambda x: x[:-len("-01-01 00:00:00")])
+            continue
+
+        if all_with_01_00_00_00:
+            _header = df.columns[col_i]
+            df[_header] = df[_header].map(lambda x: x[:-len("-01 00:00:00")])
+            continue
+
+        if all_with_00_00_00:
+            _header = df.columns[col_i]
+            df[_header] = df[_header].map(lambda x: x[:-len(" 00:00:00")])
+            continue
+
+    # Do header and cell value lower case
+    if lower_case:
+        new_columns = []
+        for header in df.columns:
+            lower_header = str(header).lower()
+            if lower_header in new_columns:
+                new_header, suffix = lower_header, 2
+                while new_header in new_columns:
+                    new_header = lower_header + '-' + str(suffix)
+                    suffix += 1
+                new_columns.append(new_header)
+            else:
+                new_columns.append(lower_header)
+        df.columns = new_columns
+        for header in df.columns:
+            # df[header] = df[header].map(lambda x: str(x).lower())
+            df[header] = df[header].map(lambda x: str(x).lower().strip())
+
+    # Recognize header type
+    for header in df.columns:
+
+        float_able = False
+        int_able = False
+        datetime_able = False
+
+        # Recognize int & float type
+        try:
+            df[header].astype("float")
+            float_able = True
+        except:
+            pass
+
+        if float_able:
+            try:
+                if all(df[header].astype("float") == df[header].astype(int)):
+                    int_able = True
+            except:
+                pass
+
+        if float_able:
+            if int_able:
+                df[header] = df[header].astype(int)
+            else:
+                df[header] = df[header].astype(float)
+
+        # Recognize datetime type
+        try:
+            df[header].astype("datetime64")
+            datetime_able = True
+        except:
+            pass
+
+        if datetime_able:
+            df[header] = df[header].astype("datetime64")
+
+    return df
+
+
+def normalize(x):
+    """ Normalize string. """
+    # Copied from WikiTableQuestions dataset official evaluator.
+    if x is None:
+        return None
+    # Remove diacritics
+    x = ''.join(c for c in unicodedata.normalize('NFKD', x)
+                if unicodedata.category(c) != 'Mn')
+    # Normalize quotes and dashes
+    x = re.sub("[‘’´`]", "'", x)
+    x = re.sub("[“”]", "\"", x)
+    x = re.sub("[‐‑‒–—−]", "-", x)
+    while True:
+        old_x = x
+        # Remove citations
+        x = re.sub("((?<!^)\[[^\]]*\]|\[\d+\]|[•♦†‡*#+])*$", "", x.strip())
+        # Remove details in parenthesis
+        x = re.sub("(?<!^)( \([^)]*\))*$", "", x.strip())
+        # Remove outermost quotation mark
+        x = re.sub('^"([^"]*)"$', r'\1', x.strip())
+        if x == old_x:
+            break
+    # Remove final '.'
+    if x and x[-1] == '.':
+        x = x[:-1]
+    # Collapse whitespaces and convert to lower case
+    x = re.sub('\s+', ' ', x, flags=re.U).lower().strip()
+    return x
+
+
+def post_process_sql(sql_str, df, table_title=None, process_program_with_fuzzy_match_on_db=True, verbose=False):
+    """Post process SQL: including basic fix and further fuzzy match on cell and SQL to process"""
+
+    def basic_fix(sql_str, all_headers, table_title=None):
+        def finditer(sub_str: str, mother_str: str):
+            result = []
+            start_index = 0
+            while True:
+                start_index = mother_str.find(sub_str, start_index, -1)
+                if start_index == -1:
+                    break
+                end_idx = start_index + len(sub_str)
+                result.append((start_index, end_idx))
+                start_index = end_idx
+            return result
+
+        if table_title:
+            sql_str = sql_str.replace("FROM " + table_title, "FROM w")
+            sql_str = sql_str.replace("FROM " + table_title.lower(), "FROM w")
+
+        """Case 1: Fix the `` missing. """
+        # Remove the null header.
+        while '' in all_headers:
+            all_headers.remove('')
+
+        # Remove the '\n' in header.
+        # This is because the WikiTQ won't actually show the str in two lines,
+        # they use '\n' to mean that, and display it in the same line when print.
+        sql_str = sql_str.replace("\\n", "\n")
+        sql_str = sql_str.replace("\n", "\\n")
+
+        # Add `` in SQL.
+
+        all_headers.sort(key=lambda x: len(x), reverse=True)
+        have_matched = [0 for i in range(len(sql_str))]
+
+        # match quotation
+        idx_s_single_quotation = [_ for _ in range(1, len(sql_str)) if
+                                  sql_str[_] in ["\'"] and sql_str[_ - 1] not in ["\'"]]
+        idx_s_double_quotation = [_ for _ in range(1, len(sql_str)) if
+                                  sql_str[_] in ["\""] and sql_str[_ - 1] not in ["\""]]
+        for idx_s in [idx_s_single_quotation, idx_s_double_quotation]:
+            if len(idx_s) % 2 == 0:
+                for idx in range(int(len(idx_s) / 2)):
+                    start_idx = idx_s[idx * 2]
+                    end_idx = idx_s[idx * 2 + 1]
+                    have_matched[start_idx: end_idx] = [2 for _ in range(end_idx - start_idx)]
+
+        # match headers
+        for header in all_headers:
+            if (header in sql_str) and (header not in ALL_KEY_WORDS):
+                all_matched_of_this_header = finditer(header, sql_str)
+                for matched_of_this_header in all_matched_of_this_header:
+                    start_idx, end_idx = matched_of_this_header
+                    if all(have_matched[start_idx: end_idx]) == 0 and (not sql_str[start_idx - 1] == "`") and (
+                            not sql_str[end_idx] == "`"):
+                        have_matched[start_idx: end_idx] = [1 for _ in range(end_idx - start_idx)]
+                        # a bit ugly, but anyway.
+
+        # re-compose sql from the matched idx.
+        start_have_matched = [0] + have_matched
+        end_have_matched = have_matched + [0]
+        start_idx_s = [idx - 1 for idx in range(1, len(start_have_matched)) if
+                       start_have_matched[idx - 1] == 0 and start_have_matched[idx] == 1]
+        end_idx_s = [idx for idx in range(len(end_have_matched) - 1) if
+                     end_have_matched[idx] == 1 and end_have_matched[idx + 1] == 0]
+        assert len(start_idx_s) == len(end_idx_s)
+        spans = []
+        current_idx = 0
+        for start_idx, end_idx in zip(start_idx_s, end_idx_s):
+            spans.append(sql_str[current_idx:start_idx])
+            spans.append(sql_str[start_idx:end_idx + 1])
+            current_idx = end_idx + 1
+        spans.append(sql_str[current_idx:])
+        sql_str = '`'.join(spans)
+
+        return sql_str
+
+    def fuzzy_match_process(sql_str, df, verbose=False):
+        """
+        Post-process SQL by fuzzy matching value with table contents.
+        """
+
+        def _get_matched_cells(value_str, df, fuzz_threshold=70):
+            """
+            Get matched table cells with value token.
+            """
+            matched_cells = []
+            for row_id, row in df.iterrows():
+                for cell in row:
+                    cell = str(cell)
+                    fuzz_score = fuzz.ratio(value_str, cell)
+                    if fuzz_score == 100:
+                        matched_cells = [(cell, fuzz_score)]
+                        return matched_cells
+                    if fuzz_score >= fuzz_threshold:
+                        matched_cells.append((cell, fuzz_score))
+
+            matched_cells = sorted(matched_cells, key=lambda x: x[1], reverse=True)
+            return matched_cells
+
+        def _check_valid_fuzzy_match(value_str, matched_cell):
+            """
+            Check if the fuzzy match is valid, now considering:
+            1. The number/date should not be disturbed, but adding new number or deleting number is valid.
+            """
+            number_pattern = "[+]?[.]?[\d]+(?:,\d\d\d)*[\.]?\d*(?:[eE][-+]?\d+)?"
+            numbers_in_value = re.findall(number_pattern, value_str)
+            numbers_in_matched_cell = re.findall(number_pattern, matched_cell)
+            try:
+                numbers_in_value = [float(num.replace(',', '')) for num in numbers_in_value]
+            except:
+                print(f"Can't convert number string {numbers_in_value} into float in _check_valid_fuzzy_match().")
+            try:
+                numbers_in_matched_cell = [float(num.replace(',', '')) for num in numbers_in_matched_cell]
+            except:
+                print(
+                    f"Can't convert number string {numbers_in_matched_cell} into float in _check_valid_fuzzy_match().")
+            numbers_in_value = set(numbers_in_value)
+            numbers_in_matched_cell = set(numbers_in_matched_cell)
+
+            if numbers_in_value.issubset(numbers_in_matched_cell) or numbers_in_matched_cell.issubset(numbers_in_value):
+                return True
+            else:
+                return False
+
+        # Drop trailing '\n```', a pattern that may appear in Codex SQL generation
+        sql_str = sql_str.rstrip('```').rstrip('\n')
+
+        # Replace QA module with placeholder
+        qa_pattern = "QA\(.+?;.*?`.+?`.*?\)"
+        qas = re.findall(qa_pattern, sql_str)
+        for idx, qa in enumerate(qas):
+            sql_str = sql_str.replace(qa, f"placeholder{idx}")
+
+        # Parse and replace SQL value with table contents
+        sql_tokens = tokenize(sql_str)
+        sql_template_tokens = extract_partial_template_from_sql(sql_str)
+        # Fix 'between' keyword bug in parsing templates
+        fixed_sql_template_tokens = []
+        sql_tok_bias = 0
+        for idx, sql_templ_tok in enumerate(sql_template_tokens):
+            sql_tok = sql_tokens[idx + sql_tok_bias]
+            if sql_tok == 'between' and sql_templ_tok == '[WHERE_OP]':
+                fixed_sql_template_tokens.extend(['[WHERE_OP]', '[VALUE]', 'and'])
+                sql_tok_bias += 2  # pass '[VALUE]', 'and'
+            else:
+                fixed_sql_template_tokens.append(sql_templ_tok)
+        sql_template_tokens = fixed_sql_template_tokens
+        for idx, tok in enumerate(sql_tokens):
+            if tok in ALL_KEY_WORDS:
+                sql_tokens[idx] = tok.upper()
+
+        if verbose:
+            print(sql_tokens)
+            print(sql_template_tokens)
+
+        assert len(sql_tokens) == len(sql_template_tokens)
+        value_indices = [idx for idx in range(len(sql_template_tokens)) if sql_template_tokens[idx] == '[VALUE]']
+        for value_idx in value_indices:
+            # Skip the value if the where condition column is QA module
+            if value_idx >= 2 and sql_tokens[value_idx - 2].startswith('placeholder'):
+                continue
+            value_str = sql_tokens[value_idx]
+            # Drop \"\" for fuzzy match
+            is_string = False
+            if value_str[0] == "\"" and value_str[-1] == "\"":
+                value_str = value_str[1:-1]
+                is_string = True
+            # If already fuzzy match, skip
+            if value_str[0] == '%' or value_str[-1] == '%':
+                continue
+            value_str = value_str.lower()
+            # Fuzzy Match
+            matched_cells = _get_matched_cells(value_str, df)
+
+            if verbose:
+                print(matched_cells)
+
+            new_value_str = value_str
+            if matched_cells:
+                # new_value_str = matched_cells[0][0]
+                for matched_cell, fuzz_score in matched_cells:
+                    if _check_valid_fuzzy_match(value_str, matched_cell):
+                        new_value_str = matched_cell
+                        if verbose and new_value_str != value_str:
+                            print("\tfuzzy match replacing!", value_str, '->', matched_cell, f'fuzz_score:{fuzz_score}')
+                        break
+            if is_string:
+                new_value_str = f"\"{new_value_str}\""
+            sql_tokens[value_idx] = new_value_str
+        # Compose new sql string
+        # Clean column name in SQL since columns may have been tokenized in the postprocessing, e.g., (ppp) -> ( ppp )
+        new_sql_str = ' '.join(sql_tokens)
+        sql_columns = re.findall('`\s(.*?)\s`', new_sql_str)
+        for sql_col in sql_columns:
+            matched_columns = []
+            for col in df.columns:
+                score = fuzz.ratio(sql_col.lower(), col)
+                if score == 100:
+                    matched_columns = [(col, score)]
+                    break
+                if score >= 80:
+                    matched_columns.append((col, score))
+            matched_columns = sorted(matched_columns, key=lambda x: x[1], reverse=True)
+            if matched_columns:
+                matched_col = matched_columns[0][0]
+                new_sql_str = new_sql_str.replace(f"` {sql_col} `", f"`{matched_col}`")
+            else:
+                new_sql_str = new_sql_str.replace(f"` {sql_col} `", f"`{sql_col}`")
+
+        # Restore QA modules
+        for idx, qa in enumerate(qas):
+            new_sql_str = new_sql_str.replace(f"placeholder{idx}", qa)
+
+        # Fix '<>' when composing the new sql
+        new_sql_str = new_sql_str.replace('< >', '<>')
+
+        return new_sql_str
+
+    sql_str = basic_fix(sql_str, list(df.columns), table_title)
+
+    if process_program_with_fuzzy_match_on_db:
+        try:
+            sql_str = fuzzy_match_process(sql_str, df, verbose)
+        except:
+            pass
+
+    return sql_str

utils/sql/all_keywords.py

@@ -0,0 +1,31 @@
+CLAUSE_KEYWORDS = (
+        "select",
+        "from",
+        "where",
+        "group",
+        "order",
+        "limit",
+        "intersect",
+        "union",
+        "except",
+    )
+JOIN_KEYWORDS = ("join", "on", "as")
+
+WHERE_OPS = (
+    "not",
+    "between",
+    "=",
+    ">",
+    "<",
+    ">=",
+    "<=",
+    "!=",
+    "in",
+    "like",
+    "is",
+    "exists",
+)
+UNIT_OPS = ("none", "-", "+", "*", "/")
+AGG_OPS = ("none", "max", "min", "count", "sum", "avg")
+
+ALL_KEY_WORDS = CLAUSE_KEYWORDS + JOIN_KEYWORDS + WHERE_OPS + UNIT_OPS + AGG_OPS

utils/sql/extraction_from_sql.py

@@ -0,0 +1,622 @@
+import argparse
+import json
+from utils.sql.process_sql import (
+  tokenize, CLAUSE_KEYWORDS, WHERE_OPS, COND_OPS, UNIT_OPS, AGG_OPS,
+  JOIN_KEYWORDS, ORDER_OPS, skip_semicolon, SQL_OPS)
+KEPT_WHERE_OP = ('not', 'in', 'exists')
+
+
+def parse_table_unit(toks, start_idx, tables_with_alias):
+  idx = start_idx
+  len_ = len(toks)
+  key = toks[idx]
+
+  if idx + 1 < len_ and toks[idx + 1] == "as":
+    tables_with_alias[toks[idx + 2]] = toks[idx]
+    idx += 3
+  else:
+    idx += 1
+
+  return idx, key
+
+def parse_col(toks, start_idx, tables_with_alias, schema, default_tables=None):
+  """
+      :returns next idx, column id
+  """
+  tok = toks[start_idx]
+  if tok == "*":
+    return start_idx + 1
+
+  if '.' in tok:  # if token is a composite
+    alias, col = tok.split('.')
+    # key = tables_with_alias[alias] + "." + col
+    table = tables_with_alias[alias]
+    """
+    Add schema
+    """
+    if table not in schema:
+      schema[table] = []
+    schema[table].append(col)
+    # We also want to normalize the column
+    toks[start_idx] = "{}.{}".format(table, col)
+    """
+    END
+    """
+    return start_idx + 1
+
+  assert default_tables is not None and len(default_tables) > 0, "Default tables should not be None or empty"
+
+  # assert len(default_tables) == 1, "Default table should only have one time"
+
+  """
+  Add schema
+  """
+  # Find the best table here
+  def choose_best_table(default_tables, tok):
+    lower_tok = tok.lower()
+    candidate = process.extractOne(lower_tok, [table.lower() for table in default_tables])[0]
+    return candidate
+
+  if len(default_tables) != 1:
+    # print(default_tables)
+    table = choose_best_table(default_tables, tok)
+    # assert len(default_tables) == 1, "Default table should only have one time"
+  else:
+    table = default_tables[0]
+  if table not in schema:
+    schema[table] = []
+  schema[table].append(tok)
+  toks[start_idx] = "{}.{}".format(table, tok)
+  return start_idx + 1
+
+  # for alias in default_tables:
+  #   table = tables_with_alias[alias]
+  #   if tok in schema.schema[table]:
+  #     key = table + "." + tok
+  #     return start_idx + 1, schema.idMap[key]
+
+  # assert False, "Error col: {}".format(tok)
+
+def parse_col_unit(toks, start_idx, tables_with_alias, schema, default_tables=None, end_idx=None):
+  """
+      :returns next idx, (agg_op id, col_id)
+  """
+  idx = start_idx
+  if end_idx is not None:
+    len_ = len(toks[start_idx:end_idx])
+  else:
+    len_ = len(toks)
+  isBlock = False
+  isDistinct = False
+  if toks[idx] == '(':
+    isBlock = True
+    idx += 1
+
+  if toks[idx] in AGG_OPS:
+    agg_id = AGG_OPS.index(toks[idx])
+    idx += 1
+    assert idx < len_ and toks[idx] == '('
+    idx += 1
+    if toks[idx] == "distinct":
+      idx += 1
+      isDistinct = True
+    idx = parse_col(toks, idx, tables_with_alias, schema, default_tables)
+    assert idx < len_ and toks[idx] == ')'
+    idx += 1
+    return idx
+
+  if toks[idx] == "distinct":
+    idx += 1
+    isDistinct = True
+  agg_id = AGG_OPS.index("none")
+  idx = parse_col(toks, idx, tables_with_alias, schema, default_tables)
+
+  if isBlock:
+    assert toks[idx] == ')'
+    idx += 1  # skip ')'
+
+  return idx
+
+def parse_val_unit(toks, start_idx, tables_with_alias, schema, default_tables=None):
+  idx = start_idx
+  len_ = len(toks)
+  isBlock = False
+  if toks[idx] == '(':
+    isBlock = True
+    idx += 1
+
+  col_unit1 = None
+  col_unit2 = None
+  unit_op = UNIT_OPS.index('none')
+
+  idx = parse_col_unit(toks, idx, tables_with_alias, schema, default_tables)
+  if idx < len_ and toks[idx] in UNIT_OPS:
+    unit_op = UNIT_OPS.index(toks[idx])
+    idx += 1
+    idx = parse_col_unit(toks, idx, tables_with_alias, schema, default_tables)
+
+  if isBlock:
+    assert toks[idx] == ')'
+    idx += 1  # skip ')'
+
+  return idx
+
+def parse_value(toks, start_idx, tables_with_alias, schema, default_tables=None):
+  idx = start_idx
+  len_ = len(toks)
+
+  isBlock = False
+  if toks[idx] == '(':
+    isBlock = True
+    idx += 1
+
+  if toks[idx] == 'select':
+    idx = parse_sql(toks, idx, schema)
+  elif "\"" in toks[idx]:  # token is a string value
+    val = toks[idx]
+    # Replace with placeholder
+    toks[idx] = "_str_value_"
+    idx += 1
+  else:
+    try:
+      val = float(toks[idx])
+      toks[idx] = "_num_value_"
+      idx += 1
+    except:
+      end_idx = idx
+      while end_idx < len_ and toks[end_idx] != ',' and toks[end_idx] != ')' \
+              and toks[end_idx] != 'and' and toks[end_idx] not in CLAUSE_KEYWORDS and toks[
+        end_idx] not in JOIN_KEYWORDS:
+        end_idx += 1
+
+      # idx = parse_col_unit(toks[start_idx: end_idx], 0, tables_with_alias, schema, default_tables)
+      idx = parse_col_unit(toks, start_idx, tables_with_alias, schema, default_tables, end_idx=end_idx)
+      idx = end_idx
+
+  if isBlock:
+    assert toks[idx] == ')'
+    idx += 1
+
+  return idx
+
+def parse_condition(toks, start_idx, tables_with_alias, schema, default_tables=None):
+  idx = start_idx
+  len_ = len(toks)
+  # conds = []
+
+  while idx < len_:
+    idx = parse_val_unit(toks, idx, tables_with_alias, schema, default_tables)
+    not_op = False
+    if toks[idx] == 'not':
+      not_op = True
+      idx += 1
+
+    assert idx < len_ and toks[idx] in WHERE_OPS, "Error condition: idx: {}, tok: {}".format(idx, toks[idx])
+    op_id = WHERE_OPS.index(toks[idx])
+    idx += 1
+    val1 = val2 = None
+    if op_id == WHERE_OPS.index('between'):  # between..and... special case: dual values
+      idx = parse_value(toks, idx, tables_with_alias, schema, default_tables)
+      assert toks[idx] == 'and'
+      idx += 1
+      idx = parse_value(toks, idx, tables_with_alias, schema, default_tables)
+    else:  # normal case: single value
+      idx = parse_value(toks, idx, tables_with_alias, schema, default_tables)
+      val2 = None
+
+    # conds.append((not_op, op_id, val_unit, val1, val2))
+
+    if idx < len_ and (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";") or toks[idx] in JOIN_KEYWORDS):
+      break
+
+    if idx < len_ and toks[idx] in COND_OPS:
+      # conds.append(toks[idx])
+      idx += 1  # skip and/or
+  return idx# , conds
+
+
+def parse_from(toks, start_idx, schema):
+  assert 'from' in toks[start_idx:], "'from' not found"
+  tables_with_alias = {}
+
+  len_ = len(toks)
+  idx = toks.index('from', start_idx) + 1
+  default_tables = []
+  table_units = []
+  conds = []
+  # print(idx, len_)
+  while idx < len_:
+    # print("idx", idx, toks[idx])
+    isBlock = False
+    if toks[idx] == '(':
+      isBlock = True
+      idx += 1
+
+    if toks[idx] == 'select':
+      idx = parse_sql(toks, idx, schema)
+      # table_units.append((TABLE_TYPE['sql'], sql))
+    else:
+      if idx < len_ and toks[idx] == 'join':
+        idx += 1  # skip join
+      idx, table_name = parse_table_unit(toks, idx, tables_with_alias)
+      # print(table_name)
+      # table_units.append((TABLE_TYPE['table_unit'], table_unit))
+      default_tables.append(table_name)
+      """
+      Add schema
+      """
+      if table_name not in schema:
+        schema[table_name] = []
+      """
+      END
+      """
+
+    if idx < len_ and toks[idx] == "on":
+      idx += 1  # skip on
+      idx = parse_condition(toks, idx, tables_with_alias, schema, default_tables)
+      # if len(conds) > 0:
+      #   conds.append('and')
+      # conds.extend(this_conds)
+
+    if isBlock:
+      assert toks[idx] == ')'
+      idx += 1
+
+    if idx < len_ and (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";")):
+      break
+
+  return idx, default_tables, tables_with_alias
+
+def parse_select(toks, start_idx, tables_with_alias, schema, default_tables=None):
+  idx = start_idx
+  len_ = len(toks)
+
+  assert toks[idx] == 'select', "'select' not found"
+  idx += 1
+  isDistinct = False
+  if idx < len_ and toks[idx] == 'distinct':
+    idx += 1
+    isDistinct = True
+  val_units = []
+
+  while idx < len_ and toks[idx] not in CLAUSE_KEYWORDS:
+    agg_id = AGG_OPS.index("none")
+    if toks[idx] in AGG_OPS:
+      agg_id = AGG_OPS.index(toks[idx])
+      idx += 1
+    idx = parse_val_unit(toks, idx, tables_with_alias, schema, default_tables)
+    # val_units.append((agg_id, val_unit))
+    if idx < len_ and toks[idx] == ',':
+      idx += 1  # skip ','
+
+  return idx
+
+def parse_where(toks, start_idx, tables_with_alias, schema, default_tables):
+  idx = start_idx
+  len_ = len(toks)
+
+  if idx >= len_ or toks[idx] != 'where':
+    return idx
+
+  idx += 1
+  idx = parse_condition(toks, idx, tables_with_alias, schema, default_tables)
+  return idx
+
+def parse_group_by(toks, start_idx, tables_with_alias, schema, default_tables):
+  idx = start_idx
+  len_ = len(toks)
+  col_units = []
+
+  if idx >= len_ or toks[idx] != 'group':
+    return idx
+
+  idx += 1
+  assert toks[idx] == 'by'
+  idx += 1
+
+  while idx < len_ and not (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";")):
+    idx = parse_col_unit(toks, idx, tables_with_alias, schema, default_tables)
+    # col_units.append(col_unit)
+    if idx < len_ and toks[idx] == ',':
+      idx += 1  # skip ','
+    else:
+      break
+
+  return idx
+
+def parse_having(toks, start_idx, tables_with_alias, schema, default_tables):
+  idx = start_idx
+  len_ = len(toks)
+
+  if idx >= len_ or toks[idx] != 'having':
+    return idx
+
+  idx += 1
+  idx = parse_condition(toks, idx, tables_with_alias, schema, default_tables)
+  return idx
+
+def parse_order_by(toks, start_idx, tables_with_alias, schema, default_tables):
+  idx = start_idx
+  len_ = len(toks)
+  val_units = []
+  order_type = 'asc'  # default type is 'asc'
+
+  if idx >= len_ or toks[idx] != 'order':
+    return idx
+
+  idx += 1
+  assert toks[idx] == 'by'
+  idx += 1
+
+  while idx < len_ and not (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";")):
+    idx = parse_val_unit(toks, idx, tables_with_alias, schema, default_tables)
+    # val_units.append(val_unit)
+    if idx < len_ and toks[idx] in ORDER_OPS:
+      order_type = toks[idx]
+      idx += 1
+    if idx < len_ and toks[idx] == ',':
+      idx += 1  # skip ','
+    else:
+      break
+
+  return idx
+
+def parse_limit(toks, start_idx):
+  idx = start_idx
+  len_ = len(toks)
+
+  if idx < len_ and toks[idx] == 'limit':
+    idx += 2
+    toks[idx - 1] = "_limit_value_"
+    # make limit value can work, cannot assume put 1 as a fake limit number
+    if type(toks[idx - 1]) != int:
+      return idx
+
+    return idx
+
+  return idx
+
+def parse_sql(toks, start_idx, schema):
+  isBlock = False  # indicate whether this is a block of sql/sub-sql
+  len_ = len(toks)
+  idx = start_idx
+
+  if toks[idx] == '(':
+    isBlock = True
+    idx += 1
+
+  from_end_idx, default_tables, tables_with_alias = parse_from(toks, start_idx, schema)
+
+  _ = parse_select(toks, idx, tables_with_alias, schema, default_tables)
+  idx = from_end_idx
+
+  idx = parse_where(toks, idx, tables_with_alias, schema, default_tables)
+  idx = parse_group_by(toks, idx, tables_with_alias, schema, default_tables)
+  idx = parse_having(toks, idx, tables_with_alias, schema, default_tables)
+  idx = parse_order_by(toks, idx, tables_with_alias, schema, default_tables)
+  idx = parse_limit(toks, idx)
+  #
+  idx = skip_semicolon(toks, idx)
+  if isBlock:
+    assert toks[idx] == ')'
+    idx += 1  # skip ')'
+  idx = skip_semicolon(toks, idx)
+
+  # for op in SQL_OPS:  # initialize IUE
+  #   sql[op] = None
+  if idx < len_ and toks[idx] in SQL_OPS:
+    sql_op = toks[idx]
+    idx += 1
+    idx = parse_sql(toks, idx, schema)
+    # sql[sql_op] = IUE_sql
+  return idx
+
+def extract_schema_from_sql(schema, sql):
+  toks = tokenize(sql)
+  parse_sql(toks=toks, start_idx=0, schema=schema)
+  return toks
+
+def extract_template_from_sql(sql, schema={}):
+  try:
+    toks = tokenize(sql)
+  except:
+    print("Tokenization error for {}".format(sql))
+    toks = []
+  # print(toks)
+  template = []
+  # ignore_follow_up_and = False
+  len_ = len(toks)
+  idx = 0
+  while idx < len_:
+    tok = toks[idx]
+    if tok == "from":
+      template.append(tok)
+      if toks[idx+1] != "(":
+        template.append("[FROM_PART]")
+        idx += 1
+        while idx < len_ and (toks[idx] not in CLAUSE_KEYWORDS and toks[idx] != ")"):
+          idx += 1
+        continue
+    elif tok in CLAUSE_KEYWORDS:
+      template.append(tok)
+    elif tok in AGG_OPS:
+      template.append(tok)
+    elif tok in [",", "*", "(", ")", "having", "by", "distinct"]:
+      template.append(tok)
+    elif tok in ["asc", "desc"]:
+      template.append("[ORDER_DIRECTION]")
+    elif tok in WHERE_OPS:
+      if tok in KEPT_WHERE_OP:
+        template.append(tok)
+      else:
+        template.append("[WHERE_OP]")
+        if tok == "between":
+          idx += 2
+    elif tok in COND_OPS:
+      template.append(tok)
+    elif template[-1] == "[WHERE_OP]":
+      template.append("[VALUE]")
+    elif template[-1] == "limit":
+      template.append("[LIMIT_VALUE]")
+    elif template[-1] != "[MASK]": # value, schema, join on as
+      template.append("[MASK]")
+    idx += 1
+  return template
+
+def extract_partial_template_from_sql(sql, schema={}):
+  toks = tokenize(sql)
+  # print(toks)
+  template = []
+  # ignore_follow_up_and = False
+  len_ = len(toks)
+  idx = 0
+  while idx < len_:
+    tok = toks[idx]
+    if tok == "from":
+      template.append(tok)
+      if toks[idx+1] != "(":
+        # template.append("[FROM_PART]")
+        idx += 1
+        while idx < len_ and (toks[idx] not in CLAUSE_KEYWORDS and toks[idx] != ")"):
+          template.append(toks[idx])
+          idx += 1
+        continue
+    elif tok in CLAUSE_KEYWORDS:
+      template.append(tok)
+    elif tok in AGG_OPS:
+      template.append(tok)
+    elif tok in [",", "*", "(", ")", "having", "by", "distinct"]:
+      template.append(tok)
+    elif tok in ["asc", "desc"]:
+      template.append("[ORDER_DIRECTION]")
+    elif tok in WHERE_OPS:
+      if tok in KEPT_WHERE_OP:
+        template.append(tok)
+      else:
+        template.append("[WHERE_OP]")
+        if tok == "between":
+          idx += 2
+    elif tok in COND_OPS:
+      template.append(tok)
+    elif template[-1] == "[WHERE_OP]":
+      template.append("[VALUE]")
+    elif template[-1] == "limit":
+      template.append("[LIMIT_VALUE]")
+    else:
+      template.append(tok)
+    idx += 1
+  return template
+
+
+def is_valid_schema(schema):
+  # There is no "." and " " in the column name
+  for table in schema:
+    if "." in table:
+      return False
+    if any([keyword == table for keyword in CLAUSE_KEYWORDS]):
+      return False
+    for column in schema[table]:
+      if "." in column or " " in column or '"' in column or "'" in column:
+        return False
+  return True
+
+def clean_sql(sql):
+  while "JOIN JOIN" in sql:
+    sql = sql.replace("JOIN JOIN", "JOIN")
+  if "JOIN WHERE" in sql:
+    sql = sql.replace("JOIN WHERE", "WHERE")
+  if "JOIN GROUP BY" in sql:
+    sql = sql.replace("JOIN GROUP BY", "GROUP BY")
+  return sql
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--input_file", type=str)
+  parser.add_argument("--output_file", type=str)
+  parser.add_argument("--mode", type=str, choices=["debug", "verbose", "silent"])
+  parser.add_argument("--task", type=str, choices=["template_extraction", "schema_extraction"])
+  args = parser.parse_args()
+
+  if args.task == "schema_extraction":
+    if args.mode == "debug":
+      sql = "SELECT count(*) FROM games"
+      sql = sql + " INTERSECT " + "SELECT sacks, year FROM players"
+      sql = sql + " EXCEPT " + 'SELECT T1.year, T1.sacks FROM players AS T1 JOIN tackles AS T2 ON T1.id = T2.player_id WHERE T2.manager = "A" and T2.season NOT IN (SELECT season FROM match WHERE match_name = "IVL" INTERSECT SELECT T1.year, T1.sacks FROM sack AS T1) GROUP BY T1.year, T1.sacks HAVING count(T1.coach) > 10 ORDER BY T2.score LIMIT 5'
+      sql = "SELECT T1.pld FROM pld AS T1 JOIN games AS T2 ON T1.crs_code = T2.crs_code JOIN GROUP BY T1.pld WHERE T2.gf = '8' AND T2.gf = '9'"
+      sql = 'select * from head where height = "6-0" or height = "6-0" order by height asc'
+      schema = {}
+      extract_schema_from_sql(schema, sql)
+      print(schema, is_valid_schema(schema))
+    elif args.mode == "verbose":
+      fout = open(args.output_file, "w")
+      with open(args.input_file) as fin:
+        for line in fin:
+          example = json.loads(line)
+          schema = {}
+          try:
+            sql = example["sql"] if "sql" in example else example["pred"]
+            sql = clean_sql(sql)
+            example["sql"] = sql
+            extract_schema_from_sql(schema, sql)
+
+          except:
+            # print(sql)
+            continue
+          for table in schema:
+            schema[table] = list(set(schema[table]))
+          if is_valid_schema(schema):
+            example["extracted_schema"] = schema
+            fout.write(json.dumps(example) + "\n")
+    elif args.mode == "verbose":
+      fout = open(args.output_file, "w")
+      with open(args.input_file) as fin:
+        for line in fin:
+          example = json.loads(line)
+          schema = {}
+          sql = example["sql"] if "sql" in example else example["pred"]
+          sql = clean_sql(sql)
+          example["sql"] = sql
+          extract_schema_from_sql(schema, sql)
+          for table in schema:
+            schema[table] = list(set(schema[table]))
+          example["extracted_schema"] = schema
+          fout.write(json.dumps(example) + "\n")
+          if is_valid_schema(schema):
+            example["extracted_schema"] = schema
+            fout.write(json.dumps(example) + "\n")
+  elif args.task == "template_extraction":
+    if args.mode == "debug":
+      sql = "SELECT avg(T1.Votes) FROM seats AS T1 JOIN votes AS T2 ON T1.Seat_ID = T2.Seat_ID WHERE T1.seats BETWEEN 1 AND 2 and T1.Seats = 1 AND T2.Votes = 10"
+      print(extract_template_from_sql(sql))
+      print(extract_partial_template_from_sql(sql))
+    elif args.mode == "verbose":
+      fout_json = open(args.output_file + ".json", "w")
+      fout_txt = open(args.output_file + ".txt", "w")
+      low_freq_txt = open(args.output_file + ".low_freq", "w")
+      high_freq_txt = open(args.output_file + ".high_freq", "w")
+      all_templates = set()
+      # for input_file in args.input_file.split(","):
+      templates = {}
+      with open(args.input_file) as fin:
+        for line in fin:
+          example = json.loads(line)
+          sql = example["sql"] if "sql" in example else example["pred"]
+          if isinstance(sql, list):
+            sql = sql[-1]
+          template = extract_template_from_sql(sql)
+          template_str = " ".join(template)
+          if template_str not in templates:
+            templates[template_str] = []
+          templates[template_str].append(sql)
+      print("{} has template {}".format(args.input_file, len(templates)))
+
+      json.dump(templates, fout_json)
+      for template in sorted(templates.keys()):
+        if len(templates[template]) > 1:
+          high_freq_txt.write(template + "\n")
+        else:
+          low_freq_txt.write(template + "\n")
+        fout_txt.write(template + "\n")
+
+
+

utils/sql/process_sql.py

@@ -0,0 +1,595 @@
+################################
+# Assumptions:
+#   1. sql is correct
+#   2. only table name has alias
+#   3. only one intersect/union/except
+#
+# val: number(float)/string(str)/sql(dict)
+# col_unit: (agg_id, col_id, isDistinct(bool))
+# val_unit: (unit_op, col_unit1, col_unit2)
+# table_unit: (table_type, col_unit/sql)
+# cond_unit: (not_op, op_id, val_unit, val1, val2)
+# condition: [cond_unit1, 'and'/'or', cond_unit2, ...]
+# sql {
+#   'select': (isDistinct(bool), [(agg_id, val_unit), (agg_id, val_unit), ...])
+#   'from': {'table_units': [table_unit1, table_unit2, ...], 'conds': condition}
+#   'where': condition
+#   'groupBy': [col_unit1, col_unit2, ...]
+#   'orderBy': ('asc'/'desc', [val_unit1, val_unit2, ...])
+#   'having': condition
+#   'limit': None/limit value
+#   'intersect': None/sql
+#   'except': None/sql
+#   'union': None/sql
+# }
+################################
+
+import json
+import sqlite3
+from nltk import word_tokenize
+
+CLAUSE_KEYWORDS = ('select', 'from', 'where', 'group', 'order', 'limit', 'intersect', 'union', 'except')
+JOIN_KEYWORDS = ('join', 'on', 'as')
+
+WHERE_OPS = ('not', 'between', '=', '>', '<', '>=', '<=', '!=', 'in', 'like', 'is', 'exists')
+UNIT_OPS = ('none', '-', '+', "*", '/')
+AGG_OPS = ('none', 'max', 'min', 'count', 'sum', 'avg')
+TABLE_TYPE = {
+    'sql': "sql",
+    'table_unit': "table_unit",
+}
+
+COND_OPS = ('and', 'or')
+SQL_OPS = ('intersect', 'union', 'except')
+ORDER_OPS = ('desc', 'asc')
+
+
+
+class Schema:
+    """
+    Simple schema which maps table&column to a unique identifier
+    """
+    def __init__(self, schema):
+        self._schema = schema
+        self._idMap = self._map(self._schema)
+
+    @property
+    def schema(self):
+        return self._schema
+
+    @property
+    def idMap(self):
+        return self._idMap
+
+    def _map(self, schema):
+        idMap = {'*': "__all__"}
+        id = 1
+        for key, vals in schema.items():
+            for val in vals:
+                idMap[key.lower() + "." + val.lower()] = "__" + key.lower() + "." + val.lower() + "__"
+                id += 1
+
+        for key in schema:
+            idMap[key.lower()] = "__" + key.lower() + "__"
+            id += 1
+
+        return idMap
+
+
+def get_schema(db):
+    """
+    Get database's schema, which is a dict with table name as key
+    and list of column names as value
+    :param db: database path
+    :return: schema dict
+    """
+
+    schema = {}
+    conn = sqlite3.connect(db)
+    cursor = conn.cursor()
+
+    # fetch table names
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = [str(table[0].lower()) for table in cursor.fetchall()]
+
+    # fetch table info
+    for table in tables:
+        cursor.execute("PRAGMA table_info({})".format(table))
+        schema[table] = [str(col[1].lower()) for col in cursor.fetchall()]
+
+    return schema
+
+
+def get_schema_from_json(fpath):
+    with open(fpath) as f:
+        data = json.load(f)
+
+    schema = {}
+    for entry in data:
+        table = str(entry['table'].lower())
+        cols = [str(col['column_name'].lower()) for col in entry['col_data']]
+        schema[table] = cols
+
+    return schema
+
+
+def tokenize(string):
+    string = str(string)
+    string = string.replace("\'", "\"")  # ensures all string values wrapped by "" problem??
+    quote_idxs = [idx for idx, char in enumerate(string) if char == '"']
+    assert len(quote_idxs) % 2 == 0, "Unexpected quote"
+
+    # keep string value as token
+    vals = {}
+    for i in range(len(quote_idxs)-1, -1, -2):
+        qidx1 = quote_idxs[i-1]
+        qidx2 = quote_idxs[i]
+        val = string[qidx1: qidx2+1]
+        key = "__val_{}_{}__".format(qidx1, qidx2)
+        string = string[:qidx1] + key + string[qidx2+1:]
+        vals[key] = val
+
+    # tokenize sql
+    toks_tmp = [word.lower() for word in word_tokenize(string)]
+    toks = []
+    for tok in toks_tmp:
+        if tok.startswith('=__val_'):
+            tok = tok[1:]
+            toks.append('=')
+        toks.append(tok)
+
+    # replace with string value token
+    for i in range(len(toks)):
+        if toks[i] in vals:
+            toks[i] = vals[toks[i]]
+
+    # find if there exists !=, >=, <=
+    eq_idxs = [idx for idx, tok in enumerate(toks) if tok == "="]
+    eq_idxs.reverse()
+    prefix = ('!', '>', '<')
+    for eq_idx in eq_idxs:
+        pre_tok = toks[eq_idx-1]
+        if pre_tok in prefix:
+            toks = toks[:eq_idx-1] + [pre_tok + "="] + toks[eq_idx+1: ]
+
+    return toks
+
+
+def scan_alias(toks):
+    """Scan the index of 'as' and build the map for all alias"""
+    as_idxs = [idx for idx, tok in enumerate(toks) if tok == 'as']
+    alias = {}
+    for idx in as_idxs:
+        alias[toks[idx+1]] = toks[idx-1]
+    return alias
+
+
+def get_tables_with_alias(schema, toks):
+    tables = scan_alias(toks)
+    for key in schema:
+        assert key not in tables, "Alias {} has the same name in table".format(key)
+        tables[key] = key
+    return tables
+
+
+def parse_col(toks, start_idx, tables_with_alias, schema, default_tables=None):
+    """
+        :returns next idx, column id
+    """
+    tok = toks[start_idx]
+    if tok == "*":
+        return start_idx + 1, schema.idMap[tok]
+
+    if '.' in tok:  # if token is a composite
+        alias, col = tok.split('.')
+        key = tables_with_alias[alias] + "." + col
+        return start_idx+1, schema.idMap[key]
+
+    assert default_tables is not None and len(default_tables) > 0, "Default tables should not be None or empty"
+
+    for alias in default_tables:
+        table = tables_with_alias[alias]
+        if tok in schema.schema[table]:
+            key = table + "." + tok
+            return start_idx+1, schema.idMap[key]
+
+    assert False, "Error col: {}".format(tok)
+
+
+def parse_col_unit(toks, start_idx, tables_with_alias, schema, default_tables=None):
+    """
+        :returns next idx, (agg_op id, col_id)
+    """
+    idx = start_idx
+    len_ = len(toks)
+    isBlock = False
+    isDistinct = False
+    if toks[idx] == '(':
+        isBlock = True
+        idx += 1
+
+    if toks[idx] in AGG_OPS:
+        agg_id = AGG_OPS.index(toks[idx])
+        idx += 1
+        assert idx < len_ and toks[idx] == '('
+        idx += 1
+        if toks[idx] == "distinct":
+            idx += 1
+            isDistinct = True
+        idx, col_id = parse_col(toks, idx, tables_with_alias, schema, default_tables)
+        assert idx < len_ and toks[idx] == ')'
+        idx += 1
+        return idx, (agg_id, col_id, isDistinct)
+
+    if toks[idx] == "distinct":
+        idx += 1
+        isDistinct = True
+    agg_id = AGG_OPS.index("none")
+    idx, col_id = parse_col(toks, idx, tables_with_alias, schema, default_tables)
+
+    if isBlock:
+        assert toks[idx] == ')'
+        idx += 1  # skip ')'
+
+    return idx, (agg_id, col_id, isDistinct)
+
+
+def parse_val_unit(toks, start_idx, tables_with_alias, schema, default_tables=None):
+    idx = start_idx
+    len_ = len(toks)
+    isBlock = False
+    if toks[idx] == '(':
+        isBlock = True
+        idx += 1
+
+    col_unit1 = None
+    col_unit2 = None
+    unit_op = UNIT_OPS.index('none')
+
+    idx, col_unit1 = parse_col_unit(toks, idx, tables_with_alias, schema, default_tables)
+    if idx < len_ and toks[idx] in UNIT_OPS:
+        unit_op = UNIT_OPS.index(toks[idx])
+        idx += 1
+        idx, col_unit2 = parse_col_unit(toks, idx, tables_with_alias, schema, default_tables)
+
+    if isBlock:
+        assert toks[idx] == ')'
+        idx += 1  # skip ')'
+
+    return idx, (unit_op, col_unit1, col_unit2)
+
+
+def parse_table_unit(toks, start_idx, tables_with_alias, schema):
+    """
+        :returns next idx, table id, table name
+    """
+    idx = start_idx
+    len_ = len(toks)
+    key = tables_with_alias[toks[idx]]
+
+    if idx + 1 < len_ and toks[idx+1] == "as":
+        idx += 3
+    else:
+        idx += 1
+
+    return idx, schema.idMap[key], key
+
+
+def parse_value(toks, start_idx, tables_with_alias, schema, default_tables=None):
+    idx = start_idx
+    len_ = len(toks)
+
+    isBlock = False
+    if toks[idx] == '(':
+        isBlock = True
+        idx += 1
+
+    if toks[idx] == 'select':
+        idx, val = parse_sql(toks, idx, tables_with_alias, schema)
+    elif "\"" in toks[idx]:  # token is a string value
+        val = toks[idx]
+        idx += 1
+    else:
+        try:
+            val = float(toks[idx])
+            idx += 1
+        except:
+            end_idx = idx
+            while end_idx < len_ and toks[end_idx] != ',' and toks[end_idx] != ')'\
+                and toks[end_idx] != 'and' and toks[end_idx] not in CLAUSE_KEYWORDS and toks[end_idx] not in JOIN_KEYWORDS:
+                    end_idx += 1
+
+            idx, val = parse_col_unit(toks[start_idx: end_idx], 0, tables_with_alias, schema, default_tables)
+            idx = end_idx
+
+    if isBlock:
+        assert toks[idx] == ')'
+        idx += 1
+
+    return idx, val
+
+
+def parse_condition(toks, start_idx, tables_with_alias, schema, default_tables=None):
+    idx = start_idx
+    len_ = len(toks)
+    conds = []
+
+    while idx < len_:
+        idx, val_unit = parse_val_unit(toks, idx, tables_with_alias, schema, default_tables)
+        not_op = False
+        if toks[idx] == 'not':
+            not_op = True
+            idx += 1
+
+        assert idx < len_ and toks[idx] in WHERE_OPS, "Error condition: idx: {}, tok: {}".format(idx, toks[idx])
+        op_id = WHERE_OPS.index(toks[idx])
+        idx += 1
+        val1 = val2 = None
+        if op_id == WHERE_OPS.index('between'):  # between..and... special case: dual values
+            idx, val1 = parse_value(toks, idx, tables_with_alias, schema, default_tables)
+            assert toks[idx] == 'and'
+            idx += 1
+            idx, val2 = parse_value(toks, idx, tables_with_alias, schema, default_tables)
+        else:  # normal case: single value
+            idx, val1 = parse_value(toks, idx, tables_with_alias, schema, default_tables)
+            val2 = None
+
+        conds.append((not_op, op_id, val_unit, val1, val2))
+
+        if idx < len_ and (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";") or toks[idx] in JOIN_KEYWORDS):
+            break
+
+        if idx < len_ and toks[idx] in COND_OPS:
+            conds.append(toks[idx])
+            idx += 1  # skip and/or
+
+    return idx, conds
+
+
+def parse_select(toks, start_idx, tables_with_alias, schema, default_tables=None):
+    idx = start_idx
+    len_ = len(toks)
+
+    assert toks[idx] == 'select', "'select' not found"
+    idx += 1
+    isDistinct = False
+    if idx < len_ and toks[idx] == 'distinct':
+        idx += 1
+        isDistinct = True
+    val_units = []
+
+    while idx < len_ and toks[idx] not in CLAUSE_KEYWORDS:
+        agg_id = AGG_OPS.index("none")
+        if toks[idx] in AGG_OPS:
+            agg_id = AGG_OPS.index(toks[idx])
+            idx += 1
+        idx, val_unit = parse_val_unit(toks, idx, tables_with_alias, schema, default_tables)
+        val_units.append((agg_id, val_unit))
+        if idx < len_ and toks[idx] == ',':
+            idx += 1  # skip ','
+
+    return idx, (isDistinct, val_units)
+
+
+def parse_from(toks, start_idx, tables_with_alias, schema):
+    """
+    Assume in the from clause, all table units are combined with join
+    """
+    assert 'from' in toks[start_idx:], "'from' not found"
+
+    len_ = len(toks)
+    idx = toks.index('from', start_idx) + 1
+    default_tables = []
+    table_units = []
+    conds = []
+
+    while idx < len_:
+        isBlock = False
+        if toks[idx] == '(':
+            isBlock = True
+            idx += 1
+
+        if toks[idx] == 'select':
+            idx, sql = parse_sql(toks, idx, tables_with_alias, schema)
+            table_units.append((TABLE_TYPE['sql'], sql))
+        else:
+            if idx < len_ and toks[idx] == 'join':
+                idx += 1  # skip join
+            idx, table_unit, table_name = parse_table_unit(toks, idx, tables_with_alias, schema)
+            table_units.append((TABLE_TYPE['table_unit'],table_unit))
+            default_tables.append(table_name)
+        if idx < len_ and toks[idx] == "on":
+            idx += 1  # skip on
+            idx, this_conds = parse_condition(toks, idx, tables_with_alias, schema, default_tables)
+            if len(conds) > 0:
+                conds.append('and')
+            conds.extend(this_conds)
+
+        if isBlock:
+            assert toks[idx] == ')'
+            idx += 1
+        if idx < len_ and (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";")):
+            break
+
+    return idx, table_units, conds, default_tables
+
+
+def parse_where(toks, start_idx, tables_with_alias, schema, default_tables):
+    idx = start_idx
+    len_ = len(toks)
+
+    if idx >= len_ or toks[idx] != 'where':
+        return idx, []
+
+    idx += 1
+    idx, conds = parse_condition(toks, idx, tables_with_alias, schema, default_tables)
+    return idx, conds
+
+
+def parse_group_by(toks, start_idx, tables_with_alias, schema, default_tables):
+    idx = start_idx
+    len_ = len(toks)
+    col_units = []
+
+    if idx >= len_ or toks[idx] != 'group':
+        return idx, col_units
+
+    idx += 1
+    assert toks[idx] == 'by'
+    idx += 1
+
+    while idx < len_ and not (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";")):
+        idx, col_unit = parse_col_unit(toks, idx, tables_with_alias, schema, default_tables)
+        col_units.append(col_unit)
+        if idx < len_ and toks[idx] == ',':
+            idx += 1  # skip ','
+        else:
+            break
+
+    return idx, col_units
+
+
+def parse_order_by(toks, start_idx, tables_with_alias, schema, default_tables):
+    idx = start_idx
+    len_ = len(toks)
+    val_units = []
+    order_type = 'asc' # default type is 'asc'
+
+    if idx >= len_ or toks[idx] != 'order':
+        return idx, val_units
+
+    idx += 1
+    assert toks[idx] == 'by'
+    idx += 1
+
+    while idx < len_ and not (toks[idx] in CLAUSE_KEYWORDS or toks[idx] in (")", ";")):
+        idx, val_unit = parse_val_unit(toks, idx, tables_with_alias, schema, default_tables)
+        val_units.append(val_unit)
+        if idx < len_ and toks[idx] in ORDER_OPS:
+            order_type = toks[idx]
+            idx += 1
+        if idx < len_ and toks[idx] == ',':
+            idx += 1  # skip ','
+        else:
+            break
+
+    return idx, (order_type, val_units)
+
+
+def parse_having(toks, start_idx, tables_with_alias, schema, default_tables):
+    idx = start_idx
+    len_ = len(toks)
+
+    if idx >= len_ or toks[idx] != 'having':
+        return idx, []
+
+    idx += 1
+    idx, conds = parse_condition(toks, idx, tables_with_alias, schema, default_tables)
+    return idx, conds
+
+
+def parse_limit(toks, start_idx):
+    idx = start_idx
+    len_ = len(toks)
+
+    if idx < len_ and toks[idx] == 'limit':
+        idx += 2
+        # make limit value can work, cannot assume put 1 as a fake limit number
+        if type(toks[idx-1]) != int:
+            return idx, 1
+
+        return idx, int(toks[idx-1])
+
+    return idx, None
+
+
+def parse_sql(toks, start_idx, tables_with_alias, schema):
+    isBlock = False # indicate whether this is a block of sql/sub-sql
+    len_ = len(toks)
+    idx = start_idx
+
+    sql = {}
+    if toks[idx] == '(':
+        isBlock = True
+        idx += 1
+
+    # parse from clause in order to get default tables
+    from_end_idx, table_units, conds, default_tables = parse_from(toks, start_idx, tables_with_alias, schema)
+    sql['from'] = {'table_units': table_units, 'conds': conds}
+    # select clause
+    _, select_col_units = parse_select(toks, idx, tables_with_alias, schema, default_tables)
+    idx = from_end_idx
+    sql['select'] = select_col_units
+    # where clause
+    idx, where_conds = parse_where(toks, idx, tables_with_alias, schema, default_tables)
+    sql['where'] = where_conds
+    # group by clause
+    idx, group_col_units = parse_group_by(toks, idx, tables_with_alias, schema, default_tables)
+    sql['groupBy'] = group_col_units
+    # having clause
+    idx, having_conds = parse_having(toks, idx, tables_with_alias, schema, default_tables)
+    sql['having'] = having_conds
+    # order by clause
+    idx, order_col_units = parse_order_by(toks, idx, tables_with_alias, schema, default_tables)
+    sql['orderBy'] = order_col_units
+    # limit clause
+    idx, limit_val = parse_limit(toks, idx)
+    sql['limit'] = limit_val
+
+    idx = skip_semicolon(toks, idx)
+    if isBlock:
+        assert toks[idx] == ')'
+        idx += 1  # skip ')'
+    idx = skip_semicolon(toks, idx)
+
+    # intersect/union/except clause
+    for op in SQL_OPS:  # initialize IUE
+        sql[op] = None
+    if idx < len_ and toks[idx] in SQL_OPS:
+        sql_op = toks[idx]
+        idx += 1
+        idx, IUE_sql = parse_sql(toks, idx, tables_with_alias, schema)
+        sql[sql_op] = IUE_sql
+    return idx, sql
+
+
+def load_data(fpath):
+    with open(fpath) as f:
+        data = json.load(f)
+    return data
+
+
+def get_sql(schema, query):
+    toks = tokenize(query)
+    tables_with_alias = get_tables_with_alias(schema.schema, toks)
+    _, sql = parse_sql(toks, 0, tables_with_alias, schema)
+
+    return sql
+
+
+def skip_semicolon(toks, start_idx):
+    idx = start_idx
+    while idx < len(toks) and toks[idx] == ";":
+        idx += 1
+    return idx
+
+def get_schemas_from_json(fpath):
+    with open(fpath) as f:
+        data = json.load(f)
+    db_names = [db['db_id'] for db in data]
+
+    tables = {}
+    schemas = {}
+    for db in data:
+        db_id = db['db_id']
+        schema = {} #{'table': [col.lower, ..., ]} * -> __all__
+        column_names_original = db['column_names_original']
+        table_names_original = db['table_names_original']
+        tables[db_id] = {'column_names_original': column_names_original, 'table_names_original': table_names_original}
+        for i, tabn in enumerate(table_names_original):
+            table = str(tabn.lower())
+            cols = [str(col.lower()) for td, col in column_names_original if td == i]
+            schema[table] = cols
+        schemas[db_id] = schema
+
+    return schemas, db_names, tables