init commit

.gitignore

@@ -0,0 +1,2 @@
+*.txt
+__pycache__

app.py

@@ -0,0 +1,39 @@
+import streamlit as st
+
+
+st.set_page_config(
+    page_title="Solis Demo",
+    page_icon="🐈",
+    layout="wide",
+    initial_sidebar_state="expanded",
+    menu_items={
+        'About': "Welcome to check Reflection-Of-Thought [website](https://reflection-of-thought.github.io/)!"
+    }
+)
+
+test_examples = [
+    "Nancy uploaded 41 pictures to Facebook. She put 37 pics into one album and put the rest into 2 different albums. How many pictures were in each album?",
+]
+
+def read_markdown(path):
+    with open(path, "r") as f:
+        output = f.read()
+    st.markdown(output, unsafe_allow_html=True)
+
+
+
+# Main Demo
+st.markdown("# Solis Demo")
+
+# Demo description
+read_markdown('md_src/demo_description.md')
+
+question = st.text_input(
+    "Ask a question which requries numerical reasoning:",
+    value=test_examples[0]
+)
+
+button = st.button("Run Solis")
+if not button:
+    st.stop()
+

llm_src/demo.py

@@ -0,0 +1,68 @@
+import argparse
+import random
+
+from utils.cot.get_prompt import get_prompt
+from utils.decoder import Decoder, answer_cleansing
+from utils.fp_substitution import fp_substitute, get_nums_from_passage
+from utils.solis.solis_solver import try_search
+from utils.solis.helper import *
+
+
+def demo(decoder: Decoder, x: str, CNT_SUM):
+    random.seed(123)
+    args = get_default_argument()
+    prompt_x = get_prompt()
+    
+    orig_nums, _ = get_nums_from_passage(x)
+    if len(orig_nums) > 3:
+        return "Too many operands!"
+    orig_x = prompt_x + f"Q: {x}\nA:"
+    # step 0, original predict
+    try:
+        orig_z = decoder.decode(args, orig_x, CNT_SUM)
+        orig_z = answer_cleansing(args, orig_z)
+    except Exception as e:
+        print(e)
+        orig_z = "Too Frequent!"
+        return orig_z
+    
+    # step 1, #TODO skip operand proposal
+    # step 2, substitute
+    fp_data_list = fp_substitute(x, args.substitute_time)
+    fp_results = []
+    for fp_data in fp_data_list:
+        fp_x = prompt_x + f"Q: {fp_data['Question']}\nA:"
+        try:
+            fp_z = decoder.decode(args, fp_x, CNT_SUM)
+            fp_z = answer_cleansing(args, fp_z)
+        except Exception as e:
+            print(e)
+            fp_z = "Too Frequent!"
+            return fp_z
+        fp_results.append({
+            "fp_nums": fp_data["Alignments"],
+            "fp_z": fp_z,
+        })
+    # step 3, arith relationship inversion
+    solis_ret = try_search(args, orig_nums, fp_results)
+    print(solis_ret)
+    return solis_ret
+
+def get_default_argument():
+    parser = argparse.ArgumentParser(description="Solis")
+    parser.add_argument("--seed", type=int, default=123)
+    parser.add_argument("--api_time_interval", type=float, default=2)
+    parser.add_argument("--max_length", type=int, default=256)
+    parser.add_argument("--substitute_time", type=int, default=5)
+    parser.add_argument("--dataset", type=str, default="multiarith")
+    parser.add_argument("--direct_answer_trigger_for_fewshot", type=str, default="The answer is")
+    args = parser.parse_args()
+    return args
+
+if __name__ == "__main__":
+    test_examples = [
+        "Nancy uploaded 41 pictures to Facebook. She put 37 pics into one album and put the rest into 2 different albums. How many pictures were in each album?",
+    ]
+    decoder = Decoder()
+    for test_example in test_examples:
+        demo(decoder, test_example, 0)

llm_src/utils/cot/get_prompt.py

@@ -0,0 +1,17 @@
+import random
+
+def get_prompt(num_cases=8):
+    """
+    select prompt examples from COT
+    """
+    COT_PROMPT = "Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?\nA: We start with 15 trees. Later we have 21 trees. The difference must be the number of trees they planted. So, they must have planted 21 - 15 = 6 trees. The answer is 6.\n\nQ: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?\nA: There are 3 cars in the parking lot already. 2 more arrive. Now there are 3 + 2 = 5 cars. The answer is 5.\n\nQ: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\nA: Leah had 32 chocolates and Leah's sister had 42. That means there were originally 32 + 42 = 74 chocolates. 35 have been eaten. So in total they still have 74 - 35 = 39 chocolates. The answer is 39.\n\nQ: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?\nA: Jason had 20 lollipops. Since he only has 12 now, he must have given the rest to Denny. The number of lollipops he has given to Denny must have been 20 - 12 = 8 lollipops. The answer is 8.\n\nQ: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?\nA: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.\n\nQ: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?\nA: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.\n\nQ: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?\nA: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.\n\nQ: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\nA: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8."
+
+    COT_prompt_list = COT_PROMPT.split('\n\n')
+    random.shuffle(COT_prompt_list)
+    COT_prompt = '\n\n'.join(COT_prompt_list[:num_cases]) + '\n\n'
+
+    return COT_prompt
+
+
+if __name__ == "__main__":
+    print(get_prompt())

llm_src/utils/decoder.py

@@ -0,0 +1,87 @@
+import re
+import openai
+import time
+
+
+class Decoder():
+    def __init__(self):
+        pass
+
+    def decode(self, args, input, CNT_SUM):
+        response = decoder_for_gpt3(args, input, CNT_SUM)
+        return response
+
+# Sentence Generator (Decoder) for GPT-3 ...
+def decoder_for_gpt3(args, input, CNT_SUM):
+    
+    # GPT-3 API allows each users execute the API within 60 times in a minute ...
+    time.sleep(args.api_time_interval)
+    
+    # https://beta.openai.com/account/api-keys
+    openai.api_key = ""
+    engine = "code-davinci-002"
+        
+    response = openai.Completion.create(
+      engine=engine,
+      prompt=input,
+      max_tokens=args.max_length,
+      temperature=0,
+      stop=['--', '\n\n', '#'],
+    )
+    
+    return response["choices"][0]["text"]
+
+# ver 0.2
+def answer_cleansing(args, pred):
+
+    print("pred_before : " + pred)
+    
+    preds = pred.split(args.direct_answer_trigger_for_fewshot)
+    answer_flag = True if len(preds) > 1 else False 
+    pred = preds[-1]
+
+    if args.dataset in ("aqua", "commonsensqa"):
+        pred = re.findall(r'A|B|C|D|E', pred)
+    elif args.dataset == "bigbench_date":
+        pred = re.findall(r'A|B|C|D|E|F', pred)
+    elif args.dataset in ("object_tracking"):
+        pred = re.findall(r'A|B|C', pred)
+    elif args.dataset in ("gsm8k", "addsub", "multiarith", "svamp", "singleeq"):
+        pred = pred.replace(",", "")
+        pred = [s for s in re.findall(r'-?\d+\.?\d*', pred)]
+    elif args.dataset in ("strategyqa", "coin_flip"):
+        pred = pred.lower()
+        pred = re.sub("\"|\'|\n|\.|\s|\:|\,"," ", pred)
+        pred = pred.split(" ")
+        pred = [i for i in pred if i in ("yes", "no")]
+    elif args.dataset == "last_letters":
+        pred = re.sub("\"|\'|\n|\.|\s","", pred)
+        pred = [pred]
+    else:
+        raise ValueError("dataset is not properly defined ...")
+
+    # If there is no candidate in list, null is set.
+    if len(pred) == 0:
+        pred = ""
+    else:
+        if answer_flag:
+            # choose the first element in list ...
+            pred = pred[0]
+        else:
+            # choose the last element in list ...
+            pred = pred[-1]
+    
+    # (For arithmetic tasks) if a word ends with period, it will be omitted ...
+    if pred != "":
+        if pred[-1] == ".":
+            pred = pred[:-1]
+    
+    print("pred_after : " + pred)
+    return pred
+
+
+if __name__ == "__main__":
+    
+    test_str = "/* Create a JavaScript dictionary of 5 countries and capitals: */\n"
+    z = decoder_for_gpt3(None, test_str, 256, 0)
+    print(z)

llm_src/utils/fp_substitution.py

@@ -0,0 +1,77 @@
+import random
+import numpy as np
+
+def get_nums_from_passage(p):
+    words = p.strip().split()
+    nums = []
+    nums_id = []
+    for idx, word in enumerate(words):
+        try:
+            word = word.replace(',','').replace('$', '')
+            num = eval(word)
+            if isinstance(num, int) or isinstance(num, float):
+                nums.append(num)
+                nums_id.append(idx)
+        except:
+            continue
+    return nums, nums_id
+
+
+def generate_fp_nums_multiarith(orig_nums, repeat_time):
+    # erase cannot be divided exactly
+    nums_candis = []
+    ks = [[1, 2, 3], [1, 2, 5], [1, 2, 7], [1, 3, 5]]
+    for x0 in [1, 2, 3, 4, 5]:
+        for k in ks:
+            nums_tmp = k
+            nums_tmp = [_ * x0 for _ in nums_tmp]
+            nums_candis.append(nums_tmp)
+    
+    fp_repl_numbers_list = []
+    num_repl = len(orig_nums)
+    for num_var in range(repeat_time):
+        idx_candi = random.randint(0, len(nums_candis) - 1)
+        num_pick = sorted(nums_candis[idx_candi])
+        num_order = np.argsort(orig_nums).tolist()
+        cur_rdx = 0
+        idx_ = 0
+        fp_nums = [-1 for _ in range(num_repl)]
+        while idx_ < num_repl:
+            fp_nums[num_order[idx_]] = num_pick[cur_rdx]
+            idx_ += 1
+            if idx_ == num_repl:
+                break
+            if orig_nums[num_order[idx_]] != orig_nums[num_order[idx_ - 1]] or \
+                orig_nums[num_order[idx_]] < 10:
+                cur_rdx += 1
+        
+        fp_repl_numbers_list.append(fp_nums)
+    
+    return fp_repl_numbers_list
+
+
+def fp_substitute(p, s_time)-> list: 
+
+    orig_nums, nums_id = get_nums_from_passage(p)
+    fp_nums_list = generate_fp_nums_multiarith(orig_nums, s_time)
+    data_fp_list = []
+    words = p.strip().split()
+    for fp_idx, fp_nums in enumerate(fp_nums_list):
+            q = []
+            num_idx = 0
+            for idx, word in enumerate(words):
+                if idx in nums_id:
+                    q.append(str(fp_nums[num_idx]))
+                    if word[-1] == '.':
+                        q[-1] += '.'
+                    num_idx += 1
+                else:
+                    q.append(word)
+            q = " ".join(q)
+            item_fp = ({
+                "Question": q,
+                "Alignments": fp_nums,
+            })
+            data_fp_list.append(item_fp)
+
+    return data_fp_list

llm_src/utils/solis/helper.py

@@ -0,0 +1,304 @@
+from sympy import simplify
+
+
+def generate_func_set(num_x, num_op, show_funx=False, op_sets=['+', '-', '*', '/'], rep=True):
+    """
+    e.g. numx = 3, num_op = 2, must return 3 lists:
+    [0, 1, 2] => var order
+    ['+', '-'] => op order
+    [0, 1] => barcket start,end pos
+    """
+    def dfs_op(tmp_op_list, max_num):
+        if len(tmp_op_list) == max_num:
+            if tmp_op_list not in op_lists:
+                op_lists.append(tmp_op_list)
+            return
+        
+        for op in op_sets:
+            # if op not in tmp_op_list:
+            dfs_op(tmp_op_list + [op], max_num)
+            # tmp_op_list = tmp_op_list[:-1]
+            # print(tmp_op_list)
+        return
+    
+    def dfs_x(tmp_x_list, max_var_num, max_num):
+        if len(tmp_x_list) == max_num:
+            if len(set(tmp_x_list)) == max_var_num:
+                x_lists.append(tmp_x_list)
+            return
+        
+        for x in x_sets:
+            # tmp_x_list.append(x)
+            if rep or (not rep and x not in tmp_x_list):
+                dfs_x(tmp_x_list + [x], max_var_num, max_num)
+            # tmp_x_list = tmp_x_list[:-1]
+        return
+
+    def dfs_parenthesis(expr_list, left, right):
+        if left > right:
+            return []
+        ans = []  
+        for i in range(left, right + 1):
+            if 'x' in expr_list[i]:
+                continue
+            ans_left = dfs_parenthesis(expr_list, left, i - 1)
+            ans_right = dfs_parenthesis(expr_list, i + 1, right)
+
+            for a in ans_left:
+                for b in ans_right:
+                    if a.count('x') > 1:
+                        a = '(' + a + ')'
+                    if b.count('x') > 1:
+                        b = '(' + b + ')'
+                    cur = ""
+                    if expr_list[i] == "+":
+                        cur = a + '+' + b
+                    elif expr_list[i] == "-":
+                        cur = a + '-' + b
+                    elif expr_list[i] == '*':
+                        cur = a + '*' + b
+                    elif expr_list[i] == '/':
+                        cur = a + '/' + b
+                        # print(cur)
+                    ans.append(cur)
+        if not ans:
+            ans.append(expr_list[left])
+            # print(ans)
+        return ans
+
+    x_sets = [_ for _ in range(num_x)]
+    # op_sets = ['/', '-']
+    n_op = len(op_sets)
+
+    op_lists = []
+    x_lists = []
+
+    # Generate all OP combinations, all X combinations
+    dfs_op([], max_num=num_op)
+    dfs_x([], max_var_num=num_x, max_num=num_op+1)
+
+    exprs = []
+    symps = []
+
+    # Make up all operations, add parentthesis, remove repeated cases
+    for op_list in op_lists:
+        for x_list in x_lists:
+            if len(x_list) != len(op_list) + 1:
+                continue
+            expr = ""
+            for i in range(len(op_list)):
+                expr = expr + "x" + str(x_list[i])# + ' '
+                expr = expr + op_list[i]# + ' '
+            # Make up expression without ()
+            expr = expr + "x" + str(x_list[-1])
+            
+            # Make up expression with ()
+            e_list = expr.replace(' ', '').replace("+", " + ").replace("-", " - ").replace("*", " * ").replace("/", " / ").split(' ')
+
+            # ic(e_list)
+            if rep:
+                exprs_parenth = dfs_parenthesis(e_list, 0, len(e_list) - 1)
+                exprs_parenth.insert(0, expr)
+            else:
+                exprs_parenth = [expr]
+
+            # remove repeat
+            for expr in exprs_parenth:
+                symp = simplify(expr)
+                if all(op not in str(symp) for op in op_sets):  # remove out of func space
+                    continue
+                if symp not in symps:
+                    symps.append(symp)
+
+                    e_list = str(symp).replace(' ', '').replace("+", " + ").replace("-", " - ").replace("*", " * ").replace("/", " / ").replace("(", " ( ").replace(")", " ) ").replace("**", " ** ").split(' ')
+
+                    if any(e.isnumeric() for e in e_list):  # do not allow const in a expression, only to add manually
+                        expr_simple = expr
+                    else:
+                        expr_simple = str(symp)
+
+                    x_sets_ = list(set([e if 'x' in e else '' for e in e_list]))
+                    x_sets_.remove('')
+                    # ic(x_sets_, x_sets)
+                    if x_sets_ is None or len(x_sets_) < len(x_sets):
+                        continue
+
+                    exprs.append(expr_simple.replace(' ',''))
+                  
+    # for _ in exprs:
+    #     ic(_)
+
+    return op_lists, x_lists, exprs
+
+
+def generate_func_set_all(num_x, num_op, show_funx=False, op_sets=['+', '-', '*', '/'], rep=True):
+    """
+    e.g. numx = 3, num_op = 2, must return 3 lists:
+    [0, 1, 2] => var order
+    ['+', '-'] => op order
+    [0, 1] => barcket start,end pos
+    """
+    def dfs_op(tmp_op_list, max_num):
+        if len(tmp_op_list) == max_num:
+            if tmp_op_list not in op_lists:
+                op_lists.append(tmp_op_list)
+            return
+        
+        elif len(tmp_op_list) < max_num and len(tmp_op_list) > 0:
+            if tmp_op_list not in op_lists:
+                op_lists.append(tmp_op_list)
+        
+        for op in op_sets:
+            dfs_op(tmp_op_list + [op], max_num)
+            # tmp_op_list = tmp_op_list[:-1]
+            # print(tmp_op_list)
+        return
+    
+    def dfs_x(tmp_x_list, max_var_num, max_num):
+        if len(tmp_x_list) == max_num:
+            if len(set(tmp_x_list)) == max_var_num:
+                x_lists.append(tmp_x_list)
+            return
+        
+        elif len(tmp_x_list) < max_num and len(tmp_x_list) > 0:
+            if tmp_x_list not in x_lists:
+                x_lists.append(tmp_x_list)
+        
+        for x in x_sets:
+            # tmp_x_list.append(x)
+            if rep or (not rep and x not in tmp_x_list):
+                dfs_x(tmp_x_list + [x], max_var_num, max_num)
+            # tmp_x_list = tmp_x_list[:-1]
+        return
+
+    def dfs_parenthesis(expr_list, left, right):
+        if left > right:
+            return []
+        ans = []  
+        for i in range(left, right + 1):
+            if 'x' in expr_list[i]:
+                continue
+            ans_left = dfs_parenthesis(expr_list, left, i - 1)
+            ans_right = dfs_parenthesis(expr_list, i + 1, right)
+
+            for a in ans_left:
+                for b in ans_right:
+                    if a.count('x') > 1:
+                        a = '(' + a + ')'
+                    if b.count('x') > 1:
+                        b = '(' + b + ')'
+                    cur = ""
+                    if expr_list[i] == "+":
+                        cur = a + '+' + b
+                    elif expr_list[i] == "-":
+                        cur = a + '-' + b
+                    elif expr_list[i] == '*':
+                        cur = a + '*' + b
+                    elif expr_list[i] == '/':
+                        cur = a + '/' + b
+                        # print(cur)
+                    ans.append(cur)
+        if not ans:
+            ans.append(expr_list[left])
+            # print(ans)
+        return ans
+
+    x_sets = [_ for _ in range(num_x)]
+    # op_sets = ['/', '-']
+    n_op = len(op_sets)
+
+    op_lists = []
+    x_lists = []
+
+    # Generate all OP combinations, all X combinations
+    dfs_op([], max_num=num_op)
+    dfs_x([], max_var_num=num_x, max_num=num_op+1)
+    
+    # ic(op_lists, x_lists)
+
+
+    exprs = []
+    symps = []
+
+    # Make up all operations, add parentthesis, remove repeated cases
+    for op_list in op_lists:
+        for x_list in x_lists:
+            if len(x_list) != len(op_list) + 1:
+                continue
+            expr = ""
+            for i in range(len(op_list)):
+                expr = expr + "x" + str(x_list[i])# + ' '
+                expr = expr + op_list[i]# + ' '
+            # Make up expression without ()
+            expr = expr + "x" + str(x_list[-1])
+            
+            # Make up expression with ()
+            e_list = expr.replace(' ', '').replace("+", " + ").replace("-", " - ").replace("*", " * ").replace("/", " / ").split(' ')
+
+            # ic(expr)
+
+            # ic(e_list)
+            if rep:
+                exprs_parenth = dfs_parenthesis(e_list, 0, len(e_list) - 1)
+                exprs_parenth.insert(0, expr)
+            else:
+                exprs_parenth = [expr]
+
+            # remove repeat
+            for expr in exprs_parenth:
+                symp = simplify(expr)
+                if all(op not in str(symp) for op in op_sets):  # remove out of func space
+                    continue
+                if symp not in symps:
+                    symps.append(symp)
+
+                    e_list = str(symp).replace(' ', '').replace("+", " + ").replace("-", " - ").replace("*", " * ").replace("/", " / ").replace("(", " ( ").replace(")", " ) ").replace("**", " ** ").split(' ')
+
+                    if any(e.isnumeric() for e in e_list):  # do not allow const in a expression, only to add manually
+                        expr_simple = expr
+                    else:
+                        expr_simple = str(symp)
+
+                    x_sets_ = list(set([e if 'x' in e else '' for e in e_list]))
+                    x_sets_.remove('')
+                    # ic(x_sets_, x_sets)
+                    if x_sets_ is None:
+                        continue
+
+                    exprs.append(expr_simple.replace(' ',''))
+                  
+    # for _ in exprs:
+    #     ic(_)
+
+    return op_lists, x_lists, exprs
+
+
+def is_func(str_1, rets, vars_sets, prec=1e-1, return_loss=False):
+    var_dict = {}
+    for _, var_set in enumerate(vars_sets):
+        for i, var in enumerate(var_set):
+            var_dict.update({
+                f"x{i}": var,
+            })
+        try:
+            ret_1 = eval(str_1, var_dict)
+            rets[_] = eval(rets[_])
+            if not isinstance(rets[_], int) and not isinstance(rets[_], float):
+                if not return_loss:
+                    return False
+                else:
+                    return False, abs(ret_1)
+        except:
+            if not return_loss:
+                return False
+            else:
+                return False, 1e20
+        if abs(ret_1 - rets[_]) > prec:
+            if not return_loss:
+                return False
+            else:
+                return False, abs(ret_1 - rets[_])
+    if not return_loss:
+        return True
+    else:
+        return True, abs(ret_1 - rets[_])

llm_src/utils/solis/solis_solver.py

@@ -0,0 +1,97 @@
+import math
+from utils.solis.helper import generate_func_set, generate_func_set_all, is_func
+
+def try_search(args, orig_nums, fp_results, func_set=None):
+
+    if func_set is None:
+        if "multiarith" in args.dataset:
+            op_set, x_set, func_set = generate_func_set(
+                num_x=3,
+                num_op=2,
+                show_funx=False,
+                op_sets=['+', '-', '*', '/'],
+                rep=True,
+            )
+        elif "addsub" in args.dataset:
+            op_set, x_set, func_set_3 = generate_func_set_all(
+                num_x=3,
+                num_op=2,
+                show_funx=False,
+                op_sets=['+', '-'],
+                rep=True,
+            )
+            op_set, x_set, func_set_2 = generate_func_set(
+                num_x=2,
+                num_op=1,
+                show_funx=False,
+                op_sets=['+', '-'],
+                rep=True,
+            )
+            if len(orig_nums) == 3:
+                func_set = func_set_3
+            elif len(orig_nums) == 2:
+                func_set = func_set_2
+            else:
+                return None, None
+    
+    errors_cnt = [0] * len(func_set)
+    losses_cnt = [0] * len(func_set)
+    for k, expr in enumerate(func_set):
+
+        for fp_result in fp_results:
+
+            pred = fp_result["fp_z"]
+            repl_numbers = fp_result["fp_nums"]
+
+            flag_, loss_ = is_func(expr, [str(pred)], [repl_numbers], return_loss=True)
+            errors_cnt[k] += (flag_ == False)
+            losses_cnt[k] += abs(loss_)
+
+    tmp_min = 10000000000000
+    tmp_error = 10000000000000
+    thresh_k = int(len(errors_cnt))
+    expr_filter = ""
+    for k, cnt in enumerate(errors_cnt):
+        if cnt <= thresh_k and errors_cnt[k] < tmp_min:
+            expr_filter = func_set[k]
+            tmp_min = errors_cnt[k]
+
+    tmp_min = losses_cnt[0]
+    if expr_filter == "":
+        for k, loss in enumerate(losses_cnt):
+            if loss < tmp_min:
+                expr_filter = func_set[k]
+                tmp_min = loss
+
+    # try calibration
+    cali_pred = ""
+    if expr_filter != "":
+        var_dict = {}
+        for i_, var in enumerate(orig_nums):
+            var_dict.update({
+                f"x{i_}": var,
+            })
+        try:
+            cali_pred = eval(expr_filter, var_dict)
+            if "multiarith" in args.dataset:
+                cali_pred = round(cali_pred, 5)
+                if int(cali_pred * 10 // 10) == cali_pred:
+                    cali_pred = int(cali_pred)
+                else:
+                    cali_pred = math.ceil(cali_pred)
+            elif "addsub" in args.dataset:
+                bit_max = 0
+                for number in orig_nums:
+                    bit = str(number).split('.')
+                    if len(bit) == 1:
+                        bit = 0
+                    else:
+                        bit = len(bit[-1])
+                        bit_max = max(bit, bit_max)
+                cali_pred = round(cali_pred, bit_max)
+        except:
+            return None, None
+
+        return expr_filter, cali_pred
+    else:
+        return None, None

md_src/demo_description.md

@@ -0,0 +1 @@
+Welcome! This is an interactive demo of [Solis](http://reflection-of-thought.github.io/).