Complete on arithmetic DSL

README.md

@@ -1,7 +1,5 @@
 # Bottom-Up Enumerative Program Synthesis
 
-🚨🚨PLEASE DO NOT GRADE YET🚨🚨
-
 Completed for [CS252R: Program Synthesis](https://synthesis.metareflection.club/) at the Harvard John A. Paulson School of Engineering and Applied Sciences, taught in Fall 2023 by Prof. Nada Amin.
 
 ## 🛠️ Background
@@ -51,7 +49,9 @@ To add additional input-output examples, modify `examples.py`. Add a new key to
 
 ## 🔎 Abstract Syntax Tree
 
-The most important data structure in this implementation is the abstract syntax tree (AST). The AST is a tree representation of a program, where each node is either a primitive or a compound expression. The AST is represented by the `OperatorNode` class in `abstract_syntax_tree.py`. My AST implementation includes functions to recursively evaluate the operator and its operands, and also to generate a string representation of the program.
+The most important data structure in this implementation is the abstract syntax tree (AST). The AST is a tree representation of a program, where each node is either a primitive or a compound expression. The AST is represented by the `OperatorNode` class in `abstract_syntax_tree.py`. My AST implementation includes functions to recursively evaluate the operator and its operands and also to generate a string representation of the program.
+
+At program evaluation time, the AST is evaluated from the bottom up. That is, the operands are evaluated first, and then the operator is evaluated on the operands. This is implemented in the `evaluate` method of the `OperatorNode` class. In the case of integers, variable inputs are represented by the `IntegerVariable` class in `arithmetic.py`. When input is not `None`, input type checking and validation is performed by the `evaluate` function in this class.
 
 ## 🔮 Virtual Environment
 

abstract_syntax_tree.py

@@ -24,10 +24,12 @@ class OperatorNode:
         add_node = OperatorNode(Add(), [IntegerVariable(0), IntegerConstant(5)])
         multiply_node.evaluate([7]) # returns 24
     '''
-    
+
     def __init__(self, operator, children):
-        self.operator = operator  # Operator object (e.g., Add, Subtract, etc.)
+        self.operator = operator  # operator object (e.g., Add, Subtract, etc.)
         self.children = children  # list of children nodes (operands)
+        self.weight = operator.weight + sum([child.weight for child in children])  # weight of the node
+        self.type = operator.return_type  # return type of the operator object
 
     def evaluate(self, input = None):
 

arithmetic.py

@@ -16,6 +16,7 @@ class IntegerVariable:
         # self.value = None           # value of the variable, initially None
         self.position = position    # zero-indexed position of the variable in the arguments to program
         self.type = int             # type of the variable
+        self.weight = 1             # weight of the variable
 
     # def assign(self, value):
     #     self.value = value
@@ -57,6 +58,7 @@ class IntegerConstant:
     def __init__(self, value):
         self.value = value  # value of the constant
         self.type = int     # type of the constant
+        self.weight = 1     # weight of the constant
 
     def evaluate(self, input = None):
         return self.value

demonstration.ipynb

@@ -29,6 +29,7 @@
     "from arithmetic import *\n",
     "from abstract_syntax_tree import OperatorNode\n",
     "from examples import example_set, check_examples\n",
+    "from synthesizer import extract_constants, observationally_equivalent, check_program\n",
     "import config"
    ]
   },
@@ -41,7 +42,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -55,233 +56,104 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "First, I define a function to check that, across all input-output pairs, all inputs are of the same length and that argument types are consistent across inputs."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "I provide examples of arithmetic operations."
+    "I define a function to extract constants from examples."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
-    "class IntegerVariable:\n",
-    "    '''\n",
-    "    Class to represent an integer variable. Note that position is the position of the variable in the input.\n",
-    "    For example, if the input is [4, 5, 6] and the variable is the third element (i.e., 6), then position = 2.\n",
-    "    '''\n",
-    "    def __init__(self, position):\n",
-    "        self.value = None           # value of the variable, initially None\n",
-    "        self.position = position    # position of the variable in the arguments to program\n",
-    "        self.type = int             # type of the variable\n",
-    "\n",
-    "    def assign(self, value):\n",
-    "        self.value = value\n",
-    "\n",
-    "class IntegerConstant:\n",
-    "    '''\n",
-    "    Class to represent an integer constant.\n",
-    "    '''\n",
-    "    def __init__(self, value):\n",
-    "        self.value = value  # value of the constant\n",
-    "        self.type = int     # type of the constant\n",
-    "\n",
-    "class Add:\n",
-    "    '''\n",
-    "    Operator to add two numerical values.\n",
-    "    '''\n",
-    "    def __init__(self):\n",
-    "        self.arity = 2                  # number of arguments\n",
-    "        self.arg_types = [int, int]     # argument types\n",
-    "        self.return_type = int          # return type\n",
-    "        self.weight = 1                 # weight\n",
-    "\n",
-    "    def __call__(self, x, y):\n",
-    "        return x + y\n",
-    "    \n",
-    "    def str(x, y):\n",
-    "        return f\"{x} + {y}\"\n",
-    "\n",
-    "class Subtract:\n",
-    "    '''\n",
-    "    Operator to subtract two numerical values.\n",
-    "    '''\n",
-    "    def __init__(self):\n",
-    "        self.arity = 2                  # number of arguments\n",
-    "        self.arg_types = [int, int]     # argument types\n",
-    "        self.return_type = int          # return type\n",
-    "        self.weight = 1                 # weight\n",
-    "\n",
-    "    def __call__(self, x, y):\n",
-    "        return x - y\n",
-    "    \n",
-    "    def str(x, y):\n",
-    "        return f\"{x} - {y}\"\n",
-    "    \n",
-    "class Multiply:\n",
-    "    '''\n",
-    "    Operator to multiply two numerical values.\n",
-    "    '''\n",
-    "    def __init__(self):\n",
-    "        self.arity = 2                  # number of arguments\n",
-    "        self.arg_types = [int, int]     # argument types\n",
-    "        self.return_type = int          # return type\n",
-    "        self.weight = 1                 # weight\n",
-    "\n",
-    "    def __call__(self, x, y):\n",
-    "        return x * y\n",
-    "    \n",
-    "    def str(x, y):\n",
-    "        return f\"{x} * {y}\" \n",
-    "\n",
-    "class Divide:\n",
-    "    '''\n",
-    "    Operator to divide two numerical values.\n",
-    "    '''\n",
-    "    def __init__(self):\n",
-    "        self.arity = 2                  # number of arguments\n",
-    "        self.arg_types = [int, int]     # argument types\n",
-    "        self.return_type = int          # return type\n",
-    "        self.weight = 1                 # weight\n",
-    "\n",
-    "    def __call__(self, x, y):\n",
-    "        try: # check for division by zero error\n",
-    "            return x / y\n",
-    "        except ZeroDivisionError:\n",
-    "            return None\n",
-    "    \n",
-    "    def str(x, y):\n",
-    "        return f\"{x} / {y}\"\n",
-    "\n",
-    "\n",
-    "'''\n",
-    "GLOBAL CONSTANTS\n",
-    "''' \n",
-    "\n",
-    "# define operators\n",
-    "arithmetic_operators = [Add(), Subtract(), Multiply(), Divide()]"
+    "# initialize program bank\n",
+    "program_bank = extract_constants(examples)\n",
+    "program_bank_str = [p.str() for p in program_bank]"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "I define a function to extract constants from examples."
+    "Next, I define the bottom-up synthesis algorithm."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(x0 + x1)\n"
+     ]
+    }
+   ],
    "source": [
-    "def extract_constants(examples):\n",
-    "    '''\n",
-    "    Extracts the constants from the input-output examples. Also constructs variables as needed\n",
-    "    based on the input-output examples, and adds them to the list of constants.\n",
-    "    '''\n",
+    "# define operators\n",
+    "operators = arithmetic_operators\n",
     "\n",
-    "    # check validity of provided examples\n",
-    "    # if valid, extract arity and argument types\n",
-    "    arity, arg_types = check_examples(examples)\n",
+    "# iterate over each level\n",
+    "for weight in range(2, max_weight):\n",
     "\n",
-    "    # initialize list of constants\n",
-    "    constants = []\n",
+    "    for op in operators:\n",
     "\n",
-    "    # get unique set of inputs\n",
-    "    inputs = [input for example in examples for input in example[0]]\n",
-    "    inputs = set(inputs)\n",
+    "        # get all possible combinations of primitives in program bank\n",
+    "        combinations = itertools.combinations(program_bank, op.arity)\n",
     "\n",
-    "    # add 1 to the set of inputs\n",
-    "    inputs.add(1)\n",
+    "        # iterate over each combination\n",
+    "        for combination in combinations:\n",
     "\n",
-    "    # extract constants in input\n",
-    "    for input in inputs:\n",
+    "            # get type signature\n",
+    "            type_signature = [p.type for p in combination]\n",
     "\n",
-    "        if type(input) == int:\n",
-    "            constants.append(IntegerConstant(input))\n",
-    "        elif type(input) == str:\n",
-    "            # constants.append(StringConstant(input))\n",
-    "            pass\n",
-    "        else:\n",
-    "            raise Exception(\"Input of unknown type.\")\n",
-    "        \n",
-    "    # initialize list of variables\n",
-    "    variables = []\n",
+    "            # check if type signature matches operator\n",
+    "            if type_signature != op.arg_types:\n",
+    "                continue\n",
     "\n",
-    "    # extract variables in input\n",
-    "    for position, arg in enumerate(arg_types):\n",
-    "        if arg == int:\n",
-    "            variables.append(IntegerVariable(position))\n",
-    "        elif arg == str:\n",
-    "            # variables.append(StringVariable(position))\n",
-    "            pass\n",
-    "        else:\n",
-    "            raise Exception(\"Input of unknown type.\")\n",
+    "            # check that sum of weights of arguments <= w\n",
+    "            if sum([p.weight for p in combination]) > weight:\n",
+    "                continue\n",
     "\n",
-    "    return constants + variables"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# initialize program bank\n",
-    "program_bank = extract_constants(examples)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "I define a function to determine observational equivalence."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def observationally_equivalent(a, b):\n",
-    "    \"\"\"\n",
-    "    Returns True if a and b are observationally equivalent, False otherwise.\n",
-    "    \"\"\"\n",
+    "            # create new program\n",
+    "            program = OperatorNode(op, combination)\n",
     "\n",
-    "    pass"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Next, I define the bottom-up synthesis algorithm."
+    "            # check if program is in program bank using string representation\n",
+    "            if program.str() in program_bank_str:\n",
+    "                continue\n",
+    "            \n",
+    "            # check if program is observationally equivalent to any program in program bank\n",
+    "            if any([observationally_equivalent(program, p, examples) for p in program_bank]):\n",
+    "                continue\n",
+    "\n",
+    "            # add program to program bank\n",
+    "            program_bank.append(program)\n",
+    "            program_bank_str.append(program.str())\n",
+    "\n",
+    "            # check if program passes all examples\n",
+    "            if check_program(program, examples):\n",
+    "                # return(program)\n",
+    "                print(program.str())"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "# iterate over each level\n",
-    "for i in range(2, max_weight):\n",
-    "\n",
-    "    # define level program bank\n",
-    "    level_program_bank = []\n",
-    "\n",
-    "    for op in arithmetic_operators:\n",
-    "\n",
-    "        break"
-   ]
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"<class 'int'>\""
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
   }
  ],
  "metadata": {

synthesizer.py

@@ -10,9 +10,12 @@ python synthesizer.py --domain arithmetic --examples addition
 # load libraries
 import numpy as np
 import argparse
+import itertools
+import time
 
 # import examples
 from arithmetic import *
+from abstract_syntax_tree import *
 from examples import example_set, check_examples
 import config
 
@@ -92,6 +95,32 @@ def extract_constants(examples):
     return constants + variables
 
 
+# CHECK OBSERVATIONAL EQUIVALENCE
+def observationally_equivalent(program_a, program_b, examples):
+    """
+    Returns True if Program A and Program B are observationally equivalent, False otherwise.
+    """
+
+    inputs = [example[0] for example in examples]
+    a_output = [program_a.evaluate(input) for input in inputs]
+    b_output = [program_b.evaluate(input) for input in inputs]
+
+    return a_output == b_output
+
+
+# CHECK CORRECTNESS
+def check_program(program, examples):
+    '''
+    Check whether the program satisfies the input-output examples.
+    '''
+    
+    inputs = [example[0] for example in examples]
+    outputs = [example[1] for example in examples]
+    program_output = [program.evaluate(input) for input in inputs]
+
+    return program_output == outputs
+
+
 # RUN SYNTHESIZER
 def run_synthesizer(args):
     '''
@@ -103,9 +132,59 @@ def run_synthesizer(args):
 
     # extract constants from examples
     program_bank = extract_constants(examples)
-    print(examples)
+    program_bank_str = [p.str() for p in program_bank]
+    print(f"- Extracted {len(program_bank)} constants from examples.")
+
+    # define operators
+    operators = arithmetic_operators
+
+    # iterate over each level
+    for weight in range(2, args.max_weight):
+
+        # print message
+        print(f"- Searching level {weight} with {len(program_bank)} primitives.")
+
+        # iterate over each operator
+        for op in operators:
+
+            # get all possible combinations of primitives in program bank
+            combinations = itertools.combinations(program_bank, op.arity)
+
+            # iterate over each combination
+            for combination in combinations:
+
+                # get type signature
+                type_signature = [p.type for p in combination]
+
+                # check if type signature matches operator
+                if type_signature != op.arg_types:
+                    continue
+
+                # check that sum of weights of arguments <= w
+                if sum([p.weight for p in combination]) > weight:
+                    continue
+
+                # create new program
+                program = OperatorNode(op, combination)
+
+                # check if program is in program bank using string representation
+                if program.str() in program_bank_str:
+                    continue
+                
+                # check if program is observationally equivalent to any program in program bank
+                if any([observationally_equivalent(program, p, examples) for p in program_bank]):
+                    continue
+
+                # add program to program bank
+                program_bank.append(program)
+                program_bank_str.append(program.str())
+
+                # check if program passes all examples
+                if check_program(program, examples):
+                    return(program)    
 
-    pass
+    # return None if no program is found
+    return None 
 
 
 if __name__ == '__main__':
@@ -115,4 +194,16 @@ if __name__ == '__main__':
     # print(args)
 
     # run bottom-up enumerative synthesis
-    run_synthesizer(args)
+    start_time = time.time()
+    program = run_synthesizer(args)
+    end_time = time.time()
+    elapsed_time = round(end_time - start_time, 4)
+
+    # check if program was found
+    if program is None:
+        print(f"Max weight of {args.max_weight} reached, no program found in {elapsed_time}s.")
+    else:
+        print(f"Program found in {elapsed_time}s.")
+        print(f"Program: {program.str()}")
+        print(f"Program weight: {program.weight}")
+        print(f"Program return type: {program.type.__name__}")